Add multi PVC versions of tests for delayed binding

Signed-off-by: Deep Debroy <ddebroy@docker.com>
pull/58/head
Deep Debroy 2018-10-29 02:30:30 -07:00
parent 8616687c90
commit 923df464fb
3 changed files with 179 additions and 130 deletions

View File

@ -1010,22 +1010,40 @@ func WaitForPersistentVolumeDeleted(c clientset.Interface, pvName string, Poll,
// WaitForPersistentVolumeClaimPhase waits for a PersistentVolumeClaim to be in a specific phase or until timeout occurs, whichever comes first.
func WaitForPersistentVolumeClaimPhase(phase v1.PersistentVolumeClaimPhase, c clientset.Interface, ns string, pvcName string, Poll, timeout time.Duration) error {
Logf("Waiting up to %v for PersistentVolumeClaim %s to have phase %s", timeout, pvcName, phase)
return WaitForPersistentVolumeClaimsPhase(phase, c, ns, []string{pvcName}, Poll, timeout, true)
}
// WaitForPersistentVolumeClaimPhase waits for any (if matchAny is true) or all (if matchAny is false) PersistentVolumeClaims
// to be in a specific phase or until timeout occurs, whichever comes first.
func WaitForPersistentVolumeClaimsPhase(phase v1.PersistentVolumeClaimPhase, c clientset.Interface, ns string, pvcNames []string, Poll, timeout time.Duration, matchAny bool) error {
if len(pvcNames) == 0 {
return fmt.Errorf("Incorrect parameter: Need at least one PVC to track. Found 0.")
}
Logf("Waiting up to %v for PersistentVolumeClaims %v to have phase %s", timeout, pvcNames, phase)
for start := time.Now(); time.Since(start) < timeout; time.Sleep(Poll) {
pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Get(pvcName, metav1.GetOptions{})
if err != nil {
Logf("Failed to get claim %q, retrying in %v. Error: %v", pvcName, Poll, err)
continue
} else {
if pvc.Status.Phase == phase {
Logf("PersistentVolumeClaim %s found and phase=%s (%v)", pvcName, phase, time.Since(start))
return nil
phaseFoundInAllClaims := true
for _, pvcName := range pvcNames {
pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Get(pvcName, metav1.GetOptions{})
if err != nil {
Logf("Failed to get claim %q, retrying in %v. Error: %v", pvcName, Poll, err)
continue
} else {
Logf("PersistentVolumeClaim %s found but phase is %s instead of %s.", pvcName, pvc.Status.Phase, phase)
if pvc.Status.Phase == phase {
Logf("PersistentVolumeClaim %s found and phase=%s (%v)", pvcName, phase, time.Since(start))
if matchAny {
return nil
}
} else {
Logf("PersistentVolumeClaim %s found but phase is %s instead of %s.", pvcName, pvc.Status.Phase, phase)
phaseFoundInAllClaims = false
}
}
}
if phaseFoundInAllClaims {
return nil
}
}
return fmt.Errorf("PersistentVolumeClaim %s not in phase %s within %v", pvcName, phase, timeout)
return fmt.Errorf("PersistentVolumeClaims %v not all in phase %s within %v", pvcNames, phase, timeout)
}
// CreateTestingNS should be used by every test, note that we append a common prefix to the provided test name.

View File

@ -74,7 +74,8 @@ var _ = utils.SIGDescribe("Regional PD", func() {
})
It("should provision storage with delayed binding [Slow]", func() {
testRegionalDelayedBinding(c, ns)
testRegionalDelayedBinding(c, ns, 1 /* pvcCount */)
testRegionalDelayedBinding(c, ns, 3 /* pvcCount */)
})
It("should provision storage in the allowedTopologies [Slow]", func() {
@ -82,7 +83,8 @@ var _ = utils.SIGDescribe("Regional PD", func() {
})
It("should provision storage in the allowedTopologies with delayed binding [Slow]", func() {
testRegionalAllowedTopologiesWithDelayedBinding(c, ns)
testRegionalAllowedTopologiesWithDelayedBinding(c, ns, 1 /* pvcCount */)
testRegionalAllowedTopologiesWithDelayedBinding(c, ns, 3 /* pvcCount */)
})
It("should failover to a different zone when all nodes in one zone become unreachable [Slow] [Disruptive]", func() {
@ -297,7 +299,7 @@ func addTaint(c clientset.Interface, ns string, nodes []v1.Node, podZone string)
}
}
func testRegionalDelayedBinding(c clientset.Interface, ns string) {
func testRegionalDelayedBinding(c clientset.Interface, ns string, pvcCount int) {
test := testsuites.StorageClassTest{
Name: "Regional PD storage class with waitForFirstConsumer test on GCE",
Provisioner: "kubernetes.io/gce-pd",
@ -311,9 +313,13 @@ func testRegionalDelayedBinding(c clientset.Interface, ns string) {
suffix := "delayed-regional"
class := newStorageClass(test, ns, suffix)
claim := newClaim(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name
pv, node := testBindingWaitForFirstConsumer(c, claim, class)
var claims []*v1.PersistentVolumeClaim
for i := 0; i < pvcCount; i++ {
claim := newClaim(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name
claims = append(claims, claim)
}
pvs, node := testBindingWaitForFirstConsumerMultiPVC(c, claims, class)
if node == nil {
framework.Failf("unexpected nil node found")
}
@ -321,7 +327,9 @@ func testRegionalDelayedBinding(c clientset.Interface, ns string) {
if !ok {
framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain)
}
checkZoneFromLabelAndAffinity(pv, zone, false)
for _, pv := range pvs {
checkZoneFromLabelAndAffinity(pv, zone, false)
}
}
func testRegionalAllowedTopologies(c clientset.Interface, ns string) {
@ -346,7 +354,7 @@ func testRegionalAllowedTopologies(c clientset.Interface, ns string) {
checkZonesFromLabelAndAffinity(pv, sets.NewString(zones...), true)
}
func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns string) {
func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns string, pvcCount int) {
test := testsuites.StorageClassTest{
Name: "Regional PD storage class with allowedTopologies and waitForFirstConsumer test on GCE",
Provisioner: "kubernetes.io/gce-pd",
@ -362,9 +370,13 @@ func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns s
class := newStorageClass(test, ns, suffix)
topoZones := getTwoRandomZones(c)
addAllowedTopologiesToStorageClass(c, class, topoZones)
claim := newClaim(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name
pv, node := testBindingWaitForFirstConsumer(c, claim, class)
var claims []*v1.PersistentVolumeClaim
for i := 0; i < pvcCount; i++ {
claim := newClaim(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name
claims = append(claims, claim)
}
pvs, node := testBindingWaitForFirstConsumerMultiPVC(c, claims, class)
if node == nil {
framework.Failf("unexpected nil node found")
}
@ -382,7 +394,9 @@ func testRegionalAllowedTopologiesWithDelayedBinding(c clientset.Interface, ns s
if !zoneFound {
framework.Failf("zones specified in AllowedTopologies: %v does not contain zone of node where PV got provisioned: %s", topoZones, nodeZone)
}
checkZonesFromLabelAndAffinity(pv, sets.NewString(topoZones...), true)
for _, pv := range pvs {
checkZonesFromLabelAndAffinity(pv, sets.NewString(topoZones...), true)
}
}
func getPVC(c clientset.Interface, ns string, pvcLabels map[string]string) *v1.PersistentVolumeClaim {

View File

@ -54,57 +54,87 @@ import (
const (
// Plugin name of the external provisioner
externalPluginName = "example.com/nfs"
// Number of PVCs for multi PVC tests
multiPVCcount = 3
)
func testBindingWaitForFirstConsumer(client clientset.Interface, claim *v1.PersistentVolumeClaim, class *storage.StorageClass) (*v1.PersistentVolume, *v1.Node) {
pvs, node := testBindingWaitForFirstConsumerMultiPVC(client, []*v1.PersistentVolumeClaim{claim}, class)
return pvs[0], node
}
func testBindingWaitForFirstConsumerMultiPVC(client clientset.Interface, claims []*v1.PersistentVolumeClaim, class *storage.StorageClass) ([]*v1.PersistentVolume, *v1.Node) {
var err error
Expect(len(claims)).ToNot(Equal(0))
namespace := claims[0].Namespace
By("creating a storage class " + class.Name)
class, err = client.StorageV1().StorageClasses().Create(class)
Expect(err).NotTo(HaveOccurred())
defer deleteStorageClass(client, class.Name)
By("creating a claim")
claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim)
Expect(err).NotTo(HaveOccurred())
By("creating claims")
var claimNames []string
var createdClaims []*v1.PersistentVolumeClaim
for _, claim := range claims {
c, err := client.CoreV1().PersistentVolumeClaims(claim.Namespace).Create(claim)
claimNames = append(claimNames, c.Name)
createdClaims = append(createdClaims, c)
Expect(err).NotTo(HaveOccurred())
}
defer func() {
framework.ExpectNoError(framework.DeletePersistentVolumeClaim(client, claim.Name, claim.Namespace), "Failed to delete PVC ", claim.Name)
var errors map[string]error
for _, claim := range createdClaims {
err := framework.DeletePersistentVolumeClaim(client, claim.Name, claim.Namespace)
if err != nil {
errors[claim.Name] = err
}
}
if len(errors) > 0 {
for claimName, err := range errors {
framework.Logf("Failed to delete PVC: %s due to error: %v", claimName, err)
}
}
}()
// Wait for ClaimProvisionTimeout and make sure the phase did not become Bound i.e. the Wait errors out
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, 2*time.Second, framework.ClaimProvisionShortTimeout)
// Wait for ClaimProvisionTimeout (across all PVCs in parallel) and make sure the phase did not become Bound i.e. the Wait errors out
By("checking the claims are in pending state")
err = framework.WaitForPersistentVolumeClaimsPhase(v1.ClaimBound, client, namespace, claimNames, 2*time.Second, framework.ClaimProvisionShortTimeout, true)
Expect(err).To(HaveOccurred())
for _, claim := range createdClaims {
// Get new copy of the claim
claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
Expect(claim.Status.Phase).To(Equal(v1.ClaimPending))
}
By("checking the claim is in pending state")
// Get new copy of the claim
claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
Expect(claim.Status.Phase).To(Equal(v1.ClaimPending))
By("creating a pod referring to the claim")
By("creating a pod referring to the claims")
// Create a pod referring to the claim and wait for it to get to running
pod, err := framework.CreateClientPod(client, claim.Namespace, claim)
pod, err := framework.CreatePod(client, namespace, nil /* nodeSelector */, createdClaims, true /* isPrivileged */, "" /* command */)
Expect(err).NotTo(HaveOccurred())
defer func() {
framework.DeletePodOrFail(client, pod.Namespace, pod.Name)
}()
By("re-checking the claim to see it binded")
// Get new copy of the claim
claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
// make sure claim did bind
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
Expect(err).NotTo(HaveOccurred())
// collect node and pv details
// collect node details
node, err := client.CoreV1().Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
By("re-checking the claims to see they binded")
var pvs []*v1.PersistentVolume
for _, claim := range createdClaims {
// Get new copy of the claim
claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
// make sure claim did bind
err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
Expect(err).NotTo(HaveOccurred())
return pv, node
pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
pvs = append(pvs, pv)
}
Expect(len(pvs)).ToNot(Equal(0))
return pvs, node
}
func checkZoneFromLabelAndAffinity(pv *v1.PersistentVolume, zone string, matchZone bool) {
@ -231,6 +261,67 @@ func checkGCEPD(volume *v1.PersistentVolume, volumeType string) error {
return nil
}
func testZonalDelayedBinding(c clientset.Interface, ns string, specifyAllowedTopology bool, pvcCount int) {
storageClassTestNameFmt := "Delayed binding %s storage class test %s"
storageClassTestNameSuffix := ""
if specifyAllowedTopology {
storageClassTestNameSuffix += " with AllowedTopologies"
}
tests := []testsuites.StorageClassTest{
{
Name: fmt.Sprintf(storageClassTestNameFmt, "EBS", storageClassTestNameSuffix),
CloudProviders: []string{"aws"},
Provisioner: "kubernetes.io/aws-ebs",
ClaimSize: "2Gi",
DelayBinding: true,
},
{
Name: fmt.Sprintf(storageClassTestNameFmt, "GCE PD", storageClassTestNameSuffix),
CloudProviders: []string{"gce", "gke"},
Provisioner: "kubernetes.io/gce-pd",
ClaimSize: "2Gi",
DelayBinding: true,
},
}
for _, test := range tests {
if !framework.ProviderIs(test.CloudProviders...) {
framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders)
continue
}
action := "creating claims with class with waitForFirstConsumer"
suffix := "delayed"
var topoZone string
class := newStorageClass(test, ns, suffix)
if specifyAllowedTopology {
action += " and allowedTopologies"
suffix += "-topo"
topoZone = getRandomCloudZone(c)
addSingleZoneAllowedTopologyToStorageClass(c, class, topoZone)
}
By(action)
var claims []*v1.PersistentVolumeClaim
for i := 0; i < pvcCount; i++ {
claim := newClaim(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name
claims = append(claims, claim)
}
pvs, node := testBindingWaitForFirstConsumerMultiPVC(c, claims, class)
if node == nil {
framework.Failf("unexpected nil node found")
}
zone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain]
if !ok {
framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain)
}
if specifyAllowedTopology && topoZone != zone {
framework.Failf("zone specified in allowedTopologies: %s does not match zone of node where PV got provisioned: %s", topoZone, zone)
}
for _, pv := range pvs {
checkZoneFromLabelAndAffinity(pv, zone, true)
}
}
}
var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
f := framework.NewDefaultFramework("volume-provisioning")
@ -846,43 +937,9 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
})
})
Describe("DynamicProvisioner delayed binding [Slow]", func() {
It("should create a persistent volume in the same zone as node after a pod mounting the claim is started", func() {
tests := []testsuites.StorageClassTest{
{
Name: "Delayed binding EBS storage class test",
CloudProviders: []string{"aws"},
Provisioner: "kubernetes.io/aws-ebs",
ClaimSize: "2Gi",
DelayBinding: true,
},
{
Name: "Delayed binding GCE PD storage class test",
CloudProviders: []string{"gce", "gke"},
Provisioner: "kubernetes.io/gce-pd",
ClaimSize: "2Gi",
DelayBinding: true,
},
}
for _, test := range tests {
if !framework.ProviderIs(test.CloudProviders...) {
framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders)
continue
}
By("creating a claim with class with waitForFirstConsumer")
suffix := "delayed"
class := newStorageClass(test, ns, suffix)
claim := newClaim(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name
pv, node := testBindingWaitForFirstConsumer(c, claim, class)
if node == nil {
framework.Failf("unexpected nil node found")
}
zone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain]
if !ok {
framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain)
}
checkZoneFromLabelAndAffinity(pv, zone, true)
}
It("should create persistent volumes in the same zone as node after a pod mounting the claims is started", func() {
testZonalDelayedBinding(c, ns, false /*specifyAllowedTopology*/, 1 /*pvcCount*/)
testZonalDelayedBinding(c, ns, false /*specifyAllowedTopology*/, 3 /*pvcCount*/)
})
})
Describe("DynamicProvisioner allowedTopologies", func() {
@ -921,51 +978,11 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
})
})
Describe("DynamicProvisioner delayed binding with allowedTopologies [Slow]", func() {
It("should create persistent volume in the same zone as specified in allowedTopologies after a pod mounting the claim is started", func() {
tests := []testsuites.StorageClassTest{
{
Name: "AllowedTopologies and delayed binding EBS storage class test",
CloudProviders: []string{"aws"},
Provisioner: "kubernetes.io/aws-ebs",
ClaimSize: "2Gi",
DelayBinding: true,
},
{
Name: "AllowedTopologies and delayed binding GCE PD storage class test",
CloudProviders: []string{"gce", "gke"},
Provisioner: "kubernetes.io/gce-pd",
ClaimSize: "2Gi",
DelayBinding: true,
},
}
for _, test := range tests {
if !framework.ProviderIs(test.CloudProviders...) {
framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders)
continue
}
By("creating a claim with class with WaitForFirstConsumer and allowedTopologies")
suffix := "delayed-topo"
class := newStorageClass(test, ns, suffix)
topoZone := getRandomCloudZone(c)
addSingleZoneAllowedTopologyToStorageClass(c, class, topoZone)
claim := newClaim(test, ns, suffix)
claim.Spec.StorageClassName = &class.Name
pv, node := testBindingWaitForFirstConsumer(c, claim, class)
if node == nil {
framework.Failf("unexpected nil node found")
}
nodeZone, ok := node.Labels[kubeletapis.LabelZoneFailureDomain]
if !ok {
framework.Failf("label %s not found on Node", kubeletapis.LabelZoneFailureDomain)
}
if topoZone != nodeZone {
framework.Failf("zone specified in allowedTopologies: %s does not match zone of node where PV got provisioned: %s", topoZone, nodeZone)
}
checkZoneFromLabelAndAffinity(pv, topoZone, true)
}
It("should create persistent volumes in the same zone as specified in allowedTopologies after a pod mounting the claims is started", func() {
testZonalDelayedBinding(c, ns, true /*specifyAllowedTopology*/, 1 /*pvcCount*/)
testZonalDelayedBinding(c, ns, true /*specifyAllowedTopology*/, 3 /*pvcCount*/)
})
})
})
func getDefaultStorageClassName(c clientset.Interface) string {