From 4fbf25f8c7681398a00f46c5fe9397b2ae0aaebe Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Thu, 2 Aug 2018 11:07:19 -0700 Subject: [PATCH] Rework multi-volume test to use StatefulSet --- test/e2e/storage/persistent_volumes.go | 134 +++++++++++++++++++------ 1 file changed, 102 insertions(+), 32 deletions(-) diff --git a/test/e2e/storage/persistent_volumes.go b/test/e2e/storage/persistent_volumes.go index acf126d4ef..d20dbf4076 100644 --- a/test/e2e/storage/persistent_volumes.go +++ b/test/e2e/storage/persistent_volumes.go @@ -23,6 +23,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -30,6 +31,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/utils" + imageutils "k8s.io/kubernetes/test/utils/image" ) // Validate PV/PVC, create and verify writer pod, delete the PVC, and validate the PV's @@ -303,50 +305,118 @@ var _ = utils.SIGDescribe("PersistentVolumes", func() { Describe("Default StorageClass", func() { Context("pods that use multiple volumes", func() { + + AfterEach(func() { + framework.DeleteAllStatefulSets(c, ns) + }) + It("should be reschedulable", func() { // Only run on providers with default storageclass framework.SkipUnlessProviderIs("openstack", "gce", "gke", "vsphere", "azure") numVols := 4 - pvcs := []*v1.PersistentVolumeClaim{} + ssTester := framework.NewStatefulSetTester(c) - By("Creating PVCs") - for i := 0; i < numVols; i++ { - pvc = framework.MakePersistentVolumeClaim(framework.PersistentVolumeClaimConfig{}, ns) - pvc, err = framework.CreatePVC(c, ns, pvc) - Expect(err).NotTo(HaveOccurred()) - pvcs = append(pvcs, pvc) - } - - By("Waiting for PVCs to be bound") - for _, pvc := range pvcs { - framework.Logf("Created PVC %q", pvc.Name) - framework.ExpectNoError(framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, ns, pvc.Name, framework.Poll, framework.ClaimProvisionTimeout)) - } - - By("Creating a pod and initializing data") + By("Creating a StatefulSet pod to initialize data") writeCmd := "true" - for i, pvc := range pvcs { - // mountPath is /mnt/volume - writeCmd += fmt.Sprintf("&& touch /mnt/volume%v/%v", i+1, pvc.Name) + for i := 0; i < numVols; i++ { + writeCmd += fmt.Sprintf("&& touch %v", getVolumeFile(i)) } - pod := framework.MakePod(ns, nil, pvcs, false, writeCmd) - pod, err = c.CoreV1().Pods(ns).Create(pod) - Expect(err).NotTo(HaveOccurred()) - framework.ExpectNoError(framework.WaitForPodSuccessInNamespace(c, pod.Name, ns)) + writeCmd += "&& sleep 10000" - By("Recreating the pod and validating the data") - framework.ExpectNoError(framework.DeletePodWithWait(f, c, pod)) - validateCmd := "true" - for i, pvc := range pvcs { - // mountPath is /mnt/volume - validateCmd += fmt.Sprintf("&& test -f /mnt/volume%v/%v", i+1, pvc.Name) + probe := &v1.Probe{ + Handler: v1.Handler{ + Exec: &v1.ExecAction{ + // Check that the last file got created + Command: []string{"test", "-f", getVolumeFile(numVols - 1)}, + }, + }, + InitialDelaySeconds: 1, + PeriodSeconds: 1, } - pod = framework.MakePod(ns, nil, pvcs, false, validateCmd) - pod, err = c.CoreV1().Pods(ns).Create(pod) + + mounts := []v1.VolumeMount{} + claims := []v1.PersistentVolumeClaim{} + for i := 0; i < numVols; i++ { + pvc := framework.MakePersistentVolumeClaim(framework.PersistentVolumeClaimConfig{}, ns) + pvc.Name = getVolName(i) + mounts = append(mounts, v1.VolumeMount{Name: pvc.Name, MountPath: getMountPath(i)}) + claims = append(claims, *pvc) + } + + spec := makeStatefulSetWithPVCs(ns, writeCmd, mounts, claims, probe) + ss, err := c.AppsV1().StatefulSets(ns).Create(spec) Expect(err).NotTo(HaveOccurred()) - framework.ExpectNoError(framework.WaitForPodSuccessInNamespace(c, pod.Name, ns)) + ssTester.WaitForRunningAndReady(1, ss) + + By("Deleting the StatefulSet but not the volumes") + // Scale down to 0 first so that the Delete is quick + ss, err = ssTester.Scale(ss, 0) + Expect(err).NotTo(HaveOccurred()) + ssTester.WaitForStatusReplicas(ss, 0) + err = c.AppsV1().StatefulSets(ns).Delete(ss.Name, &metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a new Statefulset and validating the data") + validateCmd := "true" + for i := 0; i < numVols; i++ { + validateCmd += fmt.Sprintf("&& test -f %v", getVolumeFile(i)) + } + validateCmd += "&& sleep 10000" + + spec = makeStatefulSetWithPVCs(ns, validateCmd, mounts, claims, probe) + ss, err = c.AppsV1().StatefulSets(ns).Create(spec) + Expect(err).NotTo(HaveOccurred()) + ssTester.WaitForRunningAndReady(1, ss) }) }) }) }) + +func getVolName(i int) string { + return fmt.Sprintf("vol%v", i) +} + +func getMountPath(i int) string { + return fmt.Sprintf("/mnt/%v", getVolName(i)) +} + +func getVolumeFile(i int) string { + return fmt.Sprintf("%v/data%v", getMountPath(i), i) +} + +func makeStatefulSetWithPVCs(ns, cmd string, mounts []v1.VolumeMount, claims []v1.PersistentVolumeClaim, readyProbe *v1.Probe) *appsv1.StatefulSet { + ssReplicas := int32(1) + + labels := map[string]string{"app": "many-volumes-test"} + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "many-volumes-test", + Namespace: ns, + }, + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "many-volumes-test"}, + }, + Replicas: &ssReplicas, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: imageutils.GetE2EImage(imageutils.NginxSlim), + Command: []string{"/bin/sh"}, + Args: []string{"-c", cmd}, + VolumeMounts: mounts, + ReadinessProbe: readyProbe, + }, + }, + }, + }, + VolumeClaimTemplates: claims, + }, + } +}