From f1cef9bde456273133de8581444507e897ef31dc Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Tue, 28 Aug 2018 15:27:41 +0200 Subject: [PATCH] Add e2e test for skipping attach --- pkg/volume/csi/csi_attacher_test.go | 4 +- pkg/volume/csi/csi_mounter_test.go | 6 +- test/e2e/framework/BUILD | 1 + test/e2e/framework/framework.go | 7 ++ test/e2e/storage/BUILD | 2 + test/e2e/storage/csi_volumes.go | 185 ++++++++++++++++++++++++++++ 6 files changed, 200 insertions(+), 5 deletions(-) diff --git a/pkg/volume/csi/csi_attacher_test.go b/pkg/volume/csi/csi_attacher_test.go index 2dea95a666..99f4f3f094 100644 --- a/pkg/volume/csi/csi_attacher_test.go +++ b/pkg/volume/csi/csi_attacher_test.go @@ -42,8 +42,8 @@ import ( ) var ( - bFalse bool = false - bTrue bool = true + bFalse = false + bTrue = true ) func makeTestAttachment(attachID, nodeName, pvName string) *storage.VolumeAttachment { diff --git a/pkg/volume/csi/csi_mounter_test.go b/pkg/volume/csi/csi_mounter_test.go index 15cc9646fc..f0334d84ff 100644 --- a/pkg/volume/csi/csi_mounter_test.go +++ b/pkg/volume/csi/csi_mounter_test.go @@ -268,14 +268,14 @@ func TestSaveVolumeData(t *testing.T) { } func getCSIDriver(name string, requiresPodInfo *bool, attachable *bool) *csiapi.CSIDriver { + podInfoMountVersion := "v1" return &csiapi.CSIDriver{ ObjectMeta: meta.ObjectMeta{ Name: name, }, Spec: csiapi.CSIDriverSpec{ - Driver: name, - PodInfoRequiredOnMount: requiresPodInfo, - AttachRequired: attachable, + PodInfoOnMountVersion: &podInfoMountVersion, + AttachRequired: attachable, }, } } diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index e468c6568e..39c32e1401 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -133,6 +133,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/remotecommand:go_default_library", "//staging/src/k8s.io/client-go/tools/watch:go_default_library", "//staging/src/k8s.io/client-go/util/retry:go_default_library", + "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset:go_default_library", "//test/e2e/framework/ginkgowrapper:go_default_library", "//test/e2e/framework/metrics:go_default_library", diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index fcb1959b41..625c0505e4 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -42,6 +42,7 @@ import ( "k8s.io/client-go/restmapper" scaleclient "k8s.io/client-go/scale" "k8s.io/client-go/tools/clientcmd" + csi "k8s.io/csi-api/pkg/client/clientset/versioned" aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" @@ -67,6 +68,7 @@ type Framework struct { ClientSet clientset.Interface KubemarkExternalClusterClientSet clientset.Interface + CSIClientSet csi.Interface InternalClientset *internalclientset.Clientset AggregatorClient *aggregatorclient.Clientset @@ -181,6 +183,11 @@ func (f *Framework) BeforeEach() { Expect(err).NotTo(HaveOccurred()) f.DynamicClient, err = dynamic.NewForConfig(config) Expect(err).NotTo(HaveOccurred()) + // csi.storage.k8s.io is based on CRD, which is served only as JSON + jsonConfig := config + jsonConfig.ContentType = "application/json" + f.CSIClientSet, err = csi.NewForConfig(jsonConfig) + Expect(err).NotTo(HaveOccurred()) // create scales getter, set GroupVersion and NegotiatedSerializer to default values // as they are required when creating a REST client. diff --git a/test/e2e/storage/BUILD b/test/e2e/storage/BUILD index 29a0f59911..53c30eaf35 100644 --- a/test/e2e/storage/BUILD +++ b/test/e2e/storage/BUILD @@ -62,6 +62,8 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", + "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", + "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/framework/metrics:go_default_library", "//test/e2e/generated:go_default_library", diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go index 82397eb008..acb989d6ef 100644 --- a/test/e2e/storage/csi_volumes.go +++ b/test/e2e/storage/csi_volumes.go @@ -23,10 +23,18 @@ import ( "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" + csi "k8s.io/csi-api/pkg/apis/csi/v1alpha1" + csiclient "k8s.io/csi-api/pkg/client/clientset/versioned" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/utils" + imageutils "k8s.io/kubernetes/test/utils/image" + + "crypto/sha256" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -55,6 +63,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { var ( cs clientset.Interface + csics csiclient.Interface ns *v1.Namespace node v1.Node config framework.VolumeTestConfig @@ -62,6 +71,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { BeforeEach(func() { cs = f.ClientSet + csics = f.CSIClientSet ns = f.Namespace nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet) node = nodes.Items[rand.Intn(len(nodes.Items))] @@ -102,8 +112,183 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { }) }) } + + // Use [Serial], because there can be only one CSIDriver for csi-hostpath driver. + Context("CSI attach test using HostPath driver [Serial][Feature:CSISkipAttach]", func() { + var ( + driver csiTestDriver + ) + BeforeEach(func() { + driver = initCSIHostpath(f, config) + driver.createCSIDriver() + }) + + AfterEach(func() { + driver.cleanupCSIDriver() + }) + + tests := []struct { + name string + driverAttachable bool + driverExists bool + expectVolumeAttachment bool + }{ + { + name: "non-attachable volume does not need VolumeAttachment", + driverAttachable: false, + driverExists: true, + expectVolumeAttachment: false, + }, + { + name: "attachable volume needs VolumeAttachment", + driverAttachable: true, + driverExists: true, + expectVolumeAttachment: true, + }, + { + name: "volume with no CSI driver needs VolumeAttachment", + driverExists: false, + expectVolumeAttachment: true, + }, + } + + for _, t := range tests { + test := t + It(test.name, func() { + if test.driverExists { + driver := createCSIDriver(csics, test.driverAttachable) + if driver != nil { + defer csics.CsiV1alpha1().CSIDrivers().Delete(driver.Name, nil) + } + } + + By("Creating pod") + t := driver.createStorageClassTest(node) + class, claim, pod := startPausePod(cs, t, ns.Name) + if class != nil { + defer cs.StorageV1().StorageClasses().Delete(class.Name, nil) + } + if claim != nil { + defer cs.CoreV1().PersistentVolumeClaims(ns.Name).Delete(claim.Name, nil) + } + if pod != nil { + // Fully delete (=unmount) the pod before deleting CSI driver + defer framework.DeletePodWithWait(f, cs, pod) + } + if pod == nil { + return + } + + err := framework.WaitForPodNameRunningInNamespace(cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "Failed to start pod: %v", err) + + By("Checking if VolumeAttachment was created for the pod") + // Check that VolumeAttachment does not exist + handle := getVolumeHandle(cs, claim) + attachmentHash := sha256.Sum256([]byte(fmt.Sprintf("%s%s%s", handle, t.provisioner, node.Name))) + attachmentName := fmt.Sprintf("csi-%x", attachmentHash) + _, err = cs.StorageV1beta1().VolumeAttachments().Get(attachmentName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + if test.expectVolumeAttachment { + framework.ExpectNoError(err, "Expected VolumeAttachment but none was found") + } + } else { + framework.ExpectNoError(err, "Failed to find VolumeAttachment") + } + } + if !test.expectVolumeAttachment { + Expect(err).To(HaveOccurred(), "Unexpected VolumeAttachment found") + } + }) + } + }) }) +func createCSIDriver(csics csiclient.Interface, attachable bool) *csi.CSIDriver { + By("Creating CSIDriver instance") + driver := &csi.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csi-hostpath", + }, + Spec: csi.CSIDriverSpec{ + AttachRequired: &attachable, + }, + } + driver, err := csics.CsiV1alpha1().CSIDrivers().Create(driver) + framework.ExpectNoError(err, "Failed to create CSIDriver: %v", err) + return driver +} + +func getVolumeHandle(cs clientset.Interface, claim *v1.PersistentVolumeClaim) string { + // re-get the claim to the the latest state with bound volume + claim, err := cs.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{}) + if err != nil { + framework.ExpectNoError(err, "Cannot get PVC") + return "" + } + pvName := claim.Spec.VolumeName + pv, err := cs.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{}) + if err != nil { + framework.ExpectNoError(err, "Cannot get PV") + return "" + } + if pv.Spec.CSI == nil { + Expect(pv.Spec.CSI).NotTo(BeNil()) + return "" + } + return pv.Spec.CSI.VolumeHandle +} + +func startPausePod(cs clientset.Interface, t storageClassTest, ns string) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) { + class := newStorageClass(t, ns, "") + class, err := cs.StorageV1().StorageClasses().Create(class) + framework.ExpectNoError(err, "Failed to create class : %v", err) + claim := newClaim(t, ns, "") + claim.Spec.StorageClassName = &class.Name + claim, err = cs.CoreV1().PersistentVolumeClaims(ns).Create(claim) + framework.ExpectNoError(err, "Failed to create claim: %v", err) + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "pvc-volume-tester-", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "volume-tester", + Image: imageutils.GetE2EImage(imageutils.Pause), + VolumeMounts: []v1.VolumeMount{ + { + Name: "my-volume", + MountPath: "/mnt/test", + }, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + Volumes: []v1.Volume{ + { + Name: "my-volume", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: claim.Name, + ReadOnly: false, + }, + }, + }, + }, + }, + } + + if len(t.nodeName) != 0 { + pod.Spec.NodeName = t.nodeName + } + pod, err = cs.CoreV1().Pods(ns).Create(pod) + framework.ExpectNoError(err, "Failed to create pod: %v", err) + return class, claim, pod +} + type hostpathCSIDriver struct { combinedClusterRoleNames []string serviceAccount *v1.ServiceAccount