diff --git a/test/e2e/storage/BUILD b/test/e2e/storage/BUILD index c3f048265e..8385ca3767 100644 --- a/test/e2e/storage/BUILD +++ b/test/e2e/storage/BUILD @@ -64,7 +64,6 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/cloud-provider/volume/helpers:go_default_library", - "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/framework/metrics:go_default_library", diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go index 10d3d12679..4ff9ebfcea 100644 --- a/test/e2e/storage/csi_volumes.go +++ b/test/e2e/storage/csi_volumes.go @@ -22,6 +22,7 @@ import ( "fmt" "regexp" "strings" + "time" "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -29,7 +30,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" - csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1" csiclient "k8s.io/csi-api/pkg/client/clientset/versioned" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/podlogs" @@ -196,57 +196,49 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { }) // The CSIDriverRegistry feature gate is needed for this test in Kubernetes 1.12. - Context("CSI attach test using HostPath driver [Feature:CSIDriverRegistry]", func() { + + Context("CSI attach test using mock driver [Feature:CSIDriverRegistry]", func() { var ( driver testsuites.TestDriver ) - BeforeEach(func() { - config := testsuites.TestConfig{ - Framework: f, - Prefix: "csi-attach", - } - driver = drivers.InitHostPathCSIDriver(config) - driver.CreateDriver() - }) - - AfterEach(func() { - driver.CleanupDriver() - }) - tests := []struct { - name string - driverAttachable bool - driverExists bool - expectVolumeAttachment bool + name string + driverAttachable bool + driverExists bool }{ { - name: "non-attachable volume does not need VolumeAttachment", - driverAttachable: false, - driverExists: true, - expectVolumeAttachment: false, + name: "should not require VolumeAttach for drivers without attachment", + driverAttachable: false, + driverExists: true, }, { - name: "attachable volume needs VolumeAttachment", - driverAttachable: true, - driverExists: true, - expectVolumeAttachment: true, + name: "should require VolumeAttach for drivers with attachment", + driverAttachable: true, + driverExists: true, }, { - name: "volume with no CSI driver needs VolumeAttachment", - driverExists: false, - expectVolumeAttachment: true, + name: "should preserve attachment policy when no CSIDriver present", + driverAttachable: true, + driverExists: false, }, } for _, t := range tests { test := t It(test.name, func() { + By("Deploying mock CSI driver") + config := testsuites.TestConfig{ + Framework: f, + Prefix: "csi-attach", + } + + driver = drivers.InitMockCSIDriver(config, test.driverExists, test.driverAttachable, nil) + driver.CreateDriver() + if test.driverExists { - csiDriver := createCSIDriver(csics, testsuites.GetUniqueDriverName(driver), test.driverAttachable, nil) - if csiDriver != nil { - defer csics.CsiV1alpha1().CSIDrivers().Delete(csiDriver.Name, nil) - } + defer destroyCSIDriver(csics, driver) + defer driver.CleanupDriver() } By("Creating pod") @@ -270,7 +262,8 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { defer cs.StorageV1().StorageClasses().Delete(class.Name, nil) } if claim != nil { - defer cs.CoreV1().PersistentVolumeClaims(ns.Name).Delete(claim.Name, nil) + // Fully delete PV before deleting CSI driver + defer deleteVolume(cs, claim) } if pod != nil { // Fully delete (=unmount) the pod before deleting CSI driver @@ -284,28 +277,27 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { framework.ExpectNoError(err, "Failed to start pod: %v", err) By("Checking if VolumeAttachment was created for the pod") - // Check that VolumeAttachment does not exist handle := getVolumeHandle(cs, claim) attachmentHash := sha256.Sum256([]byte(fmt.Sprintf("%s%s%s", handle, scTest.Provisioner, nodeName))) attachmentName := fmt.Sprintf("csi-%x", attachmentHash) _, err = cs.StorageV1beta1().VolumeAttachments().Get(attachmentName, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { - if test.expectVolumeAttachment { + if test.driverAttachable { framework.ExpectNoError(err, "Expected VolumeAttachment but none was found") } } else { framework.ExpectNoError(err, "Failed to find VolumeAttachment") } } - if !test.expectVolumeAttachment { + if !test.driverAttachable { Expect(err).To(HaveOccurred(), "Unexpected VolumeAttachment found") } }) } }) - Context("CSI workload information [Feature:CSIDriverRegistry]", func() { + Context("CSI workload information using mock driver [Feature:CSIDriverRegistry]", func() { var ( driver testsuites.TestDriver podInfoV1 = "v1" @@ -313,19 +305,6 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { podInfoEmpty = "" ) - BeforeEach(func() { - config := testsuites.TestConfig{ - Framework: f, - Prefix: "csi-workload", - } - driver = drivers.InitMockCSIDriver(config) - driver.CreateDriver() - }) - - AfterEach(func() { - driver.CleanupDriver() - }) - tests := []struct { name string podInfoOnMountVersion *string @@ -365,11 +344,18 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { for _, t := range tests { test := t It(test.name, func() { + By("Deploying mock CSI driver") + config := testsuites.TestConfig{ + Framework: f, + Prefix: "csi-workload", + } + + driver = drivers.InitMockCSIDriver(config, test.driverExists, true, test.podInfoOnMountVersion) + driver.CreateDriver() + if test.driverExists { - csiDriver := createCSIDriver(csics, testsuites.GetUniqueDriverName(driver), true, test.podInfoOnMountVersion) - if csiDriver != nil { - defer csics.CsiV1alpha1().CSIDrivers().Delete(csiDriver.Name, nil) - } + defer destroyCSIDriver(csics, driver) + defer driver.CleanupDriver() } By("Creating pod") @@ -397,7 +383,8 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { defer cs.StorageV1().StorageClasses().Delete(class.Name, nil) } if claim != nil { - defer cs.CoreV1().PersistentVolumeClaims(ns.Name).Delete(claim.Name, nil) + // Fully delete PV before deleting CSI driver + defer deleteVolume(cs, claim) } if pod != nil { // Fully delete (=unmount) the pod before deleting CSI driver @@ -472,20 +459,15 @@ func testTopologyNegative(cs clientset.Interface, suffix, namespace string, dela } } -func createCSIDriver(csics csiclient.Interface, name string, attachable bool, podInfoOnMountVersion *string) *csiv1alpha1.CSIDriver { - By("Creating CSIDriver instance") - driver := &csiv1alpha1.CSIDriver{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: csiv1alpha1.CSIDriverSpec{ - AttachRequired: &attachable, - PodInfoOnMountVersion: podInfoOnMountVersion, - }, +func destroyCSIDriver(csics csiclient.Interface, driver testsuites.TestDriver) { + driverName := testsuites.GetUniqueDriverName(driver) + driverGet, err := csics.CsiV1alpha1().CSIDrivers().Get(driverName, metav1.GetOptions{}) + if err == nil { + framework.Logf("deleting %s.%s: %s", driverGet.TypeMeta.APIVersion, driverGet.TypeMeta.Kind, driverGet.ObjectMeta.Name) + // Uncomment the following line to get full dump of CSIDriver object + // framework.Logf("%s", framework.PrettyPrint(driverGet)) + csics.CsiV1alpha1().CSIDrivers().Delete(driverName, nil) } - driver, err := csics.CsiV1alpha1().CSIDrivers().Create(driver) - framework.ExpectNoError(err, "Failed to create CSIDriver: %v", err) - return driver } func getVolumeHandle(cs clientset.Interface, claim *v1.PersistentVolumeClaim) string { @@ -508,6 +490,15 @@ func getVolumeHandle(cs clientset.Interface, claim *v1.PersistentVolumeClaim) st return pv.Spec.CSI.VolumeHandle } +func deleteVolume(cs clientset.Interface, claim *v1.PersistentVolumeClaim) { + // re-get the claim to the latest state with bound volume + claim, err := cs.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{}) + if err == nil { + cs.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, nil) + framework.WaitForPersistentVolumeDeleted(cs, claim.Spec.VolumeName, 2*time.Second, 2*time.Minute) + } +} + func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node testsuites.NodeSelection, ns string) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) { class := newStorageClass(t, ns, "") class, err := cs.StorageV1().StorageClasses().Create(class) diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 8bfbabf579..9d384c2e4f 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -166,15 +166,39 @@ func (h *hostpathCSIDriver) CleanupDriver() { // mockCSI type mockCSIDriver struct { - cleanup func() - driverInfo testsuites.DriverInfo + cleanup func() + driverInfo testsuites.DriverInfo + manifests []string + podInfoVersion *string } var _ testsuites.TestDriver = &mockCSIDriver{} var _ testsuites.DynamicPVTestDriver = &mockCSIDriver{} // InitMockCSIDriver returns a mockCSIDriver that implements TestDriver interface -func InitMockCSIDriver(config testsuites.TestConfig) testsuites.TestDriver { +func InitMockCSIDriver(config testsuites.TestConfig, registerDriver, driverAttachable bool, podInfoVersion *string) testsuites.TestDriver { + driverManifests := []string{ + "test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml", + "test/e2e/testing-manifests/storage-csi/mock/csi-storageclass.yaml", + "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml", + } + + config.ServerConfig = &framework.VolumeTestConfig{} + + if registerDriver { + driverManifests = append(driverManifests, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml") + } + + if driverAttachable { + driverManifests = append(driverManifests, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-attacher.yaml") + } else { + config.ServerConfig.ServerArgs = append(config.ServerConfig.ServerArgs, "--disable-attach") + } + return &mockCSIDriver{ driverInfo: testsuites.DriverInfo{ Name: "csi-mock", @@ -190,6 +214,8 @@ func InitMockCSIDriver(config testsuites.TestConfig) testsuites.TestDriver { }, Config: config, }, + manifests: driverManifests, + podInfoVersion: podInfoVersion, } } @@ -223,26 +249,28 @@ func (m *mockCSIDriver) CreateDriver() { node := nodes.Items[rand.Intn(len(nodes.Items))] m.driverInfo.Config.ClientNodeName = node.Name + containerArgs := []string{"--name=csi-mock-" + f.UniqueName} + + if m.driverInfo.Config.ServerConfig != nil && m.driverInfo.Config.ServerConfig.ServerArgs != nil { + containerArgs = append(containerArgs, m.driverInfo.Config.ServerConfig.ServerArgs...) + } + // TODO (?): the storage.csi.image.version and storage.csi.image.registry // settings are ignored for this test. We could patch the image definitions. o := utils.PatchCSIOptions{ - OldDriverName: "csi-mock", - NewDriverName: "csi-mock-" + f.UniqueName, - DriverContainerName: "mock", - DriverContainerArguments: []string{"--name=csi-mock-" + f.UniqueName}, - ProvisionerContainerName: "csi-provisioner", - NodeName: m.driverInfo.Config.ClientNodeName, + OldDriverName: "csi-mock", + NewDriverName: "csi-mock-" + f.UniqueName, + DriverContainerName: "mock", + DriverContainerArguments: containerArgs, + ProvisionerContainerName: "csi-provisioner", + ClusterRegistrarContainerName: "csi-cluster-driver-registrar", + NodeName: m.driverInfo.Config.ClientNodeName, + PodInfoVersion: m.podInfoVersion, } cleanup, err := f.CreateFromManifests(func(item interface{}) error { return utils.PatchCSIDeployment(f, o, item) }, - "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", - "test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml", - "test/e2e/testing-manifests/storage-csi/mock/csi-storageclass.yaml", - "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml", - ) + m.manifests...) m.cleanup = cleanup if err != nil { framework.Failf("deploying csi mock driver: %v", err) diff --git a/test/e2e/storage/utils/deployment.go b/test/e2e/storage/utils/deployment.go index e1885eb9e4..e899f2ca88 100644 --- a/test/e2e/storage/utils/deployment.go +++ b/test/e2e/storage/utils/deployment.go @@ -97,6 +97,10 @@ func PatchCSIDeployment(f *framework.Framework, o PatchCSIOptions, object interf // Driver name is expected to be the same // as the snapshotter here. container.Args = append(container.Args, "--snapshotter="+o.NewDriverName) + case o.ClusterRegistrarContainerName: + if o.PodInfoVersion != nil { + container.Args = append(container.Args, "--pod-info-mount-version="+*o.PodInfoVersion) + } } } } @@ -153,6 +157,12 @@ type PatchCSIOptions struct { // If non-empty, --snapshotter with new name will be appended // to the argument list. SnapshotterContainerName string + // The name of the container which has the cluster-driver-registrar + // binary. + ClusterRegistrarContainerName string // If non-empty, all pods are forced to run on this node. NodeName string + // If not nil, the argument to pass to the cluster-driver-registrar's + // pod-info-mount-version argument. + PodInfoVersion *string } diff --git a/test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/README.md b/test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/README.md new file mode 100644 index 0000000000..6247e1c98d --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/README.md @@ -0,0 +1 @@ +The original file is (or will be) https://github.com/kubernetes-csi/cluster-driver-registrar/blob/master/deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml b/test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml new file mode 100644 index 0000000000..c3c593537e --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml @@ -0,0 +1,38 @@ +# This YAML file contains all RBAC objects that are necessary to run +# cluster-driver-registrar. +# +# In production, each CSI driver deployment has to be customized: +# - to avoid conflicts, use non-default namespace and different names +# for non-namespaced entities like the ClusterRole + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-cluster-driver-registrar + # replace with non-default namespace name + namespace: default + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cluster-driver-registrar-runner +rules: + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csidrivers"] + verbs: ["create", "delete"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-cluster-driver-registrar-role +subjects: + - kind: ServiceAccount + name: csi-cluster-driver-registrar + # replace with non-default namespace name + namespace: default +roleRef: + kind: ClusterRole + name: cluster-driver-registrar-runner + apiGroup: rbac.authorization.k8s.io diff --git a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml new file mode 100644 index 0000000000..72f0ebf1bc --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml @@ -0,0 +1,33 @@ +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-mockplugin-cluster-driver-registrar +spec: + selector: + matchLabels: + app: csi-mockplugin-cluster-driver-registrar + replicas: 1 + template: + metadata: + labels: + app: csi-mockplugin-cluster-driver-registrar + spec: + serviceAccountName: csi-mock + containers: + - name: csi-cluster-driver-registrar + image: quay.io/k8scsi/csi-cluster-driver-registrar:canary + args: + - --v=5 + - --csi-address=$(ADDRESS) + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: Always + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-mock + type: DirectoryOrCreate + name: socket-dir diff --git a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-attacher.yaml b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-attacher.yaml new file mode 100644 index 0000000000..ef9a1cbb64 --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver-attacher.yaml @@ -0,0 +1,34 @@ +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-mockplugin-attacher +spec: + selector: + matchLabels: + app: csi-mockplugin-attacher + replicas: 1 + template: + metadata: + labels: + app: csi-mockplugin-attacher + spec: + serviceAccountName: csi-mock + containers: + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v1.0.1 + args: + - --v=5 + - --csi-address=$(ADDRESS) + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: Always + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-mock + type: DirectoryOrCreate + name: socket-dir + diff --git a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml index 1592ccda52..6aeb02b0aa 100644 --- a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml +++ b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml @@ -16,18 +16,6 @@ spec: containers: - - name: csi-attacher - image: quay.io/k8scsi/csi-attacher:v1.0.1 - args: - - --v=5 - - --csi-address=$(ADDRESS) - env: - - name: ADDRESS - value: /csi/csi.sock - imagePullPolicy: Always - volumeMounts: - - mountPath: /csi - name: socket-dir - name: csi-provisioner image: quay.io/k8scsi/csi-provisioner:v1.0.1 args: diff --git a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml index dd09a7dc44..dff014b6f2 100644 --- a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml @@ -31,6 +31,20 @@ roleRef: name: external-provisioner-runner apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-controller-cluster-driver-registrar-role +subjects: + - kind: ServiceAccount + name: csi-mock + namespace: default +roleRef: + kind: ClusterRole + name: cluster-driver-registrar-runner + apiGroup: rbac.authorization.k8s.io + --- # priviledged Pod Security Policy, previously defined via PrivilegedTestPSPClusterRoleBinding() kind: ClusterRoleBinding