From d69e0ff3f0547cc1acc6b35af1e6b93b90ed0e45 Mon Sep 17 00:00:00 2001 From: Xing Yang Date: Fri, 1 Mar 2019 18:30:24 -0800 Subject: [PATCH] Enable CSI test suite and fix test failures --- test/e2e/storage/csi_mock_volume.go | 1 - test/e2e/storage/csi_volumes.go | 6 +- test/integration/auth/BUILD | 4 +- test/integration/auth/node_test.go | 94 +++++++++++------------------ test/integration/auth/rbac_test.go | 8 --- test/integration/etcd/data.go | 2 +- 6 files changed, 37 insertions(+), 78 deletions(-) diff --git a/test/e2e/storage/csi_mock_volume.go b/test/e2e/storage/csi_mock_volume.go index 28b0769f22..31ebd2830e 100644 --- a/test/e2e/storage/csi_mock_volume.go +++ b/test/e2e/storage/csi_mock_volume.go @@ -33,7 +33,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" - //csiclient "k8s.io/csi-api/pkg/client/clientset/versioned" volumeutil "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/drivers" diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go index 11a5e91b4f..d4234434cf 100644 --- a/test/e2e/storage/csi_volumes.go +++ b/test/e2e/storage/csi_volumes.go @@ -55,11 +55,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { curDriver := initDriver() Context(testsuites.GetDriverNameWithFeatureTags(curDriver), func() { - // TODO(xyang): Disable the CSI tests until the sidecar container images - // are updated to support CSINodeInfo and CSIDriver Core APIs in the - // following PR: - // https://github.com/kubernetes/kubernetes/pull/73883 - //testsuites.DefineTestSuite(curDriver, csiTestSuites) + testsuites.DefineTestSuite(curDriver, csiTestSuites) }) } diff --git a/test/integration/auth/BUILD b/test/integration/auth/BUILD index 31405070a5..545af5ad2d 100644 --- a/test/integration/auth/BUILD +++ b/test/integration/auth/BUILD @@ -52,6 +52,7 @@ go_test( "//staging/src/k8s.io/api/authentication/v1beta1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", + "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", @@ -87,11 +88,8 @@ go_test( "//staging/src/k8s.io/client-go/transport:go_default_library", "//staging/src/k8s.io/client-go/util/keyutil:go_default_library", "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", - "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", - "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", "//test/e2e/lifecycle/bootstrap:go_default_library", "//test/integration:go_default_library", - "//test/integration/etcd:go_default_library", "//test/integration/framework:go_default_library", "//vendor/gopkg.in/square/go-jose.v2/jwt:go_default_library", "//vendor/k8s.io/klog:go_default_library", diff --git a/test/integration/auth/node_test.go b/test/integration/auth/node_test.go index 868418c3ad..bd652c38cf 100644 --- a/test/integration/auth/node_test.go +++ b/test/integration/auth/node_test.go @@ -24,6 +24,7 @@ import ( "time" storagev1 "k8s.io/api/storage/v1" + storagev1beta1 "k8s.io/api/storage/v1beta1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -34,8 +35,6 @@ import ( utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" externalclientset "k8s.io/client-go/kubernetes" - csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1" - csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/coordination" @@ -44,7 +43,6 @@ import ( "k8s.io/kubernetes/pkg/apis/policy" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/features" - "k8s.io/kubernetes/test/integration/etcd" "k8s.io/kubernetes/test/integration/framework" "k8s.io/utils/pointer" ) @@ -67,7 +65,7 @@ func TestNodeAuthorizer(t *testing.T) { // Enable NodeLease feature so that nodes can create leases defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NodeLease, true)() - // Enable CSINodeInfo feature so that nodes can create CSINodeInfo objects. + // Enable CSINodeInfo feature so that nodes can create CSINode objects. defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSINodeInfo, true)() tokenFile, err := ioutil.TempFile("", "kubeconfig") @@ -95,7 +93,6 @@ func TestNodeAuthorizer(t *testing.T) { // Build client config and superuser clientset clientConfig := server.ClientConfig superuserClient, superuserClientExternal := clientsetForToken(tokenMaster, clientConfig) - superuserCRDClient := crdClientsetForToken(tokenMaster, clientConfig) // Wait for a healthy server for { @@ -158,13 +155,6 @@ func TestNodeAuthorizer(t *testing.T) { t.Fatal(err) } - csiNodeInfoCRD, err := crdFromManifest("../../../staging/src/k8s.io/csi-api/pkg/crd/manifests/csinodeinfo.yaml") - if err != nil { - t.Fatal(err) - } - - etcd.CreateTestCRDs(t, superuserCRDClient, false, csiNodeInfoCRD) - getSecret := func(client clientset.Interface) func() error { return func() error { _, err := client.Core().Secrets("ns").Get("mysecret", metav1.GetOptions{}) @@ -412,84 +402,68 @@ func TestNodeAuthorizer(t *testing.T) { } } - getNode1CSINodeInfo := func(client csiclientset.Interface) func() error { + getNode1CSINode := func(client externalclientset.Interface) func() error { return func() error { - _, err := client.CsiV1alpha1().CSINodeInfos().Get("node1", metav1.GetOptions{}) + _, err := client.StorageV1beta1().CSINodes().Get("node1", metav1.GetOptions{}) return err } } - createNode1CSINodeInfo := func(client csiclientset.Interface) func() error { + createNode1CSINode := func(client externalclientset.Interface) func() error { return func() error { - nodeInfo := &csiv1alpha1.CSINodeInfo{ + nodeInfo := &storagev1beta1.CSINode{ ObjectMeta: metav1.ObjectMeta{ Name: "node1", }, - Spec: csiv1alpha1.CSINodeInfoSpec{ - Drivers: []csiv1alpha1.CSIDriverInfoSpec{ + Spec: storagev1beta1.CSINodeSpec{ + Drivers: []storagev1beta1.CSINodeDriver{ { - Name: "com.example.csi/driver1", + Name: "com.example.csi.driver1", NodeID: "com.example.csi/node1", TopologyKeys: []string{"com.example.csi/zone"}, }, }, }, - Status: csiv1alpha1.CSINodeInfoStatus{ - Drivers: []csiv1alpha1.CSIDriverInfoStatus{ - { - Name: "com.example.csi/driver1", - Available: true, - VolumePluginMechanism: csiv1alpha1.VolumePluginMechanismInTree, - }, - }, - }, } - _, err := client.CsiV1alpha1().CSINodeInfos().Create(nodeInfo) + _, err := client.StorageV1beta1().CSINodes().Create(nodeInfo) return err } } - updateNode1CSINodeInfo := func(client csiclientset.Interface) func() error { + updateNode1CSINode := func(client externalclientset.Interface) func() error { return func() error { - nodeInfo, err := client.CsiV1alpha1().CSINodeInfos().Get("node1", metav1.GetOptions{}) + nodeInfo, err := client.StorageV1beta1().CSINodes().Get("node1", metav1.GetOptions{}) if err != nil { return err } - nodeInfo.Spec.Drivers = []csiv1alpha1.CSIDriverInfoSpec{ + nodeInfo.Spec.Drivers = []storagev1beta1.CSINodeDriver{ { - Name: "com.example.csi/driver1", + Name: "com.example.csi.driver2", NodeID: "com.example.csi/node1", TopologyKeys: []string{"com.example.csi/rack"}, }, } - nodeInfo.Status.Drivers = []csiv1alpha1.CSIDriverInfoStatus{ - { - Name: "com.example.csi/driver1", - Available: true, - VolumePluginMechanism: csiv1alpha1.VolumePluginMechanismInTree, - }, - } - _, err = client.CsiV1alpha1().CSINodeInfos().Update(nodeInfo) + _, err = client.StorageV1beta1().CSINodes().Update(nodeInfo) return err } } - patchNode1CSINodeInfo := func(client csiclientset.Interface) func() error { + patchNode1CSINode := func(client externalclientset.Interface) func() error { return func() error { - bs := []byte(fmt.Sprintf(`{"csiDrivers": [ { "driver": "net.example.storage/driver2", "nodeID": "net.example.storage/node1", "topologyKeys": [ "net.example.storage/region" ] } ] }`)) + bs := []byte(fmt.Sprintf(`{"csiDrivers": [ { "driver": "net.example.storage.driver2", "nodeID": "net.example.storage/node1", "topologyKeys": [ "net.example.storage/region" ] } ] }`)) // StrategicMergePatch is unsupported by CRs. Falling back to MergePatch - _, err := client.CsiV1alpha1().CSINodeInfos().Patch("node1", types.MergePatchType, bs) + _, err := client.StorageV1beta1().CSINodes().Patch("node1", types.MergePatchType, bs) return err } } - deleteNode1CSINodeInfo := func(client csiclientset.Interface) func() error { + deleteNode1CSINode := func(client externalclientset.Interface) func() error { return func() error { - return client.CsiV1alpha1().CSINodeInfos().Delete("node1", &metav1.DeleteOptions{}) + return client.StorageV1beta1().CSINodes().Delete("node1", &metav1.DeleteOptions{}) } } nodeanonClient, _ := clientsetForToken(tokenNodeUnknown, clientConfig) node1Client, node1ClientExternal := clientsetForToken(tokenNode1, clientConfig) node2Client, node2ClientExternal := clientsetForToken(tokenNode2, clientConfig) - csiNode1Client := csiClientsetForToken(tokenNode1, clientConfig) - csiNode2Client := csiClientsetForToken(tokenNode2, clientConfig) + _, csiNode1Client := clientsetForToken(tokenNode1, clientConfig) + _, csiNode2Client := clientsetForToken(tokenNode2, clientConfig) // all node requests from node1 and unknown node fail expectForbidden(t, getSecret(nodeanonClient)) @@ -645,18 +619,18 @@ func TestNodeAuthorizer(t *testing.T) { expectForbidden(t, patchNode1Lease(node2Client)) expectForbidden(t, deleteNode1Lease(node2Client)) - // node1 allowed to operate on its own CSINodeInfo - expectAllowed(t, createNode1CSINodeInfo(csiNode1Client)) - expectAllowed(t, getNode1CSINodeInfo(csiNode1Client)) - expectAllowed(t, updateNode1CSINodeInfo(csiNode1Client)) - expectAllowed(t, patchNode1CSINodeInfo(csiNode1Client)) - expectAllowed(t, deleteNode1CSINodeInfo(csiNode1Client)) - // node2 not allowed to operate on another node's CSINodeInfo - expectForbidden(t, createNode1CSINodeInfo(csiNode2Client)) - expectForbidden(t, getNode1CSINodeInfo(csiNode2Client)) - expectForbidden(t, updateNode1CSINodeInfo(csiNode2Client)) - expectForbidden(t, patchNode1CSINodeInfo(csiNode2Client)) - expectForbidden(t, deleteNode1CSINodeInfo(csiNode2Client)) + // node1 allowed to operate on its own CSINode + expectAllowed(t, createNode1CSINode(csiNode1Client)) + expectAllowed(t, getNode1CSINode(csiNode1Client)) + expectAllowed(t, updateNode1CSINode(csiNode1Client)) + expectAllowed(t, patchNode1CSINode(csiNode1Client)) + expectAllowed(t, deleteNode1CSINode(csiNode1Client)) + // node2 not allowed to operate on another node's CSINode + expectForbidden(t, createNode1CSINode(csiNode2Client)) + expectForbidden(t, getNode1CSINode(csiNode2Client)) + expectForbidden(t, updateNode1CSINode(csiNode2Client)) + expectForbidden(t, patchNode1CSINode(csiNode2Client)) + expectForbidden(t, deleteNode1CSINode(csiNode2Client)) } // expect executes a function a set number of times until it either returns the diff --git a/test/integration/auth/rbac_test.go b/test/integration/auth/rbac_test.go index e95e107f49..950b1e94e2 100644 --- a/test/integration/auth/rbac_test.go +++ b/test/integration/auth/rbac_test.go @@ -45,7 +45,6 @@ import ( restclient "k8s.io/client-go/rest" watchtools "k8s.io/client-go/tools/watch" "k8s.io/client-go/transport" - csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned" "k8s.io/klog" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/api/testapi" @@ -86,13 +85,6 @@ func crdClientsetForToken(user string, config *restclient.Config) apiextensionsc return apiextensionsclient.NewForConfigOrDie(&configCopy) } -func csiClientsetForToken(user string, config *restclient.Config) csiclientset.Interface { - configCopy := *config - configCopy.BearerToken = user - configCopy.ContentType = "application/json" // // csi client works with CRDs that support json only - return csiclientset.NewForConfigOrDie(&configCopy) -} - type testRESTOptionsGetter struct { config *master.Config } diff --git a/test/integration/etcd/data.go b/test/integration/etcd/data.go index 0bbe937cd1..f6d84a8d7c 100644 --- a/test/integration/etcd/data.go +++ b/test/integration/etcd/data.go @@ -491,7 +491,7 @@ func GetEtcdStorageData() map[schema.GroupVersionResource]StorageData { // add csinodes if CSINodeInfo feature gate is enabled if utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) { etcdStorageData[gvr("storage.k8s.io", "v1beta1", "csinodes")] = StorageData{ - Stub: `{"metadata": {"name": "csini1"}, "spec": {"drivers": [{"name": "test-driver", "nodeID": "localhost", "topologyKeys": ["company.com/zone1", "company.com/zone2"]}]}`, + Stub: `{"metadata": {"name": "csini1"}, "spec": {"drivers": [{"name": "test-driver", "nodeID": "localhost", "topologyKeys": ["company.com/zone1", "company.com/zone2"]}]}}`, ExpectedEtcdPath: "/registry/csinodes/csini1", } }