Merge pull request #68579 from verult/adc-crd-access

Updating CSI e2e test to create CSI CRDs

cmd/kube-controller-manager/app/BUILD

@@ -107,7 +107,6 @@ go_library(
         "//pkg/volume/util:go_default_library",
         "//pkg/volume/vsphere_volume:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",

cmd/kube-controller-manager/app/core.go

@@ -31,7 +31,6 @@ import (
 	"net/http"
 
 	"k8s.io/api/core/v1"
-	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	cacheddiscovery "k8s.io/client-go/discovery/cached"
@@ -199,13 +198,10 @@ func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, err
 	// csiClient works with CRDs that support json only
 	csiClientConfig.ContentType = "application/json"
 
-	crdClientConfig := ctx.ClientBuilder.ConfigOrDie("attachdetach-controller")
-
 	attachDetachController, attachDetachControllerErr :=
 		attachdetach.NewAttachDetachController(
 			ctx.ClientBuilder.ClientOrDie("attachdetach-controller"),
 			csiclientset.NewForConfigOrDie(csiClientConfig),
-			apiextensionsclient.NewForConfigOrDie(crdClientConfig),
 			ctx.InformerFactory.Core().V1().Pods(),
 			ctx.InformerFactory.Core().V1().Nodes(),
 			ctx.InformerFactory.Core().V1().PersistentVolumeClaims(),

pkg/controller/volume/attachdetach/BUILD

@@ -19,7 +19,6 @@ go_library(
         "//pkg/controller/volume/attachdetach/reconciler:go_default_library",
         "//pkg/controller/volume/attachdetach/statusupdater:go_default_library",
         "//pkg/controller/volume/attachdetach/util:go_default_library",
-        "//pkg/features:go_default_library",
         "//pkg/util/mount:go_default_library",
         "//pkg/volume:go_default_library",
         "//pkg/volume/util:go_default_library",
@@ -27,15 +26,11 @@ go_library(
         "//pkg/volume/util/volumepathhandler:go_default_library",
         "//staging/src/k8s.io/api/authentication/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
@@ -44,7 +39,6 @@ go_library(
         "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
         "//staging/src/k8s.io/client-go/tools/record:go_default_library",
         "//staging/src/k8s.io/client-go/util/workqueue:go_default_library",
-        "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library",
         "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
     ],
@@ -60,7 +54,6 @@ go_test(
         "//pkg/controller/volume/attachdetach/testing:go_default_library",
         "//pkg/volume:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",

pkg/controller/volume/attachdetach/attach_detach_controller.go

@@ -21,21 +21,16 @@ package attachdetach
 import (
 	"fmt"
 	"net"
-	"reflect"
 	"time"
 
 	"github.com/golang/glog"
 	authenticationv1 "k8s.io/api/authentication/v1"
 	"k8s.io/api/core/v1"
-	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
-	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	coreinformers "k8s.io/client-go/informers/core/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -44,7 +39,6 @@ import (
 	kcache "k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
-	csiapiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1"
 	csiclient "k8s.io/csi-api/pkg/client/clientset/versioned"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/controller"
@@ -54,7 +48,6 @@ import (
 	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/reconciler"
 	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/statusupdater"
 	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/util"
-	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/util/mount"
 	"k8s.io/kubernetes/pkg/volume"
 	volumeutil "k8s.io/kubernetes/pkg/volume/util"
@@ -105,7 +98,6 @@ type AttachDetachController interface {
 func NewAttachDetachController(
 	kubeClient clientset.Interface,
 	csiClient csiclient.Interface,
-	crdClient apiextensionsclient.Interface,
 	podInformer coreinformers.PodInformer,
 	nodeInformer coreinformers.NodeInformer,
 	pvcInformer coreinformers.PersistentVolumeClaimInformer,
@@ -133,7 +125,6 @@ func NewAttachDetachController(
 	adc := &attachDetachController{
 		kubeClient:  kubeClient,
 		csiClient:   csiClient,
-		crdClient:   crdClient,
 		pvcLister:   pvcInformer.Lister(),
 		pvcsSynced:  pvcInformer.Informer().HasSynced,
 		pvLister:    pvInformer.Lister(),
@@ -147,14 +138,6 @@ func NewAttachDetachController(
 		pvcQueue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pvcs"),
 	}
 
-	// Install required CSI CRDs on API server
-	if utilfeature.DefaultFeatureGate.Enabled(features.CSIDriverRegistry) {
-		adc.installCSIDriverCRD()
-	}
-	if utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) {
-		adc.installCSINodeInfoCRD()
-	}
-
 	if err := adc.volumePluginMgr.InitPlugins(plugins, prober, adc); err != nil {
 		return nil, fmt.Errorf("Could not initialize volume plugins for Attach/Detach Controller: %+v", err)
 	}
@@ -257,14 +240,10 @@ type attachDetachController struct {
 	// the API server.
 	kubeClient clientset.Interface
 
-	// csiClient is the client used to read/write csi.storage.k8s.io API objects
+	// csiClient is the csi.storage.k8s.io API client used by volumehost to communicate with
-	// from the API server.
+	// the API server.
 	csiClient csiclient.Interface
 
-	// crdClient is the client used to read/write apiextensions.k8s.io objects
-	// from the API server.
-	crdClient apiextensionsclient.Interface
-
 	// pvcLister is the shared PVC lister used to fetch and store PVC
 	// objects from the API server. It is shared with other controllers and
 	// therefore the PVC objects in its store should be treated as immutable.
@@ -670,71 +649,6 @@ func (adc *attachDetachController) processVolumesInUse(
 	}
 }
 
-func (adc *attachDetachController) installCSIDriverCRD() error {
-	crd := &apiextensionsv1beta1.CustomResourceDefinition{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: csiapiv1alpha1.CsiDriverResourcePlural + "." + csiapiv1alpha1.GroupName,
-		},
-		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
-			Group:   csiapiv1alpha1.GroupName,
-			Version: csiapiv1alpha1.SchemeGroupVersion.Version,
-			Scope:   apiextensionsv1beta1.ClusterScoped,
-			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
-				Plural: csiapiv1alpha1.CsiDriverResourcePlural,
-				Kind:   reflect.TypeOf(csiapiv1alpha1.CSIDriver{}).Name(),
-			},
-		},
-	}
-	res, err := adc.crdClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
-
-	if err == nil {
-		glog.Infof("CSIDrivers CRD created successfully: %#v",
-			res)
-	} else if apierrors.IsAlreadyExists(err) {
-		glog.Warningf("CSIDrivers CRD already exists: %#v, err: %#v",
-			res, err)
-	} else {
-		glog.Errorf("failed to create CSIDrivers CRD: %#v, err: %#v",
-			res, err)
-		return err
-	}
-
-	return nil
-}
-
-// installCRDs creates the specified CustomResourceDefinition for the CSIDrivers object.
-func (adc *attachDetachController) installCSINodeInfoCRD() error {
-	crd := &apiextensionsv1beta1.CustomResourceDefinition{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: csiapiv1alpha1.CsiNodeInfoResourcePlural + "." + csiapiv1alpha1.GroupName,
-		},
-		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
-			Group:   csiapiv1alpha1.GroupName,
-			Version: csiapiv1alpha1.SchemeGroupVersion.Version,
-			Scope:   apiextensionsv1beta1.ClusterScoped,
-			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
-				Plural: csiapiv1alpha1.CsiNodeInfoResourcePlural,
-				Kind:   reflect.TypeOf(csiapiv1alpha1.CSINodeInfo{}).Name(),
-			},
-		},
-	}
-	res, err := adc.crdClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
-
-	if err == nil {
-		glog.Infof("CSINodeInfo CRD created successfully: %#v",
-			res)
-	} else if apierrors.IsAlreadyExists(err) {
-		glog.Warningf("CSINodeInfo CRD already exists: %#v, err: %#v",
-			res, err)
-	} else {
-		glog.Errorf("failed to create CSINodeInfo CRD: %#v, err: %#v",
-			res, err)
-		return err
-	}
-
-	return nil
-}
-
 // VolumeHost implementation
 // This is an unfortunate requirement of the current factoring of volume plugin
 // initializing code. It requires kubelet specific methods used by the mounting

pkg/controller/volume/attachdetach/attach_detach_controller_test.go

@@ -22,7 +22,6 @@ import (
 	"time"
 
 	"k8s.io/api/core/v1"
-	fakeapiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
@@ -37,13 +36,11 @@ func Test_NewAttachDetachController_Positive(t *testing.T) {
 	// Arrange
 	fakeKubeClient := controllervolumetesting.CreateTestClient()
 	informerFactory := informers.NewSharedInformerFactory(fakeKubeClient, controller.NoResyncPeriodFunc())
-	fakeApiExtensionsClient := fakeapiextensionsclient.NewSimpleClientset()
 
 	// Act
 	_, err := NewAttachDetachController(
 		fakeKubeClient,
 		nil, /* csiClient */
-		fakeApiExtensionsClient, /* crdClient */
 		informerFactory.Core().V1().Pods(),
 		informerFactory.Core().V1().Nodes(),
 		informerFactory.Core().V1().PersistentVolumeClaims(),
@@ -149,7 +146,6 @@ func Test_AttachDetachControllerRecovery(t *testing.T) {
 
 func attachDetachRecoveryTestCase(t *testing.T, extraPods1 []*v1.Pod, extraPods2 []*v1.Pod) {
 	fakeKubeClient := controllervolumetesting.CreateTestClient()
-	fakeApiExtensionsClient := fakeapiextensionsclient.NewSimpleClientset()
 	informerFactory := informers.NewSharedInformerFactory(fakeKubeClient, time.Second*1)
 	//informerFactory := informers.NewSharedInformerFactory(fakeKubeClient, time.Second*1)
 	plugins := controllervolumetesting.CreateTestPlugin()
@@ -220,7 +216,6 @@ func attachDetachRecoveryTestCase(t *testing.T, extraPods1 []*v1.Pod, extraPods2
 	adcObj, err := NewAttachDetachController(
 		fakeKubeClient,
 		nil, /* csiClient */
-		fakeApiExtensionsClient, /* crdClient */
 		informerFactory.Core().V1().Pods(),
 		informerFactory.Core().V1().Nodes(),
 		informerFactory.Core().V1().PersistentVolumeClaims(),

pkg/volume/csi/nodeinfomanager/nodeinfomanager.go

@@ -353,7 +353,7 @@ func (nim *nodeInfoManager) createNodeInfoObject(
 		return fmt.Errorf("error getting CSI client")
 	}
 
-	var topologyKeys []string
+	topologyKeys := []string{} // must be an empty slice instead of nil to satisfy CRD OpenAPI Schema validation
 	if topology != nil {
 		for k := range topology.Segments {
 			topologyKeys = append(topologyKeys, k)

staging/src/BUILD

@@ -206,6 +206,7 @@ filegroup(
         "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:all-srcs",
         "//staging/src/k8s.io/csi-api/pkg/client/informers/externalversions:all-srcs",
         "//staging/src/k8s.io/csi-api/pkg/client/listers/csi/v1alpha1:all-srcs",
+        "//staging/src/k8s.io/csi-api/pkg/crd:all-srcs",
         "//staging/src/k8s.io/kube-aggregator:all-srcs",
         "//staging/src/k8s.io/kube-controller-manager/config/v1alpha1:all-srcs",
         "//staging/src/k8s.io/kube-proxy/config/v1alpha1:all-srcs",

staging/src/k8s.io/csi-api/Godeps/Godeps.json

@@ -290,6 +290,18 @@
 			"ImportPath": "k8s.io/api/storage/v1beta1",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
+		{
+			"ImportPath": "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions",
+			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		},
+		{
+			"ImportPath": "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1",
+			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		},
+		{
+			"ImportPath": "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme",
+			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/errors",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

staging/src/k8s.io/csi-api/pkg/crd/BUILD

@@ -0,0 +1,42 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["crd.go"],
+    importmap = "k8s.io/kubernetes/vendor/k8s.io/csi-api/pkg/crd",
+    importpath = "k8s.io/csi-api/pkg/crd",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["crd_test.go"],
+    data = glob(["testdata/**"]),
+    embed = [":go_default_library"],
+    deps = [
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
+        "//vendor/github.com/ghodss/yaml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

staging/src/k8s.io/csi-api/pkg/crd/crd.go

@@ -0,0 +1,119 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd
+
+import (
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	csiapiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1"
+	"reflect"
+)
+
+// NOTE: the CRD functions here and the associated unit tests are non-ideal temporary measures in
+// release 1.12 in order to aid manual CRD installation. This installation will be automated in
+// subsequent releases and as a result this package will be removed.
+
+// CSIDriverCRD returns the CustomResourceDefinition for CSIDriver object.
+func CSIDriverCRD() *apiextensionsv1beta1.CustomResourceDefinition {
+	return &apiextensionsv1beta1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: csiapiv1alpha1.CsiDriverResourcePlural + "." + csiapiv1alpha1.GroupName,
+		},
+		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+			Group:   csiapiv1alpha1.GroupName,
+			Version: csiapiv1alpha1.SchemeGroupVersion.Version,
+			Scope:   apiextensionsv1beta1.ClusterScoped,
+			Validation: &apiextensionsv1beta1.CustomResourceValidation{
+				OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
+					Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
+						"spec": {
+							Description: "Specification of the CSI Driver.",
+							Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
+								"attachRequired": {
+									Description: "Indicates this CSI volume driver requires an attach operation," +
+										" and that Kubernetes should call attach and wait for any attach operation to" +
+										" complete before proceeding to mount.",
+									Type: "boolean",
+								},
+								"podInfoOnMountVersion": {
+									Description: "Indicates this CSI volume driver requires additional pod" +
+										" information (like podName, podUID, etc.) during mount operations.",
+									Type: "string",
+								},
+							},
+						},
+					},
+				},
+			},
+			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+				Plural: csiapiv1alpha1.CsiDriverResourcePlural,
+				Kind:   reflect.TypeOf(csiapiv1alpha1.CSIDriver{}).Name(),
+			},
+		},
+	}
+}
+
+// CSINodeInfoCRD returns the CustomResourceDefinition for CSINodeInfo object.
+func CSINodeInfoCRD() *apiextensionsv1beta1.CustomResourceDefinition {
+	return &apiextensionsv1beta1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: csiapiv1alpha1.CsiNodeInfoResourcePlural + "." + csiapiv1alpha1.GroupName,
+		},
+		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+			Group:   csiapiv1alpha1.GroupName,
+			Version: csiapiv1alpha1.SchemeGroupVersion.Version,
+			Scope:   apiextensionsv1beta1.ClusterScoped,
+			Validation: &apiextensionsv1beta1.CustomResourceValidation{
+				OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
+					Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
+						"csiDrivers": {
+							Description: "List of CSI drivers running on the node and their properties.",
+							Type:        "array",
+							Items: &apiextensionsv1beta1.JSONSchemaPropsOrArray{
+								Schema: &apiextensionsv1beta1.JSONSchemaProps{
+									Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
+										"driver": {
+											Description: "The CSI driver that this object refers to.",
+											Type:        "string",
+										},
+										"nodeID": {
+											Description: "The node from the driver point of view.",
+											Type:        "string",
+										},
+										"topologyKeys": {
+											Description: "List of keys supported by the driver.",
+											Type:        "array",
+											Items: &apiextensionsv1beta1.JSONSchemaPropsOrArray{
+												Schema: &apiextensionsv1beta1.JSONSchemaProps{
+													Type: "string",
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+				Plural: csiapiv1alpha1.CsiNodeInfoResourcePlural,
+				Kind:   reflect.TypeOf(csiapiv1alpha1.CSINodeInfo{}).Name(),
+			},
+		},
+	}
+}

staging/src/k8s.io/csi-api/pkg/crd/crd_test.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/ghodss/yaml"
+	"io/ioutil"
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsscheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/diff"
+	"k8s.io/csi-api/pkg/crd"
+	"os"
+)
+
+func TestBootstrapCRDs(t *testing.T) {
+	testObjects(t, crd.CSIDriverCRD(), "csidriver.yaml")
+	testObjects(t, crd.CSINodeInfoCRD(), "csinodeinfo.yaml")
+}
+
+func testObjects(t *testing.T, crd *apiextensionsv1beta1.CustomResourceDefinition, fixtureFilename string) {
+	filename := filepath.Join("testdata", fixtureFilename)
+	expectedYAML, err := ioutil.ReadFile(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	jsonData, err := runtime.Encode(apiextensionsscheme.Codecs.LegacyCodec(apiextensionsv1beta1.SchemeGroupVersion), crd)
+	if err != nil {
+		t.Fatal(err)
+	}
+	yamlData, err := yaml.JSONToYAML(jsonData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(yamlData) != string(expectedYAML) {
+		t.Errorf("Bootstrap CRD data does not match the test fixture in %s", filename)
+
+		const updateEnvVar = "UPDATE_CSI_CRD_FIXTURE_DATA"
+		if os.Getenv(updateEnvVar) == "true" {
+			if err := ioutil.WriteFile(filename, []byte(yamlData), os.FileMode(0755)); err == nil {
+				t.Logf("Updated data in %s", filename)
+				t.Logf("Verify the diff, commit changes, and rerun the tests")
+			} else {
+				t.Logf("Could not update data in %s: %v", filename, err)
+			}
+		} else {
+			t.Logf("Diff between data in code and fixture data in %s:\n-------------\n%s", filename, diff.StringDiff(string(yamlData), string(expectedYAML)))
+			t.Logf("If the change is expected, re-run with %s=true to update the fixtures", updateEnvVar)
+		}
+	}
+}

staging/src/k8s.io/csi-api/pkg/crd/testdata/csidriver.yaml

@@ -0,0 +1,33 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  creationTimestamp: null
+  name: csidrivers.csi.storage.k8s.io
+spec:
+  group: csi.storage.k8s.io
+  names:
+    kind: CSIDriver
+    plural: csidrivers
+  scope: Cluster
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          description: Specification of the CSI Driver.
+          properties:
+            attachRequired:
+              description: Indicates this CSI volume driver requires an attach operation,
+                and that Kubernetes should call attach and wait for any attach operation
+                to complete before proceeding to mount.
+              type: boolean
+            podInfoOnMountVersion:
+              description: Indicates this CSI volume driver requires additional pod
+                information (like podName, podUID, etc.) during mount operations.
+              type: string
+  version: v1alpha1
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: null
+  storedVersions: null

staging/src/k8s.io/csi-api/pkg/crd/testdata/csinodeinfo.yaml

@@ -0,0 +1,37 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  creationTimestamp: null
+  name: csinodeinfos.csi.storage.k8s.io
+spec:
+  group: csi.storage.k8s.io
+  names:
+    kind: CSINodeInfo
+    plural: csinodeinfos
+  scope: Cluster
+  validation:
+    openAPIV3Schema:
+      properties:
+        csiDrivers:
+          description: List of CSI drivers running on the node and their properties.
+          items:
+            properties:
+              driver:
+                description: The CSI driver that this object refers to.
+                type: string
+              nodeID:
+                description: The node from the driver point of view.
+                type: string
+              topologyKeys:
+                description: List of keys supported by the driver.
+                items:
+                  type: string
+                type: array
+          type: array
+  version: v1alpha1
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: null
+  storedVersions: null

test/e2e/framework/framework.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"k8s.io/api/core/v1"
+	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -67,6 +68,7 @@ type Framework struct {
 
 	ClientSet                        clientset.Interface
 	KubemarkExternalClusterClientSet clientset.Interface
+	APIExtensionsClientSet           apiextensionsclient.Interface
 	CSIClientSet                     csi.Interface
 
 	InternalClientset *internalclientset.Clientset
@@ -176,6 +178,8 @@ func (f *Framework) BeforeEach() {
 		}
 		f.ClientSet, err = clientset.NewForConfig(config)
 		Expect(err).NotTo(HaveOccurred())
+		f.APIExtensionsClientSet, err = apiextensionsclient.NewForConfig(config)
+		Expect(err).NotTo(HaveOccurred())
 		f.InternalClientset, err = internalclientset.NewForConfig(config)
 		Expect(err).NotTo(HaveOccurred())
 		f.AggregatorClient, err = aggregatorclient.NewForConfig(config)

test/e2e/storage/BUILD

@@ -44,6 +44,8 @@ go_library(
         "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library",
         "//staging/src/k8s.io/api/storage/v1:go_default_library",
         "//staging/src/k8s.io/api/storage/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
@@ -64,6 +66,7 @@ go_library(
         "//staging/src/k8s.io/client-go/rest:go_default_library",
         "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library",
         "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library",
+        "//staging/src/k8s.io/csi-api/pkg/crd:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/e2e/framework/metrics:go_default_library",
         "//test/e2e/generated:go_default_library",

test/e2e/storage/csi_objects.go

@@ -35,6 +35,10 @@ import (
 	"k8s.io/kubernetes/test/e2e/manifest"
 
 	. "github.com/onsi/ginkgo"
+
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	csicrd "k8s.io/csi-api/pkg/crd"
 )
 
 var csiImageVersions = map[string]string{
@@ -427,3 +431,26 @@ func deployGCEPDCSIDriver(
 	framework.ExpectNoError(err, "Failed to create DaemonSet: %v", nodeds.Name)
 
 }
+
+func createCSICRDs(c apiextensionsclient.Interface) {
+	By("Creating CSI CRDs")
+	crds := []*apiextensionsv1beta1.CustomResourceDefinition{
+		csicrd.CSIDriverCRD(),
+		csicrd.CSINodeInfoCRD(),
+	}
+
+	for _, crd := range crds {
+		_, err := c.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
+		framework.ExpectNoError(err, "Failed to create CSI CRD %q: %v", crd.Name, err)
+	}
+}
+
+func deleteCSICRDs(c apiextensionsclient.Interface) {
+	By("Deleting CSI CRDs")
+	csiDriverCRDName := csicrd.CSIDriverCRD().Name
+	csiNodeInfoCRDName := csicrd.CSINodeInfoCRD().Name
+	err := c.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(csiDriverCRDName, &metav1.DeleteOptions{})
+	framework.ExpectNoError(err, "Failed to delete CSI CRD %q: %v", csiDriverCRDName, err)
+	err = c.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(csiNodeInfoCRDName, &metav1.DeleteOptions{})
+	framework.ExpectNoError(err, "Failed to delete CSI CRD %q: %v", csiNodeInfoCRDName, err)
+}

test/e2e/storage/csi_volumes.go

@@ -24,10 +24,11 @@ import (
 	"k8s.io/api/core/v1"
 
 	storagev1 "k8s.io/api/storage/v1"
+	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
-	csi "k8s.io/csi-api/pkg/apis/csi/v1alpha1"
+	csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1"
 	csiclient "k8s.io/csi-api/pkg/client/clientset/versioned"
 	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -58,19 +59,21 @@ var csiTestDrivers = map[string]func(f *framework.Framework, config framework.Vo
 	"[Feature: GCE PD CSI Plugin] gcePD": initCSIgcePD,
 }
 
-var _ = utils.SIGDescribe("CSI Volumes", func() {
+var _ = utils.SIGDescribe("[Serial] CSI Volumes", func() {
 	f := framework.NewDefaultFramework("csi-mock-plugin")
 
 	var (
-		cs     clientset.Interface
+		cs        clientset.Interface
-		csics  csiclient.Interface
+		crdclient apiextensionsclient.Interface
-		ns     *v1.Namespace
+		csics     csiclient.Interface
-		node   v1.Node
+		ns        *v1.Namespace
-		config framework.VolumeTestConfig
+		node      v1.Node
+		config    framework.VolumeTestConfig
 	)
 
 	BeforeEach(func() {
 		cs = f.ClientSet
+		crdclient = f.APIExtensionsClientSet
 		csics = f.CSIClientSet
 		ns = f.Namespace
 		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
@@ -83,13 +86,18 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
 			WaitForCompletion: true,
 		}
 		csiDriverRegistrarClusterRole(config)
+		createCSICRDs(crdclient)
+	})
+
+	AfterEach(func() {
+		deleteCSICRDs(crdclient)
 	})
 
 	for driverName, initCSIDriver := range csiTestDrivers {
 		curDriverName := driverName
 		curInitCSIDriver := initCSIDriver
 
-		Context(fmt.Sprintf("CSI plugin test using CSI driver: %s", curDriverName), func() {
+		Context(fmt.Sprintf("CSI plugin test using CSI driver: %s [Serial]", curDriverName), func() {
 			var (
 				driver csiTestDriver
 			)
@@ -205,13 +213,13 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
 	})
 })
 
-func createCSIDriver(csics csiclient.Interface, attachable bool) *csi.CSIDriver {
+func createCSIDriver(csics csiclient.Interface, attachable bool) *csiv1alpha1.CSIDriver {
 	By("Creating CSIDriver instance")
-	driver := &csi.CSIDriver{
+	driver := &csiv1alpha1.CSIDriver{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "csi-hostpath",
 		},
-		Spec: csi.CSIDriverSpec{
+		Spec: csiv1alpha1.CSIDriverSpec{
 			AttachRequired: &attachable,
 		},
 	}

test/integration/volume/BUILD

@@ -26,7 +26,6 @@ go_test(
         "//pkg/volume/util:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/storage/v1:go_default_library",
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",

test/integration/volume/attach_detach_test.go

@@ -23,7 +23,6 @@ import (
 	"time"
 
 	"k8s.io/api/core/v1"
-	fakeapiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -394,7 +393,6 @@ func createAdClients(ns *v1.Namespace, t *testing.T, server *httptest.Server, sy
 	}
 	resyncPeriod := 12 * time.Hour
 	testClient := clientset.NewForConfigOrDie(&config)
-	fakeApiExtensionsClient := fakeapiextensionsclient.NewSimpleClientset()
 
 	host := volumetest.NewFakeVolumeHost("/tmp/fake", nil, nil)
 	plugin := &volumetest.FakeVolumePlugin{
@@ -415,7 +413,6 @@ func createAdClients(ns *v1.Namespace, t *testing.T, server *httptest.Server, sy
 	ctrl, err := attachdetach.NewAttachDetachController(
 		testClient,
 		nil, /* csiClient */
-		fakeApiExtensionsClient, /* crdClient */
 		informers.Core().V1().Pods(),
 		informers.Core().V1().Nodes(),
 		informers.Core().V1().PersistentVolumeClaims(),