diff --git a/cmd/kube-controller-manager/app/BUILD b/cmd/kube-controller-manager/app/BUILD index f183dda468..cd36af5aff 100644 --- a/cmd/kube-controller-manager/app/BUILD +++ b/cmd/kube-controller-manager/app/BUILD @@ -107,7 +107,6 @@ go_library( "//pkg/volume/util:go_default_library", "//pkg/volume/vsphere_volume:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", diff --git a/cmd/kube-controller-manager/app/core.go b/cmd/kube-controller-manager/app/core.go index f6a273385a..eca5648cd0 100644 --- a/cmd/kube-controller-manager/app/core.go +++ b/cmd/kube-controller-manager/app/core.go @@ -31,7 +31,6 @@ import ( "net/http" "k8s.io/api/core/v1" - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/runtime/schema" utilfeature "k8s.io/apiserver/pkg/util/feature" cacheddiscovery "k8s.io/client-go/discovery/cached" @@ -199,13 +198,10 @@ func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, err // csiClient works with CRDs that support json only csiClientConfig.ContentType = "application/json" - crdClientConfig := ctx.ClientBuilder.ConfigOrDie("attachdetach-controller") - attachDetachController, attachDetachControllerErr := attachdetach.NewAttachDetachController( ctx.ClientBuilder.ClientOrDie("attachdetach-controller"), csiclientset.NewForConfigOrDie(csiClientConfig), - apiextensionsclient.NewForConfigOrDie(crdClientConfig), ctx.InformerFactory.Core().V1().Pods(), ctx.InformerFactory.Core().V1().Nodes(), ctx.InformerFactory.Core().V1().PersistentVolumeClaims(), diff --git a/pkg/controller/volume/attachdetach/BUILD b/pkg/controller/volume/attachdetach/BUILD index 13da43f203..b3e919746b 100644 --- a/pkg/controller/volume/attachdetach/BUILD +++ b/pkg/controller/volume/attachdetach/BUILD @@ -19,7 +19,6 @@ go_library( "//pkg/controller/volume/attachdetach/reconciler:go_default_library", "//pkg/controller/volume/attachdetach/statusupdater:go_default_library", "//pkg/controller/volume/attachdetach/util:go_default_library", - "//pkg/features:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", @@ -27,15 +26,11 @@ go_library( "//pkg/volume/util/volumepathhandler:go_default_library", "//staging/src/k8s.io/api/authentication/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", @@ -44,7 +39,6 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", - "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", "//vendor/github.com/golang/glog:go_default_library", ], @@ -60,7 +54,6 @@ go_test( "//pkg/controller/volume/attachdetach/testing:go_default_library", "//pkg/volume:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller.go b/pkg/controller/volume/attachdetach/attach_detach_controller.go index 8895a211c6..b7f07b16c3 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller.go @@ -21,21 +21,16 @@ package attachdetach import ( "fmt" "net" - "reflect" "time" "github.com/golang/glog" authenticationv1 "k8s.io/api/authentication/v1" "k8s.io/api/core/v1" - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" - utilfeature "k8s.io/apiserver/pkg/util/feature" coreinformers "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" @@ -44,7 +39,6 @@ import ( kcache "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" - csiapiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1" csiclient "k8s.io/csi-api/pkg/client/clientset/versioned" "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/controller" @@ -54,7 +48,6 @@ import ( "k8s.io/kubernetes/pkg/controller/volume/attachdetach/reconciler" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/statusupdater" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/util" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" @@ -105,7 +98,6 @@ type AttachDetachController interface { func NewAttachDetachController( kubeClient clientset.Interface, csiClient csiclient.Interface, - crdClient apiextensionsclient.Interface, podInformer coreinformers.PodInformer, nodeInformer coreinformers.NodeInformer, pvcInformer coreinformers.PersistentVolumeClaimInformer, @@ -133,7 +125,6 @@ func NewAttachDetachController( adc := &attachDetachController{ kubeClient: kubeClient, csiClient: csiClient, - crdClient: crdClient, pvcLister: pvcInformer.Lister(), pvcsSynced: pvcInformer.Informer().HasSynced, pvLister: pvInformer.Lister(), @@ -147,14 +138,6 @@ func NewAttachDetachController( pvcQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pvcs"), } - // Install required CSI CRDs on API server - if utilfeature.DefaultFeatureGate.Enabled(features.CSIDriverRegistry) { - adc.installCSIDriverCRD() - } - if utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) { - adc.installCSINodeInfoCRD() - } - if err := adc.volumePluginMgr.InitPlugins(plugins, prober, adc); err != nil { return nil, fmt.Errorf("Could not initialize volume plugins for Attach/Detach Controller: %+v", err) } @@ -257,14 +240,10 @@ type attachDetachController struct { // the API server. kubeClient clientset.Interface - // csiClient is the client used to read/write csi.storage.k8s.io API objects - // from the API server. + // csiClient is the csi.storage.k8s.io API client used by volumehost to communicate with + // the API server. csiClient csiclient.Interface - // crdClient is the client used to read/write apiextensions.k8s.io objects - // from the API server. - crdClient apiextensionsclient.Interface - // pvcLister is the shared PVC lister used to fetch and store PVC // objects from the API server. It is shared with other controllers and // therefore the PVC objects in its store should be treated as immutable. @@ -670,71 +649,6 @@ func (adc *attachDetachController) processVolumesInUse( } } -func (adc *attachDetachController) installCSIDriverCRD() error { - crd := &apiextensionsv1beta1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: csiapiv1alpha1.CsiDriverResourcePlural + "." + csiapiv1alpha1.GroupName, - }, - Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ - Group: csiapiv1alpha1.GroupName, - Version: csiapiv1alpha1.SchemeGroupVersion.Version, - Scope: apiextensionsv1beta1.ClusterScoped, - Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ - Plural: csiapiv1alpha1.CsiDriverResourcePlural, - Kind: reflect.TypeOf(csiapiv1alpha1.CSIDriver{}).Name(), - }, - }, - } - res, err := adc.crdClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd) - - if err == nil { - glog.Infof("CSIDrivers CRD created successfully: %#v", - res) - } else if apierrors.IsAlreadyExists(err) { - glog.Warningf("CSIDrivers CRD already exists: %#v, err: %#v", - res, err) - } else { - glog.Errorf("failed to create CSIDrivers CRD: %#v, err: %#v", - res, err) - return err - } - - return nil -} - -// installCRDs creates the specified CustomResourceDefinition for the CSIDrivers object. -func (adc *attachDetachController) installCSINodeInfoCRD() error { - crd := &apiextensionsv1beta1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: csiapiv1alpha1.CsiNodeInfoResourcePlural + "." + csiapiv1alpha1.GroupName, - }, - Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ - Group: csiapiv1alpha1.GroupName, - Version: csiapiv1alpha1.SchemeGroupVersion.Version, - Scope: apiextensionsv1beta1.ClusterScoped, - Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ - Plural: csiapiv1alpha1.CsiNodeInfoResourcePlural, - Kind: reflect.TypeOf(csiapiv1alpha1.CSINodeInfo{}).Name(), - }, - }, - } - res, err := adc.crdClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd) - - if err == nil { - glog.Infof("CSINodeInfo CRD created successfully: %#v", - res) - } else if apierrors.IsAlreadyExists(err) { - glog.Warningf("CSINodeInfo CRD already exists: %#v, err: %#v", - res, err) - } else { - glog.Errorf("failed to create CSINodeInfo CRD: %#v, err: %#v", - res, err) - return err - } - - return nil -} - // VolumeHost implementation // This is an unfortunate requirement of the current factoring of volume plugin // initializing code. It requires kubelet specific methods used by the mounting diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller_test.go b/pkg/controller/volume/attachdetach/attach_detach_controller_test.go index 1ded525fd8..f850302521 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller_test.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller_test.go @@ -22,7 +22,6 @@ import ( "time" "k8s.io/api/core/v1" - fakeapiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" @@ -37,13 +36,11 @@ func Test_NewAttachDetachController_Positive(t *testing.T) { // Arrange fakeKubeClient := controllervolumetesting.CreateTestClient() informerFactory := informers.NewSharedInformerFactory(fakeKubeClient, controller.NoResyncPeriodFunc()) - fakeApiExtensionsClient := fakeapiextensionsclient.NewSimpleClientset() // Act _, err := NewAttachDetachController( fakeKubeClient, nil, /* csiClient */ - fakeApiExtensionsClient, /* crdClient */ informerFactory.Core().V1().Pods(), informerFactory.Core().V1().Nodes(), informerFactory.Core().V1().PersistentVolumeClaims(), @@ -149,7 +146,6 @@ func Test_AttachDetachControllerRecovery(t *testing.T) { func attachDetachRecoveryTestCase(t *testing.T, extraPods1 []*v1.Pod, extraPods2 []*v1.Pod) { fakeKubeClient := controllervolumetesting.CreateTestClient() - fakeApiExtensionsClient := fakeapiextensionsclient.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(fakeKubeClient, time.Second*1) //informerFactory := informers.NewSharedInformerFactory(fakeKubeClient, time.Second*1) plugins := controllervolumetesting.CreateTestPlugin() @@ -220,7 +216,6 @@ func attachDetachRecoveryTestCase(t *testing.T, extraPods1 []*v1.Pod, extraPods2 adcObj, err := NewAttachDetachController( fakeKubeClient, nil, /* csiClient */ - fakeApiExtensionsClient, /* crdClient */ informerFactory.Core().V1().Pods(), informerFactory.Core().V1().Nodes(), informerFactory.Core().V1().PersistentVolumeClaims(), diff --git a/pkg/volume/csi/nodeinfomanager/nodeinfomanager.go b/pkg/volume/csi/nodeinfomanager/nodeinfomanager.go index f13a4419be..462914f968 100644 --- a/pkg/volume/csi/nodeinfomanager/nodeinfomanager.go +++ b/pkg/volume/csi/nodeinfomanager/nodeinfomanager.go @@ -353,7 +353,7 @@ func (nim *nodeInfoManager) createNodeInfoObject( return fmt.Errorf("error getting CSI client") } - var topologyKeys []string + topologyKeys := []string{} // must be an empty slice instead of nil to satisfy CRD OpenAPI Schema validation if topology != nil { for k := range topology.Segments { topologyKeys = append(topologyKeys, k) diff --git a/staging/src/BUILD b/staging/src/BUILD index 31e4f9ca52..6045d02b21 100644 --- a/staging/src/BUILD +++ b/staging/src/BUILD @@ -206,6 +206,7 @@ filegroup( "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:all-srcs", "//staging/src/k8s.io/csi-api/pkg/client/informers/externalversions:all-srcs", "//staging/src/k8s.io/csi-api/pkg/client/listers/csi/v1alpha1:all-srcs", + "//staging/src/k8s.io/csi-api/pkg/crd:all-srcs", "//staging/src/k8s.io/kube-aggregator:all-srcs", "//staging/src/k8s.io/kube-controller-manager/config/v1alpha1:all-srcs", "//staging/src/k8s.io/kube-proxy/config/v1alpha1:all-srcs", diff --git a/staging/src/k8s.io/csi-api/Godeps/Godeps.json b/staging/src/k8s.io/csi-api/Godeps/Godeps.json index f9cc4e25f1..d326856ae1 100644 --- a/staging/src/k8s.io/csi-api/Godeps/Godeps.json +++ b/staging/src/k8s.io/csi-api/Godeps/Godeps.json @@ -290,6 +290,18 @@ "ImportPath": "k8s.io/api/storage/v1beta1", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "ImportPath": "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + { + "ImportPath": "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apimachinery/pkg/api/errors", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" diff --git a/staging/src/k8s.io/csi-api/pkg/crd/BUILD b/staging/src/k8s.io/csi-api/pkg/crd/BUILD new file mode 100644 index 0000000000..3a9a64a5c8 --- /dev/null +++ b/staging/src/k8s.io/csi-api/pkg/crd/BUILD @@ -0,0 +1,42 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["crd.go"], + importmap = "k8s.io/kubernetes/vendor/k8s.io/csi-api/pkg/crd", + importpath = "k8s.io/csi-api/pkg/crd", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["crd_test.go"], + data = glob(["testdata/**"]), + embed = [":go_default_library"], + deps = [ + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//vendor/github.com/ghodss/yaml:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/csi-api/pkg/crd/crd.go b/staging/src/k8s.io/csi-api/pkg/crd/crd.go new file mode 100644 index 0000000000..4a7936e414 --- /dev/null +++ b/staging/src/k8s.io/csi-api/pkg/crd/crd.go @@ -0,0 +1,119 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crd + +import ( + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + csiapiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1" + "reflect" +) + +// NOTE: the CRD functions here and the associated unit tests are non-ideal temporary measures in +// release 1.12 in order to aid manual CRD installation. This installation will be automated in +// subsequent releases and as a result this package will be removed. + +// CSIDriverCRD returns the CustomResourceDefinition for CSIDriver object. +func CSIDriverCRD() *apiextensionsv1beta1.CustomResourceDefinition { + return &apiextensionsv1beta1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: csiapiv1alpha1.CsiDriverResourcePlural + "." + csiapiv1alpha1.GroupName, + }, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: csiapiv1alpha1.GroupName, + Version: csiapiv1alpha1.SchemeGroupVersion.Version, + Scope: apiextensionsv1beta1.ClusterScoped, + Validation: &apiextensionsv1beta1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "spec": { + Description: "Specification of the CSI Driver.", + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "attachRequired": { + Description: "Indicates this CSI volume driver requires an attach operation," + + " and that Kubernetes should call attach and wait for any attach operation to" + + " complete before proceeding to mount.", + Type: "boolean", + }, + "podInfoOnMountVersion": { + Description: "Indicates this CSI volume driver requires additional pod" + + " information (like podName, podUID, etc.) during mount operations.", + Type: "string", + }, + }, + }, + }, + }, + }, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: csiapiv1alpha1.CsiDriverResourcePlural, + Kind: reflect.TypeOf(csiapiv1alpha1.CSIDriver{}).Name(), + }, + }, + } +} + +// CSINodeInfoCRD returns the CustomResourceDefinition for CSINodeInfo object. +func CSINodeInfoCRD() *apiextensionsv1beta1.CustomResourceDefinition { + return &apiextensionsv1beta1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: csiapiv1alpha1.CsiNodeInfoResourcePlural + "." + csiapiv1alpha1.GroupName, + }, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: csiapiv1alpha1.GroupName, + Version: csiapiv1alpha1.SchemeGroupVersion.Version, + Scope: apiextensionsv1beta1.ClusterScoped, + Validation: &apiextensionsv1beta1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "csiDrivers": { + Description: "List of CSI drivers running on the node and their properties.", + Type: "array", + Items: &apiextensionsv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextensionsv1beta1.JSONSchemaProps{ + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "driver": { + Description: "The CSI driver that this object refers to.", + Type: "string", + }, + "nodeID": { + Description: "The node from the driver point of view.", + Type: "string", + }, + "topologyKeys": { + Description: "List of keys supported by the driver.", + Type: "array", + Items: &apiextensionsv1beta1.JSONSchemaPropsOrArray{ + Schema: &apiextensionsv1beta1.JSONSchemaProps{ + Type: "string", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: csiapiv1alpha1.CsiNodeInfoResourcePlural, + Kind: reflect.TypeOf(csiapiv1alpha1.CSINodeInfo{}).Name(), + }, + }, + } +} diff --git a/staging/src/k8s.io/csi-api/pkg/crd/crd_test.go b/staging/src/k8s.io/csi-api/pkg/crd/crd_test.go new file mode 100644 index 0000000000..369a3b458b --- /dev/null +++ b/staging/src/k8s.io/csi-api/pkg/crd/crd_test.go @@ -0,0 +1,69 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crd_test + +import ( + "path/filepath" + "testing" + + "github.com/ghodss/yaml" + "io/ioutil" + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsscheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/csi-api/pkg/crd" + "os" +) + +func TestBootstrapCRDs(t *testing.T) { + testObjects(t, crd.CSIDriverCRD(), "csidriver.yaml") + testObjects(t, crd.CSINodeInfoCRD(), "csinodeinfo.yaml") +} + +func testObjects(t *testing.T, crd *apiextensionsv1beta1.CustomResourceDefinition, fixtureFilename string) { + filename := filepath.Join("testdata", fixtureFilename) + expectedYAML, err := ioutil.ReadFile(filename) + if err != nil { + t.Fatal(err) + } + + jsonData, err := runtime.Encode(apiextensionsscheme.Codecs.LegacyCodec(apiextensionsv1beta1.SchemeGroupVersion), crd) + if err != nil { + t.Fatal(err) + } + yamlData, err := yaml.JSONToYAML(jsonData) + if err != nil { + t.Fatal(err) + } + if string(yamlData) != string(expectedYAML) { + t.Errorf("Bootstrap CRD data does not match the test fixture in %s", filename) + + const updateEnvVar = "UPDATE_CSI_CRD_FIXTURE_DATA" + if os.Getenv(updateEnvVar) == "true" { + if err := ioutil.WriteFile(filename, []byte(yamlData), os.FileMode(0755)); err == nil { + t.Logf("Updated data in %s", filename) + t.Logf("Verify the diff, commit changes, and rerun the tests") + } else { + t.Logf("Could not update data in %s: %v", filename, err) + } + } else { + t.Logf("Diff between data in code and fixture data in %s:\n-------------\n%s", filename, diff.StringDiff(string(yamlData), string(expectedYAML))) + t.Logf("If the change is expected, re-run with %s=true to update the fixtures", updateEnvVar) + } + } +} diff --git a/staging/src/k8s.io/csi-api/pkg/crd/testdata/csidriver.yaml b/staging/src/k8s.io/csi-api/pkg/crd/testdata/csidriver.yaml new file mode 100644 index 0000000000..d950cbf494 --- /dev/null +++ b/staging/src/k8s.io/csi-api/pkg/crd/testdata/csidriver.yaml @@ -0,0 +1,33 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: csidrivers.csi.storage.k8s.io +spec: + group: csi.storage.k8s.io + names: + kind: CSIDriver + plural: csidrivers + scope: Cluster + validation: + openAPIV3Schema: + properties: + spec: + description: Specification of the CSI Driver. + properties: + attachRequired: + description: Indicates this CSI volume driver requires an attach operation, + and that Kubernetes should call attach and wait for any attach operation + to complete before proceeding to mount. + type: boolean + podInfoOnMountVersion: + description: Indicates this CSI volume driver requires additional pod + information (like podName, podUID, etc.) during mount operations. + type: string + version: v1alpha1 +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null diff --git a/staging/src/k8s.io/csi-api/pkg/crd/testdata/csinodeinfo.yaml b/staging/src/k8s.io/csi-api/pkg/crd/testdata/csinodeinfo.yaml new file mode 100644 index 0000000000..6ea408d5e7 --- /dev/null +++ b/staging/src/k8s.io/csi-api/pkg/crd/testdata/csinodeinfo.yaml @@ -0,0 +1,37 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: csinodeinfos.csi.storage.k8s.io +spec: + group: csi.storage.k8s.io + names: + kind: CSINodeInfo + plural: csinodeinfos + scope: Cluster + validation: + openAPIV3Schema: + properties: + csiDrivers: + description: List of CSI drivers running on the node and their properties. + items: + properties: + driver: + description: The CSI driver that this object refers to. + type: string + nodeID: + description: The node from the driver point of view. + type: string + topologyKeys: + description: List of keys supported by the driver. + items: + type: string + type: array + type: array + version: v1alpha1 +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index c51efa17b2..0b8f64cf9d 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -26,6 +26,7 @@ import ( "time" "k8s.io/api/core/v1" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -67,6 +68,7 @@ type Framework struct { ClientSet clientset.Interface KubemarkExternalClusterClientSet clientset.Interface + APIExtensionsClientSet apiextensionsclient.Interface CSIClientSet csi.Interface InternalClientset *internalclientset.Clientset @@ -176,6 +178,8 @@ func (f *Framework) BeforeEach() { } f.ClientSet, err = clientset.NewForConfig(config) Expect(err).NotTo(HaveOccurred()) + f.APIExtensionsClientSet, err = apiextensionsclient.NewForConfig(config) + Expect(err).NotTo(HaveOccurred()) f.InternalClientset, err = internalclientset.NewForConfig(config) Expect(err).NotTo(HaveOccurred()) f.AggregatorClient, err = aggregatorclient.NewForConfig(config) diff --git a/test/e2e/storage/BUILD b/test/e2e/storage/BUILD index 53c30eaf35..a5ed5fc648 100644 --- a/test/e2e/storage/BUILD +++ b/test/e2e/storage/BUILD @@ -44,6 +44,8 @@ go_library( "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -64,6 +66,7 @@ go_library( "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", + "//staging/src/k8s.io/csi-api/pkg/crd:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/framework/metrics:go_default_library", "//test/e2e/generated:go_default_library", diff --git a/test/e2e/storage/csi_objects.go b/test/e2e/storage/csi_objects.go index aadf4dfe60..4239d7cdce 100644 --- a/test/e2e/storage/csi_objects.go +++ b/test/e2e/storage/csi_objects.go @@ -35,6 +35,10 @@ import ( "k8s.io/kubernetes/test/e2e/manifest" . "github.com/onsi/ginkgo" + + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + csicrd "k8s.io/csi-api/pkg/crd" ) var csiImageVersions = map[string]string{ @@ -427,3 +431,26 @@ func deployGCEPDCSIDriver( framework.ExpectNoError(err, "Failed to create DaemonSet: %v", nodeds.Name) } + +func createCSICRDs(c apiextensionsclient.Interface) { + By("Creating CSI CRDs") + crds := []*apiextensionsv1beta1.CustomResourceDefinition{ + csicrd.CSIDriverCRD(), + csicrd.CSINodeInfoCRD(), + } + + for _, crd := range crds { + _, err := c.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd) + framework.ExpectNoError(err, "Failed to create CSI CRD %q: %v", crd.Name, err) + } +} + +func deleteCSICRDs(c apiextensionsclient.Interface) { + By("Deleting CSI CRDs") + csiDriverCRDName := csicrd.CSIDriverCRD().Name + csiNodeInfoCRDName := csicrd.CSINodeInfoCRD().Name + err := c.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(csiDriverCRDName, &metav1.DeleteOptions{}) + framework.ExpectNoError(err, "Failed to delete CSI CRD %q: %v", csiDriverCRDName, err) + err = c.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(csiNodeInfoCRDName, &metav1.DeleteOptions{}) + framework.ExpectNoError(err, "Failed to delete CSI CRD %q: %v", csiNodeInfoCRDName, err) +} diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go index acb989d6ef..20ac056e93 100644 --- a/test/e2e/storage/csi_volumes.go +++ b/test/e2e/storage/csi_volumes.go @@ -24,10 +24,11 @@ import ( "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" - csi "k8s.io/csi-api/pkg/apis/csi/v1alpha1" + csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1" csiclient "k8s.io/csi-api/pkg/client/clientset/versioned" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "k8s.io/kubernetes/test/e2e/framework" @@ -58,19 +59,21 @@ var csiTestDrivers = map[string]func(f *framework.Framework, config framework.Vo "[Feature: GCE PD CSI Plugin] gcePD": initCSIgcePD, } -var _ = utils.SIGDescribe("CSI Volumes", func() { +var _ = utils.SIGDescribe("[Serial] CSI Volumes", func() { f := framework.NewDefaultFramework("csi-mock-plugin") var ( - cs clientset.Interface - csics csiclient.Interface - ns *v1.Namespace - node v1.Node - config framework.VolumeTestConfig + cs clientset.Interface + crdclient apiextensionsclient.Interface + csics csiclient.Interface + ns *v1.Namespace + node v1.Node + config framework.VolumeTestConfig ) BeforeEach(func() { cs = f.ClientSet + crdclient = f.APIExtensionsClientSet csics = f.CSIClientSet ns = f.Namespace nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet) @@ -83,13 +86,18 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { WaitForCompletion: true, } csiDriverRegistrarClusterRole(config) + createCSICRDs(crdclient) + }) + + AfterEach(func() { + deleteCSICRDs(crdclient) }) for driverName, initCSIDriver := range csiTestDrivers { curDriverName := driverName curInitCSIDriver := initCSIDriver - Context(fmt.Sprintf("CSI plugin test using CSI driver: %s", curDriverName), func() { + Context(fmt.Sprintf("CSI plugin test using CSI driver: %s [Serial]", curDriverName), func() { var ( driver csiTestDriver ) @@ -205,13 +213,13 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { }) }) -func createCSIDriver(csics csiclient.Interface, attachable bool) *csi.CSIDriver { +func createCSIDriver(csics csiclient.Interface, attachable bool) *csiv1alpha1.CSIDriver { By("Creating CSIDriver instance") - driver := &csi.CSIDriver{ + driver := &csiv1alpha1.CSIDriver{ ObjectMeta: metav1.ObjectMeta{ Name: "csi-hostpath", }, - Spec: csi.CSIDriverSpec{ + Spec: csiv1alpha1.CSIDriverSpec{ AttachRequired: &attachable, }, } diff --git a/test/integration/volume/BUILD b/test/integration/volume/BUILD index 156b58ab2d..0e6231fde6 100644 --- a/test/integration/volume/BUILD +++ b/test/integration/volume/BUILD @@ -26,7 +26,6 @@ go_test( "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/test/integration/volume/attach_detach_test.go b/test/integration/volume/attach_detach_test.go index 956b90c587..aa72838c05 100644 --- a/test/integration/volume/attach_detach_test.go +++ b/test/integration/volume/attach_detach_test.go @@ -23,7 +23,6 @@ import ( "time" "k8s.io/api/core/v1" - fakeapiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -394,7 +393,6 @@ func createAdClients(ns *v1.Namespace, t *testing.T, server *httptest.Server, sy } resyncPeriod := 12 * time.Hour testClient := clientset.NewForConfigOrDie(&config) - fakeApiExtensionsClient := fakeapiextensionsclient.NewSimpleClientset() host := volumetest.NewFakeVolumeHost("/tmp/fake", nil, nil) plugin := &volumetest.FakeVolumePlugin{ @@ -415,7 +413,6 @@ func createAdClients(ns *v1.Namespace, t *testing.T, server *httptest.Server, sy ctrl, err := attachdetach.NewAttachDetachController( testClient, nil, /* csiClient */ - fakeApiExtensionsClient, /* crdClient */ informers.Core().V1().Pods(), informers.Core().V1().Nodes(), informers.Core().V1().PersistentVolumeClaims(),