From a679486a5ed66a07cd0ce3b8dc9143212744b624 Mon Sep 17 00:00:00 2001 From: saad-ali Date: Wed, 24 Oct 2018 10:04:06 -0700 Subject: [PATCH] Register CSI CRDs as addon --- cluster/addons/storage-crds/OWNERS | 6 + cluster/addons/storage-crds/README.md | 13 +++ .../addons/storage-crds}/csidriver.yaml | 9 +- .../addons/storage-crds}/csinodeinfo.yaml | 9 +- cluster/gce/config-default.sh | 16 ++- cluster/gce/gci/configure-helper.sh | 3 + staging/src/k8s.io/csi-api/Godeps/Godeps.json | 12 -- staging/src/k8s.io/csi-api/pkg/crd/BUILD | 26 ++--- staging/src/k8s.io/csi-api/pkg/crd/crd.go | 107 +----------------- .../src/k8s.io/csi-api/pkg/crd/crd_test.go | 75 +++++++----- .../csi-api/pkg/crd/manifests/csidriver.yaml | 28 +++++ .../pkg/crd/manifests/csinodeinfo.yaml | 32 ++++++ test/e2e/storage/BUILD | 3 - test/e2e/storage/csi_objects.go | 26 ----- test/e2e/storage/csi_volumes.go | 16 +-- test/integration/auth/BUILD | 6 +- test/integration/auth/node_test.go | 25 +++- 17 files changed, 194 insertions(+), 218 deletions(-) create mode 100644 cluster/addons/storage-crds/OWNERS create mode 100644 cluster/addons/storage-crds/README.md rename {staging/src/k8s.io/csi-api/pkg/crd/testdata => cluster/addons/storage-crds}/csidriver.yaml (88%) rename {staging/src/k8s.io/csi-api/pkg/crd/testdata => cluster/addons/storage-crds}/csinodeinfo.yaml (88%) create mode 100644 staging/src/k8s.io/csi-api/pkg/crd/manifests/csidriver.yaml create mode 100644 staging/src/k8s.io/csi-api/pkg/crd/manifests/csinodeinfo.yaml diff --git a/cluster/addons/storage-crds/OWNERS b/cluster/addons/storage-crds/OWNERS new file mode 100644 index 0000000000..852f35cac6 --- /dev/null +++ b/cluster/addons/storage-crds/OWNERS @@ -0,0 +1,6 @@ +approvers: +- saad-ali +- jsafrane +- msau42 +reviewers: +- davidz627 \ No newline at end of file diff --git a/cluster/addons/storage-crds/README.md b/cluster/addons/storage-crds/README.md new file mode 100644 index 0000000000..0ac4681c13 --- /dev/null +++ b/cluster/addons/storage-crds/README.md @@ -0,0 +1,13 @@ +# Kubernetes CSI CRDs + +The Kubernetes Container Storage Interface implementation defines some API objects as CRDs that Kubernetes components +including the Attach/Detach controller depend on. + +If you are using CSI, it is recommended that you enable the relevant feature gates (e.g. `CSIDriverRegistry`, `CSINodeInfo`, etc.), and ensure the CRDs in this directory are installed. + +These objects and their CRDs are defined in `staging/src/k8s.io/csi-api/pkg/crd/manifests`, the source of truth. +They are copied from that CRD manifest directory to this addon directory. +A unit test in `staging/src/k8s.io/csi-api/pkg/crd` verifies that this (and any other) copies of the manifest outside of `staging/src/k8s.io/csi-api/pkg/crd/manifests` do not drift from that source of truth. +If you need to make changes please make changes in the `staging/src/k8s.io/csi-api/pkg/crd/manifests` directory and then update this copy. + +For more information, see: https://kubernetes-csi.github.io/docs/ diff --git a/staging/src/k8s.io/csi-api/pkg/crd/testdata/csidriver.yaml b/cluster/addons/storage-crds/csidriver.yaml similarity index 88% rename from staging/src/k8s.io/csi-api/pkg/crd/testdata/csidriver.yaml rename to cluster/addons/storage-crds/csidriver.yaml index d950cbf494..54416b24f9 100644 --- a/staging/src/k8s.io/csi-api/pkg/crd/testdata/csidriver.yaml +++ b/cluster/addons/storage-crds/csidriver.yaml @@ -1,8 +1,9 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: - creationTimestamp: null name: csidrivers.csi.storage.k8s.io + labels: + addonmanager.kubernetes.io/mode: Reconcile spec: group: csi.storage.k8s.io names: @@ -25,9 +26,3 @@ spec: information (like podName, podUID, etc.) during mount operations. type: string version: v1alpha1 -status: - acceptedNames: - kind: "" - plural: "" - conditions: null - storedVersions: null diff --git a/staging/src/k8s.io/csi-api/pkg/crd/testdata/csinodeinfo.yaml b/cluster/addons/storage-crds/csinodeinfo.yaml similarity index 88% rename from staging/src/k8s.io/csi-api/pkg/crd/testdata/csinodeinfo.yaml rename to cluster/addons/storage-crds/csinodeinfo.yaml index 6ea408d5e7..a9556fc1e8 100644 --- a/staging/src/k8s.io/csi-api/pkg/crd/testdata/csinodeinfo.yaml +++ b/cluster/addons/storage-crds/csinodeinfo.yaml @@ -1,8 +1,9 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: - creationTimestamp: null name: csinodeinfos.csi.storage.k8s.io + labels: + addonmanager.kubernetes.io/mode: Reconcile spec: group: csi.storage.k8s.io names: @@ -29,9 +30,3 @@ spec: type: array type: array version: v1alpha1 -status: - acceptedNames: - kind: "" - plural: "" - conditions: null - storedVersions: null diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 671c1a5db7..20d4d722bc 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -246,10 +246,6 @@ fi # Optional: customize runtime config RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}" -if [[ "${KUBE_FEATURE_GATES:-}" == "AllAlpha=true" ]]; then - RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-api/all=true}" -fi - # Optional: set feature gates FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}" @@ -260,6 +256,18 @@ if [[ ! -z "${NODE_ACCELERATORS}" ]]; then fi fi +if [[ "${KUBE_FEATURE_GATES:-}" == "AllAlpha=true" ]]; then + RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-api/all=true}" + if ! [[ "${KUBE_FEATURE_GATES:-}" =~ "CSIDriverRegistry" ]]; then + # If not explicitly specified, default to true. + FEATURE_GATES="${FEATURE_GATES},CSIDriverRegistry=true" + fi + if ! [[ "${KUBE_FEATURE_GATES:-}" =~ "CSINodeInfo" ]]; then + # If not explicitly specified, default to true. + FEATURE_GATES="${FEATURE_GATES},CSINodeInfo=true" + fi +fi + # Optional: Install cluster DNS. # Set CLUSTER_DNS_CORE_DNS to 'false' to install kube-dns instead of CoreDNS. CLUSTER_DNS_CORE_DNS="${CLUSTER_DNS_CORE_DNS:-true}" diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 411b529b28..10a3a0e663 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -2567,6 +2567,9 @@ EOF if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then setup-addon-manifests "addons" "storage-class/gce" fi + if [[ "${FEATURE_GATES:-}" =~ "CSIDriverRegistry=true" || "${FEATURE_GATES:-}" =~ "CSINodeInfo=true" ]]; then + setup-addon-manifests "addons" "storage-crds" + fi if [[ "${ENABLE_IP_MASQ_AGENT:-}" == "true" ]]; then setup-addon-manifests "addons" "ip-masq-agent" fi diff --git a/staging/src/k8s.io/csi-api/Godeps/Godeps.json b/staging/src/k8s.io/csi-api/Godeps/Godeps.json index 091b3c0e16..f5aba5bd57 100644 --- a/staging/src/k8s.io/csi-api/Godeps/Godeps.json +++ b/staging/src/k8s.io/csi-api/Godeps/Godeps.json @@ -298,18 +298,6 @@ "ImportPath": "k8s.io/api/storage/v1beta1", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, - { - "ImportPath": "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, - { - "ImportPath": "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme", - "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - }, { "ImportPath": "k8s.io/apimachinery/pkg/api/errors", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" diff --git a/staging/src/k8s.io/csi-api/pkg/crd/BUILD b/staging/src/k8s.io/csi-api/pkg/crd/BUILD index 3a9a64a5c8..71a403cd57 100644 --- a/staging/src/k8s.io/csi-api/pkg/crd/BUILD +++ b/staging/src/k8s.io/csi-api/pkg/crd/BUILD @@ -6,25 +6,17 @@ go_library( importmap = "k8s.io/kubernetes/vendor/k8s.io/csi-api/pkg/crd", importpath = "k8s.io/csi-api/pkg/crd", visibility = ["//visibility:public"], - deps = [ - "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", - ], ) go_test( name = "go_default_test", srcs = ["crd_test.go"], - data = glob(["testdata/**"]), - embed = [":go_default_library"], - deps = [ - "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", + data = [ + "//cluster/addons:addon-srcs", + "//staging/src/k8s.io/csi-api/pkg/crd:manifests", ], + embed = [":go_default_library"], + deps = ["//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library"], ) filegroup( @@ -34,6 +26,14 @@ filegroup( visibility = ["//visibility:private"], ) +filegroup( + name = "csi-manifests", + srcs = [ + "//staging/src/k8s.io/csi-api/pkg/crd:manifests", + ], + visibility = ["//visibility:public"], +) + filegroup( name = "all-srcs", srcs = [":package-srcs"], diff --git a/staging/src/k8s.io/csi-api/pkg/crd/crd.go b/staging/src/k8s.io/csi-api/pkg/crd/crd.go index 4a7936e414..02e5b15281 100644 --- a/staging/src/k8s.io/csi-api/pkg/crd/crd.go +++ b/staging/src/k8s.io/csi-api/pkg/crd/crd.go @@ -14,106 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Package crd is only for running tests to verify the manifest files +// in this package and the addons are in sync. +// The manifest file is currently manually generated, in the future, we +// should invest in tooling that will automatically generate the CRD +// manifest from the CR schema. package crd - -import ( - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - csiapiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1" - "reflect" -) - -// NOTE: the CRD functions here and the associated unit tests are non-ideal temporary measures in -// release 1.12 in order to aid manual CRD installation. This installation will be automated in -// subsequent releases and as a result this package will be removed. - -// CSIDriverCRD returns the CustomResourceDefinition for CSIDriver object. -func CSIDriverCRD() *apiextensionsv1beta1.CustomResourceDefinition { - return &apiextensionsv1beta1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: csiapiv1alpha1.CsiDriverResourcePlural + "." + csiapiv1alpha1.GroupName, - }, - Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ - Group: csiapiv1alpha1.GroupName, - Version: csiapiv1alpha1.SchemeGroupVersion.Version, - Scope: apiextensionsv1beta1.ClusterScoped, - Validation: &apiextensionsv1beta1.CustomResourceValidation{ - OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ - Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ - "spec": { - Description: "Specification of the CSI Driver.", - Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ - "attachRequired": { - Description: "Indicates this CSI volume driver requires an attach operation," + - " and that Kubernetes should call attach and wait for any attach operation to" + - " complete before proceeding to mount.", - Type: "boolean", - }, - "podInfoOnMountVersion": { - Description: "Indicates this CSI volume driver requires additional pod" + - " information (like podName, podUID, etc.) during mount operations.", - Type: "string", - }, - }, - }, - }, - }, - }, - Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ - Plural: csiapiv1alpha1.CsiDriverResourcePlural, - Kind: reflect.TypeOf(csiapiv1alpha1.CSIDriver{}).Name(), - }, - }, - } -} - -// CSINodeInfoCRD returns the CustomResourceDefinition for CSINodeInfo object. -func CSINodeInfoCRD() *apiextensionsv1beta1.CustomResourceDefinition { - return &apiextensionsv1beta1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: csiapiv1alpha1.CsiNodeInfoResourcePlural + "." + csiapiv1alpha1.GroupName, - }, - Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ - Group: csiapiv1alpha1.GroupName, - Version: csiapiv1alpha1.SchemeGroupVersion.Version, - Scope: apiextensionsv1beta1.ClusterScoped, - Validation: &apiextensionsv1beta1.CustomResourceValidation{ - OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ - Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ - "csiDrivers": { - Description: "List of CSI drivers running on the node and their properties.", - Type: "array", - Items: &apiextensionsv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextensionsv1beta1.JSONSchemaProps{ - Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ - "driver": { - Description: "The CSI driver that this object refers to.", - Type: "string", - }, - "nodeID": { - Description: "The node from the driver point of view.", - Type: "string", - }, - "topologyKeys": { - Description: "List of keys supported by the driver.", - Type: "array", - Items: &apiextensionsv1beta1.JSONSchemaPropsOrArray{ - Schema: &apiextensionsv1beta1.JSONSchemaProps{ - Type: "string", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ - Plural: csiapiv1alpha1.CsiNodeInfoResourcePlural, - Kind: reflect.TypeOf(csiapiv1alpha1.CSINodeInfo{}).Name(), - }, - }, - } -} diff --git a/staging/src/k8s.io/csi-api/pkg/crd/crd_test.go b/staging/src/k8s.io/csi-api/pkg/crd/crd_test.go index 369a3b458b..2bb155baff 100644 --- a/staging/src/k8s.io/csi-api/pkg/crd/crd_test.go +++ b/staging/src/k8s.io/csi-api/pkg/crd/crd_test.go @@ -14,56 +14,69 @@ See the License for the specific language governing permissions and limitations under the License. */ +// These tests verify the manifest files in this package and the +// addons directory are in sync. package crd_test import ( + "io/ioutil" + "os" "path/filepath" "testing" - "github.com/ghodss/yaml" - "io/ioutil" - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - apiextensionsscheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/diff" - "k8s.io/csi-api/pkg/crd" - "os" ) func TestBootstrapCRDs(t *testing.T) { - testObjects(t, crd.CSIDriverCRD(), "csidriver.yaml") - testObjects(t, crd.CSINodeInfoCRD(), "csinodeinfo.yaml") + verifyCopiesAreInSync( + t, + "csidriver.yaml", /* filename */ + "manifests", /* sourceOfTruthDir */ + []string{"../../../../../../cluster/addons/storage-crds"}, /* copyDirs */ + ) + verifyCopiesAreInSync( + t, + "csinodeinfo.yaml", /* filename */ + "manifests", /* sourceOfTruthDir */ + []string{"../../../../../../cluster/addons/storage-crds"}, /* copyDirs */ + ) } -func testObjects(t *testing.T, crd *apiextensionsv1beta1.CustomResourceDefinition, fixtureFilename string) { - filename := filepath.Join("testdata", fixtureFilename) - expectedYAML, err := ioutil.ReadFile(filename) +// verifyCopiesAreInSync fails if any copies are different from source of truth. +func verifyCopiesAreInSync(t *testing.T, filename string, sourceOfTruthDir string, copyDirs []string) { + sourceOfTruthFilename := filepath.Join(sourceOfTruthDir, filename) + + if len(copyDirs) <= 0 { + t.Fatalf("copyDirs is empty. There are no copies to validate.") + } + + expectedYAML, err := ioutil.ReadFile(sourceOfTruthFilename) if err != nil { t.Fatal(err) } - jsonData, err := runtime.Encode(apiextensionsscheme.Codecs.LegacyCodec(apiextensionsv1beta1.SchemeGroupVersion), crd) - if err != nil { - t.Fatal(err) - } - yamlData, err := yaml.JSONToYAML(jsonData) - if err != nil { - t.Fatal(err) - } - if string(yamlData) != string(expectedYAML) { - t.Errorf("Bootstrap CRD data does not match the test fixture in %s", filename) + for _, copyDir := range copyDirs { + copyFilename := filepath.Join(copyDir, filename) + actualYAML, err := ioutil.ReadFile(copyFilename) + if err != nil { + t.Fatal(err) + } - const updateEnvVar = "UPDATE_CSI_CRD_FIXTURE_DATA" - if os.Getenv(updateEnvVar) == "true" { - if err := ioutil.WriteFile(filename, []byte(yamlData), os.FileMode(0755)); err == nil { - t.Logf("Updated data in %s", filename) - t.Logf("Verify the diff, commit changes, and rerun the tests") + if string(actualYAML) != string(expectedYAML) { + t.Errorf("Data in %q does not match source of truth in %q.", copyFilename, sourceOfTruthFilename) + + const updateEnvVar = "UPDATE_CSI_CRD_FIXTURE_DATA" + if os.Getenv(updateEnvVar) == "true" { + if err := ioutil.WriteFile(copyFilename, []byte(expectedYAML), os.FileMode(0755)); err == nil { + t.Logf("Updated data in %s", copyFilename) + t.Logf("Verify the diff, commit changes, and rerun the tests") + } else { + t.Logf("Could not update data in %s: %v", copyFilename, err) + } } else { - t.Logf("Could not update data in %s: %v", filename, err) + t.Logf("Diff between source of truth data and copy data in %s:\n-------------\n%s", copyFilename, diff.StringDiff(string(actualYAML), string(expectedYAML))) + t.Logf("If the change is expected, re-run with %s=true to update the copy data", updateEnvVar) } - } else { - t.Logf("Diff between data in code and fixture data in %s:\n-------------\n%s", filename, diff.StringDiff(string(yamlData), string(expectedYAML))) - t.Logf("If the change is expected, re-run with %s=true to update the fixtures", updateEnvVar) } } } diff --git a/staging/src/k8s.io/csi-api/pkg/crd/manifests/csidriver.yaml b/staging/src/k8s.io/csi-api/pkg/crd/manifests/csidriver.yaml new file mode 100644 index 0000000000..54416b24f9 --- /dev/null +++ b/staging/src/k8s.io/csi-api/pkg/crd/manifests/csidriver.yaml @@ -0,0 +1,28 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: csidrivers.csi.storage.k8s.io + labels: + addonmanager.kubernetes.io/mode: Reconcile +spec: + group: csi.storage.k8s.io + names: + kind: CSIDriver + plural: csidrivers + scope: Cluster + validation: + openAPIV3Schema: + properties: + spec: + description: Specification of the CSI Driver. + properties: + attachRequired: + description: Indicates this CSI volume driver requires an attach operation, + and that Kubernetes should call attach and wait for any attach operation + to complete before proceeding to mount. + type: boolean + podInfoOnMountVersion: + description: Indicates this CSI volume driver requires additional pod + information (like podName, podUID, etc.) during mount operations. + type: string + version: v1alpha1 diff --git a/staging/src/k8s.io/csi-api/pkg/crd/manifests/csinodeinfo.yaml b/staging/src/k8s.io/csi-api/pkg/crd/manifests/csinodeinfo.yaml new file mode 100644 index 0000000000..a9556fc1e8 --- /dev/null +++ b/staging/src/k8s.io/csi-api/pkg/crd/manifests/csinodeinfo.yaml @@ -0,0 +1,32 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: csinodeinfos.csi.storage.k8s.io + labels: + addonmanager.kubernetes.io/mode: Reconcile +spec: + group: csi.storage.k8s.io + names: + kind: CSINodeInfo + plural: csinodeinfos + scope: Cluster + validation: + openAPIV3Schema: + properties: + csiDrivers: + description: List of CSI drivers running on the node and their properties. + items: + properties: + driver: + description: The CSI driver that this object refers to. + type: string + nodeID: + description: The node from the driver point of view. + type: string + topologyKeys: + description: List of keys supported by the driver. + items: + type: string + type: array + type: array + version: v1alpha1 diff --git a/test/e2e/storage/BUILD b/test/e2e/storage/BUILD index e800048a36..5e7c6d9a6f 100644 --- a/test/e2e/storage/BUILD +++ b/test/e2e/storage/BUILD @@ -45,8 +45,6 @@ go_library( "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -68,7 +66,6 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", - "//staging/src/k8s.io/csi-api/pkg/crd:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e/framework/metrics:go_default_library", "//test/e2e/framework/podlogs:go_default_library", diff --git a/test/e2e/storage/csi_objects.go b/test/e2e/storage/csi_objects.go index 5907a6f426..a7db54f0c8 100644 --- a/test/e2e/storage/csi_objects.go +++ b/test/e2e/storage/csi_objects.go @@ -28,18 +28,11 @@ import ( "path/filepath" "k8s.io/api/core/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" - - . "github.com/onsi/ginkgo" - - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - csicrd "k8s.io/csi-api/pkg/crd" ) var ( @@ -64,25 +57,6 @@ func csiContainerImage(image string) string { return fullName } -func createCSICRDs(c apiextensionsclient.Interface) { - By("Creating CSI CRDs") - crds := []*apiextensionsv1beta1.CustomResourceDefinition{ - csicrd.CSIDriverCRD(), - csicrd.CSINodeInfoCRD(), - } - - for _, crd := range crds { - _, err := c.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{}) - if err == nil { - continue - } else if !apierrs.IsNotFound(err) { - framework.ExpectNoError(err, "Failed to check for existing of CSI CRD %q: %v", crd.Name, err) - } - _, err = c.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd) - framework.ExpectNoError(err, "Failed to create CSI CRD %q: %v", crd.Name, err) - } -} - func shredFile(filePath string) { if _, err := os.Stat(filePath); os.IsNotExist(err) { framework.Logf("File %v was not found, skipping shredding", filePath) diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go index e93a20f5e1..24ac665821 100644 --- a/test/e2e/storage/csi_volumes.go +++ b/test/e2e/storage/csi_volumes.go @@ -25,7 +25,6 @@ import ( "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" @@ -60,13 +59,12 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { f := framework.NewDefaultFramework("csi-volumes") var ( - cancel context.CancelFunc - cs clientset.Interface - crdclient apiextensionsclient.Interface - csics csiclient.Interface - ns *v1.Namespace - node v1.Node - config framework.VolumeTestConfig + cancel context.CancelFunc + cs clientset.Interface + csics csiclient.Interface + ns *v1.Namespace + node v1.Node + config framework.VolumeTestConfig ) BeforeEach(func() { @@ -74,7 +72,6 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { cancel = c cs = f.ClientSet - crdclient = f.APIExtensionsClientSet csics = f.CSIClientSet ns = f.Namespace @@ -114,7 +111,6 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { ServerNodeName: node.Name, WaitForCompletion: true, } - createCSICRDs(crdclient) }) AfterEach(func() { diff --git a/test/integration/auth/BUILD b/test/integration/auth/BUILD index 6a640ac36a..48ea61d250 100644 --- a/test/integration/auth/BUILD +++ b/test/integration/auth/BUILD @@ -17,6 +17,9 @@ go_test( "rbac_test.go", "svcaccttoken_test.go", ], + data = [ + "//staging/src/k8s.io/csi-api/pkg/crd:csi-manifests", + ], tags = ["integration"], deps = [ "//cmd/kube-apiserver/app/testing:go_default_library", @@ -49,11 +52,13 @@ go_test( "//staging/src/k8s.io/api/authentication/v1beta1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", @@ -80,7 +85,6 @@ go_test( "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", - "//staging/src/k8s.io/csi-api/pkg/crd:go_default_library", "//test/e2e/lifecycle/bootstrap:go_default_library", "//test/integration:go_default_library", "//test/integration/etcd:go_default_library", diff --git a/test/integration/auth/node_test.go b/test/integration/auth/node_test.go index e3eeb9cf00..3d4038efb6 100644 --- a/test/integration/auth/node_test.go +++ b/test/integration/auth/node_test.go @@ -24,9 +24,11 @@ import ( "time" storagev1beta1 "k8s.io/api/storage/v1beta1" + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -34,8 +36,8 @@ import ( externalclientset "k8s.io/client-go/kubernetes" csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1" csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned" - csicrd "k8s.io/csi-api/pkg/crd" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" + "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/coordination" "k8s.io/kubernetes/pkg/apis/core" api "k8s.io/kubernetes/pkg/apis/core" @@ -156,7 +158,12 @@ func TestNodeAuthorizer(t *testing.T) { t.Fatal(err) } - etcd.CreateTestCRDs(t, superuserCRDClient, false, csicrd.CSINodeInfoCRD()) + csiNodeInfoCRD, err := crdFromManifest("../../../staging/src/k8s.io/csi-api/pkg/crd/manifests/csinodeinfo.yaml") + if err != nil { + t.Fatal(err) + } + + etcd.CreateTestCRDs(t, superuserCRDClient, false, csiNodeInfoCRD) getSecret := func(client clientset.Interface) func() error { return func() error { @@ -664,3 +671,17 @@ func expectAllowed(t *testing.T, f func() error) { t.Errorf("Expected no error, got %v", err) } } + +// crdFromManifest reads a .json/yaml file and returns the CRD in it. +func crdFromManifest(filename string) (*apiextensionsv1beta1.CustomResourceDefinition, error) { + var crd apiextensionsv1beta1.CustomResourceDefinition + data, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), data, &crd); err != nil { + return nil, err + } + return &crd, nil +}