diff --git a/cmd/kube-apiserver/app/BUILD b/cmd/kube-apiserver/app/BUILD index 7830ca5d08..9b55641912 100644 --- a/cmd/kube-apiserver/app/BUILD +++ b/cmd/kube-apiserver/app/BUILD @@ -39,6 +39,7 @@ go_library( "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", diff --git a/cmd/kube-apiserver/app/aggregator.go b/cmd/kube-apiserver/app/aggregator.go index acf5b071ca..2cedb55cdc 100644 --- a/cmd/kube-apiserver/app/aggregator.go +++ b/cmd/kube-apiserver/app/aggregator.go @@ -30,6 +30,7 @@ import ( apiextensionsinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/admission" @@ -79,6 +80,7 @@ func createAggregatorConfig( etcdOptions := *commandOptions.Etcd etcdOptions.StorageConfig.Paging = utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking) etcdOptions.StorageConfig.Codec = aggregatorscheme.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion) + etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1beta1.SchemeGroupVersion, schema.GroupKind{Group: v1beta1.GroupName}) genericConfig.RESTOptionsGetter = &genericoptions.SimpleRestOptionsFactory{Options: etcdOptions} // override MergedResourceConfig with aggregator defaults and registry diff --git a/cmd/kube-apiserver/app/apiextensions.go b/cmd/kube-apiserver/app/apiextensions.go index 2db21153cd..f2da1ca989 100644 --- a/cmd/kube-apiserver/app/apiextensions.go +++ b/cmd/kube-apiserver/app/apiextensions.go @@ -23,6 +23,8 @@ import ( "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" apiextensionsoptions "k8s.io/apiextensions-apiserver/pkg/cmd/server/options" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/features" genericapiserver "k8s.io/apiserver/pkg/server" @@ -61,6 +63,7 @@ func createAPIExtensionsConfig( etcdOptions := *commandOptions.Etcd etcdOptions.StorageConfig.Paging = utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking) etcdOptions.StorageConfig.Codec = apiextensionsapiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion) + etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1beta1.SchemeGroupVersion, schema.GroupKind{Group: v1beta1.GroupName}) genericConfig.RESTOptionsGetter = &genericoptions.SimpleRestOptionsFactory{Options: etcdOptions} // override MergedResourceConfig with apiextensions defaults and registry diff --git a/pkg/master/BUILD b/pkg/master/BUILD index 28bf854721..0bec722bf3 100644 --- a/pkg/master/BUILD +++ b/pkg/master/BUILD @@ -141,10 +141,13 @@ go_test( deps = [ "//pkg/api/legacyscheme:go_default_library", "//pkg/api/testapi:go_default_library", + "//pkg/apis/batch:go_default_library", "//pkg/apis/core:go_default_library", + "//pkg/apis/storage:go_default_library", "//pkg/generated/openapi:go_default_library", "//pkg/kubelet/client:go_default_library", "//pkg/master/reconcilers:go_default_library", + "//pkg/master/storageversionhashdata:go_default_library", "//pkg/registry/certificates/rest:go_default_library", "//pkg/registry/core/rest:go_default_library", "//pkg/registry/registrytest:go_default_library", @@ -154,16 +157,23 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/api/apitesting/naming:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/version:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/openapi:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/features:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server/resourceconfig:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", + "//staging/src/k8s.io/client-go/discovery:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", @@ -191,6 +201,7 @@ filegroup( "//pkg/master/controller/crdregistration:all-srcs", "//pkg/master/ports:all-srcs", "//pkg/master/reconcilers:all-srcs", + "//pkg/master/storageversionhashdata:all-srcs", "//pkg/master/tunneler:all-srcs", ], tags = ["automanaged"], diff --git a/pkg/master/master_test.go b/pkg/master/master_test.go index efac9d45f0..90239fa0e0 100644 --- a/pkg/master/master_test.go +++ b/pkg/master/master_test.go @@ -31,22 +31,32 @@ import ( certificatesapiv1beta1 "k8s.io/api/certificates/v1beta1" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/version" + "k8s.io/apiserver/pkg/authorization/authorizerfactory" + "k8s.io/apiserver/pkg/features" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/options" + "k8s.io/apiserver/pkg/server/resourceconfig" serverstorage "k8s.io/apiserver/pkg/server/storage" etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + "k8s.io/client-go/discovery" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" restclient "k8s.io/client-go/rest" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/api/testapi" + "k8s.io/kubernetes/pkg/apis/batch" api "k8s.io/kubernetes/pkg/apis/core" + apisstorage "k8s.io/kubernetes/pkg/apis/storage" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" "k8s.io/kubernetes/pkg/master/reconcilers" + "k8s.io/kubernetes/pkg/master/storageversionhashdata" certificatesrest "k8s.io/kubernetes/pkg/registry/certificates/rest" corerest "k8s.io/kubernetes/pkg/registry/core/rest" "k8s.io/kubernetes/pkg/registry/registrytest" @@ -70,6 +80,14 @@ func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, *assert.Assertion } resourceEncoding := serverstorage.NewDefaultResourceEncodingConfig(legacyscheme.Scheme) + // This configures the testing master the same way the real master is + // configured. The storage versions of these resources are different + // from the storage versions of other resources in their group. + resourceEncodingOverrides := []schema.GroupVersionResource{ + batch.Resource("cronjobs").WithVersion("v1beta1"), + apisstorage.Resource("volumeattachments").WithVersion("v1beta1"), + } + resourceEncoding = resourceconfig.MergeResourceEncodingConfigs(resourceEncoding, resourceEncodingOverrides) storageFactory := serverstorage.NewDefaultStorageFactory(*storageConfig, testapi.StorageMediaType(), legacyscheme.Codecs, resourceEncoding, DefaultAPIResourceConfigSource(), nil) etcdOptions := options.NewEtcdOptions(storageConfig) @@ -81,12 +99,12 @@ func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, *assert.Assertion } kubeVersion := kubeversion.Get() + config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer() config.GenericConfig.Version = &kubeVersion config.ExtraConfig.StorageFactory = storageFactory config.GenericConfig.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}} config.GenericConfig.PublicAddress = net.ParseIP("192.168.10.4") config.GenericConfig.LegacyAPIGroupPrefixes = sets.NewString("/api") - config.GenericConfig.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}} config.ExtraConfig.KubeletClientConfig = kubeletclient.KubeletClientConfig{Port: 10250} config.ExtraConfig.ProxyTransport = utilnet.SetTransportDefaults(&http.Transport{ DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { return nil, nil }, @@ -363,6 +381,112 @@ func TestAPIVersionOfDiscoveryEndpoints(t *testing.T) { } +// This test doesn't cover the apiregistration and apiextensions group, as they are installed by other apiservers. +func TestStorageVersionHashes(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionHash, true)() + master, etcdserver, _, _ := newMaster(t) + defer etcdserver.Terminate(t) + + server := httptest.NewServer(master.GenericAPIServer.Handler.GoRestfulContainer.ServeMux) + + c := &restclient.Config{ + Host: server.URL, + APIPath: "/api", + ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}, + } + discover := discovery.NewDiscoveryClientForConfigOrDie(c) + all, err := discover.ServerResources() + if err != nil { + t.Error(err) + } + var count int + for _, g := range all { + for _, r := range g.APIResources { + if strings.Contains(r.Name, "/") || + storageversionhashdata.NoStorageVersionHash.Has(g.GroupVersion+"/"+r.Name) { + if r.StorageVersionHash != "" { + t.Errorf("expect resource %s/%s to have empty storageVersionHash, got hash %q", g.GroupVersion, r.Name, r.StorageVersionHash) + } + continue + } + if r.StorageVersionHash == "" { + t.Errorf("expect the storageVersionHash of %s/%s to exist", g.GroupVersion, r.Name) + continue + } + // Uncomment the following line if you want to update storageversionhash/data.go + // fmt.Printf("\"%s/%s\": \"%s\",\n", g.GroupVersion, r.Name, r.StorageVersionHash) + expected := storageversionhashdata.GVRToStorageVersionHash[g.GroupVersion+"/"+r.Name] + if r.StorageVersionHash != expected { + t.Errorf("expect the storageVersionHash of %s/%s to be %q, got %q", g.GroupVersion, r.Name, expected, r.StorageVersionHash) + } + count++ + } + } + if count != len(storageversionhashdata.GVRToStorageVersionHash) { + t.Errorf("please remove the redundant entries from GVRToStorageVersionHash") + } +} + +func TestStorageVersionHashEqualities(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionHash, true)() + master, etcdserver, _, assert := newMaster(t) + defer etcdserver.Terminate(t) + + server := httptest.NewServer(master.GenericAPIServer.Handler.GoRestfulContainer.ServeMux) + + // Test 1: extensions/v1beta1/replicasets and apps/v1/replicasets have + // the same storage version hash. + resp, err := http.Get(server.URL + "/apis/extensions/v1beta1") + assert.Empty(err) + extList := metav1.APIResourceList{} + assert.NoError(decodeResponse(resp, &extList)) + var extReplicasetHash, appsReplicasetHash string + for _, r := range extList.APIResources { + if r.Name == "replicasets" { + extReplicasetHash = r.StorageVersionHash + } + } + assert.NotEmpty(extReplicasetHash) + + resp, err = http.Get(server.URL + "/apis/apps/v1") + assert.Empty(err) + appsList := metav1.APIResourceList{} + assert.NoError(decodeResponse(resp, &appsList)) + for _, r := range appsList.APIResources { + if r.Name == "replicasets" { + appsReplicasetHash = r.StorageVersionHash + } + } + assert.Equal(extReplicasetHash, appsReplicasetHash) + + // Test 2: batch/v1/jobs and batch/v1beta1/cronjobs have different + // storage version hashes. + resp, err = http.Get(server.URL + "/apis/batch/v1") + assert.Empty(err) + batchv1 := metav1.APIResourceList{} + assert.NoError(decodeResponse(resp, &batchv1)) + var jobsHash string + for _, r := range batchv1.APIResources { + if r.Name == "jobs" { + jobsHash = r.StorageVersionHash + } + } + assert.NotEmpty(jobsHash) + + resp, err = http.Get(server.URL + "/apis/batch/v1beta1") + assert.Empty(err) + batchv1beta1 := metav1.APIResourceList{} + assert.NoError(decodeResponse(resp, &batchv1beta1)) + var cronjobsHash string + for _, r := range batchv1beta1.APIResources { + if r.Name == "cronjobs" { + cronjobsHash = r.StorageVersionHash + } + } + assert.NotEmpty(cronjobsHash) + assert.NotEqual(jobsHash, cronjobsHash) +} + func TestNoAlphaVersionsEnabledByDefault(t *testing.T) { config := DefaultAPIResourceConfigSource() for gv, enable := range config.GroupVersionConfigs { diff --git a/pkg/master/storageversionhashdata/BUILD b/pkg/master/storageversionhashdata/BUILD new file mode 100644 index 0000000000..7eca8cf1e5 --- /dev/null +++ b/pkg/master/storageversionhashdata/BUILD @@ -0,0 +1,23 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["data.go"], + importpath = "k8s.io/kubernetes/pkg/master/storageversionhashdata", + visibility = ["//visibility:public"], + deps = ["//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/master/storageversionhashdata/OWNERS b/pkg/master/storageversionhashdata/OWNERS new file mode 100644 index 0000000000..8f7783f9f0 --- /dev/null +++ b/pkg/master/storageversionhashdata/OWNERS @@ -0,0 +1,4 @@ +approvers: +- api-approvers +reviewers: +- api-reviewers diff --git a/pkg/master/storageversionhashdata/data.go b/pkg/master/storageversionhashdata/data.go new file mode 100644 index 0000000000..2a8b50365b --- /dev/null +++ b/pkg/master/storageversionhashdata/data.go @@ -0,0 +1,111 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package storageversionhashdata is for test only. +package storageversionhashdata + +import ( + "k8s.io/apimachinery/pkg/util/sets" +) + +// NoStorageVersionHash lists resources that legitimately with empty storage +// version hash. +var NoStorageVersionHash = sets.NewString( + "v1/bindings", + "v1/componentstatuses", + "authentication.k8s.io/v1/tokenreviews", + "authorization.k8s.io/v1/localsubjectaccessreviews", + "authorization.k8s.io/v1/selfsubjectaccessreviews", + "authorization.k8s.io/v1/selfsubjectrulesreviews", + "authorization.k8s.io/v1/subjectaccessreviews", + "authentication.k8s.io/v1beta1/tokenreviews", + "authorization.k8s.io/v1beta1/localsubjectaccessreviews", + "authorization.k8s.io/v1beta1/selfsubjectaccessreviews", + "authorization.k8s.io/v1beta1/selfsubjectrulesreviews", + "authorization.k8s.io/v1beta1/subjectaccessreviews", + "extensions/v1beta1/replicationcontrollers", +) + +// GVRToStorageVersionHash shouldn't change unless we intentionally change the +// storage version of a resource. +var GVRToStorageVersionHash = map[string]string{ + "v1/configmaps": "qFsyl6wFWjQ=", + "v1/endpoints": "fWeeMqaN/OA=", + "v1/events": "r2yiGXH7wu8=", + "v1/limitranges": "EBKMFVe6cwo=", + "v1/namespaces": "Q3oi5N2YM8M=", + "v1/nodes": "XwShjMxG9Fs=", + "v1/persistentvolumeclaims": "QWTyNDq0dC4=", + "v1/persistentvolumes": "HN/zwEC+JgM=", + "v1/pods": "xPOwRZ+Yhw8=", + "v1/podtemplates": "LIXB2x4IFpk=", + "v1/replicationcontrollers": "Jond2If31h0=", + "v1/resourcequotas": "8uhSgffRX6w=", + "v1/secrets": "S6u1pOWzb84=", + "v1/serviceaccounts": "pbx9ZvyFpBE=", + "v1/services": "0/CO1lhkEBI=", + "autoscaling/v1/horizontalpodautoscalers": "oQlkt7f5j/A=", + "autoscaling/v2beta1/horizontalpodautoscalers": "oQlkt7f5j/A=", + "autoscaling/v2beta2/horizontalpodautoscalers": "oQlkt7f5j/A=", + "batch/v1/jobs": "mudhfqk/qZY=", + "batch/v1beta1/cronjobs": "h/JlFAZkyyY=", + "certificates.k8s.io/v1beta1/certificatesigningrequests": "UQh3YTCDIf0=", + "coordination.k8s.io/v1beta1/leases": "/sY7hl8ol1U=", + "coordination.k8s.io/v1/leases": "/sY7hl8ol1U=", + "extensions/v1beta1/daemonsets": "dd7pWHUlMKQ=", + "extensions/v1beta1/deployments": "8aSe+NMegvE=", + "extensions/v1beta1/ingresses": "Ejja63IbU0E=", + "extensions/v1beta1/networkpolicies": "YpfwF18m1G8=", + "extensions/v1beta1/podsecuritypolicies": "khBLobUXkqA=", + "extensions/v1beta1/replicasets": "P1RzHs8/mWQ=", + "networking.k8s.io/v1/networkpolicies": "YpfwF18m1G8=", + "networking.k8s.io/v1beta1/ingresses": "Ejja63IbU0E=", + "node.k8s.io/v1beta1/runtimeclasses": "8nMHWqj34s0=", + "policy/v1beta1/poddisruptionbudgets": "6BGBu0kpHtk=", + "policy/v1beta1/podsecuritypolicies": "khBLobUXkqA=", + "rbac.authorization.k8s.io/v1/clusterrolebindings": "48tpQ8gZHFc=", + "rbac.authorization.k8s.io/v1/clusterroles": "bYE5ZWDrJ44=", + "rbac.authorization.k8s.io/v1/rolebindings": "eGsCzGH6b1g=", + "rbac.authorization.k8s.io/v1/roles": "7FuwZcIIItM=", + "rbac.authorization.k8s.io/v1beta1/clusterrolebindings": "48tpQ8gZHFc=", + "rbac.authorization.k8s.io/v1beta1/clusterroles": "bYE5ZWDrJ44=", + "rbac.authorization.k8s.io/v1beta1/rolebindings": "eGsCzGH6b1g=", + "rbac.authorization.k8s.io/v1beta1/roles": "7FuwZcIIItM=", + "scheduling.k8s.io/v1beta1/priorityclasses": "D3vHs+OgrtA=", + "scheduling.k8s.io/v1/priorityclasses": "D3vHs+OgrtA=", + "storage.k8s.io/v1/storageclasses": "K+m6uJwbjGY=", + "storage.k8s.io/v1/volumeattachments": "vQAqD28V4AY=", + "storage.k8s.io/v1beta1/csidrivers": "hL6j/rwBV5w=", + "storage.k8s.io/v1beta1/csinodes": "Pe62DkZtjuo=", + "storage.k8s.io/v1beta1/storageclasses": "K+m6uJwbjGY=", + "storage.k8s.io/v1beta1/volumeattachments": "vQAqD28V4AY=", + "apps/v1/controllerrevisions": "85nkx63pcBU=", + "apps/v1/daemonsets": "dd7pWHUlMKQ=", + "apps/v1/deployments": "8aSe+NMegvE=", + "apps/v1/replicasets": "P1RzHs8/mWQ=", + "apps/v1/statefulsets": "H+vl74LkKdo=", + "apps/v1beta2/controllerrevisions": "85nkx63pcBU=", + "apps/v1beta2/daemonsets": "dd7pWHUlMKQ=", + "apps/v1beta2/deployments": "8aSe+NMegvE=", + "apps/v1beta2/replicasets": "P1RzHs8/mWQ=", + "apps/v1beta2/statefulsets": "H+vl74LkKdo=", + "apps/v1beta1/controllerrevisions": "85nkx63pcBU=", + "apps/v1beta1/deployments": "8aSe+NMegvE=", + "apps/v1beta1/statefulsets": "H+vl74LkKdo=", + "admissionregistration.k8s.io/v1beta1/mutatingwebhookconfigurations": "yxW1cpLtfp8=", + "admissionregistration.k8s.io/v1beta1/validatingwebhookconfigurations": "P9NhrezfnWE=", + "events.k8s.io/v1beta1/events": "r2yiGXH7wu8=", +} diff --git a/pkg/registry/core/namespace/storage/storage.go b/pkg/registry/core/namespace/storage/storage.go index c8f0c278cb..2fe25f1ee5 100644 --- a/pkg/registry/core/namespace/storage/storage.go +++ b/pkg/registry/core/namespace/storage/storage.go @@ -226,6 +226,12 @@ func (r *REST) ShortNames() []string { return []string{"ns"} } +var _ rest.StorageVersionProvider = &REST{} + +func (r *REST) StorageVersion() runtime.GroupVersioner { + return r.store.StorageVersion() +} + func (r *StatusREST) New() runtime.Object { return r.store.New() } diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go index fe337d247f..4ed99d1791 100644 --- a/pkg/registry/core/service/storage/rest.go +++ b/pkg/registry/core/service/storage/rest.go @@ -77,6 +77,7 @@ type ServiceStorage interface { rest.Watcher rest.TableConvertor rest.Exporter + rest.StorageVersionProvider } type EndpointsStorage interface { @@ -108,11 +109,16 @@ func NewREST( } var ( - _ ServiceStorage = &REST{} - _ rest.CategoriesProvider = &REST{} - _ rest.ShortNamesProvider = &REST{} + _ ServiceStorage = &REST{} + _ rest.CategoriesProvider = &REST{} + _ rest.ShortNamesProvider = &REST{} + _ rest.StorageVersionProvider = &REST{} ) +func (rs *REST) StorageVersion() runtime.GroupVersioner { + return rs.services.StorageVersion() +} + // ShortNames implements the ShortNamesProvider interface. Returns a list of short names for a resource. func (rs *REST) ShortNames() []string { return []string{"svc"} diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go index e7f4560b5f..b226a87eac 100644 --- a/pkg/registry/core/service/storage/rest_test.go +++ b/pkg/registry/core/service/storage/rest_test.go @@ -159,6 +159,10 @@ func (s *serviceStorage) Export(ctx context.Context, name string, opts metav1.Ex panic("not implemented") } +func (s *serviceStorage) StorageVersion() runtime.GroupVersioner { + panic("not implemented") +} + func generateRandomNodePort() int32 { return int32(rand.IntnRange(30001, 30999)) } diff --git a/pkg/registry/rbac/clusterrole/policybased/storage.go b/pkg/registry/rbac/clusterrole/policybased/storage.go index 4505d9aba6..e3a9dd6ff2 100644 --- a/pkg/registry/rbac/clusterrole/policybased/storage.go +++ b/pkg/registry/rbac/clusterrole/policybased/storage.go @@ -50,6 +50,16 @@ func (r *Storage) NamespaceScoped() bool { return false } +func (r *Storage) StorageVersion() runtime.GroupVersioner { + svp, ok := r.StandardStorage.(rest.StorageVersionProvider) + if !ok { + return nil + } + return svp.StorageVersion() +} + +var _ rest.StorageVersionProvider = &Storage{} + var fullAuthority = []rbac.PolicyRule{ rbac.NewRule("*").Groups("*").Resources("*").RuleOrDie(), rbac.NewRule("*").URLs("*").RuleOrDie(), diff --git a/pkg/registry/rbac/clusterrolebinding/policybased/storage.go b/pkg/registry/rbac/clusterrolebinding/policybased/storage.go index 2267394aaa..6a4b2c8f2e 100644 --- a/pkg/registry/rbac/clusterrolebinding/policybased/storage.go +++ b/pkg/registry/rbac/clusterrolebinding/policybased/storage.go @@ -51,6 +51,16 @@ func (r *Storage) NamespaceScoped() bool { return false } +func (r *Storage) StorageVersion() runtime.GroupVersioner { + svp, ok := r.StandardStorage.(rest.StorageVersionProvider) + if !ok { + return nil + } + return svp.StorageVersion() +} + +var _ rest.StorageVersionProvider = &Storage{} + func (s *Storage) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) { if rbacregistry.EscalationAllowed(ctx) { return s.StandardStorage.Create(ctx, obj, createValidation, options) diff --git a/pkg/registry/rbac/role/policybased/storage.go b/pkg/registry/rbac/role/policybased/storage.go index 25a622fe07..34281be975 100644 --- a/pkg/registry/rbac/role/policybased/storage.go +++ b/pkg/registry/rbac/role/policybased/storage.go @@ -49,6 +49,16 @@ func (r *Storage) NamespaceScoped() bool { return true } +func (r *Storage) StorageVersion() runtime.GroupVersioner { + svp, ok := r.StandardStorage.(rest.StorageVersionProvider) + if !ok { + return nil + } + return svp.StorageVersion() +} + +var _ rest.StorageVersionProvider = &Storage{} + func (s *Storage) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) { if rbacregistry.EscalationAllowed(ctx) || rbacregistry.RoleEscalationAuthorized(ctx, s.authorizer) { return s.StandardStorage.Create(ctx, obj, createValidation, options) diff --git a/pkg/registry/rbac/rolebinding/policybased/storage.go b/pkg/registry/rbac/rolebinding/policybased/storage.go index 2a0603e740..d73a1e1f93 100644 --- a/pkg/registry/rbac/rolebinding/policybased/storage.go +++ b/pkg/registry/rbac/rolebinding/policybased/storage.go @@ -52,6 +52,16 @@ func (r *Storage) NamespaceScoped() bool { return true } +func (r *Storage) StorageVersion() runtime.GroupVersioner { + svp, ok := r.StandardStorage.(rest.StorageVersionProvider) + if !ok { + return nil + } + return svp.StorageVersion() +} + +var _ rest.StorageVersionProvider = &Storage{} + func (s *Storage) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) { if rbacregistry.EscalationAllowed(ctx) { return s.StandardStorage.Create(ctx, obj, createValidation, options) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go index 86203b2ba0..f1592444a4 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go @@ -95,6 +95,7 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error { } foundThisVersion := false + var storageVersionHash string for _, v := range crd.Spec.Versions { if !v.Served { continue @@ -113,6 +114,9 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error { if v.Name == version.Version { foundThisVersion = true } + if v.Storage { + storageVersionHash = discovery.StorageVersionHash(gv.Group, gv.Version, crd.Spec.Names.Kind) + } } if !foundThisVersion { @@ -127,13 +131,14 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error { } apiResourcesForDiscovery = append(apiResourcesForDiscovery, metav1.APIResource{ - Name: crd.Status.AcceptedNames.Plural, - SingularName: crd.Status.AcceptedNames.Singular, - Namespaced: crd.Spec.Scope == apiextensions.NamespaceScoped, - Kind: crd.Status.AcceptedNames.Kind, - Verbs: verbs, - ShortNames: crd.Status.AcceptedNames.ShortNames, - Categories: crd.Status.AcceptedNames.Categories, + Name: crd.Status.AcceptedNames.Plural, + SingularName: crd.Status.AcceptedNames.Singular, + Namespaced: crd.Spec.Scope == apiextensions.NamespaceScoped, + Kind: crd.Status.AcceptedNames.Kind, + Verbs: verbs, + ShortNames: crd.Status.AcceptedNames.ShortNames, + Categories: crd.Status.AcceptedNames.Categories, + StorageVersionHash: storageVersionHash, }) subresources, err := apiextensions.GetSubresourcesForVersion(crd, version.Version) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/BUILD index f6395aeca8..d80a60ea96 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/BUILD @@ -32,6 +32,7 @@ go_library( "group.go", "legacy.go", "root.go", + "storageversionhash.go", "util.go", "version.go", ], diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/storageversionhash.go b/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/storageversionhash.go new file mode 100644 index 0000000000..a1b00decba --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/storageversionhash.go @@ -0,0 +1,40 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package discovery + +import ( + "crypto/sha256" + "encoding/base64" +) + +// StorageVersionHash calculates the storage version hash for a +// tuple. +// WARNING: this function is subject to change. Clients shouldn't depend on +// this function. +func StorageVersionHash(group, version, kind string) string { + gvk := group + "/" + version + "/" + kind + if gvk == "" { + return "" + } + bytes := sha256.Sum256([]byte(gvk)) + // Assuming there are N kinds in the cluster, and the hash is X-byte long, + // the chance of colliding hash P(N,X) approximates to 1-e^(-(N^2)/2^(8X+1)). + // P(10,000, 8) ~= 2.7*10^(-12), which is low enough. + // See https://en.wikipedia.org/wiki/Birthday_problem#Approximations. + return base64.StdEncoding.EncodeToString( + bytes[:8]) +} diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go index 8440624329..c48c2d61f6 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go @@ -33,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/admission" + "k8s.io/apiserver/pkg/endpoints/discovery" "k8s.io/apiserver/pkg/endpoints/handlers" "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager" "k8s.io/apiserver/pkg/endpoints/handlers/negotiation" @@ -133,6 +134,20 @@ func (a *APIInstaller) newWebService() *restful.WebService { return ws } +// calculate the storage gvk, the gvk objects are converted to before persisted to the etcd. +func getStorageVersionKind(storageVersioner runtime.GroupVersioner, storage rest.Storage, typer runtime.ObjectTyper) (schema.GroupVersionKind, error) { + object := storage.New() + fqKinds, _, err := typer.ObjectKinds(object) + if err != nil { + return schema.GroupVersionKind{}, err + } + gvk, ok := storageVersioner.KindForGroupVersionKinds(fqKinds) + if !ok { + return schema.GroupVersionKind{}, fmt.Errorf("cannot find the storage version kind for %v", reflect.TypeOf(object)) + } + return gvk, nil +} + // GetResourceKind returns the external group version kind registered for the given storage // object. If the storage object is a subresource and has an override supplied for it, it returns // the group version kind supplied in the override. @@ -227,6 +242,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag watcher, isWatcher := storage.(rest.Watcher) connecter, isConnecter := storage.(rest.Connecter) storageMeta, isMetadata := storage.(rest.StorageMetadata) + storageVersionProvider, isStorageVersionProvider := storage.(rest.StorageVersionProvider) if !isMetadata { storageMeta = defaultStorageMetadata{} } @@ -365,6 +381,17 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag tableProvider, _ := storage.(rest.TableConvertor) var apiResource metav1.APIResource + if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionHash) && + isStorageVersionProvider && + storageVersionProvider.StorageVersion() != nil { + versioner := storageVersionProvider.StorageVersion() + gvk, err := getStorageVersionKind(versioner, storage, a.group.Typer) + if err != nil { + return nil, err + } + apiResource.StorageVersionHash = discovery.StorageVersionHash(gvk.Group, gvk.Version, gvk.Kind) + } + // Get the list of actions for the given scope. switch { case !namespaceScoped: diff --git a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go index e99045896f..587a6dfe54 100644 --- a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go @@ -89,6 +89,13 @@ const ( // Server-side apply. Merging happens on the server. ServerSideApply utilfeature.Feature = "ServerSideApply" + // owner: @caesarxuchao + // alpha: v1.14 + // + // Allow apiservers to expose the storage version hash in the discovery + // document. + StorageVersionHash utilfeature.Feature = "StorageVersionHash" + // owner: @ksubrmnn // alpha: v1.14 // @@ -118,6 +125,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS APIListChunking: {Default: true, PreRelease: utilfeature.Beta}, DryRun: {Default: true, PreRelease: utilfeature.Beta}, ServerSideApply: {Default: false, PreRelease: utilfeature.Alpha}, + StorageVersionHash: {Default: false, PreRelease: utilfeature.Alpha}, WinOverlay: {Default: false, PreRelease: utilfeature.Alpha}, WinDSR: {Default: false, PreRelease: utilfeature.Alpha}, } diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go index 5489596e63..c84a3276c4 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go @@ -177,6 +177,12 @@ type Store struct { // resource. It is wrapped into a "DryRunnableStorage" that will // either pass-through or simply dry-run. Storage DryRunnableStorage + // StorageVersioner outputs the an object will be + // converted to before persisted in etcd, given a list of possible + // kinds of the object. + // If the StorageVersioner is nil, apiserver will leave the + // storageVersionHash as empty in the discovery document. + StorageVersioner runtime.GroupVersioner // Called to cleanup clients used by the underlying Storage; optional. DestroyFunc func() } @@ -1287,6 +1293,7 @@ func (e *Store) CompleteWithOptions(options *generic.StoreOptions) error { attrFunc, triggerFunc, ) + e.StorageVersioner = opts.StorageConfig.EncodeVersioner if opts.CountMetricPollPeriod > 0 { stopFunc := e.startObservingCount(opts.CountMetricPollPeriod) @@ -1327,3 +1334,7 @@ func (e *Store) ConvertToTable(ctx context.Context, object runtime.Object, table } return rest.NewDefaultTableConvertor(e.qualifiedResourceFromContext(ctx)).ConvertToTable(ctx, object, tableOptions) } + +func (e *Store) StorageVersion() runtime.GroupVersioner { + return e.StorageVersioner +} diff --git a/staging/src/k8s.io/apiserver/pkg/registry/rest/rest.go b/staging/src/k8s.io/apiserver/pkg/registry/rest/rest.go index 9fad541348..b16f7f677b 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/rest/rest.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/rest/rest.go @@ -332,3 +332,12 @@ type StorageMetadata interface { // it is not nil. Only the type of the return object matters, the value will be ignored. ProducesObject(verb string) interface{} } + +// StorageVersionProvider is an optional interface that a storage object can +// implement if it wishes to disclose its storage version. +type StorageVersionProvider interface { + // StorageVersion returns a group versioner, which will outputs the gvk + // an object will be converted to before persisted in etcd, given a + // list of kinds the object might belong to. + StorageVersion() runtime.GroupVersioner +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_codec.go b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_codec.go index e2f91bf13d..96faa17122 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_codec.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_codec.go @@ -40,15 +40,15 @@ type StorageCodecConfig struct { // NewStorageCodec assembles a storage codec for the provided storage media type, the provided serializer, and the requested // storage and memory versions. -func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, error) { +func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, runtime.GroupVersioner, error) { mediaType, _, err := mime.ParseMediaType(opts.StorageMediaType) if err != nil { - return nil, fmt.Errorf("%q is not a valid mime-type", opts.StorageMediaType) + return nil, nil, fmt.Errorf("%q is not a valid mime-type", opts.StorageMediaType) } serializer, ok := runtime.SerializerInfoForMediaType(opts.StorageSerializer.SupportedMediaTypes(), mediaType) if !ok { - return nil, fmt.Errorf("unable to find serializer for %q", mediaType) + return nil, nil, fmt.Errorf("unable to find serializer for %q", mediaType) } s := serializer.Serializer @@ -74,14 +74,16 @@ func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, error) { decoders = opts.DecoderDecoratorFn(decoders) } + encodeVersioner := runtime.NewMultiGroupVersioner( + opts.StorageVersion, + schema.GroupKind{Group: opts.StorageVersion.Group}, + schema.GroupKind{Group: opts.MemoryVersion.Group}, + ) + // Ensure the storage receives the correct version. encoder = opts.StorageSerializer.EncoderForVersion( encoder, - runtime.NewMultiGroupVersioner( - opts.StorageVersion, - schema.GroupKind{Group: opts.StorageVersion.Group}, - schema.GroupKind{Group: opts.MemoryVersion.Group}, - ), + encodeVersioner, ) decoder := opts.StorageSerializer.DecoderToVersion( recognizer.NewDecoder(decoders...), @@ -92,5 +94,5 @@ func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, error) { ), ) - return runtime.NewCodec(encoder, decoder), nil + return runtime.NewCodec(encoder, decoder), encodeVersioner, nil } diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go index c3bb6ecd6d..267de1370b 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go @@ -86,7 +86,7 @@ type DefaultStorageFactory struct { APIResourceConfigSource APIResourceConfigSource // newStorageCodecFn exists to be overwritten for unit testing. - newStorageCodecFn func(opts StorageCodecConfig) (codec runtime.Codec, err error) + newStorageCodecFn func(opts StorageCodecConfig) (codec runtime.Codec, encodeVersioner runtime.GroupVersioner, err error) } type groupResourceOverrides struct { @@ -278,7 +278,7 @@ func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (* } codecConfig.Config = storageConfig - storageConfig.Codec, err = s.newStorageCodecFn(codecConfig) + storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go index c36a103942..d4bc7fb49d 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go @@ -57,6 +57,11 @@ type Config struct { Paging bool Codec runtime.Codec + // EncodeVersioner is the same groupVersioner used to build the + // storage encoder. Given a list of kinds the input object might belong + // to, the EncodeVersioner outputs the gvk the object will be + // converted to before persisted in etcd. + EncodeVersioner runtime.GroupVersioner // Transformer allows the value to be transformed prior to persisting into etcd. Transformer value.Transformer diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/BUILD b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/BUILD index 9f704e55c1..0a71e2a768 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/BUILD +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/BUILD @@ -11,10 +11,13 @@ go_library( importmap = "k8s.io/kubernetes/vendor/k8s.io/sample-apiserver/pkg/cmd/server", importpath = "k8s.io/sample-apiserver/pkg/cmd/server", deps = [ + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/sample-apiserver/pkg/admission/plugin/banflunder:go_default_library", "//staging/src/k8s.io/sample-apiserver/pkg/admission/wardleinitializer:go_default_library", "//staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1:go_default_library", diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go index 48aa538ba0..66409efad6 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go @@ -23,10 +23,13 @@ import ( "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apiserver/pkg/admission" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/sample-apiserver/pkg/admission/plugin/banflunder" "k8s.io/sample-apiserver/pkg/admission/wardleinitializer" "k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1" @@ -56,7 +59,7 @@ func NewWardleServerOptions(out, errOut io.Writer) *WardleServerOptions { StdOut: out, StdErr: errOut, } - + o.RecommendedOptions.Etcd.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1alpha1.SchemeGroupVersion, schema.GroupKind{Group: v1alpha1.GroupName}) return o } @@ -83,6 +86,7 @@ func NewCommandStartWardleServer(defaults *WardleServerOptions, stopCh <-chan st flags := cmd.Flags() o.RecommendedOptions.AddFlags(flags) + utilfeature.DefaultMutableFeatureGate.AddFlag(flags) return cmd } diff --git a/test/e2e/apimachinery/BUILD b/test/e2e/apimachinery/BUILD index 136dd18f70..e32bde872d 100644 --- a/test/e2e/apimachinery/BUILD +++ b/test/e2e/apimachinery/BUILD @@ -15,6 +15,7 @@ go_library( "crd_publish_openapi.go", "crd_watch.go", "custom_resource_definition.go", + "discovery.go", "etcd_failure.go", "framework.go", "garbage_collector.go", @@ -65,6 +66,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/endpoints/discovery:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", "//staging/src/k8s.io/client-go/discovery:go_default_library", diff --git a/test/e2e/apimachinery/discovery.go b/test/e2e/apimachinery/discovery.go new file mode 100644 index 0000000000..b2e42db58e --- /dev/null +++ b/test/e2e/apimachinery/discovery.go @@ -0,0 +1,78 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apimachinery + +import ( + utilversion "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apiserver/pkg/endpoints/discovery" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" +) + +var storageVersionServerVersion = utilversion.MustParseSemantic("v1.13.99") +var _ = SIGDescribe("Discovery", func() { + f := framework.NewDefaultFramework("discovery") + + var namespaceName string + + BeforeEach(func() { + namespaceName = f.Namespace.Name + + framework.SkipUnlessServerVersionGTE(storageVersionServerVersion, f.ClientSet.Discovery()) + + By("Setting up server cert") + setupServerCert(namespaceName, serviceName) + }) + + It("[Feature:StorageVersionHash] Custom resource should have storage version hash", func() { + testcrd, err := framework.CreateTestCRD(f) + if err != nil { + return + } + defer testcrd.CleanUp() + spec := testcrd.Crd.Spec + resources, err := testcrd.ApiExtensionClient.Discovery().ServerResourcesForGroupVersion(spec.Group + "/" + spec.Versions[0].Name) + if err != nil { + framework.Failf("failed to find the discovery doc for %v: %v", resources, err) + } + found := false + var storageVersion string + for _, v := range spec.Versions { + if v.Storage { + storageVersion = v.Name + } + } + // DISCLAIMER: the algorithm of deriving the storageVersionHash + // is an implementation detail, which shouldn't be relied on by + // the clients. The following calculation is for test purpose + // only. + expected := discovery.StorageVersionHash(spec.Group, storageVersion, spec.Names.Kind) + + for _, r := range resources.APIResources { + if r.Name == spec.Names.Plural { + found = true + if r.StorageVersionHash != expected { + framework.Failf("expected storageVersionHash of %s/%s/%s to be %s, got %s", r.Group, r.Version, r.Name, expected, r.StorageVersionHash) + } + } + } + if !found { + framework.Failf("didn't find resource %s in the discovery doc", spec.Names.Plural) + } + }) +})