github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Expose storage version hash

pull/564/head
Chao Xu 2019-01-14 19:31:25 -08:00
parent 4ea48886df
commit 3b618af0d4
23 changed files with 510 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/kube-apiserver/app/aggregator.go
View File

@ -30,6 +30,7 @@ import (
apiextensionsinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/admission"
@ -79,6 +80,7 @@ func createAggregatorConfig(
etcdOptions := *commandOptions.Etcd
etcdOptions.StorageConfig.Paging = utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking)
etcdOptions.StorageConfig.Codec = aggregatorscheme.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion)
etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1beta1.SchemeGroupVersion, schema.GroupKind{Group: v1beta1.GroupName})
genericConfig.RESTOptionsGetter = &genericoptions.SimpleRestOptionsFactory{Options: etcdOptions}
// override MergedResourceConfig with aggregator defaults and registry

3
cmd/kube-apiserver/app/apiextensions.go
View File

@ -23,6 +23,8 @@ import (
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
apiextensionsoptions "k8s.io/apiextensions-apiserver/pkg/cmd/server/options"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/features"
genericapiserver "k8s.io/apiserver/pkg/server"
@ -61,6 +63,7 @@ func createAPIExtensionsConfig(
etcdOptions := *commandOptions.Etcd
etcdOptions.StorageConfig.Paging = utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking)
etcdOptions.StorageConfig.Codec = apiextensionsapiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion)
etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1beta1.SchemeGroupVersion, schema.GroupKind{Group: v1beta1.GroupName})
genericConfig.RESTOptionsGetter = &genericoptions.SimpleRestOptionsFactory{Options: etcdOptions}
// override MergedResourceConfig with apiextensions defaults and registry

126
pkg/master/master_test.go
View File

@ -31,22 +31,32 @@ import (
certificatesapiv1beta1 "k8s.io/api/certificates/v1beta1"
apiv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/version"
"k8s.io/apiserver/pkg/authorization/authorizerfactory"
"k8s.io/apiserver/pkg/features"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/server/resourceconfig"
serverstorage "k8s.io/apiserver/pkg/server/storage"
etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
"k8s.io/client-go/discovery"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/fake"
restclient "k8s.io/client-go/rest"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/api/testapi"
"k8s.io/kubernetes/pkg/apis/batch"
api "k8s.io/kubernetes/pkg/apis/core"
apisstorage "k8s.io/kubernetes/pkg/apis/storage"
kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
"k8s.io/kubernetes/pkg/master/reconcilers"
"k8s.io/kubernetes/pkg/master/storageversionhashdata"
certificatesrest "k8s.io/kubernetes/pkg/registry/certificates/rest"
corerest "k8s.io/kubernetes/pkg/registry/core/rest"
"k8s.io/kubernetes/pkg/registry/registrytest"
@ -70,6 +80,14 @@ func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, *assert.Assertion
}
resourceEncoding := serverstorage.NewDefaultResourceEncodingConfig(legacyscheme.Scheme)
// This configures the testing master the same way the real master is
// configured. The storage versions of these resources are different
// from the storage versions of other resources in their group.
resourceEncodingOverrides := []schema.GroupVersionResource{
batch.Resource("cronjobs").WithVersion("v1beta1"),
apisstorage.Resource("volumeattachments").WithVersion("v1beta1"),
}
resourceEncoding = resourceconfig.MergeResourceEncodingConfigs(resourceEncoding, resourceEncodingOverrides)
storageFactory := serverstorage.NewDefaultStorageFactory(*storageConfig, testapi.StorageMediaType(), legacyscheme.Codecs, resourceEncoding, DefaultAPIResourceConfigSource(), nil)
etcdOptions := options.NewEtcdOptions(storageConfig)
@ -81,12 +99,12 @@ func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, *assert.Assertion
}
kubeVersion := kubeversion.Get()
config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
config.GenericConfig.Version = &kubeVersion
config.ExtraConfig.StorageFactory = storageFactory
config.GenericConfig.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}}
config.GenericConfig.PublicAddress = net.ParseIP("192.168.10.4")
config.GenericConfig.LegacyAPIGroupPrefixes = sets.NewString("/api")
config.GenericConfig.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}}
config.ExtraConfig.KubeletClientConfig = kubeletclient.KubeletClientConfig{Port: 10250}
config.ExtraConfig.ProxyTransport = utilnet.SetTransportDefaults(&http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { return nil, nil },
@ -363,6 +381,112 @@ func TestAPIVersionOfDiscoveryEndpoints(t *testing.T) {
}
// This test doesn't cover the apiregistration and apiextensions group, as they are installed by other apiservers.
func TestStorageVersionHashes(t *testing.T) {
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionHash, true)()
master, etcdserver, _, _ := newMaster(t)
defer etcdserver.Terminate(t)
server := httptest.NewServer(master.GenericAPIServer.Handler.GoRestfulContainer.ServeMux)
c := &restclient.Config{
Host: server.URL,
APIPath: "/api",
ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs},
}
discover := discovery.NewDiscoveryClientForConfigOrDie(c)
all, err := discover.ServerResources()
if err != nil {
t.Error(err)
}
var count int
for _, g := range all {
for _, r := range g.APIResources {
if strings.Contains(r.Name, "/") ||
storageversionhashdata.NoStorageVersionHash.Has(g.GroupVersion+"/"+r.Name) {
if r.StorageVersionHash != "" {
t.Errorf("expect resource %s/%s to have empty storageVersionHash, got hash %q", g.GroupVersion, r.Name, r.StorageVersionHash)
}
continue
}
if r.StorageVersionHash == "" {
t.Errorf("expect the storageVersionHash of %s/%s to exist", g.GroupVersion, r.Name)
continue
}
// Uncomment the following line if you want to update storageversionhash/data.go
// fmt.Printf("\"%s/%s\": \"%s\",\n", g.GroupVersion, r.Name, r.StorageVersionHash)
expected := storageversionhashdata.GVRToStorageVersionHash[g.GroupVersion+"/"+r.Name]
if r.StorageVersionHash != expected {
t.Errorf("expect the storageVersionHash of %s/%s to be %q, got %q", g.GroupVersion, r.Name, expected, r.StorageVersionHash)
}
count++
}
}
if count != len(storageversionhashdata.GVRToStorageVersionHash) {
t.Errorf("please remove the redundant entries from GVRToStorageVersionHash")
}
}
func TestStorageVersionHashEqualities(t *testing.T) {
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionHash, true)()
master, etcdserver, _, assert := newMaster(t)
defer etcdserver.Terminate(t)
server := httptest.NewServer(master.GenericAPIServer.Handler.GoRestfulContainer.ServeMux)
// Test 1: extensions/v1beta1/replicasets and apps/v1/replicasets have
// the same storage version hash.
resp, err := http.Get(server.URL + "/apis/extensions/v1beta1")
assert.Empty(err)
extList := metav1.APIResourceList{}
assert.NoError(decodeResponse(resp, &extList))
var extReplicasetHash, appsReplicasetHash string
for _, r := range extList.APIResources {
if r.Name == "replicasets" {
extReplicasetHash = r.StorageVersionHash
}
}
assert.NotEmpty(extReplicasetHash)
resp, err = http.Get(server.URL + "/apis/apps/v1")
assert.Empty(err)
appsList := metav1.APIResourceList{}
assert.NoError(decodeResponse(resp, &appsList))
for _, r := range appsList.APIResources {
if r.Name == "replicasets" {
appsReplicasetHash = r.StorageVersionHash
}
}
assert.Equal(extReplicasetHash, appsReplicasetHash)
// Test 2: batch/v1/jobs and batch/v1beta1/cronjobs have different
// storage version hashes.
resp, err = http.Get(server.URL + "/apis/batch/v1")
assert.Empty(err)
batchv1 := metav1.APIResourceList{}
assert.NoError(decodeResponse(resp, &batchv1))
var jobsHash string
for _, r := range batchv1.APIResources {
if r.Name == "jobs" {
jobsHash = r.StorageVersionHash
}
}
assert.NotEmpty(jobsHash)
resp, err = http.Get(server.URL + "/apis/batch/v1beta1")
assert.Empty(err)
batchv1beta1 := metav1.APIResourceList{}
assert.NoError(decodeResponse(resp, &batchv1beta1))
var cronjobsHash string
for _, r := range batchv1beta1.APIResources {
if r.Name == "cronjobs" {
cronjobsHash = r.StorageVersionHash
}
}
assert.NotEmpty(cronjobsHash)
assert.NotEqual(jobsHash, cronjobsHash)
}
func TestNoAlphaVersionsEnabledByDefault(t *testing.T) {
config := DefaultAPIResourceConfigSource()
for gv, enable := range config.GroupVersionConfigs {

4
pkg/master/storageversionhashdata/OWNERS Normal file
View File

@ -0,0 +1,4 @@
approvers:
- api-approvers
reviewers:
- api-reviewers

111
pkg/master/storageversionhashdata/data.go Normal file
View File

@ -0,0 +1,111 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package storageversionhashdata is for test only.
package storageversionhashdata
import (
"k8s.io/apimachinery/pkg/util/sets"
)
// NoStorageVersionHash lists resources that legitimately with empty storage
// version hash.
var NoStorageVersionHash = sets.NewString(
"v1/bindings",
"v1/componentstatuses",
"authentication.k8s.io/v1/tokenreviews",
"authorization.k8s.io/v1/localsubjectaccessreviews",
"authorization.k8s.io/v1/selfsubjectaccessreviews",
"authorization.k8s.io/v1/selfsubjectrulesreviews",
"authorization.k8s.io/v1/subjectaccessreviews",
"authentication.k8s.io/v1beta1/tokenreviews",
"authorization.k8s.io/v1beta1/localsubjectaccessreviews",
"authorization.k8s.io/v1beta1/selfsubjectaccessreviews",
"authorization.k8s.io/v1beta1/selfsubjectrulesreviews",
"authorization.k8s.io/v1beta1/subjectaccessreviews",
"extensions/v1beta1/replicationcontrollers",
)
// GVRToStorageVersionHash shouldn't change unless we intentionally change the
// storage version of a resource.
var GVRToStorageVersionHash = map[string]string{
"v1/configmaps": "qFsyl6wFWjQ=",
"v1/endpoints": "fWeeMqaN/OA=",
"v1/events": "r2yiGXH7wu8=",
"v1/limitranges": "EBKMFVe6cwo=",
"v1/namespaces": "Q3oi5N2YM8M=",
"v1/nodes": "XwShjMxG9Fs=",
"v1/persistentvolumeclaims": "QWTyNDq0dC4=",
"v1/persistentvolumes": "HN/zwEC+JgM=",
"v1/pods": "xPOwRZ+Yhw8=",
"v1/podtemplates": "LIXB2x4IFpk=",
"v1/replicationcontrollers": "Jond2If31h0=",
"v1/resourcequotas": "8uhSgffRX6w=",
"v1/secrets": "S6u1pOWzb84=",
"v1/serviceaccounts": "pbx9ZvyFpBE=",
"v1/services": "0/CO1lhkEBI=",
"autoscaling/v1/horizontalpodautoscalers": "oQlkt7f5j/A=",
"autoscaling/v2beta1/horizontalpodautoscalers": "oQlkt7f5j/A=",
"autoscaling/v2beta2/horizontalpodautoscalers": "oQlkt7f5j/A=",
"batch/v1/jobs": "mudhfqk/qZY=",
"batch/v1beta1/cronjobs": "h/JlFAZkyyY=",
"certificates.k8s.io/v1beta1/certificatesigningrequests": "UQh3YTCDIf0=",
"coordination.k8s.io/v1beta1/leases": "/sY7hl8ol1U=",
"coordination.k8s.io/v1/leases": "/sY7hl8ol1U=",
"extensions/v1beta1/daemonsets": "dd7pWHUlMKQ=",
"extensions/v1beta1/deployments": "8aSe+NMegvE=",
"extensions/v1beta1/ingresses": "Ejja63IbU0E=",
"extensions/v1beta1/networkpolicies": "YpfwF18m1G8=",
"extensions/v1beta1/podsecuritypolicies": "khBLobUXkqA=",
"extensions/v1beta1/replicasets": "P1RzHs8/mWQ=",
"networking.k8s.io/v1/networkpolicies": "YpfwF18m1G8=",
"networking.k8s.io/v1beta1/ingresses": "Ejja63IbU0E=",
"node.k8s.io/v1beta1/runtimeclasses": "8nMHWqj34s0=",
"policy/v1beta1/poddisruptionbudgets": "6BGBu0kpHtk=",
"policy/v1beta1/podsecuritypolicies": "khBLobUXkqA=",
"rbac.authorization.k8s.io/v1/clusterrolebindings": "48tpQ8gZHFc=",
"rbac.authorization.k8s.io/v1/clusterroles": "bYE5ZWDrJ44=",
"rbac.authorization.k8s.io/v1/rolebindings": "eGsCzGH6b1g=",
"rbac.authorization.k8s.io/v1/roles": "7FuwZcIIItM=",
"rbac.authorization.k8s.io/v1beta1/clusterrolebindings": "48tpQ8gZHFc=",
"rbac.authorization.k8s.io/v1beta1/clusterroles": "bYE5ZWDrJ44=",
"rbac.authorization.k8s.io/v1beta1/rolebindings": "eGsCzGH6b1g=",
"rbac.authorization.k8s.io/v1beta1/roles": "7FuwZcIIItM=",
"scheduling.k8s.io/v1beta1/priorityclasses": "D3vHs+OgrtA=",
"scheduling.k8s.io/v1/priorityclasses": "D3vHs+OgrtA=",
"storage.k8s.io/v1/storageclasses": "K+m6uJwbjGY=",
"storage.k8s.io/v1/volumeattachments": "vQAqD28V4AY=",
"storage.k8s.io/v1beta1/csidrivers": "hL6j/rwBV5w=",
"storage.k8s.io/v1beta1/csinodes": "Pe62DkZtjuo=",
"storage.k8s.io/v1beta1/storageclasses": "K+m6uJwbjGY=",
"storage.k8s.io/v1beta1/volumeattachments": "vQAqD28V4AY=",
"apps/v1/controllerrevisions": "85nkx63pcBU=",
"apps/v1/daemonsets": "dd7pWHUlMKQ=",
"apps/v1/deployments": "8aSe+NMegvE=",
"apps/v1/replicasets": "P1RzHs8/mWQ=",
"apps/v1/statefulsets": "H+vl74LkKdo=",
"apps/v1beta2/controllerrevisions": "85nkx63pcBU=",
"apps/v1beta2/daemonsets": "dd7pWHUlMKQ=",
"apps/v1beta2/deployments": "8aSe+NMegvE=",
"apps/v1beta2/replicasets": "P1RzHs8/mWQ=",
"apps/v1beta2/statefulsets": "H+vl74LkKdo=",
"apps/v1beta1/controllerrevisions": "85nkx63pcBU=",
"apps/v1beta1/deployments": "8aSe+NMegvE=",
"apps/v1beta1/statefulsets": "H+vl74LkKdo=",
"admissionregistration.k8s.io/v1beta1/mutatingwebhookconfigurations": "yxW1cpLtfp8=",
"admissionregistration.k8s.io/v1beta1/validatingwebhookconfigurations": "P9NhrezfnWE=",
"events.k8s.io/v1beta1/events": "r2yiGXH7wu8=",
}

6
pkg/registry/core/namespace/storage/storage.go
View File

@ -226,6 +226,12 @@ func (r *REST) ShortNames() []string {
return []string{"ns"}
}
var _ rest.StorageVersionProvider = &REST{}
func (r *REST) StorageVersion() runtime.GroupVersioner {
return r.store.StorageVersion()
}
func (r *StatusREST) New() runtime.Object {
return r.store.New()
}

12
pkg/registry/core/service/storage/rest.go
View File

@ -77,6 +77,7 @@ type ServiceStorage interface {
rest.Watcher
rest.TableConvertor
rest.Exporter
rest.StorageVersionProvider
}
type EndpointsStorage interface {
@ -108,11 +109,16 @@ func NewREST(
}
var (
_ ServiceStorage = &REST{}
_ rest.CategoriesProvider = &REST{}
_ rest.ShortNamesProvider = &REST{}
_ ServiceStorage = &REST{}
_ rest.CategoriesProvider = &REST{}
_ rest.ShortNamesProvider = &REST{}
_ rest.StorageVersionProvider = &REST{}
)
func (rs *REST) StorageVersion() runtime.GroupVersioner {
return rs.services.StorageVersion()
}
// ShortNames implements the ShortNamesProvider interface. Returns a list of short names for a resource.
func (rs *REST) ShortNames() []string {
return []string{"svc"}

4
pkg/registry/core/service/storage/rest_test.go
View File

@ -159,6 +159,10 @@ func (s *serviceStorage) Export(ctx context.Context, name string, opts metav1.Ex
panic("not implemented")
}
func (s *serviceStorage) StorageVersion() runtime.GroupVersioner {
panic("not implemented")
}
func generateRandomNodePort() int32 {
return int32(rand.IntnRange(30001, 30999))
}

10
pkg/registry/rbac/clusterrole/policybased/storage.go
View File

@ -50,6 +50,16 @@ func (r *Storage) NamespaceScoped() bool {
return false
}
func (r *Storage) StorageVersion() runtime.GroupVersioner {
svp, ok := r.StandardStorage.(rest.StorageVersionProvider)
if !ok {
return nil
}
return svp.StorageVersion()
}
var _ rest.StorageVersionProvider = &Storage{}
var fullAuthority = []rbac.PolicyRule{
rbac.NewRule("*").Groups("*").Resources("*").RuleOrDie(),
rbac.NewRule("*").URLs("*").RuleOrDie(),

10
pkg/registry/rbac/clusterrolebinding/policybased/storage.go
View File

@ -51,6 +51,16 @@ func (r *Storage) NamespaceScoped() bool {
return false
}
func (r *Storage) StorageVersion() runtime.GroupVersioner {
svp, ok := r.StandardStorage.(rest.StorageVersionProvider)
if !ok {
return nil
}
return svp.StorageVersion()
}
var _ rest.StorageVersionProvider = &Storage{}
func (s *Storage) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
if rbacregistry.EscalationAllowed(ctx) {
return s.StandardStorage.Create(ctx, obj, createValidation, options)

10
pkg/registry/rbac/role/policybased/storage.go
View File

@ -49,6 +49,16 @@ func (r *Storage) NamespaceScoped() bool {
return true
}
func (r *Storage) StorageVersion() runtime.GroupVersioner {
svp, ok := r.StandardStorage.(rest.StorageVersionProvider)
if !ok {
return nil
}
return svp.StorageVersion()
}
var _ rest.StorageVersionProvider = &Storage{}
func (s *Storage) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
if rbacregistry.EscalationAllowed(ctx) || rbacregistry.RoleEscalationAuthorized(ctx, s.authorizer) {
return s.StandardStorage.Create(ctx, obj, createValidation, options)

10
pkg/registry/rbac/rolebinding/policybased/storage.go
View File

@ -52,6 +52,16 @@ func (r *Storage) NamespaceScoped() bool {
return true
}
func (r *Storage) StorageVersion() runtime.GroupVersioner {
svp, ok := r.StandardStorage.(rest.StorageVersionProvider)
if !ok {
return nil
}
return svp.StorageVersion()
}
var _ rest.StorageVersionProvider = &Storage{}
func (s *Storage) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
if rbacregistry.EscalationAllowed(ctx) {
return s.StandardStorage.Create(ctx, obj, createValidation, options)

19
staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go
View File

@ -95,6 +95,7 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
}
foundThisVersion := false
var storageVersionHash string
for _, v := range crd.Spec.Versions {
if !v.Served {
continue
@ -113,6 +114,9 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
if v.Name == version.Version {
foundThisVersion = true
}
if v.Storage {
storageVersionHash = discovery.StorageVersionHash(gv.Group, gv.Version, crd.Spec.Names.Kind)
}
}
if !foundThisVersion {
@ -127,13 +131,14 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error {
}
apiResourcesForDiscovery = append(apiResourcesForDiscovery, metav1.APIResource{
Name: crd.Status.AcceptedNames.Plural,
SingularName: crd.Status.AcceptedNames.Singular,
Namespaced: crd.Spec.Scope == apiextensions.NamespaceScoped,
Kind: crd.Status.AcceptedNames.Kind,
Verbs: verbs,
ShortNames: crd.Status.AcceptedNames.ShortNames,
Categories: crd.Status.AcceptedNames.Categories,
Name: crd.Status.AcceptedNames.Plural,
SingularName: crd.Status.AcceptedNames.Singular,
Namespaced: crd.Spec.Scope == apiextensions.NamespaceScoped,
Kind: crd.Status.AcceptedNames.Kind,
Verbs: verbs,
ShortNames: crd.Status.AcceptedNames.ShortNames,
Categories: crd.Status.AcceptedNames.Categories,
StorageVersionHash: storageVersionHash,
})
subresources, err := apiextensions.GetSubresourcesForVersion(crd, version.Version)

40
staging/src/k8s.io/apiserver/pkg/endpoints/discovery/storageversionhash.go Normal file
View File

@ -0,0 +1,40 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package discovery
import (
"crypto/sha256"
"encoding/base64"
)
// StorageVersionHash calculates the storage version hash for a
// <group/version/kind> tuple.
// WARNING: this function is subject to change. Clients shouldn't depend on
// this function.
func StorageVersionHash(group, version, kind string) string {
gvk := group + "/" + version + "/" + kind
if gvk == "" {
return ""
}
bytes := sha256.Sum256([]byte(gvk))
// Assuming there are N kinds in the cluster, and the hash is X-byte long,
// the chance of colliding hash P(N,X) approximates to 1-e^(-(N^2)/2^(8X+1)).
// P(10,000, 8) ~= 2.7*10^(-12), which is low enough.
// See https://en.wikipedia.org/wiki/Birthday_problem#Approximations.
return base64.StdEncoding.EncodeToString(
bytes[:8])
}

27
staging/src/k8s.io/apiserver/pkg/endpoints/installer.go
View File

@ -33,6 +33,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/endpoints/discovery"
"k8s.io/apiserver/pkg/endpoints/handlers"
"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
@ -133,6 +134,20 @@ func (a *APIInstaller) newWebService() *restful.WebService {
return ws
}
// calculate the storage gvk, the gvk objects are converted to before persisted to the etcd.
func getStorageVersionKind(storageVersioner runtime.GroupVersioner, storage rest.Storage, typer runtime.ObjectTyper) (schema.GroupVersionKind, error) {
object := storage.New()
fqKinds, _, err := typer.ObjectKinds(object)
if err != nil {
return schema.GroupVersionKind{}, err
}
gvk, ok := storageVersioner.KindForGroupVersionKinds(fqKinds)
if !ok {
return schema.GroupVersionKind{}, fmt.Errorf("cannot find the storage version kind for %v", reflect.TypeOf(object))
}
return gvk, nil
}
// GetResourceKind returns the external group version kind registered for the given storage
// object. If the storage object is a subresource and has an override supplied for it, it returns
// the group version kind supplied in the override.
@ -227,6 +242,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
watcher, isWatcher := storage.(rest.Watcher)
connecter, isConnecter := storage.(rest.Connecter)
storageMeta, isMetadata := storage.(rest.StorageMetadata)
storageVersionProvider, isStorageVersionProvider := storage.(rest.StorageVersionProvider)
if !isMetadata {
storageMeta = defaultStorageMetadata{}
}
@ -365,6 +381,17 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
tableProvider, _ := storage.(rest.TableConvertor)
var apiResource metav1.APIResource
if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionHash) &&
isStorageVersionProvider &&
storageVersionProvider.StorageVersion() != nil {
versioner := storageVersionProvider.StorageVersion()
gvk, err := getStorageVersionKind(versioner, storage, a.group.Typer)
if err != nil {
return nil, err
}
apiResource.StorageVersionHash = discovery.StorageVersionHash(gvk.Group, gvk.Version, gvk.Kind)
}
// Get the list of actions for the given scope.
switch {
case !namespaceScoped:

8
staging/src/k8s.io/apiserver/pkg/features/kube_features.go
View File

@ -89,6 +89,13 @@ const (
// Server-side apply. Merging happens on the server.
ServerSideApply utilfeature.Feature = "ServerSideApply"
// owner: @caesarxuchao
// alpha: v1.14
//
// Allow apiservers to expose the storage version hash in the discovery
// document.
StorageVersionHash utilfeature.Feature = "StorageVersionHash"
// owner: @ksubrmnn
// alpha: v1.14
//
@ -118,6 +125,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
APIListChunking: {Default: true, PreRelease: utilfeature.Beta},
DryRun: {Default: true, PreRelease: utilfeature.Beta},
ServerSideApply: {Default: false, PreRelease: utilfeature.Alpha},
StorageVersionHash: {Default: false, PreRelease: utilfeature.Alpha},
WinOverlay: {Default: false, PreRelease: utilfeature.Alpha},
WinDSR: {Default: false, PreRelease: utilfeature.Alpha},
}

11
staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go
View File

@ -177,6 +177,12 @@ type Store struct {
// resource. It is wrapped into a "DryRunnableStorage" that will
// either pass-through or simply dry-run.
Storage DryRunnableStorage
// StorageVersioner outputs the <group/version/kind> an object will be
// converted to before persisted in etcd, given a list of possible
// kinds of the object.
// If the StorageVersioner is nil, apiserver will leave the
// storageVersionHash as empty in the discovery document.
StorageVersioner runtime.GroupVersioner
// Called to cleanup clients used by the underlying Storage; optional.
DestroyFunc func()
}
@ -1287,6 +1293,7 @@ func (e *Store) CompleteWithOptions(options *generic.StoreOptions) error {
attrFunc,
triggerFunc,
)
e.StorageVersioner = opts.StorageConfig.EncodeVersioner
if opts.CountMetricPollPeriod > 0 {
stopFunc := e.startObservingCount(opts.CountMetricPollPeriod)
@ -1327,3 +1334,7 @@ func (e *Store) ConvertToTable(ctx context.Context, object runtime.Object, table
}
return rest.NewDefaultTableConvertor(e.qualifiedResourceFromContext(ctx)).ConvertToTable(ctx, object, tableOptions)
}
func (e *Store) StorageVersion() runtime.GroupVersioner {
return e.StorageVersioner
}

9
staging/src/k8s.io/apiserver/pkg/registry/rest/rest.go
View File

@ -332,3 +332,12 @@ type StorageMetadata interface {
// it is not nil. Only the type of the return object matters, the value will be ignored.
ProducesObject(verb string) interface{}
}
// StorageVersionProvider is an optional interface that a storage object can
// implement if it wishes to disclose its storage version.
type StorageVersionProvider interface {
// StorageVersion returns a group versioner, which will outputs the gvk
// an object will be converted to before persisted in etcd, given a
// list of kinds the object might belong to.
StorageVersion() runtime.GroupVersioner
}

20
staging/src/k8s.io/apiserver/pkg/server/storage/storage_codec.go
View File

@ -40,15 +40,15 @@ type StorageCodecConfig struct {
// NewStorageCodec assembles a storage codec for the provided storage media type, the provided serializer, and the requested
// storage and memory versions.
func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, error) {
func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, runtime.GroupVersioner, error) {
mediaType, _, err := mime.ParseMediaType(opts.StorageMediaType)
if err != nil {
return nil, fmt.Errorf("%q is not a valid mime-type", opts.StorageMediaType)
return nil, nil, fmt.Errorf("%q is not a valid mime-type", opts.StorageMediaType)
}
serializer, ok := runtime.SerializerInfoForMediaType(opts.StorageSerializer.SupportedMediaTypes(), mediaType)
if !ok {
return nil, fmt.Errorf("unable to find serializer for %q", mediaType)
return nil, nil, fmt.Errorf("unable to find serializer for %q", mediaType)
}
s := serializer.Serializer
@ -74,14 +74,16 @@ func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, error) {
decoders = opts.DecoderDecoratorFn(decoders)
}
encodeVersioner := runtime.NewMultiGroupVersioner(
opts.StorageVersion,
schema.GroupKind{Group: opts.StorageVersion.Group},
schema.GroupKind{Group: opts.MemoryVersion.Group},
)
// Ensure the storage receives the correct version.
encoder = opts.StorageSerializer.EncoderForVersion(
encoder,
runtime.NewMultiGroupVersioner(
opts.StorageVersion,
schema.GroupKind{Group: opts.StorageVersion.Group},
schema.GroupKind{Group: opts.MemoryVersion.Group},
),
encodeVersioner,
)
decoder := opts.StorageSerializer.DecoderToVersion(
recognizer.NewDecoder(decoders...),
@ -92,5 +94,5 @@ func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, error) {
),
)
return runtime.NewCodec(encoder, decoder), nil
return runtime.NewCodec(encoder, decoder), encodeVersioner, nil
}

4
staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go
View File

@ -86,7 +86,7 @@ type DefaultStorageFactory struct {
APIResourceConfigSource APIResourceConfigSource
// newStorageCodecFn exists to be overwritten for unit testing.
newStorageCodecFn func(opts StorageCodecConfig) (codec runtime.Codec, err error)
newStorageCodecFn func(opts StorageCodecConfig) (codec runtime.Codec, encodeVersioner runtime.GroupVersioner, err error)
}
type groupResourceOverrides struct {
@ -278,7 +278,7 @@ func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*
}
codecConfig.Config = storageConfig
storageConfig.Codec, err = s.newStorageCodecFn(codecConfig)
storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig)
if err != nil {
return nil, err
}

5
staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go
View File

@ -57,6 +57,11 @@ type Config struct {
Paging bool
Codec runtime.Codec
// EncodeVersioner is the same groupVersioner used to build the
// storage encoder. Given a list of kinds the input object might belong
// to, the EncodeVersioner outputs the gvk the object will be
// converted to before persisted in etcd.
EncodeVersioner runtime.GroupVersioner
// Transformer allows the value to be transformed prior to persisting into etcd.
Transformer value.Transformer

4
staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go
View File

@ -23,6 +23,8 @@ import (
"github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apiserver/pkg/admission"
genericapiserver "k8s.io/apiserver/pkg/server"
@ -56,7 +58,7 @@ func NewWardleServerOptions(out, errOut io.Writer) *WardleServerOptions {
StdOut: out,
StdErr: errOut,
}
o.RecommendedOptions.Etcd.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1alpha1.SchemeGroupVersion, schema.GroupKind{Group: v1alpha1.GroupName})
return o
}

78
test/e2e/apimachinery/discovery.go Normal file
View File

@ -0,0 +1,78 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package apimachinery
import (
utilversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/endpoints/discovery"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
)
var storageVersionServerVersion = utilversion.MustParseSemantic("v1.13.99")
var _ = SIGDescribe("Discovery", func() {
f := framework.NewDefaultFramework("discovery")
var namespaceName string
BeforeEach(func() {
namespaceName = f.Namespace.Name
framework.SkipUnlessServerVersionGTE(storageVersionServerVersion, f.ClientSet.Discovery())
By("Setting up server cert")
setupServerCert(namespaceName, serviceName)
})
It("[Feature:StorageVersionHash] Custom resource should have storage version hash", func() {
testcrd, err := framework.CreateTestCRD(f)
if err != nil {
return
}
defer testcrd.CleanUp()
spec := testcrd.Crd.Spec
resources, err := testcrd.ApiExtensionClient.Discovery().ServerResourcesForGroupVersion(spec.Group + "/" + spec.Versions[0].Name)
if err != nil {
framework.Failf("failed to find the discovery doc for %v: %v", resources, err)
}
found := false
var storageVersion string
for _, v := range spec.Versions {
if v.Storage {
storageVersion = v.Name
}
}
// DISCLAIMER: the algorithm of deriving the storageVersionHash
// is an implementation detail, which shouldn't be relied on by
// the clients. The following calculation is for test purpose
// only.
expected := discovery.StorageVersionHash(spec.Group, storageVersion, spec.Names.Kind)
for _, r := range resources.APIResources {
if r.Name == spec.Names.Plural {
found = true
if r.StorageVersionHash != expected {
framework.Failf("expected storageVersionHash of %s/%s/%s to be %s, got %s", r.Group, r.Version, r.Name, expected, r.StorageVersionHash)
}
}
}
if !found {
framework.Failf("didn't find resource %s in the discovery doc", spec.Names.Plural)
}
})
})