From 1fd5bd7de90ae93e0f8b6790afdd470e74b56017 Mon Sep 17 00:00:00 2001 From: Silvan Kaiser Date: Mon, 25 Feb 2019 12:03:34 +0100 Subject: [PATCH 01/16] Adds missing user mapping option for Quobyte mounts --- pkg/volume/quobyte/quobyte.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/volume/quobyte/quobyte.go b/pkg/volume/quobyte/quobyte.go index 0a51568e79..087bdaecbf 100644 --- a/pkg/volume/quobyte/quobyte.go +++ b/pkg/volume/quobyte/quobyte.go @@ -254,6 +254,7 @@ func (mounter *quobyteMounter) SetUpAt(dir string, fsGroup *int64) error { os.MkdirAll(dir, 0750) var options []string + options = append(options, "allow-usermapping-in-volumename") if mounter.readOnly { options = append(options, "ro") } From 6dce4d87a335495b3097dd4bfaa30d0d232fb2ff Mon Sep 17 00:00:00 2001 From: wojtekt Date: Thu, 28 Feb 2019 08:47:29 +0100 Subject: [PATCH 02/16] Fix secret/configmap management for terminated pods --- pkg/kubelet/pod/pod_manager.go | 32 +++++++++++++++---- .../util/manager/cache_based_manager_test.go | 31 ++++++++++++++++++ pkg/kubelet/util/manager/manager.go | 4 +++ 3 files changed, 61 insertions(+), 6 deletions(-) diff --git a/pkg/kubelet/pod/pod_manager.go b/pkg/kubelet/pod/pod_manager.go index ce5c1c30c6..17f54184b1 100644 --- a/pkg/kubelet/pod/pod_manager.go +++ b/pkg/kubelet/pod/pod_manager.go @@ -168,20 +168,40 @@ func (pm *basicManager) UpdatePod(pod *v1.Pod) { } } +func isPodInTerminatedState(pod *v1.Pod) bool { + return pod.Status.Phase == v1.PodFailed || pod.Status.Phase == v1.PodSucceeded +} + // updatePodsInternal replaces the given pods in the current state of the // manager, updating the various indices. The caller is assumed to hold the // lock. func (pm *basicManager) updatePodsInternal(pods ...*v1.Pod) { for _, pod := range pods { if pm.secretManager != nil { - // TODO: Consider detecting only status update and in such case do - // not register pod, as it doesn't really matter. - pm.secretManager.RegisterPod(pod) + if isPodInTerminatedState(pod) { + // Pods that are in terminated state and no longer running can be + // ignored as they no longer require access to secrets. + // It is especially important in watch-based manager, to avoid + // unnecessary watches for terminated pods waiting for GC. + pm.secretManager.UnregisterPod(pod) + } else { + // TODO: Consider detecting only status update and in such case do + // not register pod, as it doesn't really matter. + pm.secretManager.RegisterPod(pod) + } } if pm.configMapManager != nil { - // TODO: Consider detecting only status update and in such case do - // not register pod, as it doesn't really matter. - pm.configMapManager.RegisterPod(pod) + if isPodInTerminatedState(pod) { + // Pods that are in terminated state and no longer running can be + // ignored as they no longer require access to configmaps. + // It is especially important in watch-based manager, to avoid + // unnecessary watches for terminated pods waiting for GC. + pm.configMapManager.UnregisterPod(pod) + } else { + // TODO: Consider detecting only status update and in such case do + // not register pod, as it doesn't really matter. + pm.configMapManager.RegisterPod(pod) + } } podFullName := kubecontainer.GetPodFullName(pod) // This logic relies on a static pod and its mirror to have the same name. diff --git a/pkg/kubelet/util/manager/cache_based_manager_test.go b/pkg/kubelet/util/manager/cache_based_manager_test.go index c21584df50..652aa12257 100644 --- a/pkg/kubelet/util/manager/cache_based_manager_test.go +++ b/pkg/kubelet/util/manager/cache_based_manager_test.go @@ -429,6 +429,37 @@ func TestCacheInvalidation(t *testing.T) { fakeClient.ClearActions() } +func TestRegisterIdempotence(t *testing.T) { + fakeClient := &fake.Clientset{} + fakeClock := clock.NewFakeClock(time.Now()) + store := newSecretStore(fakeClient, fakeClock, noObjectTTL, time.Minute) + manager := newCacheBasedSecretManager(store) + + s1 := secretsToAttach{ + imagePullSecretNames: []string{"s1"}, + } + + refs := func(ns, name string) int { + store.lock.Lock() + defer store.lock.Unlock() + item, ok := store.items[objectKey{ns, name}] + if !ok { + return 0 + } + return item.refCount + } + + manager.RegisterPod(podWithSecrets("ns1", "name1", s1)) + assert.Equal(t, 1, refs("ns1", "s1")) + manager.RegisterPod(podWithSecrets("ns1", "name1", s1)) + assert.Equal(t, 1, refs("ns1", "s1")) + + manager.UnregisterPod(podWithSecrets("ns1", "name1", s1)) + assert.Equal(t, 0, refs("ns1", "s1")) + manager.UnregisterPod(podWithSecrets("ns1", "name1", s1)) + assert.Equal(t, 0, refs("ns1", "s1")) +} + func TestCacheRefcounts(t *testing.T) { fakeClient := &fake.Clientset{} fakeClock := clock.NewFakeClock(time.Now()) diff --git a/pkg/kubelet/util/manager/manager.go b/pkg/kubelet/util/manager/manager.go index 4d4b958d0a..2c983d35d2 100644 --- a/pkg/kubelet/util/manager/manager.go +++ b/pkg/kubelet/util/manager/manager.go @@ -32,10 +32,14 @@ type Manager interface { // i.e. should not block on network operations. // RegisterPod registers all objects referenced from a given pod. + // + // NOTE: All implementations of RegisterPod should be idempotent. RegisterPod(pod *v1.Pod) // UnregisterPod unregisters objects referenced from a given pod that are not // used by any other registered pod. + // + // NOTE: All implementations of UnregisterPod should be idempotent. UnregisterPod(pod *v1.Pod) } From c75fc3688964954b95fdfc37c43ea5598c622d21 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Tue, 5 Mar 2019 13:00:38 -0500 Subject: [PATCH 03/16] Deprecate make-symlink parameter in hyperkube Change-Id: I07e7f5f2a4e9050de92d3f0230dae0f869b77529 --- cmd/hyperkube/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/hyperkube/main.go b/cmd/hyperkube/main.go index b3ff9ea5c6..0c54a70993 100644 --- a/cmd/hyperkube/main.go +++ b/cmd/hyperkube/main.go @@ -142,6 +142,7 @@ func NewHyperKubeCommand(stopCh <-chan struct{}) (*cobra.Command, []func() *cobr } cmd.Flags().BoolVar(&makeSymlinksFlag, "make-symlinks", makeSymlinksFlag, "create a symlink for each server in current directory") cmd.Flags().MarkHidden("make-symlinks") // hide this flag from appearing in servers' usage output + cmd.Flags().MarkDeprecated("make-symlinks", "This feature will be removed in a later release.") for i := range commandFns { cmd.AddCommand(commandFns[i]()) From f7752c645a1838718749a80aadd6619b67067515 Mon Sep 17 00:00:00 2001 From: Niko Pen <42466421+nikopen@users.noreply.github.com> Date: Wed, 6 Mar 2019 00:49:48 +0000 Subject: [PATCH 04/16] rebase image to distroless/static --- test/images/audit-proxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/images/audit-proxy/Dockerfile b/test/images/audit-proxy/Dockerfile index 7bbeb62b45..ef7ef7fafb 100644 --- a/test/images/audit-proxy/Dockerfile +++ b/test/images/audit-proxy/Dockerfile @@ -12,6 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM scratch +FROM gcr.io/distroless/static:latest COPY audit-proxy / ENTRYPOINT ["/audit-proxy"] From 58c7b5de9c64b5efdcb8f8af01c78a58a6304521 Mon Sep 17 00:00:00 2001 From: Niko Pen <42466421+nikopen@users.noreply.github.com> Date: Wed, 6 Mar 2019 12:33:38 +0000 Subject: [PATCH 05/16] version bump --- test/images/audit-proxy/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/images/audit-proxy/VERSION b/test/images/audit-proxy/VERSION index d3827e75a5..9459d4ba2a 100644 --- a/test/images/audit-proxy/VERSION +++ b/test/images/audit-proxy/VERSION @@ -1 +1 @@ -1.0 +1.1 From 52913c59d15c58bbac9235897d6e3e5b3060dbbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20=C5=81ukaszewicz?= Date: Thu, 7 Mar 2019 17:02:51 +0100 Subject: [PATCH 06/16] Test with 2 pods, to make it more reliable. --- pkg/kubelet/util/manager/cache_based_manager_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/kubelet/util/manager/cache_based_manager_test.go b/pkg/kubelet/util/manager/cache_based_manager_test.go index 652aa12257..159ef4f74d 100644 --- a/pkg/kubelet/util/manager/cache_based_manager_test.go +++ b/pkg/kubelet/util/manager/cache_based_manager_test.go @@ -24,7 +24,7 @@ import ( "testing" "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -453,10 +453,14 @@ func TestRegisterIdempotence(t *testing.T) { assert.Equal(t, 1, refs("ns1", "s1")) manager.RegisterPod(podWithSecrets("ns1", "name1", s1)) assert.Equal(t, 1, refs("ns1", "s1")) + manager.RegisterPod(podWithSecrets("ns1", "name2", s1)) + assert.Equal(t, 2, refs("ns1", "s1")) manager.UnregisterPod(podWithSecrets("ns1", "name1", s1)) - assert.Equal(t, 0, refs("ns1", "s1")) + assert.Equal(t, 1, refs("ns1", "s1")) manager.UnregisterPod(podWithSecrets("ns1", "name1", s1)) + assert.Equal(t, 1, refs("ns1", "s1")) + manager.UnregisterPod(podWithSecrets("ns1", "name2", s1)) assert.Equal(t, 0, refs("ns1", "s1")) } From 8302b5b262c506ff7d363338f980ce204d574354 Mon Sep 17 00:00:00 2001 From: Tomas Nozicka Date: Fri, 8 Mar 2019 09:34:56 +0100 Subject: [PATCH 07/16] Handle unstructured status in RetryWatcher --- staging/src/k8s.io/client-go/tools/watch/retrywatcher.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/staging/src/k8s.io/client-go/tools/watch/retrywatcher.go b/staging/src/k8s.io/client-go/tools/watch/retrywatcher.go index 62c14b0784..e45d58ec15 100644 --- a/staging/src/k8s.io/client-go/tools/watch/retrywatcher.go +++ b/staging/src/k8s.io/client-go/tools/watch/retrywatcher.go @@ -184,13 +184,17 @@ func (rw *RetryWatcher) doReceive() (bool, time.Duration) { continue case watch.Error: - status, ok := event.Object.(*metav1.Status) + // This round trip allows us to handle unstructured status + errObject := apierrors.FromObject(event.Object) + statusErr, ok := errObject.(*apierrors.StatusError) if !ok { klog.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object)) // Retry unknown errors return false, 0 } + status := statusErr.ErrStatus + statusDelay := time.Duration(0) if status.Details != nil { statusDelay = time.Duration(status.Details.RetryAfterSeconds) * time.Second From 2790d9151a09391ed7a68abafbc05287f59cd8ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Osipiuk?= Date: Fri, 8 Mar 2019 14:01:08 +0100 Subject: [PATCH 08/16] Update Cluster Autscaler version to 1.14.0-beta.1 --- cluster/gce/manifests/cluster-autoscaler.manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/gce/manifests/cluster-autoscaler.manifest b/cluster/gce/manifests/cluster-autoscaler.manifest index 049f520200..2554a6150f 100644 --- a/cluster/gce/manifests/cluster-autoscaler.manifest +++ b/cluster/gce/manifests/cluster-autoscaler.manifest @@ -17,7 +17,7 @@ "containers": [ { "name": "cluster-autoscaler", - "image": "k8s.gcr.io/cluster-autoscaler:v1.13.0", + "image": "k8s.gcr.io/cluster-autoscaler:v1.14.0-beta.1", "livenessProbe": { "httpGet": { "path": "/health-check", From 5b0099785e70712d8dee215e089726d1c248e0d3 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Fri, 8 Mar 2019 13:44:39 -0500 Subject: [PATCH 09/16] deflake nodelease test --- test/e2e/common/node_lease.go | 61 +++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/test/e2e/common/node_lease.go b/test/e2e/common/node_lease.go index b3a113b86d..88d9833518 100644 --- a/test/e2e/common/node_lease.go +++ b/test/e2e/common/node_lease.go @@ -101,20 +101,37 @@ var _ = framework.KubeDescribe("NodeLease", func() { By("verify NodeStatus report period is longer than lease duration") // NodeStatus is reported from node to master when there is some change or // enough time has passed. So for here, keep checking the time diff - // between 2 NodeStatus report, until it is longer than lease duration ( - // the same as nodeMonitorGracePeriod). - heartbeatTime := getNextReadyConditionHeartbeatTime(f.ClientSet, nodeName, metav1.Time{}) + // between 2 NodeStatus report, until it is longer than lease duration + // (the same as nodeMonitorGracePeriod), or it doesn't change for at least leaseDuration + lastHeartbeatTime := getReadyConditionHeartbeatTime(f.ClientSet, nodeName) + lastObserved := time.Now() Eventually(func() error { - nextHeartbeatTime := getNextReadyConditionHeartbeatTime(f.ClientSet, nodeName, heartbeatTime) + currentHeartbeatTime := getReadyConditionHeartbeatTime(f.ClientSet, nodeName) + currentObserved := time.Now() - if nextHeartbeatTime.Time.After(heartbeatTime.Time.Add(leaseDuration)) { - return nil + switch { + case currentHeartbeatTime == lastHeartbeatTime: + if currentObserved.Sub(lastObserved) > 2*leaseDuration { + // heartbeat hasn't changed while watching for at least 2*leaseDuration, success! + framework.Logf("node status heartbeat is unchanged for %s, was waiting for at least %s, success!", currentObserved.Sub(lastObserved), 2*leaseDuration) + return nil + } + framework.Logf("node status heartbeat is unchanged for %s, waiting for %s", currentObserved.Sub(lastObserved), 2*leaseDuration) + return fmt.Errorf("node status heartbeat is unchanged for %s, waiting for %s", currentObserved.Sub(lastObserved), 2*leaseDuration) + + case currentHeartbeatTime != lastHeartbeatTime: + if currentHeartbeatTime.Sub(lastHeartbeatTime) > leaseDuration { + // heartbeat time changed, but the diff was greater than leaseDuration, success! + framework.Logf("node status heartbeat changed in %s, was waiting for at least %s, success!", currentHeartbeatTime.Sub(lastHeartbeatTime), leaseDuration) + return nil + } + lastHeartbeatTime = currentHeartbeatTime + lastObserved = currentObserved + framework.Logf("node status heartbeat changed in %s, waiting for %s", currentHeartbeatTime.Sub(lastHeartbeatTime), leaseDuration) + return fmt.Errorf("node status heartbeat changed in %s, waiting for %s", currentHeartbeatTime.Sub(lastHeartbeatTime), leaseDuration) } - heartbeatTime = nextHeartbeatTime - return fmt.Errorf("node status report period is shorter than lease duration") - - // Enter next round immediately. - }, 5*time.Minute, time.Nanosecond).Should(BeNil()) + return nil + }, 5*time.Minute, time.Second).Should(BeNil()) By("verify node is still in ready status even though node status report is infrequent") // This check on node status is only meaningful when this e2e test is @@ -128,22 +145,12 @@ var _ = framework.KubeDescribe("NodeLease", func() { }) }) -func getNextReadyConditionHeartbeatTime(clientSet clientset.Interface, nodeName string, prevHeartbeatTime metav1.Time) metav1.Time { - var newHeartbeatTime metav1.Time - Eventually(func() error { - node, err := clientSet.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) - if err != nil { - return err - } - _, readyCondition := testutils.GetNodeCondition(&node.Status, corev1.NodeReady) - Expect(readyCondition.Status).To(Equal(corev1.ConditionTrue)) - newHeartbeatTime = readyCondition.LastHeartbeatTime - if prevHeartbeatTime.Before(&newHeartbeatTime) { - return nil - } - return fmt.Errorf("heartbeat has not changed yet") - }, 5*time.Minute, 5*time.Second).Should(BeNil()) - return newHeartbeatTime +func getReadyConditionHeartbeatTime(clientSet clientset.Interface, nodeName string) time.Time { + node, err := clientSet.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) + Expect(err).To(BeNil()) + _, readyCondition := testutils.GetNodeCondition(&node.Status, corev1.NodeReady) + Expect(readyCondition.Status).To(Equal(corev1.ConditionTrue)) + return readyCondition.LastHeartbeatTime.Time } func expectLease(lease *coordv1beta1.Lease, nodeName string) error { From 7ffa7da1122fd6a7925ddc730a6cb508e7f2f389 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Sun, 24 Feb 2019 23:24:31 -0800 Subject: [PATCH 10/16] Configure logrotate for pod logs. --- cluster/gce/config-default.sh | 8 ++++++++ cluster/gce/config-test.sh | 8 ++++++++ cluster/gce/gci/configure-helper.sh | 15 +++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 34753dc7a5..a7e057927d 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -415,6 +415,14 @@ if [[ -n "${LOGROTATE_MAX_SIZE:-}" ]]; then PROVIDER_VARS="${PROVIDER_VARS:-} LOGROTATE_MAX_SIZE" fi +if [[ -n "${POD_LOG_MAX_FILE:-}" ]]; then + PROVIDER_VARS="${PROVIDER_VARS:-} POD_LOG_MAX_FILE" +fi + +if [[ -n "${POD_LOG_MAX_SIZE:-}" ]]; then + PROVIDER_VARS="${PROVIDER_VARS:-} POD_LOG_MAX_SIZE" +fi + # Fluentd requirements # YAML exists to trigger a configuration refresh when changes are made. FLUENTD_GCP_YAML_VERSION="v3.2.0" diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index cce1b431ed..9fbfac7a4a 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -432,6 +432,14 @@ if [[ -n "${LOGROTATE_MAX_SIZE:-}" ]]; then PROVIDER_VARS="${PROVIDER_VARS:-} LOGROTATE_MAX_SIZE" fi +if [[ -n "${POD_LOG_MAX_FILE:-}" ]]; then + PROVIDER_VARS="${PROVIDER_VARS:-} POD_LOG_MAX_FILE" +fi + +if [[ -n "${POD_LOG_MAX_SIZE:-}" ]]; then + PROVIDER_VARS="${PROVIDER_VARS:-} POD_LOG_MAX_SIZE" +fi + # Fluentd requirements # YAML exists to trigger a configuration refresh when changes are made. FLUENTD_GCP_YAML_VERSION="v3.2.0" diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index a2e429f6a4..de00f1ae60 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -374,6 +374,21 @@ function setup-logrotate() { } EOF + # Configure log rotation for pod logs in /var/log/pods/NAMESPACE_NAME_UID. + cat > /etc/logrotate.d/allpodlogs < Date: Sat, 9 Mar 2019 13:44:08 +0800 Subject: [PATCH 11/16] Delay CSI client initialization --- pkg/volume/csi/csi_block.go | 22 +++++++++++++++----- pkg/volume/csi/csi_client.go | 34 +++++++++++++++++++++++++++++++ pkg/volume/csi/csi_mounter.go | 14 ++++++++++--- pkg/volume/csi/csi_plugin.go | 20 ++++-------------- pkg/volume/csi/csi_plugin_test.go | 17 ++++++++++------ 5 files changed, 77 insertions(+), 30 deletions(-) diff --git a/pkg/volume/csi/csi_block.go b/pkg/volume/csi/csi_block.go index d8296c9eda..a1daf21cfe 100644 --- a/pkg/volume/csi/csi_block.go +++ b/pkg/volume/csi/csi_block.go @@ -37,8 +37,8 @@ import ( ) type csiBlockMapper struct { + csiClientGetter k8s kubernetes.Interface - csiClient csiClient plugin *csiPlugin driverName csiDriverName specName string @@ -247,14 +247,20 @@ func (m *csiBlockMapper) SetUpDevice() (string, error) { ctx, cancel := context.WithTimeout(context.Background(), csiTimeout) defer cancel() + csiClient, err := m.csiClientGetter.Get() + if err != nil { + klog.Error(log("blockMapper.SetUpDevice failed to get CSI client: %v", err)) + return "", err + } + // Call NodeStageVolume - stagingPath, err := m.stageVolumeForBlock(ctx, m.csiClient, accessMode, csiSource, attachment) + stagingPath, err := m.stageVolumeForBlock(ctx, csiClient, accessMode, csiSource, attachment) if err != nil { return "", err } // Call NodePublishVolume - publishPath, err := m.publishVolumeForBlock(ctx, m.csiClient, accessMode, csiSource, attachment, stagingPath) + publishPath, err := m.publishVolumeForBlock(ctx, csiClient, accessMode, csiSource, attachment, stagingPath) if err != nil { return "", err } @@ -326,6 +332,12 @@ func (m *csiBlockMapper) TearDownDevice(globalMapPath, devicePath string) error ctx, cancel := context.WithTimeout(context.Background(), csiTimeout) defer cancel() + csiClient, err := m.csiClientGetter.Get() + if err != nil { + klog.Error(log("blockMapper.TearDownDevice failed to get CSI client: %v", err)) + return err + } + // Call NodeUnpublishVolume publishPath := m.getPublishPath() if _, err := os.Stat(publishPath); err != nil { @@ -335,7 +347,7 @@ func (m *csiBlockMapper) TearDownDevice(globalMapPath, devicePath string) error return err } } else { - err := m.unpublishVolumeForBlock(ctx, m.csiClient, publishPath) + err := m.unpublishVolumeForBlock(ctx, csiClient, publishPath) if err != nil { return err } @@ -350,7 +362,7 @@ func (m *csiBlockMapper) TearDownDevice(globalMapPath, devicePath string) error return err } } else { - err := m.unstageVolumeForBlock(ctx, m.csiClient, stagingPath) + err := m.unstageVolumeForBlock(ctx, csiClient, stagingPath) if err != nil { return err } diff --git a/pkg/volume/csi/csi_client.go b/pkg/volume/csi/csi_client.go index 075123080b..272c5eee78 100644 --- a/pkg/volume/csi/csi_client.go +++ b/pkg/volume/csi/csi_client.go @@ -23,6 +23,7 @@ import ( "io" "net" "strings" + "sync" "time" csipbv1 "github.com/container-storage-interface/spec/lib/go/csi" @@ -807,3 +808,36 @@ func versionRequiresV0Client(version *utilversion.Version) bool { return false } + +// CSI client getter with cache. +// This provides a method to initialize CSI client with driver name and caches +// it for later use. When CSI clients have not been discovered yet (e.g. +// on kubelet restart), client initialization will fail. Users of CSI client (e.g. +// mounter manager and block mapper) can use this to delay CSI client +// initialization until needed. +type csiClientGetter struct { + sync.RWMutex + csiClient csiClient + driverName csiDriverName +} + +func (c *csiClientGetter) Get() (csiClient, error) { + c.RLock() + if c.csiClient != nil { + c.RUnlock() + return c.csiClient, nil + } + c.RUnlock() + c.Lock() + defer c.Unlock() + // Double-checking locking criterion. + if c.csiClient != nil { + return c.csiClient, nil + } + csi, err := newCsiDriverClient(c.driverName) + if err != nil { + return nil, err + } + c.csiClient = csi + return c.csiClient, nil +} diff --git a/pkg/volume/csi/csi_mounter.go b/pkg/volume/csi/csi_mounter.go index f61ccc33d2..ccdce2ace2 100644 --- a/pkg/volume/csi/csi_mounter.go +++ b/pkg/volume/csi/csi_mounter.go @@ -56,7 +56,7 @@ var ( ) type csiMountMgr struct { - csiClient csiClient + csiClientGetter k8s kubernetes.Interface plugin *csiPlugin driverName csiDriverName @@ -111,7 +111,11 @@ func (c *csiMountMgr) SetUpAt(dir string, fsGroup *int64) error { return nil } - csi := c.csiClient + csi, err := c.csiClientGetter.Get() + if err != nil { + klog.Error(log("mounter.SetUpAt failed to get CSI client: %v", err)) + return err + } ctx, cancel := context.WithTimeout(context.Background(), csiTimeout) defer cancel() @@ -343,7 +347,11 @@ func (c *csiMountMgr) TearDownAt(dir string) error { klog.V(4).Infof(log("Unmounter.TearDown(%s)", dir)) volID := c.volumeID - csi := c.csiClient + csi, err := c.csiClientGetter.Get() + if err != nil { + klog.Error(log("mounter.SetUpAt failed to get CSI client: %v", err)) + return err + } ctx, cancel := context.WithTimeout(context.Background(), csiTimeout) defer cancel() diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go index f1ac67aced..ade21aa109 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go @@ -383,11 +383,6 @@ func (p *csiPlugin) NewMounter( return nil, errors.New("failed to get a Kubernetes client") } - csi, err := newCsiDriverClient(csiDriverName(driverName)) - if err != nil { - return nil, err - } - mounter := &csiMountMgr{ plugin: p, k8s: k8s, @@ -398,9 +393,9 @@ func (p *csiPlugin) NewMounter( driverMode: driverMode, volumeID: volumeHandle, specVolumeID: spec.Name(), - csiClient: csi, readOnly: readOnly, } + mounter.csiClientGetter.driverName = csiDriverName(driverName) // Save volume info in pod dir dir := mounter.GetPath() @@ -458,10 +453,7 @@ func (p *csiPlugin) NewUnmounter(specName string, podUID types.UID) (volume.Unmo } unmounter.driverName = csiDriverName(data[volDataKey.driverName]) unmounter.volumeID = data[volDataKey.volHandle] - unmounter.csiClient, err = newCsiDriverClient(unmounter.driverName) - if err != nil { - return nil, err - } + unmounter.csiClientGetter.driverName = unmounter.driverName return unmounter, nil } @@ -638,10 +630,6 @@ func (p *csiPlugin) NewBlockVolumeMapper(spec *volume.Spec, podRef *api.Pod, opt } klog.V(4).Info(log("setting up block mapper for [volume=%v,driver=%v]", pvSource.VolumeHandle, pvSource.Driver)) - client, err := newCsiDriverClient(csiDriverName(pvSource.Driver)) - if err != nil { - return nil, err - } k8s := p.host.GetKubeClient() if k8s == nil { @@ -650,7 +638,6 @@ func (p *csiPlugin) NewBlockVolumeMapper(spec *volume.Spec, podRef *api.Pod, opt } mapper := &csiBlockMapper{ - csiClient: client, k8s: k8s, plugin: p, volumeID: pvSource.VolumeHandle, @@ -660,6 +647,7 @@ func (p *csiPlugin) NewBlockVolumeMapper(spec *volume.Spec, podRef *api.Pod, opt specName: spec.Name(), podUID: podRef.UID, } + mapper.csiClientGetter.driverName = csiDriverName(pvSource.Driver) // Save volume info in pod dir dataDir := getVolumeDeviceDataDir(spec.Name(), p.host) @@ -714,7 +702,7 @@ func (p *csiPlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (vo } unmapper.driverName = csiDriverName(data[volDataKey.driverName]) unmapper.volumeID = data[volDataKey.volHandle] - unmapper.csiClient, err = newCsiDriverClient(unmapper.driverName) + unmapper.csiClientGetter.driverName = unmapper.driverName if err != nil { return nil, err } diff --git a/pkg/volume/csi/csi_plugin_test.go b/pkg/volume/csi/csi_plugin_test.go index a45e0dcb99..4747ba4db3 100644 --- a/pkg/volume/csi/csi_plugin_test.go +++ b/pkg/volume/csi/csi_plugin_test.go @@ -614,7 +614,8 @@ func TestPluginNewMounter(t *testing.T) { if string(csiMounter.podUID) != string(test.podUID) { t.Error("mounter podUID not set") } - if csiMounter.csiClient == nil { + csiClient, err := csiMounter.csiClientGetter.Get() + if csiClient == nil { t.Error("mounter csiClient is nil") } if csiMounter.driverMode != test.driverMode { @@ -732,7 +733,8 @@ func TestPluginNewMounterWithInline(t *testing.T) { if string(csiMounter.podUID) != string(test.podUID) { t.Error("mounter podUID not set") } - if csiMounter.csiClient == nil { + csiClient, err := csiMounter.csiClientGetter.Get() + if csiClient == nil { t.Error("mounter csiClient is nil") } if csiMounter.driverMode != test.driverMode { @@ -815,8 +817,9 @@ func TestPluginNewUnmounter(t *testing.T) { t.Error("podUID not set") } - if csiUnmounter.csiClient == nil { - t.Error("unmounter csiClient is nil") + csiClient, err := csiUnmounter.csiClientGetter.Get() + if csiClient == nil { + t.Error("mounter csiClient is nil") } } @@ -932,7 +935,8 @@ func TestPluginNewBlockMapper(t *testing.T) { if csiMapper.podUID == types.UID("") { t.Error("CSI block mapper missing pod.UID") } - if csiMapper.csiClient == nil { + csiClient, err := csiMapper.csiClientGetter.Get() + if csiClient == nil { t.Error("mapper csiClient is nil") } @@ -994,7 +998,8 @@ func TestPluginNewUnmapper(t *testing.T) { t.Error("specName not set") } - if csiUnmapper.csiClient == nil { + csiClient, err := csiUnmapper.csiClientGetter.Get() + if csiClient == nil { t.Error("unmapper csiClient is nil") } From 720a5e20d8da0bc5c95296c056c6771ca5d70d4e Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Thu, 7 Mar 2019 05:38:06 +0000 Subject: [PATCH 12/16] fix smb unmount issue on Windows fix log warning use IsCorruptedMnt in GetMountRefs on Windows use errorno in IsCorruptedMnt check fix comments: add more error code add more error no checking change year fix comments --- pkg/util/mount/BUILD | 4 +- ...mount_helper.go => mount_helper_common.go} | 21 ------ pkg/util/mount/mount_helper_unix.go | 44 ++++++++++++ pkg/util/mount/mount_helper_windows.go | 68 +++++++++++++++++++ pkg/util/mount/mount_windows.go | 11 +-- 5 files changed, 121 insertions(+), 27 deletions(-) rename pkg/util/mount/{mount_helper.go => mount_helper_common.go} (85%) create mode 100644 pkg/util/mount/mount_helper_unix.go create mode 100644 pkg/util/mount/mount_helper_windows.go diff --git a/pkg/util/mount/BUILD b/pkg/util/mount/BUILD index 5b22b2b5aa..f50a934a9d 100644 --- a/pkg/util/mount/BUILD +++ b/pkg/util/mount/BUILD @@ -9,7 +9,9 @@ go_library( "exec_mount_unsupported.go", "fake.go", "mount.go", - "mount_helper.go", + "mount_helper_common.go", + "mount_helper_unix.go", + "mount_helper_windows.go", "mount_linux.go", "mount_unsupported.go", "mount_windows.go", diff --git a/pkg/util/mount/mount_helper.go b/pkg/util/mount/mount_helper_common.go similarity index 85% rename from pkg/util/mount/mount_helper.go rename to pkg/util/mount/mount_helper_common.go index 9984705183..cff1d89588 100644 --- a/pkg/util/mount/mount_helper.go +++ b/pkg/util/mount/mount_helper_common.go @@ -19,7 +19,6 @@ package mount import ( "fmt" "os" - "syscall" "k8s.io/klog" ) @@ -102,23 +101,3 @@ func PathExists(path string) (bool, error) { return false, err } } - -// IsCorruptedMnt return true if err is about corrupted mount point -func IsCorruptedMnt(err error) bool { - if err == nil { - return false - } - var underlyingError error - switch pe := err.(type) { - case nil: - return false - case *os.PathError: - underlyingError = pe.Err - case *os.LinkError: - underlyingError = pe.Err - case *os.SyscallError: - underlyingError = pe.Err - } - - return underlyingError == syscall.ENOTCONN || underlyingError == syscall.ESTALE || underlyingError == syscall.EIO || underlyingError == syscall.EACCES -} diff --git a/pkg/util/mount/mount_helper_unix.go b/pkg/util/mount/mount_helper_unix.go new file mode 100644 index 0000000000..880a89e159 --- /dev/null +++ b/pkg/util/mount/mount_helper_unix.go @@ -0,0 +1,44 @@ +// +build !windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "os" + "syscall" +) + +// IsCorruptedMnt return true if err is about corrupted mount point +func IsCorruptedMnt(err error) bool { + if err == nil { + return false + } + var underlyingError error + switch pe := err.(type) { + case nil: + return false + case *os.PathError: + underlyingError = pe.Err + case *os.LinkError: + underlyingError = pe.Err + case *os.SyscallError: + underlyingError = pe.Err + } + + return underlyingError == syscall.ENOTCONN || underlyingError == syscall.ESTALE || underlyingError == syscall.EIO || underlyingError == syscall.EACCES +} diff --git a/pkg/util/mount/mount_helper_windows.go b/pkg/util/mount/mount_helper_windows.go new file mode 100644 index 0000000000..e9b3c65779 --- /dev/null +++ b/pkg/util/mount/mount_helper_windows.go @@ -0,0 +1,68 @@ +// +build windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "os" + "syscall" + + "k8s.io/klog" +) + +// following failure codes are from https://docs.microsoft.com/en-us/windows/desktop/debug/system-error-codes--1300-1699- +// ERROR_BAD_NETPATH = 53 +// ERROR_NETWORK_BUSY = 54 +// ERROR_UNEXP_NET_ERR = 59 +// ERROR_NETNAME_DELETED = 64 +// ERROR_NETWORK_ACCESS_DENIED = 65 +// ERROR_BAD_DEV_TYPE = 66 +// ERROR_BAD_NET_NAME = 67 +// ERROR_SESSION_CREDENTIAL_CONFLICT = 1219 +// ERROR_LOGON_FAILURE = 1326 +var errorNoList = [...]int{53, 54, 59, 64, 65, 66, 67, 1219, 1326} + +// IsCorruptedMnt return true if err is about corrupted mount point +func IsCorruptedMnt(err error) bool { + if err == nil { + return false + } + + var underlyingError error + switch pe := err.(type) { + case nil: + return false + case *os.PathError: + underlyingError = pe.Err + case *os.LinkError: + underlyingError = pe.Err + case *os.SyscallError: + underlyingError = pe.Err + } + + if ee, ok := underlyingError.(syscall.Errno); ok { + for _, errno := range errorNoList { + if int(ee) == errno { + klog.Warningf("IsCorruptedMnt failed with error: %v, error code: %v", err, errno) + return true + } + } + } + + return false +} diff --git a/pkg/util/mount/mount_windows.go b/pkg/util/mount/mount_windows.go index 84aa18421b..53dda930b7 100644 --- a/pkg/util/mount/mount_windows.go +++ b/pkg/util/mount/mount_windows.go @@ -378,14 +378,15 @@ func getAllParentLinks(path string) ([]string, error) { // GetMountRefs : empty implementation here since there is no place to query all mount points on Windows func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { - pathExists, pathErr := PathExists(normalizeWindowsPath(pathname)) - // TODO(#75012): Need a Windows specific IsCorruptedMnt function that checks against whatever errno's - // Windows emits when we try to Stat a corrupted mount - // https://golang.org/pkg/syscall/?GOOS=windows&GOARCH=amd64#Errno + windowsPath := normalizeWindowsPath(pathname) + pathExists, pathErr := PathExists(windowsPath) if !pathExists { return []string{}, nil + } else if IsCorruptedMnt(pathErr) { + klog.Warningf("GetMountRefs found corrupted mount at %s, treating as unmounted path", windowsPath) + return []string{}, nil } else if pathErr != nil { - return nil, fmt.Errorf("error checking path %s: %v", normalizeWindowsPath(pathname), pathErr) + return nil, fmt.Errorf("error checking path %s: %v", windowsPath, pathErr) } return []string{pathname}, nil } From c5c4cd2580dd6ce9f6c35f2caaebd4f922e46fae Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Sat, 9 Mar 2019 00:57:11 +0200 Subject: [PATCH 13/16] kubeadm: print key inside the upload-certs phase of init The standalone execution of upload-certs phase does not print the key that that user should use for the newly uploaded encrypted secret. Print this key in the upload-certs phase in both standalone mode or if executed in the standard init workflow. Make it possible to omit the printing if the user passes --skip-certificate-key-print. Also: - Uppercase string in Printf call in copycerts.go - Don't use V(1) for the "Skipping phase" message in uploadcerts.go instead always print a message that the user case use --experimental-upload-certs. This solves a problem if the user tried the standalone phase but didn't pass --experimental-upload-certs. --- cmd/kubeadm/app/cmd/init.go | 5 +++++ cmd/kubeadm/app/cmd/phases/init/data.go | 1 + cmd/kubeadm/app/cmd/phases/init/data_test.go | 1 + cmd/kubeadm/app/cmd/phases/init/uploadcerts.go | 7 +++++-- cmd/kubeadm/app/phases/copycerts/copycerts.go | 2 +- 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 85c5eec165..b55aa9e87c 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -379,6 +379,11 @@ func (d *initData) SetCertificateKey(key string) { d.certificateKey = key } +// SkipCertificateKeyPrint returns the skipCertificateKeyPrint flag. +func (d *initData) SkipCertificateKeyPrint() bool { + return d.skipCertificateKeyPrint +} + // Cfg returns initConfiguration. func (d *initData) Cfg() *kubeadmapi.InitConfiguration { return d.cfg diff --git a/cmd/kubeadm/app/cmd/phases/init/data.go b/cmd/kubeadm/app/cmd/phases/init/data.go index f30a9c3577..38e7a88ec2 100644 --- a/cmd/kubeadm/app/cmd/phases/init/data.go +++ b/cmd/kubeadm/app/cmd/phases/init/data.go @@ -30,6 +30,7 @@ type InitData interface { UploadCerts() bool CertificateKey() string SetCertificateKey(key string) + SkipCertificateKeyPrint() bool Cfg() *kubeadmapi.InitConfiguration DryRun() bool SkipTokenPrint() bool diff --git a/cmd/kubeadm/app/cmd/phases/init/data_test.go b/cmd/kubeadm/app/cmd/phases/init/data_test.go index 4d51efef76..a33cf8189f 100644 --- a/cmd/kubeadm/app/cmd/phases/init/data_test.go +++ b/cmd/kubeadm/app/cmd/phases/init/data_test.go @@ -33,6 +33,7 @@ var _ InitData = &testInitData{} func (t *testInitData) UploadCerts() bool { return false } func (t *testInitData) CertificateKey() string { return "" } func (t *testInitData) SetCertificateKey(key string) {} +func (t *testInitData) SkipCertificateKeyPrint() bool { return false } func (t *testInitData) Cfg() *kubeadmapi.InitConfiguration { return nil } func (t *testInitData) DryRun() bool { return false } func (t *testInitData) SkipTokenPrint() bool { return false } diff --git a/cmd/kubeadm/app/cmd/phases/init/uploadcerts.go b/cmd/kubeadm/app/cmd/phases/init/uploadcerts.go index bd24cbcff1..f62815a446 100644 --- a/cmd/kubeadm/app/cmd/phases/init/uploadcerts.go +++ b/cmd/kubeadm/app/cmd/phases/init/uploadcerts.go @@ -21,7 +21,6 @@ import ( "github.com/pkg/errors" - "k8s.io/klog" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" @@ -40,6 +39,7 @@ func NewUploadCertsPhase() workflow.Phase { options.CfgPath, options.UploadCerts, options.CertificateKey, + options.SkipCertificateKeyPrint, }, } } @@ -51,7 +51,7 @@ func runUploadCerts(c workflow.RunData) error { } if !data.UploadCerts() { - klog.V(1).Infoln("[upload-certs] Skipping certs upload") + fmt.Printf("[upload-certs] Skipping phase. Please see --%s\n", options.UploadCerts) return nil } client, err := data.Client() @@ -70,5 +70,8 @@ func runUploadCerts(c workflow.RunData) error { if err := copycerts.UploadCerts(client, data.Cfg(), data.CertificateKey()); err != nil { return errors.Wrap(err, "error uploading certs") } + if !data.SkipCertificateKeyPrint() { + fmt.Printf("[upload-certs] Using certificate key:\n%s\n", data.CertificateKey()) + } return nil } diff --git a/cmd/kubeadm/app/phases/copycerts/copycerts.go b/cmd/kubeadm/app/phases/copycerts/copycerts.go index 229be0a22f..44899f178e 100644 --- a/cmd/kubeadm/app/phases/copycerts/copycerts.go +++ b/cmd/kubeadm/app/phases/copycerts/copycerts.go @@ -85,7 +85,7 @@ func CreateCertificateKey() (string, error) { //UploadCerts save certs needs to join a new control-plane on kubeadm-certs sercret. func UploadCerts(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, key string) error { - fmt.Printf("[upload-certs] storing the certificates in ConfigMap %q in the %q Namespace\n", kubeadmconstants.KubeadmCertsSecret, metav1.NamespaceSystem) + fmt.Printf("[upload-certs] Storing the certificates in ConfigMap %q in the %q Namespace\n", kubeadmconstants.KubeadmCertsSecret, metav1.NamespaceSystem) decodedKey, err := hex.DecodeString(key) if err != nil { return err From 3b618af0d435628feedf06f97bd1c69340d07d95 Mon Sep 17 00:00:00 2001 From: Chao Xu Date: Mon, 14 Jan 2019 19:31:25 -0800 Subject: [PATCH 14/16] Expose storage version hash --- cmd/kube-apiserver/app/aggregator.go | 2 + cmd/kube-apiserver/app/apiextensions.go | 3 + pkg/master/master_test.go | 126 +++++++++++++++++- pkg/master/storageversionhashdata/OWNERS | 4 + pkg/master/storageversionhashdata/data.go | 111 +++++++++++++++ .../core/namespace/storage/storage.go | 6 + pkg/registry/core/service/storage/rest.go | 12 +- .../core/service/storage/rest_test.go | 4 + .../rbac/clusterrole/policybased/storage.go | 10 ++ .../clusterrolebinding/policybased/storage.go | 10 ++ pkg/registry/rbac/role/policybased/storage.go | 10 ++ .../rbac/rolebinding/policybased/storage.go | 10 ++ .../customresource_discovery_controller.go | 19 ++- .../endpoints/discovery/storageversionhash.go | 40 ++++++ .../apiserver/pkg/endpoints/installer.go | 27 ++++ .../apiserver/pkg/features/kube_features.go | 8 ++ .../pkg/registry/generic/registry/store.go | 11 ++ .../apiserver/pkg/registry/rest/rest.go | 9 ++ .../pkg/server/storage/storage_codec.go | 20 +-- .../pkg/server/storage/storage_factory.go | 4 +- .../pkg/storage/storagebackend/config.go | 5 + .../sample-apiserver/pkg/cmd/server/start.go | 4 +- test/e2e/apimachinery/discovery.go | 78 +++++++++++ 23 files changed, 510 insertions(+), 23 deletions(-) create mode 100644 pkg/master/storageversionhashdata/OWNERS create mode 100644 pkg/master/storageversionhashdata/data.go create mode 100644 staging/src/k8s.io/apiserver/pkg/endpoints/discovery/storageversionhash.go create mode 100644 test/e2e/apimachinery/discovery.go diff --git a/cmd/kube-apiserver/app/aggregator.go b/cmd/kube-apiserver/app/aggregator.go index acf5b071ca..2cedb55cdc 100644 --- a/cmd/kube-apiserver/app/aggregator.go +++ b/cmd/kube-apiserver/app/aggregator.go @@ -30,6 +30,7 @@ import ( apiextensionsinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/admission" @@ -79,6 +80,7 @@ func createAggregatorConfig( etcdOptions := *commandOptions.Etcd etcdOptions.StorageConfig.Paging = utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking) etcdOptions.StorageConfig.Codec = aggregatorscheme.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion) + etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1beta1.SchemeGroupVersion, schema.GroupKind{Group: v1beta1.GroupName}) genericConfig.RESTOptionsGetter = &genericoptions.SimpleRestOptionsFactory{Options: etcdOptions} // override MergedResourceConfig with aggregator defaults and registry diff --git a/cmd/kube-apiserver/app/apiextensions.go b/cmd/kube-apiserver/app/apiextensions.go index 2db21153cd..f2da1ca989 100644 --- a/cmd/kube-apiserver/app/apiextensions.go +++ b/cmd/kube-apiserver/app/apiextensions.go @@ -23,6 +23,8 @@ import ( "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" apiextensionsoptions "k8s.io/apiextensions-apiserver/pkg/cmd/server/options" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/features" genericapiserver "k8s.io/apiserver/pkg/server" @@ -61,6 +63,7 @@ func createAPIExtensionsConfig( etcdOptions := *commandOptions.Etcd etcdOptions.StorageConfig.Paging = utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking) etcdOptions.StorageConfig.Codec = apiextensionsapiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion) + etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1beta1.SchemeGroupVersion, schema.GroupKind{Group: v1beta1.GroupName}) genericConfig.RESTOptionsGetter = &genericoptions.SimpleRestOptionsFactory{Options: etcdOptions} // override MergedResourceConfig with apiextensions defaults and registry diff --git a/pkg/master/master_test.go b/pkg/master/master_test.go index efac9d45f0..90239fa0e0 100644 --- a/pkg/master/master_test.go +++ b/pkg/master/master_test.go @@ -31,22 +31,32 @@ import ( certificatesapiv1beta1 "k8s.io/api/certificates/v1beta1" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/version" + "k8s.io/apiserver/pkg/authorization/authorizerfactory" + "k8s.io/apiserver/pkg/features" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/options" + "k8s.io/apiserver/pkg/server/resourceconfig" serverstorage "k8s.io/apiserver/pkg/server/storage" etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + "k8s.io/client-go/discovery" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" restclient "k8s.io/client-go/rest" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/api/testapi" + "k8s.io/kubernetes/pkg/apis/batch" api "k8s.io/kubernetes/pkg/apis/core" + apisstorage "k8s.io/kubernetes/pkg/apis/storage" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" "k8s.io/kubernetes/pkg/master/reconcilers" + "k8s.io/kubernetes/pkg/master/storageversionhashdata" certificatesrest "k8s.io/kubernetes/pkg/registry/certificates/rest" corerest "k8s.io/kubernetes/pkg/registry/core/rest" "k8s.io/kubernetes/pkg/registry/registrytest" @@ -70,6 +80,14 @@ func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, *assert.Assertion } resourceEncoding := serverstorage.NewDefaultResourceEncodingConfig(legacyscheme.Scheme) + // This configures the testing master the same way the real master is + // configured. The storage versions of these resources are different + // from the storage versions of other resources in their group. + resourceEncodingOverrides := []schema.GroupVersionResource{ + batch.Resource("cronjobs").WithVersion("v1beta1"), + apisstorage.Resource("volumeattachments").WithVersion("v1beta1"), + } + resourceEncoding = resourceconfig.MergeResourceEncodingConfigs(resourceEncoding, resourceEncodingOverrides) storageFactory := serverstorage.NewDefaultStorageFactory(*storageConfig, testapi.StorageMediaType(), legacyscheme.Codecs, resourceEncoding, DefaultAPIResourceConfigSource(), nil) etcdOptions := options.NewEtcdOptions(storageConfig) @@ -81,12 +99,12 @@ func setUp(t *testing.T) (*etcdtesting.EtcdTestServer, Config, *assert.Assertion } kubeVersion := kubeversion.Get() + config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer() config.GenericConfig.Version = &kubeVersion config.ExtraConfig.StorageFactory = storageFactory config.GenericConfig.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}} config.GenericConfig.PublicAddress = net.ParseIP("192.168.10.4") config.GenericConfig.LegacyAPIGroupPrefixes = sets.NewString("/api") - config.GenericConfig.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}} config.ExtraConfig.KubeletClientConfig = kubeletclient.KubeletClientConfig{Port: 10250} config.ExtraConfig.ProxyTransport = utilnet.SetTransportDefaults(&http.Transport{ DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { return nil, nil }, @@ -363,6 +381,112 @@ func TestAPIVersionOfDiscoveryEndpoints(t *testing.T) { } +// This test doesn't cover the apiregistration and apiextensions group, as they are installed by other apiservers. +func TestStorageVersionHashes(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionHash, true)() + master, etcdserver, _, _ := newMaster(t) + defer etcdserver.Terminate(t) + + server := httptest.NewServer(master.GenericAPIServer.Handler.GoRestfulContainer.ServeMux) + + c := &restclient.Config{ + Host: server.URL, + APIPath: "/api", + ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}, + } + discover := discovery.NewDiscoveryClientForConfigOrDie(c) + all, err := discover.ServerResources() + if err != nil { + t.Error(err) + } + var count int + for _, g := range all { + for _, r := range g.APIResources { + if strings.Contains(r.Name, "/") || + storageversionhashdata.NoStorageVersionHash.Has(g.GroupVersion+"/"+r.Name) { + if r.StorageVersionHash != "" { + t.Errorf("expect resource %s/%s to have empty storageVersionHash, got hash %q", g.GroupVersion, r.Name, r.StorageVersionHash) + } + continue + } + if r.StorageVersionHash == "" { + t.Errorf("expect the storageVersionHash of %s/%s to exist", g.GroupVersion, r.Name) + continue + } + // Uncomment the following line if you want to update storageversionhash/data.go + // fmt.Printf("\"%s/%s\": \"%s\",\n", g.GroupVersion, r.Name, r.StorageVersionHash) + expected := storageversionhashdata.GVRToStorageVersionHash[g.GroupVersion+"/"+r.Name] + if r.StorageVersionHash != expected { + t.Errorf("expect the storageVersionHash of %s/%s to be %q, got %q", g.GroupVersion, r.Name, expected, r.StorageVersionHash) + } + count++ + } + } + if count != len(storageversionhashdata.GVRToStorageVersionHash) { + t.Errorf("please remove the redundant entries from GVRToStorageVersionHash") + } +} + +func TestStorageVersionHashEqualities(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionHash, true)() + master, etcdserver, _, assert := newMaster(t) + defer etcdserver.Terminate(t) + + server := httptest.NewServer(master.GenericAPIServer.Handler.GoRestfulContainer.ServeMux) + + // Test 1: extensions/v1beta1/replicasets and apps/v1/replicasets have + // the same storage version hash. + resp, err := http.Get(server.URL + "/apis/extensions/v1beta1") + assert.Empty(err) + extList := metav1.APIResourceList{} + assert.NoError(decodeResponse(resp, &extList)) + var extReplicasetHash, appsReplicasetHash string + for _, r := range extList.APIResources { + if r.Name == "replicasets" { + extReplicasetHash = r.StorageVersionHash + } + } + assert.NotEmpty(extReplicasetHash) + + resp, err = http.Get(server.URL + "/apis/apps/v1") + assert.Empty(err) + appsList := metav1.APIResourceList{} + assert.NoError(decodeResponse(resp, &appsList)) + for _, r := range appsList.APIResources { + if r.Name == "replicasets" { + appsReplicasetHash = r.StorageVersionHash + } + } + assert.Equal(extReplicasetHash, appsReplicasetHash) + + // Test 2: batch/v1/jobs and batch/v1beta1/cronjobs have different + // storage version hashes. + resp, err = http.Get(server.URL + "/apis/batch/v1") + assert.Empty(err) + batchv1 := metav1.APIResourceList{} + assert.NoError(decodeResponse(resp, &batchv1)) + var jobsHash string + for _, r := range batchv1.APIResources { + if r.Name == "jobs" { + jobsHash = r.StorageVersionHash + } + } + assert.NotEmpty(jobsHash) + + resp, err = http.Get(server.URL + "/apis/batch/v1beta1") + assert.Empty(err) + batchv1beta1 := metav1.APIResourceList{} + assert.NoError(decodeResponse(resp, &batchv1beta1)) + var cronjobsHash string + for _, r := range batchv1beta1.APIResources { + if r.Name == "cronjobs" { + cronjobsHash = r.StorageVersionHash + } + } + assert.NotEmpty(cronjobsHash) + assert.NotEqual(jobsHash, cronjobsHash) +} + func TestNoAlphaVersionsEnabledByDefault(t *testing.T) { config := DefaultAPIResourceConfigSource() for gv, enable := range config.GroupVersionConfigs { diff --git a/pkg/master/storageversionhashdata/OWNERS b/pkg/master/storageversionhashdata/OWNERS new file mode 100644 index 0000000000..8f7783f9f0 --- /dev/null +++ b/pkg/master/storageversionhashdata/OWNERS @@ -0,0 +1,4 @@ +approvers: +- api-approvers +reviewers: +- api-reviewers diff --git a/pkg/master/storageversionhashdata/data.go b/pkg/master/storageversionhashdata/data.go new file mode 100644 index 0000000000..2a8b50365b --- /dev/null +++ b/pkg/master/storageversionhashdata/data.go @@ -0,0 +1,111 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package storageversionhashdata is for test only. +package storageversionhashdata + +import ( + "k8s.io/apimachinery/pkg/util/sets" +) + +// NoStorageVersionHash lists resources that legitimately with empty storage +// version hash. +var NoStorageVersionHash = sets.NewString( + "v1/bindings", + "v1/componentstatuses", + "authentication.k8s.io/v1/tokenreviews", + "authorization.k8s.io/v1/localsubjectaccessreviews", + "authorization.k8s.io/v1/selfsubjectaccessreviews", + "authorization.k8s.io/v1/selfsubjectrulesreviews", + "authorization.k8s.io/v1/subjectaccessreviews", + "authentication.k8s.io/v1beta1/tokenreviews", + "authorization.k8s.io/v1beta1/localsubjectaccessreviews", + "authorization.k8s.io/v1beta1/selfsubjectaccessreviews", + "authorization.k8s.io/v1beta1/selfsubjectrulesreviews", + "authorization.k8s.io/v1beta1/subjectaccessreviews", + "extensions/v1beta1/replicationcontrollers", +) + +// GVRToStorageVersionHash shouldn't change unless we intentionally change the +// storage version of a resource. +var GVRToStorageVersionHash = map[string]string{ + "v1/configmaps": "qFsyl6wFWjQ=", + "v1/endpoints": "fWeeMqaN/OA=", + "v1/events": "r2yiGXH7wu8=", + "v1/limitranges": "EBKMFVe6cwo=", + "v1/namespaces": "Q3oi5N2YM8M=", + "v1/nodes": "XwShjMxG9Fs=", + "v1/persistentvolumeclaims": "QWTyNDq0dC4=", + "v1/persistentvolumes": "HN/zwEC+JgM=", + "v1/pods": "xPOwRZ+Yhw8=", + "v1/podtemplates": "LIXB2x4IFpk=", + "v1/replicationcontrollers": "Jond2If31h0=", + "v1/resourcequotas": "8uhSgffRX6w=", + "v1/secrets": "S6u1pOWzb84=", + "v1/serviceaccounts": "pbx9ZvyFpBE=", + "v1/services": "0/CO1lhkEBI=", + "autoscaling/v1/horizontalpodautoscalers": "oQlkt7f5j/A=", + "autoscaling/v2beta1/horizontalpodautoscalers": "oQlkt7f5j/A=", + "autoscaling/v2beta2/horizontalpodautoscalers": "oQlkt7f5j/A=", + "batch/v1/jobs": "mudhfqk/qZY=", + "batch/v1beta1/cronjobs": "h/JlFAZkyyY=", + "certificates.k8s.io/v1beta1/certificatesigningrequests": "UQh3YTCDIf0=", + "coordination.k8s.io/v1beta1/leases": "/sY7hl8ol1U=", + "coordination.k8s.io/v1/leases": "/sY7hl8ol1U=", + "extensions/v1beta1/daemonsets": "dd7pWHUlMKQ=", + "extensions/v1beta1/deployments": "8aSe+NMegvE=", + "extensions/v1beta1/ingresses": "Ejja63IbU0E=", + "extensions/v1beta1/networkpolicies": "YpfwF18m1G8=", + "extensions/v1beta1/podsecuritypolicies": "khBLobUXkqA=", + "extensions/v1beta1/replicasets": "P1RzHs8/mWQ=", + "networking.k8s.io/v1/networkpolicies": "YpfwF18m1G8=", + "networking.k8s.io/v1beta1/ingresses": "Ejja63IbU0E=", + "node.k8s.io/v1beta1/runtimeclasses": "8nMHWqj34s0=", + "policy/v1beta1/poddisruptionbudgets": "6BGBu0kpHtk=", + "policy/v1beta1/podsecuritypolicies": "khBLobUXkqA=", + "rbac.authorization.k8s.io/v1/clusterrolebindings": "48tpQ8gZHFc=", + "rbac.authorization.k8s.io/v1/clusterroles": "bYE5ZWDrJ44=", + "rbac.authorization.k8s.io/v1/rolebindings": "eGsCzGH6b1g=", + "rbac.authorization.k8s.io/v1/roles": "7FuwZcIIItM=", + "rbac.authorization.k8s.io/v1beta1/clusterrolebindings": "48tpQ8gZHFc=", + "rbac.authorization.k8s.io/v1beta1/clusterroles": "bYE5ZWDrJ44=", + "rbac.authorization.k8s.io/v1beta1/rolebindings": "eGsCzGH6b1g=", + "rbac.authorization.k8s.io/v1beta1/roles": "7FuwZcIIItM=", + "scheduling.k8s.io/v1beta1/priorityclasses": "D3vHs+OgrtA=", + "scheduling.k8s.io/v1/priorityclasses": "D3vHs+OgrtA=", + "storage.k8s.io/v1/storageclasses": "K+m6uJwbjGY=", + "storage.k8s.io/v1/volumeattachments": "vQAqD28V4AY=", + "storage.k8s.io/v1beta1/csidrivers": "hL6j/rwBV5w=", + "storage.k8s.io/v1beta1/csinodes": "Pe62DkZtjuo=", + "storage.k8s.io/v1beta1/storageclasses": "K+m6uJwbjGY=", + "storage.k8s.io/v1beta1/volumeattachments": "vQAqD28V4AY=", + "apps/v1/controllerrevisions": "85nkx63pcBU=", + "apps/v1/daemonsets": "dd7pWHUlMKQ=", + "apps/v1/deployments": "8aSe+NMegvE=", + "apps/v1/replicasets": "P1RzHs8/mWQ=", + "apps/v1/statefulsets": "H+vl74LkKdo=", + "apps/v1beta2/controllerrevisions": "85nkx63pcBU=", + "apps/v1beta2/daemonsets": "dd7pWHUlMKQ=", + "apps/v1beta2/deployments": "8aSe+NMegvE=", + "apps/v1beta2/replicasets": "P1RzHs8/mWQ=", + "apps/v1beta2/statefulsets": "H+vl74LkKdo=", + "apps/v1beta1/controllerrevisions": "85nkx63pcBU=", + "apps/v1beta1/deployments": "8aSe+NMegvE=", + "apps/v1beta1/statefulsets": "H+vl74LkKdo=", + "admissionregistration.k8s.io/v1beta1/mutatingwebhookconfigurations": "yxW1cpLtfp8=", + "admissionregistration.k8s.io/v1beta1/validatingwebhookconfigurations": "P9NhrezfnWE=", + "events.k8s.io/v1beta1/events": "r2yiGXH7wu8=", +} diff --git a/pkg/registry/core/namespace/storage/storage.go b/pkg/registry/core/namespace/storage/storage.go index c8f0c278cb..2fe25f1ee5 100644 --- a/pkg/registry/core/namespace/storage/storage.go +++ b/pkg/registry/core/namespace/storage/storage.go @@ -226,6 +226,12 @@ func (r *REST) ShortNames() []string { return []string{"ns"} } +var _ rest.StorageVersionProvider = &REST{} + +func (r *REST) StorageVersion() runtime.GroupVersioner { + return r.store.StorageVersion() +} + func (r *StatusREST) New() runtime.Object { return r.store.New() } diff --git a/pkg/registry/core/service/storage/rest.go b/pkg/registry/core/service/storage/rest.go index fe337d247f..4ed99d1791 100644 --- a/pkg/registry/core/service/storage/rest.go +++ b/pkg/registry/core/service/storage/rest.go @@ -77,6 +77,7 @@ type ServiceStorage interface { rest.Watcher rest.TableConvertor rest.Exporter + rest.StorageVersionProvider } type EndpointsStorage interface { @@ -108,11 +109,16 @@ func NewREST( } var ( - _ ServiceStorage = &REST{} - _ rest.CategoriesProvider = &REST{} - _ rest.ShortNamesProvider = &REST{} + _ ServiceStorage = &REST{} + _ rest.CategoriesProvider = &REST{} + _ rest.ShortNamesProvider = &REST{} + _ rest.StorageVersionProvider = &REST{} ) +func (rs *REST) StorageVersion() runtime.GroupVersioner { + return rs.services.StorageVersion() +} + // ShortNames implements the ShortNamesProvider interface. Returns a list of short names for a resource. func (rs *REST) ShortNames() []string { return []string{"svc"} diff --git a/pkg/registry/core/service/storage/rest_test.go b/pkg/registry/core/service/storage/rest_test.go index e7f4560b5f..b226a87eac 100644 --- a/pkg/registry/core/service/storage/rest_test.go +++ b/pkg/registry/core/service/storage/rest_test.go @@ -159,6 +159,10 @@ func (s *serviceStorage) Export(ctx context.Context, name string, opts metav1.Ex panic("not implemented") } +func (s *serviceStorage) StorageVersion() runtime.GroupVersioner { + panic("not implemented") +} + func generateRandomNodePort() int32 { return int32(rand.IntnRange(30001, 30999)) } diff --git a/pkg/registry/rbac/clusterrole/policybased/storage.go b/pkg/registry/rbac/clusterrole/policybased/storage.go index 4505d9aba6..e3a9dd6ff2 100644 --- a/pkg/registry/rbac/clusterrole/policybased/storage.go +++ b/pkg/registry/rbac/clusterrole/policybased/storage.go @@ -50,6 +50,16 @@ func (r *Storage) NamespaceScoped() bool { return false } +func (r *Storage) StorageVersion() runtime.GroupVersioner { + svp, ok := r.StandardStorage.(rest.StorageVersionProvider) + if !ok { + return nil + } + return svp.StorageVersion() +} + +var _ rest.StorageVersionProvider = &Storage{} + var fullAuthority = []rbac.PolicyRule{ rbac.NewRule("*").Groups("*").Resources("*").RuleOrDie(), rbac.NewRule("*").URLs("*").RuleOrDie(), diff --git a/pkg/registry/rbac/clusterrolebinding/policybased/storage.go b/pkg/registry/rbac/clusterrolebinding/policybased/storage.go index 2267394aaa..6a4b2c8f2e 100644 --- a/pkg/registry/rbac/clusterrolebinding/policybased/storage.go +++ b/pkg/registry/rbac/clusterrolebinding/policybased/storage.go @@ -51,6 +51,16 @@ func (r *Storage) NamespaceScoped() bool { return false } +func (r *Storage) StorageVersion() runtime.GroupVersioner { + svp, ok := r.StandardStorage.(rest.StorageVersionProvider) + if !ok { + return nil + } + return svp.StorageVersion() +} + +var _ rest.StorageVersionProvider = &Storage{} + func (s *Storage) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) { if rbacregistry.EscalationAllowed(ctx) { return s.StandardStorage.Create(ctx, obj, createValidation, options) diff --git a/pkg/registry/rbac/role/policybased/storage.go b/pkg/registry/rbac/role/policybased/storage.go index 25a622fe07..34281be975 100644 --- a/pkg/registry/rbac/role/policybased/storage.go +++ b/pkg/registry/rbac/role/policybased/storage.go @@ -49,6 +49,16 @@ func (r *Storage) NamespaceScoped() bool { return true } +func (r *Storage) StorageVersion() runtime.GroupVersioner { + svp, ok := r.StandardStorage.(rest.StorageVersionProvider) + if !ok { + return nil + } + return svp.StorageVersion() +} + +var _ rest.StorageVersionProvider = &Storage{} + func (s *Storage) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) { if rbacregistry.EscalationAllowed(ctx) || rbacregistry.RoleEscalationAuthorized(ctx, s.authorizer) { return s.StandardStorage.Create(ctx, obj, createValidation, options) diff --git a/pkg/registry/rbac/rolebinding/policybased/storage.go b/pkg/registry/rbac/rolebinding/policybased/storage.go index 2a0603e740..d73a1e1f93 100644 --- a/pkg/registry/rbac/rolebinding/policybased/storage.go +++ b/pkg/registry/rbac/rolebinding/policybased/storage.go @@ -52,6 +52,16 @@ func (r *Storage) NamespaceScoped() bool { return true } +func (r *Storage) StorageVersion() runtime.GroupVersioner { + svp, ok := r.StandardStorage.(rest.StorageVersionProvider) + if !ok { + return nil + } + return svp.StorageVersion() +} + +var _ rest.StorageVersionProvider = &Storage{} + func (s *Storage) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) { if rbacregistry.EscalationAllowed(ctx) { return s.StandardStorage.Create(ctx, obj, createValidation, options) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go index 86203b2ba0..f1592444a4 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_discovery_controller.go @@ -95,6 +95,7 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error { } foundThisVersion := false + var storageVersionHash string for _, v := range crd.Spec.Versions { if !v.Served { continue @@ -113,6 +114,9 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error { if v.Name == version.Version { foundThisVersion = true } + if v.Storage { + storageVersionHash = discovery.StorageVersionHash(gv.Group, gv.Version, crd.Spec.Names.Kind) + } } if !foundThisVersion { @@ -127,13 +131,14 @@ func (c *DiscoveryController) sync(version schema.GroupVersion) error { } apiResourcesForDiscovery = append(apiResourcesForDiscovery, metav1.APIResource{ - Name: crd.Status.AcceptedNames.Plural, - SingularName: crd.Status.AcceptedNames.Singular, - Namespaced: crd.Spec.Scope == apiextensions.NamespaceScoped, - Kind: crd.Status.AcceptedNames.Kind, - Verbs: verbs, - ShortNames: crd.Status.AcceptedNames.ShortNames, - Categories: crd.Status.AcceptedNames.Categories, + Name: crd.Status.AcceptedNames.Plural, + SingularName: crd.Status.AcceptedNames.Singular, + Namespaced: crd.Spec.Scope == apiextensions.NamespaceScoped, + Kind: crd.Status.AcceptedNames.Kind, + Verbs: verbs, + ShortNames: crd.Status.AcceptedNames.ShortNames, + Categories: crd.Status.AcceptedNames.Categories, + StorageVersionHash: storageVersionHash, }) subresources, err := apiextensions.GetSubresourcesForVersion(crd, version.Version) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/storageversionhash.go b/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/storageversionhash.go new file mode 100644 index 0000000000..a1b00decba --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/storageversionhash.go @@ -0,0 +1,40 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package discovery + +import ( + "crypto/sha256" + "encoding/base64" +) + +// StorageVersionHash calculates the storage version hash for a +// tuple. +// WARNING: this function is subject to change. Clients shouldn't depend on +// this function. +func StorageVersionHash(group, version, kind string) string { + gvk := group + "/" + version + "/" + kind + if gvk == "" { + return "" + } + bytes := sha256.Sum256([]byte(gvk)) + // Assuming there are N kinds in the cluster, and the hash is X-byte long, + // the chance of colliding hash P(N,X) approximates to 1-e^(-(N^2)/2^(8X+1)). + // P(10,000, 8) ~= 2.7*10^(-12), which is low enough. + // See https://en.wikipedia.org/wiki/Birthday_problem#Approximations. + return base64.StdEncoding.EncodeToString( + bytes[:8]) +} diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go index 8440624329..c48c2d61f6 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/installer.go @@ -33,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/admission" + "k8s.io/apiserver/pkg/endpoints/discovery" "k8s.io/apiserver/pkg/endpoints/handlers" "k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager" "k8s.io/apiserver/pkg/endpoints/handlers/negotiation" @@ -133,6 +134,20 @@ func (a *APIInstaller) newWebService() *restful.WebService { return ws } +// calculate the storage gvk, the gvk objects are converted to before persisted to the etcd. +func getStorageVersionKind(storageVersioner runtime.GroupVersioner, storage rest.Storage, typer runtime.ObjectTyper) (schema.GroupVersionKind, error) { + object := storage.New() + fqKinds, _, err := typer.ObjectKinds(object) + if err != nil { + return schema.GroupVersionKind{}, err + } + gvk, ok := storageVersioner.KindForGroupVersionKinds(fqKinds) + if !ok { + return schema.GroupVersionKind{}, fmt.Errorf("cannot find the storage version kind for %v", reflect.TypeOf(object)) + } + return gvk, nil +} + // GetResourceKind returns the external group version kind registered for the given storage // object. If the storage object is a subresource and has an override supplied for it, it returns // the group version kind supplied in the override. @@ -227,6 +242,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag watcher, isWatcher := storage.(rest.Watcher) connecter, isConnecter := storage.(rest.Connecter) storageMeta, isMetadata := storage.(rest.StorageMetadata) + storageVersionProvider, isStorageVersionProvider := storage.(rest.StorageVersionProvider) if !isMetadata { storageMeta = defaultStorageMetadata{} } @@ -365,6 +381,17 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag tableProvider, _ := storage.(rest.TableConvertor) var apiResource metav1.APIResource + if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionHash) && + isStorageVersionProvider && + storageVersionProvider.StorageVersion() != nil { + versioner := storageVersionProvider.StorageVersion() + gvk, err := getStorageVersionKind(versioner, storage, a.group.Typer) + if err != nil { + return nil, err + } + apiResource.StorageVersionHash = discovery.StorageVersionHash(gvk.Group, gvk.Version, gvk.Kind) + } + // Get the list of actions for the given scope. switch { case !namespaceScoped: diff --git a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go index e99045896f..587a6dfe54 100644 --- a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go @@ -89,6 +89,13 @@ const ( // Server-side apply. Merging happens on the server. ServerSideApply utilfeature.Feature = "ServerSideApply" + // owner: @caesarxuchao + // alpha: v1.14 + // + // Allow apiservers to expose the storage version hash in the discovery + // document. + StorageVersionHash utilfeature.Feature = "StorageVersionHash" + // owner: @ksubrmnn // alpha: v1.14 // @@ -118,6 +125,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS APIListChunking: {Default: true, PreRelease: utilfeature.Beta}, DryRun: {Default: true, PreRelease: utilfeature.Beta}, ServerSideApply: {Default: false, PreRelease: utilfeature.Alpha}, + StorageVersionHash: {Default: false, PreRelease: utilfeature.Alpha}, WinOverlay: {Default: false, PreRelease: utilfeature.Alpha}, WinDSR: {Default: false, PreRelease: utilfeature.Alpha}, } diff --git a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go index 5489596e63..c84a3276c4 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/generic/registry/store.go @@ -177,6 +177,12 @@ type Store struct { // resource. It is wrapped into a "DryRunnableStorage" that will // either pass-through or simply dry-run. Storage DryRunnableStorage + // StorageVersioner outputs the an object will be + // converted to before persisted in etcd, given a list of possible + // kinds of the object. + // If the StorageVersioner is nil, apiserver will leave the + // storageVersionHash as empty in the discovery document. + StorageVersioner runtime.GroupVersioner // Called to cleanup clients used by the underlying Storage; optional. DestroyFunc func() } @@ -1287,6 +1293,7 @@ func (e *Store) CompleteWithOptions(options *generic.StoreOptions) error { attrFunc, triggerFunc, ) + e.StorageVersioner = opts.StorageConfig.EncodeVersioner if opts.CountMetricPollPeriod > 0 { stopFunc := e.startObservingCount(opts.CountMetricPollPeriod) @@ -1327,3 +1334,7 @@ func (e *Store) ConvertToTable(ctx context.Context, object runtime.Object, table } return rest.NewDefaultTableConvertor(e.qualifiedResourceFromContext(ctx)).ConvertToTable(ctx, object, tableOptions) } + +func (e *Store) StorageVersion() runtime.GroupVersioner { + return e.StorageVersioner +} diff --git a/staging/src/k8s.io/apiserver/pkg/registry/rest/rest.go b/staging/src/k8s.io/apiserver/pkg/registry/rest/rest.go index 9fad541348..b16f7f677b 100644 --- a/staging/src/k8s.io/apiserver/pkg/registry/rest/rest.go +++ b/staging/src/k8s.io/apiserver/pkg/registry/rest/rest.go @@ -332,3 +332,12 @@ type StorageMetadata interface { // it is not nil. Only the type of the return object matters, the value will be ignored. ProducesObject(verb string) interface{} } + +// StorageVersionProvider is an optional interface that a storage object can +// implement if it wishes to disclose its storage version. +type StorageVersionProvider interface { + // StorageVersion returns a group versioner, which will outputs the gvk + // an object will be converted to before persisted in etcd, given a + // list of kinds the object might belong to. + StorageVersion() runtime.GroupVersioner +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_codec.go b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_codec.go index e2f91bf13d..96faa17122 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_codec.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_codec.go @@ -40,15 +40,15 @@ type StorageCodecConfig struct { // NewStorageCodec assembles a storage codec for the provided storage media type, the provided serializer, and the requested // storage and memory versions. -func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, error) { +func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, runtime.GroupVersioner, error) { mediaType, _, err := mime.ParseMediaType(opts.StorageMediaType) if err != nil { - return nil, fmt.Errorf("%q is not a valid mime-type", opts.StorageMediaType) + return nil, nil, fmt.Errorf("%q is not a valid mime-type", opts.StorageMediaType) } serializer, ok := runtime.SerializerInfoForMediaType(opts.StorageSerializer.SupportedMediaTypes(), mediaType) if !ok { - return nil, fmt.Errorf("unable to find serializer for %q", mediaType) + return nil, nil, fmt.Errorf("unable to find serializer for %q", mediaType) } s := serializer.Serializer @@ -74,14 +74,16 @@ func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, error) { decoders = opts.DecoderDecoratorFn(decoders) } + encodeVersioner := runtime.NewMultiGroupVersioner( + opts.StorageVersion, + schema.GroupKind{Group: opts.StorageVersion.Group}, + schema.GroupKind{Group: opts.MemoryVersion.Group}, + ) + // Ensure the storage receives the correct version. encoder = opts.StorageSerializer.EncoderForVersion( encoder, - runtime.NewMultiGroupVersioner( - opts.StorageVersion, - schema.GroupKind{Group: opts.StorageVersion.Group}, - schema.GroupKind{Group: opts.MemoryVersion.Group}, - ), + encodeVersioner, ) decoder := opts.StorageSerializer.DecoderToVersion( recognizer.NewDecoder(decoders...), @@ -92,5 +94,5 @@ func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, error) { ), ) - return runtime.NewCodec(encoder, decoder), nil + return runtime.NewCodec(encoder, decoder), encodeVersioner, nil } diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go index c3bb6ecd6d..267de1370b 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go @@ -86,7 +86,7 @@ type DefaultStorageFactory struct { APIResourceConfigSource APIResourceConfigSource // newStorageCodecFn exists to be overwritten for unit testing. - newStorageCodecFn func(opts StorageCodecConfig) (codec runtime.Codec, err error) + newStorageCodecFn func(opts StorageCodecConfig) (codec runtime.Codec, encodeVersioner runtime.GroupVersioner, err error) } type groupResourceOverrides struct { @@ -278,7 +278,7 @@ func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (* } codecConfig.Config = storageConfig - storageConfig.Codec, err = s.newStorageCodecFn(codecConfig) + storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go index c36a103942..d4bc7fb49d 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go @@ -57,6 +57,11 @@ type Config struct { Paging bool Codec runtime.Codec + // EncodeVersioner is the same groupVersioner used to build the + // storage encoder. Given a list of kinds the input object might belong + // to, the EncodeVersioner outputs the gvk the object will be + // converted to before persisted in etcd. + EncodeVersioner runtime.GroupVersioner // Transformer allows the value to be transformed prior to persisting into etcd. Transformer value.Transformer diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go index 48aa538ba0..4b2b19b7a8 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go @@ -23,6 +23,8 @@ import ( "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apiserver/pkg/admission" genericapiserver "k8s.io/apiserver/pkg/server" @@ -56,7 +58,7 @@ func NewWardleServerOptions(out, errOut io.Writer) *WardleServerOptions { StdOut: out, StdErr: errOut, } - + o.RecommendedOptions.Etcd.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1alpha1.SchemeGroupVersion, schema.GroupKind{Group: v1alpha1.GroupName}) return o } diff --git a/test/e2e/apimachinery/discovery.go b/test/e2e/apimachinery/discovery.go new file mode 100644 index 0000000000..b2e42db58e --- /dev/null +++ b/test/e2e/apimachinery/discovery.go @@ -0,0 +1,78 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apimachinery + +import ( + utilversion "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apiserver/pkg/endpoints/discovery" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" +) + +var storageVersionServerVersion = utilversion.MustParseSemantic("v1.13.99") +var _ = SIGDescribe("Discovery", func() { + f := framework.NewDefaultFramework("discovery") + + var namespaceName string + + BeforeEach(func() { + namespaceName = f.Namespace.Name + + framework.SkipUnlessServerVersionGTE(storageVersionServerVersion, f.ClientSet.Discovery()) + + By("Setting up server cert") + setupServerCert(namespaceName, serviceName) + }) + + It("[Feature:StorageVersionHash] Custom resource should have storage version hash", func() { + testcrd, err := framework.CreateTestCRD(f) + if err != nil { + return + } + defer testcrd.CleanUp() + spec := testcrd.Crd.Spec + resources, err := testcrd.ApiExtensionClient.Discovery().ServerResourcesForGroupVersion(spec.Group + "/" + spec.Versions[0].Name) + if err != nil { + framework.Failf("failed to find the discovery doc for %v: %v", resources, err) + } + found := false + var storageVersion string + for _, v := range spec.Versions { + if v.Storage { + storageVersion = v.Name + } + } + // DISCLAIMER: the algorithm of deriving the storageVersionHash + // is an implementation detail, which shouldn't be relied on by + // the clients. The following calculation is for test purpose + // only. + expected := discovery.StorageVersionHash(spec.Group, storageVersion, spec.Names.Kind) + + for _, r := range resources.APIResources { + if r.Name == spec.Names.Plural { + found = true + if r.StorageVersionHash != expected { + framework.Failf("expected storageVersionHash of %s/%s/%s to be %s, got %s", r.Group, r.Version, r.Name, expected, r.StorageVersionHash) + } + } + } + if !found { + framework.Failf("didn't find resource %s in the discovery doc", spec.Names.Plural) + } + }) +}) From 0ace8b3da4a732f07443e6ba3f7d11afdd0e4fe0 Mon Sep 17 00:00:00 2001 From: Chao Xu Date: Tue, 12 Feb 2019 14:21:31 -0800 Subject: [PATCH 15/16] Let the sample apiserver show feature gates flag --- staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go index 4b2b19b7a8..66409efad6 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/start.go @@ -29,6 +29,7 @@ import ( "k8s.io/apiserver/pkg/admission" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/sample-apiserver/pkg/admission/plugin/banflunder" "k8s.io/sample-apiserver/pkg/admission/wardleinitializer" "k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1" @@ -85,6 +86,7 @@ func NewCommandStartWardleServer(defaults *WardleServerOptions, stopCh <-chan st flags := cmd.Flags() o.RecommendedOptions.AddFlags(flags) + utilfeature.DefaultMutableFeatureGate.AddFlag(flags) return cmd } From 887cb93d8d74ac6491df850edef051bf607fceb6 Mon Sep 17 00:00:00 2001 From: Chao Xu Date: Tue, 15 Jan 2019 13:41:53 -0800 Subject: [PATCH 16/16] generated BUILD generated proto --- cmd/kube-apiserver/app/BUILD | 1 + pkg/master/BUILD | 11 +++++++++ pkg/master/storageversionhashdata/BUILD | 23 +++++++++++++++++++ .../apiserver/pkg/endpoints/discovery/BUILD | 1 + .../sample-apiserver/pkg/cmd/server/BUILD | 3 +++ test/e2e/apimachinery/BUILD | 2 ++ 6 files changed, 41 insertions(+) create mode 100644 pkg/master/storageversionhashdata/BUILD diff --git a/cmd/kube-apiserver/app/BUILD b/cmd/kube-apiserver/app/BUILD index 7830ca5d08..9b55641912 100644 --- a/cmd/kube-apiserver/app/BUILD +++ b/cmd/kube-apiserver/app/BUILD @@ -39,6 +39,7 @@ go_library( "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", diff --git a/pkg/master/BUILD b/pkg/master/BUILD index 28bf854721..0bec722bf3 100644 --- a/pkg/master/BUILD +++ b/pkg/master/BUILD @@ -141,10 +141,13 @@ go_test( deps = [ "//pkg/api/legacyscheme:go_default_library", "//pkg/api/testapi:go_default_library", + "//pkg/apis/batch:go_default_library", "//pkg/apis/core:go_default_library", + "//pkg/apis/storage:go_default_library", "//pkg/generated/openapi:go_default_library", "//pkg/kubelet/client:go_default_library", "//pkg/master/reconcilers:go_default_library", + "//pkg/master/storageversionhashdata:go_default_library", "//pkg/registry/certificates/rest:go_default_library", "//pkg/registry/core/rest:go_default_library", "//pkg/registry/registrytest:go_default_library", @@ -154,16 +157,23 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/api/apitesting/naming:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/version:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/openapi:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/features:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/server/resourceconfig:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", + "//staging/src/k8s.io/client-go/discovery:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", @@ -191,6 +201,7 @@ filegroup( "//pkg/master/controller/crdregistration:all-srcs", "//pkg/master/ports:all-srcs", "//pkg/master/reconcilers:all-srcs", + "//pkg/master/storageversionhashdata:all-srcs", "//pkg/master/tunneler:all-srcs", ], tags = ["automanaged"], diff --git a/pkg/master/storageversionhashdata/BUILD b/pkg/master/storageversionhashdata/BUILD new file mode 100644 index 0000000000..7eca8cf1e5 --- /dev/null +++ b/pkg/master/storageversionhashdata/BUILD @@ -0,0 +1,23 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["data.go"], + importpath = "k8s.io/kubernetes/pkg/master/storageversionhashdata", + visibility = ["//visibility:public"], + deps = ["//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/BUILD index f6395aeca8..d80a60ea96 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/discovery/BUILD @@ -32,6 +32,7 @@ go_library( "group.go", "legacy.go", "root.go", + "storageversionhash.go", "util.go", "version.go", ], diff --git a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/BUILD b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/BUILD index 9f704e55c1..0a71e2a768 100644 --- a/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/BUILD +++ b/staging/src/k8s.io/sample-apiserver/pkg/cmd/server/BUILD @@ -11,10 +11,13 @@ go_library( importmap = "k8s.io/kubernetes/vendor/k8s.io/sample-apiserver/pkg/cmd/server", importpath = "k8s.io/sample-apiserver/pkg/cmd/server", deps = [ + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/sample-apiserver/pkg/admission/plugin/banflunder:go_default_library", "//staging/src/k8s.io/sample-apiserver/pkg/admission/wardleinitializer:go_default_library", "//staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1:go_default_library", diff --git a/test/e2e/apimachinery/BUILD b/test/e2e/apimachinery/BUILD index 136dd18f70..e32bde872d 100644 --- a/test/e2e/apimachinery/BUILD +++ b/test/e2e/apimachinery/BUILD @@ -15,6 +15,7 @@ go_library( "crd_publish_openapi.go", "crd_watch.go", "custom_resource_definition.go", + "discovery.go", "etcd_failure.go", "framework.go", "garbage_collector.go", @@ -65,6 +66,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/endpoints/discovery:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", "//staging/src/k8s.io/client-go/discovery:go_default_library",