From aab3523e0ec97c73e62d3a2d3b8340c52caa3610 Mon Sep 17 00:00:00 2001 From: Tim Allclair Date: Wed, 23 Jan 2019 14:26:02 -0800 Subject: [PATCH 1/3] Migrate RuntimeClass support to the generated typed client --- cmd/kubelet/app/BUILD | 2 +- cmd/kubelet/app/server.go | 21 +++--- pkg/kubelet/BUILD | 2 +- pkg/kubelet/kubelet.go | 10 +-- .../kuberuntime/kuberuntime_sandbox_test.go | 4 +- pkg/kubelet/runtimeclass/BUILD | 8 +-- .../runtimeclass/runtimeclass_manager.go | 67 ++++++++---------- .../runtimeclass/runtimeclass_manager_test.go | 6 +- pkg/kubelet/runtimeclass/testing/BUILD | 11 ++- .../runtimeclass/testing/fake_manager.go | 69 ++++++++----------- 10 files changed, 86 insertions(+), 114 deletions(-) diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index 59e5a93605..82724f8d7f 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -123,7 +123,6 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library", - "//staging/src/k8s.io/client-go/dynamic:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/authentication/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/authorization/v1beta1:go_default_library", @@ -137,6 +136,7 @@ go_library( "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", + "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library", "//vendor/github.com/coreos/go-systemd/daemon:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index c84979fd61..da09e0f239 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -48,7 +48,6 @@ import ( "k8s.io/apiserver/pkg/server/healthz" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/flag" - "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" v1core "k8s.io/client-go/kubernetes/typed/core/v1" @@ -96,6 +95,7 @@ import ( "k8s.io/kubernetes/pkg/util/rlimit" "k8s.io/kubernetes/pkg/version" "k8s.io/kubernetes/pkg/version/verflag" + nodeapiclientset "k8s.io/node-api/pkg/client/clientset/versioned" "k8s.io/utils/exec" ) @@ -542,12 +542,11 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan switch { case standaloneMode: kubeDeps.KubeClient = nil - kubeDeps.DynamicKubeClient = nil kubeDeps.EventClient = nil kubeDeps.HeartbeatClient = nil klog.Warningf("standalone mode, no API client") - case kubeDeps.KubeClient == nil, kubeDeps.EventClient == nil, kubeDeps.HeartbeatClient == nil, kubeDeps.DynamicKubeClient == nil: + case kubeDeps.KubeClient == nil, kubeDeps.EventClient == nil, kubeDeps.HeartbeatClient == nil: clientConfig, closeAllConns, err := buildKubeletClientConfig(s, nodeName) if err != nil { return err @@ -559,11 +558,6 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan return fmt.Errorf("failed to initialize kubelet client: %v", err) } - kubeDeps.DynamicKubeClient, err = dynamic.NewForConfig(clientConfig) - if err != nil { - return fmt.Errorf("failed to initialize kubelet dynamic client: %v", err) - } - // make a separate client for events eventClientConfig := *clientConfig eventClientConfig.QPS = float32(s.EventRecordQPS) @@ -590,12 +584,17 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan } // CRDs are JSON only, and client renegotiation for streaming is not correct as per #67803 - csiClientConfig := restclient.CopyConfig(clientConfig) - csiClientConfig.ContentType = "application/json" - kubeDeps.CSIClient, err = csiclientset.NewForConfig(csiClientConfig) + crdClientConfig := restclient.CopyConfig(clientConfig) + crdClientConfig.ContentType = "application/json" + kubeDeps.CSIClient, err = csiclientset.NewForConfig(crdClientConfig) if err != nil { return fmt.Errorf("failed to initialize kubelet storage client: %v", err) } + + kubeDeps.NodeAPIClient, err = nodeapiclientset.NewForConfig(crdClientConfig) + if err != nil { + return fmt.Errorf("failed to initialize kubelet node-api client: %v", err) + } } // If the kubelet config controller is available, and dynamic config is enabled, start the config and status sync loops diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index 61e9e649ed..b708314fcd 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -132,7 +132,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//staging/src/k8s.io/client-go/dynamic:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", @@ -143,6 +142,7 @@ go_library( "//staging/src/k8s.io/client-go/util/integer:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", + "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library", "//third_party/forked/golang/expansion:go_default_library", "//vendor/github.com/golang/groupcache/lru:go_default_library", "//vendor/github.com/google/cadvisor/events:go_default_library", diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 45ae42629f..44ea48c67d 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -43,7 +43,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" v1core "k8s.io/client-go/kubernetes/typed/core/v1" corelisters "k8s.io/client-go/listers/core/v1" @@ -115,6 +114,7 @@ import ( "k8s.io/kubernetes/pkg/util/oom" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/csi" + nodeapiclientset "k8s.io/node-api/pkg/client/clientset/versioned" utilexec "k8s.io/utils/exec" ) @@ -249,7 +249,7 @@ type Dependencies struct { OnHeartbeatFailure func() KubeClient clientset.Interface CSIClient csiclientset.Interface - DynamicKubeClient dynamic.Interface + NodeAPIClient nodeapiclientset.Interface Mounter mount.Interface OOMAdjuster *oom.OOMAdjuster OSInterface kubecontainer.OSInterface @@ -658,8 +658,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, } klet.runtimeService = runtimeService - if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) && kubeDeps.DynamicKubeClient != nil { - klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.DynamicKubeClient) + if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) && kubeDeps.NodeAPIClient != nil { + klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.NodeAPIClient) } runtime, err := kuberuntime.NewKubeGenericRuntimeManager( @@ -1426,7 +1426,7 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) { // Start syncing RuntimeClasses if enabled. if kl.runtimeClassManager != nil { - go kl.runtimeClassManager.Run(wait.NeverStop) + kl.runtimeClassManager.Start(wait.NeverStop) } // Start the pod lifecycle event generator. diff --git a/pkg/kubelet/kuberuntime/kuberuntime_sandbox_test.go b/pkg/kubelet/kuberuntime/kuberuntime_sandbox_test.go index 6d23710e8b..02854dba10 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_sandbox_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_sandbox_test.go @@ -61,8 +61,8 @@ func TestCreatePodSandbox(t *testing.T) { func TestCreatePodSandbox_RuntimeClass(t *testing.T) { defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RuntimeClass, true)() - rcm := runtimeclass.NewManager(rctest.NewPopulatedDynamicClient()) - defer rctest.StartManagerSync(t, rcm)() + rcm := runtimeclass.NewManager(rctest.NewPopulatedClient()) + defer rctest.StartManagerSync(rcm)() fakeRuntime, _, m, err := createTestRuntimeManager() require.NoError(t, err) diff --git a/pkg/kubelet/runtimeclass/BUILD b/pkg/kubelet/runtimeclass/BUILD index 6b7dcdde4b..8e59955f3c 100644 --- a/pkg/kubelet/runtimeclass/BUILD +++ b/pkg/kubelet/runtimeclass/BUILD @@ -7,12 +7,10 @@ go_library( visibility = ["//visibility:public"], deps = [ "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/client-go/dynamic:go_default_library", - "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library", + "//staging/src/k8s.io/node-api/pkg/client/informers/externalversions:go_default_library", + "//staging/src/k8s.io/node-api/pkg/client/listers/node/v1alpha1:go_default_library", ], ) diff --git a/pkg/kubelet/runtimeclass/runtimeclass_manager.go b/pkg/kubelet/runtimeclass/runtimeclass_manager.go index 892240aebd..b65d5b9bf1 100644 --- a/pkg/kubelet/runtimeclass/runtimeclass_manager.go +++ b/pkg/kubelet/runtimeclass/runtimeclass_manager.go @@ -20,12 +20,10 @@ import ( "fmt" "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/tools/cache" + nodeapiclient "k8s.io/node-api/pkg/client/clientset/versioned" + nodeapiinformer "k8s.io/node-api/pkg/client/informers/externalversions" + nodev1alpha1 "k8s.io/node-api/pkg/client/listers/node/v1alpha1" ) var ( @@ -38,33 +36,36 @@ var ( // Manager caches RuntimeClass API objects, and provides accessors to the Kubelet. type Manager struct { - informer cache.SharedInformer + informerFactory nodeapiinformer.SharedInformerFactory + lister nodev1alpha1.RuntimeClassLister } // NewManager returns a new RuntimeClass Manager. Run must be called before the manager can be used. -func NewManager(client dynamic.Interface) *Manager { - rc := client.Resource(runtimeClassGVR) - lw := &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return rc.List(options) - }, - WatchFunc: rc.Watch, - } - informer := cache.NewSharedInformer(lw, &unstructured.Unstructured{}, 0) +func NewManager(client nodeapiclient.Interface) *Manager { + const resyncPeriod = 0 + factory := nodeapiinformer.NewSharedInformerFactory(client, resyncPeriod) + lister := factory.Node().V1alpha1().RuntimeClasses().Lister() return &Manager{ - informer: informer, + informerFactory: factory, + lister: lister, } } -// Run starts syncing the RuntimeClass cache with the apiserver. -func (m *Manager) Run(stopCh <-chan struct{}) { - m.informer.Run(stopCh) +// Start starts syncing the RuntimeClass cache with the apiserver. +func (m *Manager) Start(stopCh <-chan struct{}) { + m.informerFactory.Start(stopCh) +} + +// WaitForCacheSync exposes the WaitForCacheSync method on the informer factory for testing +// purposes. +func (m *Manager) WaitForCacheSync(stopCh <-chan struct{}) { + m.informerFactory.WaitForCacheSync(stopCh) } // LookupRuntimeHandler returns the RuntimeHandler string associated with the given RuntimeClass // name (or the default of "" for nil). If the RuntimeClass is not found, it returns an -// apierrors.NotFound error. +// errors.NotFound error. func (m *Manager) LookupRuntimeHandler(runtimeClassName *string) (string, error) { if runtimeClassName == nil || *runtimeClassName == "" { // The default RuntimeClass always resolves to the empty runtime handler. @@ -72,26 +73,18 @@ func (m *Manager) LookupRuntimeHandler(runtimeClassName *string) (string, error) } name := *runtimeClassName - item, exists, err := m.informer.GetStore().GetByKey(name) + + rc, err := m.lister.Get(name) if err != nil { + if errors.IsNotFound(err) { + return "", err + } return "", fmt.Errorf("Failed to lookup RuntimeClass %s: %v", name, err) } - if !exists { - return "", errors.NewNotFound(schema.GroupResource{ - Group: runtimeClassGVR.Group, - Resource: runtimeClassGVR.Resource, - }, name) - } - rc, ok := item.(*unstructured.Unstructured) - if !ok { - return "", fmt.Errorf("unexpected RuntimeClass type %T", item) + handler := rc.Spec.RuntimeHandler + if handler == nil { + return "", nil } - - handler, _, err := unstructured.NestedString(rc.Object, "spec", "runtimeHandler") - if err != nil { - return "", fmt.Errorf("Invalid RuntimeClass object: %v", err) - } - - return handler, nil + return *handler, nil } diff --git a/pkg/kubelet/runtimeclass/runtimeclass_manager_test.go b/pkg/kubelet/runtimeclass/runtimeclass_manager_test.go index 7b7af8a0dc..004157a112 100644 --- a/pkg/kubelet/runtimeclass/runtimeclass_manager_test.go +++ b/pkg/kubelet/runtimeclass/runtimeclass_manager_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "k8s.io/kubernetes/pkg/kubelet/runtimeclass" rctest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing" "k8s.io/utils/pointer" @@ -36,12 +37,11 @@ func TestLookupRuntimeHandler(t *testing.T) { {rcn: pointer.StringPtr(""), expected: ""}, {rcn: pointer.StringPtr(rctest.EmptyRuntimeClass), expected: ""}, {rcn: pointer.StringPtr(rctest.SandboxRuntimeClass), expected: "kata-containers"}, - {rcn: pointer.StringPtr(rctest.InvalidRuntimeClass), expectError: true}, {rcn: pointer.StringPtr("phantom"), expectError: true}, } - manager := runtimeclass.NewManager(rctest.NewPopulatedDynamicClient()) - defer rctest.StartManagerSync(t, manager)() + manager := runtimeclass.NewManager(rctest.NewPopulatedClient()) + defer rctest.StartManagerSync(manager)() for _, test := range tests { tname := "nil" diff --git a/pkg/kubelet/runtimeclass/testing/BUILD b/pkg/kubelet/runtimeclass/testing/BUILD index 9774d3492b..f9609f323f 100644 --- a/pkg/kubelet/runtimeclass/testing/BUILD +++ b/pkg/kubelet/runtimeclass/testing/BUILD @@ -7,14 +7,11 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/kubelet/runtimeclass:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/client-go/dynamic:go_default_library", - "//staging/src/k8s.io/client-go/dynamic/fake:go_default_library", - "//vendor/github.com/stretchr/testify/require:go_default_library", - "//vendor/k8s.io/utils/pointer:go_default_library", + "//staging/src/k8s.io/node-api/pkg/apis/node/v1alpha1:go_default_library", + "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library", + "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned/fake:go_default_library", ], ) diff --git a/pkg/kubelet/runtimeclass/testing/fake_manager.go b/pkg/kubelet/runtimeclass/testing/fake_manager.go index 79ecfe5bfe..a7b8d23c5e 100644 --- a/pkg/kubelet/runtimeclass/testing/fake_manager.go +++ b/pkg/kubelet/runtimeclass/testing/fake_manager.go @@ -17,18 +17,12 @@ limitations under the License. package testing import ( - "testing" - "time" - - "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/dynamic" - fakedynamic "k8s.io/client-go/dynamic/fake" "k8s.io/kubernetes/pkg/kubelet/runtimeclass" - "k8s.io/utils/pointer" + nodev1alpha1 "k8s.io/node-api/pkg/apis/node/v1alpha1" + nodeapiclient "k8s.io/node-api/pkg/client/clientset/versioned" + nodeapifake "k8s.io/node-api/pkg/client/clientset/versioned/fake" ) const ( @@ -39,51 +33,42 @@ const ( // EmptyRuntimeClass is a valid RuntimeClass without a handler pre-populated in the populated dynamic client. EmptyRuntimeClass = "native" - // InvalidRuntimeClass is an invalid RuntimeClass pre-populated in the populated dynamic client. - InvalidRuntimeClass = "foo" ) -// NewPopulatedDynamicClient creates a dynamic client for use with the runtimeclass.Manager, +// NewPopulatedClient creates a node-api client for use with the runtimeclass.Manager, // and populates it with a few test RuntimeClass objects. -func NewPopulatedDynamicClient() dynamic.Interface { - invalidRC := NewUnstructuredRuntimeClass(InvalidRuntimeClass, "") - invalidRC.Object["spec"].(map[string]interface{})["runtimeHandler"] = true - - client := fakedynamic.NewSimpleDynamicClient(runtime.NewScheme(), - NewUnstructuredRuntimeClass(EmptyRuntimeClass, ""), - NewUnstructuredRuntimeClass(SandboxRuntimeClass, SandboxRuntimeHandler), - invalidRC, +func NewPopulatedClient() nodeapiclient.Interface { + return nodeapifake.NewSimpleClientset( + NewRuntimeClass(EmptyRuntimeClass, ""), + NewRuntimeClass(SandboxRuntimeClass, SandboxRuntimeHandler), ) - return client } -// StartManagerSync runs the manager, and waits for startup by polling for the expected "native" -// RuntimeClass to be populated. Returns a function to stop the manager, which should be called with -// a defer: +// StartManagerSync starts the manager, and waits for the informer cache to sync. +// Returns a function to stop the manager, which should be called with a defer: // defer StartManagerSync(t, m)() -// Any errors are considered fatal to the test. -func StartManagerSync(t *testing.T, m *runtimeclass.Manager) func() { +func StartManagerSync(m *runtimeclass.Manager) func() { stopCh := make(chan struct{}) - go m.Run(stopCh) - - // Wait for informer to populate. - err := wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { - _, err := m.LookupRuntimeHandler(pointer.StringPtr(EmptyRuntimeClass)) - if err != nil { - if errors.IsNotFound(err) { - return false, nil - } - return false, err - } - return true, nil - }) - require.NoError(t, err, "Failed to start manager") - + m.Start(stopCh) + m.WaitForCacheSync(stopCh) return func() { close(stopCh) } } +// NewRuntimeClass is a helper to generate a RuntimeClass resource with +// the given name & handler. +func NewRuntimeClass(name, handler string) *nodev1alpha1.RuntimeClass { + return &nodev1alpha1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: nodev1alpha1.RuntimeClassSpec{ + RuntimeHandler: &handler, + }, + } +} + // NewUnstructuredRuntimeClass is a helper to generate an unstructured RuntimeClass resource with // the given name & handler. func NewUnstructuredRuntimeClass(name, handler string) *unstructured.Unstructured { From 291d0cbd2f4b114dde46ec94a59243becc1af244 Mon Sep 17 00:00:00 2001 From: Tim Allclair Date: Wed, 23 Jan 2019 17:40:42 -0800 Subject: [PATCH 2/3] Fix deleted RuntimeClass CRD recovery test flake --- test/e2e/node/runtimeclass.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/e2e/node/runtimeclass.go b/test/e2e/node/runtimeclass.go index 692966aa50..ba971ba460 100644 --- a/test/e2e/node/runtimeclass.go +++ b/test/e2e/node/runtimeclass.go @@ -132,7 +132,11 @@ var _ = SIGDescribe("RuntimeClass [Feature:RuntimeClass]", func() { rcName := createRuntimeClass(f, "valid", "") pod := createRuntimeClassPod(f, rcName) - expectPodSuccess(f, pod) + + // Before the pod can be run, the RuntimeClass informer must time out, by which time the Kubelet + // will probably be in a backoff state, so the pod can take a long time to start. + framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow( + f.ClientSet, pod.Name, f.Namespace.Name)) }) // TODO(tallclair): Test an actual configured non-default runtimeHandler. From 954f43a10dea19b1a03482e53bbfae850824718a Mon Sep 17 00:00:00 2001 From: Tim Allclair Date: Thu, 24 Jan 2019 11:08:19 -0800 Subject: [PATCH 3/3] Migrate RuntimeClass E2E to typed client --- pkg/kubelet/runtimeclass/testing/BUILD | 1 - .../runtimeclass/testing/fake_manager.go | 18 ------------------ test/e2e/framework/BUILD | 1 + test/e2e/framework/framework.go | 5 +++++ test/e2e/node/runtimeclass.go | 11 ++++++----- 5 files changed, 12 insertions(+), 24 deletions(-) diff --git a/pkg/kubelet/runtimeclass/testing/BUILD b/pkg/kubelet/runtimeclass/testing/BUILD index f9609f323f..b48519c73a 100644 --- a/pkg/kubelet/runtimeclass/testing/BUILD +++ b/pkg/kubelet/runtimeclass/testing/BUILD @@ -8,7 +8,6 @@ go_library( deps = [ "//pkg/kubelet/runtimeclass:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", "//staging/src/k8s.io/node-api/pkg/apis/node/v1alpha1:go_default_library", "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library", "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned/fake:go_default_library", diff --git a/pkg/kubelet/runtimeclass/testing/fake_manager.go b/pkg/kubelet/runtimeclass/testing/fake_manager.go index a7b8d23c5e..99b19ab8c0 100644 --- a/pkg/kubelet/runtimeclass/testing/fake_manager.go +++ b/pkg/kubelet/runtimeclass/testing/fake_manager.go @@ -18,7 +18,6 @@ package testing import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/kubernetes/pkg/kubelet/runtimeclass" nodev1alpha1 "k8s.io/node-api/pkg/apis/node/v1alpha1" nodeapiclient "k8s.io/node-api/pkg/client/clientset/versioned" @@ -68,20 +67,3 @@ func NewRuntimeClass(name, handler string) *nodev1alpha1.RuntimeClass { }, } } - -// NewUnstructuredRuntimeClass is a helper to generate an unstructured RuntimeClass resource with -// the given name & handler. -func NewUnstructuredRuntimeClass(name, handler string) *unstructured.Unstructured { - return &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "node.k8s.io/v1alpha1", - "kind": "RuntimeClass", - "metadata": map[string]interface{}{ - "name": name, - }, - "spec": map[string]interface{}{ - "runtimeHandler": handler, - }, - }, - } -} diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index fae77f670a..e5dcba85b7 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -128,6 +128,7 @@ go_library( "//staging/src/k8s.io/client-go/util/retry:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset:go_default_library", + "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library", "//test/e2e/framework/ginkgowrapper:go_default_library", "//test/e2e/framework/metrics:go_default_library", "//test/e2e/framework/testfiles:go_default_library", diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 8bd4c931b5..4acc0bf985 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -52,6 +52,7 @@ import ( "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/test/e2e/framework/metrics" testutils "k8s.io/kubernetes/test/utils" + nodeapiclient "k8s.io/node-api/pkg/client/clientset/versioned" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -78,6 +79,7 @@ type Framework struct { KubemarkExternalClusterClientSet clientset.Interface APIExtensionsClientSet apiextensionsclient.Interface CSIClientSet csi.Interface + NodeAPIClientSet nodeapiclient.Interface InternalClientset *internalclientset.Clientset AggregatorClient *aggregatorclient.Clientset @@ -197,6 +199,9 @@ func (f *Framework) BeforeEach() { jsonConfig.ContentType = "application/json" f.CSIClientSet, err = csi.NewForConfig(jsonConfig) Expect(err).NotTo(HaveOccurred()) + // node.k8s.io is also based on CRD + f.NodeAPIClientSet, err = nodeapiclient.NewForConfig(jsonConfig) + Expect(err).NotTo(HaveOccurred()) // create scales getter, set GroupVersion and NegotiatedSerializer to default values // as they are required when creating a REST client. diff --git a/test/e2e/node/runtimeclass.go b/test/e2e/node/runtimeclass.go index ba971ba460..e5fe7f8b32 100644 --- a/test/e2e/node/runtimeclass.go +++ b/test/e2e/node/runtimeclass.go @@ -69,14 +69,15 @@ var _ = SIGDescribe("RuntimeClass [Feature:RuntimeClass]", func() { It("should reject a Pod requesting a deleted RuntimeClass", func() { rcName := createRuntimeClass(f, "delete-me", "") + rcClient := f.NodeAPIClientSet.NodeV1alpha1().RuntimeClasses() By("Deleting RuntimeClass "+rcName, func() { - err := f.DynamicClient.Resource(runtimeClassGVR).Delete(rcName, nil) + err := rcClient.Delete(rcName, nil) framework.ExpectNoError(err, "failed to delete RuntimeClass %s", rcName) By("Waiting for the RuntimeClass to disappear") framework.ExpectNoError(wait.PollImmediate(framework.Poll, time.Minute, func() (bool, error) { - _, err := f.DynamicClient.Resource(runtimeClassGVR).Get(rcName, metav1.GetOptions{}) + _, err := rcClient.Get(rcName, metav1.GetOptions{}) if errors.IsNotFound(err) { return true, nil // done } @@ -143,11 +144,11 @@ var _ = SIGDescribe("RuntimeClass [Feature:RuntimeClass]", func() { }) // createRuntimeClass generates a RuntimeClass with the desired handler and a "namespaced" name, -// synchronously creates it with the dynamic client, and returns the resulting name. +// synchronously creates it, and returns the generated name. func createRuntimeClass(f *framework.Framework, name, handler string) string { uniqueName := fmt.Sprintf("%s-%s", f.Namespace.Name, name) - rc := runtimeclasstest.NewUnstructuredRuntimeClass(uniqueName, handler) - rc, err := f.DynamicClient.Resource(runtimeClassGVR).Create(rc, metav1.CreateOptions{}) + rc := runtimeclasstest.NewRuntimeClass(uniqueName, handler) + rc, err := f.NodeAPIClientSet.NodeV1alpha1().RuntimeClasses().Create(rc) framework.ExpectNoError(err, "failed to create RuntimeClass resource") return rc.GetName() }