To inject ca.crt into container when projected volume was specified, configmap should be created in each namespace.

This patch add a controller called "root-ca-cert-publisher" to complete above job as well as some bootstrap rbac policies.

cmd/kube-controller-manager/app/BUILD

@@ -43,6 +43,7 @@ go_library(
         "//pkg/controller/bootstrap:go_default_library",
         "//pkg/controller/certificates/approver:go_default_library",
         "//pkg/controller/certificates/cleaner:go_default_library",
+        "//pkg/controller/certificates/rootcacertpublisher:go_default_library",
         "//pkg/controller/certificates/signer:go_default_library",
         "//pkg/controller/clusterroleaggregation:go_default_library",
         "//pkg/controller/cronjob:go_default_library",

cmd/kube-controller-manager/app/certificates.go

@@ -29,10 +29,13 @@ import (
 	"net/http"
 
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	kubeoptions "k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
 	"k8s.io/kubernetes/pkg/controller/certificates/approver"
 	"k8s.io/kubernetes/pkg/controller/certificates/cleaner"
+	"k8s.io/kubernetes/pkg/controller/certificates/rootcacertpublisher"
 	"k8s.io/kubernetes/pkg/controller/certificates/signer"
+	"k8s.io/kubernetes/pkg/features"
 )
 
 func startCSRSigningController(ctx ControllerContext) (http.Handler, bool, error) {
@@ -120,3 +123,33 @@ func startCSRCleanerController(ctx ControllerContext) (http.Handler, bool, error
 	go cleaner.Run(1, ctx.Stop)
 	return nil, true, nil
 }
+
+func startRootCACertPublisher(ctx ControllerContext) (http.Handler, bool, error) {
+	if !utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) {
+		return nil, false, nil
+	}
+
+	var (
+		rootCA []byte
+		err    error
+	)
+	if ctx.ComponentConfig.SAController.RootCAFile != "" {
+		if rootCA, err = readCA(ctx.ComponentConfig.SAController.RootCAFile); err != nil {
+			return nil, true, fmt.Errorf("error parsing root-ca-file at %s: %v", ctx.ComponentConfig.SAController.RootCAFile, err)
+		}
+	} else {
+		rootCA = ctx.ClientBuilder.ConfigOrDie("root-ca-cert-publisher").CAData
+	}
+
+	sac, err := rootcacertpublisher.NewPublisher(
+		ctx.InformerFactory.Core().V1().ConfigMaps(),
+		ctx.InformerFactory.Core().V1().Namespaces(),
+		ctx.ClientBuilder.ClientOrDie("root-ca-cert-publisher"),
+		rootCA,
+	)
+	if err != nil {
+		return nil, true, fmt.Errorf("error creating root CA certificate publisher: %v", err)
+	}
+	go sac.Run(1, ctx.Stop)
+	return nil, true, nil
+}

cmd/kube-controller-manager/app/controllermanager.go

@@ -39,6 +39,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/mux"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	apiserverflag "k8s.io/apiserver/pkg/util/flag"
 	cacheddiscovery "k8s.io/client-go/discovery/cached"
 	"k8s.io/client-go/informers"
@@ -54,6 +55,7 @@ import (
 	"k8s.io/kubernetes/pkg/controller"
 	kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config"
 	serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/serviceaccount"
 	"k8s.io/kubernetes/pkg/util/configz"
 	utilflag "k8s.io/kubernetes/pkg/util/flag"
@@ -333,6 +335,7 @@ func KnownControllers() []string {
 var ControllersDisabledByDefault = sets.NewString(
 	"bootstrapsigner",
 	"tokencleaner",
+	"root_ca_crt_publisher",
 )
 
 const (
@@ -379,6 +382,9 @@ func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc
 	controllers["pvc-protection"] = startPVCProtectionController
 	controllers["pv-protection"] = startPVProtectionController
 	controllers["ttl-after-finished"] = startTTLAfterFinishedController
+	if utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) {
+		controllers["root_ca_crt_publisher"] = startRootCACertPublisher
+	}
 
 	return controllers
 }
@@ -524,11 +530,7 @@ func (c serviceAccountTokenControllerStarter) startServiceAccountTokenController
 
 	var rootCA []byte
 	if ctx.ComponentConfig.SAController.RootCAFile != "" {
-		rootCA, err = ioutil.ReadFile(ctx.ComponentConfig.SAController.RootCAFile)
+		if rootCA, err = readCA(ctx.ComponentConfig.SAController.RootCAFile); err != nil {
-		if err != nil {
-			return nil, true, fmt.Errorf("error reading root-ca-file at %s: %v", ctx.ComponentConfig.SAController.RootCAFile, err)
-		}
-		if _, err := certutil.ParseCertsPEM(rootCA); err != nil {
 			return nil, true, fmt.Errorf("error parsing root-ca-file at %s: %v", ctx.ComponentConfig.SAController.RootCAFile, err)
 		}
 	} else {
@@ -558,3 +560,15 @@ func (c serviceAccountTokenControllerStarter) startServiceAccountTokenController
 
 	return nil, true, nil
 }
+
+func readCA(file string) ([]byte, error) {
+	rootCA, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := certutil.ParseCertsPEM(rootCA); err != nil {
+		return nil, err
+	}
+
+	return rootCA, err
+}

pkg/controller/certificates/BUILD

@@ -43,6 +43,7 @@ filegroup(
         ":package-srcs",
         "//pkg/controller/certificates/approver:all-srcs",
         "//pkg/controller/certificates/cleaner:all-srcs",
+        "//pkg/controller/certificates/rootcacertpublisher:all-srcs",
         "//pkg/controller/certificates/signer:all-srcs",
     ],
     tags = ["automanaged"],

pkg/controller/certificates/rootcacertpublisher/BUILD

@@ -0,0 +1,51 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["root_ca_cert_publisher.go"],
+    importpath = "k8s.io/kubernetes/pkg/controller/certificates/rootcacertpublisher",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/controller:go_default_library",
+        "//pkg/util/metrics:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
+        "//staging/src/k8s.io/client-go/util/workqueue:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["root_ca_cert_publisher_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//pkg/controller:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/informers:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
+        "//staging/src/k8s.io/client-go/testing:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

pkg/controller/certificates/rootcacertpublisher/root_ca_cert_publisher.go

@@ -0,0 +1,232 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rootcacertpublisher
+
+import (
+	"fmt"
+	"reflect"
+	"time"
+
+	"github.com/golang/glog"
+	"k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	coreinformers "k8s.io/client-go/informers/core/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	corelisters "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/kubernetes/pkg/controller"
+	"k8s.io/kubernetes/pkg/util/metrics"
+)
+
+// RootCACertCofigMapName is name of the configmap which stores certificates to access api-server
+const RootCACertCofigMapName = "kube-root-ca.crt"
+
+// NewPublisher construct a new controller which would manage the configmap which stores
+// certificates in each namespace. It will make sure certificate configmap exists in each namespace.
+func NewPublisher(cmInformer coreinformers.ConfigMapInformer, nsInformer coreinformers.NamespaceInformer, cl clientset.Interface, rootCA []byte) (*Publisher, error) {
+	e := &Publisher{
+		client: cl,
+		configMap: v1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: RootCACertCofigMapName,
+			},
+			Data: map[string]string{
+				"ca.crt": string(rootCA),
+			},
+		},
+		queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "root-ca-crt-publisher"),
+	}
+	if cl.CoreV1().RESTClient().GetRateLimiter() != nil {
+		if err := metrics.RegisterMetricAndTrackRateLimiterUsage("root_ca_crt_publisher", cl.CoreV1().RESTClient().GetRateLimiter()); err != nil {
+			return nil, err
+		}
+	}
+
+	cmInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		DeleteFunc: e.configMapDeleted,
+		UpdateFunc: e.configMapUpdated,
+	})
+	e.cmLister = cmInformer.Lister()
+	e.cmListerSynced = cmInformer.Informer().HasSynced
+
+	nsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc:    e.namespaceAdded,
+		UpdateFunc: e.namespaceUpdated,
+	})
+	e.nsLister = nsInformer.Lister()
+	e.nsListerSynced = nsInformer.Informer().HasSynced
+
+	e.syncHandler = e.syncNamespace
+
+	return e, nil
+
+}
+
+// Publisher manages certificate ConfigMap objects inside Namespaces
+type Publisher struct {
+	client    clientset.Interface
+	configMap v1.ConfigMap
+
+	// To allow injection for testing.
+	syncHandler func(key string) error
+
+	cmLister       corelisters.ConfigMapLister
+	cmListerSynced cache.InformerSynced
+
+	nsLister       corelisters.NamespaceLister
+	nsListerSynced cache.InformerSynced
+
+	queue workqueue.RateLimitingInterface
+}
+
+// Run starts process
+func (c *Publisher) Run(workers int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	glog.Infof("Starting root CA certificate configmap publisher")
+	defer glog.Infof("Shutting down root CA certificate configmap publisher")
+
+	if !controller.WaitForCacheSync("crt configmap", stopCh, c.cmListerSynced, c.nsListerSynced) {
+		return
+	}
+
+	for i := 0; i < workers; i++ {
+		go wait.Until(c.runWorker, time.Second, stopCh)
+	}
+
+	<-stopCh
+}
+
+func (c *Publisher) configMapDeleted(obj interface{}) {
+	cm, err := convertToCM(obj)
+	if err != nil {
+		utilruntime.HandleError(err)
+		return
+	}
+	if cm.Name != RootCACertCofigMapName {
+		return
+	}
+	c.queue.Add(cm.Namespace)
+}
+
+func (c *Publisher) configMapUpdated(_, newObj interface{}) {
+	newConfigMap, err := convertToCM(newObj)
+	if err != nil {
+		utilruntime.HandleError(err)
+		return
+	}
+	if newConfigMap.Name != RootCACertCofigMapName {
+		return
+	}
+
+	if reflect.DeepEqual(c.configMap.Data, newConfigMap.Data) {
+		return
+	}
+
+	newConfigMap.Data = make(map[string]string)
+	newConfigMap.Data["ca.crt"] = c.configMap.Data["ca.crt"]
+	if _, err := c.client.CoreV1().ConfigMaps(newConfigMap.Namespace).Update(newConfigMap); err != nil && !apierrs.IsAlreadyExists(err) {
+		utilruntime.HandleError(fmt.Errorf("configmap creation failure:%v", err))
+	}
+}
+
+func (c *Publisher) namespaceAdded(obj interface{}) {
+	namespace := obj.(*v1.Namespace)
+	c.queue.Add(namespace.Name)
+}
+
+func (c *Publisher) namespaceUpdated(oldObj interface{}, newObj interface{}) {
+	newNamespace := newObj.(*v1.Namespace)
+	if newNamespace.Status.Phase != v1.NamespaceActive {
+		return
+	}
+	c.queue.Add(newNamespace.Name)
+}
+
+func (c *Publisher) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+// processNextWorkItem deals with one key off the queue.  It returns false when it's time to quit.
+func (c *Publisher) processNextWorkItem() bool {
+	key, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(key)
+
+	err := c.syncHandler(key.(string))
+	if err == nil {
+		c.queue.Forget(key)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", key, err))
+	c.queue.AddRateLimited(key)
+
+	return true
+}
+
+func (c *Publisher) syncNamespace(key string) error {
+	startTime := time.Now()
+	defer func() {
+		glog.V(4).Infof("Finished syncing namespace %q (%v)", key, time.Since(startTime))
+	}()
+
+	ns, err := c.nsLister.Get(key)
+	if apierrs.IsNotFound(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	switch _, err := c.cmLister.ConfigMaps(ns.Name).Get(c.configMap.Name); {
+	case err == nil:
+		return nil
+	case apierrs.IsNotFound(err):
+	case err != nil:
+		return err
+	}
+
+	cm := c.configMap.DeepCopy()
+	if _, err := c.client.CoreV1().ConfigMaps(ns.Name).Create(cm); err != nil && !apierrs.IsAlreadyExists(err) {
+		return err
+	}
+	return nil
+}
+
+func convertToCM(obj interface{}) (*v1.ConfigMap, error) {
+	cm, ok := obj.(*v1.ConfigMap)
+	if !ok {
+		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+		if !ok {
+			return nil, fmt.Errorf("Couldn't get object from tombstone %#v", obj)
+		}
+		cm, ok = tombstone.Obj.(*v1.ConfigMap)
+		if !ok {
+			return nil, fmt.Errorf("Tombstone contained object that is not a ConfigMap %#v", obj)
+		}
+	}
+	return cm, nil
+}

pkg/controller/certificates/rootcacertpublisher/root_ca_cert_publisher_test.go

@@ -0,0 +1,218 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rootcacertpublisher
+
+import (
+	"testing"
+	"time"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes/fake"
+	core "k8s.io/client-go/testing"
+	"k8s.io/kubernetes/pkg/controller"
+)
+
+func TestConfigMapCreation(t *testing.T) {
+	ns := metav1.NamespaceDefault
+	fakeRootCA := []byte("fake-root-ca")
+
+	caConfigMap := defaultCrtConfigMapPtr(fakeRootCA)
+	addFieldCM := defaultCrtConfigMapPtr(fakeRootCA)
+	addFieldCM.Data["test"] = "test"
+	modifyFieldCM := defaultCrtConfigMapPtr([]byte("abc"))
+	otherConfigMap := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            "other",
+			Namespace:       ns,
+			ResourceVersion: "1",
+		},
+	}
+	updateOtherConfigMap := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            "other",
+			Namespace:       ns,
+			ResourceVersion: "1",
+		},
+		Data: map[string]string{"aa": "bb"},
+	}
+
+	existNS := &v1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{Name: ns},
+		Status: v1.NamespaceStatus{
+			Phase: v1.NamespaceActive,
+		},
+	}
+	newNs := &v1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{Name: "new"},
+		Status: v1.NamespaceStatus{
+			Phase: v1.NamespaceActive,
+		},
+	}
+	terminatingNS := &v1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{Name: ns},
+		Status: v1.NamespaceStatus{
+			Phase: v1.NamespaceTerminating,
+		},
+	}
+
+	type action struct {
+		verb string
+		name string
+	}
+	testcases := map[string]struct {
+		ExistingNamespace  *v1.Namespace
+		ExistingConfigMaps []*v1.ConfigMap
+		AddedNamespace     *v1.Namespace
+		UpdatedNamespace   *v1.Namespace
+		DeletedConfigMap   *v1.ConfigMap
+		UpdatedConfigMap   []*v1.ConfigMap
+		ExpectActions      []action
+	}{
+		"create new namesapce": {
+			AddedNamespace: newNs,
+			ExpectActions:  []action{{verb: "create", name: RootCACertCofigMapName}},
+		},
+
+		"delete other configmap": {
+			ExistingNamespace:  existNS,
+			ExistingConfigMaps: []*v1.ConfigMap{otherConfigMap, caConfigMap},
+			DeletedConfigMap:   otherConfigMap,
+		},
+		"delete ca configmap": {
+			ExistingNamespace:  existNS,
+			ExistingConfigMaps: []*v1.ConfigMap{otherConfigMap, caConfigMap},
+			DeletedConfigMap:   caConfigMap,
+			ExpectActions:      []action{{verb: "create", name: RootCACertCofigMapName}},
+		},
+		"update ca configmap with adding field": {
+			ExistingNamespace:  existNS,
+			ExistingConfigMaps: []*v1.ConfigMap{caConfigMap},
+			UpdatedConfigMap:   []*v1.ConfigMap{caConfigMap, addFieldCM},
+			ExpectActions:      []action{{verb: "update", name: RootCACertCofigMapName}},
+		},
+		"update ca configmap with modifying field": {
+			ExistingNamespace:  existNS,
+			ExistingConfigMaps: []*v1.ConfigMap{caConfigMap},
+			UpdatedConfigMap:   []*v1.ConfigMap{caConfigMap, modifyFieldCM},
+			ExpectActions:      []action{{verb: "update", name: RootCACertCofigMapName}},
+		},
+		"update with other configmap": {
+			ExistingNamespace:  existNS,
+			ExistingConfigMaps: []*v1.ConfigMap{caConfigMap, otherConfigMap},
+			UpdatedConfigMap:   []*v1.ConfigMap{otherConfigMap, updateOtherConfigMap},
+		},
+		"update namespace with terminating state": {
+			ExistingNamespace: existNS,
+			UpdatedNamespace:  terminatingNS,
+		},
+	}
+
+	for k, tc := range testcases {
+		client := fake.NewSimpleClientset(caConfigMap, existNS)
+		informers := informers.NewSharedInformerFactory(fake.NewSimpleClientset(), controller.NoResyncPeriodFunc())
+		cmInformer := informers.Core().V1().ConfigMaps()
+		nsInformer := informers.Core().V1().Namespaces()
+		controller, err := NewPublisher(cmInformer, nsInformer, client, fakeRootCA)
+		if err != nil {
+			t.Fatalf("error creating ServiceAccounts controller: %v", err)
+		}
+		controller.cmListerSynced = alwaysReady
+		controller.nsListerSynced = alwaysReady
+
+		cmStore := cmInformer.Informer().GetStore()
+		nsStore := nsInformer.Informer().GetStore()
+
+		syncCalls := make(chan struct{})
+		controller.syncHandler = func(key string) error {
+			err := controller.syncNamespace(key)
+			if err != nil {
+				t.Logf("%s: %v", k, err)
+			}
+			syncCalls <- struct{}{}
+			return err
+		}
+		stopCh := make(chan struct{})
+		defer close(stopCh)
+		go controller.Run(1, stopCh)
+
+		if tc.ExistingNamespace != nil {
+			nsStore.Add(tc.ExistingNamespace)
+		}
+		for _, s := range tc.ExistingConfigMaps {
+			cmStore.Add(s)
+		}
+
+		if tc.AddedNamespace != nil {
+			nsStore.Add(tc.AddedNamespace)
+			controller.namespaceAdded(tc.AddedNamespace)
+		}
+		if tc.UpdatedNamespace != nil {
+			controller.namespaceUpdated(tc.ExistingNamespace, tc.UpdatedNamespace)
+		}
+
+		if tc.DeletedConfigMap != nil {
+			cmStore.Delete(tc.DeletedConfigMap)
+			controller.configMapDeleted(tc.DeletedConfigMap)
+		}
+
+		if tc.UpdatedConfigMap != nil {
+			old := tc.UpdatedConfigMap[0]
+			new := tc.UpdatedConfigMap[1]
+			controller.configMapUpdated(old, new)
+		}
+
+		// wait to be called
+		select {
+		case <-syncCalls:
+		case <-time.After(5 * time.Second):
+		}
+
+		actions := client.Actions()
+		if len(tc.ExpectActions) != len(actions) {
+			t.Errorf("%s: Expected to create configmap %#v. Actual actions were: %#v", k, tc.ExpectActions, actions)
+			continue
+		}
+		for i, expectAction := range tc.ExpectActions {
+			action := actions[i]
+			if !action.Matches(expectAction.verb, "configmaps") {
+				t.Errorf("%s: Unexpected action %s", k, action)
+				break
+			}
+			cm := action.(core.CreateAction).GetObject().(*v1.ConfigMap)
+			if cm.Name != expectAction.name {
+				t.Errorf("%s: Expected %s to be %s, got %s be %s", k, expectAction.name, expectAction.verb, cm.Name, action.GetVerb())
+			}
+		}
+	}
+}
+
+var alwaysReady = func() bool { return true }
+
+func defaultCrtConfigMapPtr(rootCA []byte) *v1.ConfigMap {
+	tmp := v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: RootCACertCofigMapName,
+		},
+		Data: map[string]string{
+			"ca.crt": string(rootCA),
+		},
+	}
+	tmp.Namespace = metav1.NamespaceDefault
+	return &tmp
+}

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go

@@ -353,6 +353,16 @@ func buildControllerRoles() ([]rbacv1.ClusterRole, []rbacv1.ClusterRoleBinding)
 		})
 	}
 
+	if utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) {
+		addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{
+			ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "root-ca-cert-publisher"},
+			Rules: []rbacv1.PolicyRule{
+				rbacv1helpers.NewRule("create", "update").Groups(legacyGroup).Resources("configmaps").RuleOrDie(),
+				eventsRule(),
+			},
+		})
+	}
+
 	return controllerRoles, controllerRoleBindings
 }
 

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-role-bindings.yaml

@@ -357,6 +357,23 @@ items:
   - kind: ServiceAccount
     name: resourcequota-controller
     namespace: kube-system
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRoleBinding
+  metadata:
+    annotations:
+      rbac.authorization.kubernetes.io/autoupdate: "true"
+    creationTimestamp: null
+    labels:
+      kubernetes.io/bootstrapping: rbac-defaults
+    name: system:controller:root-ca-cert-publisher
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: system:controller:root-ca-cert-publisher
+  subjects:
+  - kind: ServiceAccount
+    name: root-ca-cert-publisher
+    namespace: kube-system
 - apiVersion: rbac.authorization.k8s.io/v1
   kind: ClusterRoleBinding
   metadata:

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml

@@ -1031,6 +1031,31 @@ items:
     - create
     - patch
     - update
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRole
+  metadata:
+    annotations:
+      rbac.authorization.kubernetes.io/autoupdate: "true"
+    creationTimestamp: null
+    labels:
+      kubernetes.io/bootstrapping: rbac-defaults
+    name: system:controller:root-ca-cert-publisher
+  rules:
+  - apiGroups:
+    - ""
+    resources:
+    - configmaps
+    verbs:
+    - create
+    - update
+  - apiGroups:
+    - ""
+    resources:
+    - events
+    verbs:
+    - create
+    - patch
+    - update
 - apiVersion: rbac.authorization.k8s.io/v1
   kind: ClusterRole
   metadata:

test/cmd/apps.sh

@@ -369,7 +369,8 @@ run_deployment_tests() {
   kubectl create -f hack/testdata/configmap.yaml "${kube_flags[@]}"
   kubectl create -f hack/testdata/secret.yaml "${kube_flags[@]}"
   kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" 'nginx-deployment:'
-  kube::test::get_object_assert configmap "{{range.items}}{{$id_field}}:{{end}}" 'test-set-env-config:'
+  #configmap is special here due to controller will create kube-root-ca.crt for each namespace automatically
+  kube::test::get_object_assert 'configmaps/test-set-env-config' "{{$id_field}}" 'test-set-env-config'
   kube::test::get_object_assert secret "{{range.items}}{{$id_field}}:{{end}}" 'test-set-env-secret:'
   # Set env of deployments by configmap from keys
   kubectl set env deployment nginx-deployment --keys=key-2 --from=configmap/test-set-env-config "${kube_flags[@]}"

test/cmd/core.sh

@@ -25,7 +25,7 @@ run_configmap_tests() {
   create_and_use_new_namespace
   kube::log::status "Testing configmaps"
   kubectl create -f test/fixtures/doc-yaml/user-guide/configmap/configmap.yaml
-  kube::test::get_object_assert configmap "{{range.items}}{{$id_field}}{{end}}" 'test-configmap'
+  kube::test::get_object_assert 'configmap/test-configmap' "{{$id_field}}" 'test-configmap'
   kubectl delete configmap test-configmap "${kube_flags[@]}"
 
   ### Create a new namespace
@@ -37,8 +37,10 @@ run_configmap_tests() {
   kube::test::get_object_assert 'namespaces/test-configmaps' "{{$id_field}}" 'test-configmaps'
 
   ### Create a generic configmap in a specific namespace
-  # Pre-condition: no configmaps namespace exists
+  # Pre-condition: configmap test-configmap and test-binary-configmap does not exist
-  kube::test::get_object_assert 'configmaps --namespace=test-configmaps' "{{range.items}}{{$id_field}}:{{end}}" ''
+  kube::test::get_object_assert 'configmaps' '{{range.items}}{{ if eq $id_field \"test-configmap\" }}found{{end}}{{end}}:' ':'
+  kube::test::get_object_assert 'configmaps' '{{range.items}}{{ if eq $id_field \"test-binary-configmap\" }}found{{end}}{{end}}:' ':'
+
   # Command
   kubectl create configmap test-configmap --from-literal=key1=value1 --namespace=test-configmaps
   kubectl create configmap test-binary-configmap --from-file <( head -c 256 /dev/urandom ) --namespace=test-configmaps
@@ -222,8 +224,11 @@ run_pod_tests() {
   kube::test::get_object_assert 'secret/test-secret --namespace=test-kubectl-describe-pod' "{{$secret_type}}" 'test-type'
 
   ### Create a generic configmap
-  # Pre-condition: no CONFIGMAP exists
+  # Pre-condition: CONFIGMAP test-configmap does not exist
-  kube::test::get_object_assert 'configmaps --namespace=test-kubectl-describe-pod' "{{range.items}}{{$id_field}}:{{end}}" ''
+  #kube::test::get_object_assert 'configmap/test-configmap --namespace=test-kubectl-describe-pod' "{{$id_field}}" ''
+  kube::test::get_object_assert 'configmaps --namespace=test-kubectl-describe-pod' '{{range.items}}{{ if eq $id_field \"test-configmap\" }}found{{end}}{{end}}:' ':'
+
+  #kube::test::get_object_assert 'configmaps --namespace=test-kubectl-describe-pod' "{{range.items}}{{$id_field}}:{{end}}" ''
   # Command
   kubectl create configmap test-configmap --from-literal=key-2=value2 --namespace=test-kubectl-describe-pod
   # Post-condition: configmap exists and has expected values

test/cmd/get.sh

@@ -130,8 +130,11 @@ run_kubectl_get_tests() {
   kube::test::if_has_string "${output_message}" "/clusterroles?limit=500 200 OK"
 
   ### Test kubectl get chunk size does not result in a --watch error when resource list is served in multiple chunks
-  # Pre-condition: no ConfigMaps exist
+  # Pre-condition: ConfigMap one two tree does not exist
-  kube::test::get_object_assert configmap "{{range.items}}{{$id_field}}:{{end}}" ''
+  kube::test::get_object_assert 'configmaps' '{{range.items}}{{ if eq $id_field \"one\" }}found{{end}}{{end}}:' ':'
+  kube::test::get_object_assert 'configmaps' '{{range.items}}{{ if eq $id_field \"two\" }}found{{end}}{{end}}:' ':'
+  kube::test::get_object_assert 'configmaps' '{{range.items}}{{ if eq $id_field \"three\" }}found{{end}}{{end}}:' ':'
+
   # Post-condition: Create three configmaps and ensure that we can --watch them with a --chunk-size of 1
   kubectl create cm one "${kube_flags[@]}"
   kubectl create cm two "${kube_flags[@]}"

test/e2e/scheduling/resource_quota.go

@@ -365,6 +365,22 @@ var _ = SIGDescribe("ResourceQuota", func() {
 	})
 
 	It("should create a ResourceQuota and capture the life of a configMap.", func() {
+		found, unchanged := 0, 0
+		wait.Poll(1*time.Second, 30*time.Second, func() (bool, error) {
+			configmaps, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).List(metav1.ListOptions{})
+			Expect(err).NotTo(HaveOccurred())
+			if len(configmaps.Items) == found {
+				// loop until the number of configmaps has stabilized for 5 seconds
+				unchanged++
+				return unchanged > 4, nil
+			}
+			unchanged = 0
+			found = len(configmaps.Items)
+			return false, nil
+		})
+		defaultConfigMaps := fmt.Sprintf("%d", found)
+		hardConfigMaps := fmt.Sprintf("%d", found+1)
+
 		By("Creating a ResourceQuota")
 		quotaName := "test-quota"
 		resourceQuota := newTestResourceQuota(quotaName)
@@ -374,6 +390,7 @@ var _ = SIGDescribe("ResourceQuota", func() {
 		By("Ensuring resource quota status is calculated")
 		usedResources := v1.ResourceList{}
 		usedResources[v1.ResourceQuotas] = resource.MustParse("1")
+		usedResources[v1.ResourceConfigMaps] = resource.MustParse(defaultConfigMaps)
 		err = waitForResourceQuota(f.ClientSet, f.Namespace.Name, quotaName, usedResources)
 		Expect(err).NotTo(HaveOccurred())
 
@@ -385,7 +402,10 @@ var _ = SIGDescribe("ResourceQuota", func() {
 		By("Ensuring resource quota status captures configMap creation")
 		usedResources = v1.ResourceList{}
 		usedResources[v1.ResourceQuotas] = resource.MustParse("1")
-		usedResources[v1.ResourceConfigMaps] = resource.MustParse("1")
+		// we expect there to be two configmaps because each namespace will receive
+		// a ca.crt configmap by default.
+		// ref:https://github.com/kubernetes/kubernetes/pull/68812
+		usedResources[v1.ResourceConfigMaps] = resource.MustParse(hardConfigMaps)
 		err = waitForResourceQuota(f.ClientSet, f.Namespace.Name, quotaName, usedResources)
 		Expect(err).NotTo(HaveOccurred())
 
@@ -394,7 +414,7 @@ var _ = SIGDescribe("ResourceQuota", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		By("Ensuring resource quota status released usage")
-		usedResources[v1.ResourceConfigMaps] = resource.MustParse("0")
+		usedResources[v1.ResourceConfigMaps] = resource.MustParse(defaultConfigMaps)
 		err = waitForResourceQuota(f.ClientSet, f.Namespace.Name, quotaName, usedResources)
 		Expect(err).NotTo(HaveOccurred())
 	})