Migrate the controller to use TokenRequest and rotate token periodically

Godeps/Godeps.json

@@ -1758,12 +1758,12 @@
 		},
 		{
 			"ImportPath": "github.com/go-openapi/jsonpointer",
-			"Comment": "v0.18.0",
+			"Comment": "v0.19.0",
 			"Rev": "ef5f0afec364d3b9396b7b77b43dbe26bf1f8004"
 		},
 		{
 			"ImportPath": "github.com/go-openapi/jsonreference",
-			"Comment": "v0.18.0",
+			"Comment": "v0.19.0",
 			"Rev": "8483a886a90412cd6858df4ea3483dce9c8e35a3"
 		},
 		{

cmd/kube-controller-manager/app/controllermanager.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -39,9 +40,11 @@ import (
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/apiserver/pkg/server/mux"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/pkg/util/term"
 	cacheddiscovery "k8s.io/client-go/discovery/cached"
 	"k8s.io/client-go/informers"
+	clientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/restmapper"
 	"k8s.io/client-go/tools/leaderelection"
@@ -58,6 +61,7 @@ import (
 	"k8s.io/kubernetes/pkg/controller"
 	kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config"
 	serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/serviceaccount"
 	"k8s.io/kubernetes/pkg/util/configz"
 	utilflag "k8s.io/kubernetes/pkg/util/flag"
@@ -199,12 +203,23 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {
 				// If one isn't, we'll timeout and exit when our client builder is unable to create the tokens.
 				klog.Warningf("--use-service-account-credentials was specified without providing a --service-account-private-key-file")
 			}
+
+			if shouldTurnOnDynamicClient(c.Client) {
+				klog.V(1).Infof("using dynamic client builder")
+				//Dynamic builder will use TokenRequest feature and refresh service account token periodically
+				clientBuilder = controller.NewDynamicClientBuilder(
+					restclient.AnonymousClientConfig(c.Kubeconfig),
+					c.Client.CoreV1(),
+					"kube-system")
+			} else {
+				klog.V(1).Infof("using legacy client builder")
 				clientBuilder = controller.SAControllerClientBuilder{
 					ClientConfig:         restclient.AnonymousClientConfig(c.Kubeconfig),
 					CoreClient:           c.Client.CoreV1(),
 					AuthenticationClient: c.Client.AuthenticationV1(),
 					Namespace:            "kube-system",
 				}
+			}
 		} else {
 			clientBuilder = rootClientBuilder
 		}
@@ -566,3 +581,24 @@ func readCA(file string) ([]byte, error) {
 
 	return rootCA, err
 }
+
+func shouldTurnOnDynamicClient(client clientset.Interface) bool {
+	if !utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) {
+		return false
+	}
+	apiResourceList, err := client.Discovery().ServerResourcesForGroupVersion(v1.SchemeGroupVersion.String())
+	if err != nil {
+		klog.Warningf("fetch api resource lists failed, use legacy client builder: %v", err)
+		return false
+	}
+
+	for _, resource := range apiResourceList.APIResources {
+		if resource.Name == "serviceaccounts/token" &&
+			resource.Group == "authentication.k8s.io" &&
+			sets.NewString(resource.Verbs...).Has("create") {
+			return true
+		}
+	}
+
+	return false
+}

pkg/controller/.import-restrictions

@@ -160,7 +160,8 @@
         "k8s.io/client-go/util/cert",
         "k8s.io/client-go/util/flowcontrol",
         "k8s.io/client-go/util/retry",
-        "k8s.io/client-go/util/workqueue"
+        "k8s.io/client-go/util/workqueue",
+	"k8s.io/client-go/transport"
       ]
     },
     {

pkg/controller/BUILD

@@ -43,6 +43,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "client_builder.go",
+        "client_builder_dynamic.go",
         "controller_ref_manager.go",
         "controller_utils.go",
         "doc.go",
@@ -85,10 +86,13 @@ go_library(
         "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
         "//staging/src/k8s.io/client-go/tools/record:go_default_library",
         "//staging/src/k8s.io/client-go/tools/watch:go_default_library",
+        "//staging/src/k8s.io/client-go/transport:go_default_library",
         "//staging/src/k8s.io/client-go/util/retry:go_default_library",
         "//vendor/github.com/golang/groupcache/lru:go_default_library",
+        "//vendor/golang.org/x/oauth2:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
         "//vendor/k8s.io/utils/integer:go_default_library",
+        "//vendor/k8s.io/utils/pointer:go_default_library",
     ],
 )
 

pkg/controller/client_builder.go

@@ -108,7 +108,7 @@ type SAControllerClientBuilder struct {
 // config returns a complete clientConfig for constructing clients.  This is separate in anticipation of composition
 // which means that not all clientsets are known here
 func (b SAControllerClientBuilder) Config(name string) (*restclient.Config, error) {
-	sa, err := b.getOrCreateServiceAccount(name)
+	sa, err := getOrCreateServiceAccount(b.CoreClient, b.Namespace, name)
 	if err != nil {
 		return nil, err
 	}
@@ -177,30 +177,6 @@ func (b SAControllerClientBuilder) Config(name string) (*restclient.Config, erro
 	return clientConfig, nil
 }
 
-func (b SAControllerClientBuilder) getOrCreateServiceAccount(name string) (*v1.ServiceAccount, error) {
-	sa, err := b.CoreClient.ServiceAccounts(b.Namespace).Get(name, metav1.GetOptions{})
-	if err == nil {
-		return sa, nil
-	}
-	if !apierrors.IsNotFound(err) {
-		return nil, err
-	}
-
-	// Create the namespace if we can't verify it exists.
-	// Tolerate errors, since we don't know whether this component has namespace creation permissions.
-	if _, err := b.CoreClient.Namespaces().Get(b.Namespace, metav1.GetOptions{}); err != nil {
-		b.CoreClient.Namespaces().Create(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: b.Namespace}})
-	}
-
-	// Create the service account
-	sa, err = b.CoreClient.ServiceAccounts(b.Namespace).Create(&v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: b.Namespace, Name: name}})
-	if apierrors.IsAlreadyExists(err) {
-		// If we're racing to init and someone else already created it, re-fetch
-		return b.CoreClient.ServiceAccounts(b.Namespace).Get(name, metav1.GetOptions{})
-	}
-	return sa, err
-}
-
 func (b SAControllerClientBuilder) getAuthenticatedConfig(sa *v1.ServiceAccount, token string) (*restclient.Config, bool, error) {
 	username := apiserverserviceaccount.MakeUsername(sa.Namespace, sa.Name)
 

pkg/controller/client_builder_dynamic.go

@@ -0,0 +1,217 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	"golang.org/x/oauth2"
+
+	v1authenticationapi "k8s.io/api/authentication/v1"
+	"k8s.io/apimachinery/pkg/util/clock"
+	"k8s.io/apimachinery/pkg/util/wait"
+	apiserverserviceaccount "k8s.io/apiserver/pkg/authentication/serviceaccount"
+	clientset "k8s.io/client-go/kubernetes"
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/transport"
+	"k8s.io/klog"
+	utilpointer "k8s.io/utils/pointer"
+)
+
+var (
+	// defaultExpirationSeconds defines the duration of a TokenRequest in seconds.
+	defaultExpirationSeconds = int64(3600)
+	// defaultLeewayPercent defines the percentage of expiration left before the client trigger a token rotation.
+	// range[0, 100]
+	defaultLeewayPercent = 20
+)
+
+type DynamicControllerClientBuilder struct {
+	// ClientConfig is a skeleton config to clone and use as the basis for each controller client
+	ClientConfig *restclient.Config
+
+	// CoreClient is used to provision service accounts if needed and watch for their associated tokens
+	// to construct a controller client
+	CoreClient v1core.CoreV1Interface
+
+	// Namespace is the namespace used to host the service accounts that will back the
+	// controllers.  It must be highly privileged namespace which normal users cannot inspect.
+	Namespace string
+
+	// roundTripperFuncMap is a cache stores the corresponding roundtripper func for each
+	// service account
+	roundTripperFuncMap map[string]func(http.RoundTripper) http.RoundTripper
+
+	// expirationSeconds defines the token expiration seconds
+	expirationSeconds int64
+
+	// leewayPercent defines the percentage of expiration left before the client trigger a token rotation.
+	leewayPercent int
+
+	mutex sync.Mutex
+
+	clock clock.Clock
+}
+
+func NewDynamicClientBuilder(clientConfig *restclient.Config, coreClient v1core.CoreV1Interface, ns string) ControllerClientBuilder {
+	builder := &DynamicControllerClientBuilder{
+		ClientConfig:        clientConfig,
+		CoreClient:          coreClient,
+		Namespace:           ns,
+		roundTripperFuncMap: map[string]func(http.RoundTripper) http.RoundTripper{},
+		expirationSeconds:   defaultExpirationSeconds,
+		leewayPercent:       defaultLeewayPercent,
+		clock:               clock.RealClock{},
+	}
+	return builder
+}
+
+// this function only for test purpose, don't call it
+func NewTestDynamicClientBuilder(clientConfig *restclient.Config, coreClient v1core.CoreV1Interface, ns string, expirationSeconds int64, leewayPercent int) ControllerClientBuilder {
+	builder := &DynamicControllerClientBuilder{
+		ClientConfig:        clientConfig,
+		CoreClient:          coreClient,
+		Namespace:           ns,
+		roundTripperFuncMap: map[string]func(http.RoundTripper) http.RoundTripper{},
+		expirationSeconds:   expirationSeconds,
+		leewayPercent:       leewayPercent,
+		clock:               clock.RealClock{},
+	}
+	return builder
+}
+
+func (t *DynamicControllerClientBuilder) Config(saName string) (*restclient.Config, error) {
+	_, err := getOrCreateServiceAccount(t.CoreClient, t.Namespace, saName)
+	if err != nil {
+		return nil, err
+	}
+
+	configCopy := constructClient(t.Namespace, saName, t.ClientConfig)
+
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	rt, ok := t.roundTripperFuncMap[saName]
+	if ok {
+		configCopy.WrapTransport = rt
+	} else {
+		cachedTokenSource := transport.NewCachedTokenSource(&tokenSourceImpl{
+			namespace:          t.Namespace,
+			serviceAccountName: saName,
+			coreClient:         t.CoreClient,
+			expirationSeconds:  t.expirationSeconds,
+			leewayPercent:      t.leewayPercent,
+		})
+		configCopy.WrapTransport = transport.TokenSourceWrapTransport(cachedTokenSource)
+
+		t.roundTripperFuncMap[saName] = configCopy.WrapTransport
+	}
+
+	return &configCopy, nil
+}
+
+func (t *DynamicControllerClientBuilder) ConfigOrDie(name string) *restclient.Config {
+	clientConfig, err := t.Config(name)
+	if err != nil {
+		klog.Fatal(err)
+	}
+	return clientConfig
+}
+
+func (t *DynamicControllerClientBuilder) Client(name string) (clientset.Interface, error) {
+	clientConfig, err := t.Config(name)
+	if err != nil {
+		return nil, err
+	}
+	return clientset.NewForConfig(clientConfig)
+}
+
+func (t *DynamicControllerClientBuilder) ClientOrDie(name string) clientset.Interface {
+	client, err := t.Client(name)
+	if err != nil {
+		klog.Fatal(err)
+	}
+	return client
+}
+
+type tokenSourceImpl struct {
+	namespace          string
+	serviceAccountName string
+	coreClient         v1core.CoreV1Interface
+	expirationSeconds  int64
+	leewayPercent      int
+}
+
+func (ts *tokenSourceImpl) Token() (*oauth2.Token, error) {
+	var retTokenRequest *v1authenticationapi.TokenRequest
+
+	backoff := wait.Backoff{
+		Duration: 500 * time.Millisecond,
+		Factor:   2, // double the timeout for every failure
+		Steps:    4,
+	}
+	if err := wait.ExponentialBackoff(backoff, func() (bool, error) {
+		if _, inErr := getOrCreateServiceAccount(ts.coreClient, ts.namespace, ts.serviceAccountName); inErr != nil {
+			klog.Warningf("get or create service account failed: %v", inErr)
+			return false, nil
+		}
+
+		tr, inErr := ts.coreClient.ServiceAccounts(ts.namespace).CreateToken(ts.serviceAccountName, &v1authenticationapi.TokenRequest{
+			Spec: v1authenticationapi.TokenRequestSpec{
+				ExpirationSeconds: utilpointer.Int64Ptr(ts.expirationSeconds),
+			},
+		})
+		if inErr != nil {
+			klog.Warningf("get token failed: %v", inErr)
+			return false, nil
+		}
+		retTokenRequest = tr
+		return true, nil
+	}); err != nil {
+		return nil, fmt.Errorf("failed to get token for %s/%s: %v", ts.namespace, ts.serviceAccountName, err)
+	}
+
+	if retTokenRequest.Spec.ExpirationSeconds == nil {
+		return nil, fmt.Errorf("nil pointer of expiration in token request")
+	}
+
+	lifetime := retTokenRequest.Status.ExpirationTimestamp.Time.Sub(time.Now())
+	if lifetime < time.Minute*10 {
+		// possible clock skew issue, pin to minimum token lifetime
+		lifetime = time.Minute * 10
+	}
+
+	leeway := time.Duration(int64(lifetime) * int64(ts.leewayPercent) / 100)
+	expiry := time.Now().Add(lifetime).Add(-1 * leeway)
+
+	return &oauth2.Token{
+		AccessToken: retTokenRequest.Status.Token,
+		TokenType:   "Bearer",
+		Expiry:      expiry,
+	}, nil
+}
+
+func constructClient(saNamespace, saName string, config *restclient.Config) restclient.Config {
+	username := apiserverserviceaccount.MakeUsername(saNamespace, saName)
+	ret := *restclient.AnonymousClientConfig(config)
+	restclient.AddUserAgent(&ret, username)
+	return ret
+}

pkg/controller/controller_utils.go

@@ -40,6 +40,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
 	clientretry "k8s.io/client-go/util/retry"
@@ -1096,3 +1097,29 @@ func AddOrUpdateLabelsOnNode(kubeClient clientset.Interface, nodeName string, la
 		return nil
 	})
 }
+
+func getOrCreateServiceAccount(coreClient v1core.CoreV1Interface, namespace, name string) (*v1.ServiceAccount, error) {
+	sa, err := coreClient.ServiceAccounts(namespace).Get(name, metav1.GetOptions{})
+	if err == nil {
+		return sa, nil
+	}
+	if !apierrors.IsNotFound(err) {
+		return nil, err
+	}
+
+	// Create the namespace if we can't verify it exists.
+	// Tolerate errors, since we don't know whether this component has namespace creation permissions.
+	if _, err := coreClient.Namespaces().Get(namespace, metav1.GetOptions{}); apierrors.IsNotFound(err) {
+		if _, err = coreClient.Namespaces().Create(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}); err != nil && !apierrors.IsAlreadyExists(err) {
+			klog.Warningf("create non-exist namespace %s failed:%v", namespace, err)
+		}
+	}
+
+	// Create the service account
+	sa, err = coreClient.ServiceAccounts(namespace).Create(&v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}})
+	if apierrors.IsAlreadyExists(err) {
+		// If we're racing to init and someone else already created it, re-fetch
+		return coreClient.ServiceAccounts(namespace).Get(name, metav1.GetOptions{})
+	}
+	return sa, err
+}

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go

@@ -413,6 +413,7 @@ func ClusterRoles() []rbacv1.ClusterRole {
 				rbacv1helpers.NewRule("create").Groups(authorizationGroup).Resources("subjectaccessreviews").RuleOrDie(),
 				// Needed for all shared informers
 				rbacv1helpers.NewRule("list", "watch").Groups("*").Resources("*").RuleOrDie(),
+				rbacv1helpers.NewRule("create").Groups(legacyGroup).Resources("serviceaccounts/token").RuleOrDie(),
 			},
 		},
 		{

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml

@@ -681,6 +681,12 @@ items:
     verbs:
     - list
     - watch
+  - apiGroups:
+    - ""
+    resources:
+    - serviceaccounts/token
+    verbs:
+    - create
 - apiVersion: rbac.authorization.k8s.io/v1
   kind: ClusterRole
   metadata:

staging/src/k8s.io/client-go/transport/token_source.go

@@ -59,6 +59,15 @@ func NewCachedFileTokenSource(path string) oauth2.TokenSource {
 	}
 }
 
+// NewCachedTokenSource returns a oauth2.TokenSource reads a token from a
+// designed TokenSource. The ts would provide the source of token.
+func NewCachedTokenSource(ts oauth2.TokenSource) oauth2.TokenSource {
+	return &cachingTokenSource{
+		now:  time.Now,
+		base: ts,
+	}
+}
+
 type tokenSourceTransport struct {
 	base http.RoundTripper
 	ort  http.RoundTripper

test/integration/auth/BUILD

@@ -12,6 +12,7 @@ go_test(
         "accessreview_test.go",
         "auth_test.go",
         "bootstraptoken_test.go",
+        "dynamic_client_test.go",
         "main_test.go",
         "node_test.go",
         "rbac_test.go",
@@ -22,6 +23,7 @@ go_test(
     ],
     tags = ["integration"],
     deps = [
+        "//cmd/kube-apiserver/app/options:go_default_library",
         "//cmd/kube-apiserver/app/testing:go_default_library",
         "//pkg/api/legacyscheme:go_default_library",
         "//pkg/api/testapi:go_default_library",
@@ -34,8 +36,10 @@ go_test(
         "//pkg/apis/rbac:go_default_library",
         "//pkg/auth/authorizer/abac:go_default_library",
         "//pkg/client/clientset_generated/internalclientset:go_default_library",
+        "//pkg/controller:go_default_library",
         "//pkg/controller/serviceaccount:go_default_library",
         "//pkg/features:go_default_library",
+        "//pkg/kubeapiserver/options:go_default_library",
         "//pkg/master:go_default_library",
         "//pkg/registry/rbac/clusterrole:go_default_library",
         "//pkg/registry/rbac/clusterrole/storage:go_default_library",

test/integration/auth/dynamic_client_test.go

@@ -0,0 +1,130 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package auth
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apiserver/pkg/authentication/authenticator"
+	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
+	clientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
+	"k8s.io/kubernetes/pkg/controller"
+	"k8s.io/kubernetes/pkg/features"
+	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
+	"k8s.io/kubernetes/pkg/master"
+	"k8s.io/kubernetes/test/integration/framework"
+)
+
+func TestDynamicClientBuilder(t *testing.T) {
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.TokenRequest, true)()
+
+	tmpfile, err := ioutil.TempFile("/tmp", "key")
+	if err != nil {
+		t.Fatalf("create temp file failed: %v", err)
+	}
+	defer os.RemoveAll(tmpfile.Name())
+
+	if err = ioutil.WriteFile(tmpfile.Name(), []byte(ecdsaPrivateKey), 0666); err != nil {
+		t.Fatalf("write file %s failed: %v", tmpfile.Name(), err)
+	}
+
+	const iss = "https://foo.bar.example.com"
+	aud := authenticator.Audiences{"api"}
+
+	maxExpirationDuration := time.Second * 60 * 60
+	if err != nil {
+		t.Fatalf("parse duration failed: %v", err)
+	}
+
+	stopCh := make(chan struct{})
+	defer close(stopCh)
+
+	baseClient, baseConfig := framework.StartTestServer(t, stopCh, framework.TestServerSetup{
+		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
+			opts.ServiceAccountSigningKeyFile = tmpfile.Name()
+			opts.ServiceAccountTokenMaxExpiration = maxExpirationDuration
+			if opts.Authentication == nil {
+				opts.Authentication = &kubeoptions.BuiltInAuthenticationOptions{}
+			}
+
+			opts.Authentication.APIAudiences = aud
+			if opts.Authentication.ServiceAccounts == nil {
+				opts.Authentication.ServiceAccounts = &kubeoptions.ServiceAccountAuthenticationOptions{}
+			}
+			opts.Authentication.ServiceAccounts.Issuer = iss
+			opts.Authentication.ServiceAccounts.KeyFiles = []string{tmpfile.Name()}
+		},
+		ModifyServerConfig: func(config *master.Config) {
+			config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
+		},
+	})
+
+	// We want to test if the token rotation works fine here.
+	// To minimize the time this test would consume, we use the minimial token expiration.
+	// The minimial token expiration is defined in:
+	// pkg/apis/authentication/validation/validation.go
+	exp := int64(600)
+	leeway := 99
+	ns := "default"
+	clientBuilder := controller.NewTestDynamicClientBuilder(
+		restclient.AnonymousClientConfig(baseConfig),
+		baseClient.CoreV1(),
+		ns, exp, leeway)
+
+	saName := "dt"
+	dymClient, err := clientBuilder.Client(saName)
+
+	if err != nil {
+		t.Fatalf("build client via dynamic client builder failed: %v", err)
+	}
+
+	if err = testClientBuilder(dymClient, ns, saName); err != nil {
+		t.Fatalf("dynamic client get resources failed befroe deleting sa: %v", err)
+	}
+
+	// We want to trigger token rotation here by deleting service account
+	// the dynamic client was using.
+	if err = dymClient.CoreV1().ServiceAccounts(ns).Delete(saName, nil); err != nil {
+		t.Fatalf("delete service account %s failed: %v", saName, err)
+	}
+	time.Sleep(time.Second * 10)
+
+	if err = testClientBuilder(dymClient, ns, saName); err != nil {
+		t.Fatalf("dynamic client get resources failed after deleting sa: %v", err)
+	}
+}
+
+func testClientBuilder(dymClient clientset.Interface, ns, saName string) error {
+	_, err := dymClient.CoreV1().Namespaces().Get(ns, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	_, err = dymClient.CoreV1().ServiceAccounts(ns).Get(saName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}