From 244b244f9d84c56ad3a5af255b70c793f6bfd39c Mon Sep 17 00:00:00 2001 From: WanLinghao Date: Tue, 19 Feb 2019 11:42:05 +0800 Subject: [PATCH] Migrate the controller to use TokenRequest and rotate token periodically --- Godeps/Godeps.json | 4 +- .../app/controllermanager.go | 46 +++- pkg/controller/.import-restrictions | 3 +- pkg/controller/BUILD | 4 + pkg/controller/client_builder.go | 26 +-- pkg/controller/client_builder_dynamic.go | 217 ++++++++++++++++++ pkg/controller/controller_utils.go | 27 +++ .../authorizer/rbac/bootstrappolicy/policy.go | 1 + .../testdata/cluster-roles.yaml | 6 + .../client-go/transport/token_source.go | 9 + test/integration/auth/BUILD | 4 + test/integration/auth/dynamic_client_test.go | 130 +++++++++++ 12 files changed, 444 insertions(+), 33 deletions(-) create mode 100644 pkg/controller/client_builder_dynamic.go create mode 100644 test/integration/auth/dynamic_client_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index fe2ee875f6..ffb39f4b56 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1758,12 +1758,12 @@ }, { "ImportPath": "github.com/go-openapi/jsonpointer", - "Comment": "v0.18.0", + "Comment": "v0.19.0", "Rev": "ef5f0afec364d3b9396b7b77b43dbe26bf1f8004" }, { "ImportPath": "github.com/go-openapi/jsonreference", - "Comment": "v0.18.0", + "Comment": "v0.19.0", "Rev": "8483a886a90412cd6858df4ea3483dce9c8e35a3" }, { diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index 319ef1771d..7d89676d40 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -31,6 +31,7 @@ import ( "github.com/spf13/cobra" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" @@ -39,9 +40,11 @@ import ( "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/mux" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/term" cacheddiscovery "k8s.io/client-go/discovery/cached" "k8s.io/client-go/informers" + clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/restmapper" "k8s.io/client-go/tools/leaderelection" @@ -58,6 +61,7 @@ import ( "k8s.io/kubernetes/pkg/controller" kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config" serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/serviceaccount" "k8s.io/kubernetes/pkg/util/configz" utilflag "k8s.io/kubernetes/pkg/util/flag" @@ -199,11 +203,22 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { // If one isn't, we'll timeout and exit when our client builder is unable to create the tokens. klog.Warningf("--use-service-account-credentials was specified without providing a --service-account-private-key-file") } - clientBuilder = controller.SAControllerClientBuilder{ - ClientConfig: restclient.AnonymousClientConfig(c.Kubeconfig), - CoreClient: c.Client.CoreV1(), - AuthenticationClient: c.Client.AuthenticationV1(), - Namespace: "kube-system", + + if shouldTurnOnDynamicClient(c.Client) { + klog.V(1).Infof("using dynamic client builder") + //Dynamic builder will use TokenRequest feature and refresh service account token periodically + clientBuilder = controller.NewDynamicClientBuilder( + restclient.AnonymousClientConfig(c.Kubeconfig), + c.Client.CoreV1(), + "kube-system") + } else { + klog.V(1).Infof("using legacy client builder") + clientBuilder = controller.SAControllerClientBuilder{ + ClientConfig: restclient.AnonymousClientConfig(c.Kubeconfig), + CoreClient: c.Client.CoreV1(), + AuthenticationClient: c.Client.AuthenticationV1(), + Namespace: "kube-system", + } } } else { clientBuilder = rootClientBuilder @@ -566,3 +581,24 @@ func readCA(file string) ([]byte, error) { return rootCA, err } + +func shouldTurnOnDynamicClient(client clientset.Interface) bool { + if !utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) { + return false + } + apiResourceList, err := client.Discovery().ServerResourcesForGroupVersion(v1.SchemeGroupVersion.String()) + if err != nil { + klog.Warningf("fetch api resource lists failed, use legacy client builder: %v", err) + return false + } + + for _, resource := range apiResourceList.APIResources { + if resource.Name == "serviceaccounts/token" && + resource.Group == "authentication.k8s.io" && + sets.NewString(resource.Verbs...).Has("create") { + return true + } + } + + return false +} diff --git a/pkg/controller/.import-restrictions b/pkg/controller/.import-restrictions index 79dbe90178..f8177ea804 100644 --- a/pkg/controller/.import-restrictions +++ b/pkg/controller/.import-restrictions @@ -160,7 +160,8 @@ "k8s.io/client-go/util/cert", "k8s.io/client-go/util/flowcontrol", "k8s.io/client-go/util/retry", - "k8s.io/client-go/util/workqueue" + "k8s.io/client-go/util/workqueue", + "k8s.io/client-go/transport" ] }, { diff --git a/pkg/controller/BUILD b/pkg/controller/BUILD index e2902c7c54..ee926ba27f 100644 --- a/pkg/controller/BUILD +++ b/pkg/controller/BUILD @@ -43,6 +43,7 @@ go_library( name = "go_default_library", srcs = [ "client_builder.go", + "client_builder_dynamic.go", "controller_ref_manager.go", "controller_utils.go", "doc.go", @@ -85,10 +86,13 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/tools/watch:go_default_library", + "//staging/src/k8s.io/client-go/transport:go_default_library", "//staging/src/k8s.io/client-go/util/retry:go_default_library", "//vendor/github.com/golang/groupcache/lru:go_default_library", + "//vendor/golang.org/x/oauth2:go_default_library", "//vendor/k8s.io/klog:go_default_library", "//vendor/k8s.io/utils/integer:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", ], ) diff --git a/pkg/controller/client_builder.go b/pkg/controller/client_builder.go index 500a51ac39..54b4dbb06f 100644 --- a/pkg/controller/client_builder.go +++ b/pkg/controller/client_builder.go @@ -108,7 +108,7 @@ type SAControllerClientBuilder struct { // config returns a complete clientConfig for constructing clients. This is separate in anticipation of composition // which means that not all clientsets are known here func (b SAControllerClientBuilder) Config(name string) (*restclient.Config, error) { - sa, err := b.getOrCreateServiceAccount(name) + sa, err := getOrCreateServiceAccount(b.CoreClient, b.Namespace, name) if err != nil { return nil, err } @@ -177,30 +177,6 @@ func (b SAControllerClientBuilder) Config(name string) (*restclient.Config, erro return clientConfig, nil } -func (b SAControllerClientBuilder) getOrCreateServiceAccount(name string) (*v1.ServiceAccount, error) { - sa, err := b.CoreClient.ServiceAccounts(b.Namespace).Get(name, metav1.GetOptions{}) - if err == nil { - return sa, nil - } - if !apierrors.IsNotFound(err) { - return nil, err - } - - // Create the namespace if we can't verify it exists. - // Tolerate errors, since we don't know whether this component has namespace creation permissions. - if _, err := b.CoreClient.Namespaces().Get(b.Namespace, metav1.GetOptions{}); err != nil { - b.CoreClient.Namespaces().Create(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: b.Namespace}}) - } - - // Create the service account - sa, err = b.CoreClient.ServiceAccounts(b.Namespace).Create(&v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: b.Namespace, Name: name}}) - if apierrors.IsAlreadyExists(err) { - // If we're racing to init and someone else already created it, re-fetch - return b.CoreClient.ServiceAccounts(b.Namespace).Get(name, metav1.GetOptions{}) - } - return sa, err -} - func (b SAControllerClientBuilder) getAuthenticatedConfig(sa *v1.ServiceAccount, token string) (*restclient.Config, bool, error) { username := apiserverserviceaccount.MakeUsername(sa.Namespace, sa.Name) diff --git a/pkg/controller/client_builder_dynamic.go b/pkg/controller/client_builder_dynamic.go new file mode 100644 index 0000000000..19aaded8bf --- /dev/null +++ b/pkg/controller/client_builder_dynamic.go @@ -0,0 +1,217 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "fmt" + "net/http" + "sync" + "time" + + "golang.org/x/oauth2" + + v1authenticationapi "k8s.io/api/authentication/v1" + "k8s.io/apimachinery/pkg/util/clock" + "k8s.io/apimachinery/pkg/util/wait" + apiserverserviceaccount "k8s.io/apiserver/pkg/authentication/serviceaccount" + clientset "k8s.io/client-go/kubernetes" + v1core "k8s.io/client-go/kubernetes/typed/core/v1" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/transport" + "k8s.io/klog" + utilpointer "k8s.io/utils/pointer" +) + +var ( + // defaultExpirationSeconds defines the duration of a TokenRequest in seconds. + defaultExpirationSeconds = int64(3600) + // defaultLeewayPercent defines the percentage of expiration left before the client trigger a token rotation. + // range[0, 100] + defaultLeewayPercent = 20 +) + +type DynamicControllerClientBuilder struct { + // ClientConfig is a skeleton config to clone and use as the basis for each controller client + ClientConfig *restclient.Config + + // CoreClient is used to provision service accounts if needed and watch for their associated tokens + // to construct a controller client + CoreClient v1core.CoreV1Interface + + // Namespace is the namespace used to host the service accounts that will back the + // controllers. It must be highly privileged namespace which normal users cannot inspect. + Namespace string + + // roundTripperFuncMap is a cache stores the corresponding roundtripper func for each + // service account + roundTripperFuncMap map[string]func(http.RoundTripper) http.RoundTripper + + // expirationSeconds defines the token expiration seconds + expirationSeconds int64 + + // leewayPercent defines the percentage of expiration left before the client trigger a token rotation. + leewayPercent int + + mutex sync.Mutex + + clock clock.Clock +} + +func NewDynamicClientBuilder(clientConfig *restclient.Config, coreClient v1core.CoreV1Interface, ns string) ControllerClientBuilder { + builder := &DynamicControllerClientBuilder{ + ClientConfig: clientConfig, + CoreClient: coreClient, + Namespace: ns, + roundTripperFuncMap: map[string]func(http.RoundTripper) http.RoundTripper{}, + expirationSeconds: defaultExpirationSeconds, + leewayPercent: defaultLeewayPercent, + clock: clock.RealClock{}, + } + return builder +} + +// this function only for test purpose, don't call it +func NewTestDynamicClientBuilder(clientConfig *restclient.Config, coreClient v1core.CoreV1Interface, ns string, expirationSeconds int64, leewayPercent int) ControllerClientBuilder { + builder := &DynamicControllerClientBuilder{ + ClientConfig: clientConfig, + CoreClient: coreClient, + Namespace: ns, + roundTripperFuncMap: map[string]func(http.RoundTripper) http.RoundTripper{}, + expirationSeconds: expirationSeconds, + leewayPercent: leewayPercent, + clock: clock.RealClock{}, + } + return builder +} + +func (t *DynamicControllerClientBuilder) Config(saName string) (*restclient.Config, error) { + _, err := getOrCreateServiceAccount(t.CoreClient, t.Namespace, saName) + if err != nil { + return nil, err + } + + configCopy := constructClient(t.Namespace, saName, t.ClientConfig) + + t.mutex.Lock() + defer t.mutex.Unlock() + + rt, ok := t.roundTripperFuncMap[saName] + if ok { + configCopy.WrapTransport = rt + } else { + cachedTokenSource := transport.NewCachedTokenSource(&tokenSourceImpl{ + namespace: t.Namespace, + serviceAccountName: saName, + coreClient: t.CoreClient, + expirationSeconds: t.expirationSeconds, + leewayPercent: t.leewayPercent, + }) + configCopy.WrapTransport = transport.TokenSourceWrapTransport(cachedTokenSource) + + t.roundTripperFuncMap[saName] = configCopy.WrapTransport + } + + return &configCopy, nil +} + +func (t *DynamicControllerClientBuilder) ConfigOrDie(name string) *restclient.Config { + clientConfig, err := t.Config(name) + if err != nil { + klog.Fatal(err) + } + return clientConfig +} + +func (t *DynamicControllerClientBuilder) Client(name string) (clientset.Interface, error) { + clientConfig, err := t.Config(name) + if err != nil { + return nil, err + } + return clientset.NewForConfig(clientConfig) +} + +func (t *DynamicControllerClientBuilder) ClientOrDie(name string) clientset.Interface { + client, err := t.Client(name) + if err != nil { + klog.Fatal(err) + } + return client +} + +type tokenSourceImpl struct { + namespace string + serviceAccountName string + coreClient v1core.CoreV1Interface + expirationSeconds int64 + leewayPercent int +} + +func (ts *tokenSourceImpl) Token() (*oauth2.Token, error) { + var retTokenRequest *v1authenticationapi.TokenRequest + + backoff := wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 2, // double the timeout for every failure + Steps: 4, + } + if err := wait.ExponentialBackoff(backoff, func() (bool, error) { + if _, inErr := getOrCreateServiceAccount(ts.coreClient, ts.namespace, ts.serviceAccountName); inErr != nil { + klog.Warningf("get or create service account failed: %v", inErr) + return false, nil + } + + tr, inErr := ts.coreClient.ServiceAccounts(ts.namespace).CreateToken(ts.serviceAccountName, &v1authenticationapi.TokenRequest{ + Spec: v1authenticationapi.TokenRequestSpec{ + ExpirationSeconds: utilpointer.Int64Ptr(ts.expirationSeconds), + }, + }) + if inErr != nil { + klog.Warningf("get token failed: %v", inErr) + return false, nil + } + retTokenRequest = tr + return true, nil + }); err != nil { + return nil, fmt.Errorf("failed to get token for %s/%s: %v", ts.namespace, ts.serviceAccountName, err) + } + + if retTokenRequest.Spec.ExpirationSeconds == nil { + return nil, fmt.Errorf("nil pointer of expiration in token request") + } + + lifetime := retTokenRequest.Status.ExpirationTimestamp.Time.Sub(time.Now()) + if lifetime < time.Minute*10 { + // possible clock skew issue, pin to minimum token lifetime + lifetime = time.Minute * 10 + } + + leeway := time.Duration(int64(lifetime) * int64(ts.leewayPercent) / 100) + expiry := time.Now().Add(lifetime).Add(-1 * leeway) + + return &oauth2.Token{ + AccessToken: retTokenRequest.Status.Token, + TokenType: "Bearer", + Expiry: expiry, + }, nil +} + +func constructClient(saNamespace, saName string, config *restclient.Config) restclient.Config { + username := apiserverserviceaccount.MakeUsername(saNamespace, saName) + ret := *restclient.AnonymousClientConfig(config) + restclient.AddUserAgent(&ret, username) + return ret +} diff --git a/pkg/controller/controller_utils.go b/pkg/controller/controller_utils.go index ecae8d7e87..e813bc2646 100644 --- a/pkg/controller/controller_utils.go +++ b/pkg/controller/controller_utils.go @@ -40,6 +40,7 @@ import ( "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + v1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" clientretry "k8s.io/client-go/util/retry" @@ -1096,3 +1097,29 @@ func AddOrUpdateLabelsOnNode(kubeClient clientset.Interface, nodeName string, la return nil }) } + +func getOrCreateServiceAccount(coreClient v1core.CoreV1Interface, namespace, name string) (*v1.ServiceAccount, error) { + sa, err := coreClient.ServiceAccounts(namespace).Get(name, metav1.GetOptions{}) + if err == nil { + return sa, nil + } + if !apierrors.IsNotFound(err) { + return nil, err + } + + // Create the namespace if we can't verify it exists. + // Tolerate errors, since we don't know whether this component has namespace creation permissions. + if _, err := coreClient.Namespaces().Get(namespace, metav1.GetOptions{}); apierrors.IsNotFound(err) { + if _, err = coreClient.Namespaces().Create(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}); err != nil && !apierrors.IsAlreadyExists(err) { + klog.Warningf("create non-exist namespace %s failed:%v", namespace, err) + } + } + + // Create the service account + sa, err = coreClient.ServiceAccounts(namespace).Create(&v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}}) + if apierrors.IsAlreadyExists(err) { + // If we're racing to init and someone else already created it, re-fetch + return coreClient.ServiceAccounts(namespace).Get(name, metav1.GetOptions{}) + } + return sa, err +} diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go index f037181279..4c31f256f8 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go @@ -413,6 +413,7 @@ func ClusterRoles() []rbacv1.ClusterRole { rbacv1helpers.NewRule("create").Groups(authorizationGroup).Resources("subjectaccessreviews").RuleOrDie(), // Needed for all shared informers rbacv1helpers.NewRule("list", "watch").Groups("*").Resources("*").RuleOrDie(), + rbacv1helpers.NewRule("create").Groups(legacyGroup).Resources("serviceaccounts/token").RuleOrDie(), }, }, { diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml index 3cb0d018d6..482471b564 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml @@ -681,6 +681,12 @@ items: verbs: - list - watch + - apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/staging/src/k8s.io/client-go/transport/token_source.go b/staging/src/k8s.io/client-go/transport/token_source.go index 8595df2716..b8cadd382a 100644 --- a/staging/src/k8s.io/client-go/transport/token_source.go +++ b/staging/src/k8s.io/client-go/transport/token_source.go @@ -59,6 +59,15 @@ func NewCachedFileTokenSource(path string) oauth2.TokenSource { } } +// NewCachedTokenSource returns a oauth2.TokenSource reads a token from a +// designed TokenSource. The ts would provide the source of token. +func NewCachedTokenSource(ts oauth2.TokenSource) oauth2.TokenSource { + return &cachingTokenSource{ + now: time.Now, + base: ts, + } +} + type tokenSourceTransport struct { base http.RoundTripper ort http.RoundTripper diff --git a/test/integration/auth/BUILD b/test/integration/auth/BUILD index 545af5ad2d..70c3226872 100644 --- a/test/integration/auth/BUILD +++ b/test/integration/auth/BUILD @@ -12,6 +12,7 @@ go_test( "accessreview_test.go", "auth_test.go", "bootstraptoken_test.go", + "dynamic_client_test.go", "main_test.go", "node_test.go", "rbac_test.go", @@ -22,6 +23,7 @@ go_test( ], tags = ["integration"], deps = [ + "//cmd/kube-apiserver/app/options:go_default_library", "//cmd/kube-apiserver/app/testing:go_default_library", "//pkg/api/legacyscheme:go_default_library", "//pkg/api/testapi:go_default_library", @@ -34,8 +36,10 @@ go_test( "//pkg/apis/rbac:go_default_library", "//pkg/auth/authorizer/abac:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", + "//pkg/controller:go_default_library", "//pkg/controller/serviceaccount:go_default_library", "//pkg/features:go_default_library", + "//pkg/kubeapiserver/options:go_default_library", "//pkg/master:go_default_library", "//pkg/registry/rbac/clusterrole:go_default_library", "//pkg/registry/rbac/clusterrole/storage:go_default_library", diff --git a/test/integration/auth/dynamic_client_test.go b/test/integration/auth/dynamic_client_test.go new file mode 100644 index 0000000000..9c1020726f --- /dev/null +++ b/test/integration/auth/dynamic_client_test.go @@ -0,0 +1,130 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "io/ioutil" + "os" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/apiserver/pkg/authorization/authorizerfactory" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + "k8s.io/kubernetes/pkg/controller" + "k8s.io/kubernetes/pkg/features" + kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" + "k8s.io/kubernetes/pkg/master" + "k8s.io/kubernetes/test/integration/framework" +) + +func TestDynamicClientBuilder(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.TokenRequest, true)() + + tmpfile, err := ioutil.TempFile("/tmp", "key") + if err != nil { + t.Fatalf("create temp file failed: %v", err) + } + defer os.RemoveAll(tmpfile.Name()) + + if err = ioutil.WriteFile(tmpfile.Name(), []byte(ecdsaPrivateKey), 0666); err != nil { + t.Fatalf("write file %s failed: %v", tmpfile.Name(), err) + } + + const iss = "https://foo.bar.example.com" + aud := authenticator.Audiences{"api"} + + maxExpirationDuration := time.Second * 60 * 60 + if err != nil { + t.Fatalf("parse duration failed: %v", err) + } + + stopCh := make(chan struct{}) + defer close(stopCh) + + baseClient, baseConfig := framework.StartTestServer(t, stopCh, framework.TestServerSetup{ + ModifyServerRunOptions: func(opts *options.ServerRunOptions) { + opts.ServiceAccountSigningKeyFile = tmpfile.Name() + opts.ServiceAccountTokenMaxExpiration = maxExpirationDuration + if opts.Authentication == nil { + opts.Authentication = &kubeoptions.BuiltInAuthenticationOptions{} + } + + opts.Authentication.APIAudiences = aud + if opts.Authentication.ServiceAccounts == nil { + opts.Authentication.ServiceAccounts = &kubeoptions.ServiceAccountAuthenticationOptions{} + } + opts.Authentication.ServiceAccounts.Issuer = iss + opts.Authentication.ServiceAccounts.KeyFiles = []string{tmpfile.Name()} + }, + ModifyServerConfig: func(config *master.Config) { + config.GenericConfig.Authorization.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer() + }, + }) + + // We want to test if the token rotation works fine here. + // To minimize the time this test would consume, we use the minimial token expiration. + // The minimial token expiration is defined in: + // pkg/apis/authentication/validation/validation.go + exp := int64(600) + leeway := 99 + ns := "default" + clientBuilder := controller.NewTestDynamicClientBuilder( + restclient.AnonymousClientConfig(baseConfig), + baseClient.CoreV1(), + ns, exp, leeway) + + saName := "dt" + dymClient, err := clientBuilder.Client(saName) + + if err != nil { + t.Fatalf("build client via dynamic client builder failed: %v", err) + } + + if err = testClientBuilder(dymClient, ns, saName); err != nil { + t.Fatalf("dynamic client get resources failed befroe deleting sa: %v", err) + } + + // We want to trigger token rotation here by deleting service account + // the dynamic client was using. + if err = dymClient.CoreV1().ServiceAccounts(ns).Delete(saName, nil); err != nil { + t.Fatalf("delete service account %s failed: %v", saName, err) + } + time.Sleep(time.Second * 10) + + if err = testClientBuilder(dymClient, ns, saName); err != nil { + t.Fatalf("dynamic client get resources failed after deleting sa: %v", err) + } +} + +func testClientBuilder(dymClient clientset.Interface, ns, saName string) error { + _, err := dymClient.CoreV1().Namespaces().Get(ns, metav1.GetOptions{}) + if err != nil { + return err + } + + _, err = dymClient.CoreV1().ServiceAccounts(ns).Get(saName, metav1.GetOptions{}) + if err != nil { + return err + } + return nil +}