diff --git a/test/e2e/auth/service_accounts.go b/test/e2e/auth/service_accounts.go index e3ca96000a..088b21a8e9 100644 --- a/test/e2e/auth/service_accounts.go +++ b/test/e2e/auth/service_accounts.go @@ -19,6 +19,8 @@ package auth import ( "fmt" "path" + "regexp" + "strings" "time" authenticationv1 "k8s.io/api/authentication/v1" @@ -38,6 +40,7 @@ import ( ) var mountImage = imageutils.GetE2EImage(imageutils.Mounttest) +var inClusterClientImage = imageutils.GetE2EImage(imageutils.InClusterClient) var _ = SIGDescribe("ServiceAccounts", func() { f := framework.NewDefaultFramework("svcaccounts") @@ -410,4 +413,138 @@ var _ = SIGDescribe("ServiceAccounts", func() { } } }) + + ginkgo.It("should support InClusterConfig with token rotation [Slow] [Feature:TokenRequestProjection]", func() { + cfg, err := framework.LoadConfig() + framework.ExpectNoError(err) + + if _, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(&v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-root-ca.crt", + }, + Data: map[string]string{ + "ca.crt": string(cfg.TLSClientConfig.CAData), + }, + }); err != nil && !apierrors.IsAlreadyExists(err) { + framework.Failf("Unexpected err creating kube-ca-crt: %v", err) + } + + tenMin := int64(10 * 60) + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "inclusterclient"}, + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Name: "inclusterclient", + Image: inClusterClientImage, + VolumeMounts: []v1.VolumeMount{{ + MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", + Name: "kube-api-access-e2e", + ReadOnly: true, + }}, + }}, + RestartPolicy: v1.RestartPolicyNever, + ServiceAccountName: "default", + Volumes: []v1.Volume{{ + Name: "kube-api-access-e2e", + VolumeSource: v1.VolumeSource{ + Projected: &v1.ProjectedVolumeSource{ + Sources: []v1.VolumeProjection{ + { + ServiceAccountToken: &v1.ServiceAccountTokenProjection{ + Path: "token", + ExpirationSeconds: &tenMin, + }, + }, + { + ConfigMap: &v1.ConfigMapProjection{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "kube-root-ca.crt", + }, + Items: []v1.KeyToPath{ + { + Key: "ca.crt", + Path: "ca.crt", + }, + }, + }, + }, + { + DownwardAPI: &v1.DownwardAPIProjection{ + Items: []v1.DownwardAPIVolumeFile{ + { + Path: "namespace", + FieldRef: &v1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.namespace", + }, + }, + }, + }, + }, + }, + }, + }, + }}, + }, + } + pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod) + framework.ExpectNoError(err) + + framework.Logf("created pod") + if !framework.CheckPodsRunningReady(f.ClientSet, f.Namespace.Name, []string{pod.Name}, time.Minute) { + framework.Failf("pod %q in ns %q never became ready", pod.Name, f.Namespace.Name) + } + + framework.Logf("pod is ready") + + var logs string + if err := wait.Poll(1*time.Minute, 20*time.Minute, func() (done bool, err error) { + framework.Logf("polling logs") + logs, err = framework.GetPodLogs(f.ClientSet, f.Namespace.Name, "inclusterclient", "inclusterclient") + if err != nil { + framework.Logf("Error pulling logs: %v", err) + return false, nil + } + tokenCount, err := parseInClusterClientLogs(logs) + if err != nil { + return false, fmt.Errorf("inclusterclient reported an error: %v", err) + } + if tokenCount < 2 { + framework.Logf("Retrying. Still waiting to see more unique tokens: got=%d, want=2", tokenCount) + return false, nil + } + return true, nil + }); err != nil { + framework.Failf("Unexpected error: %v\n%s", err, logs) + } + }) }) + +var reportLogsParser = regexp.MustCompile("([a-zA-Z0-9-_]*)=([a-zA-Z0-9-_]*)$") + +func parseInClusterClientLogs(logs string) (int, error) { + seenTokens := map[string]struct{}{} + + lines := strings.Split(logs, "\n") + for _, line := range lines { + parts := reportLogsParser.FindStringSubmatch(line) + if len(parts) != 3 { + continue + } + + key, value := parts[1], parts[2] + switch key { + case "authz_header": + if value == "" { + return 0, fmt.Errorf("saw empty Authorization header") + } + seenTokens[value] = struct{}{} + case "status": + if value == "failed" { + return 0, fmt.Errorf("saw status=failed") + } + } + } + + return len(seenTokens), nil +} diff --git a/test/images/BUILD b/test/images/BUILD index de05303960..3516f67e89 100644 --- a/test/images/BUILD +++ b/test/images/BUILD @@ -18,6 +18,7 @@ filegroup( "//test/images/echoserver:all-srcs", "//test/images/entrypoint-tester:all-srcs", "//test/images/fakegitserver:all-srcs", + "//test/images/inclusterclient:all-srcs", "//test/images/liveness:all-srcs", "//test/images/logs-generator:all-srcs", "//test/images/metadata-concealment:all-srcs", diff --git a/test/images/inclusterclient/BUILD b/test/images/inclusterclient/BUILD new file mode 100644 index 0000000000..07cfc21064 --- /dev/null +++ b/test/images/inclusterclient/BUILD @@ -0,0 +1,34 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["main.go"], + importpath = "k8s.io/kubernetes/test/images/inclusterclient", + visibility = ["//visibility:private"], + deps = [ + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", + "//staging/src/k8s.io/component-base/logs:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], +) + +go_binary( + name = "inclusterconfig", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/images/inclusterclient/Dockerfile b/test/images/inclusterclient/Dockerfile new file mode 100644 index 0000000000..478ccddefe --- /dev/null +++ b/test/images/inclusterclient/Dockerfile @@ -0,0 +1,18 @@ +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/distroless/static:latest + +ADD inclusterclient /inclusterclient +ENTRYPOINT ["/inclusterclient"] diff --git a/test/images/inclusterclient/Makefile b/test/images/inclusterclient/Makefile new file mode 100644 index 0000000000..0ce0b522c4 --- /dev/null +++ b/test/images/inclusterclient/Makefile @@ -0,0 +1,25 @@ +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SRCS = inclusterclient +ARCH ?= amd64 +TARGET ?= $(CURDIR) +GOLANG_VERSION ?= latest +SRC_DIR = $(notdir $(shell pwd)) +export + +bin: + ../image-util.sh bin $(SRCS) + +.PHONY: bin diff --git a/test/images/inclusterclient/VERSION b/test/images/inclusterclient/VERSION new file mode 100644 index 0000000000..d3827e75a5 --- /dev/null +++ b/test/images/inclusterclient/VERSION @@ -0,0 +1 @@ +1.0 diff --git a/test/images/inclusterclient/main.go b/test/images/inclusterclient/main.go new file mode 100644 index 0000000000..bf7a0ac694 --- /dev/null +++ b/test/images/inclusterclient/main.go @@ -0,0 +1,84 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "crypto/sha256" + "encoding/base64" + "flag" + "fmt" + "log" + "net/http" + "time" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/component-base/logs" + "k8s.io/klog" +) + +func main() { + logs.InitLogs() + defer logs.FlushLogs() + + pollInterval := flag.Int("poll-interval", 30, "poll interval of call to /healhtz in seconds") + flag.Set("logtostderr", "true") + flag.Parse() + + klog.Infof("started") + + cfg, err := rest.InClusterConfig() + if err != nil { + log.Fatalf("err: %v", err) + } + + cfg.Wrap(func(rt http.RoundTripper) http.RoundTripper { + return &debugRt{ + rt: rt, + } + }) + + c := kubernetes.NewForConfigOrDie(cfg).RESTClient() + + t := time.Tick(time.Duration(*pollInterval) * time.Second) + for { + <-t + klog.Infof("calling /healthz") + b, err := c.Get().AbsPath("/healthz").Do().Raw() + if err != nil { + klog.Errorf("status=failed") + klog.Errorf("error checking /healthz: %v\n%s\n", err, string(b)) + } + } +} + +type debugRt struct { + rt http.RoundTripper +} + +func (rt *debugRt) RoundTrip(req *http.Request) (*http.Response, error) { + authHeader := req.Header.Get("Authorization") + if len(authHeader) != 0 { + authHash := sha256.Sum256([]byte(fmt.Sprintf("%s|%s", "salt", authHeader))) + klog.Infof("authz_header=%s", base64.RawURLEncoding.EncodeToString(authHash[:])) + } else { + klog.Errorf("authz_header=") + } + return rt.rt.RoundTrip(req) +} + +func (rt *debugRt) WrappedRoundTripper() http.RoundTripper { return rt.rt } diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index ca79a70882..e1eea1a98b 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -130,6 +130,8 @@ const ( GBRedisSlave // Hostexec image Hostexec + // InClusterClient image + InClusterClient // IpcUtils image IpcUtils // Iperf image @@ -211,6 +213,7 @@ func initImageConfigs() map[int]Config { configs[GBFrontend] = Config{sampleRegistry, "gb-frontend", "v6"} configs[GBRedisSlave] = Config{sampleRegistry, "gb-redisslave", "v3"} configs[Hostexec] = Config{e2eRegistry, "hostexec", "1.1"} + configs[InClusterClient] = Config{e2eRegistry, "inclusterclient", "1.0"} configs[IpcUtils] = Config{e2eRegistry, "ipc-utils", "1.0"} configs[Iperf] = Config{e2eRegistry, "iperf", "1.0"} configs[JessieDnsutils] = Config{e2eRegistry, "jessie-dnsutils", "1.0"}