diff --git a/cmd/kube-controller-manager/app/BUILD b/cmd/kube-controller-manager/app/BUILD index 56bc8f57d0..d7b8a488ce 100644 --- a/cmd/kube-controller-manager/app/BUILD +++ b/cmd/kube-controller-manager/app/BUILD @@ -157,6 +157,7 @@ filegroup( ":package-srcs", "//cmd/kube-controller-manager/app/config:all-srcs", "//cmd/kube-controller-manager/app/options:all-srcs", + "//cmd/kube-controller-manager/app/testing:all-srcs", ], tags = ["automanaged"], visibility = ["//visibility:public"], diff --git a/cmd/kube-controller-manager/app/testing/BUILD b/cmd/kube-controller-manager/app/testing/BUILD new file mode 100644 index 0000000000..75c8ff763e --- /dev/null +++ b/cmd/kube-controller-manager/app/testing/BUILD @@ -0,0 +1,31 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_library( + name = "go_default_library", + srcs = ["testserver.go"], + importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing", + visibility = ["//visibility:public"], + deps = [ + "//cmd/kube-controller-manager/app:go_default_library", + "//cmd/kube-controller-manager/app/config:go_default_library", + "//cmd/kube-controller-manager/app/options:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", + "//vendor/github.com/spf13/pflag:go_default_library", + ], +) diff --git a/cmd/kube-controller-manager/app/testing/testserver.go b/cmd/kube-controller-manager/app/testing/testserver.go new file mode 100644 index 0000000000..34ba72c5db --- /dev/null +++ b/cmd/kube-controller-manager/app/testing/testserver.go @@ -0,0 +1,175 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "fmt" + "io/ioutil" + "net" + "os" + "time" + + "github.com/spf13/pflag" + + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/kubernetes/cmd/kube-controller-manager/app" + kubecontrollerconfig "k8s.io/kubernetes/cmd/kube-controller-manager/app/config" + "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" +) + +// TearDownFunc is to be called to tear down a test server. +type TearDownFunc func() + +// TestServer return values supplied by kube-test-ApiServer +type TestServer struct { + LoopbackClientConfig *restclient.Config // Rest client config using the magic token + Options *options.KubeControllerManagerOptions + Config *kubecontrollerconfig.Config + TearDownFn TearDownFunc // TearDown function + TmpDir string // Temp Dir used, by the apiserver +} + +// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie +type Logger interface { + Errorf(format string, args ...interface{}) + Fatalf(format string, args ...interface{}) + Logf(format string, args ...interface{}) +} + +// StartTestServer starts a kube-controller-manager. A rest client config and a tear-down func, +// and location of the tmpdir are returned. +// +// Note: we return a tear-down func instead of a stop channel because the later will leak temporary +// files that because Golang testing's call to os.Exit will not give a stop channel go routine +// enough time to remove temporary files. +func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) { + stopCh := make(chan struct{}) + tearDown := func() { + close(stopCh) + if len(result.TmpDir) != 0 { + os.RemoveAll(result.TmpDir) + } + } + defer func() { + if result.TearDownFn == nil { + tearDown() + } + }() + + result.TmpDir, err = ioutil.TempDir("", "kube-controller-manager") + if err != nil { + return result, fmt.Errorf("failed to create temp dir: %v", err) + } + + fs := pflag.NewFlagSet("test", pflag.PanicOnError) + + s, err := options.NewKubeControllerManagerOptions() + if err != nil { + return TestServer{}, err + } + all, disabled := app.KnownControllers(), app.ControllersDisabledByDefault.List() + namedFlagSets := s.Flags(all, disabled) + for _, f := range namedFlagSets.FlagSets { + fs.AddFlagSet(f) + } + fs.Parse(customFlags) + + if s.SecureServing.BindPort != 0 { + s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort() + if err != nil { + return result, fmt.Errorf("failed to create listener: %v", err) + } + s.SecureServing.ServerCert.CertDirectory = result.TmpDir + + t.Logf("kube-controller-manager will listen securely on port %d...", s.SecureServing.BindPort) + } + + if s.InsecureServing.BindPort != 0 { + s.InsecureServing.Listener, s.InsecureServing.BindPort, err = createListenerOnFreePort() + if err != nil { + return result, fmt.Errorf("failed to create listener: %v", err) + } + + t.Logf("kube-controller-manager will listen insecurely on port %d...", s.InsecureServing.BindPort) + } + + config, err := s.Config(all, disabled) + if err != nil { + return result, fmt.Errorf("failed to create config from options: %v", err) + } + + go func(stopCh <-chan struct{}) { + if err := app.Run(config.Complete(), stopCh); err != nil { + t.Errorf("kube-apiserver failed run: %v", err) + } + }(stopCh) + + t.Logf("Waiting for /healthz to be ok...") + client, err := kubernetes.NewForConfig(config.LoopbackClientConfig) + if err != nil { + return result, fmt.Errorf("failed to create a client: %v", err) + } + err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) { + result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do() + status := 0 + result.StatusCode(&status) + if status == 200 { + return true, nil + } + return false, nil + }) + if err != nil { + return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err) + } + + // from here the caller must call tearDown + result.LoopbackClientConfig = config.LoopbackClientConfig + result.Options = s + result.Config = config + result.TearDownFn = tearDown + + return result, nil +} + +// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed. +func StartTestServerOrDie(t Logger, flags []string) *TestServer { + result, err := StartTestServer(t, flags) + if err == nil { + return &result + } + + t.Fatalf("failed to launch server: %v", err) + return nil +} + +func createListenerOnFreePort() (net.Listener, int, error) { + ln, err := net.Listen("tcp", ":0") + if err != nil { + return nil, 0, err + } + + // get port + tcpAddr, ok := ln.Addr().(*net.TCPAddr) + if !ok { + ln.Close() + return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String()) + } + + return ln, tcpAddr.Port, nil +} diff --git a/test/integration/BUILD b/test/integration/BUILD index 3b4e63ca48..adbed5c54a 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -51,6 +51,7 @@ filegroup( "//test/integration/framework:all-srcs", "//test/integration/garbagecollector:all-srcs", "//test/integration/ipamperf:all-srcs", + "//test/integration/kube_controller_manager:all-srcs", "//test/integration/master:all-srcs", "//test/integration/metrics:all-srcs", "//test/integration/objectmeta:all-srcs", diff --git a/test/integration/kube_controller_manager/BUILD b/test/integration/kube_controller_manager/BUILD new file mode 100644 index 0000000000..3891344a36 --- /dev/null +++ b/test/integration/kube_controller_manager/BUILD @@ -0,0 +1,40 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_test", +) + +go_test( + name = "go_default_test", + size = "large", + srcs = [ + "main_test.go", + "serving_test.go", + ], + tags = [ + "etcd", + "integration", + ], + deps = [ + "//cmd/kube-apiserver/app/testing:go_default_library", + "//cmd/kube-controller-manager/app/testing:go_default_library", + "//staging/src/k8s.io/api/rbac/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//test/integration/framework:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/test/integration/kube_controller_manager/main_test.go b/test/integration/kube_controller_manager/main_test.go new file mode 100644 index 0000000000..e53224d085 --- /dev/null +++ b/test/integration/kube_controller_manager/main_test.go @@ -0,0 +1,27 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubecontrollermanager + +import ( + "testing" + + "k8s.io/kubernetes/test/integration/framework" +) + +func TestMain(m *testing.M) { + framework.EtcdMain(m.Run) +} diff --git a/test/integration/kube_controller_manager/serving_test.go b/test/integration/kube_controller_manager/serving_test.go new file mode 100644 index 0000000000..80a21e25c6 --- /dev/null +++ b/test/integration/kube_controller_manager/serving_test.go @@ -0,0 +1,291 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubecontrollermanager + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + "net/http" + "os" + "path" + "strings" + "testing" + + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" + ctrlmgrtesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing" + "k8s.io/kubernetes/test/integration/framework" +) + +func TestStartTestServer(t *testing.T) { + // Insulate this test from picking up in-cluster config when run inside a pod + // We can't assume we have permissions to write to /var/run/secrets/... from a unit test to mock in-cluster config for testing + originalHost := os.Getenv("KUBERNETES_SERVICE_HOST") + if len(originalHost) > 0 { + os.Setenv("KUBERNETES_SERVICE_HOST", "") + defer os.Setenv("KUBERNETES_SERVICE_HOST", originalHost) + } + + // authenticate to apiserver via bearer token + token := "flwqkenfjasasdfmwerasd" + tokenFile, err := ioutil.TempFile("", "kubeconfig") + if err != nil { + t.Fatal(err) + } + tokenFile.WriteString(fmt.Sprintf(` +%s,kube-controller-manager,kube-controller-manager,"" +`, token)) + tokenFile.Close() + + // start apiserver + server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{ + "--token-auth-file", tokenFile.Name(), + "--authorization-mode", "RBAC", + }, framework.SharedEtcd()) + defer server.TearDownFn() + + // allow kube-controller-manager to do SubjectAccessReview + client, err := kubernetes.NewForConfig(server.ClientConfig) + if err != nil { + t.Fatalf("unexpected error creating client config: %v", err) + } + _, err = client.RbacV1().ClusterRoleBindings().Create(&rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "kube-controller-manager:system:auth-delegator"}, + Subjects: []rbacv1.Subject{{ + Kind: "User", + Name: "kube-controller-manager", + }}, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: "system:auth-delegator", + }, + }) + if err != nil { + t.Fatalf("failed to create system:auth-delegator rbac cluster role binding: %v", err) + } + + // allow kube-controller-manager to read kube-system/extension-apiserver-authentication + _, err = client.RbacV1().RoleBindings("kube-system").Create(&rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "kube-controller-manager:extension-apiserver-authentication-reader"}, + Subjects: []rbacv1.Subject{{ + Kind: "User", + Name: "kube-controller-manager", + }}, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: "extension-apiserver-authentication-reader", + }, + }) + if err != nil { + t.Fatalf("failed to create kube-controller-manager:extension-apiserver-authentication-reader rbac role binding: %v", err) + } + + // create kubeconfig for the apiserver + apiserverConfig, err := ioutil.TempFile("", "kubeconfig") + if err != nil { + t.Fatal(err) + } + apiserverConfig.WriteString(fmt.Sprintf(` +apiVersion: v1 +kind: Config +clusters: +- cluster: + server: %s + certificate-authority: %s + name: integration +contexts: +- context: + cluster: integration + user: kube-controller-manager + name: default-context +current-context: default-context +users: +- name: kube-controller-manager + user: + token: %s +`, server.ClientConfig.Host, server.ServerOpts.SecureServing.ServerCert.CertKey.CertFile, token)) + apiserverConfig.Close() + + // create BROKEN kubeconfig for the apiserver + brokenApiserverConfig, err := ioutil.TempFile("", "kubeconfig") + if err != nil { + t.Fatal(err) + } + brokenApiserverConfig.WriteString(fmt.Sprintf(` +apiVersion: v1 +kind: Config +clusters: +- cluster: + server: %s + certificate-authority: %s + name: integration +contexts: +- context: + cluster: integration + user: kube-controller-manager + name: default-context +current-context: default-context +users: +- name: kube-controller-manager + user: + token: WRONGTOKEN +`, server.ClientConfig.Host, server.ServerOpts.SecureServing.ServerCert.CertKey.CertFile)) + brokenApiserverConfig.Close() + + tests := []struct { + name string + flags []string + path string + anonymous bool // to use the token or not + wantErr bool + wantSecureCode, wantInsecureCode *int + }{ + {"no-flags", nil, "/healthz", false, true, nil, nil}, + {"insecurely /healthz", []string{ + "--secure-port=0", + "--kubeconfig", apiserverConfig.Name(), + "--leader-elect=false", + }, "/healthz", true, false, nil, intPtr(http.StatusOK)}, + {"insecurely /metrics", []string{ + "--secure-port=0", + "--kubeconfig", apiserverConfig.Name(), + "--leader-elect=false", + }, "/metrics", true, false, nil, intPtr(http.StatusOK)}, + {"/healthz without authn/authz", []string{ + "--port=0", + "--kubeconfig", apiserverConfig.Name(), + "--leader-elect=false", + }, "/healthz", true, false, intPtr(http.StatusOK), nil}, + {"authorization skipped for /healthz with authn/authz", []string{ + "--port=0", + "--authentication-kubeconfig", apiserverConfig.Name(), + "--authorization-kubeconfig", apiserverConfig.Name(), + "--kubeconfig", apiserverConfig.Name(), + "--leader-elect=false", + }, "/healthz", false, false, intPtr(http.StatusOK), nil}, + {"authorization skipped for /healthz with BROKEN authn/authz", []string{ + "--port=0", + "--authentication-skip-lookup", // to survive unaccessible extensions-apiserver-authentication configmap + "--authentication-kubeconfig", brokenApiserverConfig.Name(), + "--authorization-kubeconfig", brokenApiserverConfig.Name(), + "--kubeconfig", apiserverConfig.Name(), + "--leader-elect=false", + }, "/healthz", false, false, intPtr(http.StatusOK), nil}, + {"not authorized /metrics", []string{ + "--port=0", + "--authentication-kubeconfig", apiserverConfig.Name(), + "--authorization-kubeconfig", apiserverConfig.Name(), + "--kubeconfig", apiserverConfig.Name(), + "--leader-elect=false", + }, "/metrics", false, false, intPtr(http.StatusForbidden), nil}, + {"not authorized /metrics with BROKEN authn/authz", []string{ + "--port=0", + "--authentication-kubeconfig", apiserverConfig.Name(), + "--authorization-kubeconfig", brokenApiserverConfig.Name(), + "--kubeconfig", apiserverConfig.Name(), + "--leader-elect=false", + }, "/metrics", false, false, intPtr(http.StatusInternalServerError), nil}, + {"always-allowed /metrics with BROKEN authn/authz", []string{ + "--port=0", + "--authentication-skip-lookup", // to survive unaccessible extensions-apiserver-authentication configmap + "--authentication-kubeconfig", apiserverConfig.Name(), + "--authorization-kubeconfig", apiserverConfig.Name(), + "--authorization-always-allow-paths", "/healthz,/metrics", + "--kubeconfig", apiserverConfig.Name(), + "--leader-elect=false", + }, "/metrics", false, false, intPtr(http.StatusOK), nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotResult, err := ctrlmgrtesting.StartTestServer(t, tt.flags) + if gotResult.TearDownFn != nil { + defer gotResult.TearDownFn() + } + if (err != nil) != tt.wantErr { + t.Fatalf("StartTestServer() error = %v, wantErr %v", err, tt.wantErr) + } + if err != nil { + return + } + + if want, got := tt.wantSecureCode != nil, gotResult.Config.SecureServing != nil; want != got { + t.Errorf("SecureServing enabled: expected=%v got=%v", want, got) + } else if want { + url := fmt.Sprintf("https://%s%s", gotResult.Config.SecureServing.Listener.Addr().String(), tt.path) + url = strings.Replace(url, "[::]", "127.0.0.1", -1) // switch to IPv4 because the self-signed cert does not support [::] + + // read self-signed server cert disk + pool := x509.NewCertPool() + serverCertPath := path.Join(gotResult.Options.SecureServing.ServerCert.CertDirectory, gotResult.Options.SecureServing.ServerCert.PairName+".crt") + serverCert, err := ioutil.ReadFile(serverCertPath) + if err != nil { + t.Fatalf("Failed to read controller-manager server cert %q: %v", serverCertPath, err) + } + pool.AppendCertsFromPEM(serverCert) + tr := &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: pool, + }, + } + + client := &http.Client{Transport: tr} + req, err := http.NewRequest("GET", url, nil) + if err != nil { + t.Fatal(err) + } + if !tt.anonymous { + req.Header.Add("Authorization", fmt.Sprintf("Token %s", token)) + } + r, err := client.Do(req) + if err != nil { + t.Fatalf("failed to GET %s from controller-manager: %v", tt.path, err) + } + + body, err := ioutil.ReadAll(r.Body) + defer r.Body.Close() + if got, expected := r.StatusCode, *tt.wantSecureCode; got != expected { + t.Fatalf("expected http %d at %s of controller-manager, got: %d %q", expected, tt.path, got, string(body)) + } + } + + if want, got := tt.wantInsecureCode != nil, gotResult.Config.InsecureServing != nil; want != got { + t.Errorf("InsecureServing enabled: expected=%v got=%v", want, got) + } else if want { + url := fmt.Sprintf("http://%s%s", gotResult.Config.InsecureServing.Listener.Addr().String(), tt.path) + r, err := http.Get(url) + if err != nil { + t.Fatalf("failed to GET %s from controller-manager: %v", tt.path, err) + } + body, err := ioutil.ReadAll(r.Body) + defer r.Body.Close() + if got, expected := r.StatusCode, *tt.wantInsecureCode; got != expected { + t.Fatalf("expected http %d at %s of controller-manager, got: %d %q", expected, tt.path, got, string(body)) + } + } + }) + } +} + +func intPtr(x int) *int { + return &x +}