From 4ba3218336c55dfae9b81d5bed3cc7296ec0a82f Mon Sep 17 00:00:00 2001 From: liz Date: Mon, 30 Apr 2018 10:00:36 -0400 Subject: [PATCH] Basic E2E tests for kubeadm Run with `go run test/e2e_kubeadm/runner/local/run_local.go -build -test-flags="--kubeconfig=$KUBECONFIG"` --- build/BUILD | 1 - hack/.golint_failures | 2 + test/e2e_kubeadm/BUILD | 3 + test/e2e_kubeadm/e2e_kubeadm_suite_test.go | 48 ++++++++ test/e2e_kubeadm/kubeadm_test.go | 136 +++++++++++++++++++++ test/e2e_kubeadm/matchers.go | 37 ++++++ test/e2e_kubeadm/runner/local/run_local.go | 106 ++++++++++++++++ 7 files changed, 332 insertions(+), 1 deletion(-) create mode 100644 test/e2e_kubeadm/e2e_kubeadm_suite_test.go create mode 100644 test/e2e_kubeadm/kubeadm_test.go create mode 100644 test/e2e_kubeadm/matchers.go create mode 100644 test/e2e_kubeadm/runner/local/run_local.go diff --git a/build/BUILD b/build/BUILD index 96c7d34c30..d325d73336 100644 --- a/build/BUILD +++ b/build/BUILD @@ -148,7 +148,6 @@ filegroup( "//cmd/kubemark", # TODO: server platforms only "//cmd/linkcheck", "//test/e2e:e2e.test", - "//test/e2e_kubeadm:e2e_kubeadm.test", # TODO: server platforms only "//test/e2e_node:e2e_node.test", # TODO: server platforms only "//vendor/github.com/onsi/ginkgo/ginkgo", ], diff --git a/hack/.golint_failures b/hack/.golint_failures index e6c0102f40..ff2e81a081 100644 --- a/hack/.golint_failures +++ b/hack/.golint_failures @@ -1,3 +1,4 @@ + cluster/images/etcd-version-monitor cmd/hyperkube cmd/kube-apiserver/app @@ -699,6 +700,7 @@ test/e2e/ui test/e2e/upgrades test/e2e/upgrades/apps test/e2e/upgrades/storage +test/e2e_kubeadm test/e2e_node test/e2e_node/builder test/e2e_node/environment diff --git a/test/e2e_kubeadm/BUILD b/test/e2e_kubeadm/BUILD index afa04ca59d..ead6cc65d8 100644 --- a/test/e2e_kubeadm/BUILD +++ b/test/e2e_kubeadm/BUILD @@ -1,3 +1,5 @@ +package(default_visibility = ["//visibility:public"]) + load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_test( @@ -7,6 +9,7 @@ go_test( "kubeadm_test.go", ], embed = [":go_default_library"], + tags = ["e2e"], deps = [ "//test/e2e/framework:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", diff --git a/test/e2e_kubeadm/e2e_kubeadm_suite_test.go b/test/e2e_kubeadm/e2e_kubeadm_suite_test.go new file mode 100644 index 0000000000..8fc3114b88 --- /dev/null +++ b/test/e2e_kubeadm/e2e_kubeadm_suite_test.go @@ -0,0 +1,48 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_kubeadm + +import ( + "flag" + "os" + "testing" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + "github.com/spf13/pflag" + + "k8s.io/kubernetes/test/e2e/framework" +) + +func init() { + framework.RegisterCommonFlags() + framework.RegisterClusterFlags() + + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) +} + +func TestMain(m *testing.M) { + pflag.Parse() + framework.AfterReadingAllFlags(&framework.TestContext) + os.Exit(m.Run()) +} + +func TestE2E(t *testing.T) { + reporters := []ginkgo.Reporter{} + gomega.RegisterFailHandler(ginkgo.Fail) + ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "E2EKubadm suite", reporters) +} diff --git a/test/e2e_kubeadm/kubeadm_test.go b/test/e2e_kubeadm/kubeadm_test.go new file mode 100644 index 0000000000..b8dc18c3c0 --- /dev/null +++ b/test/e2e_kubeadm/kubeadm_test.go @@ -0,0 +1,136 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_kubeadm + +import ( + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + bootstrapapi "k8s.io/client-go/tools/bootstrap/token/api" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const ( + masterTaint = "node-role.kubernetes.io/master" + kubeadmConfigNamespace = "kube-system" + kubeadmConfigName = "kubeadm-config" + clusterInfoNamespace = "kube-public" + clusterInfoName = "cluster-info" + bootstrapSignerRoleNamespace = "kube-system" + bootstrapSignerRoleName = "system:controller:bootstrap-signer" +) + +var _ = framework.KubeDescribe("Kubeadm [Feature:Kubeadm]", func() { + f := framework.NewDefaultFramework("kubeadm") + + Describe("kubeadm master", func() { + It("should be labelled and tainted", func() { + selector := labels.Set{masterTaint: ""}.AsSelector() + master, err := f.ClientSet.CoreV1().Nodes(). + List(metav1.ListOptions{LabelSelector: selector.String()}) + framework.ExpectNoError(err, "couldn't find a master node") + Expect(master.Items).NotTo(BeEmpty()) + for _, master := range master.Items { + Expect(master.Spec.Taints).To( + ContainElement(taint(masterTaint, corev1.TaintEffectNoSchedule)), + ) + } + }) + }) + + Describe("kubeadm-config config map", func() { + It("should exist", func() { + _, err := f.ClientSet.CoreV1(). + ConfigMaps(kubeadmConfigNamespace). + Get(kubeadmConfigName, metav1.GetOptions{}) + framework.ExpectNoError(err) + }) + }) + + Describe("cluster-info", func() { + It("should have expected keys", func() { + clientInfo, err := f.ClientSet.CoreV1(). + ConfigMaps(clusterInfoNamespace). + Get(clusterInfoName, metav1.GetOptions{}) + framework.ExpectNoError(err, "couldn't find config map") + + Expect(clientInfo.Data).To(HaveKey(HavePrefix(bootstrapapi.JWSSignatureKeyPrefix))) + Expect(clientInfo.Data).To(HaveKey(bootstrapapi.KubeConfigKey)) + }) + + It("should be public", func() { + cfg, err := framework.LoadConfig() + framework.ExpectNoError(err, "couldn't get config") + cfg = rest.AnonymousClientConfig(cfg) + client, err := kubernetes.NewForConfig(cfg) + framework.ExpectNoError(err, "couldn't create client") + + _, err = client.CoreV1().ConfigMaps(clusterInfoNamespace). + Get(clusterInfoName, metav1.GetOptions{}) + framework.ExpectNoError(err, "couldn't anonymously access config") + }) + }) + + Describe("bootstrap signer RBAC role", func() { + It("should exist", func() { + _, err := f.ClientSet.RbacV1(). + Roles(bootstrapSignerRoleNamespace). + Get(bootstrapSignerRoleName, metav1.GetOptions{}) + framework.ExpectNoError(err, "doesn't exist") + }) + }) + + Describe("kubeadm:kubelet-bootstrap cluster role binding", func() { + It("should exist", func() { + binding, err := f.ClientSet.RbacV1(). + ClusterRoleBindings(). + Get("kubeadm:kubelet-bootstrap", metav1.GetOptions{}) + framework.ExpectNoError(err, "couldn't get clusterrolebinding") + Expect(binding.Subjects).To( + ContainElement(subject( + "system:bootstrappers:kubeadm:default-node-token", + rbacv1.GroupKind, + )), + ) + Expect(binding.RoleRef.Name).To(Equal("system:node-bootstrapper")) + }) + }) + + Describe("autoapproval for new bootstrap token", func() { + It("should create a clusterrolebinding", func() { + binding, err := f.ClientSet.RbacV1(). + ClusterRoleBindings(). + Get("kubeadm:node-autoapprove-bootstrap", metav1.GetOptions{}) + framework.ExpectNoError(err, "couldn't get clusterrolebinding") + Expect(binding.Subjects).To( + ContainElement(subject( + "system:bootstrappers:kubeadm:default-node-token", + rbacv1.GroupKind, + )), + ) + Expect(binding.RoleRef.Name).To( + Equal("system:certificates.k8s.io:certificatesigningrequests:nodeclient"), + ) + }) + }) +}) diff --git a/test/e2e_kubeadm/matchers.go b/test/e2e_kubeadm/matchers.go new file mode 100644 index 0000000000..8af786616b --- /dev/null +++ b/test/e2e_kubeadm/matchers.go @@ -0,0 +1,37 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_kubeadm + +import ( + "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" + corev1 "k8s.io/api/core/v1" +) + +func subject(name, kind string) gomega.OmegaMatcher { + return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Name": gomega.Equal(name), + "Kind": gomega.Equal(kind), + }) +} + +func taint(key string, effect corev1.TaintEffect) gomega.OmegaMatcher { + return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "Key": gomega.Equal(key), + "Effect": gomega.Equal(effect), + }) +} diff --git a/test/e2e_kubeadm/runner/local/run_local.go b/test/e2e_kubeadm/runner/local/run_local.go new file mode 100644 index 0000000000..9ccdfcb730 --- /dev/null +++ b/test/e2e_kubeadm/runner/local/run_local.go @@ -0,0 +1,106 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/golang/glog" + "k8s.io/kubernetes/test/utils" +) + +func bazelBuild() error { + targets := []string{ + "//vendor/github.com/onsi/ginkgo/ginkgo", + "//test/e2e_kubeadm:e2e_kubeadm.test", + } + + args := append([]string{"build"}, targets...) + + return execCommand("bazel", args...) +} + +var ginkgoFlags = flag.String("ginkgo-flags", "", "Space-separated list of arguments to pass to Ginkgo test runner.") +var testFlags = flag.String("test-flags", "", "Space-separated list of arguments to pass to node e2e test.") +var build = flag.Bool("build", false, "use Bazel to build binaries before testing") + +func main() { + flag.Parse() + + if *build { + if err := bazelBuild(); err != nil { + glog.Exitf("couldn't build with bazel: %v", err) + } + } + + ginkgo, err := getBazelGinkgo() + if err != nil { + glog.Fatalf("Failed to get ginkgo binary: %v", err) + } + + test, err := getBazelTestBin() + if err != nil { + glog.Fatalf("Failed to get test file: %v", err) + } + + args := append(strings.Split(*ginkgoFlags, " "), test, "--") + args = append(args, strings.Split(*testFlags, " ")...) + + if execCommand(ginkgo, args...); err != nil { + glog.Exitf("Test failed: %v", err) + } + +} + +func getBazelTestBin() (string, error) { + k8sRoot, err := utils.GetK8sRootDir() + if err != nil { + return "", err + } + buildFile := filepath.Join(k8sRoot, "bazel-bin/test/e2e_kubeadm/e2e_kubeadm.test") + if _, err := os.Stat(buildFile); err != nil { + return "", err + } + return buildFile, nil + +} + +func getBazelGinkgo() (string, error) { + k8sRoot, err := utils.GetK8sRootDir() + if err != nil { + return "", err + } + buildOutputDir := filepath.Join(k8sRoot, "bazel-bin", "vendor/github.com/onsi/ginkgo/ginkgo", fmt.Sprintf("%s_%s_stripped", runtime.GOOS, runtime.GOARCH), "ginkgo") + if _, err := os.Stat(buildOutputDir); err != nil { + return "", err + } + return buildOutputDir, nil +} + +func execCommand(binary string, args ...string) error { + fmt.Printf("Running command: %v %v\n", binary, strings.Join(args, " ")) + cmd := exec.Command(binary, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +}