e2e-kubeadm-new-test

k3s-v1.15.3
fabriziopandini 2019-03-26 12:16:57 +01:00
parent 7dfcacd1cf
commit 5b675d6f00
8 changed files with 597 additions and 2 deletions

View File

@ -12,8 +12,13 @@ go_test(
"bootstrap_token_test.go", "bootstrap_token_test.go",
"cluster_info_test.go", "cluster_info_test.go",
"controlplane_nodes_test.go", "controlplane_nodes_test.go",
"dns_addon_test.go",
"e2e_kubeadm_suite_test.go", "e2e_kubeadm_suite_test.go",
"kubeadm_certs_test.go",
"kubeadm_config_test.go", "kubeadm_config_test.go",
"kubelet_config_test.go",
"nodes_test.go",
"proxy_addon_test.go",
], ],
out = "e2e_kubeadm.test", out = "e2e_kubeadm.test",
embed = [":go_default_library"], embed = [":go_default_library"],
@ -24,6 +29,7 @@ go_test(
"//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library",
"//test/e2e/framework:go_default_library", "//test/e2e/framework:go_default_library",
@ -63,6 +69,7 @@ go_library(
importpath = "k8s.io/kubernetes/test/e2e_kubeadm", importpath = "k8s.io/kubernetes/test/e2e_kubeadm",
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
deps = [ deps = [
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/authorization/v1:go_default_library", "//staging/src/k8s.io/api/authorization/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library",

View File

@ -21,6 +21,4 @@ const (
kubeSystemNamespace = "kube-system" kubeSystemNamespace = "kube-system"
anonymousUser = "system:anonymous" anonymousUser = "system:anonymous"
nodesGroup = "system:nodes"
) )

View File

@ -0,0 +1,150 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e_kubeadm
import (
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
const (
dnsService = "kube-dns"
coreDNSServiceAccountName = "coredns"
coreDNSConfigMap = "coredns"
coreDNSConfigMapKey = "Corefile"
coreDNSRoleName = "system:coredns"
coreDNSRoleBindingName = coreDNSRoleName
coreDNSDeploymentName = "coredns"
kubeDNSServiceAccountName = "kube-dns"
kubeDNSDeploymentName = "kube-dns"
)
var (
dnsType = ""
)
// Define container for all the test specification aimed at verifying
// that kubeadm configures the dns as expected
var _ = KubeadmDescribe("DNS addon", func() {
// Get an instance of the k8s test framework
f := framework.NewDefaultFramework("DNS")
// Tests in this container are not expected to create new objects in the cluster
// so we are disabling the creation of a namespace in order to get a faster execution
f.SkipNamespaceCreation = true
// kubeadm supports two type of DNS addon, and so
// it is necessary to get it from the kubeadm-config ConfigMap before testing
BeforeEach(func() {
// if the dnsType name is already known exit
if dnsType != "" {
return
}
// gets the ClusterConfiguration from the kubeadm kubeadm-config ConfigMap as a untyped map
m := getClusterConfiguration(f.ClientSet)
// Extract the dnsType
dnsType = "CoreDNS"
if _, ok := m["dns"]; ok {
d := m["dns"].(map[interface{}]interface{})
if t, ok := d["type"]; ok {
dnsType = t.(string)
}
}
})
Context("kube-dns", func() {
Context("kube-dns ServiceAccount", func() {
It("should exist", func() {
if dnsType != "kube-dns" {
framework.Skipf("Skipping because DNS type is %s", dnsType)
}
ExpectServiceAccount(f.ClientSet, kubeSystemNamespace, kubeDNSServiceAccountName)
})
})
Context("kube-dns Deployment", func() {
It("should exist and be properly configured", func() {
if dnsType != "kube-dns" {
framework.Skipf("Skipping because DNS type is %s", dnsType)
}
d := GetDeployment(f.ClientSet, kubeSystemNamespace, kubeDNSDeploymentName)
Expect(d.Spec.Template.Spec.ServiceAccountName).To(Equal(kubeDNSServiceAccountName))
})
})
})
Context("CoreDNS", func() {
Context("CoreDNS ServiceAccount", func() {
It("should exist", func() {
if dnsType != "CoreDNS" {
framework.Skipf("Skipping because DNS type is %s", dnsType)
}
ExpectServiceAccount(f.ClientSet, kubeSystemNamespace, coreDNSServiceAccountName)
})
It("should have related ClusterRole and ClusterRoleBinding", func() {
if dnsType != "CoreDNS" {
framework.Skipf("Skipping because DNS type is %s", dnsType)
}
ExpectClusterRole(f.ClientSet, coreDNSRoleName)
ExpectClusterRoleBinding(f.ClientSet, coreDNSRoleBindingName)
})
})
Context("CoreDNS ConfigMap", func() {
It("should exist and be properly configured", func() {
if dnsType != "CoreDNS" {
framework.Skipf("Skipping because DNS type is %s", dnsType)
}
cm := GetConfigMap(f.ClientSet, kubeSystemNamespace, coreDNSConfigMap)
Expect(cm.Data).To(HaveKey(coreDNSConfigMapKey))
})
})
Context("CoreDNS Deployment", func() {
It("should exist and be properly configured", func() {
if dnsType != "CoreDNS" {
framework.Skipf("Skipping because DNS type is %s", dnsType)
}
d := GetDeployment(f.ClientSet, kubeSystemNamespace, coreDNSDeploymentName)
Expect(d.Spec.Template.Spec.ServiceAccountName).To(Equal(coreDNSServiceAccountName))
})
})
})
Context("DNS Service", func() {
It("should exist", func() {
ExpectService(f.ClientSet, kubeSystemNamespace, dnsService)
})
})
})

View File

@ -0,0 +1,116 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e_kubeadm
import (
"fmt"
authv1 "k8s.io/api/authorization/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
const (
kubeadmCertsSecretName = "kubeadm-certs"
)
var (
kubeadmCertsRoleName = fmt.Sprintf("kubeadm:%s", kubeadmCertsSecretName)
kubeadmCertsRoleBindingName = kubeadmCertsRoleName
kubeadmCertsSecretResource = &authv1.ResourceAttributes{
Namespace: kubeSystemNamespace,
Name: kubeadmCertsSecretName,
Resource: "secrets",
Verb: "get",
}
)
// Define container for all the test specification aimed at verifying
// that kubeadm creates the kubeadm-certs Secret, that it is properly configured
// and that all the related RBAC rules are in place
// Important! please note that kubeadm-certs is not created by default (still alpha)
// in case you want to skip this test use SKIP=copy-certs
var _ = KubeadmDescribe("kubeadm-certs [copy-certs]", func() {
// Get an instance of the k8s test framework
f := framework.NewDefaultFramework("kubeadm-certs")
// Tests in this container are not expected to create new objects in the cluster
// so we are disabling the creation of a namespace in order to get a faster execution
f.SkipNamespaceCreation = true
It("should exist and be properly configured", func() {
s := GetSecret(f.ClientSet, kubeSystemNamespace, kubeadmCertsSecretName)
// Checks the kubeadm-certs is ownen by a time lived token
Expect(s.OwnerReferences).To(HaveLen(1), "%s should have one owner reference", kubeadmCertsSecretName)
ownRef := s.OwnerReferences[0]
Expect(ownRef.Kind).To(Equal("Secret"), "%s should be owned by a secret", kubeadmCertsSecretName)
Expect(*ownRef.BlockOwnerDeletion).To(BeTrue(), "%s should be deleted on owner deletion", kubeadmCertsSecretName)
o := GetSecret(f.ClientSet, kubeSystemNamespace, ownRef.Name)
Expect(o.Type).To(Equal(corev1.SecretTypeBootstrapToken), "%s should have an owner reference that refers to a bootstrap-token", kubeadmCertsSecretName)
Expect(o.Data).To(HaveKey("expiration"), "%s should have an owner reference with an expiration", kubeadmCertsSecretName)
// gets the ClusterConfiguration from the kubeadm kubeadm-config ConfigMap as a untyped map
m := getClusterConfiguration(f.ClientSet)
// Extract the etcd Type
etcdType := "local"
if _, ok := m["etcd"]; ok {
d := m["etcd"].(map[interface{}]interface{})
if _, ok := d["external"]; ok {
etcdType = "external"
}
}
// check if all the expected key exists
Expect(s.Data).To(HaveKey("ca.crt"))
Expect(s.Data).To(HaveKey("ca.key"))
Expect(s.Data).To(HaveKey("front-proxy-ca.crt"))
Expect(s.Data).To(HaveKey("front-proxy-ca.key"))
Expect(s.Data).To(HaveKey("sa.pub"))
Expect(s.Data).To(HaveKey("sa.key"))
if etcdType == "local" {
Expect(s.Data).To(HaveKey("etcd-ca.crt"))
Expect(s.Data).To(HaveKey("etcd-ca.key"))
} else {
Expect(s.Data).To(HaveKey("external-etcd-ca.crt"))
Expect(s.Data).To(HaveKey("external-etcd.crt"))
Expect(s.Data).To(HaveKey("external-etcd.key"))
}
})
It("should have related Role and RoleBinding", func() {
ExpectRole(f.ClientSet, kubeSystemNamespace, kubeadmCertsRoleName)
ExpectRoleBinding(f.ClientSet, kubeSystemNamespace, kubeadmCertsRoleBindingName)
})
It("should be accessible for bootstrap tokens", func() {
ExpectSubjectHasAccessToResource(f.ClientSet,
rbacv1.GroupKind, bootstrapTokensGroup,
kubeadmCertsSecretResource,
)
})
})

View File

@ -0,0 +1,110 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e_kubeadm
import (
"fmt"
authv1 "k8s.io/api/authorization/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
const (
kubeletConfigConfigMapKey = "kubelet"
)
var (
kubeletConfigConfigMapName string
kubeletConfigRoleName string
kubeletConfigRoleBindingName string
kubeletConfigConfigMapResource = &authv1.ResourceAttributes{
Namespace: kubeSystemNamespace,
Name: "",
Resource: "configmaps",
Verb: "get",
}
)
// Define container for all the test specification aimed at verifying
// that kubeadm creates the kubelet-config ConfigMap, that it is properly configured
// and that all the related RBAC rules are in place
var _ = KubeadmDescribe("kubelet-config ConfigMap", func() {
// Get an instance of the k8s test framework
f := framework.NewDefaultFramework("kubelet-config")
// Tests in this container are not expected to create new objects in the cluster
// so we are disabling the creation of a namespace in order to get a faster execution
f.SkipNamespaceCreation = true
// kubelet-config map is named using the kubernetesVersion as a suffix, and so
// it is necessary to get it from the kubeadm-config ConfigMap before testing
BeforeEach(func() {
// if the kubelet-config map name is already known exit
if kubeletConfigConfigMapName != "" {
return
}
// gets the ClusterConfiguration from the kubeadm kubeadm-config ConfigMap as a untyped map
m := getClusterConfiguration(f.ClientSet)
// Extract the kubernetesVersion
Expect(m).To(HaveKey("kubernetesVersion"))
k8sVersionString := m["kubernetesVersion"].(string)
k8sVersion, err := version.ParseSemantic(k8sVersionString)
if err != nil {
framework.Failf("error reading kubernetesVersion from %s ConfigMap: %v", kubeadmConfigName, err)
}
// Computes all the names derived from the kubernetesVersion
kubeletConfigConfigMapName = fmt.Sprintf("kubelet-config-%d.%d", k8sVersion.Major(), k8sVersion.Minor())
kubeletConfigRoleName = fmt.Sprintf("kubeadm:kubelet-config-%d.%d", k8sVersion.Major(), k8sVersion.Minor())
kubeletConfigRoleBindingName = kubeletConfigRoleName
kubeletConfigConfigMapResource.Name = kubeletConfigConfigMapName
})
It("should exist and be properly configured", func() {
cm := GetConfigMap(f.ClientSet, kubeSystemNamespace, kubeletConfigConfigMapName)
Expect(cm.Data).To(HaveKey(kubeletConfigConfigMapKey))
})
It("should have related Role and RoleBinding", func() {
ExpectRole(f.ClientSet, kubeSystemNamespace, kubeletConfigRoleName)
ExpectRoleBinding(f.ClientSet, kubeSystemNamespace, kubeletConfigRoleBindingName)
})
It("should be accessible for bootstrap tokens", func() {
ExpectSubjectHasAccessToResource(f.ClientSet,
rbacv1.GroupKind, bootstrapTokensGroup,
kubeadmConfigConfigMapResource,
)
})
It("should be accessible for nodes", func() {
ExpectSubjectHasAccessToResource(f.ClientSet,
rbacv1.GroupKind, nodesGroup,
kubeadmConfigConfigMapResource,
)
})
})

View File

@ -0,0 +1,65 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e_kubeadm
import (
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
const (
nodesGroup = "system:nodes"
nodesCertificateRotationClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:selfnodeclient"
nodesCertificateRotationClusterRoleBinding = "kubeadm:node-autoapprove-certificate-rotation"
nodesCRISocketAnnotation = "kubeadm.alpha.kubernetes.io/cri-socket"
)
// Define container for all the test specification aimed at verifying
// that kubeadm configures the nodes and system:nodes group as expected
var _ = KubeadmDescribe("nodes", func() {
// Get an instance of the k8s test framework
f := framework.NewDefaultFramework("nodes")
// Tests in this container are not expected to create new objects in the cluster
// so we are disabling the creation of a namespace in order to get a faster execution
f.SkipNamespaceCreation = true
It("should have CRI annotation", func() {
nodes, err := f.ClientSet.CoreV1().Nodes().
List(metav1.ListOptions{})
framework.ExpectNoError(err, "error reading nodes")
// checks that the nodes have the CRI annotation
for _, node := range nodes.Items {
Expect(node.Annotations).To(HaveKey(nodesCRISocketAnnotation))
}
})
It("should be allowed to rotate CSR", func() {
// Nb. this is technically implemented a part of the bootstrap-token phase
ExpectClusterRoleBindingWithSubjectAndRole(f.ClientSet,
nodesCertificateRotationClusterRoleBinding,
rbacv1.GroupKind, nodesGroup,
nodesCertificateRotationClusterRoleName,
)
})
})

View File

@ -0,0 +1,100 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e_kubeadm
import (
authv1 "k8s.io/api/authorization/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
const (
kubeProxyServiceAccountName = "kube-proxy"
kubeProxyConfigMap = "kube-proxy"
kubeProxyConfigMapKey = "config.conf"
kubeProxyClusterRoleName = "system:node-proxier"
kubeProxyClusterRoleBindingName = "kubeadm:node-proxier"
kubeProxyRoleName = "kube-proxy"
kubeProxyRoleBindingName = kubeProxyRoleName
kubeProxyDaemonSetName = "kube-proxy"
)
var (
kubeProxyConfigMapResource = &authv1.ResourceAttributes{
Namespace: kubeSystemNamespace,
Name: kubeProxyConfigMap,
Resource: "configmaps",
Verb: "get",
}
)
// Define container for all the test specification aimed at verifying
// that kubeadm configures the proxy addon as expected
var _ = KubeadmDescribe("proxy addon", func() {
// Get an instance of the k8s test framework
f := framework.NewDefaultFramework("proxy")
// Tests in this container are not expected to create new objects in the cluster
// so we are disabling the creation of a namespace in order to get a faster execution
f.SkipNamespaceCreation = true
Context("kube-proxy ServiceAccount", func() {
It("should exist", func() {
ExpectServiceAccount(f.ClientSet, kubeSystemNamespace, kubeProxyServiceAccountName)
})
It("should be binded to the system:node-proxier cluster role", func() {
ExpectClusterRoleBindingWithSubjectAndRole(f.ClientSet,
kubeProxyClusterRoleBindingName,
rbacv1.ServiceAccountKind, kubeProxyServiceAccountName,
kubeProxyClusterRoleName,
)
})
})
Context("kube-proxy ConfigMap", func() {
It("should exist and be properly configured", func() {
cm := GetConfigMap(f.ClientSet, kubeSystemNamespace, kubeProxyConfigMap)
Expect(cm.Data).To(HaveKey(kubeProxyConfigMapKey))
})
It("should have related Role and RoleBinding", func() {
ExpectRole(f.ClientSet, kubeSystemNamespace, kubeProxyRoleName)
ExpectRoleBinding(f.ClientSet, kubeSystemNamespace, kubeProxyRoleBindingName)
})
It("should be accessible by bootstrap tokens", func() {
ExpectSubjectHasAccessToResource(f.ClientSet,
rbacv1.GroupKind, bootstrapTokensGroup,
kubeProxyConfigMapResource,
)
})
})
Context("kube-proxy DeamonSet", func() {
It("should exist and be properly configured", func() {
ds := GetDaemonSet(f.ClientSet, kubeSystemNamespace, kubeProxyDaemonSetName)
Expect(ds.Spec.Template.Spec.ServiceAccountName).To(Equal(kubeProxyServiceAccountName))
})
})
})

View File

@ -17,6 +17,7 @@ limitations under the License.
package e2e_kubeadm package e2e_kubeadm
import ( import (
appsv1 "k8s.io/api/apps/v1"
authv1 "k8s.io/api/authorization/v1" authv1 "k8s.io/api/authorization/v1"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1" rbacv1 "k8s.io/api/rbac/v1"
@ -28,6 +29,25 @@ import (
"github.com/onsi/gomega/gstruct" "github.com/onsi/gomega/gstruct"
) )
// ServiceAccounts utils
func ExpectServiceAccount(c clientset.Interface, namespace, name string) {
_, err := c.CoreV1().
ServiceAccounts(namespace).
Get(name, metav1.GetOptions{})
framework.ExpectNoError(err, "error getting ServiceAccount %q from namespace %q", name, namespace)
}
// Secret utils
func GetSecret(c clientset.Interface, namespace, name string) *corev1.Secret {
r, err := c.CoreV1().
Secrets(namespace).
Get(name, metav1.GetOptions{})
framework.ExpectNoError(err, "error getting Secret %q from namespace %q", name, namespace)
return r
}
// ConfigMaps utils // ConfigMaps utils
func GetConfigMap(c clientset.Interface, namespace, name string) *corev1.ConfigMap { func GetConfigMap(c clientset.Interface, namespace, name string) *corev1.ConfigMap {
@ -38,6 +58,35 @@ func GetConfigMap(c clientset.Interface, namespace, name string) *corev1.ConfigM
return r return r
} }
// Service utils
func ExpectService(c clientset.Interface, namespace, name string) {
_, err := c.CoreV1().
Services(namespace).
Get(name, metav1.GetOptions{})
framework.ExpectNoError(err, "error getting Service %q from namespace %q", name, namespace)
}
// Deployments utils
func GetDeployment(c clientset.Interface, namespace, name string) *appsv1.Deployment {
r, err := c.AppsV1().
Deployments(namespace).
Get(name, metav1.GetOptions{})
framework.ExpectNoError(err, "error getting Deployment %q from namespace %q", name, namespace)
return r
}
// DaemonSets utils
func GetDaemonSet(c clientset.Interface, namespace, name string) *appsv1.DaemonSet {
r, err := c.AppsV1().
DaemonSets(namespace).
Get(name, metav1.GetOptions{})
framework.ExpectNoError(err, "error getting DaemonSet %q from namespace %q", name, namespace)
return r
}
// RBAC utils // RBAC utils
func ExpectRole(c clientset.Interface, namespace, name string) { func ExpectRole(c clientset.Interface, namespace, name string) {