github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

E2E Test

pull/58/head
Mehdy Bohlool 2018-08-15 16:16:44 -07:00
parent 8235e389fb
commit ea54a0c504
4 changed files with 466 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

396
test/e2e/apimachinery/crd_conversion_webhook.go Normal file
View File

@ -0,0 +1,396 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package apimachinery
import (
"k8s.io/kubernetes/staging/src/k8s.io/apiextensions-apiserver/test/integration"
"time"
apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/intstr"
utilversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/client-go/dynamic"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
imageutils "k8s.io/kubernetes/test/utils/image"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
_ "github.com/stretchr/testify/assert"
)
const (
secretCRDName = "sample-custom-resource-conversion-webhook-secret"
deploymentCRDName = "sample-crd-conversion-webhook-deployment"
serviceCRDName = "e2e-test-crd-conversion-webhook"
roleBindingCRDName = "crd-conversion-webhook-auth-reader"
)
var serverCRDConversionWebhookVersion = utilversion.MustParseSemantic("v1.13.0-alpha")
var apiVersions = []v1beta1.CustomResourceDefinitionVersion{
{
Name: "v1",
Served: true,
Storage: true,
},
{
Name: "v2",
Served: true,
Storage: false,
},
}
var alternativeApiVersions = []v1beta1.CustomResourceDefinitionVersion{
{
Name: "v1",
Served: true,
Storage: false,
},
{
Name: "v2",
Served: true,
Storage: true,
},
}
var _ = SIGDescribe("CustomResourceConversionWebhook [Feature:CustomResourceWebhookConversion]", func() {
var context *certContext
f := framework.NewDefaultFramework("crd-webhook")
var client clientset.Interface
var namespaceName string
BeforeEach(func() {
client = f.ClientSet
namespaceName = f.Namespace.Name
// Make sure the relevant provider supports conversion webhook
framework.SkipUnlessServerVersionGTE(serverCRDConversionWebhookVersion, f.ClientSet.Discovery())
By("Setting up server cert")
context = setupServerCert(f.Namespace.Name, serviceCRDName)
createAuthReaderRoleBindingForCRDConversion(f, f.Namespace.Name)
deployCustomResourceWebhookAndService(f, imageutils.GetE2EImage(imageutils.CRDConversionWebhook), context)
})
AfterEach(func() {
cleanCRDWebhookTest(client, namespaceName)
})
It("Should be able to convert from CR v1 to CR v2", func() {
testcrd, err := framework.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions,
&v1beta1.WebhookClientConfig{
CABundle: context.signingCert,
Service: &v1beta1.ServiceReference{
Namespace: f.Namespace.Name,
Name: serviceCRDName,
Path: strPtr("/crdconvert"),
}})
if err != nil {
return
}
defer testcrd.CleanUp()
testCustomResourceConversionWebhook(f, testcrd.Crd, testcrd.DynamicClients)
})
It("Should be able to convert a non homogeneous list of CRs", func() {
testcrd, err := framework.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions,
&v1beta1.WebhookClientConfig{
CABundle: context.signingCert,
Service: &v1beta1.ServiceReference{
Namespace: f.Namespace.Name,
Name: serviceCRDName,
Path: strPtr("/crdconvert"),
}})
if err != nil {
return
}
defer testcrd.CleanUp()
testCRListConversion(f, testcrd)
})
})
func cleanCRDWebhookTest(client clientset.Interface, namespaceName string) {
_ = client.CoreV1().Services(namespaceName).Delete(serviceCRDName, nil)
_ = client.AppsV1().Deployments(namespaceName).Delete(deploymentCRDName, nil)
_ = client.CoreV1().Secrets(namespaceName).Delete(secretCRDName, nil)
_ = client.RbacV1().RoleBindings("kube-system").Delete(roleBindingCRDName, nil)
}
func createAuthReaderRoleBindingForCRDConversion(f *framework.Framework, namespace string) {
By("Create role binding to let cr conversion webhook read extension-apiserver-authentication")
client := f.ClientSet
// Create the role binding to allow the webhook read the extension-apiserver-authentication configmap
_, err := client.RbacV1().RoleBindings("kube-system").Create(&rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: roleBindingCRDName,
},
RoleRef: rbacv1.RoleRef{
APIGroup: "",
Kind: "Role",
Name: "extension-apiserver-authentication-reader",
},
// Webhook uses the default service account.
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: "default",
Namespace: namespace,
},
},
})
if err != nil && errors.IsAlreadyExists(err) {
framework.Logf("role binding %s already exists", roleBindingCRDName)
} else {
framework.ExpectNoError(err, "creating role binding %s:webhook to access configMap", namespace)
}
}
func deployCustomResourceWebhookAndService(f *framework.Framework, image string, context *certContext) {
By("Deploying the custom resource conversion webhook pod")
client := f.ClientSet
// Creating the secret that contains the webhook's cert.
secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretCRDName,
},
Type: v1.SecretTypeOpaque,
Data: map[string][]byte{
"tls.crt": context.cert,
"tls.key": context.key,
},
}
namespace := f.Namespace.Name
_, err := client.CoreV1().Secrets(namespace).Create(secret)
framework.ExpectNoError(err, "creating secret %q in namespace %q", secretName, namespace)
// Create the deployment of the webhook
podLabels := map[string]string{"app": "sample-crd-conversion-webhook", "crd-webhook": "true"}
replicas := int32(1)
zero := int64(0)
mounts := []v1.VolumeMount{
{
Name: "crd-conversion-webhook-certs",
ReadOnly: true,
MountPath: "/webhook.local.config/certificates",
},
}
volumes := []v1.Volume{
{
Name: "crd-conversion-webhook-certs",
VolumeSource: v1.VolumeSource{
Secret: &v1.SecretVolumeSource{SecretName: secretCRDName},
},
},
}
containers := []v1.Container{
{
Name: "sample-crd-conversion-webhook",
VolumeMounts: mounts,
Args: []string{
"--tls-cert-file=/webhook.local.config/certificates/tls.crt",
"--tls-private-key-file=/webhook.local.config/certificates/tls.key",
"--alsologtostderr",
"-v=4",
"2>&1",
},
Image: image,
},
}
d := &apps.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: deploymentCRDName,
Labels: podLabels,
},
Spec: apps.DeploymentSpec{
Replicas: &replicas,
Selector: &metav1.LabelSelector{
MatchLabels: podLabels,
},
Strategy: apps.DeploymentStrategy{
Type: apps.RollingUpdateDeploymentStrategyType,
},
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: podLabels,
},
Spec: v1.PodSpec{
TerminationGracePeriodSeconds: &zero,
Containers: containers,
Volumes: volumes,
},
},
},
}
deployment, err := client.AppsV1().Deployments(namespace).Create(d)
framework.ExpectNoError(err, "creating deployment %s in namespace %s", deploymentCRDName, namespace)
By("Wait for the deployment to be ready")
err = framework.WaitForDeploymentRevisionAndImage(client, namespace, deploymentCRDName, "1", image)
framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", image, deploymentName, namespace)
err = framework.WaitForDeploymentComplete(client, deployment)
framework.ExpectNoError(err, "waiting for the deployment status valid", image, deploymentCRDName, namespace)
By("Deploying the webhook service")
serviceLabels := map[string]string{"crd-webhook": "true"}
service := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: serviceCRDName,
Labels: map[string]string{"test": "crd-webhook"},
},
Spec: v1.ServiceSpec{
Selector: serviceLabels,
Ports: []v1.ServicePort{
{
Protocol: "TCP",
Port: 443,
TargetPort: intstr.FromInt(443),
},
},
},
}
_, err = client.CoreV1().Services(namespace).Create(service)
framework.ExpectNoError(err, "creating service %s in namespace %s", serviceCRDName, namespace)
By("Verifying the service has paired with the endpoint")
err = framework.WaitForServiceEndpointsNum(client, namespace, serviceCRDName, 1, 1*time.Second, 30*time.Second)
framework.ExpectNoError(err, "waiting for service %s/%s have %d endpoint", namespace, serviceCRDName, 1)
}
func verifyV1Object(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, obj *unstructured.Unstructured) {
Expect(obj.GetAPIVersion()).To(BeEquivalentTo(crd.Spec.Group + "/v1"))
hostPort, exists := obj.Object["hostPort"]
Expect(exists).To(BeTrue())
Expect(hostPort).To(BeEquivalentTo("localhost:8080"))
_, hostExists := obj.Object["host"]
Expect(hostExists).To(BeFalse())
_, portExists := obj.Object["port"]
Expect(portExists).To(BeFalse())
}
func verifyV2Object(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, obj *unstructured.Unstructured) {
Expect(obj.GetAPIVersion()).To(BeEquivalentTo(crd.Spec.Group + "/v2"))
_, hostPortExists := obj.Object["hostPort"]
Expect(hostPortExists).To(BeFalse())
host, hostExists := obj.Object["host"]
Expect(hostExists).To(BeTrue())
Expect(host).To(BeEquivalentTo("localhost"))
port, portExists := obj.Object["port"]
Expect(portExists).To(BeTrue())
Expect(port).To(BeEquivalentTo("8080"))
}
func testCustomResourceConversionWebhook(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, customResourceClients map[string]dynamic.ResourceInterface) {
name := "cr-instance-1"
By("Creating a v1 custom resource")
crInstance := &unstructured.Unstructured{
Object: map[string]interface{}{
"kind": crd.Spec.Names.Kind,
"apiVersion": crd.Spec.Group + "/v1",
"metadata": map[string]interface{}{
"name": name,
"namespace": f.Namespace.Name,
},
"hostPort": "localhost:8080",
},
}
_, err := customResourceClients["v1"].Create(crInstance, metav1.CreateOptions{})
Expect(err).To(BeNil())
By("v2 custom resource should be converted")
v2crd, err := customResourceClients["v2"].Get(name, metav1.GetOptions{})
verifyV2Object(f, crd, v2crd)
}
func testCRListConversion(f *framework.Framework, testCrd *framework.TestCrd) {
crd := testCrd.Crd
customResourceClients := testCrd.DynamicClients
name1 := "cr-instance-1"
name2 := "cr-instance-2"
By("Creating a v1 custom resource")
crInstance := &unstructured.Unstructured{
Object: map[string]interface{}{
"kind": crd.Spec.Names.Kind,
"apiVersion": crd.Spec.Group + "/v1",
"metadata": map[string]interface{}{
"name": name1,
"namespace": f.Namespace.Name,
},
"hostPort": "localhost:8080",
},
}
_, err := customResourceClients["v1"].Create(crInstance, metav1.CreateOptions{})
Expect(err).To(BeNil())
// Now cr-instance-1 is stored as v1. lets change storage version
crd, err = integration.UpdateCustomResourceDefinitionWithRetry(testCrd.ApiExtensionClient, crd.Name, func(c *v1beta1.CustomResourceDefinition) {
c.Spec.Versions = alternativeApiVersions
})
Expect(err).To(BeNil())
By("Create a v2 custom resource")
crInstance = &unstructured.Unstructured{
Object: map[string]interface{}{
"kind": crd.Spec.Names.Kind,
"apiVersion": crd.Spec.Group + "/v1",
"metadata": map[string]interface{}{
"name": name2,
"namespace": f.Namespace.Name,
},
"hostPort": "localhost:8080",
},
}
// After changing a CRD, the resources for versions will be re-created that can be result in
// cancelled connection (e.g. "grpc connection closed" or "context canceled").
// Just retrying fixes that.
for i := 0; i < 5; i++ {
_, err = customResourceClients["v1"].Create(crInstance, metav1.CreateOptions{})
if err == nil {
break
}
}
Expect(err).To(BeNil())
// Now that we have a v1 and v2 object, both list operation in v1 and v2 should work as expected.
By("List CRs in v1")
list, err := customResourceClients["v1"].List(metav1.ListOptions{})
Expect(err).To(BeNil())
Expect(len(list.Items)).To(BeIdenticalTo(2))
Expect((list.Items[0].GetName() == name1 && list.Items[1].GetName() == name2) ||
(list.Items[0].GetName() == name2 && list.Items[1].GetName() == name1)).To(BeTrue())
verifyV1Object(f, crd, &list.Items[0])
verifyV1Object(f, crd, &list.Items[1])
By("List CRs in v2")
list, err = customResourceClients["v2"].List(metav1.ListOptions{})
Expect(err).To(BeNil())
Expect(len(list.Items)).To(BeIdenticalTo(2))
Expect((list.Items[0].GetName() == name1 && list.Items[1].GetName() == name2) ||
(list.Items[0].GetName() == name2 && list.Items[1].GetName() == name1)).To(BeTrue())
verifyV2Object(f, crd, &list.Items[0])
verifyV2Object(f, crd, &list.Items[1])
}

30
test/e2e/apimachinery/webhook.go
View File

@ -136,7 +136,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() {
defer testcrd.CleanUp()
webhookCleanup := registerWebhookForCustomResource(f, context, testcrd)
defer webhookCleanup()
testCustomResourceWebhook(f, testcrd.Crd, testcrd.DynamicClient)
testCustomResourceWebhook(f, testcrd.Crd, testcrd.GetV1DynamicClient())
})
It("Should unconditionally reject operations on fail closed webhook", func() {
@ -173,7 +173,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() {
defer testcrd.CleanUp()
webhookCleanup := registerMutatingWebhookForCustomResource(f, context, testcrd)
defer webhookCleanup()
testMutatingCustomResourceWebhook(f, testcrd.Crd, testcrd.DynamicClient)
testMutatingCustomResourceWebhook(f, testcrd.Crd, testcrd.GetV1DynamicClient())
})
It("Should deny crd creation", func() {
@ -1157,7 +1157,7 @@ func registerWebhookForCustomResource(f *framework.Framework, context *certConte
Operations: []v1beta1.OperationType{v1beta1.Create},
Rule: v1beta1.Rule{
APIGroups: []string{testcrd.ApiGroup},
APIVersions: []string{testcrd.ApiVersion},
APIVersions: testcrd.GetAPIVersions(),
Resources: []string{testcrd.GetPluralName()},
},
}},
@ -1198,7 +1198,7 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, context *c
Operations: []v1beta1.OperationType{v1beta1.Create},
Rule: v1beta1.Rule{
APIGroups: []string{testcrd.ApiGroup},
APIVersions: []string{testcrd.ApiVersion},
APIVersions: testcrd.GetAPIVersions(),
Resources: []string{testcrd.GetPluralName()},
},
}},
@ -1217,7 +1217,7 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, context *c
Operations: []v1beta1.OperationType{v1beta1.Create},
Rule: v1beta1.Rule{
APIGroups: []string{testcrd.ApiGroup},
APIVersions: []string{testcrd.ApiVersion},
APIVersions: testcrd.GetAPIVersions(),
Resources: []string{testcrd.GetPluralName()},
},
}},
@ -1343,12 +1343,18 @@ func testCRDDenyWebhook(f *framework.Framework) {
name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, "deny")
kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, "deny")
group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName)
apiVersion := "v1"
apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{
{
Name: "v1",
Served: true,
Storage: true,
},
}
testcrd := &framework.TestCrd{
Name: name,
Kind: kind,
ApiGroup: group,
ApiVersion: apiVersion,
Name: name,
Kind: kind,
ApiGroup: group,
Versions: apiVersions,
}
// Creating a custom resource definition for use by assorted tests.
@ -1370,8 +1376,8 @@ func testCRDDenyWebhook(f *framework.Framework) {
},
},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: testcrd.ApiGroup,
Version: testcrd.ApiVersion,
Group: testcrd.ApiGroup,
Versions: testcrd.Versions,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Plural: testcrd.GetPluralName(),
Singular: testcrd.Name,

65
test/e2e/framework/crd_util.go
View File

@ -35,25 +35,23 @@ type TestCrd struct {
Name string
Kind string
ApiGroup string
ApiVersion string
Versions []apiextensionsv1beta1.CustomResourceDefinitionVersion
ApiExtensionClient *crdclientset.Clientset
Crd *apiextensionsv1beta1.CustomResourceDefinition
DynamicClient dynamic.ResourceInterface
DynamicClients map[string]dynamic.ResourceInterface
CleanUp CleanCrdFn
}
// CreateTestCRD creates a new CRD specifically for the calling test.
func CreateTestCRD(f *Framework) (*TestCrd, error) {
func CreateMultiVersionTestCRD(f *Framework, group string, apiVersions []apiextensionsv1beta1.CustomResourceDefinitionVersion, conversionWebhook *apiextensionsv1beta1.WebhookClientConfig) (*TestCrd, error) {
suffix := randomSuffix()
name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, suffix)
kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, suffix)
group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName)
apiVersion := "v1"
testcrd := &TestCrd{
Name: name,
Kind: kind,
ApiGroup: group,
ApiVersion: apiVersion,
Name: name,
Kind: kind,
ApiGroup: group,
Versions: apiVersions,
}
// Creating a custom resource definition for use by assorted tests.
@ -75,6 +73,13 @@ func CreateTestCRD(f *Framework) (*TestCrd, error) {
crd := newCRDForTest(testcrd)
if conversionWebhook != nil {
crd.Spec.Conversion = &apiextensionsv1beta1.CustomResourceConversion{
Strategy: "Webhook",
WebhookClientConfig: conversionWebhook,
}
}
//create CRD and waits for the resource to be recognized and available.
crd, err = fixtures.CreateNewCustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient)
if err != nil {
@ -82,12 +87,17 @@ func CreateTestCRD(f *Framework) (*TestCrd, error) {
return nil, err
}
gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: crd.Spec.Version, Resource: crd.Spec.Names.Plural}
resourceClient := dynamicClient.Resource(gvr).Namespace(f.Namespace.Name)
resourceClients := map[string]dynamic.ResourceInterface{}
for _, v := range crd.Spec.Versions {
if v.Served {
gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: v.Name, Resource: crd.Spec.Names.Plural}
resourceClients[v.Name] = dynamicClient.Resource(gvr).Namespace(f.Namespace.Name)
}
}
testcrd.ApiExtensionClient = apiExtensionClient
testcrd.Crd = crd
testcrd.DynamicClient = resourceClient
testcrd.DynamicClients = resourceClients
testcrd.CleanUp = func() error {
err := fixtures.DeleteCustomResourceDefinition(crd, apiExtensionClient)
if err != nil {
@ -98,13 +108,26 @@ func CreateTestCRD(f *Framework) (*TestCrd, error) {
return testcrd, nil
}
// CreateTestCRD creates a new CRD specifically for the calling test.
func CreateTestCRD(f *Framework) (*TestCrd, error) {
group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName)
apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{
{
Name: "v1",
Served: true,
Storage: true,
},
}
return CreateMultiVersionTestCRD(f, group, apiVersions, nil)
}
// newCRDForTest generates a CRD definition for the test
func newCRDForTest(testcrd *TestCrd) *apiextensionsv1beta1.CustomResourceDefinition {
return &apiextensionsv1beta1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{Name: testcrd.GetMetaName()},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: testcrd.ApiGroup,
Version: testcrd.ApiVersion,
Group: testcrd.ApiGroup,
Versions: testcrd.Versions,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Plural: testcrd.GetPluralName(),
Singular: testcrd.Name,
@ -130,3 +153,17 @@ func (c *TestCrd) GetPluralName() string {
func (c *TestCrd) GetListName() string {
return c.Name + "List"
}
func (c *TestCrd) GetAPIVersions() []string {
ret := []string{}
for _, v := range c.Versions {
if v.Served {
ret = append(ret, v.Name)
}
}
return ret
}
func (c *TestCrd) GetV1DynamicClient() dynamic.ResourceInterface {
return c.DynamicClients["v1"]
}

1
test/utils/image/manifest.go
View File

@ -92,6 +92,7 @@ var (
// Preconfigured image configs
var (
CRDConversionWebhook = Config{e2eRegistry, "crd-conversion-webhook", "1.13rev2"}
AdmissionWebhook = Config{e2eRegistry, "webhook", "1.13v1"}
APIServer = Config{e2eRegistry, "sample-apiserver", "1.10"}
AppArmorLoader = Config{e2eRegistry, "apparmor-loader", "1.0"}