Add CRDs to etcd storage path test

This change updates the etcd storage path test to exercise custom
resource storage by creating custom resource definitions before
running the test.

Duplicated custom resource definition test logic was consolidated.

Signed-off-by: Monis Khan <mkhan@redhat.com>

test/integration/auth/BUILD

@@ -49,7 +49,6 @@ go_test(
         "//staging/src/k8s.io/api/authentication/v1beta1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/storage/v1beta1:go_default_library",
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
@@ -84,6 +83,7 @@ go_test(
         "//staging/src/k8s.io/csi-api/pkg/crd:go_default_library",
         "//test/e2e/lifecycle/bootstrap:go_default_library",
         "//test/integration:go_default_library",
+        "//test/integration/etcd:go_default_library",
         "//test/integration/framework:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/gopkg.in/square/go-jose.v2/jwt:go_default_library",

test/integration/auth/node_test.go

@@ -18,11 +18,12 @@ package auth
 
 import (
 	"fmt"
+	"io/ioutil"
+	"strings"
 	"testing"
 	"time"
 
 	storagev1beta1 "k8s.io/api/storage/v1beta1"
-	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -32,21 +33,18 @@ import (
 	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
 	externalclientset "k8s.io/client-go/kubernetes"
 	csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1"
+	csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
+	csicrd "k8s.io/csi-api/pkg/crd"
+	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
 	"k8s.io/kubernetes/pkg/apis/coordination"
+	"k8s.io/kubernetes/pkg/apis/core"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/apis/policy"
 	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	"k8s.io/kubernetes/pkg/features"
+	"k8s.io/kubernetes/test/integration/etcd"
 	"k8s.io/kubernetes/test/integration/framework"
 	"k8s.io/utils/pointer"
-
-	"io/ioutil"
-	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
-	csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
-	csicrd "k8s.io/csi-api/pkg/crd"
-	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
-	"k8s.io/kubernetes/pkg/apis/core"
-	"strings"
 )
 
 func TestNodeAuthorizer(t *testing.T) {
@@ -158,13 +156,7 @@ func TestNodeAuthorizer(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	crd, err := superuserCRDClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(csicrd.CSINodeInfoCRD())
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := waitForEstablishedCRD(superuserCRDClient, crd.Name); err != nil {
-		t.Fatalf("Failed to establish CSINodeInfo CRD: %v", err)
-	}
+	etcd.CreateTestCRDs(t, superuserCRDClient, false, csicrd.CSINodeInfoCRD())
 
 	getSecret := func(client clientset.Interface) func() error {
 		return func() error {
@@ -672,25 +664,3 @@ func expectAllowed(t *testing.T, f func() error) {
 		t.Errorf("Expected no error, got %v", err)
 	}
 }
-
-func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error {
-	return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
-		crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{})
-		if err != nil {
-			return false, err
-		}
-		for _, cond := range crd.Status.Conditions {
-			switch cond.Type {
-			case apiextensionsv1beta1.Established:
-				if cond.Status == apiextensionsv1beta1.ConditionTrue {
-					return true, err
-				}
-			case apiextensionsv1beta1.NamesAccepted:
-				if cond.Status == apiextensionsv1beta1.ConditionFalse {
-					fmt.Printf("Name conflict: %v\n", cond.Reason)
-				}
-			}
-		}
-		return false, nil
-	})
-}

test/integration/etcd/BUILD

@@ -53,6 +53,8 @@ go_library(
         "//cmd/kube-apiserver/app:go_default_library",
         "//cmd/kube-apiserver/app/options:go_default_library",
         "//pkg/master:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",

test/integration/etcd/data.go

@@ -16,7 +16,11 @@ limitations under the License.
 
 package etcd
 
-import "k8s.io/apimachinery/pkg/runtime/schema"
+import (
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
 
 // GetEtcdStorageData returns etcd data for all persisted objects.
 // It is exported so that it can be reused across multiple tests.
@@ -427,6 +431,23 @@ func GetEtcdStorageData() map[schema.GroupVersionResource]StorageData {
 			Stub:             `{"metadata": {"name": "openshiftwebconsoleconfigs.webconsole.operator.openshift.io"},"spec": {"scope": "Cluster","group": "webconsole.operator.openshift.io","version": "v1alpha1","names": {"kind": "OpenShiftWebConsoleConfig","plural": "openshiftwebconsoleconfigs","singular": "openshiftwebconsoleconfig"}}}`,
 			ExpectedEtcdPath: "/registry/apiextensions.k8s.io/customresourcedefinitions/openshiftwebconsoleconfigs.webconsole.operator.openshift.io",
 		},
+		gvr("cr.bar.com", "v1", "foos"): {
+			Stub:             `{"kind": "Foo", "apiVersion": "cr.bar.com/v1", "metadata": {"name": "cr1foo"}, "color": "blue"}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper
+			ExpectedEtcdPath: "/registry/cr.bar.com/foos/etcdstoragepathtestnamespace/cr1foo",
+		},
+		gvr("custom.fancy.com", "v2", "pants"): {
+			Stub:             `{"kind": "Pant", "apiVersion": "custom.fancy.com/v2", "metadata": {"name": "cr2pant"}, "isFancy": true}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper
+			ExpectedEtcdPath: "/registry/custom.fancy.com/pants/cr2pant",
+		},
+		gvr("awesome.bears.com", "v1", "pandas"): {
+			Stub:             `{"kind": "Panda", "apiVersion": "awesome.bears.com/v1", "metadata": {"name": "cr3panda"}, "weight": 100}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper
+			ExpectedEtcdPath: "/registry/awesome.bears.com/pandas/cr3panda",
+		},
+		gvr("awesome.bears.com", "v3", "pandas"): {
+			Stub:             `{"kind": "Panda", "apiVersion": "awesome.bears.com/v3", "metadata": {"name": "cr4panda"}, "weight": 300}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper
+			ExpectedEtcdPath: "/registry/awesome.bears.com/pandas/cr4panda",
+			ExpectedGVK:      gvkP("awesome.bears.com", "v1", "Panda"),
+		},
 		// --
 	}
 }
@@ -446,6 +467,74 @@ type Prerequisite struct {
 	Stub    string
 }
 
+// GetCustomResourceDefinitionData returns the resource definitions that back the custom resources
+// included in GetEtcdStorageData.  They should be created using CreateTestCRDs before running any tests.
+func GetCustomResourceDefinitionData() []*apiextensionsv1beta1.CustomResourceDefinition {
+	return []*apiextensionsv1beta1.CustomResourceDefinition{
+		// namespaced with legacy version field
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "foos.cr.bar.com",
+			},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group:   "cr.bar.com",
+				Version: "v1",
+				Scope:   apiextensionsv1beta1.NamespaceScoped,
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Plural: "foos",
+					Kind:   "Foo",
+				},
+			},
+		},
+		// cluster scoped with legacy version field
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "pants.custom.fancy.com",
+			},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group:   "custom.fancy.com",
+				Version: "v2",
+				Scope:   apiextensionsv1beta1.ClusterScoped,
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Plural: "pants",
+					Kind:   "Pant",
+				},
+			},
+		},
+		// cluster scoped with versions field
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "pandas.awesome.bears.com",
+			},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group: "awesome.bears.com",
+				Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{
+					{
+						Name:    "v1",
+						Served:  true,
+						Storage: true,
+					},
+					{
+						Name:    "v2",
+						Served:  false,
+						Storage: false,
+					},
+					{
+						Name:    "v3",
+						Served:  true,
+						Storage: false,
+					},
+				},
+				Scope: apiextensionsv1beta1.ClusterScoped,
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Plural: "pandas",
+					Kind:   "Panda",
+				},
+			},
+		},
+	}
+}
+
 func gvr(g, v, r string) schema.GroupVersionResource {
 	return schema.GroupVersionResource{Group: g, Version: v, Resource: r}
 }

test/integration/etcd/etcd_storage_path_test.go

@@ -99,6 +99,10 @@ func TestEtcdStoragePath(t *testing.T) {
 				if input, err = jsonToMetaObject([]byte(testData.Stub)); err != nil || input.isEmpty() {
 					t.Fatalf("invalid test data for %s: %v", gvResource, err)
 				}
+				// unset type meta fields - we only set these in the CRD test data and it makes
+				// any CRD test with an expectedGVK override fail the DeepDerivative test
+				input.Kind = ""
+				input.APIVersion = ""
 			}
 
 			all := &[]cleanupData{}

test/integration/etcd/server.go

@@ -30,6 +30,8 @@ import (
 	"github.com/coreos/etcd/clientv3"
 	"github.com/coreos/etcd/clientv3/concurrency"
 
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -153,6 +155,9 @@ func StartRealMasterOrDie(t *testing.T) *Master {
 		t.Fatal(err)
 	}
 
+	// create CRDs so we can make sure that custom resources do not get lost
+	CreateTestCRDs(t, apiextensionsclientset.NewForConfigOrDie(kubeClientConfig), false, GetCustomResourceDefinitionData()...)
+
 	// force cached discovery reset
 	discoveryClient := cacheddiscovery.NewMemCacheClient(kubeClient.Discovery())
 	restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
@@ -169,6 +174,9 @@ func StartRealMasterOrDie(t *testing.T) *Master {
 		}
 		close(stopCh)
 		lock.Unlock()
+		if err := session.Close(); err != nil {
+			t.Log(err)
+		}
 	}
 
 	return &Master{
@@ -281,3 +289,79 @@ func JSONToUnstructured(stub, namespace string, mapping *meta.RESTMapping, dynam
 
 	return dynamicClient.Resource(mapping.Resource).Namespace(namespace), &unstructured.Unstructured{Object: typeMetaAdder}, nil
 }
+
+// CreateTestCRDs creates the given CRDs, any failure causes the test to Fatal.
+// If skipCrdExistsInDiscovery is true, the CRDs are only checked for the Established condition via their Status.
+// If skipCrdExistsInDiscovery is false, the CRDs are checked via discovery, see CrdExistsInDiscovery.
+func CreateTestCRDs(t *testing.T, client apiextensionsclientset.Interface, skipCrdExistsInDiscovery bool, crds ...*apiextensionsv1beta1.CustomResourceDefinition) {
+	for _, crd := range crds {
+		createTestCRD(t, client, skipCrdExistsInDiscovery, crd)
+	}
+}
+
+func createTestCRD(t *testing.T, client apiextensionsclientset.Interface, skipCrdExistsInDiscovery bool, crd *apiextensionsv1beta1.CustomResourceDefinition) {
+	if _, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil {
+		t.Fatalf("Failed to create %s CRD; %v", crd.Name, err)
+	}
+	if skipCrdExistsInDiscovery {
+		if err := waitForEstablishedCRD(client, crd.Name); err != nil {
+			t.Fatalf("Failed to establish %s CRD; %v", crd.Name, err)
+		}
+		return
+	}
+	if err := wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
+		return CrdExistsInDiscovery(client, crd), nil
+	}); err != nil {
+		t.Fatalf("Failed to see %s in discovery: %v", crd.Name, err)
+	}
+}
+
+func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error {
+	return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
+		crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		for _, cond := range crd.Status.Conditions {
+			switch cond.Type {
+			case apiextensionsv1beta1.Established:
+				if cond.Status == apiextensionsv1beta1.ConditionTrue {
+					return true, nil
+				}
+			}
+		}
+		return false, nil
+	})
+}
+
+// CrdExistsInDiscovery checks to see if the given CRD exists in discovery at all served versions.
+func CrdExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) bool {
+	var versions []string
+	if len(crd.Spec.Version) != 0 {
+		versions = append(versions, crd.Spec.Version)
+	}
+	for _, v := range crd.Spec.Versions {
+		if v.Served {
+			versions = append(versions, v.Name)
+		}
+	}
+	for _, v := range versions {
+		if !crdVersionExistsInDiscovery(client, crd, v) {
+			return false
+		}
+	}
+	return true
+}
+
+func crdVersionExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition, version string) bool {
+	resourceList, err := client.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + version)
+	if err != nil {
+		return false
+	}
+	for _, resource := range resourceList.APIResources {
+		if resource.Name == crd.Spec.Names.Plural {
+			return true
+		}
+	}
+	return false
+}

test/integration/master/BUILD

@@ -62,6 +62,7 @@ go_test(
         "//staging/src/k8s.io/client-go/rest:go_default_library",
         "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration:go_default_library",
         "//test/integration:go_default_library",
+        "//test/integration/etcd:go_default_library",
         "//test/integration/framework:go_default_library",
         "//test/utils:go_default_library",
         "//vendor/github.com/evanphx/json-patch:go_default_library",

test/integration/master/crd_test.go

@@ -18,7 +18,6 @@ package master
 
 import (
 	"encoding/json"
-	"fmt"
 	"testing"
 	"time"
 
@@ -37,6 +36,7 @@ import (
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
+	"k8s.io/kubernetes/test/integration/etcd"
 	"k8s.io/kubernetes/test/integration/framework"
 )
 
@@ -81,12 +81,8 @@ func TestCRDShadowGroup(t *testing.T) {
 			},
 		},
 	}
-	if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil {
-		t.Fatalf("Failed to create networking group CRD: %v", err)
-	}
-	if err := waitForEstablishedCRD(apiextensionsclient, crd.Name); err != nil {
-		t.Fatalf("Failed to establish networking group CRD: %v", err)
-	}
+	etcd.CreateTestCRDs(t, apiextensionsclient, true, crd)
+
 	// wait to give aggregator time to update
 	time.Sleep(2 * time.Second)
 
@@ -97,11 +93,7 @@ func TestCRDShadowGroup(t *testing.T) {
 	}
 
 	t.Logf("Checking that crd resource does not show up in networking group")
-	found, err := crdExistsInDiscovery(apiextensionsclient, crd)
-	if err != nil {
-		t.Fatalf("unexpected discovery error: %v", err)
-	}
-	if found {
+	if etcd.CrdExistsInDiscovery(apiextensionsclient, crd) {
 		t.Errorf("CRD resource shows up in discovery, but shouldn't.")
 	}
 }
@@ -137,17 +129,7 @@ func TestCRD(t *testing.T) {
 			},
 		},
 	}
-	if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil {
-		t.Fatalf("Failed to create foos.cr.bar.com CRD; %v", err)
-	}
-	if err := waitForEstablishedCRD(apiextensionsclient, crd.Name); err != nil {
-		t.Fatalf("Failed to establish foos.cr.bar.com CRD: %v", err)
-	}
-	if err := wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) {
-		return crdExistsInDiscovery(apiextensionsclient, crd)
-	}); err != nil {
-		t.Fatalf("Failed to see foos.cr.bar.com in discovery: %v", err)
-	}
+	etcd.CreateTestCRDs(t, apiextensionsclient, false, crd)
 
 	t.Logf("Trying to access foos.cr.bar.com with dynamic client")
 	dynamicClient, err := dynamic.NewForConfig(result.ClientConfig)
@@ -306,38 +288,3 @@ func unstructuredFoo(foo *Foo) (*unstructured.Unstructured, error) {
 	}
 	return ret, nil
 }
-
-func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error {
-	return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
-		crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{})
-		if err != nil {
-			return false, err
-		}
-		for _, cond := range crd.Status.Conditions {
-			switch cond.Type {
-			case apiextensionsv1beta1.Established:
-				if cond.Status == apiextensionsv1beta1.ConditionTrue {
-					return true, err
-				}
-			case apiextensionsv1beta1.NamesAccepted:
-				if cond.Status == apiextensionsv1beta1.ConditionFalse {
-					fmt.Printf("Name conflict: %v\n", cond.Reason)
-				}
-			}
-		}
-		return false, nil
-	})
-}
-
-func crdExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) (bool, error) {
-	resourceList, err := client.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + crd.Spec.Version)
-	if err != nil {
-		return false, nil
-	}
-	for _, resource := range resourceList.APIResources {
-		if resource.Name == crd.Spec.Names.Plural {
-			return true, nil
-		}
-	}
-	return false, nil
-}