Merge pull request #46030 from sdminonne/apiextensions-server-storage

Automatic merge from submit-queue

Api-extensions server integraton test: etcd storage

@deads2k 
here is the test we talked about yesterday.
Few comments:


SelfLink for CR Instances looks broken (my first test was not enough, sorry) please have a look [here](https://github.com/sdminonne/kubernetes/blob/apiextensions-server-storage/staging/src/k8s.io/kube-apiextensions-server/test/integration/registration_test.go#L435) and [here](https://github.com/sdminonne/kubernetes/blob/apiextensions-server-storage/staging/src/k8s.io/kube-apiextensions-server/test/integration/registration_test.go#L409)



Not fully sure about the way etcd client works.
I had to concatenate two times the prefix to get the value. The first time from the caller ([example](https://github.com/sdminonne/kubernetes/blob/apiextensions-server-storage/staging/src/k8s.io/kube-apiextensions-server/test/integration/registration_test.go#L428)) and the second time in the [get function](https://github.com/sdminonne/kubernetes/blob/apiextensions-server-storage/staging/src/k8s.io/kube-apiextensions-server/test/integration/registration_test.go#L473).

Not sure if it's a problem or not, here is the `etcdctl` output for example: 

```
$ ETCDCTL_API=3 etcdctl get "" --from-key
/7b02b490-8e8e-4649-ab92-aad1173314fb/7b02b490-8e8e-4649-ab92-aad1173314fb/apiextensions.k8s.io/customresourcedefinition
s/noxus.mygroup.example.com
{"kind":"CustomResourceDefinition","apiVersion":"apiextensions.k8s.io/v1alpha1","metadata":{"name":"noxus.mygroup.exampl
e.com","selfLink":"/apis/apiextensions.k8s.io/v1alpha1/customresourcedefinitions/noxus.mygroup.example.com","uid":"9a08f
664-3b17-11e7-94b1-847beb037559","creationTimestamp":"2017-05-17T15:43:41Z"},"spec":{"group":"mygroup.example.com","vers
ion":"v1alpha1","names":{"plural":"noxus","singular":"nonenglishnoxu","shortNames":["foo","bar","abc","def"],"kind":"Wis
hIHadChosenNoxu","listKind":"NoxuItemList"},"scope":"Namespaced"},"status":{"conditions":[{"type":"NameConflict","status
":"False","lastTransitionTime":null,"reason":"NoConflicts","message":"no conflicts found"}],"acceptedNames":{"plural":"n
oxus","singular":"nonenglishnoxu","shortNames":["foo","bar","abc","def"],"kind":"WishIHadChosenNoxu","listKind":"NoxuIte
mList"}}}

/7b02b490-8e8e-4649-ab92-aad1173314fb/7b02b490-8e8e-4649-ab92-aad1173314fb/mygroup.example.com/noxus/not-the-default/foo
{"apiVersion":"mygroup.example.com/v1alpha1","content":{"key":"value"},"kind":"WishIHadChosenNoxu","metadata":{"clusterN
ame":"","creationTimestamp":"2017-05-17T15:43:41Z","deletionGracePeriodSeconds":null,"deletionTimestamp":null,"name":"fo
o","namespace":"not-the-default","selfLink":"","uid":"9a174a53-3b17-11e7-94b1-847beb037559"}}
```

staging/src/k8s.io/kube-apiextensions-server/test/integration/BUILD

@@ -18,6 +18,7 @@ go_test(
         "integration",
     ],
     deps = [
+        "//vendor/github.com/coreos/etcd/clientv3:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
@@ -25,6 +26,7 @@ go_test(
         "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
         "//vendor/k8s.io/client-go/dynamic:go_default_library",
         "//vendor/k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1:go_default_library",
+        "//vendor/k8s.io/kube-apiextensions-server/pkg/apiserver:go_default_library",
         "//vendor/k8s.io/kube-apiextensions-server/test/integration/testserver:go_default_library",
     ],
 )

staging/src/k8s.io/kube-apiextensions-server/test/integration/registration_test.go

@@ -17,10 +17,17 @@ limitations under the License.
 package integration
 
 import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
 	"reflect"
 	"testing"
 	"time"
 
+	"github.com/coreos/etcd/clientv3"
+
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -28,6 +35,7 @@ import (
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/dynamic"
 	apiextensionsv1alpha1 "k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1"
+	extensionsapiserver "k8s.io/kube-apiextensions-server/pkg/apiserver"
 	"k8s.io/kube-apiextensions-server/test/integration/testserver"
 )
 
@@ -217,7 +225,7 @@ func TestMultipleRegistration(t *testing.T) {
 		t.Errorf("expected %v, got %v", e, a)
 	}
 
-	curletDefinition := testserver.NewCurletCustomResourceDefinition()
+	curletDefinition := testserver.NewCurletCustomResourceDefinition(apiextensionsv1alpha1.NamespaceScoped)
 	curletVersionClient, err := testserver.CreateNewCustomResourceDefinition(curletDefinition, apiExtensionClient, clientPool)
 	if err != nil {
 		t.Fatal(err)
@@ -338,3 +346,153 @@ func TestDeRegistrationAndReRegistration(t *testing.T) {
 		}
 	}()
 }
+
+func TestEtcdStorage(t *testing.T) {
+	config, err := testserver.DefaultServerConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	stopCh, apiExtensionClient, clientPool, err := testserver.StartServer(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer close(stopCh)
+
+	etcdPrefix := getPrefixFromConfig(t, config)
+
+	ns1 := "another-default-is-possible"
+	curletDefinition := testserver.NewCurletCustomResourceDefinition(apiextensionsv1alpha1.ClusterScoped)
+	curletVersionClient, err := testserver.CreateNewCustomResourceDefinition(curletDefinition, apiExtensionClient, clientPool)
+	if err != nil {
+		t.Fatal(err)
+	}
+	curletNamespacedResourceClient := NewNamespacedCustomResourceClient(ns1, curletVersionClient, curletDefinition)
+	if _, err := instantiateCustomResource(t, testserver.NewCurletInstance(ns1, "bar"), curletNamespacedResourceClient, curletDefinition); err != nil {
+		t.Fatalf("unable to create curlet cluster scoped Instance:%v", err)
+	}
+
+	ns2 := "the-cruel-default"
+	noxuDefinition := testserver.NewNoxuCustomResourceDefinition(apiextensionsv1alpha1.NamespaceScoped)
+	noxuVersionClient, err := testserver.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, clientPool)
+	if err != nil {
+		t.Fatal(err)
+	}
+	noxuNamespacedResourceClient := NewNamespacedCustomResourceClient(ns2, noxuVersionClient, noxuDefinition)
+	if _, err := instantiateCustomResource(t, testserver.NewNoxuInstance(ns2, "foo"), noxuNamespacedResourceClient, noxuDefinition); err != nil {
+		t.Fatalf("unable to create noxu namespace scoped Instance:%v", err)
+	}
+
+	testcases := map[string]struct {
+		etcdPath       string
+		expectedObject *metaObject
+	}{
+		"namespacedNoxuDefinition": {
+			etcdPath: path.Join("/", etcdPrefix, "apiextensions.k8s.io/customresourcedefinitions/noxus.mygroup.example.com"), // TODO: Double check this, no namespace?
+			expectedObject: &metaObject{
+				Kind:       "CustomResourceDefinition",
+				APIVersion: "apiextensions.k8s.io/v1alpha1",
+				Metadata: Metadata{
+					Name:      "noxus.mygroup.example.com",
+					Namespace: "",
+					SelfLink:  "/apis/apiextensions.k8s.io/v1alpha1/customresourcedefinitions/noxus.mygroup.example.com",
+				},
+			},
+		},
+		"namespacedNoxuInstance": {
+			etcdPath: path.Join("/", etcdPrefix, "mygroup.example.com/noxus/the-cruel-default/foo"),
+			expectedObject: &metaObject{
+				Kind:       "WishIHadChosenNoxu",
+				APIVersion: "mygroup.example.com/v1alpha1",
+				Metadata: Metadata{
+					Name:      "foo",
+					Namespace: "the-cruel-default",
+					SelfLink:  "", // TODO double check: empty?
+				},
+			},
+		},
+
+		"clusteredCurletDefinition": {
+			etcdPath: path.Join("/", etcdPrefix, "apiextensions.k8s.io/customresourcedefinitions/curlets.mygroup.example.com"),
+			expectedObject: &metaObject{
+				Kind:       "CustomResourceDefinition",
+				APIVersion: "apiextensions.k8s.io/v1alpha1",
+				Metadata: Metadata{
+					Name:      "curlets.mygroup.example.com",
+					Namespace: "",
+					SelfLink:  "/apis/apiextensions.k8s.io/v1alpha1/customresourcedefinitions/curlets.mygroup.example.com",
+				},
+			},
+		},
+
+		"clusteredCurletInstance": {
+			etcdPath: path.Join("/", etcdPrefix, "mygroup.example.com/curlets/bar"),
+			expectedObject: &metaObject{
+				Kind:       "Curlet",
+				APIVersion: "mygroup.example.com/v1alpha1",
+				Metadata: Metadata{
+					Name:      "bar",
+					Namespace: "",
+					SelfLink:  "", // TODO double check: empty?
+				},
+			},
+		},
+	}
+
+	etcdURL, ok := os.LookupEnv("KUBE_INTEGRATION_ETCD_URL")
+	if !ok {
+		etcdURL = "http://127.0.0.1:2379"
+	}
+	cfg := clientv3.Config{
+		Endpoints: []string{etcdURL},
+	}
+	c, err := clientv3.New(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	kv := clientv3.NewKV(c)
+	for testName, tc := range testcases {
+		output, err := getFromEtcd(kv, etcdPrefix, tc.etcdPath)
+		if err != nil {
+			t.Fatalf("%s - no path gotten from etcd:%v", testName, err)
+		}
+		if e, a := tc.expectedObject, output; !reflect.DeepEqual(e, a) {
+			t.Errorf("%s - expected %#v\n got %#v\n", testName, e, a)
+		}
+	}
+}
+
+func getPrefixFromConfig(t *testing.T, config *extensionsapiserver.Config) string {
+	extensionsOptionsGetter, ok := config.CustomResourceDefinitionRESTOptionsGetter.(extensionsapiserver.CustomResourceDefinitionRESTOptionsGetter)
+	if !ok {
+		t.Fatal("can't obtain etcd prefix: unable to cast config.CustomResourceDefinitionRESTOptionsGetter to extensionsapiserver.CustomResourceDefinitionRESTOptionsGetter")
+	}
+	return extensionsOptionsGetter.StoragePrefix
+}
+
+func getFromEtcd(keys clientv3.KV, prefix, localPath string) (*metaObject, error) {
+	internalPath := path.Join("/", prefix, localPath) // TODO: Double check, should we concatenate two prefixes?
+	response, err := keys.Get(context.Background(), internalPath)
+	if err != nil {
+		return nil, err
+	}
+	if response.More || response.Count != 1 || len(response.Kvs) != 1 {
+		return nil, fmt.Errorf("Invalid etcd response (not found == %v): %#v", response.Count == 0, response)
+	}
+	obj := &metaObject{}
+	if err := json.Unmarshal(response.Kvs[0].Value, obj); err != nil {
+		return nil, err
+	}
+	return obj, nil
+}
+
+type metaObject struct {
+	Kind       string `json:"kind,omitempty" protobuf:"bytes,1,opt,name=kind"`
+	APIVersion string `json:"apiVersion,omitempty" protobuf:"bytes,2,opt,name=apiVersion"`
+	Metadata   `json:"metadata,omitempty" protobuf:"bytes,3,opt,name=metadata"`
+}
+
+type Metadata struct {
+	Name      string `json:"name,omitempty" protobuf:"bytes,1,opt,name=name"`
+	Namespace string `json:"namespace,omitempty" protobuf:"bytes,2,opt,name=namespace"`
+	SelfLink  string `json:"selfLink,omitempty" protobuf:"bytes,3,opt,name=selfLink"`
+}

staging/src/k8s.io/kube-apiextensions-server/test/integration/testserver/resources.go

@@ -63,7 +63,7 @@ func NewNoxuInstance(namespace, name string) *unstructured.Unstructured {
 	}
 }
 
-func NewCurletCustomResourceDefinition() *apiextensionsv1alpha1.CustomResourceDefinition {
+func NewCurletCustomResourceDefinition(scope apiextensionsv1alpha1.ResourceScope) *apiextensionsv1alpha1.CustomResourceDefinition {
 	return &apiextensionsv1alpha1.CustomResourceDefinition{
 		ObjectMeta: metav1.ObjectMeta{Name: "curlets.mygroup.example.com"},
 		Spec: apiextensionsv1alpha1.CustomResourceDefinitionSpec{
@@ -75,7 +75,7 @@ func NewCurletCustomResourceDefinition() *apiextensionsv1alpha1.CustomResourceDe
 				Kind:     "Curlet",
 				ListKind: "CurletList",
 			},
-			Scope: apiextensionsv1alpha1.NamespaceScoped,
+			Scope: scope,
 		},
 	}
 }