embed apiextensions server into kube-apiserver

cmd/kube-apiserver/app/BUILD

@@ -11,6 +11,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "aggregator.go",
+        "apiextensions.go",
         "plugins.go",
         "server.go",
     ],
@@ -91,6 +92,8 @@ go_library(
         "//vendor/k8s.io/kube-aggregator/pkg/apiserver:go_default_library",
         "//vendor/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/typed/apiregistration/internalversion:go_default_library",
         "//vendor/k8s.io/kube-aggregator/pkg/controllers/autoregister:go_default_library",
+        "//vendor/k8s.io/kube-apiextensions-server/pkg/apiserver:go_default_library",
+        "//vendor/k8s.io/kube-apiextensions-server/pkg/cmd/server:go_default_library",
     ],
 )
 

cmd/kube-apiserver/app/apiextensions.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package app does all of the work necessary to create a Kubernetes
+// APIServer by binding together the API, master and APIServer infrastructure.
+// It can be configured and called directly or via the hyperkube framework.
+package app
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	genericoptions "k8s.io/apiserver/pkg/server/options"
+	apiextensionsapiserver "k8s.io/kube-apiextensions-server/pkg/apiserver"
+	apiextensionscmd "k8s.io/kube-apiextensions-server/pkg/cmd/server"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
+)
+
+func createAPIExtensionsConfig(kubeAPIServerConfig genericapiserver.Config, commandOptions *options.ServerRunOptions) (*apiextensionsapiserver.Config, error) {
+	// make a shallow copy to let us twiddle a few things
+	// most of the config actually remains the same.  We only need to mess with a couple items related to the particulars of the apiextensions
+	genericConfig := kubeAPIServerConfig
+
+	// the apiextensions doesn't wire these up.  It just delegates them to the kubeapiserver
+	genericConfig.EnableSwaggerUI = false
+
+	// TODO these need to be sorted out.  There's an issue open
+	genericConfig.OpenAPIConfig = nil
+	genericConfig.SwaggerConfig = nil
+
+	// copy the loopbackclientconfig.  We're going to change the contenttype back to json until we get protobuf serializations for it
+	t := *kubeAPIServerConfig.LoopbackClientConfig
+	genericConfig.LoopbackClientConfig = &t
+	genericConfig.LoopbackClientConfig.ContentConfig.ContentType = ""
+
+	// copy the etcd options so we don't mutate originals.
+	etcdOptions := *commandOptions.Etcd
+	etcdOptions.StorageConfig.Codec = apiextensionsapiserver.Codecs.LegacyCodec(schema.GroupVersion{Group: "apiextensions.k8s.io", Version: "v1alpha1"})
+	etcdOptions.StorageConfig.Copier = apiextensionsapiserver.Scheme
+	genericConfig.RESTOptionsGetter = &genericoptions.SimpleRestOptionsFactory{Options: etcdOptions}
+
+	apiextensionsConfig := &apiextensionsapiserver.Config{
+		GenericConfig:        &genericConfig,
+		CRDRESTOptionsGetter: apiextensionscmd.NewCRDRESTOptionsGetter(etcdOptions),
+	}
+
+	return apiextensionsConfig, nil
+
+}
+
+func createAPIExtensionsServer(apiextensionsConfig *apiextensionsapiserver.Config, delegateAPIServer genericapiserver.DelegationTarget) (*apiextensionsapiserver.CustomResourceDefinitions, error) {
+	apiextensionsServer, err := apiextensionsConfig.Complete().New(delegateAPIServer)
+	if err != nil {
+		return nil, err
+	}
+
+	return apiextensionsServer, nil
+}

cmd/kube-apiserver/app/server.go

@@ -104,7 +104,21 @@ func Run(runOptions *options.ServerRunOptions, stopCh <-chan struct{}) error {
 	if err != nil {
 		return err
 	}
-	kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, sharedInformers)
+
+	// kubeAPIServer is at the base for now.  This ensures that CustomResourceDefinitions trump TPRs
+	kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, genericapiserver.EmptyDelegate, sharedInformers)
+	if err != nil {
+		return err
+	}
+
+	// TPRs are enabled and not yet beta, since this these are the successor, they fall under the same rule
+	// Subsequent API servers in between here and kube-apiserver will need to be gated.
+	// These come first so that if someone registers both a TPR and a CRD, the CRD is preferred.
+	apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, runOptions)
+	if err != nil {
+		return err
+	}
+	apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, kubeAPIServer.GenericAPIServer)
 	if err != nil {
 		return err
 	}
@@ -126,11 +140,13 @@ func Run(runOptions *options.ServerRunOptions, stopCh <-chan struct{}) error {
 	// otherwise go down the normal path of standing the aggregator up in front of the API server
 	// this wires up openapi
 	kubeAPIServer.GenericAPIServer.PrepareRun()
+
+	// aggregator comes last in the chain
 	aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, runOptions)
 	if err != nil {
 		return err
 	}
-	aggregatorServer, err := createAggregatorServer(aggregatorConfig, kubeAPIServer.GenericAPIServer, sharedInformers)
+	aggregatorServer, err := createAggregatorServer(aggregatorConfig, apiExtensionsServer.GenericAPIServer, sharedInformers)
 	if err != nil {
 		// we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines
 		return err
@@ -139,8 +155,8 @@ func Run(runOptions *options.ServerRunOptions, stopCh <-chan struct{}) error {
 }
 
 // CreateKubeAPIServer creates and wires a workable kube-apiserver
-func CreateKubeAPIServer(kubeAPIServerConfig *master.Config, sharedInformers informers.SharedInformerFactory) (*master.Master, error) {
+func CreateKubeAPIServer(kubeAPIServerConfig *master.Config, delegateAPIServer genericapiserver.DelegationTarget, sharedInformers informers.SharedInformerFactory) (*master.Master, error) {
-	kubeAPIServer, err := kubeAPIServerConfig.Complete().New(genericapiserver.EmptyDelegate)
+	kubeAPIServer, err := kubeAPIServerConfig.Complete().New(delegateAPIServer)
 	if err != nil {
 		return nil, err
 	}

staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/apiserver.go

@@ -74,7 +74,7 @@ func init() {
 type Config struct {
 	GenericConfig *genericapiserver.Config
 
-	CustomResourceDefinitionRESTOptionsGetter genericregistry.RESTOptionsGetter
+	CRDRESTOptionsGetter genericregistry.RESTOptionsGetter
 }
 
 type CustomResourceDefinitions struct {
@@ -105,7 +105,7 @@ func (c *Config) SkipComplete() completedConfig {
 
 // New returns a new instance of CustomResourceDefinitions from the given config.
 func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*CustomResourceDefinitions, error) {
-	genericServer, err := c.Config.GenericConfig.SkipComplete().New(genericapiserver.EmptyDelegate) // completion is done in Complete, no need for a second time
+	genericServer, err := c.Config.GenericConfig.SkipComplete().New(delegationTarget) // completion is done in Complete, no need for a second time
 	if err != nil {
 		return nil, err
 	}
@@ -151,7 +151,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
 		s.GenericAPIServer.RequestContextMapper(),
 		customResourceDefinitionInformers.Apiextensions().InternalVersion().CustomResourceDefinitions().Lister(),
 		delegateHandler,
-		c.CustomResourceDefinitionRESTOptionsGetter,
+		c.CRDRESTOptionsGetter,
 		c.GenericConfig.AdmissionControl,
 	)
 	s.GenericAPIServer.Handler.PostGoRestfulMux.Handle("/apis", customResourceDefinitionHandler)

staging/src/k8s.io/kube-apiextensions-server/pkg/apiserver/customresource_handler.go

@@ -379,7 +379,7 @@ type UnstructuredDefaulter struct{}
 
 func (UnstructuredDefaulter) Default(in runtime.Object) {}
 
-type CustomResourceDefinitionRESTOptionsGetter struct {
+type CRDRESTOptionsGetter struct {
 	StorageConfig           storagebackend.Config
 	StoragePrefix           string
 	EnableWatchCache        bool
@@ -388,7 +388,7 @@ type CustomResourceDefinitionRESTOptionsGetter struct {
 	DeleteCollectionWorkers int
 }
 
-func (t CustomResourceDefinitionRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
+func (t CRDRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
 	ret := generic.RESTOptions{
 		StorageConfig:           &t.StorageConfig,
 		Decorator:               generic.UndecoratedStorage,

staging/src/k8s.io/kube-apiextensions-server/pkg/cmd/server/BUILD

@@ -14,6 +14,7 @@ go_library(
     deps = [
         "//vendor/github.com/spf13/cobra:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/registry/generic:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server/options:go_default_library",
         "//vendor/k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1:go_default_library",

staging/src/k8s.io/kube-apiextensions-server/pkg/cmd/server/start.go

@@ -24,6 +24,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	genericregistry "k8s.io/apiserver/pkg/registry/generic"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
 	"k8s.io/kube-apiextensions-server/pkg/apis/apiextensions/v1alpha1"
@@ -95,24 +96,28 @@ func (o CustomResourceDefinitionsServerOptions) Config() (*apiserver.Config, err
 		return nil, err
 	}
 
-	customResourceDefinitionRESTOptionsGetter := apiserver.CustomResourceDefinitionRESTOptionsGetter{
-		StorageConfig:           o.RecommendedOptions.Etcd.StorageConfig,
-		StoragePrefix:           o.RecommendedOptions.Etcd.StorageConfig.Prefix,
-		EnableWatchCache:        o.RecommendedOptions.Etcd.EnableWatchCache,
-		DefaultWatchCacheSize:   o.RecommendedOptions.Etcd.DefaultWatchCacheSize,
-		EnableGarbageCollection: o.RecommendedOptions.Etcd.EnableGarbageCollection,
-		DeleteCollectionWorkers: o.RecommendedOptions.Etcd.DeleteCollectionWorkers,
-	}
-	customResourceDefinitionRESTOptionsGetter.StorageConfig.Codec = unstructured.UnstructuredJSONScheme
-	customResourceDefinitionRESTOptionsGetter.StorageConfig.Copier = apiserver.UnstructuredCopier{}
-
 	config := &apiserver.Config{
-		GenericConfig:                             serverConfig,
+		GenericConfig:        serverConfig,
-		CustomResourceDefinitionRESTOptionsGetter: customResourceDefinitionRESTOptionsGetter,
+		CRDRESTOptionsGetter: NewCRDRESTOptionsGetter(*o.RecommendedOptions.Etcd),
 	}
 	return config, nil
 }
 
+func NewCRDRESTOptionsGetter(etcdOptions genericoptions.EtcdOptions) genericregistry.RESTOptionsGetter {
+	ret := apiserver.CRDRESTOptionsGetter{
+		StorageConfig:           etcdOptions.StorageConfig,
+		StoragePrefix:           etcdOptions.StorageConfig.Prefix,
+		EnableWatchCache:        etcdOptions.EnableWatchCache,
+		DefaultWatchCacheSize:   etcdOptions.DefaultWatchCacheSize,
+		EnableGarbageCollection: etcdOptions.EnableGarbageCollection,
+		DeleteCollectionWorkers: etcdOptions.DeleteCollectionWorkers,
+	}
+	ret.StorageConfig.Codec = unstructured.UnstructuredJSONScheme
+	ret.StorageConfig.Copier = apiserver.UnstructuredCopier{}
+
+	return ret
+}
+
 func (o CustomResourceDefinitionsServerOptions) RunCustomResourceDefinitionsServer(stopCh <-chan struct{}) error {
 	config, err := o.Config()
 	if err != nil {

staging/src/k8s.io/kube-apiextensions-server/test/integration/registration_test.go

@@ -457,9 +457,9 @@ func TestEtcdStorage(t *testing.T) {
 }
 
 func getPrefixFromConfig(t *testing.T, config *extensionsapiserver.Config) string {
-	extensionsOptionsGetter, ok := config.CustomResourceDefinitionRESTOptionsGetter.(extensionsapiserver.CustomResourceDefinitionRESTOptionsGetter)
+	extensionsOptionsGetter, ok := config.CRDRESTOptionsGetter.(extensionsapiserver.CRDRESTOptionsGetter)
 	if !ok {
-		t.Fatal("can't obtain etcd prefix: unable to cast config.CustomResourceDefinitionRESTOptionsGetter to extensionsapiserver.CustomResourceDefinitionRESTOptionsGetter")
+		t.Fatal("can't obtain etcd prefix: unable to cast config.CRDRESTOptionsGetter to extensionsapiserver.CRDRESTOptionsGetter")
 	}
 	return extensionsOptionsGetter.StoragePrefix
 }

staging/src/k8s.io/kube-apiextensions-server/test/integration/testserver/start.go

@@ -76,7 +76,7 @@ func DefaultServerConfig() (*extensionsapiserver.Config, error) {
 		return nil, err
 	}
 
-	customResourceDefinitionRESTOptionsGetter := extensionsapiserver.CustomResourceDefinitionRESTOptionsGetter{
+	customResourceDefinitionRESTOptionsGetter := extensionsapiserver.CRDRESTOptionsGetter{
 		StorageConfig:           options.RecommendedOptions.Etcd.StorageConfig,
 		StoragePrefix:           options.RecommendedOptions.Etcd.StorageConfig.Prefix,
 		EnableWatchCache:        options.RecommendedOptions.Etcd.EnableWatchCache,
@@ -88,8 +88,8 @@ func DefaultServerConfig() (*extensionsapiserver.Config, error) {
 	customResourceDefinitionRESTOptionsGetter.StorageConfig.Copier = extensionsapiserver.UnstructuredCopier{}
 
 	config := &extensionsapiserver.Config{
-		GenericConfig:                             genericConfig,
+		GenericConfig:        genericConfig,
-		CustomResourceDefinitionRESTOptionsGetter: customResourceDefinitionRESTOptionsGetter,
+		CRDRESTOptionsGetter: customResourceDefinitionRESTOptionsGetter,
 	}
 
 	return config, nil