Allowing runtimeConfig to support enabling/disabling specific extension resources

cmd/kube-apiserver/app/server.go

@@ -408,30 +408,11 @@ func (s *APIServer) Run(_ []string) error {
 		glog.Fatalf("Failure to start kubelet client: %v", err)
 	}
 
-	// "api/all=false" allows users to selectively enable specific api versions.
+	apiGroupVersionOverrides, err := s.parseRuntimeConfig()
-	disableAllAPIs := false
+	if err != nil {
-	allAPIFlagValue, ok := s.RuntimeConfig["api/all"]
+		glog.Fatalf("error in parsing runtime-config: %s", err)
-	if ok && allAPIFlagValue == "false" {
-		disableAllAPIs = true
 	}
 
-	// "api/legacy=false" allows users to disable legacy api versions.
-	disableLegacyAPIs := false
-	legacyAPIFlagValue, ok := s.RuntimeConfig["api/legacy"]
-	if ok && legacyAPIFlagValue == "false" {
-		disableLegacyAPIs = true
-	}
-	_ = disableLegacyAPIs // hush the compiler while we don't have legacy APIs to disable.
-
-	// "api/v1={true|false} allows users to enable/disable v1 API.
-	// This takes preference over api/all and api/legacy, if specified.
-	disableV1 := disableAllAPIs
-	disableV1 = !s.getRuntimeConfigValue("api/v1", !disableV1)
-
-	// "extensions/v1beta1={true|false} allows users to enable/disable the experimental API.
-	// This takes preference over api/all, if specified.
-	enableExp := s.getRuntimeConfigValue("extensions/v1beta1", false)
-
 	clientConfig := &client.Config{
 		Host:    net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)),
 		Version: s.DeprecatedStorageVersion,
@@ -458,17 +439,17 @@ func (s *APIServer) Run(_ []string) error {
 	}
 	storageDestinations.AddAPIGroup("", etcdStorage)
 
-	if enableExp {
+	if !apiGroupVersionOverrides["extensions/v1beta1"].Disable {
 		expGroup, err := latest.Group("extensions")
 		if err != nil {
-			glog.Fatalf("Experimental API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err)
+			glog.Fatalf("Extensions API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err)
 		}
 		if _, found := storageVersions[expGroup.Group]; !found {
 			glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", expGroup.Group, storageVersions)
 		}
 		expEtcdStorage, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, expGroup.InterfacesFor, storageVersions[expGroup.Group], s.EtcdPathPrefix)
 		if err != nil {
-			glog.Fatalf("Invalid experimental storage version or misconfigured etcd: %v", err)
+			glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err)
 		}
 		storageDestinations.AddAPIGroup("extensions", expEtcdStorage)
 	}
@@ -558,8 +539,7 @@ func (s *APIServer) Run(_ []string) error {
 		SupportsBasicAuth:         len(s.BasicAuthFile) > 0,
 		Authorizer:                authorizer,
 		AdmissionControl:          admissionController,
-		DisableV1:                 disableV1,
+		APIGroupVersionOverrides:  apiGroupVersionOverrides,
-		EnableExp:                 enableExp,
 		MasterServiceNamespace:    s.MasterServiceNamespace,
 		ClusterName:               s.ClusterName,
 		ExternalHost:              s.ExternalHost,
@@ -680,3 +660,61 @@ func (s *APIServer) getRuntimeConfigValue(apiKey string, defaultValue bool) bool
 	}
 	return defaultValue
 }
+
+// Parses the given runtime-config and formats it into map[string]ApiGroupVersionOverride
+func (s *APIServer) parseRuntimeConfig() (map[string]master.APIGroupVersionOverride, error) {
+	// "api/all=false" allows users to selectively enable specific api versions.
+	disableAllAPIs := false
+	allAPIFlagValue, ok := s.RuntimeConfig["api/all"]
+	if ok && allAPIFlagValue == "false" {
+		disableAllAPIs = true
+	}
+
+	// "api/legacy=false" allows users to disable legacy api versions.
+	disableLegacyAPIs := false
+	legacyAPIFlagValue, ok := s.RuntimeConfig["api/legacy"]
+	if ok && legacyAPIFlagValue == "false" {
+		disableLegacyAPIs = true
+	}
+	_ = disableLegacyAPIs // hush the compiler while we don't have legacy APIs to disable.
+
+	// "api/v1={true|false} allows users to enable/disable v1 API.
+	// This takes preference over api/all and api/legacy, if specified.
+	disableV1 := disableAllAPIs
+	v1GroupVersion := "api/v1"
+	disableV1 = !s.getRuntimeConfigValue(v1GroupVersion, !disableV1)
+	apiGroupVersionOverrides := map[string]master.APIGroupVersionOverride{}
+	if disableV1 {
+		apiGroupVersionOverrides[v1GroupVersion] = master.APIGroupVersionOverride{
+			Disable: true,
+		}
+	}
+
+	// "extensions/v1beta1={true|false} allows users to enable/disable the extensions API.
+	// This takes preference over api/all, if specified.
+	disableExtensions := disableAllAPIs
+	extensionsGroupVersion := "extensions/v1beta1"
+	// TODO: Make this a loop over all group/versions when there are more of them.
+	disableExtensions = !s.getRuntimeConfigValue(extensionsGroupVersion, !disableExtensions)
+	if disableExtensions {
+		apiGroupVersionOverrides[extensionsGroupVersion] = master.APIGroupVersionOverride{
+			Disable: true,
+		}
+	}
+
+	for key := range s.RuntimeConfig {
+		if strings.HasPrefix(key, v1GroupVersion+"/") {
+			return nil, fmt.Errorf("api/v1 resources cannot be enabled/disabled individually")
+		} else if strings.HasPrefix(key, extensionsGroupVersion+"/") {
+			resource := strings.TrimPrefix(key, extensionsGroupVersion+"/")
+
+			apiGroupVersionOverride := apiGroupVersionOverrides[extensionsGroupVersion]
+			if apiGroupVersionOverride.ResourceOverrides == nil {
+				apiGroupVersionOverride.ResourceOverrides = map[string]bool{}
+			}
+			apiGroupVersionOverride.ResourceOverrides[resource] = s.getRuntimeConfigValue(key, false)
+			apiGroupVersionOverrides[extensionsGroupVersion] = apiGroupVersionOverride
+		}
+	}
+	return apiGroupVersionOverrides, nil
+}

cmd/kube-apiserver/app/server_test.go

@@ -154,3 +154,96 @@ func TestUpdateEtcdOverrides(t *testing.T) {
 		}
 	}
 }
+
+func TestParseRuntimeConfig(t *testing.T) {
+	testCases := []struct {
+		runtimeConfig            map[string]string
+		apiGroupVersionOverrides map[string]master.APIGroupVersionOverride
+		err                      bool
+	}{
+		{
+			runtimeConfig:            map[string]string{},
+			apiGroupVersionOverrides: map[string]master.APIGroupVersionOverride{},
+			err: false,
+		},
+		{
+			// Cannot override v1 resources.
+			runtimeConfig: map[string]string{
+				"api/v1/pods": "false",
+			},
+			apiGroupVersionOverrides: map[string]master.APIGroupVersionOverride{},
+			err: true,
+		},
+		{
+			// Disable v1.
+			runtimeConfig: map[string]string{
+				"api/v1": "false",
+			},
+			apiGroupVersionOverrides: map[string]master.APIGroupVersionOverride{
+				"api/v1": {
+					Disable: true,
+				},
+			},
+			err: false,
+		},
+		{
+			// Disable extensions.
+			runtimeConfig: map[string]string{
+				"extensions/v1beta1": "false",
+			},
+			apiGroupVersionOverrides: map[string]master.APIGroupVersionOverride{
+				"extensions/v1beta1": {
+					Disable: true,
+				},
+			},
+			err: false,
+		},
+		{
+			// Disable deployments.
+			runtimeConfig: map[string]string{
+				"extensions/v1beta1/deployments": "false",
+			},
+			apiGroupVersionOverrides: map[string]master.APIGroupVersionOverride{
+				"extensions/v1beta1": {
+					ResourceOverrides: map[string]bool{
+						"deployments": false,
+					},
+				},
+			},
+			err: false,
+		},
+		{
+			// Enable deployments and disable jobs.
+			runtimeConfig: map[string]string{
+				"extensions/v1beta1/deployments": "true",
+				"extensions/v1beta1/jobs":        "false",
+			},
+			apiGroupVersionOverrides: map[string]master.APIGroupVersionOverride{
+				"extensions/v1beta1": {
+					ResourceOverrides: map[string]bool{
+						"deployments": true,
+						"jobs":        false,
+					},
+				},
+			},
+			err: false,
+		},
+	}
+	for _, test := range testCases {
+		s := &APIServer{
+			RuntimeConfig: test.runtimeConfig,
+		}
+		apiGroupVersionOverrides, err := s.parseRuntimeConfig()
+
+		if err == nil && test.err {
+			t.Fatalf("expected error for test: %q", test)
+		} else if err != nil && !test.err {
+			t.Fatalf("unexpected error: %s, for test: %q", err, test)
+		}
+
+		if err == nil && !reflect.DeepEqual(apiGroupVersionOverrides, test.apiGroupVersionOverrides) {
+			t.Fatalf("unexpected apiGroupVersionOverrides. Actual: %q, expected: %q", apiGroupVersionOverrides, test.apiGroupVersionOverrides)
+		}
+	}
+
+}

pkg/master/master.go

@@ -80,6 +80,7 @@ import (
 	"k8s.io/kubernetes/pkg/ui"
 	"k8s.io/kubernetes/pkg/util"
 	"k8s.io/kubernetes/pkg/util/sets"
+	utilSets "k8s.io/kubernetes/pkg/util/sets"
 
 	daemonetcd "k8s.io/kubernetes/pkg/registry/daemonset/etcd"
 	horizontalpodautoscaleretcd "k8s.io/kubernetes/pkg/registry/horizontalpodautoscaler/etcd"
@@ -166,6 +167,15 @@ func (s *StorageDestinations) backends() []string {
 	return backends.List()
 }
 
+// Specifies the overrides for various API group versions.
+// This can be used to enable/disable entire group versions or specific resources.
+type APIGroupVersionOverride struct {
+	// Whether to enable or disable this group version.
+	Disable bool
+	// List of overrides for individual resources in this group version.
+	ResourceOverrides map[string]bool
+}
+
 // Config is a structure used to configure a Master.
 type Config struct {
 	StorageDestinations StorageDestinations
@@ -180,9 +190,8 @@ type Config struct {
 	EnableUISupport       bool
 	// allow downstream consumers to disable swagger
 	EnableSwaggerSupport bool
-	// allow api versions to be conditionally disabled
+	// Allows api group versions or specific resources to be conditionally enabled/disabled.
-	DisableV1 bool
+	APIGroupVersionOverrides map[string]APIGroupVersionOverride
-	EnableExp bool
 	// allow downstream consumers to disable the index route
 	EnableIndex           bool
 	EnableProfiling       bool
@@ -271,26 +280,25 @@ type Master struct {
 	cacheTimeout          time.Duration
 	minRequestTimeout     time.Duration
 
-	mux                   apiserver.Mux
+	mux                      apiserver.Mux
-	muxHelper             *apiserver.MuxHelper
+	muxHelper                *apiserver.MuxHelper
-	handlerContainer      *restful.Container
+	handlerContainer         *restful.Container
-	rootWebService        *restful.WebService
+	rootWebService           *restful.WebService
-	enableCoreControllers bool
+	enableCoreControllers    bool
-	enableLogsSupport     bool
+	enableLogsSupport        bool
-	enableUISupport       bool
+	enableUISupport          bool
-	enableSwaggerSupport  bool
+	enableSwaggerSupport     bool
-	enableProfiling       bool
+	enableProfiling          bool
-	enableWatchCache      bool
+	enableWatchCache         bool
-	apiPrefix             string
+	apiPrefix                string
-	apiGroupPrefix        string
+	apiGroupPrefix           string
-	corsAllowedOriginList []string
+	corsAllowedOriginList    []string
-	authenticator         authenticator.Request
+	authenticator            authenticator.Request
-	authorizer            authorizer.Authorizer
+	authorizer               authorizer.Authorizer
-	admissionControl      admission.Interface
+	admissionControl         admission.Interface
-	masterCount           int
+	masterCount              int
-	v1                    bool
+	apiGroupVersionOverrides map[string]APIGroupVersionOverride
-	exp                   bool
+	requestContextMapper     api.RequestContextMapper
-	requestContextMapper  api.RequestContextMapper
 
 	// External host is the name that should be used in external (public internet) URLs for this master
 	externalHost string
@@ -435,24 +443,23 @@ func New(c *Config) *Master {
 	}
 
 	m := &Master{
-		serviceClusterIPRange: c.ServiceClusterIPRange,
+		serviceClusterIPRange:    c.ServiceClusterIPRange,
-		serviceNodePortRange:  c.ServiceNodePortRange,
+		serviceNodePortRange:     c.ServiceNodePortRange,
-		rootWebService:        new(restful.WebService),
+		rootWebService:           new(restful.WebService),
-		enableCoreControllers: c.EnableCoreControllers,
+		enableCoreControllers:    c.EnableCoreControllers,
-		enableLogsSupport:     c.EnableLogsSupport,
+		enableLogsSupport:        c.EnableLogsSupport,
-		enableUISupport:       c.EnableUISupport,
+		enableUISupport:          c.EnableUISupport,
-		enableSwaggerSupport:  c.EnableSwaggerSupport,
+		enableSwaggerSupport:     c.EnableSwaggerSupport,
-		enableProfiling:       c.EnableProfiling,
+		enableProfiling:          c.EnableProfiling,
-		enableWatchCache:      c.EnableWatchCache,
+		enableWatchCache:         c.EnableWatchCache,
-		apiPrefix:             c.APIPrefix,
+		apiPrefix:                c.APIPrefix,
-		apiGroupPrefix:        c.APIGroupPrefix,
+		apiGroupPrefix:           c.APIGroupPrefix,
-		corsAllowedOriginList: c.CorsAllowedOriginList,
+		corsAllowedOriginList:    c.CorsAllowedOriginList,
-		authenticator:         c.Authenticator,
+		authenticator:            c.Authenticator,
-		authorizer:            c.Authorizer,
+		authorizer:               c.Authorizer,
-		admissionControl:      c.AdmissionControl,
+		admissionControl:         c.AdmissionControl,
-		v1:                    !c.DisableV1,
+		apiGroupVersionOverrides: c.APIGroupVersionOverrides,
-		exp:                   c.EnableExp,
+		requestContextMapper:     c.RequestContextMapper,
-		requestContextMapper:  c.RequestContextMapper,
 
 		cacheTimeout:      c.CacheTimeout,
 		minRequestTimeout: time.Duration(c.MinRequestTimeout) * time.Second,
@@ -624,7 +631,8 @@ func (m *Master) init(c *Config) {
 	}
 
 	apiVersions := []string{}
-	if m.v1 {
+	// Install v1 unless disabled.
+	if !m.apiGroupVersionOverrides["api/v1"].Disable {
 		if err := m.api_v1().InstallREST(m.handlerContainer); err != nil {
 			glog.Fatalf("Unable to setup API v1: %v", err)
 		}
@@ -637,7 +645,8 @@ func (m *Master) init(c *Config) {
 
 	// allGroups records all supported groups at /apis
 	allGroups := []unversioned.APIGroup{}
-	if m.exp {
+	// Install extensions unless disabled.
+	if !m.apiGroupVersionOverrides["extensions/v1beta1"].Disable {
 		m.thirdPartyStorage = c.StorageDestinations.APIGroups["extensions"].Default
 		m.thirdPartyResources = map[string]*thirdpartyresourcedataetcd.REST{}
 
@@ -1023,45 +1032,71 @@ func (m *Master) thirdpartyapi(group, kind, version string) *apiserver.APIGroupV
 
 // experimental returns the resources and codec for the experimental api
 func (m *Master) experimental(c *Config) *apiserver.APIGroupVersion {
-	controllerStorage := expcontrolleretcd.NewStorage(c.StorageDestinations.get("", "replicationControllers"))
+	// All resources except these are disabled by default.
+	enabledResources := utilSets.NewString("jobs", "horizontalpodautoscalers", "ingress")
+	resourceOverrides := m.apiGroupVersionOverrides["extensions/v1beta1"].ResourceOverrides
+	isEnabled := func(resource string) bool {
+		// Check if the resource has been overriden.
+		enabled, ok := resourceOverrides[resource]
+		if !ok {
+			return enabledResources.Has(resource)
+		}
+		return enabled
+	}
 	dbClient := func(resource string) storage.Interface {
 		return c.StorageDestinations.get("extensions", resource)
 	}
-	autoscalerStorage, autoscalerStatusStorage := horizontalpodautoscaleretcd.NewREST(dbClient("horizonalpodautoscalers"))
-	thirdPartyResourceStorage := thirdpartyresourceetcd.NewREST(dbClient("thirdpartyresources"))
-	daemonSetStorage, daemonSetStatusStorage := daemonetcd.NewREST(dbClient("daemonsets"))
-	deploymentStorage := deploymentetcd.NewStorage(dbClient("deployments"))
-	jobStorage, jobStatusStorage := jobetcd.NewREST(dbClient("jobs"))
-	ingressStorage, ingressStatusStorage := ingressetcd.NewREST(dbClient("ingress"))
 
-	thirdPartyControl := ThirdPartyController{
+	storage := map[string]rest.Storage{}
-		master: m,
+	if isEnabled("replicationcontrollers") {
-		thirdPartyResourceRegistry: thirdPartyResourceStorage,
+		controllerStorage := expcontrolleretcd.NewStorage(c.StorageDestinations.get("", "replicationControllers"))
+		storage["replicationcontrollers"] = controllerStorage.ReplicationController
+		storage["replicationcontrollers/scale"] = controllerStorage.Scale
 	}
-	go func() {
+	if isEnabled("horizontalpodautoscalers") {
-		util.Forever(func() {
+		autoscalerStorage, autoscalerStatusStorage := horizontalpodautoscaleretcd.NewREST(dbClient("horizonalpodautoscalers"))
-			if err := thirdPartyControl.SyncResources(); err != nil {
+		storage["horizontalpodautoscalers"] = autoscalerStorage
-				glog.Warningf("third party resource sync failed: %v", err)
+		storage["horizontalpodautoscalers/status"] = autoscalerStatusStorage
-			}
+	}
-		}, 10*time.Second)
+	if isEnabled("thirdpartyresources") {
-	}()
+		thirdPartyResourceStorage := thirdpartyresourceetcd.NewREST(dbClient("thirdpartyresources"))
-	storage := map[string]rest.Storage{
+		thirdPartyControl := ThirdPartyController{
-		strings.ToLower("replicationControllers"):          controllerStorage.ReplicationController,
+			master: m,
-		strings.ToLower("replicationControllers/scale"):    controllerStorage.Scale,
+			thirdPartyResourceRegistry: thirdPartyResourceStorage,
-		strings.ToLower("horizontalpodautoscalers"):        autoscalerStorage,
+		}
-		strings.ToLower("horizontalpodautoscalers/status"): autoscalerStatusStorage,
+		go func() {
-		strings.ToLower("thirdpartyresources"):             thirdPartyResourceStorage,
+			util.Forever(func() {
-		strings.ToLower("daemonsets"):                      daemonSetStorage,
+				if err := thirdPartyControl.SyncResources(); err != nil {
-		strings.ToLower("daemonsets/status"):               daemonSetStatusStorage,
+					glog.Warningf("third party resource sync failed: %v", err)
-		strings.ToLower("deployments"):                     deploymentStorage.Deployment,
+				}
-		strings.ToLower("deployments/scale"):               deploymentStorage.Scale,
+			}, 10*time.Second)
-		strings.ToLower("jobs"):                            jobStorage,
+		}()
-		strings.ToLower("jobs/status"):                     jobStatusStorage,
+
-		strings.ToLower("ingress"):                         ingressStorage,
+		storage["thirdpartyresources"] = thirdPartyResourceStorage
-		strings.ToLower("ingress/status"):                  ingressStatusStorage,
 	}
 
-	expMeta := latest.GroupOrDie("extensions")
+	if isEnabled("daemonsets") {
+		daemonSetStorage, daemonSetStatusStorage := daemonetcd.NewREST(dbClient("daemonsets"))
+		storage["daemonsets"] = daemonSetStorage
+		storage["daemonsets/status"] = daemonSetStatusStorage
+	}
+	if isEnabled("deployments") {
+		deploymentStorage := deploymentetcd.NewStorage(dbClient("deployments"))
+		storage["deployments"] = deploymentStorage.Deployment
+		storage["deployments/scale"] = deploymentStorage.Scale
+	}
+	if isEnabled("jobs") {
+		jobStorage, jobStatusStorage := jobetcd.NewREST(dbClient("jobs"))
+		storage["jobs"] = jobStorage
+		storage["jobs/status"] = jobStatusStorage
+	}
+	if isEnabled("ingress") {
+		ingressStorage, ingressStatusStorage := ingressetcd.NewREST(dbClient("ingress"))
+		storage["ingress"] = ingressStorage
+		storage["ingress/status"] = ingressStatusStorage
+	}
+
+	extensionsGroup := latest.GroupOrDie("extensions")
 
 	return &apiserver.APIGroupVersion{
 		Root: m.apiGroupPrefix,
@@ -1071,11 +1106,11 @@ func (m *Master) experimental(c *Config) *apiserver.APIGroupVersion {
 		Convertor: api.Scheme,
 		Typer:     api.Scheme,
 
-		Mapper:        expMeta.RESTMapper,
+		Mapper:        extensionsGroup.RESTMapper,
-		Codec:         expMeta.Codec,
+		Codec:         extensionsGroup.Codec,
-		Linker:        expMeta.SelfLinker,
+		Linker:        extensionsGroup.SelfLinker,
 		Storage:       storage,
-		Version:       expMeta.GroupVersion,
+		Version:       extensionsGroup.GroupVersion,
 		ServerVersion: latest.GroupOrDie("").GroupVersion,
 
 		Admit:   m.admissionControl,

pkg/master/master_test.go

@@ -100,8 +100,7 @@ func TestNew(t *testing.T) {
 	assert.Equal(master.authenticator, config.Authenticator)
 	assert.Equal(master.authorizer, config.Authorizer)
 	assert.Equal(master.admissionControl, config.AdmissionControl)
-	assert.Equal(master.v1, !config.DisableV1)
+	assert.Equal(master.apiGroupVersionOverrides, config.APIGroupVersionOverrides)
-	assert.Equal(master.exp, config.EnableExp)
 	assert.Equal(master.requestContextMapper, config.RequestContextMapper)
 	assert.Equal(master.cacheTimeout, config.CacheTimeout)
 	assert.Equal(master.masterCount, config.MasterCount)
@@ -366,7 +365,6 @@ func TestGetNodeAddresses(t *testing.T) {
 
 func TestDiscoveryAtAPIS(t *testing.T) {
 	master, config, assert := setUp(t)
-	master.exp = true
 	// ================= preparation for master.init() ======================
 	portRange := util.PortRange{Base: 10, Size: 10}
 	master.serviceNodePortRange = portRange

test/integration/framework/master_utils.go

@@ -144,7 +144,6 @@ func startMasterOrDie(masterConfig *master.Config) (*master.Master, *httptest.Se
 			StorageDestinations:  storageDestinations,
 			StorageVersions:      storageVersions,
 			KubeletClient:        client.FakeKubeletClient{},
-			EnableExp:            true,
 			EnableLogsSupport:    false,
 			EnableProfiling:      true,
 			EnableSwaggerSupport: true,
@@ -292,7 +291,6 @@ func RunAMaster(t *testing.T) (*master.Master, *httptest.Server) {
 		EnableUISupport:     false,
 		APIPrefix:           "/api",
 		APIGroupPrefix:      "/apis",
-		EnableExp:           true,
 		Authorizer:          apiserver.NewAlwaysAllowAuthorizer(),
 		AdmissionControl:    admit.NewAlwaysAdmit(),
 		StorageVersions:     storageVersions,