add --controller flag for cloud-controller manager

cmd/cloud-controller-manager/app/controllermanager.go

@@ -20,12 +20,14 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"net/http"
 	"os"
 	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
 
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/server"
@@ -72,7 +74,7 @@ the cloud specific control loops shipped with Kubernetes.`,
 			verflag.PrintAndExitIfRequested()
 			utilflag.PrintFlags(cmd.Flags())
 
-			c, err := s.Config()
+			c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List())
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "%v\n", err)
 				os.Exit(1)
@@ -87,7 +89,7 @@ the cloud specific control loops shipped with Kubernetes.`,
 	}
 
 	fs := cmd.Flags()
-	namedFlagSets := s.Flags()
+	namedFlagSets := s.Flags(KnownControllers(), ControllersDisabledByDefault.List())
 	verflag.AddFlags(namedFlagSets.FlagSet("global"))
 	globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())
 	cmoptions.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic"))
@@ -297,3 +299,29 @@ func startControllers(c *cloudcontrollerconfig.CompletedConfig, stop <-chan stru
 
 	select {}
 }
+
+// initFunc is used to launch a particular controller.  It may run additional "should I activate checks".
+// Any error returned will cause the controller process to `Fatal`
+// The bool indicates whether the controller was enabled.
+type initFunc func(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stop <-chan struct{}) (debuggingHandler http.Handler, enabled bool, err error)
+
+// KnownControllers indicate the default controller we are known.
+func KnownControllers() []string {
+	ret := sets.StringKeySet(newControllerInitializers())
+	return ret.List()
+}
+
+// ControllersDisabledByDefault is the controller disabled default when starting cloud-controller managers.
+var ControllersDisabledByDefault = sets.NewString()
+
+// newControllerInitializers is a private map of named controller groups (you can start more than one in an init func)
+// paired to their initFunc.  This allows for structured downstream composition and subdivision.
+func newControllerInitializers() map[string]initFunc {
+	controllers := map[string]initFunc{}
+	controllers["cloud-node"] = startCloudNodeController
+	controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleController
+	controllers["persistentvolume-binder"] = startPersistentVolumeLabelController
+	controllers["service"] = startServiceController
+	controllers["route"] = startRouteController
+	return controllers
+}

cmd/cloud-controller-manager/app/options/options.go

@@ -22,8 +22,6 @@ import (
 	"net"
 	"time"
 
-	"k8s.io/klog"
-
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
@@ -36,6 +34,7 @@ import (
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/record"
+	"k8s.io/klog"
 	ccmconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config"
 	ccmconfigscheme "k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/scheme"
 	ccmconfigv1alpha1 "k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1"
@@ -125,11 +124,9 @@ func NewDefaultComponentConfig(insecurePort int32) (*ccmconfig.CloudControllerMa
 }
 
 // Flags returns flags for a specific APIServer by section name
-func (o *CloudControllerManagerOptions) Flags() apiserverflag.NamedFlagSets {
+func (o *CloudControllerManagerOptions) Flags(allControllers, disabledByDefaultControllers []string) apiserverflag.NamedFlagSets {
 	fss := apiserverflag.NamedFlagSets{}
-	o.Generic.AddFlags(&fss, []string{}, []string{})
+	o.Generic.AddFlags(&fss, allControllers, disabledByDefaultControllers)
-	// TODO: Implement the --controllers flag fully for the ccm
-	fss.FlagSet("generic").MarkHidden("controllers")
 	o.KubeCloudShared.AddFlags(fss.FlagSet("generic"))
 	o.ServiceController.AddFlags(fss.FlagSet("service controller"))
 
@@ -219,10 +216,10 @@ func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config,
 }
 
 // Validate is used to validate config before launching the cloud controller manager
-func (o *CloudControllerManagerOptions) Validate() error {
+func (o *CloudControllerManagerOptions) Validate(allControllers, disabledByDefaultControllers []string) error {
 	errors := []error{}
 
-	errors = append(errors, o.Generic.Validate(nil, nil)...)
+	errors = append(errors, o.Generic.Validate(allControllers, disabledByDefaultControllers)...)
 	errors = append(errors, o.KubeCloudShared.Validate()...)
 	errors = append(errors, o.ServiceController.Validate()...)
 	errors = append(errors, o.SecureServing.Validate()...)
@@ -246,8 +243,8 @@ func resyncPeriod(c *cloudcontrollerconfig.Config) func() time.Duration {
 }
 
 // Config return a cloud controller manager config objective
-func (o *CloudControllerManagerOptions) Config() (*cloudcontrollerconfig.Config, error) {
+func (o *CloudControllerManagerOptions) Config(allControllers, disabledByDefaultControllers []string) (*cloudcontrollerconfig.Config, error) {
-	if err := o.Validate(); err != nil {
+	if err := o.Validate(allControllers, disabledByDefaultControllers); err != nil {
 		return nil, err
 	}
 

cmd/cloud-controller-manager/app/options/options_test.go

@@ -115,7 +115,7 @@ func TestDefaultFlags(t *testing.T) {
 func TestAddFlags(t *testing.T) {
 	fs := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError)
 	s, _ := NewCloudControllerManagerOptions()
-	for _, f := range s.Flags().FlagSets {
+	for _, f := range s.Flags([]string{""}, []string{""}).FlagSets {
 		fs.AddFlagSet(f)
 	}
 
@@ -131,6 +131,7 @@ func TestAddFlags(t *testing.T) {
 		"--configure-cloud-routes=false",
 		"--contention-profiling=true",
 		"--controller-start-interval=2m",
+		"--controllers=foo,bar",
 		"--http2-max-streams-per-connection=47",
 		"--kube-api-burst=100",
 		"--kube-api-content-type=application/vnd.kubernetes.protobuf",
@@ -173,7 +174,7 @@ func TestAddFlags(t *testing.T) {
 			Debugging: &cmoptions.DebuggingOptions{
 				EnableContentionProfiling: true,
 			},
-			Controllers: []string{"*"},
+			Controllers: []string{"foo", "bar"},
 		},
 		KubeCloudShared: &cmoptions.KubeCloudSharedOptions{
 			CloudProvider: &cmoptions.CloudProviderOptions{

cmd/cloud-controller-manager/app/testing/testserver.go

@@ -83,7 +83,8 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err
 	if err != nil {
 		return TestServer{}, err
 	}
-	namedFlagSets := s.Flags()
+	all, disabled := app.KnownControllers(), app.ControllersDisabledByDefault.List()
+	namedFlagSets := s.Flags(all, disabled)
 	for _, f := range namedFlagSets.FlagSets {
 		fs.AddFlagSet(f)
 	}
@@ -108,7 +109,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err
 		t.Logf("cloud-controller-manager will listen insecurely on port %d...", s.InsecureServing.BindPort)
 	}
 
-	config, err := s.Config()
+	config, err := s.Config(all, disabled)
 	if err != nil {
 		return result, fmt.Errorf("failed to create config from options: %v", err)
 	}

cmd/controller-manager/app/options/generic.go

@@ -71,7 +71,6 @@ func (o *GenericControllerManagerConfigurationOptions) AddFlags(fss *apiserverfl
 	genericfs.Float32Var(&o.ClientConnection.QPS, "kube-api-qps", o.ClientConnection.QPS, "QPS to use while talking with kubernetes apiserver.")
 	genericfs.Int32Var(&o.ClientConnection.Burst, "kube-api-burst", o.ClientConnection.Burst, "Burst to use while talking with kubernetes apiserver.")
 	genericfs.DurationVar(&o.ControllerStartInterval.Duration, "controller-start-interval", o.ControllerStartInterval.Duration, "Interval between starting controller managers.")
-	// TODO: complete the work of the cloud-controller-manager (and possibly other consumers of this code) respecting the --controllers flag
 	genericfs.StringSliceVar(&o.Controllers, "controllers", o.Controllers, fmt.Sprintf(""+
 		"A list of controllers to enable. '*' enables all on-by-default controllers, 'foo' enables the controller "+
 		"named 'foo', '-foo' disables the controller named 'foo'.\nAll controllers: %s\nDisabled-by-default controllers: %s",

cmd/kube-controller-manager/app/controllermanager.go

@@ -387,7 +387,7 @@ func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc
 	if loopMode == IncludeCloudLoops {
 		controllers["service"] = startServiceController
 		controllers["route"] = startRouteController
-		controllers["cloudnodelifecycle"] = startCloudNodeLifecycleController
+		controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleController
 		// TODO: volume controller into the IncludeCloudLoops only set.
 	}
 	controllers["persistentvolume-binder"] = startPersistentVolumeBinderController