Seperate loop and plugin control

Seperate loop and plugin control in the kube-controller-manager.
Adding an "--external-plugin" flag to specify a plugin to load when
cloud-provider is set to "external". Flag has no effect currently
when the cloud-provider is not set to external. The expectation is
that the cloud provider and external plugin flags would go away once
all cloud providers are on stage 2 cloud-controller-manager solutions.

Managing the control loops more directly based on start up flags.
Addressing issue brought up by @wlan0

Switched to using the main node controller in CCM.
Changes to enable full NodeController to start in CCM.
Fix related tests.
Unifying some common code between KCM and CCM.
Fix related tests and comments.
Folded in feedback from @jhorwit2 and @wlan0

cmd/BUILD

@@ -13,6 +13,7 @@ filegroup(
         ":package-srcs",
         "//cmd/clicheck:all-srcs",
         "//cmd/cloud-controller-manager:all-srcs",
+        "//cmd/controller-manager/app/options:all-srcs",
         "//cmd/gendocs:all-srcs",
         "//cmd/genkubedocs:all-srcs",
         "//cmd/genman:all-srcs",

cmd/cloud-controller-manager/app/controllermanager.go

@@ -147,7 +147,7 @@ func Run(s *options.CloudControllerManagerServer) error {
 			clientBuilder = rootClientBuilder
 		}
 
-		if err := StartControllers(s, kubeconfig, clientBuilder, stop, recorder, cloud); err != nil {
+		if err := StartControllers(s, kubeconfig, rootClientBuilder, clientBuilder, stop, recorder, cloud); err != nil {
 			glog.Fatalf("error running controllers: %v", err)
 		}
 	}
@@ -193,7 +193,7 @@ func Run(s *options.CloudControllerManagerServer) error {
 }
 
 // StartControllers starts the cloud specific controller loops.
-func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restclient.Config, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}, recorder record.EventRecorder, cloud cloudprovider.Interface) error {
+func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restclient.Config, rootClientBuilder, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}, recorder record.EventRecorder, cloud cloudprovider.Interface) error {
 	// Function to build the kube client object
 	client := func(serviceAccountName string) clientset.Interface {
 		return clientBuilder.ClientOrDie(serviceAccountName)
@@ -204,7 +204,7 @@ func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restc
 		cloud.Initialize(clientBuilder)
 	}
 
-	versionedClient := client("shared-informers")
+	versionedClient := rootClientBuilder.ClientOrDie("shared-informers")
 	sharedInformers := informers.NewSharedInformerFactory(versionedClient, resyncPeriod(s)())
 
 	// Start the CloudNodeController

cmd/cloud-controller-manager/app/options/BUILD

@@ -11,7 +11,7 @@ go_library(
     srcs = ["options.go"],
     importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/options",
     deps = [
-        "//pkg/apis/componentconfig:go_default_library",
+        "//cmd/controller-manager/app/options:go_default_library",
         "//pkg/client/leaderelectionconfig:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/master/ports:go_default_library",
@@ -40,6 +40,7 @@ go_test(
     importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/options",
     library = ":go_default_library",
     deps = [
+        "//cmd/controller-manager/app/options:go_default_library",
         "//pkg/apis/componentconfig:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

cmd/cloud-controller-manager/app/options/options.go

@@ -21,7 +21,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
 	"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
 	"k8s.io/kubernetes/pkg/master/ports"
 
@@ -33,10 +33,7 @@ import (
 
 // CloudControllerManagerServer is the main context object for the controller manager.
 type CloudControllerManagerServer struct {
-	componentconfig.KubeControllerManagerConfiguration
+	cmoptions.ControllerManagerServer
-
-	Master     string
-	Kubeconfig string
 
 	// NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status
 	NodeStatusUpdateFrequency metav1.Duration
@@ -45,22 +42,10 @@ type CloudControllerManagerServer struct {
 // NewCloudControllerManagerServer creates a new ExternalCMServer with a default config.
 func NewCloudControllerManagerServer() *CloudControllerManagerServer {
 	s := CloudControllerManagerServer{
-		// Part of these default values also present in 'cmd/kube-controller-manager/app/options/options.go'.
+		// The common/default are kept in 'cmd/kube-controller-manager/app/options/util.go'.
-		// Please keep them in sync when doing update.
+		// Please make common changes there and put anything cloud specific here.
-		KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
+		ControllerManagerServer: cmoptions.ControllerManagerServer{
-			Port:                      ports.CloudControllerManagerPort,
+			KubeControllerManagerConfiguration: cmoptions.GetDefaultControllerOptions(ports.CloudControllerManagerPort),
-			Address:                   "0.0.0.0",
-			ConcurrentServiceSyncs:    1,
-			MinResyncPeriod:           metav1.Duration{Duration: 12 * time.Hour},
-			NodeMonitorPeriod:         metav1.Duration{Duration: 5 * time.Second},
-			ClusterName:               "kubernetes",
-			ConfigureCloudRoutes:      true,
-			ContentType:               "application/vnd.kubernetes.protobuf",
-			KubeAPIQPS:                20.0,
-			KubeAPIBurst:              30,
-			LeaderElection:            leaderelectionconfig.DefaultLeaderElectionConfiguration(),
-			ControllerStartInterval:   metav1.Duration{Duration: 0 * time.Second},
-			RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second},
 		},
 		NodeStatusUpdateFrequency: metav1.Duration{Duration: 5 * time.Minute},
 	}
@@ -70,34 +55,14 @@ func NewCloudControllerManagerServer() *CloudControllerManagerServer {
 
 // AddFlags adds flags for a specific ExternalCMServer to the specified FlagSet
 func (s *CloudControllerManagerServer) AddFlags(fs *pflag.FlagSet) {
-	fs.Int32Var(&s.Port, "port", s.Port, "The port that the cloud-controller-manager's http service runs on.")
+	cmoptions.AddDefaultControllerFlags(&s.ControllerManagerServer, fs)
-	fs.Var(componentconfig.IPVar{Val: &s.Address}, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces).")
 	fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider of cloud services. Cannot be empty.")
-	fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.")
-	fs.BoolVar(&s.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances. This is a legacy mode of operation and a cluster-id will be required in the future.")
-	fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release. A cluster-id will be required on cloud instances.")
-	fs.DurationVar(&s.MinResyncPeriod.Duration, "min-resync-period", s.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod.")
-	fs.DurationVar(&s.NodeMonitorPeriod.Duration, "node-monitor-period", s.NodeMonitorPeriod.Duration,
-		"The period for syncing NodeStatus in NodeController.")
 	fs.DurationVar(&s.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", s.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.")
 	// TODO: remove --service-account-private-key-file 6 months after 1.8 is released (~1.10)
 	fs.StringVar(&s.ServiceAccountKeyFile, "service-account-private-key-file", s.ServiceAccountKeyFile, "Filename containing a PEM-encoded private RSA or ECDSA key used to sign service account tokens.")
 	fs.MarkDeprecated("service-account-private-key-file", "This flag is currently no-op and will be deleted.")
-	fs.BoolVar(&s.UseServiceAccountCredentials, "use-service-account-credentials", s.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.")
-	fs.DurationVar(&s.RouteReconciliationPeriod.Duration, "route-reconciliation-period", s.RouteReconciliationPeriod.Duration, "The period for reconciling routes created for Nodes by cloud provider.")
-	fs.BoolVar(&s.ConfigureCloudRoutes, "configure-cloud-routes", true, "Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider.")
-	fs.BoolVar(&s.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/.")
-	fs.BoolVar(&s.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled.")
-	fs.StringVar(&s.ClusterCIDR, "cluster-cidr", s.ClusterCIDR, "CIDR Range for Pods in cluster.")
-	fs.StringVar(&s.ClusterName, "cluster-name", s.ClusterName, "The instance prefix for the cluster.")
-	fs.BoolVar(&s.AllocateNodeCIDRs, "allocate-node-cidrs", false, "Should CIDRs for Pods be allocated and set on the cloud provider.")
-	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
-	fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
-	fs.StringVar(&s.ContentType, "kube-api-content-type", s.ContentType, "Content type of requests sent to apiserver.")
-	fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver.")
-	fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver.")
-	fs.DurationVar(&s.ControllerStartInterval.Duration, "controller-start-interval", s.ControllerStartInterval.Duration, "Interval between starting controller managers.")
 	fs.Int32Var(&s.ConcurrentServiceSyncs, "concurrent-service-syncs", s.ConcurrentServiceSyncs, "The number of services that are allowed to sync concurrently. Larger number = more responsive service management, but more CPU (and network) load")
+
 	leaderelectionconfig.BindFlags(&s.LeaderElection, fs)
 
 	utilfeature.DefaultFeatureGate.AddFlag(fs)

cmd/cloud-controller-manager/app/options/options_test.go

@@ -25,6 +25,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/diff"
+	cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 )
 
@@ -65,34 +66,82 @@ func TestAddFlags(t *testing.T) {
 	f.Parse(args)
 
 	expected := &CloudControllerManagerServer{
-		KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
+		ControllerManagerServer: cmoptions.ControllerManagerServer{
-			CloudProvider:             "gce",
+			KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
-			CloudConfigFile:           "/cloud-config",
+				CloudProvider:                                   "gce",
-			Port:                      10000,
+				CloudConfigFile:                                 "/cloud-config",
-			Address:                   "192.168.4.10",
+				Port:                                            10000,
-			ConcurrentServiceSyncs:    1,
+				Address:                                         "192.168.4.10",
-			MinResyncPeriod:           metav1.Duration{Duration: 100 * time.Minute},
+				ConcurrentEndpointSyncs:                         5,
-			NodeMonitorPeriod:         metav1.Duration{Duration: 5 * time.Second},
+				ConcurrentRSSyncs:                               5,
-			ClusterName:               "k8s",
+				ConcurrentResourceQuotaSyncs:                    5,
-			ConfigureCloudRoutes:      false,
+				ConcurrentDeploymentSyncs:                       5,
-			AllocateNodeCIDRs:         true,
+				ConcurrentDaemonSetSyncs:                        2,
-			ContentType:               "application/vnd.kubernetes.protobuf",
+				ConcurrentJobSyncs:                              5,
-			EnableContentionProfiling: true,
+				ConcurrentNamespaceSyncs:                        10,
-			KubeAPIQPS:                50.0,
+				ConcurrentSATokenSyncs:                          5,
-			KubeAPIBurst:              100,
+				ConcurrentServiceSyncs:                          1,
-			LeaderElection: componentconfig.LeaderElectionConfiguration{
+				ConcurrentGCSyncs:                               20,
-				ResourceLock:  "configmap",
+				ConcurrentRCSyncs:                               5,
-				LeaderElect:   false,
+				MinResyncPeriod:                                 metav1.Duration{Duration: 100 * time.Minute},
-				LeaseDuration: metav1.Duration{Duration: 30 * time.Second},
+				NodeMonitorPeriod:                               metav1.Duration{Duration: 5 * time.Second},
-				RenewDeadline: metav1.Duration{Duration: 15 * time.Second},
+				ServiceSyncPeriod:                               metav1.Duration{Duration: 5 * time.Minute},
-				RetryPeriod:   metav1.Duration{Duration: 5 * time.Second},
+				ResourceQuotaSyncPeriod:                         metav1.Duration{Duration: 5 * time.Minute},
+				NamespaceSyncPeriod:                             metav1.Duration{Duration: 5 * time.Minute},
+				PVClaimBinderSyncPeriod:                         metav1.Duration{Duration: 15 * time.Second},
+				HorizontalPodAutoscalerSyncPeriod:               metav1.Duration{Duration: 30 * time.Second},
+				HorizontalPodAutoscalerUpscaleForbiddenWindow:   metav1.Duration{Duration: 3 * time.Minute},
+				HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute},
+				HorizontalPodAutoscalerTolerance:                0.1,
+				DeploymentControllerSyncPeriod:                  metav1.Duration{Duration: 30 * time.Second},
+				PodEvictionTimeout:                              metav1.Duration{Duration: 5 * time.Minute},
+				NodeMonitorGracePeriod:                          metav1.Duration{Duration: 40 * time.Second},
+				NodeStartupGracePeriod:                          metav1.Duration{Duration: 1 * time.Minute},
+				ClusterSigningDuration:                          metav1.Duration{Duration: 8760 * time.Hour},
+				ReconcilerSyncLoopPeriod:                        metav1.Duration{Duration: 1 * time.Minute},
+				TerminatedPodGCThreshold:                        12500,
+				RegisterRetryCount:                              10,
+				ClusterName:                                     "k8s",
+				ConfigureCloudRoutes:                            false,
+				AllocateNodeCIDRs:                               true,
+				EnableGarbageCollector:                          true,
+				EnableTaintManager:                              true,
+				HorizontalPodAutoscalerUseRESTClients:           true,
+				VolumeConfiguration: componentconfig.VolumeConfiguration{
+					EnableDynamicProvisioning:  true,
+					EnableHostPathProvisioning: false,
+					FlexVolumePluginDir:        "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
+					PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{
+						MaximumRetry:             3,
+						MinimumTimeoutNFS:        300,
+						IncrementTimeoutNFS:      30,
+						MinimumTimeoutHostPath:   60,
+						IncrementTimeoutHostPath: 30,
+					},
+				},
+				ContentType:               "application/vnd.kubernetes.protobuf",
+				ClusterSigningCertFile:    "/etc/kubernetes/ca/ca.pem",
+				ClusterSigningKeyFile:     "/etc/kubernetes/ca/ca.key",
+				EnableContentionProfiling: true,
+				KubeAPIQPS:                50.0,
+				KubeAPIBurst:              100,
+				LeaderElection: componentconfig.LeaderElectionConfiguration{
+					ResourceLock:  "configmap",
+					LeaderElect:   false,
+					LeaseDuration: metav1.Duration{Duration: 30 * time.Second},
+					RenewDeadline: metav1.Duration{Duration: 15 * time.Second},
+					RetryPeriod:   metav1.Duration{Duration: 5 * time.Second},
+				},
+				ControllerStartInterval:   metav1.Duration{Duration: 2 * time.Minute},
+				RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second},
+				ClusterCIDR:               "1.2.3.4/24",
+				NodeCIDRMaskSize:          24,
+				CIDRAllocatorType:         "RangeAllocator",
+				Controllers:               []string{"*"},
 			},
-			ControllerStartInterval:   metav1.Duration{Duration: 2 * time.Minute},
+			Kubeconfig: "/kubeconfig",
-			RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second},
+			Master:     "192.168.4.20",
-			ClusterCIDR:               "1.2.3.4/24",
 		},
-		Kubeconfig: "/kubeconfig",
-		Master:     "192.168.4.20",
 		NodeStatusUpdateFrequency: metav1.Duration{Duration: 10 * time.Minute},
 	}
 	if !reflect.DeepEqual(expected, s) {

cmd/controller-manager/app/options/BUILD

@@ -0,0 +1,29 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["utils.go"],
+    importpath = "k8s.io/kubernetes/cmd/controller-manager/app/options",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/apis/componentconfig:go_default_library",
+        "//pkg/client/leaderelectionconfig:go_default_library",
+        "//vendor/github.com/cloudflare/cfssl/helpers:go_default_library",
+        "//vendor/github.com/spf13/pflag:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

cmd/controller-manager/app/options/utils.go

@@ -0,0 +1,140 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"github.com/cloudflare/cfssl/helpers"
+	"time"
+
+	"github.com/spf13/pflag"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
+)
+
+// ControllerManagerServer is the common structure for a controller manager. It works with GetDefaultControllerOptions
+// and AddDefaultControllerFlags to create the common components of kube-controller-manager and cloud-controller-manager.
+type ControllerManagerServer struct {
+	componentconfig.KubeControllerManagerConfiguration
+
+	Master     string
+	Kubeconfig string
+}
+
+const (
+	// These defaults are deprecated and exported so that we can warn if
+	// they are being used.
+
+	// DefaultClusterSigningCertFile is deprecated. Do not use.
+	DefaultClusterSigningCertFile = "/etc/kubernetes/ca/ca.pem"
+	// DefaultClusterSigningKeyFile is deprecated. Do not use.
+	DefaultClusterSigningKeyFile = "/etc/kubernetes/ca/ca.key"
+)
+
+// GetDefaultControllerOptions returns common/default configuration values for both
+// the kube-controller-manager and the cloud-contoller-manager. Any common changes should
+// be made here. Any individual changes should be made in that controller.
+func GetDefaultControllerOptions(port int32) componentconfig.KubeControllerManagerConfiguration {
+	return componentconfig.KubeControllerManagerConfiguration{
+		Controllers:                                     []string{"*"},
+		Port:                                            port,
+		Address:                                         "0.0.0.0",
+		ConcurrentEndpointSyncs:                         5,
+		ConcurrentServiceSyncs:                          1,
+		ConcurrentRCSyncs:                               5,
+		ConcurrentRSSyncs:                               5,
+		ConcurrentDaemonSetSyncs:                        2,
+		ConcurrentJobSyncs:                              5,
+		ConcurrentResourceQuotaSyncs:                    5,
+		ConcurrentDeploymentSyncs:                       5,
+		ConcurrentNamespaceSyncs:                        10,
+		ConcurrentSATokenSyncs:                          5,
+		ServiceSyncPeriod:                               metav1.Duration{Duration: 5 * time.Minute},
+		RouteReconciliationPeriod:                       metav1.Duration{Duration: 10 * time.Second},
+		ResourceQuotaSyncPeriod:                         metav1.Duration{Duration: 5 * time.Minute},
+		NamespaceSyncPeriod:                             metav1.Duration{Duration: 5 * time.Minute},
+		PVClaimBinderSyncPeriod:                         metav1.Duration{Duration: 15 * time.Second},
+		HorizontalPodAutoscalerSyncPeriod:               metav1.Duration{Duration: 30 * time.Second},
+		HorizontalPodAutoscalerUpscaleForbiddenWindow:   metav1.Duration{Duration: 3 * time.Minute},
+		HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute},
+		HorizontalPodAutoscalerTolerance:                0.1,
+		DeploymentControllerSyncPeriod:                  metav1.Duration{Duration: 30 * time.Second},
+		MinResyncPeriod:                                 metav1.Duration{Duration: 12 * time.Hour},
+		RegisterRetryCount:                              10,
+		PodEvictionTimeout:                              metav1.Duration{Duration: 5 * time.Minute},
+		NodeMonitorGracePeriod:                          metav1.Duration{Duration: 40 * time.Second},
+		NodeStartupGracePeriod:                          metav1.Duration{Duration: 60 * time.Second},
+		NodeMonitorPeriod:                               metav1.Duration{Duration: 5 * time.Second},
+		ClusterName:                                     "kubernetes",
+		NodeCIDRMaskSize:                                24,
+		ConfigureCloudRoutes:                            true,
+		TerminatedPodGCThreshold:                        12500,
+		VolumeConfiguration: componentconfig.VolumeConfiguration{
+			EnableHostPathProvisioning: false,
+			EnableDynamicProvisioning:  true,
+			PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{
+				MaximumRetry:             3,
+				MinimumTimeoutNFS:        300,
+				IncrementTimeoutNFS:      30,
+				MinimumTimeoutHostPath:   60,
+				IncrementTimeoutHostPath: 30,
+			},
+			FlexVolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
+		},
+		ContentType:                           "application/vnd.kubernetes.protobuf",
+		KubeAPIQPS:                            20.0,
+		KubeAPIBurst:                          30,
+		LeaderElection:                        leaderelectionconfig.DefaultLeaderElectionConfiguration(),
+		ControllerStartInterval:               metav1.Duration{Duration: 0 * time.Second},
+		EnableGarbageCollector:                true,
+		ConcurrentGCSyncs:                     20,
+		ClusterSigningCertFile:                DefaultClusterSigningCertFile,
+		ClusterSigningKeyFile:                 DefaultClusterSigningKeyFile,
+		ClusterSigningDuration:                metav1.Duration{Duration: helpers.OneYear},
+		ReconcilerSyncLoopPeriod:              metav1.Duration{Duration: 60 * time.Second},
+		EnableTaintManager:                    true,
+		HorizontalPodAutoscalerUseRESTClients: true,
+	}
+}
+
+// AddDefaultControllerFlags adds common/default flags for both the kube and cloud Controller Manager Server to the
+// specified FlagSet. Any common changes should be made here. Any individual changes should be made in that controller.
+func AddDefaultControllerFlags(s *ControllerManagerServer, fs *pflag.FlagSet) {
+	fs.Int32Var(&s.Port, "port", s.Port, "The port that the controller-manager's http service runs on.")
+	fs.Var(componentconfig.IPVar{Val: &s.Address}, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces).")
+	fs.BoolVar(&s.UseServiceAccountCredentials, "use-service-account-credentials", s.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.")
+	fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.")
+	fs.BoolVar(&s.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances. This is a legacy mode of operation and a cluster-id will be required in the future.")
+	fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release. A cluster-id will be required on cloud instances.")
+	fs.DurationVar(&s.RouteReconciliationPeriod.Duration, "route-reconciliation-period", s.RouteReconciliationPeriod.Duration, "The period for reconciling routes created for Nodes by cloud provider.")
+	fs.DurationVar(&s.MinResyncPeriod.Duration, "min-resync-period", s.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod.")
+	fs.DurationVar(&s.NodeMonitorPeriod.Duration, "node-monitor-period", s.NodeMonitorPeriod.Duration,
+		"The period for syncing NodeStatus in NodeController.")
+	fs.BoolVar(&s.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/")
+	fs.BoolVar(&s.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled.")
+	fs.StringVar(&s.ClusterName, "cluster-name", s.ClusterName, "The instance prefix for the cluster.")
+	fs.StringVar(&s.ClusterCIDR, "cluster-cidr", s.ClusterCIDR, "CIDR Range for Pods in cluster. Requires --allocate-node-cidrs to be true")
+	fs.BoolVar(&s.AllocateNodeCIDRs, "allocate-node-cidrs", false, "Should CIDRs for Pods be allocated and set on the cloud provider.")
+	fs.StringVar(&s.CIDRAllocatorType, "cidr-allocator-type", "RangeAllocator", "Type of CIDR allocator to use")
+	fs.BoolVar(&s.ConfigureCloudRoutes, "configure-cloud-routes", true, "Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider.")
+	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
+	fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
+	fs.StringVar(&s.ContentType, "kube-api-content-type", s.ContentType, "Content type of requests sent to apiserver.")
+	fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver.")
+	fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver.")
+	fs.DurationVar(&s.ControllerStartInterval.Duration, "controller-start-interval", s.ControllerStartInterval.Duration, "Interval between starting controller managers.")
+}

cmd/kube-controller-manager/app/BUILD

@@ -24,6 +24,7 @@ go_library(
     ],
     importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app",
     deps = [
+        "//cmd/controller-manager/app/options:go_default_library",
         "//cmd/kube-controller-manager/app/options:go_default_library",
         "//pkg/api/legacyscheme:go_default_library",
         "//pkg/apis/apps/install:go_default_library",

cmd/kube-controller-manager/app/certificates.go

@@ -27,7 +27,7 @@ import (
 	"github.com/golang/glog"
 
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
+	cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
 	"k8s.io/kubernetes/pkg/controller/certificates/approver"
 	"k8s.io/kubernetes/pkg/controller/certificates/cleaner"
 	"k8s.io/kubernetes/pkg/controller/certificates/signer"
@@ -55,12 +55,12 @@ func startCSRSigningController(ctx ControllerContext) (bool, error) {
 	_, err := os.Stat(ctx.Options.ClusterSigningCertFile)
 	certFileExists = !os.IsNotExist(err)
 
-	certUsesDefault = (ctx.Options.ClusterSigningCertFile == options.DefaultClusterSigningCertFile)
+	certUsesDefault = (ctx.Options.ClusterSigningCertFile == cmoptions.DefaultClusterSigningCertFile)
 
 	_, err = os.Stat(ctx.Options.ClusterSigningKeyFile)
 	keyFileExists = !os.IsNotExist(err)
 
-	keyUsesDefault = (ctx.Options.ClusterSigningKeyFile == options.DefaultClusterSigningKeyFile)
+	keyUsesDefault = (ctx.Options.ClusterSigningKeyFile == cmoptions.DefaultClusterSigningKeyFile)
 
 	switch {
 	case (keyFileExists && keyUsesDefault) || (certFileExists && certUsesDefault):

cmd/kube-controller-manager/app/controllermanager.go

@@ -71,6 +71,13 @@ const (
 	ControllerStartJitter = 1.0
 )
 
+type ControllerLoopMode int
+
+const (
+	IncludeCloudLoops ControllerLoopMode = iota
+	ExternalLoops
+)
+
 // NewControllerManagerCommand creates a *cobra.Command object with default parameters
 func NewControllerManagerCommand() *cobra.Command {
 	s := options.NewCMServer()
@@ -151,7 +158,7 @@ func Run(s *options.CMServer) error {
 		}
 		saTokenControllerInitFunc := serviceAccountTokenControllerStarter{rootClientBuilder: rootClientBuilder}.startServiceAccountTokenController
 
-		if err := StartControllers(ctx, saTokenControllerInitFunc, NewControllerInitializers()); err != nil {
+		if err := StartControllers(ctx, saTokenControllerInitFunc, NewControllerInitializers(ctx.LoopMode)); err != nil {
 			glog.Fatalf("error starting controllers: %v", err)
 		}
 
@@ -262,6 +269,11 @@ type ControllerContext struct {
 	// It must be initialized and ready to use.
 	Cloud cloudprovider.Interface
 
+	// Control for which control loops to be run
+	// IncludeCloudLoops is for a kube-controller-manager running all loops
+	// ExternalLoops is for a kube-controller-manager running with a cloud-controller-manager
+	LoopMode ControllerLoopMode
+
 	// Stop is the stop channel
 	Stop <-chan struct{}
 
@@ -305,7 +317,7 @@ func IsControllerEnabled(name string, disabledByDefaultControllers sets.String,
 type InitFunc func(ctx ControllerContext) (bool, error)
 
 func KnownControllers() []string {
-	ret := sets.StringKeySet(NewControllerInitializers())
+	ret := sets.StringKeySet(NewControllerInitializers(IncludeCloudLoops))
 
 	// add "special" controllers that aren't initialized normally.  These controllers cannot be initialized
 	// using a normal function.  The only known special case is the SA token controller which *must* be started
@@ -329,7 +341,7 @@ const (
 
 // NewControllerInitializers is a public map of named controller groups (you can start more than one in an init func)
 // paired to their InitFunc.  This allows for structured downstream composition and subdivision.
-func NewControllerInitializers() map[string]InitFunc {
+func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc {
 	controllers := map[string]InitFunc{}
 	controllers["endpoint"] = startEndpointController
 	controllers["replicationcontroller"] = startReplicationController
@@ -352,9 +364,12 @@ func NewControllerInitializers() map[string]InitFunc {
 	controllers["ttl"] = startTTLController
 	controllers["bootstrapsigner"] = startBootstrapSignerController
 	controllers["tokencleaner"] = startTokenCleanerController
-	controllers["service"] = startServiceController
+	if loopMode == IncludeCloudLoops {
+		controllers["service"] = startServiceController
+		controllers["route"] = startRouteController
+		// TODO: Move node controller and volume controller into the IncludeCloudLoops only set.
+	}
 	controllers["node"] = startNodeController
-	controllers["route"] = startRouteController
 	controllers["persistentvolume-binder"] = startPersistentVolumeBinderController
 	controllers["attachdetach"] = startAttachDetachController
 	controllers["persistentvolume-expander"] = startVolumeExpandController
@@ -430,7 +445,17 @@ func CreateControllerContext(s *options.CMServer, rootClientBuilder, clientBuild
 		return ControllerContext{}, err
 	}
 
-	cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
+	var cloud cloudprovider.Interface
+	var loopMode ControllerLoopMode
+	if cloudprovider.IsExternal(s.CloudProvider) {
+		loopMode = ExternalLoops
+		if s.ExternalCloudVolumePlugin != "" {
+			cloud, err = cloudprovider.InitCloudProvider(s.ExternalCloudVolumePlugin, s.CloudConfigFile)
+		}
+	} else {
+		loopMode = IncludeCloudLoops
+		cloud, err = cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
+	}
 	if err != nil {
 		return ControllerContext{}, fmt.Errorf("cloud provider could not be initialized: %v", err)
 	}
@@ -453,6 +478,7 @@ func CreateControllerContext(s *options.CMServer, rootClientBuilder, clientBuild
 		Options:            *s,
 		AvailableResources: availableResources,
 		Cloud:              cloud,
+		LoopMode:           loopMode,
 		Stop:               stop,
 		InformersStarted:   make(chan struct{}),
 	}

cmd/kube-controller-manager/app/options/BUILD

@@ -11,14 +11,13 @@ go_library(
     srcs = ["options.go"],
     importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app/options",
     deps = [
+        "//cmd/controller-manager/app/options:go_default_library",
         "//pkg/apis/componentconfig:go_default_library",
         "//pkg/client/leaderelectionconfig:go_default_library",
         "//pkg/controller/garbagecollector:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/master/ports:go_default_library",
-        "//vendor/github.com/cloudflare/cfssl/helpers:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
@@ -45,6 +44,7 @@ go_test(
     library = ":go_default_library",
     tags = ["automanaged"],
     deps = [
+        "//cmd/controller-manager/app/options:go_default_library",
         "//pkg/apis/componentconfig:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

cmd/kube-controller-manager/app/options/options.go

@@ -21,12 +21,11 @@ package options
 import (
 	"fmt"
 	"strings"
-	"time"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
 	"k8s.io/kubernetes/pkg/controller/garbagecollector"
@@ -35,26 +34,12 @@ import (
 	// add the kubernetes feature gates
 	_ "k8s.io/kubernetes/pkg/features"
 
-	"github.com/cloudflare/cfssl/helpers"
 	"github.com/spf13/pflag"
 )
 
-const (
-	// These defaults are deprecated and exported so that we can warn if
-	// they are being used.
-
-	// DefaultClusterSigningCertFile is deprecated. Do not use.
-	DefaultClusterSigningCertFile = "/etc/kubernetes/ca/ca.pem"
-	// DefaultClusterSigningKeyFile is deprecated. Do not use.
-	DefaultClusterSigningKeyFile = "/etc/kubernetes/ca/ca.key"
-)
-
 // CMServer is the main context object for the controller manager.
 type CMServer struct {
-	componentconfig.KubeControllerManagerConfiguration
+	cmoptions.ControllerManagerServer
-
-	Master     string
-	Kubeconfig string
 }
 
 // NewCMServer creates a new CMServer with a default config.
@@ -65,91 +50,32 @@ func NewCMServer() *CMServer {
 	}
 
 	s := CMServer{
-		// Part of these default values also present in 'cmd/cloud-controller-manager/app/options/options.go'.
+		// The common/default are kept in 'cmd/kube-controller-manager/app/options/util.go'.
-		// Please keep them in sync when doing update.
+		// Please make common changes there but put anything kube-controller specific here.
-		KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
+		ControllerManagerServer: cmoptions.ControllerManagerServer{
-			Controllers:                                     []string{"*"},
+			KubeControllerManagerConfiguration: cmoptions.GetDefaultControllerOptions(ports.ControllerManagerPort),
-			Port:                                            ports.ControllerManagerPort,
-			Address:                                         "0.0.0.0",
-			ConcurrentEndpointSyncs:                         5,
-			ConcurrentServiceSyncs:                          1,
-			ConcurrentRCSyncs:                               5,
-			ConcurrentRSSyncs:                               5,
-			ConcurrentDaemonSetSyncs:                        2,
-			ConcurrentJobSyncs:                              5,
-			ConcurrentResourceQuotaSyncs:                    5,
-			ConcurrentDeploymentSyncs:                       5,
-			ConcurrentNamespaceSyncs:                        10,
-			ConcurrentSATokenSyncs:                          5,
-			ServiceSyncPeriod:                               metav1.Duration{Duration: 5 * time.Minute},
-			RouteReconciliationPeriod:                       metav1.Duration{Duration: 10 * time.Second},
-			ResourceQuotaSyncPeriod:                         metav1.Duration{Duration: 5 * time.Minute},
-			NamespaceSyncPeriod:                             metav1.Duration{Duration: 5 * time.Minute},
-			PVClaimBinderSyncPeriod:                         metav1.Duration{Duration: 15 * time.Second},
-			HorizontalPodAutoscalerSyncPeriod:               metav1.Duration{Duration: 30 * time.Second},
-			HorizontalPodAutoscalerUpscaleForbiddenWindow:   metav1.Duration{Duration: 3 * time.Minute},
-			HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute},
-			HorizontalPodAutoscalerTolerance:                0.1,
-			DeploymentControllerSyncPeriod:                  metav1.Duration{Duration: 30 * time.Second},
-			MinResyncPeriod:                                 metav1.Duration{Duration: 12 * time.Hour},
-			RegisterRetryCount:                              10,
-			PodEvictionTimeout:                              metav1.Duration{Duration: 5 * time.Minute},
-			NodeMonitorGracePeriod:                          metav1.Duration{Duration: 40 * time.Second},
-			NodeStartupGracePeriod:                          metav1.Duration{Duration: 60 * time.Second},
-			NodeMonitorPeriod:                               metav1.Duration{Duration: 5 * time.Second},
-			ClusterName:                                     "kubernetes",
-			NodeCIDRMaskSize:                                24,
-			ConfigureCloudRoutes:                            true,
-			TerminatedPodGCThreshold:                        12500,
-			VolumeConfiguration: componentconfig.VolumeConfiguration{
-				EnableHostPathProvisioning: false,
-				EnableDynamicProvisioning:  true,
-				PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{
-					MaximumRetry:             3,
-					MinimumTimeoutNFS:        300,
-					IncrementTimeoutNFS:      30,
-					MinimumTimeoutHostPath:   60,
-					IncrementTimeoutHostPath: 30,
-				},
-				FlexVolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
-			},
-			ContentType:                           "application/vnd.kubernetes.protobuf",
-			KubeAPIQPS:                            20.0,
-			KubeAPIBurst:                          30,
-			LeaderElection:                        leaderelectionconfig.DefaultLeaderElectionConfiguration(),
-			ControllerStartInterval:               metav1.Duration{Duration: 0 * time.Second},
-			EnableGarbageCollector:                true,
-			ConcurrentGCSyncs:                     20,
-			GCIgnoredResources:                    gcIgnoredResources,
-			ClusterSigningCertFile:                DefaultClusterSigningCertFile,
-			ClusterSigningKeyFile:                 DefaultClusterSigningKeyFile,
-			ClusterSigningDuration:                metav1.Duration{Duration: helpers.OneYear},
-			ReconcilerSyncLoopPeriod:              metav1.Duration{Duration: 60 * time.Second},
-			EnableTaintManager:                    true,
-			HorizontalPodAutoscalerUseRESTClients: true,
 		},
 	}
+	s.KubeControllerManagerConfiguration.GCIgnoredResources = gcIgnoredResources
 	s.LeaderElection.LeaderElect = true
 	return &s
 }
 
 // AddFlags adds flags for a specific CMServer to the specified FlagSet
 func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabledByDefaultControllers []string) {
+	cmoptions.AddDefaultControllerFlags(&s.ControllerManagerServer, fs)
+
 	fs.StringSliceVar(&s.Controllers, "controllers", s.Controllers, fmt.Sprintf(""+
 		"A list of controllers to enable.  '*' enables all on-by-default controllers, 'foo' enables the controller "+
 		"named 'foo', '-foo' disables the controller named 'foo'.\nAll controllers: %s\nDisabled-by-default controllers: %s",
 		strings.Join(allControllers, ", "), strings.Join(disabledByDefaultControllers, ", ")))
-	fs.Int32Var(&s.Port, "port", s.Port, "The port that the controller-manager's http service runs on")
-	fs.Var(componentconfig.IPVar{Val: &s.Address}, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces)")
-	fs.BoolVar(&s.UseServiceAccountCredentials, "use-service-account-credentials", s.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.")
 	fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider for cloud services.  Empty string for no provider.")
-	fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file.  Empty string for no configuration file.")
+	fs.StringVar(&s.ExternalCloudVolumePlugin, "external-cloud-volume-plugin", s.ExternalCloudVolumePlugin, "The plugin to use when cloud provider is set to external. Can be empty, should only be set when cloud-provider is external. Currently used to allow node and volume controllers to work for in tree cloud providers.")
-	fs.BoolVar(&s.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances.  This is a legacy mode of operation and a cluster-id will be required in the future.")
-	fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release.  A cluster-id will be required on cloud instances")
 	fs.Int32Var(&s.ConcurrentEndpointSyncs, "concurrent-endpoint-syncs", s.ConcurrentEndpointSyncs, "The number of endpoint syncing operations that will be done concurrently. Larger number = faster endpoint updating, but more CPU (and network) load")
 	fs.Int32Var(&s.ConcurrentServiceSyncs, "concurrent-service-syncs", s.ConcurrentServiceSyncs, "The number of services that are allowed to sync concurrently. Larger number = more responsive service management, but more CPU (and network) load")
 	fs.Int32Var(&s.ConcurrentRCSyncs, "concurrent_rc_syncs", s.ConcurrentRCSyncs, "The number of replication controllers that are allowed to sync concurrently. Larger number = more responsive replica management, but more CPU (and network) load")
 	fs.Int32Var(&s.ConcurrentRSSyncs, "concurrent-replicaset-syncs", s.ConcurrentRSSyncs, "The number of replica sets that are allowed to sync concurrently. Larger number = more responsive replica management, but more CPU (and network) load")
+
 	fs.Int32Var(&s.ConcurrentResourceQuotaSyncs, "concurrent-resource-quota-syncs", s.ConcurrentResourceQuotaSyncs, "The number of resource quotas that are allowed to sync concurrently. Larger number = more responsive quota management, but more CPU (and network) load")
 	fs.Int32Var(&s.ConcurrentDeploymentSyncs, "concurrent-deployment-syncs", s.ConcurrentDeploymentSyncs, "The number of deployment objects that are allowed to sync concurrently. Larger number = more responsive deployments, but more CPU (and network) load")
 	fs.Int32Var(&s.ConcurrentNamespaceSyncs, "concurrent-namespace-syncs", s.ConcurrentNamespaceSyncs, "The number of namespace objects that are allowed to sync concurrently. Larger number = more responsive namespace termination, but more CPU (and network) load")
@@ -159,11 +85,9 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabled
 		"This flag is deprecated and will be removed in future releases. See node-monitor-period for Node health checking or "+
 		"route-reconciliation-period for cloud provider's route configuration settings.")
 	fs.MarkDeprecated("node-sync-period", "This flag is currently no-op and will be deleted.")
-	fs.DurationVar(&s.RouteReconciliationPeriod.Duration, "route-reconciliation-period", s.RouteReconciliationPeriod.Duration, "The period for reconciling routes created for Nodes by cloud provider.")
 	fs.DurationVar(&s.ResourceQuotaSyncPeriod.Duration, "resource-quota-sync-period", s.ResourceQuotaSyncPeriod.Duration, "The period for syncing quota usage status in the system")
 	fs.DurationVar(&s.NamespaceSyncPeriod.Duration, "namespace-sync-period", s.NamespaceSyncPeriod.Duration, "The period for syncing namespace life-cycle updates")
 	fs.DurationVar(&s.PVClaimBinderSyncPeriod.Duration, "pvclaimbinder-sync-period", s.PVClaimBinderSyncPeriod.Duration, "The period for syncing persistent volumes and persistent volume claims")
-	fs.DurationVar(&s.MinResyncPeriod.Duration, "min-resync-period", s.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod")
 	fs.StringVar(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, "pv-recycler-pod-template-filepath-nfs", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, "The file path to a pod definition used as a template for NFS persistent volume recycling")
 	fs.Int32Var(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS, "pv-recycler-minimum-timeout-nfs", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS, "The minimum ActiveDeadlineSeconds to use for an NFS Recycler pod")
 	fs.Int32Var(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS, "pv-recycler-increment-timeout-nfs", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS, "the increment of time added per Gi to ActiveDeadlineSeconds for an NFS scrubber pod")
@@ -193,8 +117,6 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabled
 			"where N means number of retries allowed for kubelet to post node status.")
 	fs.DurationVar(&s.NodeStartupGracePeriod.Duration, "node-startup-grace-period", s.NodeStartupGracePeriod.Duration,
 		"Amount of time which we allow starting Node to be unresponsive before marking it unhealthy.")
-	fs.DurationVar(&s.NodeMonitorPeriod.Duration, "node-monitor-period", s.NodeMonitorPeriod.Duration,
-		"The period for syncing NodeStatus in NodeController.")
 	fs.StringVar(&s.ServiceAccountKeyFile, "service-account-private-key-file", s.ServiceAccountKeyFile, "Filename containing a PEM-encoded private RSA or ECDSA key used to sign service account tokens.")
 	fs.StringVar(&s.ClusterSigningCertFile, "cluster-signing-cert-file", s.ClusterSigningCertFile, "Filename containing a PEM-encoded X509 CA certificate used to issue cluster-scoped certificates")
 	fs.StringVar(&s.ClusterSigningKeyFile, "cluster-signing-key-file", s.ClusterSigningKeyFile, "Filename containing a PEM-encoded RSA or ECDSA private key used to sign cluster-scoped certificates")
@@ -202,34 +124,19 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabled
 	var dummy string
 	fs.MarkDeprecated("insecure-experimental-approve-all-kubelet-csrs-for-group", "This flag does nothing.")
 	fs.StringVar(&dummy, "insecure-experimental-approve-all-kubelet-csrs-for-group", "", "This flag does nothing.")
-	fs.BoolVar(&s.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/")
-	fs.BoolVar(&s.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled")
-	fs.StringVar(&s.ClusterName, "cluster-name", s.ClusterName, "The instance prefix for the cluster")
-	fs.StringVar(&s.ClusterCIDR, "cluster-cidr", s.ClusterCIDR, "CIDR Range for Pods in cluster. Requires --allocate-node-cidrs to be true")
 	fs.StringVar(&s.ServiceCIDR, "service-cluster-ip-range", s.ServiceCIDR, "CIDR Range for Services in cluster. Requires --allocate-node-cidrs to be true")
 	fs.Int32Var(&s.NodeCIDRMaskSize, "node-cidr-mask-size", s.NodeCIDRMaskSize, "Mask size for node cidr in cluster.")
-	fs.BoolVar(&s.AllocateNodeCIDRs, "allocate-node-cidrs", false,
-		"Should CIDRs for Pods be allocated and set on the cloud provider.")
-	fs.StringVar(&s.CIDRAllocatorType, "cidr-allocator-type", "RangeAllocator",
-		"Type of CIDR allocator to use")
-	fs.BoolVar(&s.ConfigureCloudRoutes, "configure-cloud-routes", true, "Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider.")
-	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
-	fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
 	fs.StringVar(&s.RootCAFile, "root-ca-file", s.RootCAFile, "If set, this root certificate authority will be included in service account's token secret. This must be a valid PEM-encoded CA bundle.")
-	fs.StringVar(&s.ContentType, "kube-api-content-type", s.ContentType, "Content type of requests sent to apiserver.")
-	fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver")
-	fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver")
-	fs.DurationVar(&s.ControllerStartInterval.Duration, "controller-start-interval", s.ControllerStartInterval.Duration, "Interval between starting controller managers.")
 	fs.BoolVar(&s.EnableGarbageCollector, "enable-garbage-collector", s.EnableGarbageCollector, "Enables the generic garbage collector. MUST be synced with the corresponding flag of the kube-apiserver.")
 	fs.Int32Var(&s.ConcurrentGCSyncs, "concurrent-gc-syncs", s.ConcurrentGCSyncs, "The number of garbage collector workers that are allowed to sync concurrently.")
-	fs.Float32Var(&s.NodeEvictionRate, "node-eviction-rate", 0.1, "Number of nodes per second on which pods are deleted in case of node failure when a zone is healthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters.")
-	fs.Float32Var(&s.SecondaryNodeEvictionRate, "secondary-node-eviction-rate", 0.01, "Number of nodes per second on which pods are deleted in case of node failure when a zone is unhealthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters. This value is implicitly overridden to 0 if the cluster size is smaller than --large-cluster-size-threshold.")
 	fs.Int32Var(&s.LargeClusterSizeThreshold, "large-cluster-size-threshold", 50, "Number of nodes from which NodeController treats the cluster as large for the eviction logic purposes. --secondary-node-eviction-rate is implicitly overridden to 0 for clusters this size or smaller.")
 	fs.Float32Var(&s.UnhealthyZoneThreshold, "unhealthy-zone-threshold", 0.55, "Fraction of Nodes in a zone which needs to be not Ready (minimum 3) for zone to be treated as unhealthy. ")
 	fs.BoolVar(&s.DisableAttachDetachReconcilerSync, "disable-attach-detach-reconcile-sync", false, "Disable volume attach detach reconciler sync. Disabling this may cause volumes to be mismatched with pods. Use wisely.")
 	fs.DurationVar(&s.ReconcilerSyncLoopPeriod.Duration, "attach-detach-reconcile-sync-period", s.ReconcilerSyncLoopPeriod.Duration, "The reconciler sync wait time between volume attach detach. This duration must be larger than one second, and increasing this value from the default may allow for volumes to be mismatched with pods.")
 	fs.BoolVar(&s.EnableTaintManager, "enable-taint-manager", s.EnableTaintManager, "WARNING: Beta feature. If set to true enables NoExecute Taints and will evict all not-tolerating Pod running on Nodes tainted with this kind of Taints.")
 	fs.BoolVar(&s.HorizontalPodAutoscalerUseRESTClients, "horizontal-pod-autoscaler-use-rest-clients", s.HorizontalPodAutoscalerUseRESTClients, "WARNING: alpha feature.  If set to true, causes the horizontal pod autoscaler controller to use REST clients through the kube-aggregator, instead of using the legacy metrics client through the API server proxy.  This is required for custom metrics support in the horizontal pod autoscaler.")
+	fs.Float32Var(&s.NodeEvictionRate, "node-eviction-rate", 0.1, "Number of nodes per second on which pods are deleted in case of node failure when a zone is healthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters.")
+	fs.Float32Var(&s.SecondaryNodeEvictionRate, "secondary-node-eviction-rate", 0.01, "Number of nodes per second on which pods are deleted in case of node failure when a zone is unhealthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters. This value is implicitly overridden to 0 if the cluster size is smaller than --large-cluster-size-threshold.")
 
 	leaderelectionconfig.BindFlags(&s.LeaderElection, fs)
 

cmd/kube-controller-manager/app/options/options_test.go

@@ -26,6 +26,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/diff"
+	cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 )
 
@@ -110,102 +111,104 @@ func TestAddFlags(t *testing.T) {
 	sort.Sort(sortedGCIgnoredResources(s.GCIgnoredResources))
 
 	expected := &CMServer{
-		KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
+		ControllerManagerServer: cmoptions.ControllerManagerServer{
-			Port:                                            10000,
+			KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
-			Address:                                         "192.168.4.10",
+				Port:                                            10000,
-			AllocateNodeCIDRs:                               true,
+				Address:                                         "192.168.4.10",
-			CloudConfigFile:                                 "/cloud-config",
+				AllocateNodeCIDRs:                               true,
-			CloudProvider:                                   "gce",
+				CloudConfigFile:                                 "/cloud-config",
-			ClusterCIDR:                                     "1.2.3.4/24",
+				CloudProvider:                                   "gce",
-			ClusterName:                                     "k8s",
+				ClusterCIDR:                                     "1.2.3.4/24",
-			ConcurrentDeploymentSyncs:                       10,
+				ClusterName:                                     "k8s",
-			ConcurrentEndpointSyncs:                         10,
+				ConcurrentDeploymentSyncs:                       10,
-			ConcurrentGCSyncs:                               30,
+				ConcurrentEndpointSyncs:                         10,
-			ConcurrentNamespaceSyncs:                        20,
+				ConcurrentGCSyncs:                               30,
-			ConcurrentRSSyncs:                               10,
+				ConcurrentNamespaceSyncs:                        20,
-			ConcurrentResourceQuotaSyncs:                    10,
+				ConcurrentRSSyncs:                               10,
-			ConcurrentServiceSyncs:                          2,
+				ConcurrentResourceQuotaSyncs:                    10,
-			ConcurrentSATokenSyncs:                          10,
+				ConcurrentServiceSyncs:                          2,
-			ConcurrentRCSyncs:                               10,
+				ConcurrentSATokenSyncs:                          10,
-			ConfigureCloudRoutes:                            false,
+				ConcurrentRCSyncs:                               10,
-			EnableContentionProfiling:                       true,
+				ConfigureCloudRoutes:                            false,
-			ControllerStartInterval:                         metav1.Duration{Duration: 2 * time.Minute},
+				EnableContentionProfiling:                       true,
-			ConcurrentDaemonSetSyncs:                        2,
+				ControllerStartInterval:                         metav1.Duration{Duration: 2 * time.Minute},
-			ConcurrentJobSyncs:                              5,
+				ConcurrentDaemonSetSyncs:                        2,
-			DeletingPodsQps:                                 0.1,
+				ConcurrentJobSyncs:                              5,
-			EnableProfiling:                                 false,
+				DeletingPodsQps:                                 0.1,
-			CIDRAllocatorType:                               "CloudAllocator",
+				EnableProfiling:                                 false,
-			NodeCIDRMaskSize:                                48,
+				CIDRAllocatorType:                               "CloudAllocator",
-			ServiceSyncPeriod:                               metav1.Duration{Duration: 2 * time.Minute},
+				NodeCIDRMaskSize:                                48,
-			ResourceQuotaSyncPeriod:                         metav1.Duration{Duration: 10 * time.Minute},
+				ServiceSyncPeriod:                               metav1.Duration{Duration: 2 * time.Minute},
-			NamespaceSyncPeriod:                             metav1.Duration{Duration: 10 * time.Minute},
+				ResourceQuotaSyncPeriod:                         metav1.Duration{Duration: 10 * time.Minute},
-			PVClaimBinderSyncPeriod:                         metav1.Duration{Duration: 30 * time.Second},
+				NamespaceSyncPeriod:                             metav1.Duration{Duration: 10 * time.Minute},
-			HorizontalPodAutoscalerSyncPeriod:               metav1.Duration{Duration: 45 * time.Second},
+				PVClaimBinderSyncPeriod:                         metav1.Duration{Duration: 30 * time.Second},
-			DeploymentControllerSyncPeriod:                  metav1.Duration{Duration: 45 * time.Second},
+				HorizontalPodAutoscalerSyncPeriod:               metav1.Duration{Duration: 45 * time.Second},
-			MinResyncPeriod:                                 metav1.Duration{Duration: 8 * time.Hour},
+				DeploymentControllerSyncPeriod:                  metav1.Duration{Duration: 45 * time.Second},
-			RegisterRetryCount:                              10,
+				MinResyncPeriod:                                 metav1.Duration{Duration: 8 * time.Hour},
-			RouteReconciliationPeriod:                       metav1.Duration{Duration: 30 * time.Second},
+				RegisterRetryCount:                              10,
-			PodEvictionTimeout:                              metav1.Duration{Duration: 2 * time.Minute},
+				RouteReconciliationPeriod:                       metav1.Duration{Duration: 30 * time.Second},
-			NodeMonitorGracePeriod:                          metav1.Duration{Duration: 30 * time.Second},
+				PodEvictionTimeout:                              metav1.Duration{Duration: 2 * time.Minute},
-			NodeStartupGracePeriod:                          metav1.Duration{Duration: 30 * time.Second},
+				NodeMonitorGracePeriod:                          metav1.Duration{Duration: 30 * time.Second},
-			NodeMonitorPeriod:                               metav1.Duration{Duration: 10 * time.Second},
+				NodeStartupGracePeriod:                          metav1.Duration{Duration: 30 * time.Second},
-			HorizontalPodAutoscalerUpscaleForbiddenWindow:   metav1.Duration{Duration: 1 * time.Minute},
+				NodeMonitorPeriod:                               metav1.Duration{Duration: 10 * time.Second},
-			HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute},
+				HorizontalPodAutoscalerUpscaleForbiddenWindow:   metav1.Duration{Duration: 1 * time.Minute},
-			HorizontalPodAutoscalerTolerance:                0.1,
+				HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute},
-			TerminatedPodGCThreshold:                        12000,
+				HorizontalPodAutoscalerTolerance:                0.1,
-			VolumeConfiguration: componentconfig.VolumeConfiguration{
+				TerminatedPodGCThreshold:                        12000,
-				EnableDynamicProvisioning:  false,
+				VolumeConfiguration: componentconfig.VolumeConfiguration{
-				EnableHostPathProvisioning: true,
+					EnableDynamicProvisioning:  false,
-				FlexVolumePluginDir:        "/flex-volume-plugin",
+					EnableHostPathProvisioning: true,
-				PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{
+					FlexVolumePluginDir:        "/flex-volume-plugin",
-					MaximumRetry:             3,
+					PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{
-					MinimumTimeoutNFS:        200,
+						MaximumRetry:             3,
-					IncrementTimeoutNFS:      45,
+						MinimumTimeoutNFS:        200,
-					MinimumTimeoutHostPath:   45,
+						IncrementTimeoutNFS:      45,
-					IncrementTimeoutHostPath: 45,
+						MinimumTimeoutHostPath:   45,
+						IncrementTimeoutHostPath: 45,
+					},
 				},
+				ContentType:  "application/json",
+				KubeAPIQPS:   50.0,
+				KubeAPIBurst: 100,
+				LeaderElection: componentconfig.LeaderElectionConfiguration{
+					ResourceLock:  "configmap",
+					LeaderElect:   false,
+					LeaseDuration: metav1.Duration{Duration: 30 * time.Second},
+					RenewDeadline: metav1.Duration{Duration: 15 * time.Second},
+					RetryPeriod:   metav1.Duration{Duration: 5 * time.Second},
+				},
+				ClusterSigningCertFile: "/cluster-signing-cert",
+				ClusterSigningKeyFile:  "/cluster-signing-key",
+				ServiceAccountKeyFile:  "/service-account-private-key",
+				ClusterSigningDuration: metav1.Duration{Duration: 10 * time.Hour},
+				EnableGarbageCollector: false,
+				GCIgnoredResources: []componentconfig.GroupResource{
+					{Group: "extensions", Resource: "replicationcontrollers"},
+					{Group: "", Resource: "bindings"},
+					{Group: "", Resource: "componentstatuses"},
+					{Group: "", Resource: "events"},
+					{Group: "authentication.k8s.io", Resource: "tokenreviews"},
+					{Group: "authorization.k8s.io", Resource: "subjectaccessreviews"},
+					{Group: "authorization.k8s.io", Resource: "selfsubjectaccessreviews"},
+					{Group: "authorization.k8s.io", Resource: "localsubjectaccessreviews"},
+					{Group: "authorization.k8s.io", Resource: "selfsubjectrulesreviews"},
+					{Group: "apiregistration.k8s.io", Resource: "apiservices"},
+					{Group: "apiextensions.k8s.io", Resource: "customresourcedefinitions"},
+				},
+				NodeEvictionRate:                      0.2,
+				SecondaryNodeEvictionRate:             0.05,
+				LargeClusterSizeThreshold:             100,
+				UnhealthyZoneThreshold:                0.6,
+				DisableAttachDetachReconcilerSync:     true,
+				ReconcilerSyncLoopPeriod:              metav1.Duration{Duration: 30 * time.Second},
+				Controllers:                           []string{"foo", "bar"},
+				EnableTaintManager:                    false,
+				HorizontalPodAutoscalerUseRESTClients: true,
+				UseServiceAccountCredentials:          true,
 			},
-			ContentType:  "application/json",
+			Kubeconfig: "/kubeconfig",
-			KubeAPIQPS:   50.0,
+			Master:     "192.168.4.20",
-			KubeAPIBurst: 100,
-			LeaderElection: componentconfig.LeaderElectionConfiguration{
-				ResourceLock:  "configmap",
-				LeaderElect:   false,
-				LeaseDuration: metav1.Duration{Duration: 30 * time.Second},
-				RenewDeadline: metav1.Duration{Duration: 15 * time.Second},
-				RetryPeriod:   metav1.Duration{Duration: 5 * time.Second},
-			},
-			ClusterSigningCertFile: "/cluster-signing-cert",
-			ClusterSigningKeyFile:  "/cluster-signing-key",
-			ServiceAccountKeyFile:  "/service-account-private-key",
-			ClusterSigningDuration: metav1.Duration{Duration: 10 * time.Hour},
-			EnableGarbageCollector: false,
-			GCIgnoredResources: []componentconfig.GroupResource{
-				{Group: "extensions", Resource: "replicationcontrollers"},
-				{Group: "", Resource: "bindings"},
-				{Group: "", Resource: "componentstatuses"},
-				{Group: "", Resource: "events"},
-				{Group: "authentication.k8s.io", Resource: "tokenreviews"},
-				{Group: "authorization.k8s.io", Resource: "subjectaccessreviews"},
-				{Group: "authorization.k8s.io", Resource: "selfsubjectaccessreviews"},
-				{Group: "authorization.k8s.io", Resource: "localsubjectaccessreviews"},
-				{Group: "authorization.k8s.io", Resource: "selfsubjectrulesreviews"},
-				{Group: "apiregistration.k8s.io", Resource: "apiservices"},
-				{Group: "apiextensions.k8s.io", Resource: "customresourcedefinitions"},
-			},
-			NodeEvictionRate:                      0.2,
-			SecondaryNodeEvictionRate:             0.05,
-			LargeClusterSizeThreshold:             100,
-			UnhealthyZoneThreshold:                0.6,
-			DisableAttachDetachReconcilerSync:     true,
-			ReconcilerSyncLoopPeriod:              metav1.Duration{Duration: 30 * time.Second},
-			Controllers:                           []string{"foo", "bar"},
-			EnableTaintManager:                    false,
-			HorizontalPodAutoscalerUseRESTClients: true,
-			UseServiceAccountCredentials:          true,
 		},
-		Kubeconfig: "/kubeconfig",
-		Master:     "192.168.4.20",
 	}
 
 	// Sort GCIgnoredResources because it's built from a map, which means the

pkg/apis/componentconfig/types.go

@@ -181,6 +181,9 @@ type KubeControllerManagerConfiguration struct {
 	CloudProvider string
 	// cloudConfigFile is the path to the cloud provider configuration file.
 	CloudConfigFile string
+	// externalCloudVolumePlugin specifies the plugin to use when cloudProvider is "external".
+	// It is currently used by the in repo cloud providers to handle node and volume control in the KCM.
+	ExternalCloudVolumePlugin string
 	// run with untagged cloud instances
 	AllowUntaggedCloud bool
 	// concurrentEndpointSyncs is the number of endpoint syncing operations

pkg/controller/node/node_controller.go

@@ -259,7 +259,7 @@ func NewNodeController(
 		}
 		mask := clusterCIDR.Mask
 		if maskSize, _ := mask.Size(); maskSize > nodeCIDRMaskSize {
-			glog.Fatal("Controller: Invalid clusterCIDR, mask size of clusterCIDR must be less than nodeCIDRMaskSize.")
+			glog.Fatalf("Controller: Invalid clusterCIDR, mask size of clusterCIDR(%d) must be less than nodeCIDRMaskSize(%d).", maskSize, nodeCIDRMaskSize)
 		}
 	}