diff --git a/cmd/BUILD b/cmd/BUILD index 75dcc63d83..e8ed83675a 100644 --- a/cmd/BUILD +++ b/cmd/BUILD @@ -13,6 +13,7 @@ filegroup( ":package-srcs", "//cmd/clicheck:all-srcs", "//cmd/cloud-controller-manager:all-srcs", + "//cmd/controller-manager/app/options:all-srcs", "//cmd/gendocs:all-srcs", "//cmd/genkubedocs:all-srcs", "//cmd/genman:all-srcs", diff --git a/cmd/cloud-controller-manager/app/controllermanager.go b/cmd/cloud-controller-manager/app/controllermanager.go index 2871e9f727..fc698b651d 100644 --- a/cmd/cloud-controller-manager/app/controllermanager.go +++ b/cmd/cloud-controller-manager/app/controllermanager.go @@ -147,7 +147,7 @@ func Run(s *options.CloudControllerManagerServer) error { clientBuilder = rootClientBuilder } - if err := StartControllers(s, kubeconfig, clientBuilder, stop, recorder, cloud); err != nil { + if err := StartControllers(s, kubeconfig, rootClientBuilder, clientBuilder, stop, recorder, cloud); err != nil { glog.Fatalf("error running controllers: %v", err) } } @@ -193,7 +193,7 @@ func Run(s *options.CloudControllerManagerServer) error { } // StartControllers starts the cloud specific controller loops. -func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restclient.Config, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}, recorder record.EventRecorder, cloud cloudprovider.Interface) error { +func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restclient.Config, rootClientBuilder, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}, recorder record.EventRecorder, cloud cloudprovider.Interface) error { // Function to build the kube client object client := func(serviceAccountName string) clientset.Interface { return clientBuilder.ClientOrDie(serviceAccountName) @@ -204,7 +204,7 @@ func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restc cloud.Initialize(clientBuilder) } - versionedClient := client("shared-informers") + versionedClient := rootClientBuilder.ClientOrDie("shared-informers") sharedInformers := informers.NewSharedInformerFactory(versionedClient, resyncPeriod(s)()) // Start the CloudNodeController diff --git a/cmd/cloud-controller-manager/app/options/BUILD b/cmd/cloud-controller-manager/app/options/BUILD index 62c8e0d01d..b02da213f5 100644 --- a/cmd/cloud-controller-manager/app/options/BUILD +++ b/cmd/cloud-controller-manager/app/options/BUILD @@ -11,7 +11,7 @@ go_library( srcs = ["options.go"], importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/options", deps = [ - "//pkg/apis/componentconfig:go_default_library", + "//cmd/controller-manager/app/options:go_default_library", "//pkg/client/leaderelectionconfig:go_default_library", "//pkg/features:go_default_library", "//pkg/master/ports:go_default_library", @@ -40,6 +40,7 @@ go_test( importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/options", library = ":go_default_library", deps = [ + "//cmd/controller-manager/app/options:go_default_library", "//pkg/apis/componentconfig:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/cmd/cloud-controller-manager/app/options/options.go b/cmd/cloud-controller-manager/app/options/options.go index 6120232659..5d606b16c1 100644 --- a/cmd/cloud-controller-manager/app/options/options.go +++ b/cmd/cloud-controller-manager/app/options/options.go @@ -21,7 +21,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/kubernetes/pkg/apis/componentconfig" + cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options" "k8s.io/kubernetes/pkg/client/leaderelectionconfig" "k8s.io/kubernetes/pkg/master/ports" @@ -33,10 +33,7 @@ import ( // CloudControllerManagerServer is the main context object for the controller manager. type CloudControllerManagerServer struct { - componentconfig.KubeControllerManagerConfiguration - - Master string - Kubeconfig string + cmoptions.ControllerManagerServer // NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status NodeStatusUpdateFrequency metav1.Duration @@ -45,22 +42,10 @@ type CloudControllerManagerServer struct { // NewCloudControllerManagerServer creates a new ExternalCMServer with a default config. func NewCloudControllerManagerServer() *CloudControllerManagerServer { s := CloudControllerManagerServer{ - // Part of these default values also present in 'cmd/kube-controller-manager/app/options/options.go'. - // Please keep them in sync when doing update. - KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{ - Port: ports.CloudControllerManagerPort, - Address: "0.0.0.0", - ConcurrentServiceSyncs: 1, - MinResyncPeriod: metav1.Duration{Duration: 12 * time.Hour}, - NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second}, - ClusterName: "kubernetes", - ConfigureCloudRoutes: true, - ContentType: "application/vnd.kubernetes.protobuf", - KubeAPIQPS: 20.0, - KubeAPIBurst: 30, - LeaderElection: leaderelectionconfig.DefaultLeaderElectionConfiguration(), - ControllerStartInterval: metav1.Duration{Duration: 0 * time.Second}, - RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second}, + // The common/default are kept in 'cmd/kube-controller-manager/app/options/util.go'. + // Please make common changes there and put anything cloud specific here. + ControllerManagerServer: cmoptions.ControllerManagerServer{ + KubeControllerManagerConfiguration: cmoptions.GetDefaultControllerOptions(ports.CloudControllerManagerPort), }, NodeStatusUpdateFrequency: metav1.Duration{Duration: 5 * time.Minute}, } @@ -70,34 +55,14 @@ func NewCloudControllerManagerServer() *CloudControllerManagerServer { // AddFlags adds flags for a specific ExternalCMServer to the specified FlagSet func (s *CloudControllerManagerServer) AddFlags(fs *pflag.FlagSet) { - fs.Int32Var(&s.Port, "port", s.Port, "The port that the cloud-controller-manager's http service runs on.") - fs.Var(componentconfig.IPVar{Val: &s.Address}, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces).") + cmoptions.AddDefaultControllerFlags(&s.ControllerManagerServer, fs) fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider of cloud services. Cannot be empty.") - fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.") - fs.BoolVar(&s.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances. This is a legacy mode of operation and a cluster-id will be required in the future.") - fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release. A cluster-id will be required on cloud instances.") - fs.DurationVar(&s.MinResyncPeriod.Duration, "min-resync-period", s.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod.") - fs.DurationVar(&s.NodeMonitorPeriod.Duration, "node-monitor-period", s.NodeMonitorPeriod.Duration, - "The period for syncing NodeStatus in NodeController.") fs.DurationVar(&s.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", s.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.") // TODO: remove --service-account-private-key-file 6 months after 1.8 is released (~1.10) fs.StringVar(&s.ServiceAccountKeyFile, "service-account-private-key-file", s.ServiceAccountKeyFile, "Filename containing a PEM-encoded private RSA or ECDSA key used to sign service account tokens.") fs.MarkDeprecated("service-account-private-key-file", "This flag is currently no-op and will be deleted.") - fs.BoolVar(&s.UseServiceAccountCredentials, "use-service-account-credentials", s.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.") - fs.DurationVar(&s.RouteReconciliationPeriod.Duration, "route-reconciliation-period", s.RouteReconciliationPeriod.Duration, "The period for reconciling routes created for Nodes by cloud provider.") - fs.BoolVar(&s.ConfigureCloudRoutes, "configure-cloud-routes", true, "Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider.") - fs.BoolVar(&s.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/.") - fs.BoolVar(&s.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled.") - fs.StringVar(&s.ClusterCIDR, "cluster-cidr", s.ClusterCIDR, "CIDR Range for Pods in cluster.") - fs.StringVar(&s.ClusterName, "cluster-name", s.ClusterName, "The instance prefix for the cluster.") - fs.BoolVar(&s.AllocateNodeCIDRs, "allocate-node-cidrs", false, "Should CIDRs for Pods be allocated and set on the cloud provider.") - fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") - fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.") - fs.StringVar(&s.ContentType, "kube-api-content-type", s.ContentType, "Content type of requests sent to apiserver.") - fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver.") - fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver.") - fs.DurationVar(&s.ControllerStartInterval.Duration, "controller-start-interval", s.ControllerStartInterval.Duration, "Interval between starting controller managers.") fs.Int32Var(&s.ConcurrentServiceSyncs, "concurrent-service-syncs", s.ConcurrentServiceSyncs, "The number of services that are allowed to sync concurrently. Larger number = more responsive service management, but more CPU (and network) load") + leaderelectionconfig.BindFlags(&s.LeaderElection, fs) utilfeature.DefaultFeatureGate.AddFlag(fs) diff --git a/cmd/cloud-controller-manager/app/options/options_test.go b/cmd/cloud-controller-manager/app/options/options_test.go index ece3bf09d2..7538169b42 100644 --- a/cmd/cloud-controller-manager/app/options/options_test.go +++ b/cmd/cloud-controller-manager/app/options/options_test.go @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" + cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options" "k8s.io/kubernetes/pkg/apis/componentconfig" ) @@ -65,34 +66,82 @@ func TestAddFlags(t *testing.T) { f.Parse(args) expected := &CloudControllerManagerServer{ - KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{ - CloudProvider: "gce", - CloudConfigFile: "/cloud-config", - Port: 10000, - Address: "192.168.4.10", - ConcurrentServiceSyncs: 1, - MinResyncPeriod: metav1.Duration{Duration: 100 * time.Minute}, - NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second}, - ClusterName: "k8s", - ConfigureCloudRoutes: false, - AllocateNodeCIDRs: true, - ContentType: "application/vnd.kubernetes.protobuf", - EnableContentionProfiling: true, - KubeAPIQPS: 50.0, - KubeAPIBurst: 100, - LeaderElection: componentconfig.LeaderElectionConfiguration{ - ResourceLock: "configmap", - LeaderElect: false, - LeaseDuration: metav1.Duration{Duration: 30 * time.Second}, - RenewDeadline: metav1.Duration{Duration: 15 * time.Second}, - RetryPeriod: metav1.Duration{Duration: 5 * time.Second}, + ControllerManagerServer: cmoptions.ControllerManagerServer{ + KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{ + CloudProvider: "gce", + CloudConfigFile: "/cloud-config", + Port: 10000, + Address: "192.168.4.10", + ConcurrentEndpointSyncs: 5, + ConcurrentRSSyncs: 5, + ConcurrentResourceQuotaSyncs: 5, + ConcurrentDeploymentSyncs: 5, + ConcurrentDaemonSetSyncs: 2, + ConcurrentJobSyncs: 5, + ConcurrentNamespaceSyncs: 10, + ConcurrentSATokenSyncs: 5, + ConcurrentServiceSyncs: 1, + ConcurrentGCSyncs: 20, + ConcurrentRCSyncs: 5, + MinResyncPeriod: metav1.Duration{Duration: 100 * time.Minute}, + NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second}, + ServiceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, + ResourceQuotaSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, + NamespaceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, + PVClaimBinderSyncPeriod: metav1.Duration{Duration: 15 * time.Second}, + HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, + HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 3 * time.Minute}, + HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute}, + HorizontalPodAutoscalerTolerance: 0.1, + DeploymentControllerSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, + PodEvictionTimeout: metav1.Duration{Duration: 5 * time.Minute}, + NodeMonitorGracePeriod: metav1.Duration{Duration: 40 * time.Second}, + NodeStartupGracePeriod: metav1.Duration{Duration: 1 * time.Minute}, + ClusterSigningDuration: metav1.Duration{Duration: 8760 * time.Hour}, + ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 1 * time.Minute}, + TerminatedPodGCThreshold: 12500, + RegisterRetryCount: 10, + ClusterName: "k8s", + ConfigureCloudRoutes: false, + AllocateNodeCIDRs: true, + EnableGarbageCollector: true, + EnableTaintManager: true, + HorizontalPodAutoscalerUseRESTClients: true, + VolumeConfiguration: componentconfig.VolumeConfiguration{ + EnableDynamicProvisioning: true, + EnableHostPathProvisioning: false, + FlexVolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/", + PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{ + MaximumRetry: 3, + MinimumTimeoutNFS: 300, + IncrementTimeoutNFS: 30, + MinimumTimeoutHostPath: 60, + IncrementTimeoutHostPath: 30, + }, + }, + ContentType: "application/vnd.kubernetes.protobuf", + ClusterSigningCertFile: "/etc/kubernetes/ca/ca.pem", + ClusterSigningKeyFile: "/etc/kubernetes/ca/ca.key", + EnableContentionProfiling: true, + KubeAPIQPS: 50.0, + KubeAPIBurst: 100, + LeaderElection: componentconfig.LeaderElectionConfiguration{ + ResourceLock: "configmap", + LeaderElect: false, + LeaseDuration: metav1.Duration{Duration: 30 * time.Second}, + RenewDeadline: metav1.Duration{Duration: 15 * time.Second}, + RetryPeriod: metav1.Duration{Duration: 5 * time.Second}, + }, + ControllerStartInterval: metav1.Duration{Duration: 2 * time.Minute}, + RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second}, + ClusterCIDR: "1.2.3.4/24", + NodeCIDRMaskSize: 24, + CIDRAllocatorType: "RangeAllocator", + Controllers: []string{"*"}, }, - ControllerStartInterval: metav1.Duration{Duration: 2 * time.Minute}, - RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second}, - ClusterCIDR: "1.2.3.4/24", + Kubeconfig: "/kubeconfig", + Master: "192.168.4.20", }, - Kubeconfig: "/kubeconfig", - Master: "192.168.4.20", NodeStatusUpdateFrequency: metav1.Duration{Duration: 10 * time.Minute}, } if !reflect.DeepEqual(expected, s) { diff --git a/cmd/controller-manager/app/options/BUILD b/cmd/controller-manager/app/options/BUILD new file mode 100644 index 0000000000..2ad13dea87 --- /dev/null +++ b/cmd/controller-manager/app/options/BUILD @@ -0,0 +1,29 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["utils.go"], + importpath = "k8s.io/kubernetes/cmd/controller-manager/app/options", + visibility = ["//visibility:public"], + deps = [ + "//pkg/apis/componentconfig:go_default_library", + "//pkg/client/leaderelectionconfig:go_default_library", + "//vendor/github.com/cloudflare/cfssl/helpers:go_default_library", + "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/cmd/controller-manager/app/options/utils.go b/cmd/controller-manager/app/options/utils.go new file mode 100644 index 0000000000..903158f2eb --- /dev/null +++ b/cmd/controller-manager/app/options/utils.go @@ -0,0 +1,140 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "github.com/cloudflare/cfssl/helpers" + "time" + + "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/apis/componentconfig" + "k8s.io/kubernetes/pkg/client/leaderelectionconfig" +) + +// ControllerManagerServer is the common structure for a controller manager. It works with GetDefaultControllerOptions +// and AddDefaultControllerFlags to create the common components of kube-controller-manager and cloud-controller-manager. +type ControllerManagerServer struct { + componentconfig.KubeControllerManagerConfiguration + + Master string + Kubeconfig string +} + +const ( + // These defaults are deprecated and exported so that we can warn if + // they are being used. + + // DefaultClusterSigningCertFile is deprecated. Do not use. + DefaultClusterSigningCertFile = "/etc/kubernetes/ca/ca.pem" + // DefaultClusterSigningKeyFile is deprecated. Do not use. + DefaultClusterSigningKeyFile = "/etc/kubernetes/ca/ca.key" +) + +// GetDefaultControllerOptions returns common/default configuration values for both +// the kube-controller-manager and the cloud-contoller-manager. Any common changes should +// be made here. Any individual changes should be made in that controller. +func GetDefaultControllerOptions(port int32) componentconfig.KubeControllerManagerConfiguration { + return componentconfig.KubeControllerManagerConfiguration{ + Controllers: []string{"*"}, + Port: port, + Address: "0.0.0.0", + ConcurrentEndpointSyncs: 5, + ConcurrentServiceSyncs: 1, + ConcurrentRCSyncs: 5, + ConcurrentRSSyncs: 5, + ConcurrentDaemonSetSyncs: 2, + ConcurrentJobSyncs: 5, + ConcurrentResourceQuotaSyncs: 5, + ConcurrentDeploymentSyncs: 5, + ConcurrentNamespaceSyncs: 10, + ConcurrentSATokenSyncs: 5, + ServiceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, + RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second}, + ResourceQuotaSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, + NamespaceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, + PVClaimBinderSyncPeriod: metav1.Duration{Duration: 15 * time.Second}, + HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, + HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 3 * time.Minute}, + HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute}, + HorizontalPodAutoscalerTolerance: 0.1, + DeploymentControllerSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, + MinResyncPeriod: metav1.Duration{Duration: 12 * time.Hour}, + RegisterRetryCount: 10, + PodEvictionTimeout: metav1.Duration{Duration: 5 * time.Minute}, + NodeMonitorGracePeriod: metav1.Duration{Duration: 40 * time.Second}, + NodeStartupGracePeriod: metav1.Duration{Duration: 60 * time.Second}, + NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second}, + ClusterName: "kubernetes", + NodeCIDRMaskSize: 24, + ConfigureCloudRoutes: true, + TerminatedPodGCThreshold: 12500, + VolumeConfiguration: componentconfig.VolumeConfiguration{ + EnableHostPathProvisioning: false, + EnableDynamicProvisioning: true, + PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{ + MaximumRetry: 3, + MinimumTimeoutNFS: 300, + IncrementTimeoutNFS: 30, + MinimumTimeoutHostPath: 60, + IncrementTimeoutHostPath: 30, + }, + FlexVolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/", + }, + ContentType: "application/vnd.kubernetes.protobuf", + KubeAPIQPS: 20.0, + KubeAPIBurst: 30, + LeaderElection: leaderelectionconfig.DefaultLeaderElectionConfiguration(), + ControllerStartInterval: metav1.Duration{Duration: 0 * time.Second}, + EnableGarbageCollector: true, + ConcurrentGCSyncs: 20, + ClusterSigningCertFile: DefaultClusterSigningCertFile, + ClusterSigningKeyFile: DefaultClusterSigningKeyFile, + ClusterSigningDuration: metav1.Duration{Duration: helpers.OneYear}, + ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 60 * time.Second}, + EnableTaintManager: true, + HorizontalPodAutoscalerUseRESTClients: true, + } +} + +// AddDefaultControllerFlags adds common/default flags for both the kube and cloud Controller Manager Server to the +// specified FlagSet. Any common changes should be made here. Any individual changes should be made in that controller. +func AddDefaultControllerFlags(s *ControllerManagerServer, fs *pflag.FlagSet) { + fs.Int32Var(&s.Port, "port", s.Port, "The port that the controller-manager's http service runs on.") + fs.Var(componentconfig.IPVar{Val: &s.Address}, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces).") + fs.BoolVar(&s.UseServiceAccountCredentials, "use-service-account-credentials", s.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.") + fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.") + fs.BoolVar(&s.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances. This is a legacy mode of operation and a cluster-id will be required in the future.") + fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release. A cluster-id will be required on cloud instances.") + fs.DurationVar(&s.RouteReconciliationPeriod.Duration, "route-reconciliation-period", s.RouteReconciliationPeriod.Duration, "The period for reconciling routes created for Nodes by cloud provider.") + fs.DurationVar(&s.MinResyncPeriod.Duration, "min-resync-period", s.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod.") + fs.DurationVar(&s.NodeMonitorPeriod.Duration, "node-monitor-period", s.NodeMonitorPeriod.Duration, + "The period for syncing NodeStatus in NodeController.") + fs.BoolVar(&s.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/") + fs.BoolVar(&s.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled.") + fs.StringVar(&s.ClusterName, "cluster-name", s.ClusterName, "The instance prefix for the cluster.") + fs.StringVar(&s.ClusterCIDR, "cluster-cidr", s.ClusterCIDR, "CIDR Range for Pods in cluster. Requires --allocate-node-cidrs to be true") + fs.BoolVar(&s.AllocateNodeCIDRs, "allocate-node-cidrs", false, "Should CIDRs for Pods be allocated and set on the cloud provider.") + fs.StringVar(&s.CIDRAllocatorType, "cidr-allocator-type", "RangeAllocator", "Type of CIDR allocator to use") + fs.BoolVar(&s.ConfigureCloudRoutes, "configure-cloud-routes", true, "Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider.") + fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") + fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.") + fs.StringVar(&s.ContentType, "kube-api-content-type", s.ContentType, "Content type of requests sent to apiserver.") + fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver.") + fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver.") + fs.DurationVar(&s.ControllerStartInterval.Duration, "controller-start-interval", s.ControllerStartInterval.Duration, "Interval between starting controller managers.") +} diff --git a/cmd/kube-controller-manager/app/BUILD b/cmd/kube-controller-manager/app/BUILD index a533b2e29f..2d4ede18b4 100644 --- a/cmd/kube-controller-manager/app/BUILD +++ b/cmd/kube-controller-manager/app/BUILD @@ -24,6 +24,7 @@ go_library( ], importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app", deps = [ + "//cmd/controller-manager/app/options:go_default_library", "//cmd/kube-controller-manager/app/options:go_default_library", "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/apps/install:go_default_library", diff --git a/cmd/kube-controller-manager/app/certificates.go b/cmd/kube-controller-manager/app/certificates.go index 7ef2958217..6c1531cae5 100644 --- a/cmd/kube-controller-manager/app/certificates.go +++ b/cmd/kube-controller-manager/app/certificates.go @@ -27,7 +27,7 @@ import ( "github.com/golang/glog" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" + cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options" "k8s.io/kubernetes/pkg/controller/certificates/approver" "k8s.io/kubernetes/pkg/controller/certificates/cleaner" "k8s.io/kubernetes/pkg/controller/certificates/signer" @@ -55,12 +55,12 @@ func startCSRSigningController(ctx ControllerContext) (bool, error) { _, err := os.Stat(ctx.Options.ClusterSigningCertFile) certFileExists = !os.IsNotExist(err) - certUsesDefault = (ctx.Options.ClusterSigningCertFile == options.DefaultClusterSigningCertFile) + certUsesDefault = (ctx.Options.ClusterSigningCertFile == cmoptions.DefaultClusterSigningCertFile) _, err = os.Stat(ctx.Options.ClusterSigningKeyFile) keyFileExists = !os.IsNotExist(err) - keyUsesDefault = (ctx.Options.ClusterSigningKeyFile == options.DefaultClusterSigningKeyFile) + keyUsesDefault = (ctx.Options.ClusterSigningKeyFile == cmoptions.DefaultClusterSigningKeyFile) switch { case (keyFileExists && keyUsesDefault) || (certFileExists && certUsesDefault): diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index d08e9658a8..8697214e09 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -71,6 +71,13 @@ const ( ControllerStartJitter = 1.0 ) +type ControllerLoopMode int + +const ( + IncludeCloudLoops ControllerLoopMode = iota + ExternalLoops +) + // NewControllerManagerCommand creates a *cobra.Command object with default parameters func NewControllerManagerCommand() *cobra.Command { s := options.NewCMServer() @@ -151,7 +158,7 @@ func Run(s *options.CMServer) error { } saTokenControllerInitFunc := serviceAccountTokenControllerStarter{rootClientBuilder: rootClientBuilder}.startServiceAccountTokenController - if err := StartControllers(ctx, saTokenControllerInitFunc, NewControllerInitializers()); err != nil { + if err := StartControllers(ctx, saTokenControllerInitFunc, NewControllerInitializers(ctx.LoopMode)); err != nil { glog.Fatalf("error starting controllers: %v", err) } @@ -262,6 +269,11 @@ type ControllerContext struct { // It must be initialized and ready to use. Cloud cloudprovider.Interface + // Control for which control loops to be run + // IncludeCloudLoops is for a kube-controller-manager running all loops + // ExternalLoops is for a kube-controller-manager running with a cloud-controller-manager + LoopMode ControllerLoopMode + // Stop is the stop channel Stop <-chan struct{} @@ -305,7 +317,7 @@ func IsControllerEnabled(name string, disabledByDefaultControllers sets.String, type InitFunc func(ctx ControllerContext) (bool, error) func KnownControllers() []string { - ret := sets.StringKeySet(NewControllerInitializers()) + ret := sets.StringKeySet(NewControllerInitializers(IncludeCloudLoops)) // add "special" controllers that aren't initialized normally. These controllers cannot be initialized // using a normal function. The only known special case is the SA token controller which *must* be started @@ -329,7 +341,7 @@ const ( // NewControllerInitializers is a public map of named controller groups (you can start more than one in an init func) // paired to their InitFunc. This allows for structured downstream composition and subdivision. -func NewControllerInitializers() map[string]InitFunc { +func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc { controllers := map[string]InitFunc{} controllers["endpoint"] = startEndpointController controllers["replicationcontroller"] = startReplicationController @@ -352,9 +364,12 @@ func NewControllerInitializers() map[string]InitFunc { controllers["ttl"] = startTTLController controllers["bootstrapsigner"] = startBootstrapSignerController controllers["tokencleaner"] = startTokenCleanerController - controllers["service"] = startServiceController + if loopMode == IncludeCloudLoops { + controllers["service"] = startServiceController + controllers["route"] = startRouteController + // TODO: Move node controller and volume controller into the IncludeCloudLoops only set. + } controllers["node"] = startNodeController - controllers["route"] = startRouteController controllers["persistentvolume-binder"] = startPersistentVolumeBinderController controllers["attachdetach"] = startAttachDetachController controllers["persistentvolume-expander"] = startVolumeExpandController @@ -430,7 +445,17 @@ func CreateControllerContext(s *options.CMServer, rootClientBuilder, clientBuild return ControllerContext{}, err } - cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) + var cloud cloudprovider.Interface + var loopMode ControllerLoopMode + if cloudprovider.IsExternal(s.CloudProvider) { + loopMode = ExternalLoops + if s.ExternalCloudVolumePlugin != "" { + cloud, err = cloudprovider.InitCloudProvider(s.ExternalCloudVolumePlugin, s.CloudConfigFile) + } + } else { + loopMode = IncludeCloudLoops + cloud, err = cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) + } if err != nil { return ControllerContext{}, fmt.Errorf("cloud provider could not be initialized: %v", err) } @@ -453,6 +478,7 @@ func CreateControllerContext(s *options.CMServer, rootClientBuilder, clientBuild Options: *s, AvailableResources: availableResources, Cloud: cloud, + LoopMode: loopMode, Stop: stop, InformersStarted: make(chan struct{}), } diff --git a/cmd/kube-controller-manager/app/options/BUILD b/cmd/kube-controller-manager/app/options/BUILD index 588b64c686..41eeba489f 100644 --- a/cmd/kube-controller-manager/app/options/BUILD +++ b/cmd/kube-controller-manager/app/options/BUILD @@ -11,14 +11,13 @@ go_library( srcs = ["options.go"], importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app/options", deps = [ + "//cmd/controller-manager/app/options:go_default_library", "//pkg/apis/componentconfig:go_default_library", "//pkg/client/leaderelectionconfig:go_default_library", "//pkg/controller/garbagecollector:go_default_library", "//pkg/features:go_default_library", "//pkg/master/ports:go_default_library", - "//vendor/github.com/cloudflare/cfssl/helpers:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", @@ -45,6 +44,7 @@ go_test( library = ":go_default_library", tags = ["automanaged"], deps = [ + "//cmd/controller-manager/app/options:go_default_library", "//pkg/apis/componentconfig:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index d681bfd189..94548812aa 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -21,12 +21,11 @@ package options import ( "fmt" "strings" - "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" + cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options" "k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/client/leaderelectionconfig" "k8s.io/kubernetes/pkg/controller/garbagecollector" @@ -35,26 +34,12 @@ import ( // add the kubernetes feature gates _ "k8s.io/kubernetes/pkg/features" - "github.com/cloudflare/cfssl/helpers" "github.com/spf13/pflag" ) -const ( - // These defaults are deprecated and exported so that we can warn if - // they are being used. - - // DefaultClusterSigningCertFile is deprecated. Do not use. - DefaultClusterSigningCertFile = "/etc/kubernetes/ca/ca.pem" - // DefaultClusterSigningKeyFile is deprecated. Do not use. - DefaultClusterSigningKeyFile = "/etc/kubernetes/ca/ca.key" -) - // CMServer is the main context object for the controller manager. type CMServer struct { - componentconfig.KubeControllerManagerConfiguration - - Master string - Kubeconfig string + cmoptions.ControllerManagerServer } // NewCMServer creates a new CMServer with a default config. @@ -65,91 +50,32 @@ func NewCMServer() *CMServer { } s := CMServer{ - // Part of these default values also present in 'cmd/cloud-controller-manager/app/options/options.go'. - // Please keep them in sync when doing update. - KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{ - Controllers: []string{"*"}, - Port: ports.ControllerManagerPort, - Address: "0.0.0.0", - ConcurrentEndpointSyncs: 5, - ConcurrentServiceSyncs: 1, - ConcurrentRCSyncs: 5, - ConcurrentRSSyncs: 5, - ConcurrentDaemonSetSyncs: 2, - ConcurrentJobSyncs: 5, - ConcurrentResourceQuotaSyncs: 5, - ConcurrentDeploymentSyncs: 5, - ConcurrentNamespaceSyncs: 10, - ConcurrentSATokenSyncs: 5, - ServiceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, - RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second}, - ResourceQuotaSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, - NamespaceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute}, - PVClaimBinderSyncPeriod: metav1.Duration{Duration: 15 * time.Second}, - HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, - HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 3 * time.Minute}, - HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute}, - HorizontalPodAutoscalerTolerance: 0.1, - DeploymentControllerSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, - MinResyncPeriod: metav1.Duration{Duration: 12 * time.Hour}, - RegisterRetryCount: 10, - PodEvictionTimeout: metav1.Duration{Duration: 5 * time.Minute}, - NodeMonitorGracePeriod: metav1.Duration{Duration: 40 * time.Second}, - NodeStartupGracePeriod: metav1.Duration{Duration: 60 * time.Second}, - NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second}, - ClusterName: "kubernetes", - NodeCIDRMaskSize: 24, - ConfigureCloudRoutes: true, - TerminatedPodGCThreshold: 12500, - VolumeConfiguration: componentconfig.VolumeConfiguration{ - EnableHostPathProvisioning: false, - EnableDynamicProvisioning: true, - PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{ - MaximumRetry: 3, - MinimumTimeoutNFS: 300, - IncrementTimeoutNFS: 30, - MinimumTimeoutHostPath: 60, - IncrementTimeoutHostPath: 30, - }, - FlexVolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/", - }, - ContentType: "application/vnd.kubernetes.protobuf", - KubeAPIQPS: 20.0, - KubeAPIBurst: 30, - LeaderElection: leaderelectionconfig.DefaultLeaderElectionConfiguration(), - ControllerStartInterval: metav1.Duration{Duration: 0 * time.Second}, - EnableGarbageCollector: true, - ConcurrentGCSyncs: 20, - GCIgnoredResources: gcIgnoredResources, - ClusterSigningCertFile: DefaultClusterSigningCertFile, - ClusterSigningKeyFile: DefaultClusterSigningKeyFile, - ClusterSigningDuration: metav1.Duration{Duration: helpers.OneYear}, - ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 60 * time.Second}, - EnableTaintManager: true, - HorizontalPodAutoscalerUseRESTClients: true, + // The common/default are kept in 'cmd/kube-controller-manager/app/options/util.go'. + // Please make common changes there but put anything kube-controller specific here. + ControllerManagerServer: cmoptions.ControllerManagerServer{ + KubeControllerManagerConfiguration: cmoptions.GetDefaultControllerOptions(ports.ControllerManagerPort), }, } + s.KubeControllerManagerConfiguration.GCIgnoredResources = gcIgnoredResources s.LeaderElection.LeaderElect = true return &s } // AddFlags adds flags for a specific CMServer to the specified FlagSet func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabledByDefaultControllers []string) { + cmoptions.AddDefaultControllerFlags(&s.ControllerManagerServer, fs) + fs.StringSliceVar(&s.Controllers, "controllers", s.Controllers, fmt.Sprintf(""+ "A list of controllers to enable. '*' enables all on-by-default controllers, 'foo' enables the controller "+ "named 'foo', '-foo' disables the controller named 'foo'.\nAll controllers: %s\nDisabled-by-default controllers: %s", strings.Join(allControllers, ", "), strings.Join(disabledByDefaultControllers, ", "))) - fs.Int32Var(&s.Port, "port", s.Port, "The port that the controller-manager's http service runs on") - fs.Var(componentconfig.IPVar{Val: &s.Address}, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces)") - fs.BoolVar(&s.UseServiceAccountCredentials, "use-service-account-credentials", s.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.") fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider for cloud services. Empty string for no provider.") - fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.") - fs.BoolVar(&s.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances. This is a legacy mode of operation and a cluster-id will be required in the future.") - fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release. A cluster-id will be required on cloud instances") + fs.StringVar(&s.ExternalCloudVolumePlugin, "external-cloud-volume-plugin", s.ExternalCloudVolumePlugin, "The plugin to use when cloud provider is set to external. Can be empty, should only be set when cloud-provider is external. Currently used to allow node and volume controllers to work for in tree cloud providers.") fs.Int32Var(&s.ConcurrentEndpointSyncs, "concurrent-endpoint-syncs", s.ConcurrentEndpointSyncs, "The number of endpoint syncing operations that will be done concurrently. Larger number = faster endpoint updating, but more CPU (and network) load") fs.Int32Var(&s.ConcurrentServiceSyncs, "concurrent-service-syncs", s.ConcurrentServiceSyncs, "The number of services that are allowed to sync concurrently. Larger number = more responsive service management, but more CPU (and network) load") fs.Int32Var(&s.ConcurrentRCSyncs, "concurrent_rc_syncs", s.ConcurrentRCSyncs, "The number of replication controllers that are allowed to sync concurrently. Larger number = more responsive replica management, but more CPU (and network) load") fs.Int32Var(&s.ConcurrentRSSyncs, "concurrent-replicaset-syncs", s.ConcurrentRSSyncs, "The number of replica sets that are allowed to sync concurrently. Larger number = more responsive replica management, but more CPU (and network) load") + fs.Int32Var(&s.ConcurrentResourceQuotaSyncs, "concurrent-resource-quota-syncs", s.ConcurrentResourceQuotaSyncs, "The number of resource quotas that are allowed to sync concurrently. Larger number = more responsive quota management, but more CPU (and network) load") fs.Int32Var(&s.ConcurrentDeploymentSyncs, "concurrent-deployment-syncs", s.ConcurrentDeploymentSyncs, "The number of deployment objects that are allowed to sync concurrently. Larger number = more responsive deployments, but more CPU (and network) load") fs.Int32Var(&s.ConcurrentNamespaceSyncs, "concurrent-namespace-syncs", s.ConcurrentNamespaceSyncs, "The number of namespace objects that are allowed to sync concurrently. Larger number = more responsive namespace termination, but more CPU (and network) load") @@ -159,11 +85,9 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabled "This flag is deprecated and will be removed in future releases. See node-monitor-period for Node health checking or "+ "route-reconciliation-period for cloud provider's route configuration settings.") fs.MarkDeprecated("node-sync-period", "This flag is currently no-op and will be deleted.") - fs.DurationVar(&s.RouteReconciliationPeriod.Duration, "route-reconciliation-period", s.RouteReconciliationPeriod.Duration, "The period for reconciling routes created for Nodes by cloud provider.") fs.DurationVar(&s.ResourceQuotaSyncPeriod.Duration, "resource-quota-sync-period", s.ResourceQuotaSyncPeriod.Duration, "The period for syncing quota usage status in the system") fs.DurationVar(&s.NamespaceSyncPeriod.Duration, "namespace-sync-period", s.NamespaceSyncPeriod.Duration, "The period for syncing namespace life-cycle updates") fs.DurationVar(&s.PVClaimBinderSyncPeriod.Duration, "pvclaimbinder-sync-period", s.PVClaimBinderSyncPeriod.Duration, "The period for syncing persistent volumes and persistent volume claims") - fs.DurationVar(&s.MinResyncPeriod.Duration, "min-resync-period", s.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod") fs.StringVar(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, "pv-recycler-pod-template-filepath-nfs", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, "The file path to a pod definition used as a template for NFS persistent volume recycling") fs.Int32Var(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS, "pv-recycler-minimum-timeout-nfs", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS, "The minimum ActiveDeadlineSeconds to use for an NFS Recycler pod") fs.Int32Var(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS, "pv-recycler-increment-timeout-nfs", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS, "the increment of time added per Gi to ActiveDeadlineSeconds for an NFS scrubber pod") @@ -193,8 +117,6 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabled "where N means number of retries allowed for kubelet to post node status.") fs.DurationVar(&s.NodeStartupGracePeriod.Duration, "node-startup-grace-period", s.NodeStartupGracePeriod.Duration, "Amount of time which we allow starting Node to be unresponsive before marking it unhealthy.") - fs.DurationVar(&s.NodeMonitorPeriod.Duration, "node-monitor-period", s.NodeMonitorPeriod.Duration, - "The period for syncing NodeStatus in NodeController.") fs.StringVar(&s.ServiceAccountKeyFile, "service-account-private-key-file", s.ServiceAccountKeyFile, "Filename containing a PEM-encoded private RSA or ECDSA key used to sign service account tokens.") fs.StringVar(&s.ClusterSigningCertFile, "cluster-signing-cert-file", s.ClusterSigningCertFile, "Filename containing a PEM-encoded X509 CA certificate used to issue cluster-scoped certificates") fs.StringVar(&s.ClusterSigningKeyFile, "cluster-signing-key-file", s.ClusterSigningKeyFile, "Filename containing a PEM-encoded RSA or ECDSA private key used to sign cluster-scoped certificates") @@ -202,34 +124,19 @@ func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabled var dummy string fs.MarkDeprecated("insecure-experimental-approve-all-kubelet-csrs-for-group", "This flag does nothing.") fs.StringVar(&dummy, "insecure-experimental-approve-all-kubelet-csrs-for-group", "", "This flag does nothing.") - fs.BoolVar(&s.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/") - fs.BoolVar(&s.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled") - fs.StringVar(&s.ClusterName, "cluster-name", s.ClusterName, "The instance prefix for the cluster") - fs.StringVar(&s.ClusterCIDR, "cluster-cidr", s.ClusterCIDR, "CIDR Range for Pods in cluster. Requires --allocate-node-cidrs to be true") fs.StringVar(&s.ServiceCIDR, "service-cluster-ip-range", s.ServiceCIDR, "CIDR Range for Services in cluster. Requires --allocate-node-cidrs to be true") fs.Int32Var(&s.NodeCIDRMaskSize, "node-cidr-mask-size", s.NodeCIDRMaskSize, "Mask size for node cidr in cluster.") - fs.BoolVar(&s.AllocateNodeCIDRs, "allocate-node-cidrs", false, - "Should CIDRs for Pods be allocated and set on the cloud provider.") - fs.StringVar(&s.CIDRAllocatorType, "cidr-allocator-type", "RangeAllocator", - "Type of CIDR allocator to use") - fs.BoolVar(&s.ConfigureCloudRoutes, "configure-cloud-routes", true, "Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider.") - fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") - fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.") fs.StringVar(&s.RootCAFile, "root-ca-file", s.RootCAFile, "If set, this root certificate authority will be included in service account's token secret. This must be a valid PEM-encoded CA bundle.") - fs.StringVar(&s.ContentType, "kube-api-content-type", s.ContentType, "Content type of requests sent to apiserver.") - fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver") - fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver") - fs.DurationVar(&s.ControllerStartInterval.Duration, "controller-start-interval", s.ControllerStartInterval.Duration, "Interval between starting controller managers.") fs.BoolVar(&s.EnableGarbageCollector, "enable-garbage-collector", s.EnableGarbageCollector, "Enables the generic garbage collector. MUST be synced with the corresponding flag of the kube-apiserver.") fs.Int32Var(&s.ConcurrentGCSyncs, "concurrent-gc-syncs", s.ConcurrentGCSyncs, "The number of garbage collector workers that are allowed to sync concurrently.") - fs.Float32Var(&s.NodeEvictionRate, "node-eviction-rate", 0.1, "Number of nodes per second on which pods are deleted in case of node failure when a zone is healthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters.") - fs.Float32Var(&s.SecondaryNodeEvictionRate, "secondary-node-eviction-rate", 0.01, "Number of nodes per second on which pods are deleted in case of node failure when a zone is unhealthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters. This value is implicitly overridden to 0 if the cluster size is smaller than --large-cluster-size-threshold.") fs.Int32Var(&s.LargeClusterSizeThreshold, "large-cluster-size-threshold", 50, "Number of nodes from which NodeController treats the cluster as large for the eviction logic purposes. --secondary-node-eviction-rate is implicitly overridden to 0 for clusters this size or smaller.") fs.Float32Var(&s.UnhealthyZoneThreshold, "unhealthy-zone-threshold", 0.55, "Fraction of Nodes in a zone which needs to be not Ready (minimum 3) for zone to be treated as unhealthy. ") fs.BoolVar(&s.DisableAttachDetachReconcilerSync, "disable-attach-detach-reconcile-sync", false, "Disable volume attach detach reconciler sync. Disabling this may cause volumes to be mismatched with pods. Use wisely.") fs.DurationVar(&s.ReconcilerSyncLoopPeriod.Duration, "attach-detach-reconcile-sync-period", s.ReconcilerSyncLoopPeriod.Duration, "The reconciler sync wait time between volume attach detach. This duration must be larger than one second, and increasing this value from the default may allow for volumes to be mismatched with pods.") fs.BoolVar(&s.EnableTaintManager, "enable-taint-manager", s.EnableTaintManager, "WARNING: Beta feature. If set to true enables NoExecute Taints and will evict all not-tolerating Pod running on Nodes tainted with this kind of Taints.") fs.BoolVar(&s.HorizontalPodAutoscalerUseRESTClients, "horizontal-pod-autoscaler-use-rest-clients", s.HorizontalPodAutoscalerUseRESTClients, "WARNING: alpha feature. If set to true, causes the horizontal pod autoscaler controller to use REST clients through the kube-aggregator, instead of using the legacy metrics client through the API server proxy. This is required for custom metrics support in the horizontal pod autoscaler.") + fs.Float32Var(&s.NodeEvictionRate, "node-eviction-rate", 0.1, "Number of nodes per second on which pods are deleted in case of node failure when a zone is healthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters.") + fs.Float32Var(&s.SecondaryNodeEvictionRate, "secondary-node-eviction-rate", 0.01, "Number of nodes per second on which pods are deleted in case of node failure when a zone is unhealthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters. This value is implicitly overridden to 0 if the cluster size is smaller than --large-cluster-size-threshold.") leaderelectionconfig.BindFlags(&s.LeaderElection, fs) diff --git a/cmd/kube-controller-manager/app/options/options_test.go b/cmd/kube-controller-manager/app/options/options_test.go index 87f92d5a9c..904535121f 100644 --- a/cmd/kube-controller-manager/app/options/options_test.go +++ b/cmd/kube-controller-manager/app/options/options_test.go @@ -26,6 +26,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" + cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options" "k8s.io/kubernetes/pkg/apis/componentconfig" ) @@ -110,102 +111,104 @@ func TestAddFlags(t *testing.T) { sort.Sort(sortedGCIgnoredResources(s.GCIgnoredResources)) expected := &CMServer{ - KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{ - Port: 10000, - Address: "192.168.4.10", - AllocateNodeCIDRs: true, - CloudConfigFile: "/cloud-config", - CloudProvider: "gce", - ClusterCIDR: "1.2.3.4/24", - ClusterName: "k8s", - ConcurrentDeploymentSyncs: 10, - ConcurrentEndpointSyncs: 10, - ConcurrentGCSyncs: 30, - ConcurrentNamespaceSyncs: 20, - ConcurrentRSSyncs: 10, - ConcurrentResourceQuotaSyncs: 10, - ConcurrentServiceSyncs: 2, - ConcurrentSATokenSyncs: 10, - ConcurrentRCSyncs: 10, - ConfigureCloudRoutes: false, - EnableContentionProfiling: true, - ControllerStartInterval: metav1.Duration{Duration: 2 * time.Minute}, - ConcurrentDaemonSetSyncs: 2, - ConcurrentJobSyncs: 5, - DeletingPodsQps: 0.1, - EnableProfiling: false, - CIDRAllocatorType: "CloudAllocator", - NodeCIDRMaskSize: 48, - ServiceSyncPeriod: metav1.Duration{Duration: 2 * time.Minute}, - ResourceQuotaSyncPeriod: metav1.Duration{Duration: 10 * time.Minute}, - NamespaceSyncPeriod: metav1.Duration{Duration: 10 * time.Minute}, - PVClaimBinderSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, - HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 45 * time.Second}, - DeploymentControllerSyncPeriod: metav1.Duration{Duration: 45 * time.Second}, - MinResyncPeriod: metav1.Duration{Duration: 8 * time.Hour}, - RegisterRetryCount: 10, - RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second}, - PodEvictionTimeout: metav1.Duration{Duration: 2 * time.Minute}, - NodeMonitorGracePeriod: metav1.Duration{Duration: 30 * time.Second}, - NodeStartupGracePeriod: metav1.Duration{Duration: 30 * time.Second}, - NodeMonitorPeriod: metav1.Duration{Duration: 10 * time.Second}, - HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 1 * time.Minute}, - HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute}, - HorizontalPodAutoscalerTolerance: 0.1, - TerminatedPodGCThreshold: 12000, - VolumeConfiguration: componentconfig.VolumeConfiguration{ - EnableDynamicProvisioning: false, - EnableHostPathProvisioning: true, - FlexVolumePluginDir: "/flex-volume-plugin", - PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{ - MaximumRetry: 3, - MinimumTimeoutNFS: 200, - IncrementTimeoutNFS: 45, - MinimumTimeoutHostPath: 45, - IncrementTimeoutHostPath: 45, + ControllerManagerServer: cmoptions.ControllerManagerServer{ + KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{ + Port: 10000, + Address: "192.168.4.10", + AllocateNodeCIDRs: true, + CloudConfigFile: "/cloud-config", + CloudProvider: "gce", + ClusterCIDR: "1.2.3.4/24", + ClusterName: "k8s", + ConcurrentDeploymentSyncs: 10, + ConcurrentEndpointSyncs: 10, + ConcurrentGCSyncs: 30, + ConcurrentNamespaceSyncs: 20, + ConcurrentRSSyncs: 10, + ConcurrentResourceQuotaSyncs: 10, + ConcurrentServiceSyncs: 2, + ConcurrentSATokenSyncs: 10, + ConcurrentRCSyncs: 10, + ConfigureCloudRoutes: false, + EnableContentionProfiling: true, + ControllerStartInterval: metav1.Duration{Duration: 2 * time.Minute}, + ConcurrentDaemonSetSyncs: 2, + ConcurrentJobSyncs: 5, + DeletingPodsQps: 0.1, + EnableProfiling: false, + CIDRAllocatorType: "CloudAllocator", + NodeCIDRMaskSize: 48, + ServiceSyncPeriod: metav1.Duration{Duration: 2 * time.Minute}, + ResourceQuotaSyncPeriod: metav1.Duration{Duration: 10 * time.Minute}, + NamespaceSyncPeriod: metav1.Duration{Duration: 10 * time.Minute}, + PVClaimBinderSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, + HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 45 * time.Second}, + DeploymentControllerSyncPeriod: metav1.Duration{Duration: 45 * time.Second}, + MinResyncPeriod: metav1.Duration{Duration: 8 * time.Hour}, + RegisterRetryCount: 10, + RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second}, + PodEvictionTimeout: metav1.Duration{Duration: 2 * time.Minute}, + NodeMonitorGracePeriod: metav1.Duration{Duration: 30 * time.Second}, + NodeStartupGracePeriod: metav1.Duration{Duration: 30 * time.Second}, + NodeMonitorPeriod: metav1.Duration{Duration: 10 * time.Second}, + HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 1 * time.Minute}, + HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute}, + HorizontalPodAutoscalerTolerance: 0.1, + TerminatedPodGCThreshold: 12000, + VolumeConfiguration: componentconfig.VolumeConfiguration{ + EnableDynamicProvisioning: false, + EnableHostPathProvisioning: true, + FlexVolumePluginDir: "/flex-volume-plugin", + PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{ + MaximumRetry: 3, + MinimumTimeoutNFS: 200, + IncrementTimeoutNFS: 45, + MinimumTimeoutHostPath: 45, + IncrementTimeoutHostPath: 45, + }, }, + ContentType: "application/json", + KubeAPIQPS: 50.0, + KubeAPIBurst: 100, + LeaderElection: componentconfig.LeaderElectionConfiguration{ + ResourceLock: "configmap", + LeaderElect: false, + LeaseDuration: metav1.Duration{Duration: 30 * time.Second}, + RenewDeadline: metav1.Duration{Duration: 15 * time.Second}, + RetryPeriod: metav1.Duration{Duration: 5 * time.Second}, + }, + ClusterSigningCertFile: "/cluster-signing-cert", + ClusterSigningKeyFile: "/cluster-signing-key", + ServiceAccountKeyFile: "/service-account-private-key", + ClusterSigningDuration: metav1.Duration{Duration: 10 * time.Hour}, + EnableGarbageCollector: false, + GCIgnoredResources: []componentconfig.GroupResource{ + {Group: "extensions", Resource: "replicationcontrollers"}, + {Group: "", Resource: "bindings"}, + {Group: "", Resource: "componentstatuses"}, + {Group: "", Resource: "events"}, + {Group: "authentication.k8s.io", Resource: "tokenreviews"}, + {Group: "authorization.k8s.io", Resource: "subjectaccessreviews"}, + {Group: "authorization.k8s.io", Resource: "selfsubjectaccessreviews"}, + {Group: "authorization.k8s.io", Resource: "localsubjectaccessreviews"}, + {Group: "authorization.k8s.io", Resource: "selfsubjectrulesreviews"}, + {Group: "apiregistration.k8s.io", Resource: "apiservices"}, + {Group: "apiextensions.k8s.io", Resource: "customresourcedefinitions"}, + }, + NodeEvictionRate: 0.2, + SecondaryNodeEvictionRate: 0.05, + LargeClusterSizeThreshold: 100, + UnhealthyZoneThreshold: 0.6, + DisableAttachDetachReconcilerSync: true, + ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 30 * time.Second}, + Controllers: []string{"foo", "bar"}, + EnableTaintManager: false, + HorizontalPodAutoscalerUseRESTClients: true, + UseServiceAccountCredentials: true, }, - ContentType: "application/json", - KubeAPIQPS: 50.0, - KubeAPIBurst: 100, - LeaderElection: componentconfig.LeaderElectionConfiguration{ - ResourceLock: "configmap", - LeaderElect: false, - LeaseDuration: metav1.Duration{Duration: 30 * time.Second}, - RenewDeadline: metav1.Duration{Duration: 15 * time.Second}, - RetryPeriod: metav1.Duration{Duration: 5 * time.Second}, - }, - ClusterSigningCertFile: "/cluster-signing-cert", - ClusterSigningKeyFile: "/cluster-signing-key", - ServiceAccountKeyFile: "/service-account-private-key", - ClusterSigningDuration: metav1.Duration{Duration: 10 * time.Hour}, - EnableGarbageCollector: false, - GCIgnoredResources: []componentconfig.GroupResource{ - {Group: "extensions", Resource: "replicationcontrollers"}, - {Group: "", Resource: "bindings"}, - {Group: "", Resource: "componentstatuses"}, - {Group: "", Resource: "events"}, - {Group: "authentication.k8s.io", Resource: "tokenreviews"}, - {Group: "authorization.k8s.io", Resource: "subjectaccessreviews"}, - {Group: "authorization.k8s.io", Resource: "selfsubjectaccessreviews"}, - {Group: "authorization.k8s.io", Resource: "localsubjectaccessreviews"}, - {Group: "authorization.k8s.io", Resource: "selfsubjectrulesreviews"}, - {Group: "apiregistration.k8s.io", Resource: "apiservices"}, - {Group: "apiextensions.k8s.io", Resource: "customresourcedefinitions"}, - }, - NodeEvictionRate: 0.2, - SecondaryNodeEvictionRate: 0.05, - LargeClusterSizeThreshold: 100, - UnhealthyZoneThreshold: 0.6, - DisableAttachDetachReconcilerSync: true, - ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 30 * time.Second}, - Controllers: []string{"foo", "bar"}, - EnableTaintManager: false, - HorizontalPodAutoscalerUseRESTClients: true, - UseServiceAccountCredentials: true, + Kubeconfig: "/kubeconfig", + Master: "192.168.4.20", }, - Kubeconfig: "/kubeconfig", - Master: "192.168.4.20", } // Sort GCIgnoredResources because it's built from a map, which means the diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index 4a57eec48c..4788176be6 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -181,6 +181,9 @@ type KubeControllerManagerConfiguration struct { CloudProvider string // cloudConfigFile is the path to the cloud provider configuration file. CloudConfigFile string + // externalCloudVolumePlugin specifies the plugin to use when cloudProvider is "external". + // It is currently used by the in repo cloud providers to handle node and volume control in the KCM. + ExternalCloudVolumePlugin string // run with untagged cloud instances AllowUntaggedCloud bool // concurrentEndpointSyncs is the number of endpoint syncing operations diff --git a/pkg/controller/node/node_controller.go b/pkg/controller/node/node_controller.go index cd8b0e219e..ab490d6a05 100644 --- a/pkg/controller/node/node_controller.go +++ b/pkg/controller/node/node_controller.go @@ -259,7 +259,7 @@ func NewNodeController( } mask := clusterCIDR.Mask if maskSize, _ := mask.Size(); maskSize > nodeCIDRMaskSize { - glog.Fatal("Controller: Invalid clusterCIDR, mask size of clusterCIDR must be less than nodeCIDRMaskSize.") + glog.Fatalf("Controller: Invalid clusterCIDR, mask size of clusterCIDR(%d) must be less than nodeCIDRMaskSize(%d).", maskSize, nodeCIDRMaskSize) } }