github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

controller-manager: switch to config/option struct pattern

pull/6/head
stewart-yu 2018-02-05 14:48:30 +08:00 committed by Dr. Stefan Schimanski
parent ba791275ce
commit 0cbe0a6034
27 changed files with 977 additions and 544 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
cmd/cloud-controller-manager/app/config/config.go Normal file
View File

@ -0,0 +1,55 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package app
import (
"time"
genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app"
)
// ExtraConfig are part of Config, also can place your custom config here.
type ExtraConfig struct {
NodeStatusUpdateFrequency time.Duration
}
// Config is the main context object for the cloud controller manager.
type Config struct {
Generic genericcontrollermanager.Config
Extra ExtraConfig
}
type completedConfig struct {
Generic genericcontrollermanager.CompletedConfig
Extra *ExtraConfig
}
// CompletedConfig same as Config, just to swap private object.
type CompletedConfig struct {
// Embed a private pointer that cannot be instantiated outside of this package.
*completedConfig
}
// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
func (c *Config) Complete() *CompletedConfig {
cc := completedConfig{
c.Generic.Complete(),
&c.Extra,
}
return &CompletedConfig{&cc}
}

161
cmd/cloud-controller-manager/app/controllermanager.go
View File

@ -20,32 +20,24 @@ import (
"fmt"
"math/rand"
"net"
"net/http"
"net/http/pprof"
"os"
goruntime "runtime"
"strconv"
"strings"
"time"
"github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
"github.com/spf13/cobra"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
"k8s.io/client-go/tools/record"
cloudcontrollerconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config"
"k8s.io/kubernetes/cmd/cloud-controller-manager/app/options"
"k8s.io/kubernetes/pkg/api/legacyscheme"
genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app"
"k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/controller"
cloudcontrollers "k8s.io/kubernetes/pkg/controller/cloud"
@ -62,7 +54,7 @@ const (
// NewCloudControllerManagerCommand creates a *cobra.Command object with default parameters
func NewCloudControllerManagerCommand() *cobra.Command {
s := options.NewCloudControllerManagerServer()
s := options.NewCloudControllerManagerOptions()
cmd := &cobra.Command{
Use: "cloud-controller-manager",
Long: `The Cloud controller manager is a daemon that embeds
@ -70,7 +62,13 @@ the cloud specific control loops shipped with Kubernetes.`,
Run: func(cmd *cobra.Command, args []string) {
verflag.PrintAndExitIfRequested()
if err := Run(s); err != nil {
c, err := s.Config()
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
if err := Run(c.Complete()); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
@ -83,84 +81,68 @@ the cloud specific control loops shipped with Kubernetes.`,
}
// resyncPeriod computes the time interval a shared informer waits before resyncing with the api server
func resyncPeriod(s *options.CloudControllerManagerServer) func() time.Duration {
func resyncPeriod(c *cloudcontrollerconfig.CompletedConfig) func() time.Duration {
return func() time.Duration {
factor := rand.Float64() + 1
return time.Duration(float64(s.MinResyncPeriod.Nanoseconds()) * factor)
return time.Duration(float64(c.Generic.ComponentConfig.MinResyncPeriod.Nanoseconds()) * factor)
}
}
// Run runs the ExternalCMServer. This should never exit.
func Run(s *options.CloudControllerManagerServer) error {
if s.CloudProvider == "" {
glog.Fatalf("--cloud-provider cannot be empty")
}
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
func Run(c *cloudcontrollerconfig.CompletedConfig) error {
cloud, err := cloudprovider.InitCloudProvider(c.Generic.ComponentConfig.CloudProvider, c.Generic.ComponentConfig.CloudConfigFile)
if err != nil {
glog.Fatalf("Cloud provider could not be initialized: %v", err)
}
if cloud == nil {
glog.Fatalf("cloud provider is nil")
}
if cloud.HasClusterID() == false {
if s.AllowUntaggedCloud == true {
if c.Generic.ComponentConfig.AllowUntaggedCloud == true {
glog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues")
} else {
glog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option")
}
}
if c, err := configz.New("componentconfig"); err == nil {
c.Set(s.KubeControllerManagerConfiguration)
// setup /configz endpoint
if cz, err := configz.New("componentconfig"); err == nil {
cz.Set(c.Generic.ComponentConfig)
} else {
glog.Errorf("unable to register configz: %s", err)
}
kubeconfig, err := clientcmd.BuildConfigFromFlags(s.Master, s.Kubeconfig)
if err != nil {
return err
glog.Errorf("unable to register configz: %c", err)
}
// Set the ContentType of the requests from kube client
kubeconfig.ContentConfig.ContentType = s.ContentType
// Override kubeconfig qps/burst settings from flags
kubeconfig.QPS = s.KubeAPIQPS
kubeconfig.Burst = int(s.KubeAPIBurst)
kubeClient, err := kubernetes.NewForConfig(restclient.AddUserAgent(kubeconfig, "cloud-controller-manager"))
if err != nil {
glog.Fatalf("Invalid API configuration: %v", err)
// Start the controller manager HTTP server
stopCh := make(chan struct{})
if c.Generic.InsecureServing != nil {
if err := genericcontrollermanager.Serve(&c.Generic, c.Generic.InsecureServing.Serve, stopCh); err != nil {
return err
}
}
leaderElectionClient := kubernetes.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "leader-election"))
// Start the external controller manager server
go startHTTP(s)
recorder := createRecorder(kubeClient)
run := func(stop <-chan struct{}) {
rootClientBuilder := controller.SimpleControllerClientBuilder{
ClientConfig: kubeconfig,
ClientConfig: c.Generic.Kubeconfig,
}
var clientBuilder controller.ControllerClientBuilder
if s.UseServiceAccountCredentials {
if c.Generic.ComponentConfig.UseServiceAccountCredentials {
clientBuilder = controller.SAControllerClientBuilder{
ClientConfig: restclient.AnonymousClientConfig(kubeconfig),
CoreClient: kubeClient.CoreV1(),
AuthenticationClient: kubeClient.AuthenticationV1(),
ClientConfig: restclient.AnonymousClientConfig(c.Generic.Kubeconfig),
CoreClient: c.Generic.Client.CoreV1(),
AuthenticationClient: c.Generic.Client.AuthenticationV1(),
Namespace: "kube-system",
}
} else {
clientBuilder = rootClientBuilder
}
if err := StartControllers(s, kubeconfig, rootClientBuilder, clientBuilder, stop, recorder, cloud); err != nil {
if err := startControllers(c, c.Generic.Kubeconfig, rootClientBuilder, clientBuilder, stop, c.Generic.EventRecorder, cloud); err != nil {
glog.Fatalf("error running controllers: %v", err)
}
}
if !s.LeaderElection.LeaderElect {
if !c.Generic.ComponentConfig.LeaderElection.LeaderElect {
run(nil)
panic("unreachable")
}
@ -174,13 +156,13 @@ func Run(s *options.CloudControllerManagerServer) error {
id = id + "_" + string(uuid.NewUUID())
// Lock required for leader election
rl, err := resourcelock.New(s.LeaderElection.ResourceLock,
rl, err := resourcelock.New(c.Generic.ComponentConfig.LeaderElection.ResourceLock,
"kube-system",
"cloud-controller-manager",
leaderElectionClient.CoreV1(),
c.Generic.LeaderElectionClient.CoreV1(),
resourcelock.ResourceLockConfig{
Identity: id,
EventRecorder: recorder,
EventRecorder: c.Generic.EventRecorder,
})
if err != nil {
glog.Fatalf("error creating lock: %v", err)
@ -189,9 +171,9 @@ func Run(s *options.CloudControllerManagerServer) error {
// Try and become the leader and start cloud controller manager loops
leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{
Lock: rl,
LeaseDuration: s.LeaderElection.LeaseDuration.Duration,
RenewDeadline: s.LeaderElection.RenewDeadline.Duration,
RetryPeriod: s.LeaderElection.RetryPeriod.Duration,
LeaseDuration: c.Generic.ComponentConfig.LeaderElection.LeaseDuration.Duration,
RenewDeadline: c.Generic.ComponentConfig.LeaderElection.RenewDeadline.Duration,
RetryPeriod: c.Generic.ComponentConfig.LeaderElection.RetryPeriod.Duration,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: run,
OnStoppedLeading: func() {
@ -202,36 +184,36 @@ func Run(s *options.CloudControllerManagerServer) error {
panic("unreachable")
}
// StartControllers starts the cloud specific controller loops.
func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restclient.Config, rootClientBuilder, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}, recorder record.EventRecorder, cloud cloudprovider.Interface) error {
// startControllers starts the cloud specific controller loops.
func startControllers(c *cloudcontrollerconfig.CompletedConfig, kubeconfig *restclient.Config, rootClientBuilder, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}, recorder record.EventRecorder, cloud cloudprovider.Interface) error {
// Function to build the kube client object
client := func(serviceAccountName string) kubernetes.Interface {
return clientBuilder.ClientOrDie(serviceAccountName)
}
if cloud != nil {
// Initialize the cloud provider with a reference to the clientBuilder
cloud.Initialize(clientBuilder)
}
// TODO: move this setup into Config
versionedClient := rootClientBuilder.ClientOrDie("shared-informers")
sharedInformers := informers.NewSharedInformerFactory(versionedClient, resyncPeriod(s)())
sharedInformers := informers.NewSharedInformerFactory(versionedClient, resyncPeriod(c)())
// Start the CloudNodeController
nodeController := cloudcontrollers.NewCloudNodeController(
sharedInformers.Core().V1().Nodes(),
client("cloud-node-controller"), cloud,
s.NodeMonitorPeriod.Duration,
s.NodeStatusUpdateFrequency.Duration)
c.Generic.ComponentConfig.NodeMonitorPeriod.Duration,
c.Extra.NodeStatusUpdateFrequency)
nodeController.Run()
time.Sleep(wait.Jitter(s.ControllerStartInterval.Duration, ControllerStartJitter))
time.Sleep(wait.Jitter(c.Generic.ComponentConfig.ControllerStartInterval.Duration, ControllerStartJitter))
// Start the PersistentVolumeLabelController
pvlController := cloudcontrollers.NewPersistentVolumeLabelController(client("pvl-controller"), cloud)
threads := 5
go pvlController.Run(threads, stop)
time.Sleep(wait.Jitter(s.ControllerStartInterval.Duration, ControllerStartJitter))
time.Sleep(wait.Jitter(c.Generic.ComponentConfig.ControllerStartInterval.Duration, ControllerStartJitter))
// Start the service controller
serviceController, err := servicecontroller.New(
@ -239,34 +221,34 @@ func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restc
client("service-controller"),
sharedInformers.Core().V1().Services(),
sharedInformers.Core().V1().Nodes(),
s.ClusterName,
c.Generic.ComponentConfig.ClusterName,
)
if err != nil {
glog.Errorf("Failed to start service controller: %v", err)
} else {
go serviceController.Run(stop, int(s.ConcurrentServiceSyncs))
time.Sleep(wait.Jitter(s.ControllerStartInterval.Duration, ControllerStartJitter))
go serviceController.Run(stop, int(c.Generic.ComponentConfig.ConcurrentServiceSyncs))
time.Sleep(wait.Jitter(c.Generic.ComponentConfig.ControllerStartInterval.Duration, ControllerStartJitter))
}
// If CIDRs should be allocated for pods and set on the CloudProvider, then start the route controller
if s.AllocateNodeCIDRs && s.ConfigureCloudRoutes {
if c.Generic.ComponentConfig.AllocateNodeCIDRs && c.Generic.ComponentConfig.ConfigureCloudRoutes {
if routes, ok := cloud.Routes(); !ok {
glog.Warning("configure-cloud-routes is set, but cloud provider does not support routes. Will not configure cloud provider routes.")
} else {
var clusterCIDR *net.IPNet
if len(strings.TrimSpace(s.ClusterCIDR)) != 0 {
_, clusterCIDR, err = net.ParseCIDR(s.ClusterCIDR)
if len(strings.TrimSpace(c.Generic.ComponentConfig.ClusterCIDR)) != 0 {
_, clusterCIDR, err = net.ParseCIDR(c.Generic.ComponentConfig.ClusterCIDR)
if err != nil {
glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", s.ClusterCIDR, err)
glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", c.Generic.ComponentConfig.ClusterCIDR, err)
}
}
routeController := routecontroller.New(routes, client("route-controller"), sharedInformers.Core().V1().Nodes(), s.ClusterName, clusterCIDR)
go routeController.Run(stop, s.RouteReconciliationPeriod.Duration)
time.Sleep(wait.Jitter(s.ControllerStartInterval.Duration, ControllerStartJitter))
routeController := routecontroller.New(routes, client("route-controller"), sharedInformers.Core().V1().Nodes(), c.Generic.ComponentConfig.ClusterName, clusterCIDR)
go routeController.Run(stop, c.Generic.ComponentConfig.RouteReconciliationPeriod.Duration)
time.Sleep(wait.Jitter(c.Generic.ComponentConfig.ControllerStartInterval.Duration, ControllerStartJitter))
}
} else {
glog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", s.AllocateNodeCIDRs, s.ConfigureCloudRoutes)
glog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", c.Generic.ComponentConfig.AllocateNodeCIDRs, c.Generic.ComponentConfig.ConfigureCloudRoutes)
}
// If apiserver is not running we should wait for some time and fail only then. This is particularly
@ -286,32 +268,3 @@ func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restc
select {}
}
func startHTTP(s *options.CloudControllerManagerServer) {
mux := http.NewServeMux()
healthz.InstallHandler(mux)
if s.EnableProfiling {
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
if s.EnableContentionProfiling {
goruntime.SetBlockProfileRate(1)
}
}
configz.InstallHandler(mux)
mux.Handle("/metrics", prometheus.Handler())
server := &http.Server{
Addr: net.JoinHostPort(s.Address, strconv.Itoa(int(s.Port))),
Handler: mux,
}
glog.Fatal(server.ListenAndServe())
}
func createRecorder(kubeClient *kubernetes.Clientset) record.EventRecorder {
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartLogging(glog.Infof)
eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.CoreV1().RESTClient()).Events("")})
return eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: "cloud-controller-manager"})
}

80
cmd/cloud-controller-manager/app/options/options.go
View File

@ -17,10 +17,13 @@ limitations under the License.
package options
import (
"fmt"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilfeature "k8s.io/apiserver/pkg/util/feature"
cloudcontrollerconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config"
cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
"k8s.io/kubernetes/pkg/master/ports"
@ -31,39 +34,78 @@ import (
"github.com/spf13/pflag"
)
// CloudControllerManagerServer is the main context object for the controller manager.
type CloudControllerManagerServer struct {
cmoptions.ControllerManagerServer
// CloudControllerManagerOptions is the main context object for the controller manager.
type CloudControllerManagerOptions struct {
Generic cmoptions.GenericControllerManagerOptions
// NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status
NodeStatusUpdateFrequency metav1.Duration
}
// NewCloudControllerManagerServer creates a new ExternalCMServer with a default config.
func NewCloudControllerManagerServer() *CloudControllerManagerServer {
s := CloudControllerManagerServer{
// NewCloudControllerManagerOptions creates a new ExternalCMServer with a default config.
func NewCloudControllerManagerOptions() *CloudControllerManagerOptions {
componentConfig := cmoptions.NewDefaultControllerManagerComponentConfig(ports.InsecureCloudControllerManagerPort)
s := CloudControllerManagerOptions{
// The common/default are kept in 'cmd/kube-controller-manager/app/options/util.go'.
// Please make common changes there and put anything cloud specific here.
ControllerManagerServer: cmoptions.ControllerManagerServer{
KubeControllerManagerConfiguration: cmoptions.GetDefaultControllerOptions(ports.CloudControllerManagerPort),
},
Generic: cmoptions.NewGenericControllerManagerOptions(componentConfig),
NodeStatusUpdateFrequency: metav1.Duration{Duration: 5 * time.Minute},
}
s.LeaderElection.LeaderElect = true
s.Generic.ComponentConfig.LeaderElection.LeaderElect = true
return &s
}
// AddFlags adds flags for a specific ExternalCMServer to the specified FlagSet
func (s *CloudControllerManagerServer) AddFlags(fs *pflag.FlagSet) {
cmoptions.AddDefaultControllerFlags(&s.ControllerManagerServer, fs)
fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider of cloud services. Cannot be empty.")
fs.DurationVar(&s.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", s.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.")
// TODO: remove --service-account-private-key-file 6 months after 1.8 is released (~1.10)
fs.StringVar(&s.ServiceAccountKeyFile, "service-account-private-key-file", s.ServiceAccountKeyFile, "Filename containing a PEM-encoded private RSA or ECDSA key used to sign service account tokens.")
fs.MarkDeprecated("service-account-private-key-file", "This flag is currently no-op and will be deleted.")
fs.Int32Var(&s.ConcurrentServiceSyncs, "concurrent-service-syncs", s.ConcurrentServiceSyncs, "The number of services that are allowed to sync concurrently. Larger number = more responsive service management, but more CPU (and network) load")
func (o *CloudControllerManagerOptions) AddFlags(fs *pflag.FlagSet) {
o.Generic.AddFlags(fs)
leaderelectionconfig.BindFlags(&s.LeaderElection, fs)
fs.StringVar(&o.Generic.ComponentConfig.CloudProvider, "cloud-provider", o.Generic.ComponentConfig.CloudProvider, "The provider of cloud services. Cannot be empty.")
fs.DurationVar(&o.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", o.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.")
// TODO: remove --service-account-private-key-file 6 months after 1.8 is released (~1.10)
fs.StringVar(&o.Generic.ComponentConfig.ServiceAccountKeyFile, "service-account-private-key-file", o.Generic.ComponentConfig.ServiceAccountKeyFile, "Filename containing a PEM-encoded private RSA or ECDSA key used to sign service account tokens.")
fs.MarkDeprecated("service-account-private-key-file", "This flag is currently no-op and will be deleted.")
fs.Int32Var(&o.Generic.ComponentConfig.ConcurrentServiceSyncs, "concurrent-service-syncs", o.Generic.ComponentConfig.ConcurrentServiceSyncs, "The number of services that are allowed to sync concurrently. Larger number = more responsive service management, but more CPU (and network) load")
leaderelectionconfig.BindFlags(&o.Generic.ComponentConfig.LeaderElection, fs)
utilfeature.DefaultFeatureGate.AddFlag(fs)
}
// ApplyTo fills up cloud controller manager config with options.
func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config) error {
if err := o.Generic.ApplyTo(&c.Generic, "cloud-controller-manager"); err != nil {
return err
}
c.Extra.NodeStatusUpdateFrequency = o.NodeStatusUpdateFrequency.Duration
return nil
}
// Validate is used to validate config before launching the cloud controller manager
func (o *CloudControllerManagerOptions) Validate() error {
errors := []error{}
errors = append(errors, o.Generic.Validate()...)
if len(o.Generic.ComponentConfig.CloudProvider) == 0 {
errors = append(errors, fmt.Errorf("--cloud-provider cannot be empty"))
}
return utilerrors.NewAggregate(errors)
}
// Config return a cloud controller manager config objective
func (o CloudControllerManagerOptions) Config() (*cloudcontrollerconfig.Config, error) {
if err := o.Validate(); err != nil {
return nil, err
}
c := &cloudcontrollerconfig.Config{}
if err := o.ApplyTo(c); err != nil {
return nil, err
}
return c, nil
}

18
cmd/cloud-controller-manager/app/options/options_test.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package options
import (
"net"
"reflect"
"testing"
"time"
@ -31,7 +32,7 @@ import (
func TestAddFlags(t *testing.T) {
f := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError)
s := NewCloudControllerManagerServer()
s := NewCloudControllerManagerOptions()
s.AddFlags(f)
args := []string{
@ -65,13 +66,13 @@ func TestAddFlags(t *testing.T) {
}
f.Parse(args)
expected := &CloudControllerManagerServer{
ControllerManagerServer: cmoptions.ControllerManagerServer{
KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
expected := &CloudControllerManagerOptions{
Generic: cmoptions.GenericControllerManagerOptions{
ComponentConfig: componentconfig.KubeControllerManagerConfiguration{
CloudProvider: "gce",
CloudConfigFile: "/cloud-config",
Port: 10000,
Address: "192.168.4.10",
Port: 10253, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config
Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config
ConcurrentEndpointSyncs: 5,
ConcurrentRSSyncs: 5,
ConcurrentResourceQuotaSyncs: 5,
@ -138,6 +139,11 @@ func TestAddFlags(t *testing.T) {
CIDRAllocatorType: "RangeAllocator",
Controllers: []string{"*"},
},
InsecureServing: &cmoptions.InsecureServingOptions{
BindAddress: net.ParseIP("192.168.4.10"),
BindPort: int(10000),
BindNetwork: "tcp",
},
Kubeconfig: "/kubeconfig",
Master: "192.168.4.20",
},

60
cmd/controller-manager/app/config.go Normal file
View File

@ -0,0 +1,60 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package app
import (
clientset "k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/record"
"k8s.io/kubernetes/pkg/apis/componentconfig"
)
// Config is the main context object for the controller manager.
type Config struct {
// TODO: split up the component config. This is not generic.
ComponentConfig componentconfig.KubeControllerManagerConfiguration
InsecureServing *InsecureServingInfo
// the general kube client
Client *clientset.Clientset
// the client only used for leader election
LeaderElectionClient *clientset.Clientset
// the rest config for the master
Kubeconfig *restclient.Config
// the event sink
EventRecorder record.EventRecorder
}
type completedConfig struct {
*Config
}
// CompletedConfig same as Config, just to swap private object.
type CompletedConfig struct {
// Embed a private pointer that cannot be instantiated outside of this package.
*completedConfig
}
// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
func (c *Config) Complete() CompletedConfig {
cc := completedConfig{c}
return CompletedConfig{&cc}
}

46
cmd/controller-manager/app/insecure_serving.go Normal file
View File

@ -0,0 +1,46 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package app
import (
"net"
"net/http"
"time"
"github.com/golang/glog"
"k8s.io/apiserver/pkg/server"
)
// InsecureServingInfo is the main context object for the insecure http server.
type InsecureServingInfo struct {
// Listener is the secure server network listener.
Listener net.Listener
}
// Serve starts an insecure http server with the given handler. It fails only if
// the initial listen call fails. It does not block.
func (s *InsecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Duration, stopCh <-chan struct{}) error {
insecureServer := &http.Server{
Addr: s.Listener.Addr().String(),
Handler: handler,
MaxHeaderBytes: 1 << 20,
}
glog.Infof("Serving insecurely on %s", s.Listener.Addr())
return server.RunServer(insecureServer, s.Listener, shutdownTimeout, stopCh)
}

112
cmd/controller-manager/app/options/insecure_serving.go Normal file
View File

@ -0,0 +1,112 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"fmt"
"net"
"github.com/spf13/pflag"
"k8s.io/apiserver/pkg/server/options"
genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app"
"k8s.io/kubernetes/pkg/apis/componentconfig"
)
// InsecureServingOptions are for creating an unauthenticated, unauthorized, insecure port.
// No one should be using these anymore.
type InsecureServingOptions struct {
BindAddress net.IP
BindPort int
// BindNetwork is the type of network to bind to - defaults to "tcp", accepts "tcp",
// "tcp4", and "tcp6".
BindNetwork string
// Listener is the secure server network listener.
// either Listener or BindAddress/BindPort/BindNetwork is set,
// if Listener is set, use it and omit BindAddress/BindPort/BindNetwork.
Listener net.Listener
}
// Validate ensures that the insecure port values within the range of the port.
func (s *InsecureServingOptions) Validate() []error {
errors := []error{}
if s == nil {
return nil
}
if s.BindPort < 0 || s.BindPort > 32767 {
errors = append(errors, fmt.Errorf("--insecure-port %v must be between 0 and 32767, inclusive. 0 for turning off insecure (HTTP) port", s.BindPort))
}
return errors
}
// AddFlags adds flags related to insecure serving for controller manager to the specified FlagSet.
func (s *InsecureServingOptions) AddFlags(fs *pflag.FlagSet) {
if s == nil {
return
}
}
// AddDeprecatedFlags adds deprecated flags related to insecure serving for controller manager to the specified FlagSet.
// TODO: remove it until kops stop using `--address`
func (s *InsecureServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) {
if s == nil {
return
}
fs.IPVar(&s.BindAddress, "address", s.BindAddress,
"DEPRECATED: the IP address on which to listen for the --port port. See --bind-address instead.")
// MarkDeprecated hides the flag from the help. We don't want that:
// fs.MarkDeprecated("address", "see --bind-address instead.")
fs.IntVar(&s.BindPort, "port", s.BindPort, "DEPRECATED: the port on which to serve HTTP insecurely without authentication and authorization. If 0, don't serve HTTPS at all. See --secure-port instead.")
// MarkDeprecated hides the flag from the help. We don't want that:
// fs.MarkDeprecated("port", "see --secure-port instead.")
}
// ApplyTo adds InsecureServingOptions to the insecureserverinfo amd kube-controller manager configuration.
// Note: the double pointer allows to set the *InsecureServingInfo to nil without referencing the struct hosting this pointer.
func (s *InsecureServingOptions) ApplyTo(c **genericcontrollermanager.InsecureServingInfo, cfg *componentconfig.KubeControllerManagerConfiguration) error {
if s == nil {
return nil
}
if s.BindPort <= 0 {
return nil
}
if s.Listener == nil {
var err error
addr := net.JoinHostPort(s.BindAddress.String(), fmt.Sprintf("%d", s.BindPort))
s.Listener, s.BindPort, err = options.CreateListener(s.BindNetwork, addr)
if err != nil {
return fmt.Errorf("failed to create listener: %v", err)
}
}
*c = &genericcontrollermanager.InsecureServingInfo{
Listener: s.Listener,
}
// sync back to component config
// TODO: find more elegant way than synching back the values.
cfg.Port = int32(s.BindPort)
cfg.Address = s.BindAddress.String()
return nil
}

214
cmd/controller-manager/app/options/options.go Normal file
View File

@ -0,0 +1,214 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"net"
"time"
"github.com/cloudflare/cfssl/helpers"
"github.com/golang/glog"
"github.com/spf13/pflag"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
clientset "k8s.io/client-go/kubernetes"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/record"
genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/apis/componentconfig"
"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
)
// GenericControllerManagerOptions is the common structure for a controller manager. It works with NewGenericControllerManagerOptions
// and AddDefaultControllerFlags to create the common components of kube-controller-manager and cloud-controller-manager.
type GenericControllerManagerOptions struct {
// TODO: turn ComponentConfig into modular option structs. This is not generic.
ComponentConfig componentconfig.KubeControllerManagerConfiguration
InsecureServing *InsecureServingOptions
Master string
Kubeconfig string
}
const (
// These defaults are deprecated and exported so that we can warn if
// they are being used.
// DefaultClusterSigningCertFile is deprecated. Do not use.
DefaultClusterSigningCertFile = "/etc/kubernetes/ca/ca.pem"
// DefaultClusterSigningKeyFile is deprecated. Do not use.
DefaultClusterSigningKeyFile = "/etc/kubernetes/ca/ca.key"
)
// NewGenericControllerManagerOptions returns common/default configuration values for both
// the kube-controller-manager and the cloud-contoller-manager. Any common changes should
// be made here. Any individual changes should be made in that controller.
func NewGenericControllerManagerOptions(componentConfig componentconfig.KubeControllerManagerConfiguration) GenericControllerManagerOptions {
o := GenericControllerManagerOptions{
ComponentConfig: componentConfig,
InsecureServing: &InsecureServingOptions{
BindAddress: net.ParseIP(componentConfig.Address),
BindPort: int(componentConfig.Port),
BindNetwork: "tcp",
},
}
return o
}
// NewDefaultControllerManagerComponentConfig returns default kube-controller manager configuration object.
func NewDefaultControllerManagerComponentConfig(insecurePort int32) componentconfig.KubeControllerManagerConfiguration {
return componentconfig.KubeControllerManagerConfiguration{
Controllers: []string{"*"},
Port: insecurePort,
Address: "0.0.0.0",
ConcurrentEndpointSyncs: 5,
ConcurrentServiceSyncs: 1,
ConcurrentRCSyncs: 5,
ConcurrentRSSyncs: 5,
ConcurrentDaemonSetSyncs: 2,
ConcurrentJobSyncs: 5,
ConcurrentResourceQuotaSyncs: 5,
ConcurrentDeploymentSyncs: 5,
ConcurrentNamespaceSyncs: 10,
ConcurrentSATokenSyncs: 5,
RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second},
ResourceQuotaSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
NamespaceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
PVClaimBinderSyncPeriod: metav1.Duration{Duration: 15 * time.Second},
HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 30 * time.Second},
HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 3 * time.Minute},
HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute},
HorizontalPodAutoscalerTolerance: 0.1,
DeploymentControllerSyncPeriod: metav1.Duration{Duration: 30 * time.Second},
MinResyncPeriod: metav1.Duration{Duration: 12 * time.Hour},
RegisterRetryCount: 10,
PodEvictionTimeout: metav1.Duration{Duration: 5 * time.Minute},
NodeMonitorGracePeriod: metav1.Duration{Duration: 40 * time.Second},
NodeStartupGracePeriod: metav1.Duration{Duration: 60 * time.Second},
NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second},
ClusterName: "kubernetes",
NodeCIDRMaskSize: 24,
ConfigureCloudRoutes: true,
TerminatedPodGCThreshold: 12500,
VolumeConfiguration: componentconfig.VolumeConfiguration{
EnableHostPathProvisioning: false,
EnableDynamicProvisioning: true,
PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{
MaximumRetry: 3,
MinimumTimeoutNFS: 300,
IncrementTimeoutNFS: 30,
MinimumTimeoutHostPath: 60,
IncrementTimeoutHostPath: 30,
},
FlexVolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
},
ContentType: "application/vnd.kubernetes.protobuf",
KubeAPIQPS: 20.0,
KubeAPIBurst: 30,
LeaderElection: leaderelectionconfig.DefaultLeaderElectionConfiguration(),
ControllerStartInterval: metav1.Duration{Duration: 0 * time.Second},
EnableGarbageCollector: true,
ConcurrentGCSyncs: 20,
ClusterSigningCertFile: DefaultClusterSigningCertFile,
ClusterSigningKeyFile: DefaultClusterSigningKeyFile,
ClusterSigningDuration: metav1.Duration{Duration: helpers.OneYear},
ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 60 * time.Second},
EnableTaintManager: true,
HorizontalPodAutoscalerUseRESTClients: true,
}
}
// AddFlags adds common/default flags for both the kube and cloud Controller Manager Server to the
// specified FlagSet. Any common changes should be made here. Any individual changes should be made in that controller.
func (o *GenericControllerManagerOptions) AddFlags(fs *pflag.FlagSet) {
fs.BoolVar(&o.ComponentConfig.UseServiceAccountCredentials, "use-service-account-credentials", o.ComponentConfig.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.")
fs.StringVar(&o.ComponentConfig.CloudConfigFile, "cloud-config", o.ComponentConfig.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.")
fs.BoolVar(&o.ComponentConfig.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances. This is a legacy mode of operation and a cluster-id will be required in the future.")
fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release. A cluster-id will be required on cloud instances.")
fs.DurationVar(&o.ComponentConfig.RouteReconciliationPeriod.Duration, "route-reconciliation-period", o.ComponentConfig.RouteReconciliationPeriod.Duration, "The period for reconciling routes created for Nodes by cloud provider.")
fs.DurationVar(&o.ComponentConfig.MinResyncPeriod.Duration, "min-resync-period", o.ComponentConfig.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod.")
fs.DurationVar(&o.ComponentConfig.NodeMonitorPeriod.Duration, "node-monitor-period", o.ComponentConfig.NodeMonitorPeriod.Duration,
"The period for syncing NodeStatus in NodeController.")
fs.BoolVar(&o.ComponentConfig.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/")
fs.BoolVar(&o.ComponentConfig.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled.")
fs.StringVar(&o.ComponentConfig.ClusterName, "cluster-name", o.ComponentConfig.ClusterName, "The instance prefix for the cluster.")
fs.StringVar(&o.ComponentConfig.ClusterCIDR, "cluster-cidr", o.ComponentConfig.ClusterCIDR, "CIDR Range for Pods in cluster. Requires --allocate-node-cidrs to be true")
fs.BoolVar(&o.ComponentConfig.AllocateNodeCIDRs, "allocate-node-cidrs", false, "Should CIDRs for Pods be allocated and set on the cloud provider.")
fs.StringVar(&o.ComponentConfig.CIDRAllocatorType, "cidr-allocator-type", "RangeAllocator", "Type of CIDR allocator to use")
fs.BoolVar(&o.ComponentConfig.ConfigureCloudRoutes, "configure-cloud-routes", true, "Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider.")
fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
fs.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
fs.StringVar(&o.ComponentConfig.ContentType, "kube-api-content-type", o.ComponentConfig.ContentType, "Content type of requests sent to apiserver.")
fs.Float32Var(&o.ComponentConfig.KubeAPIQPS, "kube-api-qps", o.ComponentConfig.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver.")
fs.Int32Var(&o.ComponentConfig.KubeAPIBurst, "kube-api-burst", o.ComponentConfig.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver.")
fs.DurationVar(&o.ComponentConfig.ControllerStartInterval.Duration, "controller-start-interval", o.ComponentConfig.ControllerStartInterval.Duration, "Interval between starting controller managers.")
o.InsecureServing.AddFlags(fs)
o.InsecureServing.AddDeprecatedFlags(fs)
}
// ApplyTo fills up controller manager config with options and userAgent
func (o *GenericControllerManagerOptions) ApplyTo(c *genericcontrollermanager.Config, userAgent string) error {
c.ComponentConfig = o.ComponentConfig
if err := o.InsecureServing.ApplyTo(&c.InsecureServing, &c.ComponentConfig); err != nil {
return err
}
var err error
c.Kubeconfig, err = clientcmd.BuildConfigFromFlags(o.Master, o.Kubeconfig)
if err != nil {
return err
}
c.Kubeconfig.ContentConfig.ContentType = o.ComponentConfig.ContentType
c.Kubeconfig.QPS = o.ComponentConfig.KubeAPIQPS
c.Kubeconfig.Burst = int(o.ComponentConfig.KubeAPIBurst)
c.Client, err = clientset.NewForConfig(restclient.AddUserAgent(c.Kubeconfig, userAgent))
if err != nil {
return err
}
c.LeaderElectionClient = clientset.NewForConfigOrDie(restclient.AddUserAgent(c.Kubeconfig, "leader-election"))
c.EventRecorder = createRecorder(c.Client, userAgent)
return nil
}
// Validate checks GenericControllerManagerOptions and return a slice of found errors.
func (o *GenericControllerManagerOptions) Validate() []error {
errors := []error{}
errors = append(errors, o.InsecureServing.Validate()...)
// TODO: validate component config, master and kubeconfig
return errors
}
func createRecorder(kubeClient *kubernetes.Clientset, userAgent string) record.EventRecorder {
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartLogging(glog.Infof)
eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.CoreV1().RESTClient()).Events("")})
return eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: userAgent})
}

139
cmd/controller-manager/app/options/utils.go
View File

@ -1,139 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"github.com/cloudflare/cfssl/helpers"
"time"
"github.com/spf13/pflag"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/apis/componentconfig"
"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
)
// ControllerManagerServer is the common structure for a controller manager. It works with GetDefaultControllerOptions
// and AddDefaultControllerFlags to create the common components of kube-controller-manager and cloud-controller-manager.
type ControllerManagerServer struct {
componentconfig.KubeControllerManagerConfiguration
Master string
Kubeconfig string
}
const (
// These defaults are deprecated and exported so that we can warn if
// they are being used.
// DefaultClusterSigningCertFile is deprecated. Do not use.
DefaultClusterSigningCertFile = "/etc/kubernetes/ca/ca.pem"
// DefaultClusterSigningKeyFile is deprecated. Do not use.
DefaultClusterSigningKeyFile = "/etc/kubernetes/ca/ca.key"
)
// GetDefaultControllerOptions returns common/default configuration values for both
// the kube-controller-manager and the cloud-contoller-manager. Any common changes should
// be made here. Any individual changes should be made in that controller.
func GetDefaultControllerOptions(port int32) componentconfig.KubeControllerManagerConfiguration {
return componentconfig.KubeControllerManagerConfiguration{
Controllers: []string{"*"},
Port: port,
Address: "0.0.0.0",
ConcurrentEndpointSyncs: 5,
ConcurrentServiceSyncs: 1,
ConcurrentRCSyncs: 5,
ConcurrentRSSyncs: 5,
ConcurrentDaemonSetSyncs: 2,
ConcurrentJobSyncs: 5,
ConcurrentResourceQuotaSyncs: 5,
ConcurrentDeploymentSyncs: 5,
ConcurrentNamespaceSyncs: 10,
ConcurrentSATokenSyncs: 5,
RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second},
ResourceQuotaSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
NamespaceSyncPeriod: metav1.Duration{Duration: 5 * time.Minute},
PVClaimBinderSyncPeriod: metav1.Duration{Duration: 15 * time.Second},
HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 30 * time.Second},
HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 3 * time.Minute},
HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute},
HorizontalPodAutoscalerTolerance: 0.1,
DeploymentControllerSyncPeriod: metav1.Duration{Duration: 30 * time.Second},
MinResyncPeriod: metav1.Duration{Duration: 12 * time.Hour},
RegisterRetryCount: 10,
PodEvictionTimeout: metav1.Duration{Duration: 5 * time.Minute},
NodeMonitorGracePeriod: metav1.Duration{Duration: 40 * time.Second},
NodeStartupGracePeriod: metav1.Duration{Duration: 60 * time.Second},
NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second},
ClusterName: "kubernetes",
NodeCIDRMaskSize: 24,
ConfigureCloudRoutes: true,
TerminatedPodGCThreshold: 12500,
VolumeConfiguration: componentconfig.VolumeConfiguration{
EnableHostPathProvisioning: false,
EnableDynamicProvisioning: true,
PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{
MaximumRetry: 3,
MinimumTimeoutNFS: 300,
IncrementTimeoutNFS: 30,
MinimumTimeoutHostPath: 60,
IncrementTimeoutHostPath: 30,
},
FlexVolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
},
ContentType: "application/vnd.kubernetes.protobuf",
KubeAPIQPS: 20.0,
KubeAPIBurst: 30,
LeaderElection: leaderelectionconfig.DefaultLeaderElectionConfiguration(),
ControllerStartInterval: metav1.Duration{Duration: 0 * time.Second},
EnableGarbageCollector: true,
ConcurrentGCSyncs: 20,
ClusterSigningCertFile: DefaultClusterSigningCertFile,
ClusterSigningKeyFile: DefaultClusterSigningKeyFile,
ClusterSigningDuration: metav1.Duration{Duration: helpers.OneYear},
ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 60 * time.Second},
EnableTaintManager: true,
HorizontalPodAutoscalerUseRESTClients: true,
}
}
// AddDefaultControllerFlags adds common/default flags for both the kube and cloud Controller Manager Server to the
// specified FlagSet. Any common changes should be made here. Any individual changes should be made in that controller.
func AddDefaultControllerFlags(s *ControllerManagerServer, fs *pflag.FlagSet) {
fs.Int32Var(&s.Port, "port", s.Port, "The port that the controller-manager's http service runs on.")
fs.Var(componentconfig.IPVar{Val: &s.Address}, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces).")
fs.BoolVar(&s.UseServiceAccountCredentials, "use-service-account-credentials", s.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.")
fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.")
fs.BoolVar(&s.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances. This is a legacy mode of operation and a cluster-id will be required in the future.")
fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release. A cluster-id will be required on cloud instances.")
fs.DurationVar(&s.RouteReconciliationPeriod.Duration, "route-reconciliation-period", s.RouteReconciliationPeriod.Duration, "The period for reconciling routes created for Nodes by cloud provider.")
fs.DurationVar(&s.MinResyncPeriod.Duration, "min-resync-period", s.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod.")
fs.DurationVar(&s.NodeMonitorPeriod.Duration, "node-monitor-period", s.NodeMonitorPeriod.Duration,
"The period for syncing NodeStatus in NodeController.")
fs.BoolVar(&s.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/")
fs.BoolVar(&s.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled.")
fs.StringVar(&s.ClusterName, "cluster-name", s.ClusterName, "The instance prefix for the cluster.")
fs.StringVar(&s.ClusterCIDR, "cluster-cidr", s.ClusterCIDR, "CIDR Range for Pods in cluster. Requires --allocate-node-cidrs to be true")
fs.BoolVar(&s.AllocateNodeCIDRs, "allocate-node-cidrs", false, "Should CIDRs for Pods be allocated and set on the cloud provider.")
fs.StringVar(&s.CIDRAllocatorType, "cidr-allocator-type", "RangeAllocator", "Type of CIDR allocator to use")
fs.BoolVar(&s.ConfigureCloudRoutes, "configure-cloud-routes", true, "Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider.")
fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
fs.StringVar(&s.ContentType, "kube-api-content-type", s.ContentType, "Content type of requests sent to apiserver.")
fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver.")
fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver.")
fs.DurationVar(&s.ControllerStartInterval.Duration, "controller-start-interval", s.ControllerStartInterval.Duration, "Interval between starting controller managers.")
}

51
cmd/controller-manager/app/serve.go Normal file
View File

@ -0,0 +1,51 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package app
import (
"net/http"
"net/http/pprof"
goruntime "runtime"
"time"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/kubernetes/pkg/util/configz"
)
type serveFunc func(handler http.Handler, shutdownTimeout time.Duration, stopCh <-chan struct{}) error
// Serve creates a base handler chain for a controller manager. It runs the
// the chain with the given serveFunc.
func Serve(c *CompletedConfig, serveFunc serveFunc, stopCh <-chan struct{}) error {
mux := http.NewServeMux()
healthz.InstallHandler(mux)
if c.ComponentConfig.EnableProfiling {
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
if c.ComponentConfig.EnableContentionProfiling {
goruntime.SetBlockProfileRate(1)
}
}
configz.InstallHandler(mux)
mux.Handle("/metrics", prometheus.Handler())
return serveFunc(mux, 0, stopCh)
}

10
cmd/kube-controller-manager/app/autoscaling.go
View File

@ -38,7 +38,7 @@ func startHPAController(ctx ControllerContext) (bool, error) {
return false, nil
}
if ctx.Options.HorizontalPodAutoscalerUseRESTClients {
if ctx.ComponentConfig.HorizontalPodAutoscalerUseRESTClients {
// use the new-style clients if support for custom metrics is enabled
return startHPAControllerWithRESTClient(ctx)
}
@ -88,7 +88,7 @@ func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient me
replicaCalc := podautoscaler.NewReplicaCalculator(
metricsClient,
hpaClient.CoreV1(),
ctx.Options.HorizontalPodAutoscalerTolerance,
ctx.ComponentConfig.HorizontalPodAutoscalerTolerance,
)
go podautoscaler.NewHorizontalController(
hpaClientGoClient.CoreV1(),
@ -97,9 +97,9 @@ func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient me
restMapper,
replicaCalc,
ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
ctx.Options.HorizontalPodAutoscalerSyncPeriod.Duration,
ctx.Options.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration,
ctx.Options.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration,
ctx.ComponentConfig.HorizontalPodAutoscalerSyncPeriod.Duration,
ctx.ComponentConfig.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration,
ctx.ComponentConfig.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration,
).Run(ctx.Stop)
return true, nil
}

2
cmd/kube-controller-manager/app/batch.go
View File

@ -36,7 +36,7 @@ func startJobController(ctx ControllerContext) (bool, error) {
ctx.InformerFactory.Core().V1().Pods(),
ctx.InformerFactory.Batch().V1().Jobs(),
ctx.ClientBuilder.ClientOrDie("job-controller"),
).Run(int(ctx.Options.ConcurrentJobSyncs), ctx.Stop)
).Run(int(ctx.ComponentConfig.ConcurrentJobSyncs), ctx.Stop)
return true, nil
}

16
cmd/kube-controller-manager/app/certificates.go
View File

@ -37,7 +37,7 @@ func startCSRSigningController(ctx ControllerContext) (bool, error) {
if !ctx.AvailableResources[schema.GroupVersionResource{Group: "certificates.k8s.io", Version: "v1beta1", Resource: "certificatesigningrequests"}] {
return false, nil
}
if ctx.Options.ClusterSigningCertFile == "" || ctx.Options.ClusterSigningKeyFile == "" {
if ctx.ComponentConfig.ClusterSigningCertFile == "" || ctx.ComponentConfig.ClusterSigningKeyFile == "" {
return false, nil
}
@ -52,15 +52,15 @@ func startCSRSigningController(ctx ControllerContext) (bool, error) {
// bail out of startController without logging.
var keyFileExists, keyUsesDefault, certFileExists, certUsesDefault bool
_, err := os.Stat(ctx.Options.ClusterSigningCertFile)
_, err := os.Stat(ctx.ComponentConfig.ClusterSigningCertFile)
certFileExists = !os.IsNotExist(err)
certUsesDefault = (ctx.Options.ClusterSigningCertFile == cmoptions.DefaultClusterSigningCertFile)
certUsesDefault = (ctx.ComponentConfig.ClusterSigningCertFile == cmoptions.DefaultClusterSigningCertFile)
_, err = os.Stat(ctx.Options.ClusterSigningKeyFile)
_, err = os.Stat(ctx.ComponentConfig.ClusterSigningKeyFile)
keyFileExists = !os.IsNotExist(err)
keyUsesDefault = (ctx.Options.ClusterSigningKeyFile == cmoptions.DefaultClusterSigningKeyFile)
keyUsesDefault = (ctx.ComponentConfig.ClusterSigningKeyFile == cmoptions.DefaultClusterSigningKeyFile)
switch {
case (keyFileExists && keyUsesDefault) || (certFileExists && certUsesDefault):
@ -84,9 +84,9 @@ func startCSRSigningController(ctx ControllerContext) (bool, error) {
signer, err := signer.NewCSRSigningController(
c,
ctx.InformerFactory.Certificates().V1beta1().CertificateSigningRequests(),
ctx.Options.ClusterSigningCertFile,
ctx.Options.ClusterSigningKeyFile,
ctx.Options.ClusterSigningDuration.Duration,
ctx.ComponentConfig.ClusterSigningCertFile,
ctx.ComponentConfig.ClusterSigningKeyFile,
ctx.ComponentConfig.ClusterSigningDuration.Duration,
)
if err != nil {
return false, fmt.Errorf("failed to start certificate controller: %v", err)

55
cmd/kube-controller-manager/app/config/config.go Normal file
View File

@ -0,0 +1,55 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import (
"time"
genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app"
)
// ExtraConfig are part of Config, also can place your custom config here.
type ExtraConfig struct {
NodeStatusUpdateFrequency time.Duration
}
// Config is the main context object for the controller manager.
type Config struct {
Generic genericcontrollermanager.Config
Extra ExtraConfig
}
type completedConfig struct {
Generic genericcontrollermanager.CompletedConfig
Extra *ExtraConfig
}
// CompletedConfig same as Config, just to swap private object.
type CompletedConfig struct {
// Embed a private pointer that cannot be instantiated outside of this package.
*completedConfig
}
// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
func (c *Config) Complete() *CompletedConfig {
cc := completedConfig{
c.Generic.Complete(),
&c.Extra,
}
return &CompletedConfig{&cc}
}

189
cmd/kube-controller-manager/app/controllermanager.go
View File

@ -24,46 +24,34 @@ import (
"fmt"
"io/ioutil"
"math/rand"
"net"
"net/http"
"net/http/pprof"
"os"
goruntime "runtime"
"strconv"
"time"
"github.com/golang/glog"
"github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/api/core/v1"
"k8s.io/client-go/discovery"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/record"
certutil "k8s.io/client-go/util/cert"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
certutil "k8s.io/client-go/util/cert"
genericcontrollerconfig "k8s.io/kubernetes/cmd/controller-manager/app"
"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/apis/componentconfig"
"k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/controller"
serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
"k8s.io/kubernetes/pkg/serviceaccount"
"k8s.io/kubernetes/pkg/util/configz"
"k8s.io/kubernetes/pkg/version"
"github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
"github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/pkg/version/verflag"
)
@ -81,7 +69,7 @@ const (
// NewControllerManagerCommand creates a *cobra.Command object with default parameters
func NewControllerManagerCommand() *cobra.Command {
s := options.NewCMServer()
s := options.NewKubeControllerManagerOptions()
cmd := &cobra.Command{
Use: "kube-controller-manager",
Long: `The Kubernetes controller manager is a daemon that embeds
@ -94,7 +82,17 @@ Kubernetes today are the replication controller, endpoints controller, namespace
controller, and serviceaccounts controller.`,
Run: func(cmd *cobra.Command, args []string) {
verflag.PrintAndExitIfRequested()
Run(s)
c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List())
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
if err := Run(c.Complete()); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
},
}
s.AddFlags(cmd.Flags(), KnownControllers(), ControllersDisabledByDefault.List())
@ -105,59 +103,53 @@ controller, and serviceaccounts controller.`,
// ResyncPeriod returns a function which generates a duration each time it is
// invoked; this is so that multiple controllers don't get into lock-step and all
// hammer the apiserver with list requests simultaneously.
func ResyncPeriod(s *options.CMServer) func() time.Duration {
func ResyncPeriod(c *config.CompletedConfig) func() time.Duration {
return func() time.Duration {
factor := rand.Float64() + 1
return time.Duration(float64(s.MinResyncPeriod.Nanoseconds()) * factor)
return time.Duration(float64(c.Generic.ComponentConfig.MinResyncPeriod.Nanoseconds()) * factor)
}
}
// Run runs the CMServer. This should never exit.
func Run(s *options.CMServer) error {
// Run runs the KubeControllerManagerOptions. This should never exit.
func Run(c *config.CompletedConfig) error {
// To help debugging, immediately log version
glog.Infof("Version: %+v", version.Get())
if err := s.Validate(KnownControllers(), ControllersDisabledByDefault.List()); err != nil {
return err
}
if c, err := configz.New("componentconfig"); err == nil {
c.Set(s.KubeControllerManagerConfiguration)
if cfgz, err := configz.New("componentconfig"); err == nil {
cfgz.Set(c.Generic.ComponentConfig)
} else {
glog.Errorf("unable to register configz: %s", err)
glog.Errorf("unable to register configz: %c", err)
}
kubeClient, leaderElectionClient, kubeconfig, err := createClients(s)
if err != nil {
return err
// Start the controller manager HTTP server
stopCh := make(chan struct{})
if c.Generic.InsecureServing != nil {
if err := genericcontrollerconfig.Serve(&c.Generic, c.Generic.InsecureServing.Serve, stopCh); err != nil {
return err
}
}
if s.Port >= 0 {
go startHTTP(s)
}
recorder := createRecorder(kubeClient)
run := func(stop <-chan struct{}) {
rootClientBuilder := controller.SimpleControllerClientBuilder{
ClientConfig: kubeconfig,
ClientConfig: c.Generic.Kubeconfig,
}
var clientBuilder controller.ControllerClientBuilder
if s.UseServiceAccountCredentials {
if len(s.ServiceAccountKeyFile) == 0 {
// It's possible another controller process is creating the tokens for us.
if c.Generic.ComponentConfig.UseServiceAccountCredentials {
if len(c.Generic.ComponentConfig.ServiceAccountKeyFile) == 0 {
// It'c possible another controller process is creating the tokens for us.
// If one isn't, we'll timeout and exit when our client builder is unable to create the tokens.
glog.Warningf("--use-service-account-credentials was specified without providing a --service-account-private-key-file")
}
clientBuilder = controller.SAControllerClientBuilder{
ClientConfig: restclient.AnonymousClientConfig(kubeconfig),
CoreClient: kubeClient.CoreV1(),
AuthenticationClient: kubeClient.AuthenticationV1(),
ClientConfig: restclient.AnonymousClientConfig(c.Generic.Kubeconfig),
CoreClient: c.Generic.Client.CoreV1(),
AuthenticationClient: c.Generic.Client.AuthenticationV1(),
Namespace: "kube-system",
}
} else {
clientBuilder = rootClientBuilder
}
ctx, err := CreateControllerContext(s, rootClientBuilder, clientBuilder, stop)
ctx, err := CreateControllerContext(c, rootClientBuilder, clientBuilder, stop)
if err != nil {
glog.Fatalf("error building controller context: %v", err)
}
@ -173,7 +165,7 @@ func Run(s *options.CMServer) error {
select {}
}
if !s.LeaderElection.LeaderElect {
if !c.Generic.ComponentConfig.LeaderElection.LeaderElect {
run(wait.NeverStop)
panic("unreachable")
}
@ -182,16 +174,16 @@ func Run(s *options.CMServer) error {
if err != nil {
return err
}
// add a uniquifier so that two processes on the same host don't accidentally both become active
id = id + "_" + string(uuid.NewUUID())
rl, err := resourcelock.New(s.LeaderElection.ResourceLock,
rl, err := resourcelock.New(c.Generic.ComponentConfig.LeaderElection.ResourceLock,
"kube-system",
"kube-controller-manager",
leaderElectionClient.CoreV1(),
c.Generic.LeaderElectionClient.CoreV1(),
resourcelock.ResourceLockConfig{
Identity: id,
EventRecorder: recorder,
EventRecorder: c.Generic.EventRecorder,
})
if err != nil {
glog.Fatalf("error creating lock: %v", err)
@ -199,9 +191,9 @@ func Run(s *options.CMServer) error {
leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{
Lock: rl,
LeaseDuration: s.LeaderElection.LeaseDuration.Duration,
RenewDeadline: s.LeaderElection.RenewDeadline.Duration,
RetryPeriod: s.LeaderElection.RetryPeriod.Duration,
LeaseDuration: c.Generic.ComponentConfig.LeaderElection.LeaseDuration.Duration,
RenewDeadline: c.Generic.ComponentConfig.LeaderElection.RenewDeadline.Duration,
RetryPeriod: c.Generic.ComponentConfig.LeaderElection.RetryPeriod.Duration,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: run,
OnStoppedLeading: func() {
@ -212,53 +204,6 @@ func Run(s *options.CMServer) error {
panic("unreachable")
}
func startHTTP(s *options.CMServer) {
mux := http.NewServeMux()
healthz.InstallHandler(mux)
if s.EnableProfiling {
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
if s.EnableContentionProfiling {
goruntime.SetBlockProfileRate(1)
}
}
configz.InstallHandler(mux)
mux.Handle("/metrics", prometheus.Handler())
server := &http.Server{
Addr: net.JoinHostPort(s.Address, strconv.Itoa(int(s.Port))),
Handler: mux,
}
glog.Fatal(server.ListenAndServe())
}
func createRecorder(kubeClient *clientset.Clientset) record.EventRecorder {
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartLogging(glog.Infof)
eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.CoreV1().RESTClient()).Events("")})
return eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: "controller-manager"})
}
func createClients(s *options.CMServer) (*clientset.Clientset, *clientset.Clientset, *restclient.Config, error) {
kubeconfig, err := clientcmd.BuildConfigFromFlags(s.Master, s.Kubeconfig)
if err != nil {
return nil, nil, nil, err
}
kubeconfig.ContentConfig.ContentType = s.ContentType
// Override kubeconfig qps/burst settings from flags
kubeconfig.QPS = s.KubeAPIQPS
kubeconfig.Burst = int(s.KubeAPIBurst)
kubeClient, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, "controller-manager"))
if err != nil {
glog.Fatalf("Invalid API configuration: %v", err)
}
leaderElectionClient := clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "leader-election"))
return kubeClient, leaderElectionClient, kubeconfig, nil
}
type ControllerContext struct {
// ClientBuilder will provide a client for this controller to use
ClientBuilder controller.ControllerClientBuilder
@ -267,7 +212,7 @@ type ControllerContext struct {
InformerFactory informers.SharedInformerFactory
// Options provides access to init options for a given controller
Options options.CMServer
ComponentConfig componentconfig.KubeControllerManagerConfiguration
// AvailableResources is a map listing currently available resources
AvailableResources map[schema.GroupVersionResource]bool
@ -287,10 +232,15 @@ type ControllerContext struct {
// InformersStarted is closed after all of the controllers have been initialized and are running. After this point it is safe,
// for an individual controller to start the shared informers. Before it is closed, they should not.
InformersStarted chan struct{}
// ResyncPeriod generates a duration each time it is invoked; this is so that
// multiple controllers don't get into lock-step and all hammer the apiserver
// with list requests simultaneously.
ResyncPeriod func() time.Duration
}
func (c ControllerContext) IsControllerEnabled(name string) bool {
return IsControllerEnabled(name, ControllersDisabledByDefault, c.Options.Controllers...)
return IsControllerEnabled(name, ControllersDisabledByDefault, c.ComponentConfig.Controllers...)
}
func IsControllerEnabled(name string, disabledByDefaultControllers sets.String, controllers ...string) bool {
@ -446,7 +396,7 @@ func GetAvailableResources(clientBuilder controller.ControllerClientBuilder) (ma
// CreateControllerContext creates a context struct containing references to resources needed by the
// controllers such as the cloud provider and clientBuilder. rootClientBuilder is only used for
// the shared-informers client and token controller.
func CreateControllerContext(s *options.CMServer, rootClientBuilder, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}) (ControllerContext, error) {
func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}) (ControllerContext, error) {
versionedClient := rootClientBuilder.ClientOrDie("shared-informers")
sharedInformers := informers.NewSharedInformerFactory(versionedClient, ResyncPeriod(s)())
@ -455,8 +405,8 @@ func CreateControllerContext(s *options.CMServer, rootClientBuilder, clientBuild
return ControllerContext{}, err
}
cloud, loopMode, err := createCloudProvider(s.CloudProvider, s.ExternalCloudVolumePlugin,
s.CloudConfigFile, s.AllowUntaggedCloud, sharedInformers)
cloud, loopMode, err := createCloudProvider(s.Generic.ComponentConfig.CloudProvider, s.Generic.ComponentConfig.ExternalCloudVolumePlugin,
s.Generic.ComponentConfig.CloudConfigFile, s.Generic.ComponentConfig.AllowUntaggedCloud, sharedInformers)
if err != nil {
return ControllerContext{}, err
}
@ -464,12 +414,13 @@ func CreateControllerContext(s *options.CMServer, rootClientBuilder, clientBuild
ctx := ControllerContext{
ClientBuilder: clientBuilder,
InformerFactory: sharedInformers,
Options: *s,
ComponentConfig: s.Generic.ComponentConfig,
AvailableResources: availableResources,
Cloud: cloud,
LoopMode: loopMode,
Stop: stop,
InformersStarted: make(chan struct{}),
ResyncPeriod: ResyncPeriod(s),
}
return ctx, nil
}
@ -493,7 +444,7 @@ func StartControllers(ctx ControllerContext, startSATokenController InitFunc, co
continue
}
time.Sleep(wait.Jitter(ctx.Options.ControllerStartInterval.Duration, ControllerStartJitter))
time.Sleep(wait.Jitter(ctx.ComponentConfig.ControllerStartInterval.Duration, ControllerStartJitter))
glog.V(1).Infof("Starting %q", controllerName)
started, err := initFn(ctx)
@ -524,23 +475,23 @@ func (c serviceAccountTokenControllerStarter) startServiceAccountTokenController
return false, nil
}
if len(ctx.Options.ServiceAccountKeyFile) == 0 {
if len(ctx.ComponentConfig.ServiceAccountKeyFile) == 0 {
glog.Warningf("%q is disabled because there is no private key", saTokenControllerName)
return false, nil
}
privateKey, err := certutil.PrivateKeyFromFile(ctx.Options.ServiceAccountKeyFile)
privateKey, err := certutil.PrivateKeyFromFile(ctx.ComponentConfig.ServiceAccountKeyFile)
if err != nil {
return true, fmt.Errorf("error reading key for service account token controller: %v", err)
}
var rootCA []byte
if ctx.Options.RootCAFile != "" {
rootCA, err = ioutil.ReadFile(ctx.Options.RootCAFile)
if ctx.ComponentConfig.RootCAFile != "" {
rootCA, err = ioutil.ReadFile(ctx.ComponentConfig.RootCAFile)
if err != nil {
return true, fmt.Errorf("error reading root-ca-file at %s: %v", ctx.Options.RootCAFile, err)
return true, fmt.Errorf("error reading root-ca-file at %s: %v", ctx.ComponentConfig.RootCAFile, err)
}
if _, err := certutil.ParseCertsPEM(rootCA); err != nil {
return true, fmt.Errorf("error parsing root-ca-file at %s: %v", ctx.Options.RootCAFile, err)
return true, fmt.Errorf("error parsing root-ca-file at %s: %v", ctx.ComponentConfig.RootCAFile, err)
}
} else {
rootCA = c.rootClientBuilder.ConfigOrDie("tokens-controller").CAData
@ -558,7 +509,7 @@ func (c serviceAccountTokenControllerStarter) startServiceAccountTokenController
if err != nil {
return true, fmt.Errorf("error creating Tokens controller: %v", err)
}
go controller.Run(int(ctx.Options.ConcurrentSATokenSyncs), ctx.Stop)
go controller.Run(int(ctx.ComponentConfig.ConcurrentSATokenSyncs), ctx.Stop)
// start the first set of informers now so that other controllers can start
ctx.InformerFactory.Start(ctx.Stop)

94
cmd/kube-controller-manager/app/core.go
View File

@ -68,33 +68,33 @@ func startServiceController(ctx ControllerContext) (bool, error) {
ctx.ClientBuilder.ClientOrDie("service-controller"),
ctx.InformerFactory.Core().V1().Services(),
ctx.InformerFactory.Core().V1().Nodes(),
ctx.Options.ClusterName,
ctx.ComponentConfig.ClusterName,
)
if err != nil {
// This error shouldn't fail. It lives like this as a legacy.
glog.Errorf("Failed to start service controller: %v", err)
return false, nil
}
go serviceController.Run(ctx.Stop, int(ctx.Options.ConcurrentServiceSyncs))
go serviceController.Run(ctx.Stop, int(ctx.ComponentConfig.ConcurrentServiceSyncs))
return true, nil
}
func startNodeIpamController(ctx ControllerContext) (bool, error) {
var clusterCIDR *net.IPNet = nil
var serviceCIDR *net.IPNet = nil
if ctx.Options.AllocateNodeCIDRs {
if ctx.ComponentConfig.AllocateNodeCIDRs {
var err error
if len(strings.TrimSpace(ctx.Options.ClusterCIDR)) != 0 {
_, clusterCIDR, err = net.ParseCIDR(ctx.Options.ClusterCIDR)
if len(strings.TrimSpace(ctx.ComponentConfig.ClusterCIDR)) != 0 {
_, clusterCIDR, err = net.ParseCIDR(ctx.ComponentConfig.ClusterCIDR)
if err != nil {
glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", ctx.Options.ClusterCIDR, err)
glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", ctx.ComponentConfig.ClusterCIDR, err)
}
}
if len(strings.TrimSpace(ctx.Options.ServiceCIDR)) != 0 {
_, serviceCIDR, err = net.ParseCIDR(ctx.Options.ServiceCIDR)
if len(strings.TrimSpace(ctx.ComponentConfig.ServiceCIDR)) != 0 {
_, serviceCIDR, err = net.ParseCIDR(ctx.ComponentConfig.ServiceCIDR)
if err != nil {
glog.Warningf("Unsuccessful parsing of service CIDR %v: %v", ctx.Options.ServiceCIDR, err)
glog.Warningf("Unsuccessful parsing of service CIDR %v: %v", ctx.ComponentConfig.ServiceCIDR, err)
}
}
}
@ -105,9 +105,9 @@ func startNodeIpamController(ctx ControllerContext) (bool, error) {
ctx.ClientBuilder.ClientOrDie("node-controller"),
clusterCIDR,
serviceCIDR,
int(ctx.Options.NodeCIDRMaskSize),
ctx.Options.AllocateNodeCIDRs,
ipam.CIDRAllocatorType(ctx.Options.CIDRAllocatorType),
int(ctx.ComponentConfig.NodeCIDRMaskSize),
ctx.ComponentConfig.AllocateNodeCIDRs,
ipam.CIDRAllocatorType(ctx.ComponentConfig.CIDRAllocatorType),
)
if err != nil {
return true, err
@ -123,15 +123,15 @@ func startNodeLifecycleController(ctx ControllerContext) (bool, error) {
ctx.InformerFactory.Extensions().V1beta1().DaemonSets(),
ctx.Cloud,
ctx.ClientBuilder.ClientOrDie("node-controller"),
ctx.Options.NodeMonitorPeriod.Duration,
ctx.Options.NodeStartupGracePeriod.Duration,
ctx.Options.NodeMonitorGracePeriod.Duration,
ctx.Options.PodEvictionTimeout.Duration,
ctx.Options.NodeEvictionRate,
ctx.Options.SecondaryNodeEvictionRate,
ctx.Options.LargeClusterSizeThreshold,
ctx.Options.UnhealthyZoneThreshold,
ctx.Options.EnableTaintManager,
ctx.ComponentConfig.NodeMonitorPeriod.Duration,
ctx.ComponentConfig.NodeStartupGracePeriod.Duration,
ctx.ComponentConfig.NodeMonitorGracePeriod.Duration,
ctx.ComponentConfig.PodEvictionTimeout.Duration,
ctx.ComponentConfig.NodeEvictionRate,
ctx.ComponentConfig.SecondaryNodeEvictionRate,
ctx.ComponentConfig.LargeClusterSizeThreshold,
ctx.ComponentConfig.UnhealthyZoneThreshold,
ctx.ComponentConfig.EnableTaintManager,
utilfeature.DefaultFeatureGate.Enabled(features.TaintBasedEvictions),
utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition),
)
@ -143,8 +143,8 @@ func startNodeLifecycleController(ctx ControllerContext) (bool, error) {
}
func startRouteController(ctx ControllerContext) (bool, error) {
if !ctx.Options.AllocateNodeCIDRs || !ctx.Options.ConfigureCloudRoutes {
glog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", ctx.Options.AllocateNodeCIDRs, ctx.Options.ConfigureCloudRoutes)
if !ctx.ComponentConfig.AllocateNodeCIDRs || !ctx.ComponentConfig.ConfigureCloudRoutes {
glog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", ctx.ComponentConfig.AllocateNodeCIDRs, ctx.ComponentConfig.ConfigureCloudRoutes)
return false, nil
}
if ctx.Cloud == nil {
@ -156,27 +156,27 @@ func startRouteController(ctx ControllerContext) (bool, error) {
glog.Warning("configure-cloud-routes is set, but cloud provider does not support routes. Will not configure cloud provider routes.")
return false, nil
}
_, clusterCIDR, err := net.ParseCIDR(ctx.Options.ClusterCIDR)
_, clusterCIDR, err := net.ParseCIDR(ctx.ComponentConfig.ClusterCIDR)
if err != nil {
glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", ctx.Options.ClusterCIDR, err)
glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", ctx.ComponentConfig.ClusterCIDR, err)
}
routeController := routecontroller.New(routes, ctx.ClientBuilder.ClientOrDie("route-controller"), ctx.InformerFactory.Core().V1().Nodes(), ctx.Options.ClusterName, clusterCIDR)
go routeController.Run(ctx.Stop, ctx.Options.RouteReconciliationPeriod.Duration)
routeController := routecontroller.New(routes, ctx.ClientBuilder.ClientOrDie("route-controller"), ctx.InformerFactory.Core().V1().Nodes(), ctx.ComponentConfig.ClusterName, clusterCIDR)
go routeController.Run(ctx.Stop, ctx.ComponentConfig.RouteReconciliationPeriod.Duration)
return true, nil
}
func startPersistentVolumeBinderController(ctx ControllerContext) (bool, error) {
params := persistentvolumecontroller.ControllerParameters{
KubeClient: ctx.ClientBuilder.ClientOrDie("persistent-volume-binder"),
SyncPeriod: ctx.Options.PVClaimBinderSyncPeriod.Duration,
VolumePlugins: ProbeControllerVolumePlugins(ctx.Cloud, ctx.Options.VolumeConfiguration),
SyncPeriod: ctx.ComponentConfig.PVClaimBinderSyncPeriod.Duration,
VolumePlugins: ProbeControllerVolumePlugins(ctx.Cloud, ctx.ComponentConfig.VolumeConfiguration),
Cloud: ctx.Cloud,
ClusterName: ctx.Options.ClusterName,
ClusterName: ctx.ComponentConfig.ClusterName,
VolumeInformer: ctx.InformerFactory.Core().V1().PersistentVolumes(),
ClaimInformer: ctx.InformerFactory.Core().V1().PersistentVolumeClaims(),
ClassInformer: ctx.InformerFactory.Storage().V1().StorageClasses(),
PodInformer: ctx.InformerFactory.Core().V1().Pods(),
EnableDynamicProvisioning: ctx.Options.VolumeConfiguration.EnableDynamicProvisioning,
EnableDynamicProvisioning: ctx.ComponentConfig.VolumeConfiguration.EnableDynamicProvisioning,
}
volumeController, volumeControllerErr := persistentvolumecontroller.NewController(params)
if volumeControllerErr != nil {
@ -187,7 +187,7 @@ func startPersistentVolumeBinderController(ctx ControllerContext) (bool, error)
}
func startAttachDetachController(ctx ControllerContext) (bool, error) {
if ctx.Options.ReconcilerSyncLoopPeriod.Duration < time.Second {
if ctx.ComponentConfig.ReconcilerSyncLoopPeriod.Duration < time.Second {
return true, fmt.Errorf("Duration time must be greater than one second as set via command line option reconcile-sync-loop-period.")
}
attachDetachController, attachDetachControllerErr :=
@ -199,9 +199,9 @@ func startAttachDetachController(ctx ControllerContext) (bool, error) {
ctx.InformerFactory.Core().V1().PersistentVolumes(),
ctx.Cloud,
ProbeAttachableVolumePlugins(),
GetDynamicPluginProber(ctx.Options.VolumeConfiguration),
ctx.Options.DisableAttachDetachReconcilerSync,
ctx.Options.ReconcilerSyncLoopPeriod.Duration,
GetDynamicPluginProber(ctx.ComponentConfig.VolumeConfiguration),
ctx.ComponentConfig.DisableAttachDetachReconcilerSync,
ctx.ComponentConfig.ReconcilerSyncLoopPeriod.Duration,
attachdetach.DefaultTimerConfig,
)
if attachDetachControllerErr != nil {
@ -218,7 +218,7 @@ func startVolumeExpandController(ctx ControllerContext) (bool, error) {
ctx.InformerFactory.Core().V1().PersistentVolumeClaims(),
ctx.InformerFactory.Core().V1().PersistentVolumes(),
ctx.Cloud,
ProbeExpandableVolumePlugins(ctx.Options.VolumeConfiguration))
ProbeExpandableVolumePlugins(ctx.ComponentConfig.VolumeConfiguration))
if expandControllerErr != nil {
return true, fmt.Errorf("Failed to start volume expand controller : %v", expandControllerErr)
@ -235,7 +235,7 @@ func startEndpointController(ctx ControllerContext) (bool, error) {
ctx.InformerFactory.Core().V1().Services(),
ctx.InformerFactory.Core().V1().Endpoints(),
ctx.ClientBuilder.ClientOrDie("endpoint-controller"),
).Run(int(ctx.Options.ConcurrentEndpointSyncs), ctx.Stop)
).Run(int(ctx.ComponentConfig.ConcurrentEndpointSyncs), ctx.Stop)
return true, nil
}
@ -245,7 +245,7 @@ func startReplicationController(ctx ControllerContext) (bool, error) {
ctx.InformerFactory.Core().V1().ReplicationControllers(),
ctx.ClientBuilder.ClientOrDie("replication-controller"),
replicationcontroller.BurstReplicas,
).Run(int(ctx.Options.ConcurrentRCSyncs), ctx.Stop)
).Run(int(ctx.ComponentConfig.ConcurrentRCSyncs), ctx.Stop)
return true, nil
}
@ -253,7 +253,7 @@ func startPodGCController(ctx ControllerContext) (bool, error) {
go podgc.NewPodGC(
ctx.ClientBuilder.ClientOrDie("pod-garbage-collector"),
ctx.InformerFactory.Core().V1().Pods(),
int(ctx.Options.TerminatedPodGCThreshold),
int(ctx.ComponentConfig.TerminatedPodGCThreshold),
).Run(ctx.Stop)
return true, nil
}
@ -267,9 +267,9 @@ func startResourceQuotaController(ctx ControllerContext) (bool, error) {
resourceQuotaControllerOptions := &resourcequotacontroller.ResourceQuotaControllerOptions{
QuotaClient: resourceQuotaControllerClient.CoreV1(),
ResourceQuotaInformer: ctx.InformerFactory.Core().V1().ResourceQuotas(),
ResyncPeriod: controller.StaticResyncPeriodFunc(ctx.Options.ResourceQuotaSyncPeriod.Duration),
ResyncPeriod: controller.StaticResyncPeriodFunc(ctx.ComponentConfig.ResourceQuotaSyncPeriod.Duration),
InformerFactory: ctx.InformerFactory,
ReplenishmentResyncPeriod: ResyncPeriod(&ctx.Options),
ReplenishmentResyncPeriod: ctx.ResyncPeriod,
DiscoveryFunc: discoveryFunc,
IgnoredResourcesFunc: quotaConfiguration.IgnoredResources,
InformersStarted: ctx.InformersStarted,
@ -285,7 +285,7 @@ func startResourceQuotaController(ctx ControllerContext) (bool, error) {
if err != nil {
return false, err
}
go resourceQuotaController.Run(int(ctx.Options.ConcurrentResourceQuotaSyncs), ctx.Stop)
go resourceQuotaController.Run(int(ctx.ComponentConfig.ConcurrentResourceQuotaSyncs), ctx.Stop)
// Periodically the quota controller to detect new resource types
go resourceQuotaController.Sync(discoveryFunc, 30*time.Second, ctx.Stop)
@ -313,10 +313,10 @@ func startNamespaceController(ctx ControllerContext) (bool, error) {
namespaceClientPool,
discoverResourcesFn,
ctx.InformerFactory.Core().V1().Namespaces(),
ctx.Options.NamespaceSyncPeriod.Duration,
ctx.ComponentConfig.NamespaceSyncPeriod.Duration,
v1.FinalizerKubernetes,
)
go namespaceController.Run(int(ctx.Options.ConcurrentNamespaceSyncs), ctx.Stop)
go namespaceController.Run(int(ctx.ComponentConfig.ConcurrentNamespaceSyncs), ctx.Stop)
return true, nil
}
@ -344,7 +344,7 @@ func startTTLController(ctx ControllerContext) (bool, error) {
}
func startGarbageCollectorController(ctx ControllerContext) (bool, error) {
if !ctx.Options.EnableGarbageCollector {
if !ctx.ComponentConfig.EnableGarbageCollector {
return false, nil
}
@ -367,7 +367,7 @@ func startGarbageCollectorController(ctx ControllerContext) (bool, error) {
// Get an initial set of deletable resources to prime the garbage collector.
deletableResources := garbagecollector.GetDeletableResources(discoveryClient)
ignoredResources := make(map[schema.GroupResource]struct{})
for _, r := range ctx.Options.GCIgnoredResources {
for _, r := range ctx.ComponentConfig.GCIgnoredResources {
ignoredResources[schema.GroupResource{Group: r.Group, Resource: r.Resource}] = struct{}{}
}
garbageCollector, err := garbagecollector.NewGarbageCollector(
@ -384,7 +384,7 @@ func startGarbageCollectorController(ctx ControllerContext) (bool, error) {
}
// Start the garbage collector.
workers := int(ctx.Options.ConcurrentGCSyncs)
workers := int(ctx.ComponentConfig.ConcurrentGCSyncs)
go garbageCollector.Run(workers, ctx.Stop)
// Periodically refresh the RESTMapper with new discovery information and sync

6
cmd/kube-controller-manager/app/extensions.go
View File

@ -43,7 +43,7 @@ func startDaemonSetController(ctx ControllerContext) (bool, error) {
if err != nil {
return true, fmt.Errorf("error creating DaemonSets controller: %v", err)
}
go dsc.Run(int(ctx.Options.ConcurrentDaemonSetSyncs), ctx.Stop)
go dsc.Run(int(ctx.ComponentConfig.ConcurrentDaemonSetSyncs), ctx.Stop)
return true, nil
}
@ -60,7 +60,7 @@ func startDeploymentController(ctx ControllerContext) (bool, error) {
if err != nil {
return true, fmt.Errorf("error creating Deployment controller: %v", err)
}
go dc.Run(int(ctx.Options.ConcurrentDeploymentSyncs), ctx.Stop)
go dc.Run(int(ctx.ComponentConfig.ConcurrentDeploymentSyncs), ctx.Stop)
return true, nil
}
@ -73,6 +73,6 @@ func startReplicaSetController(ctx ControllerContext) (bool, error) {
ctx.InformerFactory.Core().V1().Pods(),
ctx.ClientBuilder.ClientOrDie("replicaset-controller"),
replicaset.BurstReplicas,
).Run(int(ctx.Options.ConcurrentRSSyncs), ctx.Stop)
).Run(int(ctx.ComponentConfig.ConcurrentRSSyncs), ctx.Stop)
return true, nil
}

168
cmd/kube-controller-manager/app/options/options.go
View File

@ -26,6 +26,7 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
utilfeature "k8s.io/apiserver/pkg/util/feature"
cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
kubecontrollerconfig "k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
"k8s.io/kubernetes/pkg/apis/componentconfig"
"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
"k8s.io/kubernetes/pkg/controller/garbagecollector"
@ -37,117 +38,124 @@ import (
"github.com/spf13/pflag"
)
// CMServer is the main context object for the controller manager.
type CMServer struct {
cmoptions.ControllerManagerServer
// KubeControllerManagerOptions is the main context object for the controller manager.
type KubeControllerManagerOptions struct {
Generic cmoptions.GenericControllerManagerOptions
}
// NewCMServer creates a new CMServer with a default config.
func NewCMServer() *CMServer {
// NewKubeControllerManagerOptions creates a new KubeControllerManagerOptions with a default config.
func NewKubeControllerManagerOptions() *KubeControllerManagerOptions {
componentConfig := cmoptions.NewDefaultControllerManagerComponentConfig(ports.InsecureKubeControllerManagerPort)
s := KubeControllerManagerOptions{
// The common/default are kept in 'cmd/kube-controller-manager/app/options/util.go'.
// Please make common changes there but put anything kube-controller specific here.
Generic: cmoptions.NewGenericControllerManagerOptions(componentConfig),
}
gcIgnoredResources := make([]componentconfig.GroupResource, 0, len(garbagecollector.DefaultIgnoredResources()))
for r := range garbagecollector.DefaultIgnoredResources() {
gcIgnoredResources = append(gcIgnoredResources, componentconfig.GroupResource{Group: r.Group, Resource: r.Resource})
}
s.Generic.ComponentConfig.GCIgnoredResources = gcIgnoredResources
s.Generic.ComponentConfig.LeaderElection.LeaderElect = true
s := CMServer{
// The common/default are kept in 'cmd/kube-controller-manager/app/options/util.go'.
// Please make common changes there but put anything kube-controller specific here.
ControllerManagerServer: cmoptions.ControllerManagerServer{
KubeControllerManagerConfiguration: cmoptions.GetDefaultControllerOptions(ports.ControllerManagerPort),
},
}
s.KubeControllerManagerConfiguration.GCIgnoredResources = gcIgnoredResources
s.LeaderElection.LeaderElect = true
return &s
}
// AddFlags adds flags for a specific CMServer to the specified FlagSet
func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabledByDefaultControllers []string) {
cmoptions.AddDefaultControllerFlags(&s.ControllerManagerServer, fs)
// AddFlags adds flags for a specific KubeControllerManagerOptions to the specified FlagSet
func (s *KubeControllerManagerOptions) AddFlags(fs *pflag.FlagSet, allControllers []string, disabledByDefaultControllers []string) {
s.Generic.AddFlags(fs)
fs.StringSliceVar(&s.Controllers, "controllers", s.Controllers, fmt.Sprintf(""+
fs.StringSliceVar(&s.Generic.ComponentConfig.Controllers, "controllers", s.Generic.ComponentConfig.Controllers, fmt.Sprintf(""+
"A list of controllers to enable. '*' enables all on-by-default controllers, 'foo' enables the controller "+
"named 'foo', '-foo' disables the controller named 'foo'.\nAll controllers: %s\nDisabled-by-default controllers: %s",
strings.Join(allControllers, ", "), strings.Join(disabledByDefaultControllers, ", ")))
fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider for cloud services. Empty string for no provider.")
fs.StringVar(&s.ExternalCloudVolumePlugin, "external-cloud-volume-plugin", s.ExternalCloudVolumePlugin, "The plugin to use when cloud provider is set to external. Can be empty, should only be set when cloud-provider is external. Currently used to allow node and volume controllers to work for in tree cloud providers.")
fs.Int32Var(&s.ConcurrentEndpointSyncs, "concurrent-endpoint-syncs", s.ConcurrentEndpointSyncs, "The number of endpoint syncing operations that will be done concurrently. Larger number = faster endpoint updating, but more CPU (and network) load")
fs.Int32Var(&s.ConcurrentServiceSyncs, "concurrent-service-syncs", s.ConcurrentServiceSyncs, "The number of services that are allowed to sync concurrently. Larger number = more responsive service management, but more CPU (and network) load")
fs.Int32Var(&s.ConcurrentRCSyncs, "concurrent_rc_syncs", s.ConcurrentRCSyncs, "The number of replication controllers that are allowed to sync concurrently. Larger number = more responsive replica management, but more CPU (and network) load")
fs.Int32Var(&s.ConcurrentRSSyncs, "concurrent-replicaset-syncs", s.ConcurrentRSSyncs, "The number of replica sets that are allowed to sync concurrently. Larger number = more responsive replica management, but more CPU (and network) load")
fs.StringVar(&s.Generic.ComponentConfig.CloudProvider, "cloud-provider", s.Generic.ComponentConfig.CloudProvider, "The provider for cloud services. Empty string for no provider.")
fs.StringVar(&s.Generic.ComponentConfig.ExternalCloudVolumePlugin, "external-cloud-volume-plugin", s.Generic.ComponentConfig.ExternalCloudVolumePlugin, "The plugin to use when cloud provider is set to external. Can be empty, should only be set when cloud-provider is external. Currently used to allow node and volume controllers to work for in tree cloud providers.")
fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentEndpointSyncs, "concurrent-endpoint-syncs", s.Generic.ComponentConfig.ConcurrentEndpointSyncs, "The number of endpoint syncing operations that will be done concurrently. Larger number = faster endpoint updating, but more CPU (and network) load")
fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentServiceSyncs, "concurrent-service-syncs", s.Generic.ComponentConfig.ConcurrentServiceSyncs, "The number of services that are allowed to sync concurrently. Larger number = more responsive service management, but more CPU (and network) load")
fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentRCSyncs, "concurrent_rc_syncs", s.Generic.ComponentConfig.ConcurrentRCSyncs, "The number of replication controllers that are allowed to sync concurrently. Larger number = more responsive replica management, but more CPU (and network) load")
fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentRSSyncs, "concurrent-replicaset-syncs", s.Generic.ComponentConfig.ConcurrentRSSyncs, "The number of replica sets that are allowed to sync concurrently. Larger number = more responsive replica management, but more CPU (and network) load")
fs.Int32Var(&s.ConcurrentResourceQuotaSyncs, "concurrent-resource-quota-syncs", s.ConcurrentResourceQuotaSyncs, "The number of resource quotas that are allowed to sync concurrently. Larger number = more responsive quota management, but more CPU (and network) load")
fs.Int32Var(&s.ConcurrentDeploymentSyncs, "concurrent-deployment-syncs", s.ConcurrentDeploymentSyncs, "The number of deployment objects that are allowed to sync concurrently. Larger number = more responsive deployments, but more CPU (and network) load")
fs.Int32Var(&s.ConcurrentNamespaceSyncs, "concurrent-namespace-syncs", s.ConcurrentNamespaceSyncs, "The number of namespace objects that are allowed to sync concurrently. Larger number = more responsive namespace termination, but more CPU (and network) load")
fs.Int32Var(&s.ConcurrentSATokenSyncs, "concurrent-serviceaccount-token-syncs", s.ConcurrentSATokenSyncs, "The number of service account token objects that are allowed to sync concurrently. Larger number = more responsive token generation, but more CPU (and network) load")
fs.DurationVar(&s.NodeSyncPeriod.Duration, "node-sync-period", 0, ""+
fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentResourceQuotaSyncs, "concurrent-resource-quota-syncs", s.Generic.ComponentConfig.ConcurrentResourceQuotaSyncs, "The number of resource quotas that are allowed to sync concurrently. Larger number = more responsive quota management, but more CPU (and network) load")
fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentDeploymentSyncs, "concurrent-deployment-syncs", s.Generic.ComponentConfig.ConcurrentDeploymentSyncs, "The number of deployment objects that are allowed to sync concurrently. Larger number = more responsive deployments, but more CPU (and network) load")
fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentNamespaceSyncs, "concurrent-namespace-syncs", s.Generic.ComponentConfig.ConcurrentNamespaceSyncs, "The number of namespace objects that are allowed to sync concurrently. Larger number = more responsive namespace termination, but more CPU (and network) load")
fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentSATokenSyncs, "concurrent-serviceaccount-token-syncs", s.Generic.ComponentConfig.ConcurrentSATokenSyncs, "The number of service account token objects that are allowed to sync concurrently. Larger number = more responsive token generation, but more CPU (and network) load")
fs.DurationVar(&s.Generic.ComponentConfig.NodeSyncPeriod.Duration, "node-sync-period", 0, ""+
"This flag is deprecated and will be removed in future releases. See node-monitor-period for Node health checking or "+
"route-reconciliation-period for cloud provider's route configuration settings.")
fs.MarkDeprecated("node-sync-period", "This flag is currently no-op and will be deleted.")
fs.DurationVar(&s.ResourceQuotaSyncPeriod.Duration, "resource-quota-sync-period", s.ResourceQuotaSyncPeriod.Duration, "The period for syncing quota usage status in the system")
fs.DurationVar(&s.NamespaceSyncPeriod.Duration, "namespace-sync-period", s.NamespaceSyncPeriod.Duration, "The period for syncing namespace life-cycle updates")
fs.DurationVar(&s.PVClaimBinderSyncPeriod.Duration, "pvclaimbinder-sync-period", s.PVClaimBinderSyncPeriod.Duration, "The period for syncing persistent volumes and persistent volume claims")
fs.StringVar(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, "pv-recycler-pod-template-filepath-nfs", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, "The file path to a pod definition used as a template for NFS persistent volume recycling")
fs.Int32Var(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS, "pv-recycler-minimum-timeout-nfs", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS, "The minimum ActiveDeadlineSeconds to use for an NFS Recycler pod")
fs.Int32Var(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS, "pv-recycler-increment-timeout-nfs", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS, "the increment of time added per Gi to ActiveDeadlineSeconds for an NFS scrubber pod")
fs.StringVar(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath, "pv-recycler-pod-template-filepath-hostpath", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath, "The file path to a pod definition used as a template for HostPath persistent volume recycling. This is for development and testing only and will not work in a multi-node cluster.")
fs.Int32Var(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutHostPath, "pv-recycler-minimum-timeout-hostpath", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutHostPath, "The minimum ActiveDeadlineSeconds to use for a HostPath Recycler pod. This is for development and testing only and will not work in a multi-node cluster.")
fs.Int32Var(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutHostPath, "pv-recycler-timeout-increment-hostpath", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutHostPath, "the increment of time added per Gi to ActiveDeadlineSeconds for a HostPath scrubber pod. This is for development and testing only and will not work in a multi-node cluster.")
fs.BoolVar(&s.VolumeConfiguration.EnableHostPathProvisioning, "enable-hostpath-provisioner", s.VolumeConfiguration.EnableHostPathProvisioning, "Enable HostPath PV provisioning when running without a cloud provider. This allows testing and development of provisioning features. HostPath provisioning is not supported in any way, won't work in a multi-node cluster, and should not be used for anything other than testing or development.")
fs.BoolVar(&s.VolumeConfiguration.EnableDynamicProvisioning, "enable-dynamic-provisioning", s.VolumeConfiguration.EnableDynamicProvisioning, "Enable dynamic provisioning for environments that support it.")
fs.StringVar(&s.VolumeConfiguration.FlexVolumePluginDir, "flex-volume-plugin-dir", s.VolumeConfiguration.FlexVolumePluginDir, "Full path of the directory in which the flex volume plugin should search for additional third party volume plugins.")
fs.Int32Var(&s.TerminatedPodGCThreshold, "terminated-pod-gc-threshold", s.TerminatedPodGCThreshold, "Number of terminated pods that can exist before the terminated pod garbage collector starts deleting terminated pods. If <= 0, the terminated pod garbage collector is disabled.")
fs.DurationVar(&s.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", s.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.")
fs.DurationVar(&s.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-upscale-delay", s.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "The period since last upscale, before another upscale can be performed in horizontal pod autoscaler.")
fs.DurationVar(&s.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-downscale-delay", s.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "The period since last downscale, before another downscale can be performed in horizontal pod autoscaler.")
fs.Float64Var(&s.HorizontalPodAutoscalerTolerance, "horizontal-pod-autoscaler-tolerance", s.HorizontalPodAutoscalerTolerance, "The minimum change (from 1.0) in the desired-to-actual metrics ratio for the horizontal pod autoscaler to consider scaling.")
fs.DurationVar(&s.DeploymentControllerSyncPeriod.Duration, "deployment-controller-sync-period", s.DeploymentControllerSyncPeriod.Duration, "Period for syncing the deployments.")
fs.DurationVar(&s.PodEvictionTimeout.Duration, "pod-eviction-timeout", s.PodEvictionTimeout.Duration, "The grace period for deleting pods on failed nodes.")
fs.Float32Var(&s.DeletingPodsQps, "deleting-pods-qps", 0.1, "Number of nodes per second on which pods are deleted in case of node failure.")
fs.DurationVar(&s.Generic.ComponentConfig.ResourceQuotaSyncPeriod.Duration, "resource-quota-sync-period", s.Generic.ComponentConfig.ResourceQuotaSyncPeriod.Duration, "The period for syncing quota usage status in the system")
fs.DurationVar(&s.Generic.ComponentConfig.NamespaceSyncPeriod.Duration, "namespace-sync-period", s.Generic.ComponentConfig.NamespaceSyncPeriod.Duration, "The period for syncing namespace life-cycle updates")
fs.DurationVar(&s.Generic.ComponentConfig.PVClaimBinderSyncPeriod.Duration, "pvclaimbinder-sync-period", s.Generic.ComponentConfig.PVClaimBinderSyncPeriod.Duration, "The period for syncing persistent volumes and persistent volume claims")
fs.StringVar(&s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, "pv-recycler-pod-template-filepath-nfs", s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, "The file path to a pod definition used as a template for NFS persistent volume recycling")
fs.Int32Var(&s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS, "pv-recycler-minimum-timeout-nfs", s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS, "The minimum ActiveDeadlineSeconds to use for an NFS Recycler pod")
fs.Int32Var(&s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS, "pv-recycler-increment-timeout-nfs", s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS, "the increment of time added per Gi to ActiveDeadlineSeconds for an NFS scrubber pod")
fs.StringVar(&s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath, "pv-recycler-pod-template-filepath-hostpath", s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath, "The file path to a pod definition used as a template for HostPath persistent volume recycling. This is for development and testing only and will not work in a multi-node cluster.")
fs.Int32Var(&s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutHostPath, "pv-recycler-minimum-timeout-hostpath", s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutHostPath, "The minimum ActiveDeadlineSeconds to use for a HostPath Recycler pod. This is for development and testing only and will not work in a multi-node cluster.")
fs.Int32Var(&s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutHostPath, "pv-recycler-timeout-increment-hostpath", s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutHostPath, "the increment of time added per Gi to ActiveDeadlineSeconds for a HostPath scrubber pod. This is for development and testing only and will not work in a multi-node cluster.")
fs.BoolVar(&s.Generic.ComponentConfig.VolumeConfiguration.EnableHostPathProvisioning, "enable-hostpath-provisioner", s.Generic.ComponentConfig.VolumeConfiguration.EnableHostPathProvisioning, "Enable HostPath PV provisioning when running without a cloud provider. This allows testing and development of provisioning features. HostPath provisioning is not supported in any way, won't work in a multi-node cluster, and should not be used for anything other than testing or development.")
fs.BoolVar(&s.Generic.ComponentConfig.VolumeConfiguration.EnableDynamicProvisioning, "enable-dynamic-provisioning", s.Generic.ComponentConfig.VolumeConfiguration.EnableDynamicProvisioning, "Enable dynamic provisioning for environments that support it.")
fs.StringVar(&s.Generic.ComponentConfig.VolumeConfiguration.FlexVolumePluginDir, "flex-volume-plugin-dir", s.Generic.ComponentConfig.VolumeConfiguration.FlexVolumePluginDir, "Full path of the directory in which the flex volume plugin should search for additional third party volume plugins.")
fs.Int32Var(&s.Generic.ComponentConfig.TerminatedPodGCThreshold, "terminated-pod-gc-threshold", s.Generic.ComponentConfig.TerminatedPodGCThreshold, "Number of terminated pods that can exist before the terminated pod garbage collector starts deleting terminated pods. If <= 0, the terminated pod garbage collector is disabled.")
fs.DurationVar(&s.Generic.ComponentConfig.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", s.Generic.ComponentConfig.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.")
fs.DurationVar(&s.Generic.ComponentConfig.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-upscale-delay", s.Generic.ComponentConfig.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "The period since last upscale, before another upscale can be performed in horizontal pod autoscaler.")
fs.DurationVar(&s.Generic.ComponentConfig.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-downscale-delay", s.Generic.ComponentConfig.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "The period since last downscale, before another downscale can be performed in horizontal pod autoscaler.")
fs.Float64Var(&s.Generic.ComponentConfig.HorizontalPodAutoscalerTolerance, "horizontal-pod-autoscaler-tolerance", s.Generic.ComponentConfig.HorizontalPodAutoscalerTolerance, "The minimum change (from 1.0) in the desired-to-actual metrics ratio for the horizontal pod autoscaler to consider scaling.")
fs.DurationVar(&s.Generic.ComponentConfig.DeploymentControllerSyncPeriod.Duration, "deployment-controller-sync-period", s.Generic.ComponentConfig.DeploymentControllerSyncPeriod.Duration, "Period for syncing the deployments.")
fs.DurationVar(&s.Generic.ComponentConfig.PodEvictionTimeout.Duration, "pod-eviction-timeout", s.Generic.ComponentConfig.PodEvictionTimeout.Duration, "The grace period for deleting pods on failed nodes.")
fs.Float32Var(&s.Generic.ComponentConfig.DeletingPodsQps, "deleting-pods-qps", 0.1, "Number of nodes per second on which pods are deleted in case of node failure.")
fs.MarkDeprecated("deleting-pods-qps", "This flag is currently no-op and will be deleted.")
fs.Int32Var(&s.DeletingPodsBurst, "deleting-pods-burst", 0, "Number of nodes on which pods are bursty deleted in case of node failure. For more details look into RateLimiter.")
fs.Int32Var(&s.Generic.ComponentConfig.DeletingPodsBurst, "deleting-pods-burst", 0, "Number of nodes on which pods are bursty deleted in case of node failure. For more details look into RateLimiter.")
fs.MarkDeprecated("deleting-pods-burst", "This flag is currently no-op and will be deleted.")
fs.Int32Var(&s.RegisterRetryCount, "register-retry-count", s.RegisterRetryCount, ""+
fs.Int32Var(&s.Generic.ComponentConfig.RegisterRetryCount, "register-retry-count", s.Generic.ComponentConfig.RegisterRetryCount, ""+
"The number of retries for initial node registration. Retry interval equals node-sync-period.")
fs.MarkDeprecated("register-retry-count", "This flag is currently no-op and will be deleted.")
fs.DurationVar(&s.NodeMonitorGracePeriod.Duration, "node-monitor-grace-period", s.NodeMonitorGracePeriod.Duration,
fs.DurationVar(&s.Generic.ComponentConfig.NodeMonitorGracePeriod.Duration, "node-monitor-grace-period", s.Generic.ComponentConfig.NodeMonitorGracePeriod.Duration,
"Amount of time which we allow running Node to be unresponsive before marking it unhealthy. "+
"Must be N times more than kubelet's nodeStatusUpdateFrequency, "+
"where N means number of retries allowed for kubelet to post node status.")
fs.DurationVar(&s.NodeStartupGracePeriod.Duration, "node-startup-grace-period", s.NodeStartupGracePeriod.Duration,
fs.DurationVar(&s.Generic.ComponentConfig.NodeStartupGracePeriod.Duration, "node-startup-grace-period", s.Generic.ComponentConfig.NodeStartupGracePeriod.Duration,
"Amount of time which we allow starting Node to be unresponsive before marking it unhealthy.")
fs.StringVar(&s.ServiceAccountKeyFile, "service-account-private-key-file", s.ServiceAccountKeyFile, "Filename containing a PEM-encoded private RSA or ECDSA key used to sign service account tokens.")
fs.StringVar(&s.ClusterSigningCertFile, "cluster-signing-cert-file", s.ClusterSigningCertFile, "Filename containing a PEM-encoded X509 CA certificate used to issue cluster-scoped certificates")
fs.StringVar(&s.ClusterSigningKeyFile, "cluster-signing-key-file", s.ClusterSigningKeyFile, "Filename containing a PEM-encoded RSA or ECDSA private key used to sign cluster-scoped certificates")
fs.DurationVar(&s.ClusterSigningDuration.Duration, "experimental-cluster-signing-duration", s.ClusterSigningDuration.Duration, "The length of duration signed certificates will be given.")
fs.StringVar(&s.Generic.ComponentConfig.ServiceAccountKeyFile, "service-account-private-key-file", s.Generic.ComponentConfig.ServiceAccountKeyFile, "Filename containing a PEM-encoded private RSA or ECDSA key used to sign service account tokens.")
fs.StringVar(&s.Generic.ComponentConfig.ClusterSigningCertFile, "cluster-signing-cert-file", s.Generic.ComponentConfig.ClusterSigningCertFile, "Filename containing a PEM-encoded X509 CA certificate used to issue cluster-scoped certificates")
fs.StringVar(&s.Generic.ComponentConfig.ClusterSigningKeyFile, "cluster-signing-key-file", s.Generic.ComponentConfig.ClusterSigningKeyFile, "Filename containing a PEM-encoded RSA or ECDSA private key used to sign cluster-scoped certificates")
fs.DurationVar(&s.Generic.ComponentConfig.ClusterSigningDuration.Duration, "experimental-cluster-signing-duration", s.Generic.ComponentConfig.ClusterSigningDuration.Duration, "The length of duration signed certificates will be given.")
var dummy string
fs.MarkDeprecated("insecure-experimental-approve-all-kubelet-csrs-for-group", "This flag does nothing.")
fs.StringVar(&dummy, "insecure-experimental-approve-all-kubelet-csrs-for-group", "", "This flag does nothing.")
fs.StringVar(&s.ServiceCIDR, "service-cluster-ip-range", s.ServiceCIDR, "CIDR Range for Services in cluster. Requires --allocate-node-cidrs to be true")
fs.Int32Var(&s.NodeCIDRMaskSize, "node-cidr-mask-size", s.NodeCIDRMaskSize, "Mask size for node cidr in cluster.")
fs.StringVar(&s.RootCAFile, "root-ca-file", s.RootCAFile, "If set, this root certificate authority will be included in service account's token secret. This must be a valid PEM-encoded CA bundle.")
fs.BoolVar(&s.EnableGarbageCollector, "enable-garbage-collector", s.EnableGarbageCollector, "Enables the generic garbage collector. MUST be synced with the corresponding flag of the kube-apiserver.")
fs.Int32Var(&s.ConcurrentGCSyncs, "concurrent-gc-syncs", s.ConcurrentGCSyncs, "The number of garbage collector workers that are allowed to sync concurrently.")
fs.Int32Var(&s.LargeClusterSizeThreshold, "large-cluster-size-threshold", 50, "Number of nodes from which NodeController treats the cluster as large for the eviction logic purposes. --secondary-node-eviction-rate is implicitly overridden to 0 for clusters this size or smaller.")
fs.Float32Var(&s.UnhealthyZoneThreshold, "unhealthy-zone-threshold", 0.55, "Fraction of Nodes in a zone which needs to be not Ready (minimum 3) for zone to be treated as unhealthy. ")
fs.BoolVar(&s.DisableAttachDetachReconcilerSync, "disable-attach-detach-reconcile-sync", false, "Disable volume attach detach reconciler sync. Disabling this may cause volumes to be mismatched with pods. Use wisely.")
fs.DurationVar(&s.ReconcilerSyncLoopPeriod.Duration, "attach-detach-reconcile-sync-period", s.ReconcilerSyncLoopPeriod.Duration, "The reconciler sync wait time between volume attach detach. This duration must be larger than one second, and increasing this value from the default may allow for volumes to be mismatched with pods.")
fs.BoolVar(&s.EnableTaintManager, "enable-taint-manager", s.EnableTaintManager, "WARNING: Beta feature. If set to true enables NoExecute Taints and will evict all not-tolerating Pod running on Nodes tainted with this kind of Taints.")
fs.BoolVar(&s.HorizontalPodAutoscalerUseRESTClients, "horizontal-pod-autoscaler-use-rest-clients", s.HorizontalPodAutoscalerUseRESTClients, "WARNING: alpha feature. If set to true, causes the horizontal pod autoscaler controller to use REST clients through the kube-aggregator, instead of using the legacy metrics client through the API server proxy. This is required for custom metrics support in the horizontal pod autoscaler.")
fs.Float32Var(&s.NodeEvictionRate, "node-eviction-rate", 0.1, "Number of nodes per second on which pods are deleted in case of node failure when a zone is healthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters.")
fs.Float32Var(&s.SecondaryNodeEvictionRate, "secondary-node-eviction-rate", 0.01, "Number of nodes per second on which pods are deleted in case of node failure when a zone is unhealthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters. This value is implicitly overridden to 0 if the cluster size is smaller than --large-cluster-size-threshold.")
fs.StringVar(&s.Generic.ComponentConfig.ServiceCIDR, "service-cluster-ip-range", s.Generic.ComponentConfig.ServiceCIDR, "CIDR Range for Services in cluster. Requires --allocate-node-cidrs to be true")
fs.Int32Var(&s.Generic.ComponentConfig.NodeCIDRMaskSize, "node-cidr-mask-size", s.Generic.ComponentConfig.NodeCIDRMaskSize, "Mask size for node cidr in cluster.")
fs.StringVar(&s.Generic.ComponentConfig.RootCAFile, "root-ca-file", s.Generic.ComponentConfig.RootCAFile, "If set, this root certificate authority will be included in service account's token secret. This must be a valid PEM-encoded CA bundle.")
fs.BoolVar(&s.Generic.ComponentConfig.EnableGarbageCollector, "enable-garbage-collector", s.Generic.ComponentConfig.EnableGarbageCollector, "Enables the generic garbage collector. MUST be synced with the corresponding flag of the kube-apiserver.")
fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentGCSyncs, "concurrent-gc-syncs", s.Generic.ComponentConfig.ConcurrentGCSyncs, "The number of garbage collector workers that are allowed to sync concurrently.")
fs.Int32Var(&s.Generic.ComponentConfig.LargeClusterSizeThreshold, "large-cluster-size-threshold", 50, "Number of nodes from which NodeController treats the cluster as large for the eviction logic purposes. --secondary-node-eviction-rate is implicitly overridden to 0 for clusters this size or smaller.")
fs.Float32Var(&s.Generic.ComponentConfig.UnhealthyZoneThreshold, "unhealthy-zone-threshold", 0.55, "Fraction of Nodes in a zone which needs to be not Ready (minimum 3) for zone to be treated as unhealthy. ")
fs.BoolVar(&s.Generic.ComponentConfig.DisableAttachDetachReconcilerSync, "disable-attach-detach-reconcile-sync", false, "Disable volume attach detach reconciler sync. Disabling this may cause volumes to be mismatched with pods. Use wisely.")
fs.DurationVar(&s.Generic.ComponentConfig.ReconcilerSyncLoopPeriod.Duration, "attach-detach-reconcile-sync-period", s.Generic.ComponentConfig.ReconcilerSyncLoopPeriod.Duration, "The reconciler sync wait time between volume attach detach. This duration must be larger than one second, and increasing this value from the default may allow for volumes to be mismatched with pods.")
fs.BoolVar(&s.Generic.ComponentConfig.EnableTaintManager, "enable-taint-manager", s.Generic.ComponentConfig.EnableTaintManager, "WARNING: Beta feature. If set to true enables NoExecute Taints and will evict all not-tolerating Pod running on Nodes tainted with this kind of Taints.")
fs.BoolVar(&s.Generic.ComponentConfig.HorizontalPodAutoscalerUseRESTClients, "horizontal-pod-autoscaler-use-rest-clients", s.Generic.ComponentConfig.HorizontalPodAutoscalerUseRESTClients, "WARNING: alpha feature. If set to true, causes the horizontal pod autoscaler controller to use REST clients through the kube-aggregator, instead of using the legacy metrics client through the API server proxy. This is required for custom metrics support in the horizontal pod autoscaler.")
fs.Float32Var(&s.Generic.ComponentConfig.NodeEvictionRate, "node-eviction-rate", 0.1, "Number of nodes per second on which pods are deleted in case of node failure when a zone is healthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters.")
fs.Float32Var(&s.Generic.ComponentConfig.SecondaryNodeEvictionRate, "secondary-node-eviction-rate", 0.01, "Number of nodes per second on which pods are deleted in case of node failure when a zone is unhealthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters. This value is implicitly overridden to 0 if the cluster size is smaller than --large-cluster-size-threshold.")
leaderelectionconfig.BindFlags(&s.LeaderElection, fs)
leaderelectionconfig.BindFlags(&s.Generic.ComponentConfig.LeaderElection, fs)
utilfeature.DefaultFeatureGate.AddFlag(fs)
}
// ApplyTo fills up controller manager config with options.
func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) error {
err := s.Generic.ApplyTo(&c.Generic, "controller-manager")
return err
}
// Validate is used to validate the options and config before launching the controller manager
func (s *CMServer) Validate(allControllers []string, disabledByDefaultControllers []string) error {
func (s *KubeControllerManagerOptions) Validate(allControllers []string, disabledByDefaultControllers []string) error {
var errs []error
allControllersSet := sets.NewString(allControllers...)
for _, controller := range s.Controllers {
for _, controller := range s.Generic.ComponentConfig.Controllers {
if controller == "*" {
continue
}
@ -162,3 +170,17 @@ func (s *CMServer) Validate(allControllers []string, disabledByDefaultController
return utilerrors.NewAggregate(errs)
}
// Config return a controller manager config objective
func (s KubeControllerManagerOptions) Config(allControllers []string, disabledByDefaultControllers []string) (*kubecontrollerconfig.Config, error) {
if err := s.Validate(allControllers, disabledByDefaultControllers); err != nil {
return nil, err
}
c := &kubecontrollerconfig.Config{}
if err := s.ApplyTo(c); err != nil {
return nil, err
}
return c, nil
}

22
cmd/kube-controller-manager/app/options/options_test.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package options
import (
"net"
"reflect"
"sort"
"testing"
@ -32,7 +33,7 @@ import (
func TestAddFlags(t *testing.T) {
f := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError)
s := NewCMServer()
s := NewKubeControllerManagerOptions()
s.AddFlags(f, []string{""}, []string{""})
args := []string{
@ -107,13 +108,13 @@ func TestAddFlags(t *testing.T) {
f.Parse(args)
// Sort GCIgnoredResources because it's built from a map, which means the
// insertion order is random.
sort.Sort(sortedGCIgnoredResources(s.GCIgnoredResources))
sort.Sort(sortedGCIgnoredResources(s.Generic.ComponentConfig.GCIgnoredResources))
expected := &CMServer{
ControllerManagerServer: cmoptions.ControllerManagerServer{
KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
Port: 10000,
Address: "192.168.4.10",
expected := &KubeControllerManagerOptions{
Generic: cmoptions.GenericControllerManagerOptions{
ComponentConfig: componentconfig.KubeControllerManagerConfiguration{
Port: 10252, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config
Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config
AllocateNodeCIDRs: true,
CloudConfigFile: "/cloud-config",
CloudProvider: "gce",
@ -204,6 +205,11 @@ func TestAddFlags(t *testing.T) {
HorizontalPodAutoscalerUseRESTClients: true,
UseServiceAccountCredentials: true,
},
InsecureServing: &cmoptions.InsecureServingOptions{
BindAddress: net.ParseIP("192.168.4.10"),
BindPort: int(10000),
BindNetwork: "tcp",
},
Kubeconfig: "/kubeconfig",
Master: "192.168.4.20",
},
@ -211,7 +217,7 @@ func TestAddFlags(t *testing.T) {
// Sort GCIgnoredResources because it's built from a map, which means the
// insertion order is random.
sort.Sort(sortedGCIgnoredResources(expected.GCIgnoredResources))
sort.Sort(sortedGCIgnoredResources(expected.Generic.ComponentConfig.GCIgnoredResources))
if !reflect.DeepEqual(expected, s) {
t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", diff.ObjectReflectDiff(expected, s))

3
cmd/kube-controller-manager/controller-manager.go
View File

@ -22,14 +22,13 @@ package main
import (
goflag "flag"
"fmt"
"math/rand"
"os"
"time"
"github.com/spf13/pflag"
"fmt"
utilflag "k8s.io/apiserver/pkg/util/flag"
"k8s.io/apiserver/pkg/util/logs"
"k8s.io/kubernetes/cmd/kube-controller-manager/app"

2
cmd/kubeadm/app/preflight/checks.go
View File

@ -530,7 +530,7 @@ func (eac ExtraArgsCheck) Check() (warnings, errors []error) {
}
if len(eac.ControllerManagerExtraArgs) > 0 {
flags := pflag.NewFlagSet("", pflag.ContinueOnError)
s := cmoptions.NewCMServer()
s := cmoptions.NewKubeControllerManagerOptions()
s.AddFlags(flags, []string{}, []string{})
warnings = append(warnings, argsCheck("kube-controller-manager", eac.ControllerManagerExtraArgs, flags)...)
}

8
pkg/master/ports/ports.go
View File

@ -26,12 +26,12 @@ const (
// SchedulerPort is the default port for the scheduler status server.
// May be overridden by a flag at startup.
SchedulerPort = 10251
// ControllerManagerPort is the default port for the controller manager status server.
// InsecureKubeControllerManagerPort is the default port for the controller manager status server.
// May be overridden by a flag at startup.
ControllerManagerPort = 10252
// CloudControllerManagerPort is the default port for the cloud controller manager server.
InsecureControllerManagerPort = 10252
// InsecureCloudControllerManagerPort is the default port for the cloud controller manager server.
// This value may be overridden by a flag at startup.
CloudControllerManagerPort = 10253
InsecureCloudControllerManagerPort = 10253
// KubeletReadOnlyPort exposes basic read-only services from the kubelet.
// May be overridden by a flag at startup.
// This is necessary for heapster to collect monitoring stats from the kubelet

2
pkg/registry/core/rest/storage_core.go
View File

@ -239,7 +239,7 @@ type componentStatusStorage struct {
func (s componentStatusStorage) serversToValidate() map[string]*componentstatus.Server {
serversToValidate := map[string]*componentstatus.Server{
"controller-manager": {Addr: "127.0.0.1", Port: ports.ControllerManagerPort, Path: "/healthz"},
"controller-manager": {Addr: "127.0.0.1", Port: ports.InsecureKubeControllerManagerPort, Path: "/healthz"},
"scheduler": {Addr: "127.0.0.1", Port: ports.SchedulerPort, Path: "/healthz"},
}

2
test/e2e/apps/daemon_restart.go
View File

@ -249,7 +249,7 @@ var _ = SIGDescribe("DaemonRestart [Disruptive]", func() {
// Requires master ssh access.
framework.SkipUnlessProviderIs("gce", "aws")
restarter := NewRestartConfig(
framework.GetMasterHost(), "kube-controller", ports.ControllerManagerPort, restartPollInterval, restartTimeout)
framework.GetMasterHost(), "kube-controller", ports.InsecureKubeControllerManagerPort, restartPollInterval, restartTimeout)
restarter.restart()
// The intent is to ensure the replication controller manager has observed and reported status of

2
test/e2e/framework/metrics/metrics_grabber.go
View File

@ -158,7 +158,7 @@ func (g *MetricsGrabber) GrabFromControllerManager() (ControllerManagerMetrics,
if !g.registeredMaster {
return ControllerManagerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ControllerManager's metrics gathering.")
}
output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName), metav1.NamespaceSystem, ports.ControllerManagerPort)
output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName), metav1.NamespaceSystem, ports.InsecureKubeControllerManagerPort)
if err != nil {
return ControllerManagerMetrics{}, err
}

2
test/e2e/framework/util.go
View File

@ -3967,7 +3967,7 @@ func RestartControllerManager() error {
}
func WaitForControllerManagerUp() error {
cmd := "curl http://localhost:" + strconv.Itoa(ports.ControllerManagerPort) + "/healthz"
cmd := "curl http://localhost:" + strconv.Itoa(ports.InsecureKubeControllerManagerPort) + "/healthz"
for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(5 * time.Second) {
result, err := SSH(cmd, net.JoinHostPort(GetMasterHost(), sshPort), TestContext.Provider)
if err != nil || result.Code != 0 {

2
test/e2e/network/firewall.go
View File

@ -170,7 +170,7 @@ var _ = SIGDescribe("Firewall rule", func() {
nodeAddrs := framework.NodeAddresses(nodes, v1.NodeExternalIP)
Expect(len(nodeAddrs)).NotTo(BeZero())
masterAddr := framework.GetMasterAddress(cs)
flag, _ := framework.TestNotReachableHTTPTimeout(masterAddr, ports.ControllerManagerPort, framework.FirewallTestTcpTimeout)
flag, _ := framework.TestNotReachableHTTPTimeout(masterAddr, ports.InsecureKubeControllerManagerPort, framework.FirewallTestTcpTimeout)
Expect(flag).To(BeTrue())
flag, _ = framework.TestNotReachableHTTPTimeout(masterAddr, ports.SchedulerPort, framework.FirewallTestTcpTimeout)
Expect(flag).To(BeTrue())