From cbdc9b671f33b0f0679e790cc462b25d1476a3af Mon Sep 17 00:00:00 2001 From: Janet Kuo Date: Tue, 14 Aug 2018 15:31:32 -0700 Subject: [PATCH] Make number of workers configurable --- api/api-rules/violation_exceptions.list | 2 + .../app/controllermanager.go | 2 +- cmd/kube-controller-manager/app/core.go | 2 +- cmd/kube-controller-manager/app/options/BUILD | 1 + .../app/options/options.go | 9 +++ .../app/options/options_test.go | 4 ++ .../app/options/ttlafterfinishedcontroller.go | 58 +++++++++++++++++++ pkg/controller/apis/config/types.go | 10 ++++ .../apis/config/v1alpha1/defaults.go | 3 + .../config/v1alpha1/types.go | 10 ++++ 10 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 cmd/kube-controller-manager/app/options/ttlafterfinishedcontroller.go diff --git a/api/api-rules/violation_exceptions.list b/api/api-rules/violation_exceptions.list index df8bfded96..8add67dc09 100644 --- a/api/api-rules/violation_exceptions.list +++ b/api/api-rules/violation_exceptions.list @@ -126,6 +126,7 @@ API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,K API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,ResourceQuotaController API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,SAController API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,ServiceController +API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,KubeControllerManagerConfiguration,TTLAfterFinishedController API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NamespaceControllerConfiguration,NamespaceSyncPeriod API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NamespaceControllerConfiguration,ConcurrentNamespaceSyncs API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,NodeIPAMControllerConfiguration,ServiceCIDR @@ -156,6 +157,7 @@ API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,S API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,SAControllerConfiguration,ConcurrentSATokenSyncs API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,SAControllerConfiguration,RootCAFile API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,ServiceControllerConfiguration,ConcurrentServiceSyncs +API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,TTLAfterFinishedControllerConfiguration,ConcurrentTTLSyncs API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,VolumeConfiguration,EnableHostPathProvisioning API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,VolumeConfiguration,EnableDynamicProvisioning API rule violation: names_match,k8s.io/kube-controller-manager/config/v1alpha1,VolumeConfiguration,PersistentVolumeRecyclerConfiguration diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index 1d0a19e759..0f33ec168c 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -378,7 +378,7 @@ func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc controllers["clusterrole-aggregation"] = startClusterRoleAggregrationController controllers["pvc-protection"] = startPVCProtectionController controllers["pv-protection"] = startPVProtectionController - controllers["ttl-after-finished-controller"] = startTTLAfterFinishedController + controllers["ttl-after-finished"] = startTTLAfterFinishedController return controllers } diff --git a/cmd/kube-controller-manager/app/core.go b/cmd/kube-controller-manager/app/core.go index 202a7f9668..f6a273385a 100644 --- a/cmd/kube-controller-manager/app/core.go +++ b/cmd/kube-controller-manager/app/core.go @@ -426,6 +426,6 @@ func startTTLAfterFinishedController(ctx ControllerContext) (http.Handler, bool, go ttlafterfinished.New( ctx.InformerFactory.Batch().V1().Jobs(), ctx.ClientBuilder.ClientOrDie("ttl-after-finished-controller"), - ).Run(5, ctx.Stop) + ).Run(int(ctx.ComponentConfig.TTLAfterFinishedController.ConcurrentTTLSyncs), ctx.Stop) return nil, true, nil } diff --git a/cmd/kube-controller-manager/app/options/BUILD b/cmd/kube-controller-manager/app/options/BUILD index 23d29b8bc0..fcae9358cf 100644 --- a/cmd/kube-controller-manager/app/options/BUILD +++ b/cmd/kube-controller-manager/app/options/BUILD @@ -28,6 +28,7 @@ go_library( "replicationcontroller.go", "resourcequotacontroller.go", "serviceaccountcontroller.go", + "ttlafterfinishedcontroller.go", ], importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app/options", deps = [ diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index 56178e9ed5..22cf24a12e 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -77,6 +77,7 @@ type KubeControllerManagerOptions struct { ReplicationController *ReplicationControllerOptions ResourceQuotaController *ResourceQuotaControllerOptions SAController *SAControllerOptions + TTLAfterFinishedController *TTLAfterFinishedControllerOptions SecureServing *apiserveroptions.SecureServingOptionsWithLoopback // TODO: remove insecure serving mode @@ -172,6 +173,9 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) { ServiceController: &cmoptions.ServiceControllerOptions{ ConcurrentServiceSyncs: componentConfig.ServiceController.ConcurrentServiceSyncs, }, + TTLAfterFinishedController: &TTLAfterFinishedControllerOptions{ + ConcurrentTTLSyncs: componentConfig.TTLAfterFinishedController.ConcurrentTTLSyncs, + }, SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(), InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{ BindAddress: net.ParseIP(componentConfig.Generic.Address), @@ -251,6 +255,7 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy s.ReplicationController.AddFlags(fss.FlagSet("replicationcontroller")) s.ResourceQuotaController.AddFlags(fss.FlagSet("resourcequota controller")) s.SAController.AddFlags(fss.FlagSet("serviceaccount controller")) + s.TTLAfterFinishedController.AddFlags(fss.FlagSet("ttl-after-finished controller")) fs := fss.FlagSet("misc") fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") @@ -328,6 +333,9 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) e if err := s.ServiceController.ApplyTo(&c.ComponentConfig.ServiceController); err != nil { return err } + if err := s.TTLAfterFinishedController.ApplyTo(&c.ComponentConfig.TTLAfterFinishedController); err != nil { + return err + } if err := s.InsecureServing.ApplyTo(&c.InsecureServing, &c.LoopbackClientConfig); err != nil { return err } @@ -376,6 +384,7 @@ func (s *KubeControllerManagerOptions) Validate(allControllers []string, disable errs = append(errs, s.ResourceQuotaController.Validate()...) errs = append(errs, s.SAController.Validate()...) errs = append(errs, s.ServiceController.Validate()...) + errs = append(errs, s.TTLAfterFinishedController.Validate()...) errs = append(errs, s.SecureServing.Validate()...) errs = append(errs, s.InsecureServing.Validate()...) errs = append(errs, s.Authentication.Validate()...) diff --git a/cmd/kube-controller-manager/app/options/options_test.go b/cmd/kube-controller-manager/app/options/options_test.go index 5a59471ac4..4129038be7 100644 --- a/cmd/kube-controller-manager/app/options/options_test.go +++ b/cmd/kube-controller-manager/app/options/options_test.go @@ -116,6 +116,7 @@ func TestAddFlags(t *testing.T) { "--cert-dir=/a/b/c", "--bind-address=192.168.4.21", "--secure-port=10001", + "--concurrent-ttl-after-finished-syncs=8", } fs.Parse(args) // Sort GCIgnoredResources because it's built from a map, which means the @@ -255,6 +256,9 @@ func TestAddFlags(t *testing.T) { ServiceController: &cmoptions.ServiceControllerOptions{ ConcurrentServiceSyncs: 2, }, + TTLAfterFinishedController: &TTLAfterFinishedControllerOptions{ + ConcurrentTTLSyncs: 8, + }, SecureServing: (&apiserveroptions.SecureServingOptions{ BindPort: 10001, BindAddress: net.ParseIP("192.168.4.21"), diff --git a/cmd/kube-controller-manager/app/options/ttlafterfinishedcontroller.go b/cmd/kube-controller-manager/app/options/ttlafterfinishedcontroller.go new file mode 100644 index 0000000000..2f7fe765ea --- /dev/null +++ b/cmd/kube-controller-manager/app/options/ttlafterfinishedcontroller.go @@ -0,0 +1,58 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "github.com/spf13/pflag" + + kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config" +) + +// TTLAfterFinishedControllerOptions holds the TTLAfterFinishedController options. +type TTLAfterFinishedControllerOptions struct { + ConcurrentTTLSyncs int32 +} + +// AddFlags adds flags related to TTLAfterFinishedController for controller manager to the specified FlagSet. +func (o *TTLAfterFinishedControllerOptions) AddFlags(fs *pflag.FlagSet) { + if o == nil { + return + } + + fs.Int32Var(&o.ConcurrentTTLSyncs, "concurrent-ttl-after-finished-syncs", o.ConcurrentTTLSyncs, "The number of TTL-after-finished controller workers that are allowed to sync concurrently.") +} + +// ApplyTo fills up TTLAfterFinishedController config with options. +func (o *TTLAfterFinishedControllerOptions) ApplyTo(cfg *kubectrlmgrconfig.TTLAfterFinishedControllerConfiguration) error { + if o == nil { + return nil + } + + cfg.ConcurrentTTLSyncs = o.ConcurrentTTLSyncs + + return nil +} + +// Validate checks validation of TTLAfterFinishedControllerOptions. +func (o *TTLAfterFinishedControllerOptions) Validate() []error { + if o == nil { + return nil + } + + errs := []error{} + return errs +} diff --git a/pkg/controller/apis/config/types.go b/pkg/controller/apis/config/types.go index b7abe6b923..c04a1b3552 100644 --- a/pkg/controller/apis/config/types.go +++ b/pkg/controller/apis/config/types.go @@ -96,6 +96,9 @@ type KubeControllerManagerConfiguration struct { // ServiceControllerConfiguration holds configuration for ServiceController // related features. ServiceController ServiceControllerConfiguration + // TTLAfterFinishedControllerConfiguration holds configuration for + // TTLAfterFinishedController related features. + TTLAfterFinishedController TTLAfterFinishedControllerConfiguration } // GenericControllerManagerConfiguration holds configuration for a generic controller-manager @@ -438,3 +441,10 @@ type PersistentVolumeRecyclerConfiguration struct { // in a multi-node cluster. IncrementTimeoutHostPath int32 } + +// TTLAfterFinishedControllerConfiguration contains elements describing TTLAfterFinishedController. +type TTLAfterFinishedControllerConfiguration struct { + // concurrentTTLSyncs is the number of TTL-after-finished collector workers that are + // allowed to sync concurrently. + ConcurrentTTLSyncs int32 +} diff --git a/pkg/controller/apis/config/v1alpha1/defaults.go b/pkg/controller/apis/config/v1alpha1/defaults.go index 6e51aa964d..af96572566 100644 --- a/pkg/controller/apis/config/v1alpha1/defaults.go +++ b/pkg/controller/apis/config/v1alpha1/defaults.go @@ -48,6 +48,9 @@ func SetDefaults_KubeControllerManagerConfiguration(obj *kubectrlmgrconfigv1alph if obj.SAController.ConcurrentSATokenSyncs == 0 { obj.SAController.ConcurrentSATokenSyncs = 5 } + if obj.TTLAfterFinishedController.ConcurrentTTLSyncs <= 0 { + obj.TTLAfterFinishedController.ConcurrentTTLSyncs = 5 + } // These defaults override the recommended defaults from the apimachineryconfigv1alpha1 package that are applied automatically // These client-connection defaults are specific to the kube-controller-manager diff --git a/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/types.go b/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/types.go index 5bcfadea6f..2e9f45b5b4 100644 --- a/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/types.go +++ b/staging/src/k8s.io/kube-controller-manager/config/v1alpha1/types.go @@ -145,6 +145,9 @@ type KubeControllerManagerConfiguration struct { // ServiceControllerConfiguration holds configuration for ServiceController // related features. ServiceController ServiceControllerConfiguration + // TTLAfterFinishedControllerConfiguration holds configuration for + // TTLAfterFinishedController related features. + TTLAfterFinishedController TTLAfterFinishedControllerConfiguration } // GenericControllerManagerConfiguration holds configuration for a generic controller-manager. @@ -438,3 +441,10 @@ type ServiceControllerConfiguration struct { // management, but more CPU (and network) load. ConcurrentServiceSyncs int32 } + +// TTLAfterFinishedControllerConfiguration contains elements describing TTLAfterFinishedController. +type TTLAfterFinishedControllerConfiguration struct { + // concurrentTTLSyncs is the number of TTL-after-finished collector workers that are + // allowed to sync concurrently. + ConcurrentTTLSyncs int32 +}