diff --git a/api/api-rules/violation_exceptions.list b/api/api-rules/violation_exceptions.list index 18e652001b..03f502184d 100644 --- a/api/api-rules/violation_exceptions.list +++ b/api/api-rules/violation_exceptions.list @@ -93,6 +93,7 @@ API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alp API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,GroupResource,Resource API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerSyncPeriod API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerUpscaleForbiddenWindow +API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerDownscaleStabilizationWindow API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerDownscaleForbiddenWindow API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerTolerance API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerUseRESTClients diff --git a/cmd/kube-controller-manager/app/autoscaling.go b/cmd/kube-controller-manager/app/autoscaling.go index 9f86fe7411..76f80d9ae1 100644 --- a/cmd/kube-controller-manager/app/autoscaling.go +++ b/cmd/kube-controller-manager/app/autoscaling.go @@ -95,7 +95,7 @@ func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient me replicaCalc, ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(), ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod.Duration, - ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, + ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration, ).Run(ctx.Stop) return nil, true, nil } diff --git a/cmd/kube-controller-manager/app/options/hpacontroller.go b/cmd/kube-controller-manager/app/options/hpacontroller.go index 55d460e2dc..40ee7f4df8 100644 --- a/cmd/kube-controller-manager/app/options/hpacontroller.go +++ b/cmd/kube-controller-manager/app/options/hpacontroller.go @@ -25,13 +25,14 @@ import ( // HPAControllerOptions holds the HPAController options. type HPAControllerOptions struct { - HorizontalPodAutoscalerUseRESTClients bool - HorizontalPodAutoscalerTolerance float64 - HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration - HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration - HorizontalPodAutoscalerSyncPeriod metav1.Duration - HorizontalPodAutoscalerCPUInitializationPeriod metav1.Duration - HorizontalPodAutoscalerInitialReadinessDelay metav1.Duration + HorizontalPodAutoscalerUseRESTClients bool + HorizontalPodAutoscalerTolerance float64 + HorizontalPodAutoscalerDownscaleStabilizationWindow metav1.Duration + HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration + HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration + HorizontalPodAutoscalerSyncPeriod metav1.Duration + HorizontalPodAutoscalerCPUInitializationPeriod metav1.Duration + HorizontalPodAutoscalerInitialReadinessDelay metav1.Duration } // AddFlags adds flags related to HPAController for controller manager to the specified FlagSet. @@ -43,7 +44,9 @@ func (o *HPAControllerOptions) AddFlags(fs *pflag.FlagSet) { fs.DurationVar(&o.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", o.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.") fs.DurationVar(&o.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-upscale-delay", o.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "The period since last upscale, before another upscale can be performed in horizontal pod autoscaler.") fs.MarkDeprecated("horizontal-pod-autoscaler-upscale-delay", "This flag is currently no-op and will be deleted.") + fs.DurationVar(&o.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration, "horizontal-pod-autoscaler-downscale-stabilization", o.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration, "The period for which autoscaler will look backwards and not scale down below any recommendation it made during that period.") fs.DurationVar(&o.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-downscale-delay", o.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "The period since last downscale, before another downscale can be performed in horizontal pod autoscaler.") + fs.MarkDeprecated("horizontal-pod-autoscaler-downscale-delay", "This flag is currently no-op and will be deleted.") fs.Float64Var(&o.HorizontalPodAutoscalerTolerance, "horizontal-pod-autoscaler-tolerance", o.HorizontalPodAutoscalerTolerance, "The minimum change (from 1.0) in the desired-to-actual metrics ratio for the horizontal pod autoscaler to consider scaling.") fs.BoolVar(&o.HorizontalPodAutoscalerUseRESTClients, "horizontal-pod-autoscaler-use-rest-clients", o.HorizontalPodAutoscalerUseRESTClients, "If set to true, causes the horizontal pod autoscaler controller to use REST clients through the kube-aggregator, instead of using the legacy metrics client through the API server proxy. This is required for custom metrics support in the horizontal pod autoscaler.") fs.DurationVar(&o.HorizontalPodAutoscalerCPUInitializationPeriod.Duration, "horizontal-pod-autoscaler-cpu-initialization-period", o.HorizontalPodAutoscalerCPUInitializationPeriod.Duration, "The period after pod start when CPU samples might be skipped.") @@ -57,7 +60,7 @@ func (o *HPAControllerOptions) ApplyTo(cfg *componentconfig.HPAControllerConfigu } cfg.HorizontalPodAutoscalerSyncPeriod = o.HorizontalPodAutoscalerSyncPeriod - cfg.HorizontalPodAutoscalerDownscaleForbiddenWindow = o.HorizontalPodAutoscalerDownscaleForbiddenWindow + cfg.HorizontalPodAutoscalerDownscaleStabilizationWindow = o.HorizontalPodAutoscalerDownscaleStabilizationWindow cfg.HorizontalPodAutoscalerTolerance = o.HorizontalPodAutoscalerTolerance cfg.HorizontalPodAutoscalerUseRESTClients = o.HorizontalPodAutoscalerUseRESTClients cfg.HorizontalPodAutoscalerCPUInitializationPeriod = o.HorizontalPodAutoscalerCPUInitializationPeriod diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index f7c9c4a260..6ffeeff71c 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -123,13 +123,14 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) { EnableGarbageCollector: componentConfig.GarbageCollectorController.EnableGarbageCollector, }, HPAController: &HPAControllerOptions{ - HorizontalPodAutoscalerSyncPeriod: componentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod, - HorizontalPodAutoscalerUpscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow, - HorizontalPodAutoscalerDownscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow, - HorizontalPodAutoscalerCPUInitializationPeriod: componentConfig.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod, - HorizontalPodAutoscalerInitialReadinessDelay: componentConfig.HPAController.HorizontalPodAutoscalerInitialReadinessDelay, - HorizontalPodAutoscalerTolerance: componentConfig.HPAController.HorizontalPodAutoscalerTolerance, - HorizontalPodAutoscalerUseRESTClients: componentConfig.HPAController.HorizontalPodAutoscalerUseRESTClients, + HorizontalPodAutoscalerSyncPeriod: componentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod, + HorizontalPodAutoscalerUpscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow, + HorizontalPodAutoscalerDownscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow, + HorizontalPodAutoscalerDownscaleStabilizationWindow: componentConfig.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow, + HorizontalPodAutoscalerCPUInitializationPeriod: componentConfig.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod, + HorizontalPodAutoscalerInitialReadinessDelay: componentConfig.HPAController.HorizontalPodAutoscalerInitialReadinessDelay, + HorizontalPodAutoscalerTolerance: componentConfig.HPAController.HorizontalPodAutoscalerTolerance, + HorizontalPodAutoscalerUseRESTClients: componentConfig.HPAController.HorizontalPodAutoscalerUseRESTClients, }, JobController: &JobControllerOptions{ ConcurrentJobSyncs: componentConfig.JobController.ConcurrentJobSyncs, diff --git a/cmd/kube-controller-manager/app/options/options_test.go b/cmd/kube-controller-manager/app/options/options_test.go index f23bc13b3a..c7d47ddaa9 100644 --- a/cmd/kube-controller-manager/app/options/options_test.go +++ b/cmd/kube-controller-manager/app/options/options_test.go @@ -76,6 +76,7 @@ func TestAddFlags(t *testing.T) { "--horizontal-pod-autoscaler-downscale-delay=2m", "--horizontal-pod-autoscaler-sync-period=45s", "--horizontal-pod-autoscaler-upscale-delay=1m", + "--horizontal-pod-autoscaler-downscale-stabilization=3m", "--horizontal-pod-autoscaler-cpu-initialization-period=90s", "--horizontal-pod-autoscaler-initial-readiness-delay=50s", "--http2-max-streams-per-connection=47", @@ -190,13 +191,14 @@ func TestAddFlags(t *testing.T) { EnableGarbageCollector: false, }, HPAController: &HPAControllerOptions{ - HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 45 * time.Second}, - HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 1 * time.Minute}, - HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute}, - HorizontalPodAutoscalerCPUInitializationPeriod: metav1.Duration{Duration: 90 * time.Second}, - HorizontalPodAutoscalerInitialReadinessDelay: metav1.Duration{Duration: 50 * time.Second}, - HorizontalPodAutoscalerTolerance: 0.1, - HorizontalPodAutoscalerUseRESTClients: true, + HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 45 * time.Second}, + HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 1 * time.Minute}, + HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute}, + HorizontalPodAutoscalerDownscaleStabilizationWindow: metav1.Duration{Duration: 3 * time.Minute}, + HorizontalPodAutoscalerCPUInitializationPeriod: metav1.Duration{Duration: 90 * time.Second}, + HorizontalPodAutoscalerInitialReadinessDelay: metav1.Duration{Duration: 50 * time.Second}, + HorizontalPodAutoscalerTolerance: 0.1, + HorizontalPodAutoscalerUseRESTClients: true, }, JobController: &JobControllerOptions{ ConcurrentJobSyncs: 5, diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index 35e02a503d..7e3cf8a8f8 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -256,6 +256,9 @@ type HPAControllerConfiguration struct { HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration // horizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed. HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration + // HorizontalPodAutoscalerDowncaleStabilizationWindow is a period for which autoscaler will look + // backwards and not scale down below any recommendation it made during that period. + HorizontalPodAutoscalerDownscaleStabilizationWindow metav1.Duration // horizontalPodAutoscalerTolerance is the tolerance for when // resource usage suggests upscaling/downscaling HorizontalPodAutoscalerTolerance float64 diff --git a/pkg/apis/componentconfig/v1alpha1/defaults.go b/pkg/apis/componentconfig/v1alpha1/defaults.go index e33deab7f4..d43432797a 100644 --- a/pkg/apis/componentconfig/v1alpha1/defaults.go +++ b/pkg/apis/componentconfig/v1alpha1/defaults.go @@ -77,6 +77,9 @@ func SetDefaults_KubeControllerManagerConfiguration(obj *KubeControllerManagerCo if obj.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow == zero { obj.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow = metav1.Duration{Duration: 3 * time.Minute} } + if obj.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow == zero { + obj.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow = metav1.Duration{Duration: 5 * time.Minute} + } if obj.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod == zero { obj.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod = metav1.Duration{Duration: 5 * time.Minute} } diff --git a/pkg/apis/componentconfig/v1alpha1/types.go b/pkg/apis/componentconfig/v1alpha1/types.go index 3a7bb59fed..f9aa952c8f 100644 --- a/pkg/apis/componentconfig/v1alpha1/types.go +++ b/pkg/apis/componentconfig/v1alpha1/types.go @@ -296,14 +296,17 @@ type GarbageCollectorControllerConfiguration struct { } type HPAControllerConfiguration struct { - // horizontalPodAutoscalerSyncPeriod is the period for syncing the number of + // HorizontalPodAutoscalerSyncPeriod is the period for syncing the number of // pods in horizontal pod autoscaler. HorizontalPodAutoscalerSyncPeriod metav1.Duration - // horizontalPodAutoscalerUpscaleForbiddenWindow is a period after which next upscale allowed. + // HorizontalPodAutoscalerUpscaleForbiddenWindow is a period after which next upscale allowed. HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration - // horizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed. + // HorizontalPodAutoscalerDowncaleStabilizationWindow is a period for which autoscaler will look + // backwards and not scale down below any recommendation it made during that period. + HorizontalPodAutoscalerDownscaleStabilizationWindow metav1.Duration + // HorizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed. HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration - // horizontalPodAutoscalerTolerance is the tolerance for when + // HorizontalPodAutoscalerTolerance is the tolerance for when // resource usage suggests upscaling/downscaling HorizontalPodAutoscalerTolerance float64 // HorizontalPodAutoscalerUseRESTClients causes the HPA controller to use REST clients diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go index 6990b98a0d..ca28fcbfe6 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go @@ -602,6 +602,7 @@ func Convert_componentconfig_GroupResource_To_v1alpha1_GroupResource(in *compone func autoConvert_v1alpha1_HPAControllerConfiguration_To_componentconfig_HPAControllerConfiguration(in *HPAControllerConfiguration, out *componentconfig.HPAControllerConfiguration, s conversion.Scope) error { out.HorizontalPodAutoscalerSyncPeriod = in.HorizontalPodAutoscalerSyncPeriod out.HorizontalPodAutoscalerUpscaleForbiddenWindow = in.HorizontalPodAutoscalerUpscaleForbiddenWindow + out.HorizontalPodAutoscalerDownscaleStabilizationWindow = in.HorizontalPodAutoscalerDownscaleStabilizationWindow out.HorizontalPodAutoscalerDownscaleForbiddenWindow = in.HorizontalPodAutoscalerDownscaleForbiddenWindow out.HorizontalPodAutoscalerTolerance = in.HorizontalPodAutoscalerTolerance if err := v1.Convert_Pointer_bool_To_bool(&in.HorizontalPodAutoscalerUseRESTClients, &out.HorizontalPodAutoscalerUseRESTClients, s); err != nil { @@ -621,6 +622,7 @@ func autoConvert_componentconfig_HPAControllerConfiguration_To_v1alpha1_HPAContr out.HorizontalPodAutoscalerSyncPeriod = in.HorizontalPodAutoscalerSyncPeriod out.HorizontalPodAutoscalerUpscaleForbiddenWindow = in.HorizontalPodAutoscalerUpscaleForbiddenWindow out.HorizontalPodAutoscalerDownscaleForbiddenWindow = in.HorizontalPodAutoscalerDownscaleForbiddenWindow + out.HorizontalPodAutoscalerDownscaleStabilizationWindow = in.HorizontalPodAutoscalerDownscaleStabilizationWindow out.HorizontalPodAutoscalerTolerance = in.HorizontalPodAutoscalerTolerance if err := v1.Convert_bool_To_Pointer_bool(&in.HorizontalPodAutoscalerUseRESTClients, &out.HorizontalPodAutoscalerUseRESTClients, s); err != nil { return err diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go index 1a601f1adf..83e233ebda 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go @@ -241,6 +241,7 @@ func (in *HPAControllerConfiguration) DeepCopyInto(out *HPAControllerConfigurati *out = *in out.HorizontalPodAutoscalerSyncPeriod = in.HorizontalPodAutoscalerSyncPeriod out.HorizontalPodAutoscalerUpscaleForbiddenWindow = in.HorizontalPodAutoscalerUpscaleForbiddenWindow + out.HorizontalPodAutoscalerDownscaleStabilizationWindow = in.HorizontalPodAutoscalerDownscaleStabilizationWindow out.HorizontalPodAutoscalerDownscaleForbiddenWindow = in.HorizontalPodAutoscalerDownscaleForbiddenWindow if in.HorizontalPodAutoscalerUseRESTClients != nil { in, out := &in.HorizontalPodAutoscalerUseRESTClients, &out.HorizontalPodAutoscalerUseRESTClients diff --git a/pkg/apis/componentconfig/zz_generated.deepcopy.go b/pkg/apis/componentconfig/zz_generated.deepcopy.go index af108acac7..dc2e3aea92 100644 --- a/pkg/apis/componentconfig/zz_generated.deepcopy.go +++ b/pkg/apis/componentconfig/zz_generated.deepcopy.go @@ -237,6 +237,7 @@ func (in *HPAControllerConfiguration) DeepCopyInto(out *HPAControllerConfigurati out.HorizontalPodAutoscalerSyncPeriod = in.HorizontalPodAutoscalerSyncPeriod out.HorizontalPodAutoscalerUpscaleForbiddenWindow = in.HorizontalPodAutoscalerUpscaleForbiddenWindow out.HorizontalPodAutoscalerDownscaleForbiddenWindow = in.HorizontalPodAutoscalerDownscaleForbiddenWindow + out.HorizontalPodAutoscalerDownscaleStabilizationWindow = in.HorizontalPodAutoscalerDownscaleStabilizationWindow out.HorizontalPodAutoscalerCPUInitializationPeriod = in.HorizontalPodAutoscalerCPUInitializationPeriod out.HorizontalPodAutoscalerInitialReadinessDelay = in.HorizontalPodAutoscalerInitialReadinessDelay return diff --git a/pkg/controller/podautoscaler/BUILD b/pkg/controller/podautoscaler/BUILD index 2988db1c21..0e8237cd88 100644 --- a/pkg/controller/podautoscaler/BUILD +++ b/pkg/controller/podautoscaler/BUILD @@ -83,6 +83,7 @@ go_test( "//staging/src/k8s.io/metrics/pkg/client/clientset/versioned/fake:go_default_library", "//staging/src/k8s.io/metrics/pkg/client/custom_metrics/fake:go_default_library", "//staging/src/k8s.io/metrics/pkg/client/external_metrics/fake:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/github.com/stretchr/testify/require:go_default_library", "//vendor/k8s.io/heapster/metrics/api/v1/types:go_default_library", diff --git a/pkg/controller/podautoscaler/horizontal.go b/pkg/controller/podautoscaler/horizontal.go index b40af893bd..289a465f96 100644 --- a/pkg/controller/podautoscaler/horizontal.go +++ b/pkg/controller/podautoscaler/horizontal.go @@ -53,6 +53,11 @@ var ( scaleUpLimitMinimum = 4.0 ) +type timestampedRecommendation struct { + recommendation int32 + timestamp time.Time +} + // HorizontalController is responsible for the synchronizing HPA objects stored // in the system with the actual deployments/replication controllers they // control. @@ -64,7 +69,7 @@ type HorizontalController struct { replicaCalc *ReplicaCalculator eventRecorder record.EventRecorder - downscaleForbiddenWindow time.Duration + downscaleStabilisationWindow time.Duration // hpaLister is able to list/get HPAs from the shared cache from the informer passed in to // NewHorizontalController. @@ -73,6 +78,9 @@ type HorizontalController struct { // Controllers that need to be synced queue workqueue.RateLimitingInterface + + // Latest unstabilized recommendations for each autoscaler. + recommendations map[string][]timestampedRecommendation } // NewHorizontalController creates a new HorizontalController. @@ -84,7 +92,7 @@ func NewHorizontalController( replicaCalc *ReplicaCalculator, hpaInformer autoscalinginformers.HorizontalPodAutoscalerInformer, resyncPeriod time.Duration, - downscaleForbiddenWindow time.Duration, + downscaleStabilisationWindow time.Duration, ) *HorizontalController { broadcaster := record.NewBroadcaster() @@ -93,13 +101,14 @@ func NewHorizontalController( recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "horizontal-pod-autoscaler"}) hpaController := &HorizontalController{ - replicaCalc: replicaCalc, - eventRecorder: recorder, - scaleNamespacer: scaleNamespacer, - hpaNamespacer: hpaNamespacer, - downscaleForbiddenWindow: downscaleForbiddenWindow, - queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"), - mapper: mapper, + replicaCalc: replicaCalc, + eventRecorder: recorder, + scaleNamespacer: scaleNamespacer, + hpaNamespacer: hpaNamespacer, + downscaleStabilisationWindow: downscaleStabilisationWindow, + queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"), + mapper: mapper, + recommendations: map[string][]timestampedRecommendation{}, } hpaInformer.Informer().AddEventHandlerWithResyncPeriod( @@ -275,10 +284,11 @@ func (a *HorizontalController) reconcileKey(key string) error { hpa, err := a.hpaLister.HorizontalPodAutoscalers(namespace).Get(name) if errors.IsNotFound(err) { glog.Infof("Horizontal Pod Autoscaler %s has been deleted in %s", name, namespace) + delete(a.recommendations, key) return nil } - return a.reconcileAutoscaler(hpa) + return a.reconcileAutoscaler(hpa, key) } // computeStatusForObjectMetric computes the desired number of replicas for the specified metric of type ObjectMetricSourceType. @@ -431,7 +441,7 @@ func (a *HorizontalController) computeStatusForExternalMetric(currentReplicas in return 0, time.Time{}, "", fmt.Errorf(errMsg) } -func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.HorizontalPodAutoscaler) error { +func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.HorizontalPodAutoscaler, key string) error { // make a copy so that we never mutate the shared informer cache (conversion can mutate the object) hpav1 := hpav1Shared.DeepCopy() // then, convert to autoscaling/v2, which makes our lives easier when calculating metrics @@ -527,24 +537,8 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho if desiredReplicas < currentReplicas { rescaleReason = "All metrics below target" } - - desiredReplicas = a.normalizeDesiredReplicas(hpa, currentReplicas, desiredReplicas) - - rescale = a.shouldScale(hpa, currentReplicas, desiredReplicas, timestamp) - backoffDown := false - backoffUp := false - if hpa.Status.LastScaleTime != nil { - if !hpa.Status.LastScaleTime.Add(a.downscaleForbiddenWindow).Before(timestamp) { - setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionFalse, "BackoffDownscale", "the time since the previous scale is still within the downscale forbidden window") - backoffDown = true - } - } - - if !backoffDown && !backoffUp { - // mark that we're not backing off - setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "ReadyForNewScale", "the last scale time was sufficiently old as to warrant a new scale") - } - + desiredReplicas = a.normalizeDesiredReplicas(hpa, key, currentReplicas, desiredReplicas) + rescale = desiredReplicas != currentReplicas } if rescale { @@ -572,9 +566,39 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho return a.updateStatusIfNeeded(hpaStatusOriginal, hpa) } +// stabilizeRecommendation: +// - replaces old recommendation with the newest recommendation, +// - returns max of recommendations that are not older than downscaleStabilisationWindow. +func (a *HorizontalController) stabilizeRecommendation(key string, prenormalizedDesiredReplicas int32) int32 { + maxRecommendation := prenormalizedDesiredReplicas + foundOldSample := false + oldSampleIndex := 0 + cutoff := time.Now().Add(-a.downscaleStabilisationWindow) + for i, rec := range a.recommendations[key] { + if rec.timestamp.Before(cutoff) { + foundOldSample = true + oldSampleIndex = i + } else if rec.recommendation > maxRecommendation { + maxRecommendation = rec.recommendation + } + } + if foundOldSample { + a.recommendations[key][oldSampleIndex] = timestampedRecommendation{prenormalizedDesiredReplicas, time.Now()} + } else { + a.recommendations[key] = append(a.recommendations[key], timestampedRecommendation{prenormalizedDesiredReplicas, time.Now()}) + } + return maxRecommendation +} + // normalizeDesiredReplicas takes the metrics desired replicas value and normalizes it based on the appropriate conditions (i.e. < maxReplicas, > // minReplicas, etc...) -func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas int32, prenormalizedDesiredReplicas int32) int32 { +func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.HorizontalPodAutoscaler, key string, currentReplicas int32, prenormalizedDesiredReplicas int32) int32 { + stabilizedRecommendation := a.stabilizeRecommendation(key, prenormalizedDesiredReplicas) + if stabilizedRecommendation != prenormalizedDesiredReplicas { + setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "ScaleDownStabilized", "recent recommendations were higher than current one, applying the highest recent recommendation") + } else { + setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "ReadyForNewScale", "recommended size matches current size") + } var minReplicas int32 if hpa.Spec.MinReplicas != nil { minReplicas = *hpa.Spec.MinReplicas @@ -582,9 +606,9 @@ func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.Horiz minReplicas = 0 } - desiredReplicas, condition, reason := convertDesiredReplicasWithRules(currentReplicas, prenormalizedDesiredReplicas, minReplicas, hpa.Spec.MaxReplicas) + desiredReplicas, condition, reason := convertDesiredReplicasWithRules(currentReplicas, stabilizedRecommendation, minReplicas, hpa.Spec.MaxReplicas) - if desiredReplicas == prenormalizedDesiredReplicas { + if desiredReplicas == stabilizedRecommendation { setCondition(hpa, autoscalingv2.ScalingLimited, v1.ConditionFalse, condition, reason) } else { setCondition(hpa, autoscalingv2.ScalingLimited, v1.ConditionTrue, condition, reason) @@ -641,29 +665,6 @@ func calculateScaleUpLimit(currentReplicas int32) int32 { return int32(math.Max(scaleUpLimitFactor*float64(currentReplicas), scaleUpLimitMinimum)) } -func (a *HorizontalController) shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int32, timestamp time.Time) bool { - if desiredReplicas == currentReplicas { - return false - } - - if hpa.Status.LastScaleTime == nil { - return true - } - - // Going down only if the usageRatio dropped significantly below the target - // and there was no rescaling in the last downscaleForbiddenWindow. - if desiredReplicas < currentReplicas && hpa.Status.LastScaleTime.Add(a.downscaleForbiddenWindow).Before(timestamp) { - return true - } - - // Going up only if the usage ratio increased significantly above the target. - if desiredReplicas > currentReplicas { - return true - } - - return false -} - // scaleForResourceMappings attempts to fetch the scale for the // resource with the given name and namespace, trying each RESTMapping // in turn until a working one is found. If none work, the first error diff --git a/pkg/controller/podautoscaler/horizontal_test.go b/pkg/controller/podautoscaler/horizontal_test.go index dfc6a45ea8..8d064501c5 100644 --- a/pkg/controller/podautoscaler/horizontal_test.go +++ b/pkg/controller/podautoscaler/horizontal_test.go @@ -51,6 +51,7 @@ import ( "github.com/stretchr/testify/assert" + "github.com/golang/glog" _ "k8s.io/kubernetes/pkg/apis/autoscaling/install" _ "k8s.io/kubernetes/pkg/apis/extensions/install" ) @@ -129,6 +130,8 @@ type testCase struct { testCMClient *cmfake.FakeCustomMetricsClient testEMClient *emfake.FakeExternalMetricsClient testScaleClient *scalefake.FakeScaleClient + + recommendations []timestampedRecommendation } // Needs to be called under a lock. @@ -662,7 +665,7 @@ func (tc *testCase) setupController(t *testing.T) (*HorizontalController, inform replicaCalc := NewReplicaCalculator(metricsClient, testClient.Core(), defaultTestingTolerance, defaultTestingCpuInitializationPeriod, defaultTestingDelayOfInitialReadinessStatus) informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc()) - defaultDownscaleForbiddenWindow := 5 * time.Minute + defaultDownscalestabilizationWindow := 5 * time.Minute hpaController := NewHorizontalController( eventClient.Core(), @@ -672,9 +675,12 @@ func (tc *testCase) setupController(t *testing.T) (*HorizontalController, inform replicaCalc, informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(), controller.NoResyncPeriodFunc(), - defaultDownscaleForbiddenWindow, + defaultDownscalestabilizationWindow, ) hpaController.hpaListerSynced = alwaysReady + if tc.recommendations != nil { + hpaController.recommendations["test-namespace/test-hpa"] = tc.recommendations + } return hpaController, informerFactory } @@ -709,6 +715,7 @@ func (tc *testCase) runTestWithController(t *testing.T, hpaController *Horizonta func (tc *testCase) runTest(t *testing.T) { hpaController, informerFactory := tc.setupController(t) tc.runTestWithController(t, hpaController, informerFactory) + glog.Errorf("recommendations: %+v", hpaController.recommendations) } func TestScaleUp(t *testing.T) { @@ -2080,8 +2087,7 @@ func TestNoBackoffUpscaleCMNoBackoffCpu(t *testing.T) { tc.runTest(t) } -func TestBackoffDownscale(t *testing.T) { - time := metav1.Time{Time: time.Now().Add(-4 * time.Minute)} +func TestStabilizeDownscale(t *testing.T) { tc := testCase{ minReplicas: 1, maxReplicas: 5, @@ -2091,16 +2097,19 @@ func TestBackoffDownscale(t *testing.T) { reportedLevels: []uint64{50, 50, 50}, reportedCPURequests: []resource.Quantity{resource.MustParse("0.1"), resource.MustParse("0.1"), resource.MustParse("0.1")}, useMetricsAPI: true, - lastScaleTime: &time, expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{ Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "ReadyForNewScale", }, autoscalingv2.HorizontalPodAutoscalerCondition{ Type: autoscalingv2.AbleToScale, - Status: v1.ConditionFalse, - Reason: "BackoffDownscale", + Status: v1.ConditionTrue, + Reason: "ScaleDownStabilized", }), + recommendations: []timestampedRecommendation{ + {10, time.Now().Add(-10 * time.Minute)}, + {4, time.Now().Add(-1 * time.Minute)}, + }, } tc.runTest(t) } @@ -2278,7 +2287,7 @@ func TestAvoidUncessaryUpdates(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if err := controller.reconcileAutoscaler(&initialHPAs.Items[0]); err != nil { + if err := controller.reconcileAutoscaler(&initialHPAs.Items[0], ""); err != nil { t.Fatalf("unexpected error: %v", err) } @@ -2353,4 +2362,85 @@ func TestConvertDesiredReplicasWithRules(t *testing.T) { } } +func TestNormalizeDesiredReplicas(t *testing.T) { + tests := []struct { + name string + key string + recommendations []timestampedRecommendation + prenormalizedDesiredReplicas int32 + expectedStabilizedReplicas int32 + expectedLogLength int + }{ + { + "empty log", + "", + []timestampedRecommendation{}, + 5, + 5, + 1, + }, + { + "stabilize", + "", + []timestampedRecommendation{ + {4, time.Now().Add(-2 * time.Minute)}, + {5, time.Now().Add(-1 * time.Minute)}, + }, + 3, + 5, + 3, + }, + { + "no stabilize", + "", + []timestampedRecommendation{ + {1, time.Now().Add(-2 * time.Minute)}, + {2, time.Now().Add(-1 * time.Minute)}, + }, + 3, + 3, + 3, + }, + { + "no stabilize - old recommendations", + "", + []timestampedRecommendation{ + {10, time.Now().Add(-10 * time.Minute)}, + {9, time.Now().Add(-9 * time.Minute)}, + }, + 3, + 3, + 2, + }, + { + "stabilize - old recommendations", + "", + []timestampedRecommendation{ + {10, time.Now().Add(-10 * time.Minute)}, + {4, time.Now().Add(-1 * time.Minute)}, + {5, time.Now().Add(-2 * time.Minute)}, + {9, time.Now().Add(-9 * time.Minute)}, + }, + 3, + 5, + 4, + }, + } + for _, tc := range tests { + hc := HorizontalController{ + downscaleStabilisationWindow: 5 * time.Minute, + recommendations: map[string][]timestampedRecommendation{ + tc.key: tc.recommendations, + }, + } + r := hc.stabilizeRecommendation(tc.key, tc.prenormalizedDesiredReplicas) + if r != tc.expectedStabilizedReplicas { + t.Errorf("[%s] got %d stabilized replicas, expected %d", tc.name, r, tc.expectedStabilizedReplicas) + } + if len(hc.recommendations[tc.key]) != tc.expectedLogLength { + t.Errorf("[%s] after stabilization recommendations log has %d entries, expected %d", tc.name, len(hc.recommendations[tc.key]), tc.expectedLogLength) + } + } +} + // TODO: add more tests diff --git a/pkg/controller/podautoscaler/legacy_horizontal_test.go b/pkg/controller/podautoscaler/legacy_horizontal_test.go index 842f363acf..bc681b543a 100644 --- a/pkg/controller/podautoscaler/legacy_horizontal_test.go +++ b/pkg/controller/podautoscaler/legacy_horizontal_test.go @@ -488,7 +488,7 @@ func (tc *legacyTestCase) runTest(t *testing.T) { replicaCalc := NewReplicaCalculator(metricsClient, testClient.Core(), defaultTestingTolerance, defaultTestingCpuInitializationPeriod, defaultTestingDelayOfInitialReadinessStatus) informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc()) - defaultDownscaleForbiddenWindow := 5 * time.Minute + defaultDownscaleStabilisationWindow := 5 * time.Minute hpaController := NewHorizontalController( eventClient.Core(), @@ -498,7 +498,7 @@ func (tc *legacyTestCase) runTest(t *testing.T) { replicaCalc, informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(), controller.NoResyncPeriodFunc(), - defaultDownscaleForbiddenWindow, + defaultDownscaleStabilisationWindow, ) hpaController.hpaListerSynced = alwaysReady