mirror of https://github.com/k3s-io/k3s
Merge pull request #68122 from krzysztof-jastrzebski/scale_down
Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions here: https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md. Replace scale down window **What this PR does / why we need it**: Replace scale down forbidden window with scale down stabilization window. This allows scale down based on more than one sample, to avoid rapidly changing size up and down for controllers with fluctuating load. A bit more in https://docs.google.com/document/d/1IdG3sqgCEaRV3urPLA29IDudCufD89RYCohfBPNeWIM This PR is copy of #67771 with resolved comments. **Release note**: ```release-note Replace scale down forbidden window with scale down stabilization window. Rather than waiting a fixed period of time between scale downs HPA now scales down to the highest recommendation it during the scale down stabilization window. ```pull/8/head
Kubernetes Submit Queue
GitHub
commit
5b355f5d40
|
|
@ -93,6 +93,7 @@ API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alp
|
|||
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,GroupResource,Resource
|
||||
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerSyncPeriod
|
||||
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerUpscaleForbiddenWindow
|
||||
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerDownscaleStabilizationWindow
|
||||
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerDownscaleForbiddenWindow
|
||||
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerTolerance
|
||||
API rule violation: names_match,k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1,HPAControllerConfiguration,HorizontalPodAutoscalerUseRESTClients
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient me
|
|||
replicaCalc,
|
||||
ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
|
||||
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod.Duration,
|
||||
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration,
|
||||
ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration,
|
||||
).Run(ctx.Stop)
|
||||
return nil, true, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,13 +25,14 @@ import (
|
|||
|
||||
// HPAControllerOptions holds the HPAController options.
|
||||
type HPAControllerOptions struct {
|
||||
HorizontalPodAutoscalerUseRESTClients bool
|
||||
HorizontalPodAutoscalerTolerance float64
|
||||
HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
|
||||
HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration
|
||||
HorizontalPodAutoscalerSyncPeriod metav1.Duration
|
||||
HorizontalPodAutoscalerCPUInitializationPeriod metav1.Duration
|
||||
HorizontalPodAutoscalerInitialReadinessDelay metav1.Duration
|
||||
HorizontalPodAutoscalerUseRESTClients bool
|
||||
HorizontalPodAutoscalerTolerance float64
|
||||
HorizontalPodAutoscalerDownscaleStabilizationWindow metav1.Duration
|
||||
HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
|
||||
HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration
|
||||
HorizontalPodAutoscalerSyncPeriod metav1.Duration
|
||||
HorizontalPodAutoscalerCPUInitializationPeriod metav1.Duration
|
||||
HorizontalPodAutoscalerInitialReadinessDelay metav1.Duration
|
||||
}
|
||||
|
||||
// AddFlags adds flags related to HPAController for controller manager to the specified FlagSet.
|
||||
|
|
@ -43,7 +44,9 @@ func (o *HPAControllerOptions) AddFlags(fs *pflag.FlagSet) {
|
|||
fs.DurationVar(&o.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", o.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.")
|
||||
fs.DurationVar(&o.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-upscale-delay", o.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "The period since last upscale, before another upscale can be performed in horizontal pod autoscaler.")
|
||||
fs.MarkDeprecated("horizontal-pod-autoscaler-upscale-delay", "This flag is currently no-op and will be deleted.")
|
||||
fs.DurationVar(&o.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration, "horizontal-pod-autoscaler-downscale-stabilization", o.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration, "The period for which autoscaler will look backwards and not scale down below any recommendation it made during that period.")
|
||||
fs.DurationVar(&o.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-downscale-delay", o.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "The period since last downscale, before another downscale can be performed in horizontal pod autoscaler.")
|
||||
fs.MarkDeprecated("horizontal-pod-autoscaler-downscale-delay", "This flag is currently no-op and will be deleted.")
|
||||
fs.Float64Var(&o.HorizontalPodAutoscalerTolerance, "horizontal-pod-autoscaler-tolerance", o.HorizontalPodAutoscalerTolerance, "The minimum change (from 1.0) in the desired-to-actual metrics ratio for the horizontal pod autoscaler to consider scaling.")
|
||||
fs.BoolVar(&o.HorizontalPodAutoscalerUseRESTClients, "horizontal-pod-autoscaler-use-rest-clients", o.HorizontalPodAutoscalerUseRESTClients, "If set to true, causes the horizontal pod autoscaler controller to use REST clients through the kube-aggregator, instead of using the legacy metrics client through the API server proxy. This is required for custom metrics support in the horizontal pod autoscaler.")
|
||||
fs.DurationVar(&o.HorizontalPodAutoscalerCPUInitializationPeriod.Duration, "horizontal-pod-autoscaler-cpu-initialization-period", o.HorizontalPodAutoscalerCPUInitializationPeriod.Duration, "The period after pod start when CPU samples might be skipped.")
|
||||
|
|
@ -57,7 +60,7 @@ func (o *HPAControllerOptions) ApplyTo(cfg *componentconfig.HPAControllerConfigu
|
|||
}
|
||||
|
||||
cfg.HorizontalPodAutoscalerSyncPeriod = o.HorizontalPodAutoscalerSyncPeriod
|
||||
cfg.HorizontalPodAutoscalerDownscaleForbiddenWindow = o.HorizontalPodAutoscalerDownscaleForbiddenWindow
|
||||
cfg.HorizontalPodAutoscalerDownscaleStabilizationWindow = o.HorizontalPodAutoscalerDownscaleStabilizationWindow
|
||||
cfg.HorizontalPodAutoscalerTolerance = o.HorizontalPodAutoscalerTolerance
|
||||
cfg.HorizontalPodAutoscalerUseRESTClients = o.HorizontalPodAutoscalerUseRESTClients
|
||||
cfg.HorizontalPodAutoscalerCPUInitializationPeriod = o.HorizontalPodAutoscalerCPUInitializationPeriod
|
||||
|
|
|
|||
|
|
@ -123,13 +123,14 @@ func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) {
|
|||
EnableGarbageCollector: componentConfig.GarbageCollectorController.EnableGarbageCollector,
|
||||
},
|
||||
HPAController: &HPAControllerOptions{
|
||||
HorizontalPodAutoscalerSyncPeriod: componentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod,
|
||||
HorizontalPodAutoscalerUpscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow,
|
||||
HorizontalPodAutoscalerDownscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow,
|
||||
HorizontalPodAutoscalerCPUInitializationPeriod: componentConfig.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod,
|
||||
HorizontalPodAutoscalerInitialReadinessDelay: componentConfig.HPAController.HorizontalPodAutoscalerInitialReadinessDelay,
|
||||
HorizontalPodAutoscalerTolerance: componentConfig.HPAController.HorizontalPodAutoscalerTolerance,
|
||||
HorizontalPodAutoscalerUseRESTClients: componentConfig.HPAController.HorizontalPodAutoscalerUseRESTClients,
|
||||
HorizontalPodAutoscalerSyncPeriod: componentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod,
|
||||
HorizontalPodAutoscalerUpscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow,
|
||||
HorizontalPodAutoscalerDownscaleForbiddenWindow: componentConfig.HPAController.HorizontalPodAutoscalerDownscaleForbiddenWindow,
|
||||
HorizontalPodAutoscalerDownscaleStabilizationWindow: componentConfig.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow,
|
||||
HorizontalPodAutoscalerCPUInitializationPeriod: componentConfig.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod,
|
||||
HorizontalPodAutoscalerInitialReadinessDelay: componentConfig.HPAController.HorizontalPodAutoscalerInitialReadinessDelay,
|
||||
HorizontalPodAutoscalerTolerance: componentConfig.HPAController.HorizontalPodAutoscalerTolerance,
|
||||
HorizontalPodAutoscalerUseRESTClients: componentConfig.HPAController.HorizontalPodAutoscalerUseRESTClients,
|
||||
},
|
||||
JobController: &JobControllerOptions{
|
||||
ConcurrentJobSyncs: componentConfig.JobController.ConcurrentJobSyncs,
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ func TestAddFlags(t *testing.T) {
|
|||
"--horizontal-pod-autoscaler-downscale-delay=2m",
|
||||
"--horizontal-pod-autoscaler-sync-period=45s",
|
||||
"--horizontal-pod-autoscaler-upscale-delay=1m",
|
||||
"--horizontal-pod-autoscaler-downscale-stabilization=3m",
|
||||
"--horizontal-pod-autoscaler-cpu-initialization-period=90s",
|
||||
"--horizontal-pod-autoscaler-initial-readiness-delay=50s",
|
||||
"--http2-max-streams-per-connection=47",
|
||||
|
|
@ -190,13 +191,14 @@ func TestAddFlags(t *testing.T) {
|
|||
EnableGarbageCollector: false,
|
||||
},
|
||||
HPAController: &HPAControllerOptions{
|
||||
HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 45 * time.Second},
|
||||
HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 1 * time.Minute},
|
||||
HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute},
|
||||
HorizontalPodAutoscalerCPUInitializationPeriod: metav1.Duration{Duration: 90 * time.Second},
|
||||
HorizontalPodAutoscalerInitialReadinessDelay: metav1.Duration{Duration: 50 * time.Second},
|
||||
HorizontalPodAutoscalerTolerance: 0.1,
|
||||
HorizontalPodAutoscalerUseRESTClients: true,
|
||||
HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 45 * time.Second},
|
||||
HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 1 * time.Minute},
|
||||
HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute},
|
||||
HorizontalPodAutoscalerDownscaleStabilizationWindow: metav1.Duration{Duration: 3 * time.Minute},
|
||||
HorizontalPodAutoscalerCPUInitializationPeriod: metav1.Duration{Duration: 90 * time.Second},
|
||||
HorizontalPodAutoscalerInitialReadinessDelay: metav1.Duration{Duration: 50 * time.Second},
|
||||
HorizontalPodAutoscalerTolerance: 0.1,
|
||||
HorizontalPodAutoscalerUseRESTClients: true,
|
||||
},
|
||||
JobController: &JobControllerOptions{
|
||||
ConcurrentJobSyncs: 5,
|
||||
|
|
|
|||
|
|
@ -256,6 +256,9 @@ type HPAControllerConfiguration struct {
|
|||
HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration
|
||||
// horizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed.
|
||||
HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
|
||||
// HorizontalPodAutoscalerDowncaleStabilizationWindow is a period for which autoscaler will look
|
||||
// backwards and not scale down below any recommendation it made during that period.
|
||||
HorizontalPodAutoscalerDownscaleStabilizationWindow metav1.Duration
|
||||
// horizontalPodAutoscalerTolerance is the tolerance for when
|
||||
// resource usage suggests upscaling/downscaling
|
||||
HorizontalPodAutoscalerTolerance float64
|
||||
|
|
|
|||
|
|
@ -77,6 +77,9 @@ func SetDefaults_KubeControllerManagerConfiguration(obj *KubeControllerManagerCo
|
|||
if obj.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow == zero {
|
||||
obj.HPAController.HorizontalPodAutoscalerUpscaleForbiddenWindow = metav1.Duration{Duration: 3 * time.Minute}
|
||||
}
|
||||
if obj.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow == zero {
|
||||
obj.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow = metav1.Duration{Duration: 5 * time.Minute}
|
||||
}
|
||||
if obj.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod == zero {
|
||||
obj.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod = metav1.Duration{Duration: 5 * time.Minute}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -296,14 +296,17 @@ type GarbageCollectorControllerConfiguration struct {
|
|||
}
|
||||
|
||||
type HPAControllerConfiguration struct {
|
||||
// horizontalPodAutoscalerSyncPeriod is the period for syncing the number of
|
||||
// HorizontalPodAutoscalerSyncPeriod is the period for syncing the number of
|
||||
// pods in horizontal pod autoscaler.
|
||||
HorizontalPodAutoscalerSyncPeriod metav1.Duration
|
||||
// horizontalPodAutoscalerUpscaleForbiddenWindow is a period after which next upscale allowed.
|
||||
// HorizontalPodAutoscalerUpscaleForbiddenWindow is a period after which next upscale allowed.
|
||||
HorizontalPodAutoscalerUpscaleForbiddenWindow metav1.Duration
|
||||
// horizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed.
|
||||
// HorizontalPodAutoscalerDowncaleStabilizationWindow is a period for which autoscaler will look
|
||||
// backwards and not scale down below any recommendation it made during that period.
|
||||
HorizontalPodAutoscalerDownscaleStabilizationWindow metav1.Duration
|
||||
// HorizontalPodAutoscalerDownscaleForbiddenWindow is a period after which next downscale allowed.
|
||||
HorizontalPodAutoscalerDownscaleForbiddenWindow metav1.Duration
|
||||
// horizontalPodAutoscalerTolerance is the tolerance for when
|
||||
// HorizontalPodAutoscalerTolerance is the tolerance for when
|
||||
// resource usage suggests upscaling/downscaling
|
||||
HorizontalPodAutoscalerTolerance float64
|
||||
// HorizontalPodAutoscalerUseRESTClients causes the HPA controller to use REST clients
|
||||
|
|
|
|||
|
|
@ -602,6 +602,7 @@ func Convert_componentconfig_GroupResource_To_v1alpha1_GroupResource(in *compone
|
|||
func autoConvert_v1alpha1_HPAControllerConfiguration_To_componentconfig_HPAControllerConfiguration(in *HPAControllerConfiguration, out *componentconfig.HPAControllerConfiguration, s conversion.Scope) error {
|
||||
out.HorizontalPodAutoscalerSyncPeriod = in.HorizontalPodAutoscalerSyncPeriod
|
||||
out.HorizontalPodAutoscalerUpscaleForbiddenWindow = in.HorizontalPodAutoscalerUpscaleForbiddenWindow
|
||||
out.HorizontalPodAutoscalerDownscaleStabilizationWindow = in.HorizontalPodAutoscalerDownscaleStabilizationWindow
|
||||
out.HorizontalPodAutoscalerDownscaleForbiddenWindow = in.HorizontalPodAutoscalerDownscaleForbiddenWindow
|
||||
out.HorizontalPodAutoscalerTolerance = in.HorizontalPodAutoscalerTolerance
|
||||
if err := v1.Convert_Pointer_bool_To_bool(&in.HorizontalPodAutoscalerUseRESTClients, &out.HorizontalPodAutoscalerUseRESTClients, s); err != nil {
|
||||
|
|
@ -621,6 +622,7 @@ func autoConvert_componentconfig_HPAControllerConfiguration_To_v1alpha1_HPAContr
|
|||
out.HorizontalPodAutoscalerSyncPeriod = in.HorizontalPodAutoscalerSyncPeriod
|
||||
out.HorizontalPodAutoscalerUpscaleForbiddenWindow = in.HorizontalPodAutoscalerUpscaleForbiddenWindow
|
||||
out.HorizontalPodAutoscalerDownscaleForbiddenWindow = in.HorizontalPodAutoscalerDownscaleForbiddenWindow
|
||||
out.HorizontalPodAutoscalerDownscaleStabilizationWindow = in.HorizontalPodAutoscalerDownscaleStabilizationWindow
|
||||
out.HorizontalPodAutoscalerTolerance = in.HorizontalPodAutoscalerTolerance
|
||||
if err := v1.Convert_bool_To_Pointer_bool(&in.HorizontalPodAutoscalerUseRESTClients, &out.HorizontalPodAutoscalerUseRESTClients, s); err != nil {
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -241,6 +241,7 @@ func (in *HPAControllerConfiguration) DeepCopyInto(out *HPAControllerConfigurati
|
|||
*out = *in
|
||||
out.HorizontalPodAutoscalerSyncPeriod = in.HorizontalPodAutoscalerSyncPeriod
|
||||
out.HorizontalPodAutoscalerUpscaleForbiddenWindow = in.HorizontalPodAutoscalerUpscaleForbiddenWindow
|
||||
out.HorizontalPodAutoscalerDownscaleStabilizationWindow = in.HorizontalPodAutoscalerDownscaleStabilizationWindow
|
||||
out.HorizontalPodAutoscalerDownscaleForbiddenWindow = in.HorizontalPodAutoscalerDownscaleForbiddenWindow
|
||||
if in.HorizontalPodAutoscalerUseRESTClients != nil {
|
||||
in, out := &in.HorizontalPodAutoscalerUseRESTClients, &out.HorizontalPodAutoscalerUseRESTClients
|
||||
|
|
|
|||
|
|
@ -237,6 +237,7 @@ func (in *HPAControllerConfiguration) DeepCopyInto(out *HPAControllerConfigurati
|
|||
out.HorizontalPodAutoscalerSyncPeriod = in.HorizontalPodAutoscalerSyncPeriod
|
||||
out.HorizontalPodAutoscalerUpscaleForbiddenWindow = in.HorizontalPodAutoscalerUpscaleForbiddenWindow
|
||||
out.HorizontalPodAutoscalerDownscaleForbiddenWindow = in.HorizontalPodAutoscalerDownscaleForbiddenWindow
|
||||
out.HorizontalPodAutoscalerDownscaleStabilizationWindow = in.HorizontalPodAutoscalerDownscaleStabilizationWindow
|
||||
out.HorizontalPodAutoscalerCPUInitializationPeriod = in.HorizontalPodAutoscalerCPUInitializationPeriod
|
||||
out.HorizontalPodAutoscalerInitialReadinessDelay = in.HorizontalPodAutoscalerInitialReadinessDelay
|
||||
return
|
||||
|
|
|
|||
|
|
@ -83,6 +83,7 @@ go_test(
|
|||
"//staging/src/k8s.io/metrics/pkg/client/clientset/versioned/fake:go_default_library",
|
||||
"//staging/src/k8s.io/metrics/pkg/client/custom_metrics/fake:go_default_library",
|
||||
"//staging/src/k8s.io/metrics/pkg/client/external_metrics/fake:go_default_library",
|
||||
"//vendor/github.com/golang/glog:go_default_library",
|
||||
"//vendor/github.com/stretchr/testify/assert:go_default_library",
|
||||
"//vendor/github.com/stretchr/testify/require:go_default_library",
|
||||
"//vendor/k8s.io/heapster/metrics/api/v1/types:go_default_library",
|
||||
|
|
|
|||
|
|
@ -53,6 +53,11 @@ var (
|
|||
scaleUpLimitMinimum = 4.0
|
||||
)
|
||||
|
||||
type timestampedRecommendation struct {
|
||||
recommendation int32
|
||||
timestamp time.Time
|
||||
}
|
||||
|
||||
// HorizontalController is responsible for the synchronizing HPA objects stored
|
||||
// in the system with the actual deployments/replication controllers they
|
||||
// control.
|
||||
|
|
@ -64,7 +69,7 @@ type HorizontalController struct {
|
|||
replicaCalc *ReplicaCalculator
|
||||
eventRecorder record.EventRecorder
|
||||
|
||||
downscaleForbiddenWindow time.Duration
|
||||
downscaleStabilisationWindow time.Duration
|
||||
|
||||
// hpaLister is able to list/get HPAs from the shared cache from the informer passed in to
|
||||
// NewHorizontalController.
|
||||
|
|
@ -73,6 +78,9 @@ type HorizontalController struct {
|
|||
|
||||
// Controllers that need to be synced
|
||||
queue workqueue.RateLimitingInterface
|
||||
|
||||
// Latest unstabilized recommendations for each autoscaler.
|
||||
recommendations map[string][]timestampedRecommendation
|
||||
}
|
||||
|
||||
// NewHorizontalController creates a new HorizontalController.
|
||||
|
|
@ -84,7 +92,7 @@ func NewHorizontalController(
|
|||
replicaCalc *ReplicaCalculator,
|
||||
hpaInformer autoscalinginformers.HorizontalPodAutoscalerInformer,
|
||||
resyncPeriod time.Duration,
|
||||
downscaleForbiddenWindow time.Duration,
|
||||
downscaleStabilisationWindow time.Duration,
|
||||
|
||||
) *HorizontalController {
|
||||
broadcaster := record.NewBroadcaster()
|
||||
|
|
@ -93,13 +101,14 @@ func NewHorizontalController(
|
|||
recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "horizontal-pod-autoscaler"})
|
||||
|
||||
hpaController := &HorizontalController{
|
||||
replicaCalc: replicaCalc,
|
||||
eventRecorder: recorder,
|
||||
scaleNamespacer: scaleNamespacer,
|
||||
hpaNamespacer: hpaNamespacer,
|
||||
downscaleForbiddenWindow: downscaleForbiddenWindow,
|
||||
queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"),
|
||||
mapper: mapper,
|
||||
replicaCalc: replicaCalc,
|
||||
eventRecorder: recorder,
|
||||
scaleNamespacer: scaleNamespacer,
|
||||
hpaNamespacer: hpaNamespacer,
|
||||
downscaleStabilisationWindow: downscaleStabilisationWindow,
|
||||
queue: workqueue.NewNamedRateLimitingQueue(NewDefaultHPARateLimiter(resyncPeriod), "horizontalpodautoscaler"),
|
||||
mapper: mapper,
|
||||
recommendations: map[string][]timestampedRecommendation{},
|
||||
}
|
||||
|
||||
hpaInformer.Informer().AddEventHandlerWithResyncPeriod(
|
||||
|
|
@ -275,10 +284,11 @@ func (a *HorizontalController) reconcileKey(key string) error {
|
|||
hpa, err := a.hpaLister.HorizontalPodAutoscalers(namespace).Get(name)
|
||||
if errors.IsNotFound(err) {
|
||||
glog.Infof("Horizontal Pod Autoscaler %s has been deleted in %s", name, namespace)
|
||||
delete(a.recommendations, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
return a.reconcileAutoscaler(hpa)
|
||||
return a.reconcileAutoscaler(hpa, key)
|
||||
}
|
||||
|
||||
// computeStatusForObjectMetric computes the desired number of replicas for the specified metric of type ObjectMetricSourceType.
|
||||
|
|
@ -431,7 +441,7 @@ func (a *HorizontalController) computeStatusForExternalMetric(currentReplicas in
|
|||
return 0, time.Time{}, "", fmt.Errorf(errMsg)
|
||||
}
|
||||
|
||||
func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.HorizontalPodAutoscaler) error {
|
||||
func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.HorizontalPodAutoscaler, key string) error {
|
||||
// make a copy so that we never mutate the shared informer cache (conversion can mutate the object)
|
||||
hpav1 := hpav1Shared.DeepCopy()
|
||||
// then, convert to autoscaling/v2, which makes our lives easier when calculating metrics
|
||||
|
|
@ -527,24 +537,8 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
|
|||
if desiredReplicas < currentReplicas {
|
||||
rescaleReason = "All metrics below target"
|
||||
}
|
||||
|
||||
desiredReplicas = a.normalizeDesiredReplicas(hpa, currentReplicas, desiredReplicas)
|
||||
|
||||
rescale = a.shouldScale(hpa, currentReplicas, desiredReplicas, timestamp)
|
||||
backoffDown := false
|
||||
backoffUp := false
|
||||
if hpa.Status.LastScaleTime != nil {
|
||||
if !hpa.Status.LastScaleTime.Add(a.downscaleForbiddenWindow).Before(timestamp) {
|
||||
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionFalse, "BackoffDownscale", "the time since the previous scale is still within the downscale forbidden window")
|
||||
backoffDown = true
|
||||
}
|
||||
}
|
||||
|
||||
if !backoffDown && !backoffUp {
|
||||
// mark that we're not backing off
|
||||
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "ReadyForNewScale", "the last scale time was sufficiently old as to warrant a new scale")
|
||||
}
|
||||
|
||||
desiredReplicas = a.normalizeDesiredReplicas(hpa, key, currentReplicas, desiredReplicas)
|
||||
rescale = desiredReplicas != currentReplicas
|
||||
}
|
||||
|
||||
if rescale {
|
||||
|
|
@ -572,9 +566,39 @@ func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.Ho
|
|||
return a.updateStatusIfNeeded(hpaStatusOriginal, hpa)
|
||||
}
|
||||
|
||||
// stabilizeRecommendation:
|
||||
// - replaces old recommendation with the newest recommendation,
|
||||
// - returns max of recommendations that are not older than downscaleStabilisationWindow.
|
||||
func (a *HorizontalController) stabilizeRecommendation(key string, prenormalizedDesiredReplicas int32) int32 {
|
||||
maxRecommendation := prenormalizedDesiredReplicas
|
||||
foundOldSample := false
|
||||
oldSampleIndex := 0
|
||||
cutoff := time.Now().Add(-a.downscaleStabilisationWindow)
|
||||
for i, rec := range a.recommendations[key] {
|
||||
if rec.timestamp.Before(cutoff) {
|
||||
foundOldSample = true
|
||||
oldSampleIndex = i
|
||||
} else if rec.recommendation > maxRecommendation {
|
||||
maxRecommendation = rec.recommendation
|
||||
}
|
||||
}
|
||||
if foundOldSample {
|
||||
a.recommendations[key][oldSampleIndex] = timestampedRecommendation{prenormalizedDesiredReplicas, time.Now()}
|
||||
} else {
|
||||
a.recommendations[key] = append(a.recommendations[key], timestampedRecommendation{prenormalizedDesiredReplicas, time.Now()})
|
||||
}
|
||||
return maxRecommendation
|
||||
}
|
||||
|
||||
// normalizeDesiredReplicas takes the metrics desired replicas value and normalizes it based on the appropriate conditions (i.e. < maxReplicas, >
|
||||
// minReplicas, etc...)
|
||||
func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas int32, prenormalizedDesiredReplicas int32) int32 {
|
||||
func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.HorizontalPodAutoscaler, key string, currentReplicas int32, prenormalizedDesiredReplicas int32) int32 {
|
||||
stabilizedRecommendation := a.stabilizeRecommendation(key, prenormalizedDesiredReplicas)
|
||||
if stabilizedRecommendation != prenormalizedDesiredReplicas {
|
||||
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "ScaleDownStabilized", "recent recommendations were higher than current one, applying the highest recent recommendation")
|
||||
} else {
|
||||
setCondition(hpa, autoscalingv2.AbleToScale, v1.ConditionTrue, "ReadyForNewScale", "recommended size matches current size")
|
||||
}
|
||||
var minReplicas int32
|
||||
if hpa.Spec.MinReplicas != nil {
|
||||
minReplicas = *hpa.Spec.MinReplicas
|
||||
|
|
@ -582,9 +606,9 @@ func (a *HorizontalController) normalizeDesiredReplicas(hpa *autoscalingv2.Horiz
|
|||
minReplicas = 0
|
||||
}
|
||||
|
||||
desiredReplicas, condition, reason := convertDesiredReplicasWithRules(currentReplicas, prenormalizedDesiredReplicas, minReplicas, hpa.Spec.MaxReplicas)
|
||||
desiredReplicas, condition, reason := convertDesiredReplicasWithRules(currentReplicas, stabilizedRecommendation, minReplicas, hpa.Spec.MaxReplicas)
|
||||
|
||||
if desiredReplicas == prenormalizedDesiredReplicas {
|
||||
if desiredReplicas == stabilizedRecommendation {
|
||||
setCondition(hpa, autoscalingv2.ScalingLimited, v1.ConditionFalse, condition, reason)
|
||||
} else {
|
||||
setCondition(hpa, autoscalingv2.ScalingLimited, v1.ConditionTrue, condition, reason)
|
||||
|
|
@ -641,29 +665,6 @@ func calculateScaleUpLimit(currentReplicas int32) int32 {
|
|||
return int32(math.Max(scaleUpLimitFactor*float64(currentReplicas), scaleUpLimitMinimum))
|
||||
}
|
||||
|
||||
func (a *HorizontalController) shouldScale(hpa *autoscalingv2.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int32, timestamp time.Time) bool {
|
||||
if desiredReplicas == currentReplicas {
|
||||
return false
|
||||
}
|
||||
|
||||
if hpa.Status.LastScaleTime == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
// Going down only if the usageRatio dropped significantly below the target
|
||||
// and there was no rescaling in the last downscaleForbiddenWindow.
|
||||
if desiredReplicas < currentReplicas && hpa.Status.LastScaleTime.Add(a.downscaleForbiddenWindow).Before(timestamp) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Going up only if the usage ratio increased significantly above the target.
|
||||
if desiredReplicas > currentReplicas {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// scaleForResourceMappings attempts to fetch the scale for the
|
||||
// resource with the given name and namespace, trying each RESTMapping
|
||||
// in turn until a working one is found. If none work, the first error
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ import (
|
|||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/golang/glog"
|
||||
_ "k8s.io/kubernetes/pkg/apis/autoscaling/install"
|
||||
_ "k8s.io/kubernetes/pkg/apis/extensions/install"
|
||||
)
|
||||
|
|
@ -129,6 +130,8 @@ type testCase struct {
|
|||
testCMClient *cmfake.FakeCustomMetricsClient
|
||||
testEMClient *emfake.FakeExternalMetricsClient
|
||||
testScaleClient *scalefake.FakeScaleClient
|
||||
|
||||
recommendations []timestampedRecommendation
|
||||
}
|
||||
|
||||
// Needs to be called under a lock.
|
||||
|
|
@ -662,7 +665,7 @@ func (tc *testCase) setupController(t *testing.T) (*HorizontalController, inform
|
|||
replicaCalc := NewReplicaCalculator(metricsClient, testClient.Core(), defaultTestingTolerance, defaultTestingCpuInitializationPeriod, defaultTestingDelayOfInitialReadinessStatus)
|
||||
|
||||
informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
|
||||
defaultDownscaleForbiddenWindow := 5 * time.Minute
|
||||
defaultDownscalestabilizationWindow := 5 * time.Minute
|
||||
|
||||
hpaController := NewHorizontalController(
|
||||
eventClient.Core(),
|
||||
|
|
@ -672,9 +675,12 @@ func (tc *testCase) setupController(t *testing.T) (*HorizontalController, inform
|
|||
replicaCalc,
|
||||
informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
|
||||
controller.NoResyncPeriodFunc(),
|
||||
defaultDownscaleForbiddenWindow,
|
||||
defaultDownscalestabilizationWindow,
|
||||
)
|
||||
hpaController.hpaListerSynced = alwaysReady
|
||||
if tc.recommendations != nil {
|
||||
hpaController.recommendations["test-namespace/test-hpa"] = tc.recommendations
|
||||
}
|
||||
|
||||
return hpaController, informerFactory
|
||||
}
|
||||
|
|
@ -709,6 +715,7 @@ func (tc *testCase) runTestWithController(t *testing.T, hpaController *Horizonta
|
|||
func (tc *testCase) runTest(t *testing.T) {
|
||||
hpaController, informerFactory := tc.setupController(t)
|
||||
tc.runTestWithController(t, hpaController, informerFactory)
|
||||
glog.Errorf("recommendations: %+v", hpaController.recommendations)
|
||||
}
|
||||
|
||||
func TestScaleUp(t *testing.T) {
|
||||
|
|
@ -2080,8 +2087,7 @@ func TestNoBackoffUpscaleCMNoBackoffCpu(t *testing.T) {
|
|||
tc.runTest(t)
|
||||
}
|
||||
|
||||
func TestBackoffDownscale(t *testing.T) {
|
||||
time := metav1.Time{Time: time.Now().Add(-4 * time.Minute)}
|
||||
func TestStabilizeDownscale(t *testing.T) {
|
||||
tc := testCase{
|
||||
minReplicas: 1,
|
||||
maxReplicas: 5,
|
||||
|
|
@ -2091,16 +2097,19 @@ func TestBackoffDownscale(t *testing.T) {
|
|||
reportedLevels: []uint64{50, 50, 50},
|
||||
reportedCPURequests: []resource.Quantity{resource.MustParse("0.1"), resource.MustParse("0.1"), resource.MustParse("0.1")},
|
||||
useMetricsAPI: true,
|
||||
lastScaleTime: &time,
|
||||
expectedConditions: statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{
|
||||
Type: autoscalingv2.AbleToScale,
|
||||
Status: v1.ConditionTrue,
|
||||
Reason: "ReadyForNewScale",
|
||||
}, autoscalingv2.HorizontalPodAutoscalerCondition{
|
||||
Type: autoscalingv2.AbleToScale,
|
||||
Status: v1.ConditionFalse,
|
||||
Reason: "BackoffDownscale",
|
||||
Status: v1.ConditionTrue,
|
||||
Reason: "ScaleDownStabilized",
|
||||
}),
|
||||
recommendations: []timestampedRecommendation{
|
||||
{10, time.Now().Add(-10 * time.Minute)},
|
||||
{4, time.Now().Add(-1 * time.Minute)},
|
||||
},
|
||||
}
|
||||
tc.runTest(t)
|
||||
}
|
||||
|
|
@ -2278,7 +2287,7 @@ func TestAvoidUncessaryUpdates(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if err := controller.reconcileAutoscaler(&initialHPAs.Items[0]); err != nil {
|
||||
if err := controller.reconcileAutoscaler(&initialHPAs.Items[0], ""); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -2353,4 +2362,85 @@ func TestConvertDesiredReplicasWithRules(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestNormalizeDesiredReplicas(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
recommendations []timestampedRecommendation
|
||||
prenormalizedDesiredReplicas int32
|
||||
expectedStabilizedReplicas int32
|
||||
expectedLogLength int
|
||||
}{
|
||||
{
|
||||
"empty log",
|
||||
"",
|
||||
[]timestampedRecommendation{},
|
||||
5,
|
||||
5,
|
||||
1,
|
||||
},
|
||||
{
|
||||
"stabilize",
|
||||
"",
|
||||
[]timestampedRecommendation{
|
||||
{4, time.Now().Add(-2 * time.Minute)},
|
||||
{5, time.Now().Add(-1 * time.Minute)},
|
||||
},
|
||||
3,
|
||||
5,
|
||||
3,
|
||||
},
|
||||
{
|
||||
"no stabilize",
|
||||
"",
|
||||
[]timestampedRecommendation{
|
||||
{1, time.Now().Add(-2 * time.Minute)},
|
||||
{2, time.Now().Add(-1 * time.Minute)},
|
||||
},
|
||||
3,
|
||||
3,
|
||||
3,
|
||||
},
|
||||
{
|
||||
"no stabilize - old recommendations",
|
||||
"",
|
||||
[]timestampedRecommendation{
|
||||
{10, time.Now().Add(-10 * time.Minute)},
|
||||
{9, time.Now().Add(-9 * time.Minute)},
|
||||
},
|
||||
3,
|
||||
3,
|
||||
2,
|
||||
},
|
||||
{
|
||||
"stabilize - old recommendations",
|
||||
"",
|
||||
[]timestampedRecommendation{
|
||||
{10, time.Now().Add(-10 * time.Minute)},
|
||||
{4, time.Now().Add(-1 * time.Minute)},
|
||||
{5, time.Now().Add(-2 * time.Minute)},
|
||||
{9, time.Now().Add(-9 * time.Minute)},
|
||||
},
|
||||
3,
|
||||
5,
|
||||
4,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
hc := HorizontalController{
|
||||
downscaleStabilisationWindow: 5 * time.Minute,
|
||||
recommendations: map[string][]timestampedRecommendation{
|
||||
tc.key: tc.recommendations,
|
||||
},
|
||||
}
|
||||
r := hc.stabilizeRecommendation(tc.key, tc.prenormalizedDesiredReplicas)
|
||||
if r != tc.expectedStabilizedReplicas {
|
||||
t.Errorf("[%s] got %d stabilized replicas, expected %d", tc.name, r, tc.expectedStabilizedReplicas)
|
||||
}
|
||||
if len(hc.recommendations[tc.key]) != tc.expectedLogLength {
|
||||
t.Errorf("[%s] after stabilization recommendations log has %d entries, expected %d", tc.name, len(hc.recommendations[tc.key]), tc.expectedLogLength)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: add more tests
|
||||
|
|
|
|||
|
|
@ -488,7 +488,7 @@ func (tc *legacyTestCase) runTest(t *testing.T) {
|
|||
replicaCalc := NewReplicaCalculator(metricsClient, testClient.Core(), defaultTestingTolerance, defaultTestingCpuInitializationPeriod, defaultTestingDelayOfInitialReadinessStatus)
|
||||
|
||||
informerFactory := informers.NewSharedInformerFactory(testClient, controller.NoResyncPeriodFunc())
|
||||
defaultDownscaleForbiddenWindow := 5 * time.Minute
|
||||
defaultDownscaleStabilisationWindow := 5 * time.Minute
|
||||
|
||||
hpaController := NewHorizontalController(
|
||||
eventClient.Core(),
|
||||
|
|
@ -498,7 +498,7 @@ func (tc *legacyTestCase) runTest(t *testing.T) {
|
|||
replicaCalc,
|
||||
informerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
|
||||
controller.NoResyncPeriodFunc(),
|
||||
defaultDownscaleForbiddenWindow,
|
||||
defaultDownscaleStabilisationWindow,
|
||||
)
|
||||
hpaController.hpaListerSynced = alwaysReady
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue