diff --git a/pkg/controller/podautoscaler/horizontal.go b/pkg/controller/podautoscaler/horizontal.go index 2ca40458d6..b3e4f87996 100644 --- a/pkg/controller/podautoscaler/horizontal.go +++ b/pkg/controller/podautoscaler/horizontal.go @@ -189,9 +189,9 @@ func (a *HorizontalController) processNextWorkItem() bool { return true } -// Computes the desired number of replicas for the metric specifications listed in the HPA, returning the maximum -// of the computed replica counts, a description of the associated metric, and the statuses of all metrics -// computed. +// computeReplicasForMetrics computes the desired number of replicas for the metric specifications listed in the HPA, +// returning the maximum of the computed replica counts, a description of the associated metric, and the statuses of +// all metrics computed. func (a *HorizontalController) computeReplicasForMetrics(hpa *autoscalingv2.HorizontalPodAutoscaler, scale *autoscalingv1.Scale, metricSpecs []autoscalingv2.MetricSpec) (replicas int32, metric string, statuses []autoscalingv2.MetricStatus, timestamp time.Time, err error) { @@ -216,126 +216,29 @@ func (a *HorizontalController) computeReplicasForMetrics(hpa *autoscalingv2.Hori } var replicaCountProposal int32 - var utilizationProposal int64 var timestampProposal time.Time var metricNameProposal string switch metricSpec.Type { case autoscalingv2.ObjectMetricSourceType: - replicaCountProposal, utilizationProposal, timestampProposal, err = a.replicaCalc.GetObjectMetricReplicas(currentReplicas, metricSpec.Object.TargetValue.MilliValue(), metricSpec.Object.MetricName, hpa.Namespace, &metricSpec.Object.Target, selector) + replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForObjectMetric(currentReplicas, metricSpec, hpa, selector, &statuses[i]) if err != nil { - a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetObjectMetric", err.Error()) - setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetObjectMetric", "the HPA was unable to compute the replica count: %v", err) return 0, "", nil, time.Time{}, fmt.Errorf("failed to get object metric value: %v", err) } - metricNameProposal = fmt.Sprintf("%s metric %s", metricSpec.Object.Target.Kind, metricSpec.Object.MetricName) - statuses[i] = autoscalingv2.MetricStatus{ - Type: autoscalingv2.ObjectMetricSourceType, - Object: &autoscalingv2.ObjectMetricStatus{ - Target: metricSpec.Object.Target, - MetricName: metricSpec.Object.MetricName, - CurrentValue: *resource.NewMilliQuantity(utilizationProposal, resource.DecimalSI), - }, - } case autoscalingv2.PodsMetricSourceType: - replicaCountProposal, utilizationProposal, timestampProposal, err = a.replicaCalc.GetMetricReplicas(currentReplicas, metricSpec.Pods.TargetAverageValue.MilliValue(), metricSpec.Pods.MetricName, hpa.Namespace, selector) + replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForPodsMetric(currentReplicas, metricSpec, hpa, selector, &statuses[i]) if err != nil { - a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetPodsMetric", err.Error()) - setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetPodsMetric", "the HPA was unable to compute the replica count: %v", err) - return 0, "", nil, time.Time{}, fmt.Errorf("failed to get pods metric value: %v", err) - } - metricNameProposal = fmt.Sprintf("pods metric %s", metricSpec.Pods.MetricName) - statuses[i] = autoscalingv2.MetricStatus{ - Type: autoscalingv2.PodsMetricSourceType, - Pods: &autoscalingv2.PodsMetricStatus{ - MetricName: metricSpec.Pods.MetricName, - CurrentAverageValue: *resource.NewMilliQuantity(utilizationProposal, resource.DecimalSI), - }, + return 0, "", nil, time.Time{}, fmt.Errorf("failed to get object metric value: %v", err) } case autoscalingv2.ResourceMetricSourceType: - if metricSpec.Resource.TargetAverageValue != nil { - var rawProposal int64 - replicaCountProposal, rawProposal, timestampProposal, err = a.replicaCalc.GetRawResourceReplicas(currentReplicas, metricSpec.Resource.TargetAverageValue.MilliValue(), metricSpec.Resource.Name, hpa.Namespace, selector) - if err != nil { - a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetResourceMetric", err.Error()) - setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetResourceMetric", "the HPA was unable to compute the replica count: %v", err) - return 0, "", nil, time.Time{}, fmt.Errorf("failed to get %s utilization: %v", metricSpec.Resource.Name, err) - } - metricNameProposal = fmt.Sprintf("%s resource", metricSpec.Resource.Name) - statuses[i] = autoscalingv2.MetricStatus{ - Type: autoscalingv2.ResourceMetricSourceType, - Resource: &autoscalingv2.ResourceMetricStatus{ - Name: metricSpec.Resource.Name, - CurrentAverageValue: *resource.NewMilliQuantity(rawProposal, resource.DecimalSI), - }, - } - } else { - // set a default utilization percentage if none is set - if metricSpec.Resource.TargetAverageUtilization == nil { - errMsg := "invalid resource metric source: neither a utilization target nor a value target was set" - a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetResourceMetric", errMsg) - setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetResourceMetric", "the HPA was unable to compute the replica count: %s", errMsg) - return 0, "", nil, time.Time{}, fmt.Errorf(errMsg) - } - - targetUtilization := *metricSpec.Resource.TargetAverageUtilization - - var percentageProposal int32 - var rawProposal int64 - replicaCountProposal, percentageProposal, rawProposal, timestampProposal, err = a.replicaCalc.GetResourceReplicas(currentReplicas, targetUtilization, metricSpec.Resource.Name, hpa.Namespace, selector) - if err != nil { - a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetResourceMetric", err.Error()) - setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetResourceMetric", "the HPA was unable to compute the replica count: %v", err) - return 0, "", nil, time.Time{}, fmt.Errorf("failed to get %s utilization: %v", metricSpec.Resource.Name, err) - } - metricNameProposal = fmt.Sprintf("%s resource utilization (percentage of request)", metricSpec.Resource.Name) - statuses[i] = autoscalingv2.MetricStatus{ - Type: autoscalingv2.ResourceMetricSourceType, - Resource: &autoscalingv2.ResourceMetricStatus{ - Name: metricSpec.Resource.Name, - CurrentAverageUtilization: &percentageProposal, - CurrentAverageValue: *resource.NewMilliQuantity(rawProposal, resource.DecimalSI), - }, - } + replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForResourceMetric(currentReplicas, metricSpec, hpa, selector, &statuses[i]) + if err != nil { + return 0, "", nil, time.Time{}, err } case autoscalingv2.ExternalMetricSourceType: - if metricSpec.External.TargetAverageValue != nil { - replicaCountProposal, utilizationProposal, timestampProposal, err = a.replicaCalc.GetExternalPerPodMetricReplicas(currentReplicas, metricSpec.External.TargetAverageValue.MilliValue(), metricSpec.External.MetricName, hpa.Namespace, metricSpec.External.MetricSelector) - if err != nil { - a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetExternalMetric", err.Error()) - setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetExternalMetric", "the HPA was unable to compute the replica count: %v", err) - return 0, "", nil, time.Time{}, fmt.Errorf("failed to get %s external metric: %v", metricSpec.External.MetricName, err) - } - metricNameProposal = fmt.Sprintf("external metric %s(%+v)", metricSpec.External.MetricName, metricSpec.External.MetricSelector) - statuses[i] = autoscalingv2.MetricStatus{ - Type: autoscalingv2.ExternalMetricSourceType, - External: &autoscalingv2.ExternalMetricStatus{ - MetricSelector: metricSpec.External.MetricSelector, - MetricName: metricSpec.External.MetricName, - CurrentAverageValue: resource.NewMilliQuantity(utilizationProposal, resource.DecimalSI), - }, - } - } else if metricSpec.External.TargetValue != nil { - replicaCountProposal, utilizationProposal, timestampProposal, err = a.replicaCalc.GetExternalMetricReplicas(currentReplicas, metricSpec.External.TargetValue.MilliValue(), metricSpec.External.MetricName, hpa.Namespace, metricSpec.External.MetricSelector, selector) - if err != nil { - a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetExternalMetric", err.Error()) - setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetExternalMetric", "the HPA was unable to compute the replica count: %v", err) - return 0, "", nil, time.Time{}, fmt.Errorf("failed to get external metric %s: %v", metricSpec.External.MetricName, err) - } - metricNameProposal = fmt.Sprintf("external metric %s(%+v)", metricSpec.External.MetricName, metricSpec.External.MetricSelector) - statuses[i] = autoscalingv2.MetricStatus{ - Type: autoscalingv2.ExternalMetricSourceType, - External: &autoscalingv2.ExternalMetricStatus{ - MetricSelector: metricSpec.External.MetricSelector, - MetricName: metricSpec.External.MetricName, - CurrentValue: *resource.NewMilliQuantity(utilizationProposal, resource.DecimalSI), - }, - } - } else { - errMsg := "invalid external metric source: neither a value target nor an average value target was set" - a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetExternalMetric", errMsg) - setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetExternalMetric", "the HPA was unable to compute the replica count: %v", err) - return 0, "", nil, time.Time{}, fmt.Errorf(errMsg) + replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForExternalMetric(currentReplicas, metricSpec, hpa, selector, &statuses[i]) + if err != nil { + return 0, "", nil, time.Time{}, err } default: errMsg := fmt.Sprintf("unknown metric source type %q", string(metricSpec.Type)) @@ -370,6 +273,134 @@ func (a *HorizontalController) reconcileKey(key string) error { return a.reconcileAutoscaler(hpa) } +// computeStatusForObjectMetric computes the desired number of replicas for the specified metric of type ObjectMetricSourceType. +func (a *HorizontalController) computeStatusForObjectMetric(currentReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (int32, time.Time, string, error) { + replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetObjectMetricReplicas(currentReplicas, metricSpec.Object.TargetValue.MilliValue(), metricSpec.Object.MetricName, hpa.Namespace, &metricSpec.Object.Target, selector) + if err != nil { + a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetObjectMetric", err.Error()) + setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetObjectMetric", "the HPA was unable to compute the replica count: %v", err) + return 0, timestampProposal, "", err + } + *status = autoscalingv2.MetricStatus{ + Type: autoscalingv2.ObjectMetricSourceType, + Object: &autoscalingv2.ObjectMetricStatus{ + Target: metricSpec.Object.Target, + MetricName: metricSpec.Object.MetricName, + CurrentValue: *resource.NewMilliQuantity(utilizationProposal, resource.DecimalSI), + }, + } + return replicaCountProposal, timestampProposal, fmt.Sprintf("%s metric %s", metricSpec.Object.Target.Kind, metricSpec.Object.MetricName), nil +} + +// computeStatusForPodsMetric computes the desired number of replicas for the specified metric of type PodsMetricSourceType. +func (a *HorizontalController) computeStatusForPodsMetric(currentReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (int32, time.Time, string, error) { + replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetMetricReplicas(currentReplicas, metricSpec.Pods.TargetAverageValue.MilliValue(), metricSpec.Pods.MetricName, hpa.Namespace, selector) + if err != nil { + a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetPodsMetric", err.Error()) + setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetPodsMetric", "the HPA was unable to compute the replica count: %v", err) + return 0, timestampProposal, "", err + } + *status = autoscalingv2.MetricStatus{ + Type: autoscalingv2.PodsMetricSourceType, + Pods: &autoscalingv2.PodsMetricStatus{ + MetricName: metricSpec.Pods.MetricName, + CurrentAverageValue: *resource.NewMilliQuantity(utilizationProposal, resource.DecimalSI), + }, + } + + return replicaCountProposal, timestampProposal, fmt.Sprintf("pods metric %s", metricSpec.Pods.MetricName), nil +} + +// computeStatusForResourceMetric computes the desired number of replicas for the specified metric of type ResourceMetricSourceType. +func (a *HorizontalController) computeStatusForResourceMetric(currentReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (int32, time.Time, string, error) { + if metricSpec.Resource.TargetAverageValue != nil { + var rawProposal int64 + replicaCountProposal, rawProposal, timestampProposal, err := a.replicaCalc.GetRawResourceReplicas(currentReplicas, metricSpec.Resource.TargetAverageValue.MilliValue(), metricSpec.Resource.Name, hpa.Namespace, selector) + if err != nil { + a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetResourceMetric", err.Error()) + setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetResourceMetric", "the HPA was unable to compute the replica count: %v", err) + return 0, time.Time{}, "", fmt.Errorf("failed to get %s utilization: %v", metricSpec.Resource.Name, err) + } + metricNameProposal := fmt.Sprintf("%s resource", metricSpec.Resource.Name) + status = &autoscalingv2.MetricStatus{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricStatus{ + Name: metricSpec.Resource.Name, + CurrentAverageValue: *resource.NewMilliQuantity(rawProposal, resource.DecimalSI), + }, + } + return replicaCountProposal, timestampProposal, metricNameProposal, nil + } else { + if metricSpec.Resource.TargetAverageUtilization == nil { + errMsg := "invalid resource metric source: neither a utilization target nor a value target was set" + a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetResourceMetric", errMsg) + setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetResourceMetric", "the HPA was unable to compute the replica count: %s", errMsg) + return 0, time.Time{}, "", fmt.Errorf(errMsg) + } + targetUtilization := *metricSpec.Resource.TargetAverageUtilization + var percentageProposal int32 + var rawProposal int64 + replicaCountProposal, percentageProposal, rawProposal, timestampProposal, err := a.replicaCalc.GetResourceReplicas(currentReplicas, targetUtilization, metricSpec.Resource.Name, hpa.Namespace, selector) + if err != nil { + a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetResourceMetric", err.Error()) + setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetResourceMetric", "the HPA was unable to compute the replica count: %v", err) + return 0, time.Time{}, "", fmt.Errorf("failed to get %s utilization: %v", metricSpec.Resource.Name, err) + } + metricNameProposal := fmt.Sprintf("%s resource utilization (percentage of request)", metricSpec.Resource.Name) + *status = autoscalingv2.MetricStatus{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricStatus{ + Name: metricSpec.Resource.Name, + CurrentAverageUtilization: &percentageProposal, + CurrentAverageValue: *resource.NewMilliQuantity(rawProposal, resource.DecimalSI), + }, + } + return replicaCountProposal, timestampProposal, metricNameProposal, nil + } +} + +// computeStatusForExternalMetric computes the desired number of replicas for the specified metric of type ExternalMetricSourceType. +func (a *HorizontalController) computeStatusForExternalMetric(currentReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (int32, time.Time, string, error) { + if metricSpec.External.TargetAverageValue != nil { + replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetExternalPerPodMetricReplicas(currentReplicas, metricSpec.External.TargetAverageValue.MilliValue(), metricSpec.External.MetricName, hpa.Namespace, metricSpec.External.MetricSelector) + if err != nil { + a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetExternalMetric", err.Error()) + setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetExternalMetric", "the HPA was unable to compute the replica count: %v", err) + return 0, time.Time{}, "", fmt.Errorf("failed to get %s external metric: %v", metricSpec.External.MetricName, err) + } + *status = autoscalingv2.MetricStatus{ + Type: autoscalingv2.ExternalMetricSourceType, + External: &autoscalingv2.ExternalMetricStatus{ + MetricSelector: metricSpec.External.MetricSelector, + MetricName: metricSpec.External.MetricName, + CurrentAverageValue: resource.NewMilliQuantity(utilizationProposal, resource.DecimalSI), + }, + } + return replicaCountProposal, timestampProposal, fmt.Sprintf("external metric %s(%+v)", metricSpec.External.MetricName, metricSpec.External.MetricSelector), nil + } + if metricSpec.External.TargetValue != nil { + replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetExternalMetricReplicas(currentReplicas, metricSpec.External.TargetValue.MilliValue(), metricSpec.External.MetricName, hpa.Namespace, metricSpec.External.MetricSelector, selector) + if err != nil { + a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetExternalMetric", err.Error()) + setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetExternalMetric", "the HPA was unable to compute the replica count: %v", err) + return 0, time.Time{}, "", fmt.Errorf("failed to get external metric %s: %v", metricSpec.External.MetricName, err) + } + *status = autoscalingv2.MetricStatus{ + Type: autoscalingv2.ExternalMetricSourceType, + External: &autoscalingv2.ExternalMetricStatus{ + MetricSelector: metricSpec.External.MetricSelector, + MetricName: metricSpec.External.MetricName, + CurrentValue: *resource.NewMilliQuantity(utilizationProposal, resource.DecimalSI), + }, + } + return replicaCountProposal, timestampProposal, fmt.Sprintf("external metric %s(%+v)", metricSpec.External.MetricName, metricSpec.External.MetricSelector), nil + } + errMsg := "invalid external metric source: neither a value target nor an average value target was set" + a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetExternalMetric", errMsg) + setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetExternalMetric", "the HPA was unable to compute the replica count: %s", errMsg) + return 0, time.Time{}, "", fmt.Errorf(errMsg) +} + func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.HorizontalPodAutoscaler) error { // make a copy so that we never mutate the shared informer cache (conversion can mutate the object) hpav1 := hpav1Shared.DeepCopy()