mirror of https://github.com/k3s-io/k3s
Merge pull request #65828 from jbartosik/hpa-improv-refactor
Automatic merge from submit-queue (batch tested with PRs 66175, 66324, 65828, 65901, 66350). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Chop computeReplicasForMetrics to smaller pieces Split computeReplicasForMetrics function into smaller pieces. **Release note**: ```release-note NONE ```pull/8/head
commit
3f191aef8b
|
@ -189,9 +189,9 @@ func (a *HorizontalController) processNextWorkItem() bool {
|
|||
return true
|
||||
}
|
||||
|
||||
// Computes the desired number of replicas for the metric specifications listed in the HPA, returning the maximum
|
||||
// of the computed replica counts, a description of the associated metric, and the statuses of all metrics
|
||||
// computed.
|
||||
// computeReplicasForMetrics computes the desired number of replicas for the metric specifications listed in the HPA,
|
||||
// returning the maximum of the computed replica counts, a description of the associated metric, and the statuses of
|
||||
// all metrics computed.
|
||||
func (a *HorizontalController) computeReplicasForMetrics(hpa *autoscalingv2.HorizontalPodAutoscaler, scale *autoscalingv1.Scale,
|
||||
metricSpecs []autoscalingv2.MetricSpec) (replicas int32, metric string, statuses []autoscalingv2.MetricStatus, timestamp time.Time, err error) {
|
||||
|
||||
|
@ -216,126 +216,29 @@ func (a *HorizontalController) computeReplicasForMetrics(hpa *autoscalingv2.Hori
|
|||
}
|
||||
|
||||
var replicaCountProposal int32
|
||||
var utilizationProposal int64
|
||||
var timestampProposal time.Time
|
||||
var metricNameProposal string
|
||||
|
||||
switch metricSpec.Type {
|
||||
case autoscalingv2.ObjectMetricSourceType:
|
||||
replicaCountProposal, utilizationProposal, timestampProposal, err = a.replicaCalc.GetObjectMetricReplicas(currentReplicas, metricSpec.Object.TargetValue.MilliValue(), metricSpec.Object.MetricName, hpa.Namespace, &metricSpec.Object.Target, selector)
|
||||
replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForObjectMetric(currentReplicas, metricSpec, hpa, selector, &statuses[i])
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetObjectMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetObjectMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, "", nil, time.Time{}, fmt.Errorf("failed to get object metric value: %v", err)
|
||||
}
|
||||
metricNameProposal = fmt.Sprintf("%s metric %s", metricSpec.Object.Target.Kind, metricSpec.Object.MetricName)
|
||||
statuses[i] = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.ObjectMetricSourceType,
|
||||
Object: &autoscalingv2.ObjectMetricStatus{
|
||||
Target: metricSpec.Object.Target,
|
||||
MetricName: metricSpec.Object.MetricName,
|
||||
CurrentValue: *resource.NewMilliQuantity(utilizationProposal, resource.DecimalSI),
|
||||
},
|
||||
}
|
||||
case autoscalingv2.PodsMetricSourceType:
|
||||
replicaCountProposal, utilizationProposal, timestampProposal, err = a.replicaCalc.GetMetricReplicas(currentReplicas, metricSpec.Pods.TargetAverageValue.MilliValue(), metricSpec.Pods.MetricName, hpa.Namespace, selector)
|
||||
replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForPodsMetric(currentReplicas, metricSpec, hpa, selector, &statuses[i])
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetPodsMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetPodsMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, "", nil, time.Time{}, fmt.Errorf("failed to get pods metric value: %v", err)
|
||||
}
|
||||
metricNameProposal = fmt.Sprintf("pods metric %s", metricSpec.Pods.MetricName)
|
||||
statuses[i] = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.PodsMetricSourceType,
|
||||
Pods: &autoscalingv2.PodsMetricStatus{
|
||||
MetricName: metricSpec.Pods.MetricName,
|
||||
CurrentAverageValue: *resource.NewMilliQuantity(utilizationProposal, resource.DecimalSI),
|
||||
},
|
||||
return 0, "", nil, time.Time{}, fmt.Errorf("failed to get object metric value: %v", err)
|
||||
}
|
||||
case autoscalingv2.ResourceMetricSourceType:
|
||||
if metricSpec.Resource.TargetAverageValue != nil {
|
||||
var rawProposal int64
|
||||
replicaCountProposal, rawProposal, timestampProposal, err = a.replicaCalc.GetRawResourceReplicas(currentReplicas, metricSpec.Resource.TargetAverageValue.MilliValue(), metricSpec.Resource.Name, hpa.Namespace, selector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetResourceMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetResourceMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, "", nil, time.Time{}, fmt.Errorf("failed to get %s utilization: %v", metricSpec.Resource.Name, err)
|
||||
}
|
||||
metricNameProposal = fmt.Sprintf("%s resource", metricSpec.Resource.Name)
|
||||
statuses[i] = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.ResourceMetricSourceType,
|
||||
Resource: &autoscalingv2.ResourceMetricStatus{
|
||||
Name: metricSpec.Resource.Name,
|
||||
CurrentAverageValue: *resource.NewMilliQuantity(rawProposal, resource.DecimalSI),
|
||||
},
|
||||
}
|
||||
} else {
|
||||
// set a default utilization percentage if none is set
|
||||
if metricSpec.Resource.TargetAverageUtilization == nil {
|
||||
errMsg := "invalid resource metric source: neither a utilization target nor a value target was set"
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetResourceMetric", errMsg)
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetResourceMetric", "the HPA was unable to compute the replica count: %s", errMsg)
|
||||
return 0, "", nil, time.Time{}, fmt.Errorf(errMsg)
|
||||
}
|
||||
|
||||
targetUtilization := *metricSpec.Resource.TargetAverageUtilization
|
||||
|
||||
var percentageProposal int32
|
||||
var rawProposal int64
|
||||
replicaCountProposal, percentageProposal, rawProposal, timestampProposal, err = a.replicaCalc.GetResourceReplicas(currentReplicas, targetUtilization, metricSpec.Resource.Name, hpa.Namespace, selector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetResourceMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetResourceMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, "", nil, time.Time{}, fmt.Errorf("failed to get %s utilization: %v", metricSpec.Resource.Name, err)
|
||||
}
|
||||
metricNameProposal = fmt.Sprintf("%s resource utilization (percentage of request)", metricSpec.Resource.Name)
|
||||
statuses[i] = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.ResourceMetricSourceType,
|
||||
Resource: &autoscalingv2.ResourceMetricStatus{
|
||||
Name: metricSpec.Resource.Name,
|
||||
CurrentAverageUtilization: &percentageProposal,
|
||||
CurrentAverageValue: *resource.NewMilliQuantity(rawProposal, resource.DecimalSI),
|
||||
},
|
||||
}
|
||||
replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForResourceMetric(currentReplicas, metricSpec, hpa, selector, &statuses[i])
|
||||
if err != nil {
|
||||
return 0, "", nil, time.Time{}, err
|
||||
}
|
||||
case autoscalingv2.ExternalMetricSourceType:
|
||||
if metricSpec.External.TargetAverageValue != nil {
|
||||
replicaCountProposal, utilizationProposal, timestampProposal, err = a.replicaCalc.GetExternalPerPodMetricReplicas(currentReplicas, metricSpec.External.TargetAverageValue.MilliValue(), metricSpec.External.MetricName, hpa.Namespace, metricSpec.External.MetricSelector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetExternalMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetExternalMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, "", nil, time.Time{}, fmt.Errorf("failed to get %s external metric: %v", metricSpec.External.MetricName, err)
|
||||
}
|
||||
metricNameProposal = fmt.Sprintf("external metric %s(%+v)", metricSpec.External.MetricName, metricSpec.External.MetricSelector)
|
||||
statuses[i] = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.ExternalMetricSourceType,
|
||||
External: &autoscalingv2.ExternalMetricStatus{
|
||||
MetricSelector: metricSpec.External.MetricSelector,
|
||||
MetricName: metricSpec.External.MetricName,
|
||||
CurrentAverageValue: resource.NewMilliQuantity(utilizationProposal, resource.DecimalSI),
|
||||
},
|
||||
}
|
||||
} else if metricSpec.External.TargetValue != nil {
|
||||
replicaCountProposal, utilizationProposal, timestampProposal, err = a.replicaCalc.GetExternalMetricReplicas(currentReplicas, metricSpec.External.TargetValue.MilliValue(), metricSpec.External.MetricName, hpa.Namespace, metricSpec.External.MetricSelector, selector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetExternalMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetExternalMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, "", nil, time.Time{}, fmt.Errorf("failed to get external metric %s: %v", metricSpec.External.MetricName, err)
|
||||
}
|
||||
metricNameProposal = fmt.Sprintf("external metric %s(%+v)", metricSpec.External.MetricName, metricSpec.External.MetricSelector)
|
||||
statuses[i] = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.ExternalMetricSourceType,
|
||||
External: &autoscalingv2.ExternalMetricStatus{
|
||||
MetricSelector: metricSpec.External.MetricSelector,
|
||||
MetricName: metricSpec.External.MetricName,
|
||||
CurrentValue: *resource.NewMilliQuantity(utilizationProposal, resource.DecimalSI),
|
||||
},
|
||||
}
|
||||
} else {
|
||||
errMsg := "invalid external metric source: neither a value target nor an average value target was set"
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetExternalMetric", errMsg)
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetExternalMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, "", nil, time.Time{}, fmt.Errorf(errMsg)
|
||||
replicaCountProposal, timestampProposal, metricNameProposal, err = a.computeStatusForExternalMetric(currentReplicas, metricSpec, hpa, selector, &statuses[i])
|
||||
if err != nil {
|
||||
return 0, "", nil, time.Time{}, err
|
||||
}
|
||||
default:
|
||||
errMsg := fmt.Sprintf("unknown metric source type %q", string(metricSpec.Type))
|
||||
|
@ -370,6 +273,134 @@ func (a *HorizontalController) reconcileKey(key string) error {
|
|||
return a.reconcileAutoscaler(hpa)
|
||||
}
|
||||
|
||||
// computeStatusForObjectMetric computes the desired number of replicas for the specified metric of type ObjectMetricSourceType.
|
||||
func (a *HorizontalController) computeStatusForObjectMetric(currentReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (int32, time.Time, string, error) {
|
||||
replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetObjectMetricReplicas(currentReplicas, metricSpec.Object.TargetValue.MilliValue(), metricSpec.Object.MetricName, hpa.Namespace, &metricSpec.Object.Target, selector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetObjectMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetObjectMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, timestampProposal, "", err
|
||||
}
|
||||
*status = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.ObjectMetricSourceType,
|
||||
Object: &autoscalingv2.ObjectMetricStatus{
|
||||
Target: metricSpec.Object.Target,
|
||||
MetricName: metricSpec.Object.MetricName,
|
||||
CurrentValue: *resource.NewMilliQuantity(utilizationProposal, resource.DecimalSI),
|
||||
},
|
||||
}
|
||||
return replicaCountProposal, timestampProposal, fmt.Sprintf("%s metric %s", metricSpec.Object.Target.Kind, metricSpec.Object.MetricName), nil
|
||||
}
|
||||
|
||||
// computeStatusForPodsMetric computes the desired number of replicas for the specified metric of type PodsMetricSourceType.
|
||||
func (a *HorizontalController) computeStatusForPodsMetric(currentReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (int32, time.Time, string, error) {
|
||||
replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetMetricReplicas(currentReplicas, metricSpec.Pods.TargetAverageValue.MilliValue(), metricSpec.Pods.MetricName, hpa.Namespace, selector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetPodsMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetPodsMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, timestampProposal, "", err
|
||||
}
|
||||
*status = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.PodsMetricSourceType,
|
||||
Pods: &autoscalingv2.PodsMetricStatus{
|
||||
MetricName: metricSpec.Pods.MetricName,
|
||||
CurrentAverageValue: *resource.NewMilliQuantity(utilizationProposal, resource.DecimalSI),
|
||||
},
|
||||
}
|
||||
|
||||
return replicaCountProposal, timestampProposal, fmt.Sprintf("pods metric %s", metricSpec.Pods.MetricName), nil
|
||||
}
|
||||
|
||||
// computeStatusForResourceMetric computes the desired number of replicas for the specified metric of type ResourceMetricSourceType.
|
||||
func (a *HorizontalController) computeStatusForResourceMetric(currentReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (int32, time.Time, string, error) {
|
||||
if metricSpec.Resource.TargetAverageValue != nil {
|
||||
var rawProposal int64
|
||||
replicaCountProposal, rawProposal, timestampProposal, err := a.replicaCalc.GetRawResourceReplicas(currentReplicas, metricSpec.Resource.TargetAverageValue.MilliValue(), metricSpec.Resource.Name, hpa.Namespace, selector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetResourceMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetResourceMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, time.Time{}, "", fmt.Errorf("failed to get %s utilization: %v", metricSpec.Resource.Name, err)
|
||||
}
|
||||
metricNameProposal := fmt.Sprintf("%s resource", metricSpec.Resource.Name)
|
||||
status = &autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.ResourceMetricSourceType,
|
||||
Resource: &autoscalingv2.ResourceMetricStatus{
|
||||
Name: metricSpec.Resource.Name,
|
||||
CurrentAverageValue: *resource.NewMilliQuantity(rawProposal, resource.DecimalSI),
|
||||
},
|
||||
}
|
||||
return replicaCountProposal, timestampProposal, metricNameProposal, nil
|
||||
} else {
|
||||
if metricSpec.Resource.TargetAverageUtilization == nil {
|
||||
errMsg := "invalid resource metric source: neither a utilization target nor a value target was set"
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetResourceMetric", errMsg)
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetResourceMetric", "the HPA was unable to compute the replica count: %s", errMsg)
|
||||
return 0, time.Time{}, "", fmt.Errorf(errMsg)
|
||||
}
|
||||
targetUtilization := *metricSpec.Resource.TargetAverageUtilization
|
||||
var percentageProposal int32
|
||||
var rawProposal int64
|
||||
replicaCountProposal, percentageProposal, rawProposal, timestampProposal, err := a.replicaCalc.GetResourceReplicas(currentReplicas, targetUtilization, metricSpec.Resource.Name, hpa.Namespace, selector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetResourceMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetResourceMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, time.Time{}, "", fmt.Errorf("failed to get %s utilization: %v", metricSpec.Resource.Name, err)
|
||||
}
|
||||
metricNameProposal := fmt.Sprintf("%s resource utilization (percentage of request)", metricSpec.Resource.Name)
|
||||
*status = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.ResourceMetricSourceType,
|
||||
Resource: &autoscalingv2.ResourceMetricStatus{
|
||||
Name: metricSpec.Resource.Name,
|
||||
CurrentAverageUtilization: &percentageProposal,
|
||||
CurrentAverageValue: *resource.NewMilliQuantity(rawProposal, resource.DecimalSI),
|
||||
},
|
||||
}
|
||||
return replicaCountProposal, timestampProposal, metricNameProposal, nil
|
||||
}
|
||||
}
|
||||
|
||||
// computeStatusForExternalMetric computes the desired number of replicas for the specified metric of type ExternalMetricSourceType.
|
||||
func (a *HorizontalController) computeStatusForExternalMetric(currentReplicas int32, metricSpec autoscalingv2.MetricSpec, hpa *autoscalingv2.HorizontalPodAutoscaler, selector labels.Selector, status *autoscalingv2.MetricStatus) (int32, time.Time, string, error) {
|
||||
if metricSpec.External.TargetAverageValue != nil {
|
||||
replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetExternalPerPodMetricReplicas(currentReplicas, metricSpec.External.TargetAverageValue.MilliValue(), metricSpec.External.MetricName, hpa.Namespace, metricSpec.External.MetricSelector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetExternalMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetExternalMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, time.Time{}, "", fmt.Errorf("failed to get %s external metric: %v", metricSpec.External.MetricName, err)
|
||||
}
|
||||
*status = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.ExternalMetricSourceType,
|
||||
External: &autoscalingv2.ExternalMetricStatus{
|
||||
MetricSelector: metricSpec.External.MetricSelector,
|
||||
MetricName: metricSpec.External.MetricName,
|
||||
CurrentAverageValue: resource.NewMilliQuantity(utilizationProposal, resource.DecimalSI),
|
||||
},
|
||||
}
|
||||
return replicaCountProposal, timestampProposal, fmt.Sprintf("external metric %s(%+v)", metricSpec.External.MetricName, metricSpec.External.MetricSelector), nil
|
||||
}
|
||||
if metricSpec.External.TargetValue != nil {
|
||||
replicaCountProposal, utilizationProposal, timestampProposal, err := a.replicaCalc.GetExternalMetricReplicas(currentReplicas, metricSpec.External.TargetValue.MilliValue(), metricSpec.External.MetricName, hpa.Namespace, metricSpec.External.MetricSelector, selector)
|
||||
if err != nil {
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetExternalMetric", err.Error())
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetExternalMetric", "the HPA was unable to compute the replica count: %v", err)
|
||||
return 0, time.Time{}, "", fmt.Errorf("failed to get external metric %s: %v", metricSpec.External.MetricName, err)
|
||||
}
|
||||
*status = autoscalingv2.MetricStatus{
|
||||
Type: autoscalingv2.ExternalMetricSourceType,
|
||||
External: &autoscalingv2.ExternalMetricStatus{
|
||||
MetricSelector: metricSpec.External.MetricSelector,
|
||||
MetricName: metricSpec.External.MetricName,
|
||||
CurrentValue: *resource.NewMilliQuantity(utilizationProposal, resource.DecimalSI),
|
||||
},
|
||||
}
|
||||
return replicaCountProposal, timestampProposal, fmt.Sprintf("external metric %s(%+v)", metricSpec.External.MetricName, metricSpec.External.MetricSelector), nil
|
||||
}
|
||||
errMsg := "invalid external metric source: neither a value target nor an average value target was set"
|
||||
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedGetExternalMetric", errMsg)
|
||||
setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "FailedGetExternalMetric", "the HPA was unable to compute the replica count: %s", errMsg)
|
||||
return 0, time.Time{}, "", fmt.Errorf(errMsg)
|
||||
}
|
||||
|
||||
func (a *HorizontalController) reconcileAutoscaler(hpav1Shared *autoscalingv1.HorizontalPodAutoscaler) error {
|
||||
// make a copy so that we never mutate the shared informer cache (conversion can mutate the object)
|
||||
hpav1 := hpav1Shared.DeepCopy()
|
||||
|
|
Loading…
Reference in New Issue