Merge pull request #53664 from gnufied/fix-volume-detach-metric-flake

Automatic merge from submit-queue (batch tested with PRs 51677, 53690, 53025, 53717, 53664). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Fix flake for volume detach metrics

Fixes https://github.com/kubernetes/kubernetes/issues/53596

cc @kubernetes/sig-storage-pr-reviews @msau42

test/e2e/storage/volume_metrics.go

@@ -100,7 +100,7 @@ var _ = SIGDescribe("[Serial] Volume metrics", func() {
 		backoff := wait.Backoff{
 			Duration: 10 * time.Second,
 			Factor:   1.2,
-			Steps:    3,
+			Steps:    21,
 		}
 
 		updatedStorageMetrics := make(map[string]int64)
@@ -113,7 +113,11 @@ var _ = SIGDescribe("[Serial] Volume metrics", func() {
 				return false, err
 			}
 			updatedStorageMetrics = getControllerStorageMetrics(updatedMetrics)
-			if len(updatedStorageMetrics) == 0 {
+			metricCount := len(updatedStorageMetrics)
+			// Usually a pod deletion does not mean immediate volume detach
+			// we will have to retry to verify volume_detach metrics
+			_, detachMetricFound := updatedStorageMetrics["volume_detach"]
+			if metricCount < 3 || !detachMetricFound {
 				framework.Logf("Volume metrics not collected yet, going to retry")
 				return false, nil
 			}