From 6dbb07c4b66017c6f8c8cd34f393d39af492fef3 Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Wed, 19 Sep 2018 12:04:32 +0200
Subject: [PATCH] CSI E2E: retry csi-pod creation

Normally the pod would get created via a DaemonSet controller, but
during testing it is easier to create it directly. We just need to
ignore errors (like 'No API token found for service account
"csi-service-account"') and retry for a while. If the error persists,
the error will still abort and report it eventually.

This problem also occurs elsewhere, so an utility function in the
framework for it seems justified.

Fixes: #68776
---
 test/e2e/framework/pods.go      | 26 ++++++++++++++++++++++++++
 test/e2e/storage/csi_objects.go | 15 +++++++++------
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/test/e2e/framework/pods.go b/test/e2e/framework/pods.go
index fb9e8a5757..03c38192e2 100644
--- a/test/e2e/framework/pods.go
+++ b/test/e2e/framework/pods.go
@@ -78,6 +78,32 @@ func (c *PodClient) Create(pod *v1.Pod) *v1.Pod {
 	return p
 }
 
+// CreateEventually retries pod creation for a while before failing
+// the test with the most recent error. This mimicks the behavior
+// of a controller (like the one for DaemonSet) and is necessary
+// because pod creation can fail while its service account is still
+// getting provisioned
+// (https://github.com/kubernetes/kubernetes/issues/68776).
+//
+// Both the timeout and polling interval are configurable as optional
+// arguments:
+// - The first optional argument is the timeout.
+// - The second optional argument is the polling interval.
+//
+// Both intervals can either be specified as time.Duration, parsable
+// duration strings or as floats/integers. In the last case they are
+// interpreted as seconds.
+func (c *PodClient) CreateEventually(pod *v1.Pod, opts ...interface{}) *v1.Pod {
+	c.mungeSpec(pod)
+	var ret *v1.Pod
+	Eventually(func() error {
+		p, err := c.PodInterface.Create(pod)
+		ret = p
+		return err
+	}, opts...).ShouldNot(HaveOccurred(), "Failed to create %q pod", pod.GetName())
+	return ret
+}
+
 // CreateSync creates a new pod according to the framework specifications in the given namespace, and waits for it to start.
 func (c *PodClient) CreateSyncInNamespace(pod *v1.Pod, namespace string) *v1.Pod {
 	p := c.Create(pod)
diff --git a/test/e2e/storage/csi_objects.go b/test/e2e/storage/csi_objects.go
index 4239d7cdce..efb7ce6742 100644
--- a/test/e2e/storage/csi_objects.go
+++ b/test/e2e/storage/csi_objects.go
@@ -202,8 +202,6 @@ func csiHostPathPod(
 	f *framework.Framework,
 	sa *v1.ServiceAccount,
 ) *v1.Pod {
-	podClient := client.CoreV1().Pods(config.Namespace)
-
 	priv := true
 	mountPropagation := v1.MountPropagationBidirectional
 	hostPathType := v1.HostPathDirectoryOrCreate
@@ -366,10 +364,15 @@ func csiHostPathPod(
 		return nil
 	}
 
-	ret, err := podClient.Create(pod)
-	if err != nil {
-		framework.ExpectNoError(err, "Failed to create %q pod: %v", pod.GetName(), err)
-	}
+	// Creating the pod can fail initially while the service
+	// account's secret isn't provisioned yet ('No API token found
+	// for service account "csi-service-account", retry after the
+	// token is automatically created and added to the service
+	// account', see https://github.com/kubernetes/kubernetes/issues/68776).
+	// We could use a DaemonSet, but then the name of the csi-pod changes
+	// during each test run. It's simpler to just try for a while here.
+	podClient := f.PodClient()
+	ret := podClient.CreateEventually(pod)
 
 	// Wait for pod to come up
 	framework.ExpectNoError(framework.WaitForPodRunningInNamespace(client, ret))