e2e: network tiers should retry on 404 errors

The feature is still Alpha and at times, the IP address previously used
by the load balancer in the test will not completely freed even after
the load balancer is long gone. In this case, the test URL with the IP
would return a 404 response. Tolerate this error and retry until the new
load balancer is fully established.

test/e2e/framework/networking_utils.go

@@ -650,11 +650,20 @@ func TestReachableHTTP(ip string, port int, request string, expect string) (bool
 	return TestReachableHTTPWithContent(ip, port, request, expect, nil)
 }
 
+func TestReachableHTTPWithRetriableErrorCodes(ip string, port int, request string, expect string, retriableErrCodes []int) (bool, error) {
+	return TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip, port, request, expect, nil, retriableErrCodes, time.Second*5)
+}
+
 func TestReachableHTTPWithContent(ip string, port int, request string, expect string, content *bytes.Buffer) (bool, error) {
 	return TestReachableHTTPWithContentTimeout(ip, port, request, expect, content, 5*time.Second)
 }
 
 func TestReachableHTTPWithContentTimeout(ip string, port int, request string, expect string, content *bytes.Buffer, timeout time.Duration) (bool, error) {
+	return TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip, port, request, expect, content, []int{}, timeout)
+}
+
+func TestReachableHTTPWithContentTimeoutWithRetriableErrorCodes(ip string, port int, request string, expect string, content *bytes.Buffer, retriableErrCodes []int, timeout time.Duration) (bool, error) {
+
 	url := fmt.Sprintf("http://%s:%d%s", ip, port, request)
 	if ip == "" {
 		Failf("Got empty IP for reachability check (%s)", url)
@@ -679,6 +688,11 @@ func TestReachableHTTPWithContentTimeout(ip string, port int, request string, ex
 		return false, nil
 	}
 	if resp.StatusCode != 200 {
+		for _, code := range retriableErrCodes {
+			if resp.StatusCode == code {
+				return false, nil
+			}
+		}
 		return false, fmt.Errorf("received non-success return status %q trying to access %s; got body: %s",
 			resp.Status, url, string(body))
 	}

test/e2e/framework/service_util.go

@@ -832,7 +832,13 @@ func (j *ServiceTestJig) LaunchEchoserverPodOnNode(f *Framework, nodeName, podNa
 }
 
 func (j *ServiceTestJig) TestReachableHTTP(host string, port int, timeout time.Duration) {
-	if err := wait.PollImmediate(Poll, timeout, func() (bool, error) { return TestReachableHTTP(host, port, "/echo?msg=hello", "hello") }); err != nil {
+	j.TestReachableHTTPWithRetriableErrorCodes(host, port, []int{}, timeout)
+}
+
+func (j *ServiceTestJig) TestReachableHTTPWithRetriableErrorCodes(host string, port int, retriableErrCodes []int, timeout time.Duration) {
+	if err := wait.PollImmediate(Poll, timeout, func() (bool, error) {
+		return TestReachableHTTPWithRetriableErrorCodes(host, port, "/echo?msg=hello", "hello", retriableErrCodes)
+	}); err != nil {
 		Failf("Could not reach HTTP service through %v:%v after %v: %v", host, port, timeout, err)
 	}
 }

test/e2e/network/network_tiers.go

@@ -18,6 +18,7 @@ package network
 
 import (
 	"fmt"
+	"net/http"
 	"time"
 
 	computealpha "google.golang.org/api/compute/v0.alpha"
@@ -153,7 +154,10 @@ func waitAndVerifyLBWithTier(jig *framework.ServiceTestJig, ns, svcName, existin
 		Expect(ingressIP).To(Equal(svc.Spec.LoadBalancerIP))
 	}
 	jig.SanityCheckService(svc, v1.ServiceTypeLoadBalancer)
-	jig.TestReachableHTTP(ingressIP, svcPort, checkTimeout)
+	// If the IP has been used by previous test, sometimes we get the lingering
+	// 404 errors even after the LB is long gone. Tolerate and retry until the
+	// the new LB is fully established since this feature is still Alpha in GCP.
+	jig.TestReachableHTTPWithRetriableErrorCodes(ingressIP, svcPort, []int{http.StatusNotFound}, checkTimeout)
 
 	// Verify the network tier matches the desired.
 	svcNetTier, err := gcecloud.GetServiceNetworkTier(svc)