Merge pull request #51805 from yujuhong/net-tiers-static-ip-test

Automatic merge from submit-queue

e2e: test using reserved IP with network tiers

test/e2e/network/network_tiers.go

@@ -70,50 +70,101 @@ var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() {
 		By("creating a pod to be part of the service " + svcName)
 		jig.RunOrFail(ns, nil)
 
+		// Test 1: create a standard tiered LB for the Service.
 		By("creating a Service of type LoadBalancer using the standard network tier")
 		svc := jig.CreateTCPServiceOrFail(ns, func(svc *v1.Service) {
 			svc.Spec.Type = v1.ServiceTypeLoadBalancer
 			setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard)
 		})
+		// Verify that service has been updated properly.
+		svcTier, err := gcecloud.GetServiceNetworkTier(svc)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(svcTier).To(Equal(gcecloud.NetworkTierStandard))
 		// Record the LB name for test cleanup.
 		serviceLBNames = append(serviceLBNames, cloudprovider.GetLoadBalancerName(svc))
 
-		svc = jig.WaitForLoadBalancerOrFail(ns, svcName, createTimeout)
-		lbIngress := &svc.Status.LoadBalancer.Ingress[0]
-		svcPort := int(svc.Spec.Ports[0].Port)
-		ingressIP := framework.GetIngressPoint(lbIngress)
-
-		By("running sanity and reachability checks")
-		jig.SanityCheckService(svc, v1.ServiceTypeLoadBalancer)
-		jig.TestReachableHTTP(ingressIP, svcPort, lagTimeout)
-		// Check the network tier of the forwarding rule.
-		netTier, err := getLBNetworkTierByIP(ingressIP)
-		Expect(err).NotTo(HaveOccurred(), "failed to get the network tier of the load balancer")
-		Expect(netTier).To(Equal(gcecloud.NetworkTierStandard))
+		// Wait and verify the LB.
+		ingressIP := waitAndVerifyLBWithTier(jig, ns, svcName, "", createTimeout, lagTimeout)
 
+		// Test 2: re-create a LB of a different tier for the updated Service.
 		By("updating the Service to use the premium (default) tier")
-		existingIP := ingressIP
 		svc = jig.UpdateServiceOrFail(ns, svcName, func(svc *v1.Service) {
 			clearNetworkTier(svc)
 		})
+		// Verify that service has been updated properly.
+		svcTier, err = gcecloud.GetServiceNetworkTier(svc)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(svcTier).To(Equal(gcecloud.NetworkTierDefault))
+
 		// Wait until the ingress IP changes. Each tier has its own pool of
 		// IPs, so changing tiers implies changing IPs.
-		svc = jig.WaitForNewIngressIPOrFail(ns, svcName, existingIP, createTimeout)
-		lbIngress = &svc.Status.LoadBalancer.Ingress[0]
-		ingressIP = framework.GetIngressPoint(lbIngress)
+		ingressIP = waitAndVerifyLBWithTier(jig, ns, svcName, ingressIP, createTimeout, lagTimeout)
 
-		By("running sanity and reachability checks")
-		jig.SanityCheckService(svc, v1.ServiceTypeLoadBalancer)
-		jig.TestReachableHTTP(ingressIP, svcPort, lagTimeout)
-		// Check the network tier of the forwarding rule.
-		netTier, err = getLBNetworkTierByIP(ingressIP)
-		Expect(err).NotTo(HaveOccurred(), "failed to get the network tier of the load balancer")
-		Expect(netTier).To(Equal(gcecloud.NetworkTierPremium))
+		// Test 3: create a standard-tierd LB with a user-requested IP.
+		By("reserving a static IP for the load balancer")
+		requestedAddrName := fmt.Sprintf("e2e-ext-lb-net-tier-%s", framework.RunId)
+		requestedIP, err := reserveAlphaRegionalAddress(requestedAddrName, gcecloud.NetworkTierStandard)
+		Expect(err).NotTo(HaveOccurred(), "failed to reserve a STANDARD tiered address")
+		defer func() {
+			if requestedAddrName != "" {
+				// Release GCE static address - this is not kube-managed and will not be automatically released.
+				if err := framework.DeleteGCEStaticIP(requestedAddrName); err != nil {
+					framework.Logf("failed to release static IP address %q: %v", requestedAddrName, err)
+				}
+			}
+		}()
+		Expect(err).NotTo(HaveOccurred())
+		framework.Logf("Allocated static IP to be used by the load balancer: %q", requestedIP)
 
-		// TODO: Add tests for user-requested IPs.
+		By("updating the Service to use the standard tier with a requested IP")
+		svc = jig.UpdateServiceOrFail(ns, svc.Name, func(svc *v1.Service) {
+			svc.Spec.LoadBalancerIP = requestedIP
+			setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard)
+		})
+		// Verify that service has been updated properly.
+		Expect(svc.Spec.LoadBalancerIP).To(Equal(requestedIP))
+		svcTier, err = gcecloud.GetServiceNetworkTier(svc)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(svcTier).To(Equal(gcecloud.NetworkTierStandard))
+
+		// Wait until the ingress IP changes and verifies the LB.
+		ingressIP = waitAndVerifyLBWithTier(jig, ns, svcName, ingressIP, createTimeout, lagTimeout)
 	})
 })
 
+func waitAndVerifyLBWithTier(jig *framework.ServiceTestJig, ns, svcName, existingIP string, waitTimeout, checkTimeout time.Duration) string {
+	var svc *v1.Service
+	if existingIP == "" {
+		// Creating the LB for the first time; wait for any ingress IP to show
+		// up.
+		svc = jig.WaitForNewIngressIPOrFail(ns, svcName, existingIP, waitTimeout)
+	} else {
+		// Re-creating the LB; wait for the ingress IP to change.
+		svc = jig.WaitForNewIngressIPOrFail(ns, svcName, existingIP, waitTimeout)
+	}
+
+	svcPort := int(svc.Spec.Ports[0].Port)
+	lbIngress := &svc.Status.LoadBalancer.Ingress[0]
+	ingressIP := framework.GetIngressPoint(lbIngress)
+
+	By("running sanity and reachability checks")
+	if svc.Spec.LoadBalancerIP != "" {
+		// Verify that the new ingress IP is the requested IP if it's set.
+		Expect(ingressIP).To(Equal(svc.Spec.LoadBalancerIP))
+	}
+	jig.SanityCheckService(svc, v1.ServiceTypeLoadBalancer)
+	jig.TestReachableHTTP(ingressIP, svcPort, checkTimeout)
+
+	// Verify the network tier matches the desired.
+	svcNetTier, err := gcecloud.GetServiceNetworkTier(svc)
+	Expect(err).NotTo(HaveOccurred())
+	netTier, err := getLBNetworkTierByIP(ingressIP)
+	Expect(err).NotTo(HaveOccurred(), "failed to get the network tier of the load balancer")
+	Expect(netTier).To(Equal(svcNetTier))
+
+	return ingressIP
+}
+
 func getLBNetworkTierByIP(ip string) (gcecloud.NetworkTier, error) {
 	var rule *computealpha.ForwardingRule
 	// Retry a few times to tolerate flakes.
@@ -163,3 +214,24 @@ func clearNetworkTier(svc *v1.Service) {
 	}
 	delete(svc.ObjectMeta.Annotations, key)
 }
+
+// TODO: add retries if this turns out to be flaky.
+// TODO(#51665): remove this helper function once Network Tiers becomes beta.
+func reserveAlphaRegionalAddress(name string, netTier gcecloud.NetworkTier) (string, error) {
+	cloud, err := framework.GetGCECloud()
+	alphaAddr := &computealpha.Address{
+		Name:        name,
+		NetworkTier: netTier.ToGCEValue(),
+	}
+
+	if err := cloud.ReserveAlphaRegionAddress(alphaAddr, cloud.Region()); err != nil {
+		return "", err
+	}
+
+	addr, err := cloud.GetRegionAddress(name, cloud.Region())
+	if err != nil {
+		return "", err
+	}
+
+	return addr.Address, nil
+}