Merge pull request #64535 from agau4779/expose-neg-e2e

Automatic merge from submit-queue (batch tested with PRs 65338, 64535). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

[GCE] e2e test for expose neg on gce ingress

**What this PR does / why we need it**:
- Adds e2e test for the expose NEG annotation (which allows for standalone NEGs)

**Special notes for your reviewer**:
Note, https://github.com/kubernetes/ingress-gce/pull/350 must be merged first before this is merged.

`[Unreleased]` tag is on this PR because it depends on code from https://github.com/kubernetes/ingress-gce/pull/350 and https://github.com/kubernetes/ingress-gce/pull/284 being in an Ingress release. Will update this test and test-infra once this is released in the next Ingress.

**Release note**:
```release-note
NONE
```

test/e2e/framework/ingress_utils.go

@@ -121,8 +121,9 @@ const (
 	// a single character of padding.
 	nameLenLimit = 62
 
-	NEGAnnotation    = "alpha.cloud.google.com/load-balancer-neg"
-	NEGUpdateTimeout = 2 * time.Minute
+	NEGAnnotation       = "cloud.google.com/neg"
+	NEGStatusAnnotation = "cloud.google.com/neg-status"
+	NEGUpdateTimeout    = 2 * time.Minute
 
 	InstanceGroupAnnotation = "ingress.gcp.kubernetes.io/instance-groups"
 
@@ -164,6 +165,16 @@ type IngressConformanceTests struct {
 	ExitLog  string
 }
 
+// NegStatus contains name and zone of the Network Endpoint Group
+// resources associated with this service.
+// Needs to be consistent with the NEG internal structs in ingress-gce.
+type NegStatus struct {
+	// NetworkEndpointGroups returns the mapping between service port and NEG
+	// resource. key is service port, value is the name of the NEG resource.
+	NetworkEndpointGroups map[int32]string `json:"network_endpoint_groups,omitempty"`
+	Zones                 []string         `json:"zones,omitempty"`
+}
+
 // CreateIngressComformanceTests generates an slice of sequential test cases:
 // a simple http ingress, ingress with HTTPS, ingress HTTPS with a modified hostname,
 // ingress https with a modified URLMap
@@ -940,7 +951,7 @@ func (cont *GCEIngressController) backendMode(svcPorts map[string]v1.ServicePort
 		bsMatch := &compute.BackendService{}
 		// Non-NEG BackendServices are named with the Nodeport in the name.
 		// NEG BackendServices' names contain the a sha256 hash of a string.
-		negString := strings.Join([]string{uid, cont.Ns, svcName, sp.TargetPort.String()}, ";")
+		negString := strings.Join([]string{uid, cont.Ns, svcName, fmt.Sprintf("%v", sp.Port)}, ";")
 		negHash := fmt.Sprintf("%x", sha256.Sum256([]byte(negString)))[:8]
 		for _, bs := range beList {
 			if strings.Contains(bs.Name, strconv.Itoa(int(sp.NodePort))) ||

test/e2e/network/ingress.go

@@ -17,6 +17,7 @@ limitations under the License.
 package network
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"path/filepath"
@@ -647,6 +648,88 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 				}
 			})
 		})
+
+		It("should sync endpoints for both Ingress-referenced NEG and standalone NEG [Unreleased]", func() {
+			name := "hostname"
+			expectedKeys := []int32{80, 443}
+
+			scaleAndValidateExposedNEG := func(num int) {
+				scale, err := f.ClientSet.ExtensionsV1beta1().Deployments(ns).GetScale(name, metav1.GetOptions{})
+				Expect(err).NotTo(HaveOccurred())
+				if scale.Spec.Replicas != int32(num) {
+					scale.Spec.Replicas = int32(num)
+					_, err = f.ClientSet.ExtensionsV1beta1().Deployments(ns).UpdateScale(name, scale)
+					Expect(err).NotTo(HaveOccurred())
+				}
+				wait.Poll(10*time.Second, framework.NEGUpdateTimeout, func() (bool, error) {
+					svc, err := f.ClientSet.CoreV1().Services(ns).Get(name, metav1.GetOptions{})
+					Expect(err).NotTo(HaveOccurred())
+
+					var status framework.NegStatus
+					v, ok := svc.Annotations[framework.NEGStatusAnnotation]
+					if !ok {
+						// Wait for NEG sync loop to find NEGs
+						framework.Logf("Waiting for %v, got: %+v", framework.NEGStatusAnnotation, svc.Annotations)
+						return false, nil
+					}
+					err = json.Unmarshal([]byte(v), &status)
+					if err != nil {
+						framework.Logf("Error in parsing Expose NEG annotation: %v", err)
+						return false, nil
+					}
+					framework.Logf("Got %v: %v", framework.NEGStatusAnnotation, v)
+
+					// Expect 2 NEGs to be created based on the test setup (neg-exposed)
+					if len(status.NetworkEndpointGroups) != 2 {
+						framework.Logf("Expected 2 NEGs, got %d", len(status.NetworkEndpointGroups))
+						return false, nil
+					}
+
+					for _, port := range expectedKeys {
+						if _, ok := status.NetworkEndpointGroups[port]; !ok {
+							framework.Logf("Expected ServicePort key %v, but does not exist", port)
+						}
+					}
+
+					if len(status.NetworkEndpointGroups) != len(expectedKeys) {
+						framework.Logf("Expected length of %+v to equal length of %+v, but does not", status.NetworkEndpointGroups, expectedKeys)
+					}
+
+					gceCloud := gceController.Cloud.Provider.(*gcecloud.GCECloud)
+					for _, neg := range status.NetworkEndpointGroups {
+						networkEndpoints, err := gceCloud.ListNetworkEndpoints(neg, gceController.Cloud.Zone, false)
+						Expect(err).NotTo(HaveOccurred())
+						if len(networkEndpoints) != num {
+							framework.Logf("Expect number of endpoints to be %d, but got %d", num, len(networkEndpoints))
+							return false, nil
+						}
+					}
+
+					return true, nil
+				})
+			}
+
+			By("Create a basic HTTP ingress using NEG")
+			jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "neg-exposed"), ns, map[string]string{}, map[string]string{})
+			jig.WaitForIngress(true)
+			usingNEG, err := gceController.BackendServiceUsingNEG(jig.GetServicePorts(false))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(usingNEG).To(BeTrue())
+			// initial replicas number is 1
+			scaleAndValidateExposedNEG(1)
+
+			By("Scale up number of backends to 5")
+			scaleAndValidateExposedNEG(5)
+
+			By("Scale down number of backends to 3")
+			scaleAndValidateExposedNEG(3)
+
+			By("Scale up number of backends to 6")
+			scaleAndValidateExposedNEG(6)
+
+			By("Scale down number of backends to 2")
+			scaleAndValidateExposedNEG(3)
+		})
 	})
 
 	Describe("GCE [Slow] [Feature:kubemci]", func() {

test/e2e/testing-manifests/ingress/neg-clusterip/svc.yaml

@@ -3,7 +3,7 @@ kind: Service
 metadata:
   name: hostname
   annotations:
-    alpha.cloud.google.com/load-balancer-neg: "true"
+    cloud.google.com/neg: '{"ingress":true}'
 spec:
   ports:
   - port: 80

test/e2e/testing-manifests/ingress/neg-exposed/ing.yaml

@@ -0,0 +1,8 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: hostname
+spec:
+  backend:
+    serviceName: hostname
+    servicePort: 80

test/e2e/testing-manifests/ingress/neg-exposed/rc.yaml

@@ -0,0 +1,31 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  labels:
+    run: hostname
+  name: hostname
+spec:
+  template:
+    metadata:
+      labels:
+        run: hostname
+    spec:
+      containers:
+      - image: gcr.io/kubernetes-e2e-test-images/serve-hostname-amd64:1.1
+        name: host1
+        command:
+        - /bin/sh
+        - -c
+        - /serve_hostname -http=true -udp=false -port=8000
+        ports:
+        - protocol: TCP
+          containerPort: 8000
+      - image: gcr.io/kubernetes-e2e-test-images/serve-hostname-amd64:1.1
+        name: host2
+        command:
+        - /bin/sh
+        - -c
+        - /serve_hostname -http=true -udp=false -port=8080
+        ports:
+        - protocol: TCP
+          containerPort: 8080

test/e2e/testing-manifests/ingress/neg-exposed/svc.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: hostname
+  annotations:
+    cloud.google.com/neg: '{"ingress":true,"exposed_ports":{"80":{},"443":{}}}'
+spec:
+  ports:
+  - port: 80
+    name: host1
+    protocol: TCP
+    targetPort: 8000
+  - port: 443
+    name: host2
+    protocol: TCP
+    targetPort: 8080
+  selector:
+    run: hostname
+  sessionAffinity: None
+  type: ClusterIP

test/e2e/testing-manifests/ingress/neg/svc.yaml

@@ -3,7 +3,7 @@ kind: Service
 metadata:
   name: hostname
   annotations:
-    alpha.cloud.google.com/load-balancer-neg: "true"
+    cloud.google.com/neg: '{"ingress":true}'
 spec:
   ports:
   - port: 80