diff --git a/test/e2e/framework/ingress_utils.go b/test/e2e/framework/ingress_utils.go index f1d5b14a86..2ebe4258de 100644 --- a/test/e2e/framework/ingress_utils.go +++ b/test/e2e/framework/ingress_utils.go @@ -121,8 +121,9 @@ const ( // a single character of padding. nameLenLimit = 62 - NEGAnnotation = "alpha.cloud.google.com/load-balancer-neg" - NEGUpdateTimeout = 2 * time.Minute + NEGAnnotation = "cloud.google.com/neg" + NEGStatusAnnotation = "cloud.google.com/neg-status" + NEGUpdateTimeout = 2 * time.Minute InstanceGroupAnnotation = "ingress.gcp.kubernetes.io/instance-groups" @@ -164,6 +165,16 @@ type IngressConformanceTests struct { ExitLog string } +// NegStatus contains name and zone of the Network Endpoint Group +// resources associated with this service. +// Needs to be consistent with the NEG internal structs in ingress-gce. +type NegStatus struct { + // NetworkEndpointGroups returns the mapping between service port and NEG + // resource. key is service port, value is the name of the NEG resource. + NetworkEndpointGroups map[int32]string `json:"network_endpoint_groups,omitempty"` + Zones []string `json:"zones,omitempty"` +} + // CreateIngressComformanceTests generates an slice of sequential test cases: // a simple http ingress, ingress with HTTPS, ingress HTTPS with a modified hostname, // ingress https with a modified URLMap @@ -940,7 +951,7 @@ func (cont *GCEIngressController) backendMode(svcPorts map[string]v1.ServicePort bsMatch := &compute.BackendService{} // Non-NEG BackendServices are named with the Nodeport in the name. // NEG BackendServices' names contain the a sha256 hash of a string. - negString := strings.Join([]string{uid, cont.Ns, svcName, sp.TargetPort.String()}, ";") + negString := strings.Join([]string{uid, cont.Ns, svcName, fmt.Sprintf("%v", sp.Port)}, ";") negHash := fmt.Sprintf("%x", sha256.Sum256([]byte(negString)))[:8] for _, bs := range beList { if strings.Contains(bs.Name, strconv.Itoa(int(sp.NodePort))) || diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go index b6b31a75eb..418377737e 100644 --- a/test/e2e/network/ingress.go +++ b/test/e2e/network/ingress.go @@ -17,6 +17,7 @@ limitations under the License. package network import ( + "encoding/json" "fmt" "net/http" "path/filepath" @@ -647,6 +648,88 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { } }) }) + + It("should sync endpoints for both Ingress-referenced NEG and standalone NEG [Unreleased]", func() { + name := "hostname" + expectedKeys := []int32{80, 443} + + scaleAndValidateExposedNEG := func(num int) { + scale, err := f.ClientSet.ExtensionsV1beta1().Deployments(ns).GetScale(name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + if scale.Spec.Replicas != int32(num) { + scale.Spec.Replicas = int32(num) + _, err = f.ClientSet.ExtensionsV1beta1().Deployments(ns).UpdateScale(name, scale) + Expect(err).NotTo(HaveOccurred()) + } + wait.Poll(10*time.Second, framework.NEGUpdateTimeout, func() (bool, error) { + svc, err := f.ClientSet.CoreV1().Services(ns).Get(name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + var status framework.NegStatus + v, ok := svc.Annotations[framework.NEGStatusAnnotation] + if !ok { + // Wait for NEG sync loop to find NEGs + framework.Logf("Waiting for %v, got: %+v", framework.NEGStatusAnnotation, svc.Annotations) + return false, nil + } + err = json.Unmarshal([]byte(v), &status) + if err != nil { + framework.Logf("Error in parsing Expose NEG annotation: %v", err) + return false, nil + } + framework.Logf("Got %v: %v", framework.NEGStatusAnnotation, v) + + // Expect 2 NEGs to be created based on the test setup (neg-exposed) + if len(status.NetworkEndpointGroups) != 2 { + framework.Logf("Expected 2 NEGs, got %d", len(status.NetworkEndpointGroups)) + return false, nil + } + + for _, port := range expectedKeys { + if _, ok := status.NetworkEndpointGroups[port]; !ok { + framework.Logf("Expected ServicePort key %v, but does not exist", port) + } + } + + if len(status.NetworkEndpointGroups) != len(expectedKeys) { + framework.Logf("Expected length of %+v to equal length of %+v, but does not", status.NetworkEndpointGroups, expectedKeys) + } + + gceCloud := gceController.Cloud.Provider.(*gcecloud.GCECloud) + for _, neg := range status.NetworkEndpointGroups { + networkEndpoints, err := gceCloud.ListNetworkEndpoints(neg, gceController.Cloud.Zone, false) + Expect(err).NotTo(HaveOccurred()) + if len(networkEndpoints) != num { + framework.Logf("Expect number of endpoints to be %d, but got %d", num, len(networkEndpoints)) + return false, nil + } + } + + return true, nil + }) + } + + By("Create a basic HTTP ingress using NEG") + jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "neg-exposed"), ns, map[string]string{}, map[string]string{}) + jig.WaitForIngress(true) + usingNEG, err := gceController.BackendServiceUsingNEG(jig.GetServicePorts(false)) + Expect(err).NotTo(HaveOccurred()) + Expect(usingNEG).To(BeTrue()) + // initial replicas number is 1 + scaleAndValidateExposedNEG(1) + + By("Scale up number of backends to 5") + scaleAndValidateExposedNEG(5) + + By("Scale down number of backends to 3") + scaleAndValidateExposedNEG(3) + + By("Scale up number of backends to 6") + scaleAndValidateExposedNEG(6) + + By("Scale down number of backends to 2") + scaleAndValidateExposedNEG(3) + }) }) Describe("GCE [Slow] [Feature:kubemci]", func() { diff --git a/test/e2e/testing-manifests/ingress/neg-clusterip/svc.yaml b/test/e2e/testing-manifests/ingress/neg-clusterip/svc.yaml index 6ca920bd26..d640800ccb 100644 --- a/test/e2e/testing-manifests/ingress/neg-clusterip/svc.yaml +++ b/test/e2e/testing-manifests/ingress/neg-clusterip/svc.yaml @@ -3,7 +3,7 @@ kind: Service metadata: name: hostname annotations: - alpha.cloud.google.com/load-balancer-neg: "true" + cloud.google.com/neg: '{"ingress":true}' spec: ports: - port: 80 diff --git a/test/e2e/testing-manifests/ingress/neg-exposed/ing.yaml b/test/e2e/testing-manifests/ingress/neg-exposed/ing.yaml new file mode 100644 index 0000000000..e266d98516 --- /dev/null +++ b/test/e2e/testing-manifests/ingress/neg-exposed/ing.yaml @@ -0,0 +1,8 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: hostname +spec: + backend: + serviceName: hostname + servicePort: 80 \ No newline at end of file diff --git a/test/e2e/testing-manifests/ingress/neg-exposed/rc.yaml b/test/e2e/testing-manifests/ingress/neg-exposed/rc.yaml new file mode 100644 index 0000000000..772253a133 --- /dev/null +++ b/test/e2e/testing-manifests/ingress/neg-exposed/rc.yaml @@ -0,0 +1,31 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + run: hostname + name: hostname +spec: + template: + metadata: + labels: + run: hostname + spec: + containers: + - image: gcr.io/kubernetes-e2e-test-images/serve-hostname-amd64:1.1 + name: host1 + command: + - /bin/sh + - -c + - /serve_hostname -http=true -udp=false -port=8000 + ports: + - protocol: TCP + containerPort: 8000 + - image: gcr.io/kubernetes-e2e-test-images/serve-hostname-amd64:1.1 + name: host2 + command: + - /bin/sh + - -c + - /serve_hostname -http=true -udp=false -port=8080 + ports: + - protocol: TCP + containerPort: 8080 diff --git a/test/e2e/testing-manifests/ingress/neg-exposed/svc.yaml b/test/e2e/testing-manifests/ingress/neg-exposed/svc.yaml new file mode 100644 index 0000000000..848f28f2ee --- /dev/null +++ b/test/e2e/testing-manifests/ingress/neg-exposed/svc.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: hostname + annotations: + cloud.google.com/neg: '{"ingress":true,"exposed_ports":{"80":{},"443":{}}}' +spec: + ports: + - port: 80 + name: host1 + protocol: TCP + targetPort: 8000 + - port: 443 + name: host2 + protocol: TCP + targetPort: 8080 + selector: + run: hostname + sessionAffinity: None + type: ClusterIP diff --git a/test/e2e/testing-manifests/ingress/neg/svc.yaml b/test/e2e/testing-manifests/ingress/neg/svc.yaml index 3a0be750a2..663930d1de 100644 --- a/test/e2e/testing-manifests/ingress/neg/svc.yaml +++ b/test/e2e/testing-manifests/ingress/neg/svc.yaml @@ -3,7 +3,7 @@ kind: Service metadata: name: hostname annotations: - alpha.cloud.google.com/load-balancer-neg: "true" + cloud.google.com/neg: '{"ingress":true}' spec: ports: - port: 80 @@ -12,4 +12,4 @@ spec: selector: run: hostname sessionAffinity: None - type: NodePort \ No newline at end of file + type: NodePort