From 34928d219c70816cc2093c4f26989622450bb1c9 Mon Sep 17 00:00:00 2001 From: Ashley Gau Date: Wed, 20 Jun 2018 17:12:32 -0700 Subject: [PATCH 1/3] add e2e test for standalone (exposed) NEG annotation --- test/e2e/framework/ingress_utils.go | 16 +++- test/e2e/network/ingress.go | 73 +++++++++++++++++++ .../ingress/neg-clusterip/svc.yaml | 2 +- .../ingress/neg-exposed/ing.yaml | 8 ++ .../ingress/neg-exposed/rc.yaml | 31 ++++++++ .../ingress/neg-exposed/svc.yaml | 20 +++++ .../testing-manifests/ingress/neg/svc.yaml | 4 +- 7 files changed, 148 insertions(+), 6 deletions(-) create mode 100644 test/e2e/testing-manifests/ingress/neg-exposed/ing.yaml create mode 100644 test/e2e/testing-manifests/ingress/neg-exposed/rc.yaml create mode 100644 test/e2e/testing-manifests/ingress/neg-exposed/svc.yaml diff --git a/test/e2e/framework/ingress_utils.go b/test/e2e/framework/ingress_utils.go index f1d5b14a86..c4cccb25b3 100644 --- a/test/e2e/framework/ingress_utils.go +++ b/test/e2e/framework/ingress_utils.go @@ -121,8 +121,9 @@ const ( // a single character of padding. nameLenLimit = 62 - NEGAnnotation = "alpha.cloud.google.com/load-balancer-neg" - NEGUpdateTimeout = 2 * time.Minute + NEGAnnotation = "cloud.google.com/neg" + NEGStatusAnnotation = "cloud.google.com/neg-status" + NEGUpdateTimeout = 2 * time.Minute InstanceGroupAnnotation = "ingress.gcp.kubernetes.io/instance-groups" @@ -164,6 +165,15 @@ type IngressConformanceTests struct { ExitLog string } +// NegStatus contains name and zone of the Network Endpoint Group +// resources associated with this service +type NegStatus struct { + // NetworkEndpointGroups returns the mapping between service port and NEG + // resource. key is service port, value is the name of the NEG resource. + NetworkEndpointGroups map[int32]string `json:"network_endpoint_groups,omitempty"` + Zones []string `json:"zones,omitempty"` +} + // CreateIngressComformanceTests generates an slice of sequential test cases: // a simple http ingress, ingress with HTTPS, ingress HTTPS with a modified hostname, // ingress https with a modified URLMap @@ -940,7 +950,7 @@ func (cont *GCEIngressController) backendMode(svcPorts map[string]v1.ServicePort bsMatch := &compute.BackendService{} // Non-NEG BackendServices are named with the Nodeport in the name. // NEG BackendServices' names contain the a sha256 hash of a string. - negString := strings.Join([]string{uid, cont.Ns, svcName, sp.TargetPort.String()}, ";") + negString := strings.Join([]string{uid, cont.Ns, svcName, fmt.Sprintf("%v", sp.Port)}, ";") negHash := fmt.Sprintf("%x", sha256.Sum256([]byte(negString)))[:8] for _, bs := range beList { if strings.Contains(bs.Name, strconv.Itoa(int(sp.NodePort))) || diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go index b6b31a75eb..827f32a51b 100644 --- a/test/e2e/network/ingress.go +++ b/test/e2e/network/ingress.go @@ -17,6 +17,7 @@ limitations under the License. package network import ( + "encoding/json" "fmt" "net/http" "path/filepath" @@ -31,6 +32,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/serviceaccount" @@ -647,6 +649,77 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { } }) }) + + It("should sync endpoints for both Ingress-referenced NEG and standalone NEG [Unreleased]", func() { + name := "hostname" + scaleAndValidateExposedNEG := func(num int) { + scale, err := f.ClientSet.ExtensionsV1beta1().Deployments(ns).GetScale(name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + if scale.Spec.Replicas != int32(num) { + scale.Spec.Replicas = int32(num) + _, err = f.ClientSet.ExtensionsV1beta1().Deployments(ns).UpdateScale(name, scale) + Expect(err).NotTo(HaveOccurred()) + } + wait.Poll(10*time.Second, framework.NEGUpdateTimeout, func() (bool, error) { + svc, err := f.ClientSet.CoreV1().Services(ns).Get(name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + negs := sets.NewString() + var status framework.NegStatus + // Wait for NEG sync loop to find NEGs + framework.Logf("Waiting for %v, got: %+v", framework.NEGStatusAnnotation, svc.Annotations) + v, ok := svc.Annotations[framework.NEGStatusAnnotation] + if !ok { + return false, nil + } + err = json.Unmarshal([]byte(v), &status) + if err != nil { + framework.Logf("Error in parsing Expose NEG annotation: %v", err) + return false, nil + } + framework.Logf("Got %v: %v", framework.NEGStatusAnnotation, v) + if len(status.NetworkEndpointGroups) == 0 { + return false, nil + } + for _, neg := range status.NetworkEndpointGroups { + negs.Insert(neg) + } + + gceCloud := gceController.Cloud.Provider.(*gcecloud.GCECloud) + for _, neg := range negs.List() { + exposedNegs, err := gceCloud.ListNetworkEndpoints(neg, gceController.Cloud.Zone, false) + framework.Logf("ExposedNegs: %v, err: %v", exposedNegs, err) + Expect(err).NotTo(HaveOccurred()) + if len(exposedNegs) != num { + return false, nil + } + } + + return len(negs.List()) > 0, nil + }) + } + + By("Create a basic HTTP ingress using NEG") + jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "neg-exposed"), ns, map[string]string{}, map[string]string{}) + jig.WaitForIngress(true) + usingNEG, err := gceController.BackendServiceUsingNEG(jig.GetServicePorts(false)) + Expect(err).NotTo(HaveOccurred()) + Expect(usingNEG).To(BeTrue()) + // initial replicas number is 1 + scaleAndValidateExposedNEG(1) + + By("Scale up number of backends to 5") + scaleAndValidateExposedNEG(5) + + By("Scale down number of backends to 3") + scaleAndValidateExposedNEG(3) + + By("Scale up number of backends to 6") + scaleAndValidateExposedNEG(6) + + By("Scale down number of backends to 2") + scaleAndValidateExposedNEG(3) + }) }) Describe("GCE [Slow] [Feature:kubemci]", func() { diff --git a/test/e2e/testing-manifests/ingress/neg-clusterip/svc.yaml b/test/e2e/testing-manifests/ingress/neg-clusterip/svc.yaml index 6ca920bd26..d640800ccb 100644 --- a/test/e2e/testing-manifests/ingress/neg-clusterip/svc.yaml +++ b/test/e2e/testing-manifests/ingress/neg-clusterip/svc.yaml @@ -3,7 +3,7 @@ kind: Service metadata: name: hostname annotations: - alpha.cloud.google.com/load-balancer-neg: "true" + cloud.google.com/neg: '{"ingress":true}' spec: ports: - port: 80 diff --git a/test/e2e/testing-manifests/ingress/neg-exposed/ing.yaml b/test/e2e/testing-manifests/ingress/neg-exposed/ing.yaml new file mode 100644 index 0000000000..e266d98516 --- /dev/null +++ b/test/e2e/testing-manifests/ingress/neg-exposed/ing.yaml @@ -0,0 +1,8 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: hostname +spec: + backend: + serviceName: hostname + servicePort: 80 \ No newline at end of file diff --git a/test/e2e/testing-manifests/ingress/neg-exposed/rc.yaml b/test/e2e/testing-manifests/ingress/neg-exposed/rc.yaml new file mode 100644 index 0000000000..772253a133 --- /dev/null +++ b/test/e2e/testing-manifests/ingress/neg-exposed/rc.yaml @@ -0,0 +1,31 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + run: hostname + name: hostname +spec: + template: + metadata: + labels: + run: hostname + spec: + containers: + - image: gcr.io/kubernetes-e2e-test-images/serve-hostname-amd64:1.1 + name: host1 + command: + - /bin/sh + - -c + - /serve_hostname -http=true -udp=false -port=8000 + ports: + - protocol: TCP + containerPort: 8000 + - image: gcr.io/kubernetes-e2e-test-images/serve-hostname-amd64:1.1 + name: host2 + command: + - /bin/sh + - -c + - /serve_hostname -http=true -udp=false -port=8080 + ports: + - protocol: TCP + containerPort: 8080 diff --git a/test/e2e/testing-manifests/ingress/neg-exposed/svc.yaml b/test/e2e/testing-manifests/ingress/neg-exposed/svc.yaml new file mode 100644 index 0000000000..848f28f2ee --- /dev/null +++ b/test/e2e/testing-manifests/ingress/neg-exposed/svc.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: hostname + annotations: + cloud.google.com/neg: '{"ingress":true,"exposed_ports":{"80":{},"443":{}}}' +spec: + ports: + - port: 80 + name: host1 + protocol: TCP + targetPort: 8000 + - port: 443 + name: host2 + protocol: TCP + targetPort: 8080 + selector: + run: hostname + sessionAffinity: None + type: ClusterIP diff --git a/test/e2e/testing-manifests/ingress/neg/svc.yaml b/test/e2e/testing-manifests/ingress/neg/svc.yaml index 3a0be750a2..663930d1de 100644 --- a/test/e2e/testing-manifests/ingress/neg/svc.yaml +++ b/test/e2e/testing-manifests/ingress/neg/svc.yaml @@ -3,7 +3,7 @@ kind: Service metadata: name: hostname annotations: - alpha.cloud.google.com/load-balancer-neg: "true" + cloud.google.com/neg: '{"ingress":true}' spec: ports: - port: 80 @@ -12,4 +12,4 @@ spec: selector: run: hostname sessionAffinity: None - type: NodePort \ No newline at end of file + type: NodePort From 90c905b4f16ce8a2fb538ac12bdb1ef3c0478949 Mon Sep 17 00:00:00 2001 From: Ashley Gau Date: Fri, 22 Jun 2018 16:14:51 -0700 Subject: [PATCH 2/3] address comments --- test/e2e/framework/ingress_utils.go | 3 ++- test/e2e/network/ingress.go | 26 +++++++++++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/test/e2e/framework/ingress_utils.go b/test/e2e/framework/ingress_utils.go index c4cccb25b3..2ebe4258de 100644 --- a/test/e2e/framework/ingress_utils.go +++ b/test/e2e/framework/ingress_utils.go @@ -166,7 +166,8 @@ type IngressConformanceTests struct { } // NegStatus contains name and zone of the Network Endpoint Group -// resources associated with this service +// resources associated with this service. +// Needs to be consistent with the NEG internal structs in ingress-gce. type NegStatus struct { // NetworkEndpointGroups returns the mapping between service port and NEG // resource. key is service port, value is the name of the NEG resource. diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go index 827f32a51b..a17db5e91f 100644 --- a/test/e2e/network/ingress.go +++ b/test/e2e/network/ingress.go @@ -652,6 +652,8 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { It("should sync endpoints for both Ingress-referenced NEG and standalone NEG [Unreleased]", func() { name := "hostname" + expectedKeys := []int32{80, 443} + scaleAndValidateExposedNEG := func(num int) { scale, err := f.ClientSet.ExtensionsV1beta1().Deployments(ns).GetScale(name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -666,10 +668,10 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { negs := sets.NewString() var status framework.NegStatus - // Wait for NEG sync loop to find NEGs - framework.Logf("Waiting for %v, got: %+v", framework.NEGStatusAnnotation, svc.Annotations) v, ok := svc.Annotations[framework.NEGStatusAnnotation] if !ok { + // Wait for NEG sync loop to find NEGs + framework.Logf("Waiting for %v, got: %+v", framework.NEGStatusAnnotation, svc.Annotations) return false, nil } err = json.Unmarshal([]byte(v), &status) @@ -678,24 +680,34 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { return false, nil } framework.Logf("Got %v: %v", framework.NEGStatusAnnotation, v) - if len(status.NetworkEndpointGroups) == 0 { + + // Expect 2 NEGs to be created based on the test setup (neg-exposed) + if len(status.NetworkEndpointGroups) != 2 { + framework.Logf("Expected 2 NEGs, got %d", len(negs.List())) return false, nil } + + for _, port := range expectedKeys { + if _, ok := status.NetworkEndpointGroups[port]; !ok { + framework.Logf("Expected ServicePort key %v, but does not exist", port) + } + } + for _, neg := range status.NetworkEndpointGroups { negs.Insert(neg) } gceCloud := gceController.Cloud.Provider.(*gcecloud.GCECloud) for _, neg := range negs.List() { - exposedNegs, err := gceCloud.ListNetworkEndpoints(neg, gceController.Cloud.Zone, false) - framework.Logf("ExposedNegs: %v, err: %v", exposedNegs, err) + networkEndpoints, err := gceCloud.ListNetworkEndpoints(neg, gceController.Cloud.Zone, false) Expect(err).NotTo(HaveOccurred()) - if len(exposedNegs) != num { + if len(networkEndpoints) != num { + framework.Logf("Expect number of endpoints to be %d, but got %d", num, len(networkEndpoints)) return false, nil } } - return len(negs.List()) > 0, nil + return true, nil }) } From c981a3349f6348ae0ceb7ba52377e266cb059be4 Mon Sep 17 00:00:00 2001 From: Ashley Gau Date: Fri, 22 Jun 2018 17:21:28 -0700 Subject: [PATCH 3/3] simplify negs checking --- test/e2e/network/ingress.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go index a17db5e91f..418377737e 100644 --- a/test/e2e/network/ingress.go +++ b/test/e2e/network/ingress.go @@ -32,7 +32,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/serviceaccount" @@ -666,7 +665,6 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { svc, err := f.ClientSet.CoreV1().Services(ns).Get(name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) - negs := sets.NewString() var status framework.NegStatus v, ok := svc.Annotations[framework.NEGStatusAnnotation] if !ok { @@ -683,7 +681,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { // Expect 2 NEGs to be created based on the test setup (neg-exposed) if len(status.NetworkEndpointGroups) != 2 { - framework.Logf("Expected 2 NEGs, got %d", len(negs.List())) + framework.Logf("Expected 2 NEGs, got %d", len(status.NetworkEndpointGroups)) return false, nil } @@ -693,12 +691,12 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { } } - for _, neg := range status.NetworkEndpointGroups { - negs.Insert(neg) + if len(status.NetworkEndpointGroups) != len(expectedKeys) { + framework.Logf("Expected length of %+v to equal length of %+v, but does not", status.NetworkEndpointGroups, expectedKeys) } gceCloud := gceController.Cloud.Provider.(*gcecloud.GCECloud) - for _, neg := range negs.List() { + for _, neg := range status.NetworkEndpointGroups { networkEndpoints, err := gceCloud.ListNetworkEndpoints(neg, gceController.Cloud.Zone, false) Expect(err).NotTo(HaveOccurred()) if len(networkEndpoints) != num {