Merge pull request #61871 from MrHohn/e2e-ingress-gce-shared-test

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

[e2e ingress-gce] Run preshared-cert and backend HTTPS tests with kubemci

**What this PR does / why we need it**:

Make preshared-cert and backside-reencryption tests compatible with kubemci.

Test is currently failing with the symptom described on https://github.com/GoogleCloudPlatform/k8s-multicluster-ingress/issues/131#issuecomment-377098544.

@nikhiljindal @g-harmon 
/hold

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #NONE 

**Release note**:

```release-note
NONE
```

test/e2e/framework/ingress_utils.go

@@ -1231,7 +1231,7 @@ func (j *IngressTestJig) TryDeleteIngress() {
 }
 
 func (j *IngressTestJig) TryDeleteGivenIngress(ing *extensions.Ingress) {
-	if err := j.runDelete(ing, j.Class); err != nil {
+	if err := j.runDelete(ing); err != nil {
 		j.Logger.Infof("Error while deleting the ingress %v/%v with class %s: %v", ing.Namespace, ing.Name, j.Class, err)
 	}
 }
@@ -1244,7 +1244,7 @@ func (j *IngressTestJig) TryDeleteGivenService(svc *v1.Service) {
 }
 
 // runDelete runs the required command to delete the given ingress.
-func (j *IngressTestJig) runDelete(ing *extensions.Ingress, class string) error {
+func (j *IngressTestJig) runDelete(ing *extensions.Ingress) error {
 	if j.Class != MulticlusterIngressClassValue {
 		return j.Client.ExtensionsV1beta1().Ingresses(ing.Namespace).Delete(ing.Name, nil)
 	}
@@ -1531,14 +1531,16 @@ func (cont *NginxIngressController) Init() {
 	Logf("ingress controller running in pod %v on ip %v", cont.pod.Name, cont.externalIP)
 }
 
-func GenerateReencryptionIngressSpec() *extensions.Ingress {
+func generateBacksideHTTPSIngressSpec(ns string) *extensions.Ingress {
 	return &extensions.Ingress{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: "echoheaders-reencryption",
+			Name:      "echoheaders-https",
+			Namespace: ns,
 		},
 		Spec: extensions.IngressSpec{
+			// Note kubemci requres a default backend.
 			Backend: &extensions.IngressBackend{
-				ServiceName: "echoheaders-reencryption",
+				ServiceName: "echoheaders-https",
 				ServicePort: intstr.IntOrString{
 					Type:   intstr.Int,
 					IntVal: 443,
@@ -1548,10 +1550,10 @@ func GenerateReencryptionIngressSpec() *extensions.Ingress {
 	}
 }
 
-func GenerateReencryptionServiceSpec() *v1.Service {
+func generateBacksideHTTPSServiceSpec() *v1.Service {
 	return &v1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: "echoheaders-reencryption",
+			Name: "echoheaders-https",
 			Annotations: map[string]string{
 				ServiceApplicationProtocolKey: `{"my-https-port":"HTTPS"}`,
 			},
@@ -1564,32 +1566,32 @@ func GenerateReencryptionServiceSpec() *v1.Service {
 				TargetPort: intstr.FromString("echo-443"),
 			}},
 			Selector: map[string]string{
-				"app": "echoheaders-reencryption",
+				"app": "echoheaders-https",
 			},
 			Type: v1.ServiceTypeNodePort,
 		},
 	}
 }
 
-func GenerateReencryptionDeploymentSpec() *extensions.Deployment {
+func generateBacksideHTTPSDeploymentSpec() *extensions.Deployment {
 	return &extensions.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: "echoheaders-reencryption",
+			Name: "echoheaders-https",
 		},
 		Spec: extensions.DeploymentSpec{
 			Selector: &metav1.LabelSelector{MatchLabels: map[string]string{
-				"app": "echoheaders-reencryption",
+				"app": "echoheaders-https",
 			}},
 			Template: v1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels: map[string]string{
-						"app": "echoheaders-reencryption",
+						"app": "echoheaders-https",
 					},
 				},
 				Spec: v1.PodSpec{
 					Containers: []v1.Container{
 						{
-							Name:  "echoheaders-reencryption",
+							Name:  "echoheaders-https",
 							Image: "k8s.gcr.io/echoserver:1.9",
 							Ports: []v1.ContainerPort{{
 								ContainerPort: 8443,
@@ -1603,26 +1605,32 @@ func GenerateReencryptionDeploymentSpec() *extensions.Deployment {
 	}
 }
 
-func CreateReencryptionIngress(cs clientset.Interface, namespace string) (*extensions.Deployment, *v1.Service, *extensions.Ingress, error) {
+// SetUpBacksideHTTPSIngress sets up deployment, service and ingress with backside HTTPS configured.
-	deployCreated, err := cs.ExtensionsV1beta1().Deployments(namespace).Create(GenerateReencryptionDeploymentSpec())
+func (j *IngressTestJig) SetUpBacksideHTTPSIngress(cs clientset.Interface, namespace string, staticIPName string) (*extensions.Deployment, *v1.Service, *extensions.Ingress, error) {
+	deployCreated, err := cs.ExtensionsV1beta1().Deployments(namespace).Create(generateBacksideHTTPSDeploymentSpec())
 	if err != nil {
 		return nil, nil, nil, err
 	}
-	svcCreated, err := cs.CoreV1().Services(namespace).Create(GenerateReencryptionServiceSpec())
+	svcCreated, err := cs.CoreV1().Services(namespace).Create(generateBacksideHTTPSServiceSpec())
 	if err != nil {
 		return nil, nil, nil, err
 	}
-	ingCreated, err := cs.ExtensionsV1beta1().Ingresses(namespace).Create(GenerateReencryptionIngressSpec())
+	ingToCreate := generateBacksideHTTPSIngressSpec(namespace)
+	if staticIPName != "" {
+		ingToCreate.Annotations[IngressStaticIPKey] = staticIPName
+	}
+	ingCreated, err := j.runCreate(ingToCreate)
 	if err != nil {
 		return nil, nil, nil, err
 	}
 	return deployCreated, svcCreated, ingCreated, nil
 }
 
-func CleanupReencryptionIngress(cs clientset.Interface, deploy *extensions.Deployment, svc *v1.Service, ing *extensions.Ingress) []error {
+// DeleteTestResource deletes given deployment, service and ingress.
+func (j *IngressTestJig) DeleteTestResource(cs clientset.Interface, deploy *extensions.Deployment, svc *v1.Service, ing *extensions.Ingress) []error {
 	var errs []error
 	if ing != nil {
-		if err := cs.ExtensionsV1beta1().Ingresses(ing.Namespace).Delete(ing.Name, nil); err != nil {
+		if err := j.runDelete(ing); err != nil {
 			errs = append(errs, fmt.Errorf("error while deleting ingress %s/%s: %v", ing.Namespace, ing.Name, err))
 		}
 	}

test/e2e/network/ingress.go

@@ -31,6 +31,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/authentication/serviceaccount"
 	gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
@@ -320,80 +321,11 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 		})
 
 		It("should create ingress with pre-shared certificate", func() {
-			preSharedCertName := "test-pre-shared-cert"
+			executePresharedCertTest(f, jig, "")
-			By(fmt.Sprintf("Creating ssl certificate %q on GCE", preSharedCertName))
-			testHostname := "test.ingress.com"
-			cert, key, err := framework.GenerateRSACerts(testHostname, true)
-			Expect(err).NotTo(HaveOccurred())
-			gceCloud, err := framework.GetGCECloud()
-			Expect(err).NotTo(HaveOccurred())
-			defer func() {
-				// We would not be able to delete the cert until ingress controller
-				// cleans up the target proxy that references it.
-				By("Deleting ingress before deleting ssl certificate")
-				if jig.Ingress != nil {
-					jig.TryDeleteIngress()
-				}
-				By(fmt.Sprintf("Deleting ssl certificate %q on GCE", preSharedCertName))
-				err := wait.Poll(framework.LoadBalancerPollInterval, framework.LoadBalancerCleanupTimeout, func() (bool, error) {
-					if err := gceCloud.DeleteSslCertificate(preSharedCertName); err != nil && !errors.IsNotFound(err) {
-						framework.Logf("Failed to delete ssl certificate %q: %v. Retrying...", preSharedCertName, err)
-						return false, nil
-					}
-					return true, nil
-				})
-				Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to delete ssl certificate %q: %v", preSharedCertName, err))
-			}()
-			_, err = gceCloud.CreateSslCertificate(&compute.SslCertificate{
-				Name:        preSharedCertName,
-				Certificate: string(cert),
-				PrivateKey:  string(key),
-				Description: "pre-shared cert for ingress testing",
-			})
-			Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to create ssl certificate %q: %v", preSharedCertName, err))
-
-			By("Creating an ingress referencing the pre-shared certificate")
-			// Create an ingress referencing this cert using pre-shared-cert annotation.
-			jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "pre-shared-cert"), ns, map[string]string{
-				framework.IngressPreSharedCertKey: preSharedCertName,
-				framework.IngressAllowHTTPKey:     "false",
-			}, map[string]string{})
-
-			By("Test that ingress works with the pre-shared certificate")
-			err = jig.WaitForIngressWithCert(true, []string{testHostname}, cert)
-			Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unexpected error while waiting for ingress: %v", err))
 		})
 
-		It("should create ingress with backside re-encryption", func() {
+		It("should create ingress with backend HTTPS", func() {
-			By("Creating a set of ingress, service and deployment that have backside re-encryption configured")
+			executeBacksideBacksideHTTPSTest(f, jig, "")
-			deployCreated, svcCreated, ingCreated, err := framework.CreateReencryptionIngress(f.ClientSet, f.Namespace.Name)
-			defer func() {
-				By("Cleaning up re-encryption ingress, service and deployment")
-				if errs := framework.CleanupReencryptionIngress(f.ClientSet, deployCreated, svcCreated, ingCreated); len(errs) > 0 {
-					framework.Failf("Failed to cleanup re-encryption ingress: %v", errs)
-				}
-			}()
-			Expect(err).NotTo(HaveOccurred(), "Failed to create re-encryption ingress")
-
-			By(fmt.Sprintf("Waiting for ingress %s to come up", ingCreated.Name))
-			ingIP, err := jig.WaitForIngressAddress(f.ClientSet, f.Namespace.Name, ingCreated.Name, framework.LoadBalancerPollTimeout)
-			Expect(err).NotTo(HaveOccurred(), "Failed to wait for ingress IP")
-
-			By(fmt.Sprintf("Polling on address %s and verify the backend is serving HTTPS", ingIP))
-			timeoutClient := &http.Client{Timeout: framework.IngressReqTimeout}
-			err = wait.PollImmediate(framework.LoadBalancerPollInterval, framework.LoadBalancerPollTimeout, func() (bool, error) {
-				resp, err := framework.SimpleGET(timeoutClient, fmt.Sprintf("http://%s", ingIP), "")
-				if err != nil {
-					framework.Logf("SimpleGET failed: %v", err)
-					return false, nil
-				}
-				if !strings.Contains(resp, "request_scheme=https") {
-					return false, fmt.Errorf("request wasn't served by HTTPS, response body: %s", resp)
-				}
-				framework.Logf("Poll succeeded, request was served by HTTPS")
-				return true, nil
-			})
-			Expect(err).NotTo(HaveOccurred(), "Failed to verify backside re-encryption ingress")
 		})
 
 		It("multicluster ingress should get instance group annotation", func() {
@@ -603,11 +535,13 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 
 	Describe("GCE [Slow] [Feature:kubemci]", func() {
 		var gceController *framework.GCEIngressController
+		var ipName string
 
 		// Platform specific setup
 		BeforeEach(func() {
 			framework.SkipUnlessProviderIs("gce", "gke")
 			jig.Class = framework.MulticlusterIngressClassValue
+			jig.PollInterval = 5 * time.Second
 			By("Initializing gce controller")
 			gceController = &framework.GCEIngressController{
 				Ns:     ns,
@@ -616,6 +550,13 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 			}
 			err := gceController.Init()
 			Expect(err).NotTo(HaveOccurred())
+
+			// TODO(https://github.com/GoogleCloudPlatform/k8s-multicluster-ingress/issues/19):
+			// Kubemci should reserve a static ip if user has not specified one.
+			ipName = "kubemci-" + string(uuid.NewUUID())
+			// ip released when the rest of lb resources are deleted in CleanupGCEIngressController
+			ipAddress := gceController.CreateStaticIP(ipName)
+			By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", ipName, ipAddress))
 		})
 
 		// Platform specific cleanup
@@ -635,12 +576,6 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 		})
 
 		It("should conform to Ingress spec", func() {
-			jig.PollInterval = 5 * time.Second
-			// Use the randomly generated namespace name as the ip address name.
-			ipName := ns
-			// ip released when the rest of lb resources are deleted in CleanupGCEIngressController
-			ipAddress := gceController.CreateStaticIP(ipName)
-			By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", ipName, ipAddress))
 			conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{
 				framework.IngressStaticIPKey: ipName,
 			})
@@ -651,6 +586,14 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 				jig.WaitForIngress(false /*waitForNodePort*/)
 			}
 		})
+
+		It("should create ingress with pre-shared certificate", func() {
+			executePresharedCertTest(f, jig, ipName)
+		})
+
+		It("should create ingress with backend HTTPS", func() {
+			executeBacksideBacksideHTTPSTest(f, jig, ipName)
+		})
 	})
 
 	// Time: borderline 5m, slow by design
@@ -704,3 +647,86 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 		})
 	})
 })
+
+func executePresharedCertTest(f *framework.Framework, jig *framework.IngressTestJig, staticIPName string) {
+	preSharedCertName := "test-pre-shared-cert"
+	By(fmt.Sprintf("Creating ssl certificate %q on GCE", preSharedCertName))
+	testHostname := "test.ingress.com"
+	cert, key, err := framework.GenerateRSACerts(testHostname, true)
+	Expect(err).NotTo(HaveOccurred())
+	gceCloud, err := framework.GetGCECloud()
+	Expect(err).NotTo(HaveOccurred())
+	defer func() {
+		// We would not be able to delete the cert until ingress controller
+		// cleans up the target proxy that references it.
+		By("Deleting ingress before deleting ssl certificate")
+		if jig.Ingress != nil {
+			jig.TryDeleteIngress()
+		}
+		By(fmt.Sprintf("Deleting ssl certificate %q on GCE", preSharedCertName))
+		err := wait.Poll(framework.LoadBalancerPollInterval, framework.LoadBalancerCleanupTimeout, func() (bool, error) {
+			if err := gceCloud.DeleteSslCertificate(preSharedCertName); err != nil && !errors.IsNotFound(err) {
+				framework.Logf("Failed to delete ssl certificate %q: %v. Retrying...", preSharedCertName, err)
+				return false, nil
+			}
+			return true, nil
+		})
+		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to delete ssl certificate %q: %v", preSharedCertName, err))
+	}()
+	_, err = gceCloud.CreateSslCertificate(&compute.SslCertificate{
+		Name:        preSharedCertName,
+		Certificate: string(cert),
+		PrivateKey:  string(key),
+		Description: "pre-shared cert for ingress testing",
+	})
+	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to create ssl certificate %q: %v", preSharedCertName, err))
+
+	By("Creating an ingress referencing the pre-shared certificate")
+	// Create an ingress referencing this cert using pre-shared-cert annotation.
+	ingAnnotations := map[string]string{
+		framework.IngressPreSharedCertKey: preSharedCertName,
+		// Disallow HTTP to save resources. This is irrelevant to the
+		// pre-shared cert test.
+		framework.IngressAllowHTTPKey: "false",
+	}
+	if staticIPName != "" {
+		ingAnnotations[framework.IngressStaticIPKey] = staticIPName
+	}
+	jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "pre-shared-cert"), f.Namespace.Name, ingAnnotations, map[string]string{})
+
+	By("Test that ingress works with the pre-shared certificate")
+	err = jig.WaitForIngressWithCert(true, []string{testHostname}, cert)
+	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unexpected error while waiting for ingress: %v", err))
+}
+
+func executeBacksideBacksideHTTPSTest(f *framework.Framework, jig *framework.IngressTestJig, staticIPName string) {
+	By("Creating a set of ingress, service and deployment that have backside re-encryption configured")
+	deployCreated, svcCreated, ingCreated, err := jig.SetUpBacksideHTTPSIngress(f.ClientSet, f.Namespace.Name, staticIPName)
+	defer func() {
+		By("Cleaning up re-encryption ingress, service and deployment")
+		if errs := jig.DeleteTestResource(f.ClientSet, deployCreated, svcCreated, ingCreated); len(errs) > 0 {
+			framework.Failf("Failed to cleanup re-encryption ingress: %v", errs)
+		}
+	}()
+	Expect(err).NotTo(HaveOccurred(), "Failed to create re-encryption ingress")
+
+	By(fmt.Sprintf("Waiting for ingress %s to come up", ingCreated.Name))
+	ingIP, err := jig.WaitForIngressAddress(f.ClientSet, f.Namespace.Name, ingCreated.Name, framework.LoadBalancerPollTimeout)
+	Expect(err).NotTo(HaveOccurred(), "Failed to wait for ingress IP")
+
+	By(fmt.Sprintf("Polling on address %s and verify the backend is serving HTTPS", ingIP))
+	timeoutClient := &http.Client{Timeout: framework.IngressReqTimeout}
+	err = wait.PollImmediate(framework.LoadBalancerPollInterval, framework.LoadBalancerPollTimeout, func() (bool, error) {
+		resp, err := framework.SimpleGET(timeoutClient, fmt.Sprintf("http://%s", ingIP), "")
+		if err != nil {
+			framework.Logf("SimpleGET failed: %v", err)
+			return false, nil
+		}
+		if !strings.Contains(resp, "request_scheme=https") {
+			return false, fmt.Errorf("request wasn't served by HTTPS, response body: %s", resp)
+		}
+		framework.Logf("Poll succeeded, request was served by HTTPS")
+		return true, nil
+	})
+	Expect(err).NotTo(HaveOccurred(), "Failed to verify backside re-encryption ingress")
+}

test/e2e/testing-manifests/ingress/pre-shared-cert/ing.yaml

@@ -6,6 +6,10 @@ metadata:
   # annotations:
   #  ingress.gcp.kubernetes.io/pre-shared-cert: "test-pre-shared-cert"
 spec:
+  # kubemci requires a default backend.
+  backend:
+    serviceName: echoheaders-https
+    servicePort: 80
   rules:
   - host: test.ingress.com
     http: