Merge pull request #61792 from rramkumar1/ingress-SNI-e2e-testing-updates

Automatic merge from submit-queue (batch tested with PRs 62209, 62281, 61792, 62285, 61973). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add ingress e2e test for multiple TLS (SNI) support

**What this PR does / why we need it**:
Add an e2e test for multiple TLS support in ingress-gce.

**Release note**:
```release-note
None
```
/assign @MrHohn 
/hold

test/e2e/framework/ingress_utils.go

@@ -1213,18 +1213,33 @@ func (j *IngressTestJig) Update(update func(ing *extensions.Ingress)) {
 
 // AddHTTPS updates the ingress to use this secret for these hosts.
 func (j *IngressTestJig) AddHTTPS(secretName string, hosts ...string) {
-	j.Ingress.Spec.TLS = []extensions.IngressTLS{{Hosts: hosts, SecretName: secretName}}
 	// TODO: Just create the secret in GetRootCAs once we're watching secrets in
 	// the ingress controller.
 	_, cert, _, err := createTLSSecret(j.Client, j.Ingress.Namespace, secretName, hosts...)
 	ExpectNoError(err)
 	j.Logger.Infof("Updating ingress %v to use secret %v for TLS termination", j.Ingress.Name, secretName)
 	j.Update(func(ing *extensions.Ingress) {
-		ing.Spec.TLS = []extensions.IngressTLS{{Hosts: hosts, SecretName: secretName}}
+		ing.Spec.TLS = append(ing.Spec.TLS, extensions.IngressTLS{Hosts: hosts, SecretName: secretName})
 	})
 	j.RootCAs[secretName] = cert
 }
 
+// RemoveHTTPS updates the ingress to not use this secret for TLS.
+// Note: Does not delete the secret.
+func (j *IngressTestJig) RemoveHTTPS(secretName string) {
+	newTLS := []extensions.IngressTLS{}
+	for _, ingressTLS := range j.Ingress.Spec.TLS {
+		if secretName != ingressTLS.SecretName {
+			newTLS = append(newTLS, ingressTLS)
+		}
+	}
+	j.Logger.Infof("Updating ingress %v to not use secret %v for TLS termination", j.Ingress.Name, secretName)
+	j.Update(func(ing *extensions.Ingress) {
+		ing.Spec.TLS = newTLS
+	})
+	delete(j.RootCAs, secretName)
+}
+
 // PrepareTLSSecret creates a TLS secret and caches the cert.
 func (j *IngressTestJig) PrepareTLSSecret(namespace, secretName string, hosts ...string) error {
 	_, cert, _, err := createTLSSecret(j.Client, namespace, secretName, hosts...)

test/e2e/network/ingress.go

@@ -311,6 +311,42 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 			executeBacksideBacksideHTTPSTest(f, jig, "")
 		})
 
+		It("should support multiple TLS certs [Unreleased]", func() {
+			By("Creating an ingress with no certs.")
+			jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "multiple-certs"), ns, map[string]string{
+				framework.IngressStaticIPKey: ns,
+			}, map[string]string{})
+
+			By("Adding multiple certs to the ingress.")
+			hosts := []string{"test1.ingress.com", "test2.ingress.com", "test3.ingress.com", "test4.ingress.com"}
+			secrets := []string{"tls-secret-1", "tls-secret-2", "tls-secret-3", "tls-secret-4"}
+			certs := [][]byte{}
+			for i, host := range hosts {
+				jig.AddHTTPS(secrets[i], host)
+				certs = append(certs, jig.GetRootCA(secrets[i]))
+			}
+			for i, host := range hosts {
+				err := jig.WaitForIngressWithCert(true, []string{host}, certs[i])
+				Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unexpected error while waiting for ingress: %v", err))
+			}
+
+			By("Remove all but one of the certs on the ingress.")
+			jig.RemoveHTTPS(secrets[1])
+			jig.RemoveHTTPS(secrets[2])
+			jig.RemoveHTTPS(secrets[3])
+
+			By("Test that the remaining cert is properly served.")
+			err := jig.WaitForIngressWithCert(true, []string{hosts[0]}, certs[0])
+			Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unexpected error while waiting for ingress: %v", err))
+
+			By("Add back one of the certs that was removed and check that all certs are served.")
+			jig.AddHTTPS(secrets[1], hosts[1])
+			for i, host := range hosts[:2] {
+				err := jig.WaitForIngressWithCert(true, []string{host}, certs[i])
+				Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Unexpected error while waiting for ingress: %v", err))
+			}
+		})
+
 		It("multicluster ingress should get instance group annotation", func() {
 			name := "echomap"
 			jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "http"), ns, map[string]string{

test/e2e/testing-manifests/ingress/multiple-certs/ing.yaml

@@ -0,0 +1,34 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: multiple-certs
+spec:
+  rules:
+  - host: test1.ingress.com
+    http:
+      paths:
+      - path: /test
+        backend:
+          serviceName: echoheaders-https
+          servicePort: 80
+  - host: test2.ingress.com
+    http:
+      paths:
+      - path: /test
+        backend:
+          serviceName: echoheaders-https
+          servicePort: 80
+  - host: test3.ingress.com
+    http:
+      paths:
+      - path: /test
+        backend:
+          serviceName: echoheaders-https
+          servicePort: 80
+  - host: test4.ingress.com
+    http:
+      paths:
+      - path: /test
+        backend:
+          serviceName: echoheaders-https
+          servicePort: 80

test/e2e/testing-manifests/ingress/multiple-certs/rc.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: echoheaders-https
+spec:
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: echoheaders-https
+    spec:
+      containers:
+      - name: echoheaders-https
+        image: gcr.io/google_containers/echoserver:1.10
+        ports:
+        - containerPort: 8080

test/e2e/testing-manifests/ingress/multiple-certs/svc.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: echoheaders-https
+  labels:
+    app: echoheaders-https
+spec:
+  type: NodePort
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    app: echoheaders-https