Merge pull request #16048 from bprashanth/IngressE2E

Auto commit by PR queue bot

hack/jenkins/e2e.sh

@@ -124,6 +124,7 @@ GKE_REQUIRED_SKIP_TESTS=(
 # Tests which cannot be run on AWS.
 AWS_REQUIRED_SKIP_TESTS=(
     "experimental\sresource\susage\stracking" # Expect --max-pods=100
+    "GCE\sL7\sLoadBalancer\sController" # GCE L7 loadbalancing
 )
 
 
@@ -153,6 +154,11 @@ GCE_FLAKY_TESTS=(
 # comments below, and for poorly implemented tests, please quote the
 # issue number tracking speed improvements.
 GCE_SLOW_TESTS=(
+    # Before enabling this loadbalancer test in any other test list you must
+    # make sure the associated project has enough quota. At the time of this
+    # writing a GCE project is allowed 3 backend services by default. This
+    # test requires at least 5.
+    "GCE\sL7\sLoadBalancer\sController"               # 10 min,       file: ingress.go,              slow by design
     "SchedulerPredicates\svalidates\sMaxPods\slimit " # 8 min,        file: scheduler_predicates.go, PR:    #13315
     "Nodes\sResize"                                   # 3 min 30 sec, file: resize_nodes.go,         issue: #13323
     "resource\susage\stracking"                       # 1 hour,       file: kubelet_perf.go,         slow by design
@@ -164,6 +170,7 @@ GCE_SLOW_TESTS=(
 
 # Tests which are not able to be run in parallel.
 GCE_PARALLEL_SKIP_TESTS=(
+    "GCE\sL7\sLoadBalancer\sController" # TODO: This cannot run in parallel with other L4 tests till quota has been bumped up.
     "Nodes\sNetwork"
     "MaxPods"
     "Resource\susage\sof\ssystem\scontainers"

test/e2e/ingress.go

@@ -0,0 +1,321 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os/exec"
+	"sort"
+	"time"
+
+	compute "google.golang.org/api/compute/v1"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/apis/extensions"
+	client "k8s.io/kubernetes/pkg/client/unversioned"
+	"k8s.io/kubernetes/pkg/fields"
+	"k8s.io/kubernetes/pkg/labels"
+	"k8s.io/kubernetes/pkg/util"
+	"k8s.io/kubernetes/pkg/util/wait"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// Before enabling this test you must make sure the associated project has
+// enough quota. At the time of this writing GCE projects are allowed 3
+// backend services by default. This test requires at least 5.
+
+// This test exercises the GCE L7 loadbalancer controller cluster-addon. It
+// will fail if the addon isn't running, or doesn't send traffic to the expected
+// backend. Common failure modes include:
+// * GCE L7 took too long to spin up
+// * GCE L7 took too long to health check a backend
+// * Repeated 404:
+//   - L7 is sending traffic to the default backend of the addon.
+//   - Backend is receiving /foo when it expects /bar.
+// * Repeated 5xx:
+//   - Out of quota (describe ing should show you if this is the case)
+//   - Mismatched service/container port, or endpoints are dead.
+
+var (
+	appPrefix         = "foo-app-"
+	pathPrefix        = "foo"
+	testImage         = "gcr.io/google_containers/n-way-http:1.0"
+	httpContainerPort = 8080
+
+	expectedLBCreationTime    = 7 * time.Minute
+	expectedLBHealthCheckTime = 7 * time.Minute
+
+	// On average it takes ~6 minutes for a single backend to come online.
+	// We *don't* expect this poll to consistently take 15 minutes for every
+	// Ingress as GCE is creating/checking backends in parallel, but at the
+	// same time, we're not testing GCE startup latency. So give it enough
+	// time, and fail if the average is too high.
+	lbPollTimeout  = 15 * time.Minute
+	lbPollInterval = 30 * time.Second
+
+	// One can scale this test by tweaking numApps and numIng, the former will
+	// create more RCs/Services and add them to a single Ingress, while the latter
+	// will create smaller, more fragmented Ingresses. The numbers 4, 2 are chosen
+	// arbitrarity, we want to test more than a single Ingress, and it should have
+	// more than 1 url endpoint going to a service.
+	numApps = 4
+	numIng  = 2
+)
+
+// timeSlice allows sorting of time.Duration
+type timeSlice []time.Duration
+
+func (p timeSlice) Len() int {
+	return len(p)
+}
+
+func (p timeSlice) Less(i, j int) bool {
+	return p[i] < p[j]
+}
+
+func (p timeSlice) Swap(i, j int) {
+	p[i], p[j] = p[j], p[i]
+}
+
+// ruleByIndex returns an IngressRule for the given index.
+func ruleByIndex(i int) extensions.IngressRule {
+	return extensions.IngressRule{
+		Host: fmt.Sprintf("foo%d.bar.com", i),
+		IngressRuleValue: extensions.IngressRuleValue{
+			HTTP: &extensions.HTTPIngressRuleValue{
+				Paths: []extensions.HTTPIngressPath{
+					{
+						Path: fmt.Sprintf("/%v%d", pathPrefix, i),
+						Backend: extensions.IngressBackend{
+							ServiceName: fmt.Sprintf("%v%d", appPrefix, i),
+							ServicePort: util.NewIntOrStringFromInt(httpContainerPort),
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+// createIngress creates an Ingress with num rules. Eg:
+// start = 1 num = 2 will given you a single Ingress with 2 rules:
+// Ingress {
+//	 foo1.bar.com: /foo1
+//	 foo2.bar.com: /foo2
+// }
+func createIngress(c *client.Client, ns string, start, num int) extensions.Ingress {
+	ing := extensions.Ingress{
+		ObjectMeta: api.ObjectMeta{
+			Name:      fmt.Sprintf("%v%d", appPrefix, start),
+			Namespace: ns,
+		},
+		Spec: extensions.IngressSpec{
+			Backend: &extensions.IngressBackend{
+				ServiceName: fmt.Sprintf("%v%d", appPrefix, start),
+				ServicePort: util.NewIntOrStringFromInt(httpContainerPort),
+			},
+			Rules: []extensions.IngressRule{},
+		},
+	}
+	for i := start; i < start+num; i++ {
+		ing.Spec.Rules = append(ing.Spec.Rules, ruleByIndex(i))
+	}
+	Logf("Creating ingress %v", start)
+	_, err := c.Extensions().Ingress(ns).Create(&ing)
+	Expect(err).NotTo(HaveOccurred())
+	return ing
+}
+
+// createApp will create a single RC and Svc. The Svc will match pods of the
+// RC using the selector: 'name'=<name arg>
+func createApp(c *client.Client, ns string, i int) {
+	name := fmt.Sprintf("%v%d", appPrefix, i)
+	l := map[string]string{}
+
+	Logf("Creating svc %v", name)
+	svc := svcByName(name, httpContainerPort)
+	svc.Spec.Type = api.ServiceTypeNodePort
+	_, err := c.Services(ns).Create(svc)
+	Expect(err).NotTo(HaveOccurred())
+
+	Logf("Creating rc %v", name)
+	rc := rcByNamePort(name, 1, testImage, httpContainerPort, l)
+	rc.Spec.Template.Spec.Containers[0].Args = []string{
+		"--num=1",
+		fmt.Sprintf("--start=%d", i),
+		fmt.Sprintf("--prefix=%v", pathPrefix),
+		fmt.Sprintf("--port=%d", httpContainerPort),
+	}
+	_, err = c.ReplicationControllers(ns).Create(rc)
+	Expect(err).NotTo(HaveOccurred())
+}
+
+// gcloudUnmarshal unmarshals json output of gcloud into given out interface.
+func gcloudUnmarshal(resource, regex string, out interface{}) {
+	output, err := exec.Command("gcloud", "compute", resource, "list",
+		fmt.Sprintf("--regex=%v", regex), "-q", "--format=json").CombinedOutput()
+	if err != nil {
+		Failf("Error unmarshalling gcloud output: %v", err)
+	}
+	if err := json.Unmarshal([]byte(output), out); err != nil {
+		Failf("Error unmarshalling gcloud output: %v", err)
+	}
+}
+
+func checkLeakedResources() error {
+	msg := ""
+	// Check all resources #16636.
+	beList := []compute.BackendService{}
+	gcloudUnmarshal("backend-services", "k8s-be-[0-9]+", &beList)
+	if len(beList) != 0 {
+		for _, b := range beList {
+			msg += fmt.Sprintf("%v\n", b.Name)
+		}
+		return fmt.Errorf("Found backend services:\n%v", msg)
+	}
+	fwList := []compute.ForwardingRule{}
+	gcloudUnmarshal("forwarding-rules", "k8s-fw-.*", &fwList)
+	if len(fwList) != 0 {
+		for _, f := range fwList {
+			msg += fmt.Sprintf("%v\n", f.Name)
+		}
+		return fmt.Errorf("Found forwarding rules:\n%v", msg)
+	}
+	return nil
+}
+
+var _ = Describe("GCE L7 LoadBalancer Controller", func() {
+	// These variables are initialized after framework's beforeEach.
+	var ns string
+	var client *client.Client
+	var responseTimes, creationTimes []time.Duration
+
+	framework := Framework{BaseName: "glbc"}
+
+	BeforeEach(func() {
+		// This test requires a GCE/GKE only cluster-addon
+		SkipUnlessProviderIs("gce", "gke")
+		framework.beforeEach()
+		client = framework.Client
+		ns = framework.Namespace.Name
+		Expect(waitForRCPodsRunning(client, "kube-system", "glbc")).NotTo(HaveOccurred())
+		Expect(checkLeakedResources()).NotTo(HaveOccurred())
+		responseTimes = []time.Duration{}
+		creationTimes = []time.Duration{}
+	})
+
+	AfterEach(func() {
+		framework.afterEach()
+		err := wait.Poll(lbPollInterval, lbPollTimeout, func() (bool, error) {
+			if err := checkLeakedResources(); err != nil {
+				Logf("Still waiting for glbc to cleanup: %v", err)
+				return false, nil
+			}
+			return true, nil
+		})
+		Logf("Average creation time %+v, health check time %+v", creationTimes, responseTimes)
+		if err != nil {
+			Failf("Failed to cleanup GCE L7 resources.")
+		}
+		Logf("Successfully verified GCE L7 loadbalancer via Ingress.")
+	})
+
+	It("should create GCE L7 loadbalancers and verify Ingress", func() {
+		// Create numApps apps, exposed via numIng Ingress each with 2 paths.
+		// Eg with numApp=10, numIng=5:
+		// apps: {foo-app-(0-10)}
+		// ingress: {foo-app-(0, 2, 4, 6, 8)}
+		// paths:
+		//  ingress foo-app-0:
+		//	  default1.bar.com
+		//	  foo0.bar.com: /foo0
+		//	  foo1.bar.com: /foo1
+		if numApps < numIng {
+			Failf("Need more apps than Ingress")
+		}
+		appsPerIngress := numApps / numIng
+		By(fmt.Sprintf("Creating %d rcs + svc, and %d apps per Ingress", numApps, appsPerIngress))
+		for appID := 0; appID < numApps; appID = appID + appsPerIngress {
+			// Creates appsPerIngress apps, then creates one Ingress with paths to all the apps.
+			for j := appID; j < appID+appsPerIngress; j++ {
+				createApp(client, ns, j)
+			}
+			createIngress(client, ns, appID, appsPerIngress)
+		}
+
+		ings, err := client.Extensions().Ingress(ns).List(
+			labels.Everything(), fields.Everything())
+		Expect(err).NotTo(HaveOccurred())
+
+		for _, ing := range ings.Items {
+			// Wait for the loadbalancer IP.
+			start := time.Now()
+			address, err := waitForIngressAddress(client, ing.Namespace, ing.Name, lbPollTimeout)
+			Expect(err).NotTo(HaveOccurred())
+			By(fmt.Sprintf("Found address %v for ingress %v, took %v to come online",
+				address, ing.Name, time.Since(start)))
+			creationTimes = append(creationTimes, time.Since(start))
+
+			// Check that all rules respond to a simple GET.
+			for _, rules := range ing.Spec.Rules {
+				// As of Kubernetes 1.1 we only support HTTP Ingress.
+				if rules.IngressRuleValue.HTTP == nil {
+					continue
+				}
+				for _, p := range rules.IngressRuleValue.HTTP.Paths {
+					route := fmt.Sprintf("http://%v%v", address, p.Path)
+					Logf("Testing route %v host %v with simple GET", route, rules.Host)
+
+					GETStart := time.Now()
+					var lastBody string
+					pollErr := wait.Poll(lbPollInterval, lbPollTimeout, func() (bool, error) {
+						var err error
+						lastBody, err = simpleGET(http.DefaultClient, route, rules.Host)
+						if err != nil {
+							Logf("host %v path %v: %v", rules.Host, route, err)
+							return false, nil
+						}
+						return true, nil
+					})
+					if pollErr != nil {
+						Failf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v",
+							lbPollTimeout, route, rules.Host, lastBody, pollErr)
+					}
+					rt := time.Since(GETStart)
+					By(fmt.Sprintf("Route %v host %v took %v to respond", route, rules.Host, rt))
+					responseTimes = append(responseTimes, rt)
+				}
+			}
+		}
+		// In most cases slow loadbalancer creation/startup translates directly to
+		// GCE api sluggishness. However this might be because of something the
+		// controller is doing, eg: maxing out QPS by repeated polling.
+		sort.Sort(timeSlice(creationTimes))
+		perc50 := creationTimes[len(creationTimes)/2]
+		if perc50 > expectedLBCreationTime {
+			Failf("Average creation time is too high: %+v", creationTimes)
+		}
+		sort.Sort(timeSlice(responseTimes))
+		perc50 = responseTimes[len(responseTimes)/2]
+		if perc50 > expectedLBHealthCheckTime {
+			Failf("Average startup time is too high: %+v", responseTimes)
+		}
+	})
+})

test/e2e/resize_nodes.go

@@ -41,6 +41,7 @@ const (
 	serveHostnameImage        = "gcr.io/google_containers/serve_hostname:1.1"
 	resizeNodeReadyTimeout    = 2 * time.Minute
 	resizeNodeNotReadyTimeout = 2 * time.Minute
+	testPort                  = 9376
 )
 
 func resizeGroup(size int) error {
@@ -111,25 +112,26 @@ func waitForGroupSize(size int) error {
 	return fmt.Errorf("timeout waiting %v for node instance group size to be %d", timeout, size)
 }
 
-func svcByName(name string) *api.Service {
+func svcByName(name string, port int) *api.Service {
 	return &api.Service{
 		ObjectMeta: api.ObjectMeta{
-			Name: "test-service",
+			Name: name,
 		},
 		Spec: api.ServiceSpec{
+			Type: api.ServiceTypeNodePort,
 			Selector: map[string]string{
 				"name": name,
 			},
 			Ports: []api.ServicePort{{
-				Port:       9376,
+				Port:       port,
-				TargetPort: util.NewIntOrStringFromInt(9376),
+				TargetPort: util.NewIntOrStringFromInt(port),
 			}},
 		},
 	}
 }
 
 func newSVCByName(c *client.Client, ns, name string) error {
-	_, err := c.Services(ns).Create(svcByName(name))
+	_, err := c.Services(ns).Create(svcByName(name, testPort))
 	return err
 }
 

test/e2e/serviceloadbalancers.go

@@ -195,7 +195,7 @@ func (s *ingManager) test(path string) error {
 	url := fmt.Sprintf("%v/hostName", path)
 	httpClient := &http.Client{}
 	return wait.Poll(pollInterval, serviceRespondingTimeout, func() (bool, error) {
-		body, err := simpleGET(httpClient, url)
+		body, err := simpleGET(httpClient, url, "")
 		if err != nil {
 			Logf("%v\n%v\n%v", url, body, err)
 			return false, nil
@@ -240,8 +240,13 @@ var _ = Describe("ServiceLoadBalancer", func() {
 })
 
 // simpleGET executes a get on the given url, returns error if non-200 returned.
-func simpleGET(c *http.Client, url string) (string, error) {
+func simpleGET(c *http.Client, url, host string) (string, error) {
-	res, err := c.Get(url)
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return "", err
+	}
+	req.Host = host
+	res, err := c.Do(req)
 	if err != nil {
 		return "", err
 	}

test/e2e/util.go

@@ -2157,3 +2157,36 @@ func OpenWebSocketForURL(url *url.URL, config *client.Config, protocols []string
 	cfg.Protocol = protocols
 	return websocket.DialConfig(cfg)
 }
+
+// getIngressAddress returns the ips/hostnames associated with the Ingress.
+func getIngressAddress(client *client.Client, ns, name string) ([]string, error) {
+	ing, err := client.Extensions().Ingress(ns).Get(name)
+	if err != nil {
+		return nil, err
+	}
+	addresses := []string{}
+	for _, a := range ing.Status.LoadBalancer.Ingress {
+		if a.IP != "" {
+			addresses = append(addresses, a.IP)
+		}
+		if a.Hostname != "" {
+			addresses = append(addresses, a.Hostname)
+		}
+	}
+	return addresses, nil
+}
+
+// waitForIngressAddress waits for the Ingress to acquire an address.
+func waitForIngressAddress(c *client.Client, ns, ingName string, timeout time.Duration) (string, error) {
+	var address string
+	err := wait.PollImmediate(10*time.Second, timeout, func() (bool, error) {
+		ipOrNameList, err := getIngressAddress(c, ns, ingName)
+		if err != nil || len(ipOrNameList) == 0 {
+			Logf("Waiting for Ingress %v to acquire IP, error %v", ingName, err)
+			return false, nil
+		}
+		address = ipOrNameList[0]
+		return true, nil
+	})
+	return address, err
+}

test/images/n-way-http/Dockerfile

@@ -0,0 +1,18 @@
+# Copyright 2015 The Kubernetes Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM busybox
+MAINTAINER Prashanth B <beeps@google.com>
+ADD server server
+ENTRYPOINT ["/server"]

test/images/n-way-http/Makefile

@@ -0,0 +1,17 @@
+all: push
+
+# 0.0 shouldn't clobber any released builds
+TAG = 0.0
+PREFIX = gcr.io/google_containers/n-way-http
+
+server: server.go
+	CGO_ENABLED=0 GOOS=linux godep go build -a -installsuffix cgo -ldflags '-w' -o server ./server.go
+
+container: server
+	docker build -t $(PREFIX):$(TAG) .
+
+push: container
+	gcloud docker push $(PREFIX):$(TAG)
+
+clean:
+	rm -f server

test/images/n-way-http/server.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// A webserver that runs n http handlers. Example invocation:
+// - server -port 8080 -prefix foo -num 10 -start 0
+// Will given you 10 /foo(i) endpoints that simply echo foo(i) when requested.
+// - server -start 3 -num 1
+// Will create just one endpoint, at /foo3
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+)
+
+var (
+	port   = flag.Int("port", 8080, "Port number for requests.")
+	prefix = flag.String("prefix", "foo", "String used as path prefix")
+	num    = flag.Int("num", 10, "Number of endpoints to create.")
+	start  = flag.Int("start", 0, "Index to start, only makes sense with --num")
+)
+
+func main() {
+	flag.Parse()
+	// This container is used to test the GCE L7 controller which expects "/"
+	// to return a 200 response.
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprint(w, "ok")
+	})
+	for i := *start; i < *start+*num; i++ {
+		path := fmt.Sprintf("%v%d", *prefix, i)
+		http.HandleFunc(fmt.Sprintf("/%v", path), func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusOK)
+			fmt.Fprint(w, path)
+		})
+	}
+	log.Printf("server -port %d -prefix %v -num %d -start %d", *port, *prefix, *num, *start)
+	http.ListenAndServe(fmt.Sprintf(":%d", *port), nil)
+}