From 672819be16716ad216828a2752ed1ecaf1de1cb6 Mon Sep 17 00:00:00 2001 From: marekbiskup Date: Fri, 8 May 2015 10:07:32 -0700 Subject: [PATCH] multiport endpoint e2e test --- test/e2e/service.go | 237 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 187 insertions(+), 50 deletions(-) diff --git a/test/e2e/service.go b/test/e2e/service.go index 3954a7638b..fb254f5d0a 100644 --- a/test/e2e/service.go +++ b/test/e2e/service.go @@ -37,14 +37,28 @@ import ( var _ = Describe("Services", func() { var c *client.Client // Use these in tests. They're unique for each test to prevent name collisions. - var namespace0, namespace1 string + var namespaces [2]string BeforeEach(func() { var err error c, err = loadClient() Expect(err).NotTo(HaveOccurred()) - namespace0 = "e2e-ns-" + dateStamp() + "-0" - namespace1 = "e2e-ns-" + dateStamp() + "-1" + + By("Building a namespace api objects") + for i := range namespaces { + namespacePtr, err := createTestingNS(fmt.Sprintf("service-%d", i), c) + Expect(err).NotTo(HaveOccurred()) + namespaces[i] = namespacePtr.Name + } + }) + + AfterEach(func() { + for _, ns := range namespaces { + By(fmt.Sprintf("Destroying namespace %v", ns)) + if err := c.Namespaces().Delete(ns); err != nil { + Failf("Couldn't delete namespace %s: %s", ns, err) + } + } }) It("should provide DNS for the cluster", func() { @@ -53,13 +67,14 @@ var _ = Describe("Services", func() { return } - podClient := c.Pods(api.NamespaceDefault) - + ns := namespaces[0] + podClient := c.Pods(ns) //TODO: Wait for skyDNS // All the names we need to be able to resolve. namesToResolve := []string{ - "kubernetes-ro", + // "kubernetes-ro" is not directly visible because we're not in the the default namespace + // TODO consider creating a service in our namespace and query it's DNS name too. "kubernetes-ro.default", "kubernetes-ro.default.cluster.local", "google.com", @@ -120,15 +135,14 @@ var _ = Describe("Services", func() { By("submitting the pod to kubernetes") defer func() { - By("deleting the pod") + By("deleting pod " + pod.Name) defer GinkgoRecover() podClient.Delete(pod.Name, nil) }() if _, err := podClient.Create(pod); err != nil { - Failf("Failed to create %s pod: %v", pod.Name, err) + Failf("Failed to create pod %s: %v", pod.Name, err) } - - expectNoError(waitForPodRunning(c, pod.Name)) + expectNoError(waitForPodRunningInNamespace(c, pod.Name, ns)) By("retrieving the pod") pod, err := podClient.Get(pod.Name) @@ -148,7 +162,7 @@ var _ = Describe("Services", func() { _, err := c.Get(). Prefix("proxy"). Resource("pods"). - Namespace(api.NamespaceDefault). + Namespace(ns). Name(pod.Name). Suffix("results", testCase). Do().Raw() @@ -194,7 +208,7 @@ var _ = Describe("Services", func() { It("should serve a basic endpoint from pods", func(done Done) { serviceName := "endpoint-test2" - ns := api.NamespaceDefault + ns := namespaces[0] labels := map[string]string{ "foo": "bar", "baz": "blah", @@ -219,9 +233,8 @@ var _ = Describe("Services", func() { } _, err := c.Services(ns).Create(service) Expect(err).NotTo(HaveOccurred()) - expectedPort := 80 - validateEndpointsOrFail(c, ns, serviceName, expectedPort, []string{}) + validateEndpointsOrFail(c, ns, serviceName, map[string][]int{}) var names []string defer func() { @@ -232,28 +245,129 @@ var _ = Describe("Services", func() { }() name1 := "test1" - addEndpointPodOrFail(c, ns, name1, labels) + addEndpointPodOrFail(c, ns, name1, labels, []api.ContainerPort{{ContainerPort: 80}}) names = append(names, name1) - validateEndpointsOrFail(c, ns, serviceName, expectedPort, names) + validateEndpointsOrFail(c, ns, serviceName, map[string][]int{name1: {80}}) name2 := "test2" - addEndpointPodOrFail(c, ns, name2, labels) + addEndpointPodOrFail(c, ns, name2, labels, []api.ContainerPort{{ContainerPort: 80}}) names = append(names, name2) - validateEndpointsOrFail(c, ns, serviceName, expectedPort, names) + validateEndpointsOrFail(c, ns, serviceName, map[string][]int{name1: {80}, name2: {80}}) err = c.Pods(ns).Delete(name1, nil) Expect(err).NotTo(HaveOccurred()) names = []string{name2} - validateEndpointsOrFail(c, ns, serviceName, expectedPort, names) + validateEndpointsOrFail(c, ns, serviceName, map[string][]int{name2: {80}}) err = c.Pods(ns).Delete(name2, nil) Expect(err).NotTo(HaveOccurred()) names = []string{} - validateEndpointsOrFail(c, ns, serviceName, expectedPort, names) + validateEndpointsOrFail(c, ns, serviceName, map[string][]int{}) + + // We deferred Gingko pieces that may Fail, we aren't done. + defer func() { + close(done) + }() + }, 240.0) + + It("should serve multiport endpoints from pods", func(done Done) { + // repacking functionality is intentionally not tested here - it's better to test it in an integration test. + serviceName := "multi-endpoint-test" + ns := namespaces[0] + + defer func() { + err := c.Services(ns).Delete(serviceName) + Expect(err).NotTo(HaveOccurred()) + }() + + labels := map[string]string{"foo": "bar"} + + svc1port := "svc1" + svc2port := "svc2" + + service := &api.Service{ + ObjectMeta: api.ObjectMeta{ + Name: serviceName, + }, + Spec: api.ServiceSpec{ + Selector: labels, + Ports: []api.ServicePort{ + { + Name: "portname1", + Port: 80, + TargetPort: util.NewIntOrStringFromString(svc1port), + }, + { + Name: "portname2", + Port: 81, + TargetPort: util.NewIntOrStringFromString(svc2port), + }, + }, + }, + } + _, err := c.Services(ns).Create(service) + Expect(err).NotTo(HaveOccurred()) + port1 := 100 + port2 := 101 + validateEndpointsOrFail(c, ns, serviceName, map[string][]int{}) + + var names []string + defer func() { + for _, name := range names { + err := c.Pods(ns).Delete(name, nil) + Expect(err).NotTo(HaveOccurred()) + } + }() + + containerPorts1 := []api.ContainerPort{ + { + Name: svc1port, + ContainerPort: port1, + }, + } + containerPorts2 := []api.ContainerPort{ + { + Name: svc2port, + ContainerPort: port2, + }, + } + + podname1 := "podname1" + addEndpointPodOrFail(c, ns, podname1, labels, containerPorts1) + names = append(names, podname1) + validateEndpointsOrFail(c, ns, serviceName, map[string][]int{podname1: {port1}}) + + podname2 := "podname2" + addEndpointPodOrFail(c, ns, podname2, labels, containerPorts2) + names = append(names, podname2) + validateEndpointsOrFail(c, ns, serviceName, map[string][]int{podname1: {port1}, podname2: {port2}}) + + podname3 := "podname3" + addEndpointPodOrFail(c, ns, podname3, labels, append(containerPorts1, containerPorts2...)) + names = append(names, podname3) + validateEndpointsOrFail(c, ns, serviceName, map[string][]int{podname1: {port1}, podname2: {port2}, podname3: {port1, port2}}) + + err = c.Pods(ns).Delete(podname1, nil) + Expect(err).NotTo(HaveOccurred()) + names = []string{podname2, podname3} + + validateEndpointsOrFail(c, ns, serviceName, map[string][]int{podname2: {port2}, podname3: {port1, port2}}) + + err = c.Pods(ns).Delete(podname2, nil) + Expect(err).NotTo(HaveOccurred()) + names = []string{podname3} + + validateEndpointsOrFail(c, ns, serviceName, map[string][]int{podname3: {port1, port2}}) + + err = c.Pods(ns).Delete(podname3, nil) + Expect(err).NotTo(HaveOccurred()) + names = []string{} + + validateEndpointsOrFail(c, ns, serviceName, map[string][]int{}) // We deferred Gingko pieces that may Fail, we aren't done. defer func() { @@ -268,7 +382,7 @@ var _ = Describe("Services", func() { } serviceName := "external-lb-test" - ns := namespace0 + ns := namespaces[0] labels := map[string]string{ "key0": "value0", } @@ -363,8 +477,7 @@ var _ = Describe("Services", func() { return } - serviceNames := []string{"s0"} // Could add more here, but then it takes longer. - namespaces := []string{namespace0, namespace1} // As above. + serviceNames := []string{"s0"} // Could add more here, but then it takes longer. labels := map[string]string{ "key0": "value0", "key1": "value1", @@ -437,54 +550,85 @@ func validateUniqueOrFail(s []string) { } } -func flattenSubsets(subsets []api.EndpointSubset, expectedPort int) util.StringSet { - ips := util.StringSet{} +func getPortsByIp(subsets []api.EndpointSubset) map[string][]int { + m := make(map[string][]int) for _, ss := range subsets { for _, port := range ss.Ports { - if port.Port == expectedPort { - for _, addr := range ss.Addresses { - ips.Insert(addr.IP) + for _, addr := range ss.Addresses { + Logf("Found IP %v and port %v", addr.IP, port.Port) + if _, ok := m[addr.IP]; !ok { + m[addr.IP] = make([]int, 0) } + m[addr.IP] = append(m[addr.IP], port.Port) } } } - return ips + return m } -func validateIPsOrFail(c *client.Client, ns string, expectedPods []string, ips util.StringSet) { - for _, name := range expectedPods { +func translatePodNameToIpOrFail(c *client.Client, ns string, expectedEndpoints map[string][]int) map[string][]int { + portsByIp := make(map[string][]int) + + for name, portList := range expectedEndpoints { pod, err := c.Pods(ns).Get(name) if err != nil { Failf("failed to get pod %s, that's pretty weird. validation failed: %s", name, err) } - if !ips.Has(pod.Status.PodIP) { - Failf("ip validation failed, expected: %v, saw: %v", ips, pod.Status.PodIP) - } + portsByIp[pod.Status.PodIP] = portList By(fmt.Sprintf("")) } - By(fmt.Sprintf("successfully validated IPs %v against expected endpoints %v on namespace %s", ips, expectedPods, ns)) + By(fmt.Sprintf("successfully translated pod names to ips: %v -> %v on namespace %s", expectedEndpoints, portsByIp, ns)) + return portsByIp } -func validateEndpointsOrFail(c *client.Client, ns, serviceName string, expectedPort int, expectedPods []string) { +func validatePortsOrFail(endpoints map[string][]int, expectedEndpoints map[string][]int) { + if len(endpoints) != len(expectedEndpoints) { + // should not happen because we check this condition before + Failf("invalid number of endpoints got %v, expected %v", endpoints, expectedEndpoints) + } + for ip := range expectedEndpoints { + if _, ok := endpoints[ip]; !ok { + Failf("endpoint %v not found", ip) + } + if len(endpoints[ip]) != len(expectedEndpoints[ip]) { + Failf("invalid list of ports for ip %v. Got %v, expected %v", ip, endpoints[ip], expectedEndpoints[ip]) + } + sort.Ints(endpoints[ip]) + sort.Ints(expectedEndpoints[ip]) + for index := range endpoints[ip] { + if endpoints[ip][index] != expectedEndpoints[ip][index] { + Failf("invalid list of ports for ip %v. Got %v, expected %v", ip, endpoints[ip], expectedEndpoints[ip]) + } + } + } +} + +func validateEndpointsOrFail(c *client.Client, ns, serviceName string, expectedEndpoints map[string][]int) { + By(fmt.Sprintf("Validating endpoints %v with on service %s/%s", expectedEndpoints, ns, serviceName)) for { endpoints, err := c.Endpoints(ns).Get(serviceName) if err == nil { - ips := flattenSubsets(endpoints.Subsets, expectedPort) - if len(ips) == len(expectedPods) { - validateIPsOrFail(c, ns, expectedPods, ips) + By(fmt.Sprintf("Found endpoints %v", endpoints)) + + portsByIp := getPortsByIp(endpoints.Subsets) + + By(fmt.Sprintf("Found ports by ip %v", portsByIp)) + if len(portsByIp) == len(expectedEndpoints) { + expectedPortsByIp := translatePodNameToIpOrFail(c, ns, expectedEndpoints) + validatePortsOrFail(portsByIp, expectedPortsByIp) break } else { - By(fmt.Sprintf("Unexpected number of endpoints: found %v, expected %v (ignoring for 1 second)", ips, expectedPods)) + By(fmt.Sprintf("Unexpected number of endpoints: found %v, expected %v (ignoring for 1 second)", portsByIp, expectedEndpoints)) } } else { By(fmt.Sprintf("Failed to get endpoints: %v (ignoring for 1 second)", err)) } time.Sleep(time.Second) } - By(fmt.Sprintf("successfully validated endpoints %v port %d on service %s/%s", expectedPods, expectedPort, ns, serviceName)) + By(fmt.Sprintf("successfully validated endpoints %v with on service %s/%s", expectedEndpoints, ns, serviceName)) } -func addEndpointPodOrFail(c *client.Client, ns, name string, labels map[string]string) { +func addEndpointPodOrFail(c *client.Client, ns, name string, labels map[string]string, containerPorts []api.ContainerPort) { By(fmt.Sprintf("Adding pod %v in namespace %v", name, ns)) pod := &api.Pod{ ObjectMeta: api.ObjectMeta{ @@ -496,7 +640,7 @@ func addEndpointPodOrFail(c *client.Client, ns, name string, labels map[string]s { Name: "test", Image: "gcr.io/google_containers/pause", - Ports: []api.ContainerPort{{ContainerPort: 80}}, + Ports: containerPorts, }, }, }, @@ -504,10 +648,3 @@ func addEndpointPodOrFail(c *client.Client, ns, name string, labels map[string]s _, err := c.Pods(ns).Create(pod) Expect(err).NotTo(HaveOccurred()) } - -// dateStamp returns the current time as a string "YYYY-MM-DDTHHMMSS" -// Handy for unique names across test runs -func dateStamp() string { - now := time.Now() - return fmt.Sprintf("%04d-%02d-%02dt%02d%02d%02d", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second()) -}