mirror of https://github.com/k3s-io/k3s
multiport endpoint e2e test
marekbiskup
Marek Biskup
parent
5010b2dde0
commit
672819be16
|
|
@ -37,14 +37,28 @@ import (
|
|||
var _ = Describe("Services", func() {
|
||||
var c *client.Client
|
||||
// Use these in tests. They're unique for each test to prevent name collisions.
|
||||
var namespace0, namespace1 string
|
||||
var namespaces [2]string
|
||||
|
||||
BeforeEach(func() {
|
||||
var err error
|
||||
c, err = loadClient()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
namespace0 = "e2e-ns-" + dateStamp() + "-0"
|
||||
namespace1 = "e2e-ns-" + dateStamp() + "-1"
|
||||
|
||||
By("Building a namespace api objects")
|
||||
for i := range namespaces {
|
||||
namespacePtr, err := createTestingNS(fmt.Sprintf("service-%d", i), c)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
namespaces[i] = namespacePtr.Name
|
||||
}
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
for _, ns := range namespaces {
|
||||
By(fmt.Sprintf("Destroying namespace %v", ns))
|
||||
if err := c.Namespaces().Delete(ns); err != nil {
|
||||
Failf("Couldn't delete namespace %s: %s", ns, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
It("should provide DNS for the cluster", func() {
|
||||
|
|
@ -53,13 +67,14 @@ var _ = Describe("Services", func() {
|
|||
return
|
||||
}
|
||||
|
||||
podClient := c.Pods(api.NamespaceDefault)
|
||||
|
||||
ns := namespaces[0]
|
||||
podClient := c.Pods(ns)
|
||||
//TODO: Wait for skyDNS
|
||||
|
||||
// All the names we need to be able to resolve.
|
||||
namesToResolve := []string{
|
||||
"kubernetes-ro",
|
||||
// "kubernetes-ro" is not directly visible because we're not in the the default namespace
|
||||
// TODO consider creating a service in our namespace and query it's DNS name too.
|
||||
"kubernetes-ro.default",
|
||||
"kubernetes-ro.default.cluster.local",
|
||||
"google.com",
|
||||
|
|
@ -120,15 +135,14 @@ var _ = Describe("Services", func() {
|
|||
|
||||
By("submitting the pod to kubernetes")
|
||||
defer func() {
|
||||
By("deleting the pod")
|
||||
By("deleting pod " + pod.Name)
|
||||
defer GinkgoRecover()
|
||||
podClient.Delete(pod.Name, nil)
|
||||
}()
|
||||
if _, err := podClient.Create(pod); err != nil {
|
||||
Failf("Failed to create %s pod: %v", pod.Name, err)
|
||||
Failf("Failed to create pod %s: %v", pod.Name, err)
|
||||
}
|
||||
|
||||
expectNoError(waitForPodRunning(c, pod.Name))
|
||||
expectNoError(waitForPodRunningInNamespace(c, pod.Name, ns))
|
||||
|
||||
By("retrieving the pod")
|
||||
pod, err := podClient.Get(pod.Name)
|
||||
|
|
@ -148,7 +162,7 @@ var _ = Describe("Services", func() {
|
|||
_, err := c.Get().
|
||||
Prefix("proxy").
|
||||
Resource("pods").
|
||||
Namespace(api.NamespaceDefault).
|
||||
Namespace(ns).
|
||||
Name(pod.Name).
|
||||
Suffix("results", testCase).
|
||||
Do().Raw()
|
||||
|
|
@ -194,7 +208,7 @@ var _ = Describe("Services", func() {
|
|||
|
||||
It("should serve a basic endpoint from pods", func(done Done) {
|
||||
serviceName := "endpoint-test2"
|
||||
ns := api.NamespaceDefault
|
||||
ns := namespaces[0]
|
||||
labels := map[string]string{
|
||||
"foo": "bar",
|
||||
"baz": "blah",
|
||||
|
|
@ -219,9 +233,8 @@ var _ = Describe("Services", func() {
|
|||
}
|
||||
_, err := c.Services(ns).Create(service)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
expectedPort := 80
|
||||
|
||||
validateEndpointsOrFail(c, ns, serviceName, expectedPort, []string{})
|
||||
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{})
|
||||
|
||||
var names []string
|
||||
defer func() {
|
||||
|
|
@ -232,28 +245,129 @@ var _ = Describe("Services", func() {
|
|||
}()
|
||||
|
||||
name1 := "test1"
|
||||
addEndpointPodOrFail(c, ns, name1, labels)
|
||||
addEndpointPodOrFail(c, ns, name1, labels, []api.ContainerPort{{ContainerPort: 80}})
|
||||
names = append(names, name1)
|
||||
|
||||
validateEndpointsOrFail(c, ns, serviceName, expectedPort, names)
|
||||
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{name1: {80}})
|
||||
|
||||
name2 := "test2"
|
||||
addEndpointPodOrFail(c, ns, name2, labels)
|
||||
addEndpointPodOrFail(c, ns, name2, labels, []api.ContainerPort{{ContainerPort: 80}})
|
||||
names = append(names, name2)
|
||||
|
||||
validateEndpointsOrFail(c, ns, serviceName, expectedPort, names)
|
||||
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{name1: {80}, name2: {80}})
|
||||
|
||||
err = c.Pods(ns).Delete(name1, nil)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
names = []string{name2}
|
||||
|
||||
validateEndpointsOrFail(c, ns, serviceName, expectedPort, names)
|
||||
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{name2: {80}})
|
||||
|
||||
err = c.Pods(ns).Delete(name2, nil)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
names = []string{}
|
||||
|
||||
validateEndpointsOrFail(c, ns, serviceName, expectedPort, names)
|
||||
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{})
|
||||
|
||||
// We deferred Gingko pieces that may Fail, we aren't done.
|
||||
defer func() {
|
||||
close(done)
|
||||
}()
|
||||
}, 240.0)
|
||||
|
||||
It("should serve multiport endpoints from pods", func(done Done) {
|
||||
// repacking functionality is intentionally not tested here - it's better to test it in an integration test.
|
||||
serviceName := "multi-endpoint-test"
|
||||
ns := namespaces[0]
|
||||
|
||||
defer func() {
|
||||
err := c.Services(ns).Delete(serviceName)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}()
|
||||
|
||||
labels := map[string]string{"foo": "bar"}
|
||||
|
||||
svc1port := "svc1"
|
||||
svc2port := "svc2"
|
||||
|
||||
service := &api.Service{
|
||||
ObjectMeta: api.ObjectMeta{
|
||||
Name: serviceName,
|
||||
},
|
||||
Spec: api.ServiceSpec{
|
||||
Selector: labels,
|
||||
Ports: []api.ServicePort{
|
||||
{
|
||||
Name: "portname1",
|
||||
Port: 80,
|
||||
TargetPort: util.NewIntOrStringFromString(svc1port),
|
||||
},
|
||||
{
|
||||
Name: "portname2",
|
||||
Port: 81,
|
||||
TargetPort: util.NewIntOrStringFromString(svc2port),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err := c.Services(ns).Create(service)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
port1 := 100
|
||||
port2 := 101
|
||||
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{})
|
||||
|
||||
var names []string
|
||||
defer func() {
|
||||
for _, name := range names {
|
||||
err := c.Pods(ns).Delete(name, nil)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
}()
|
||||
|
||||
containerPorts1 := []api.ContainerPort{
|
||||
{
|
||||
Name: svc1port,
|
||||
ContainerPort: port1,
|
||||
},
|
||||
}
|
||||
containerPorts2 := []api.ContainerPort{
|
||||
{
|
||||
Name: svc2port,
|
||||
ContainerPort: port2,
|
||||
},
|
||||
}
|
||||
|
||||
podname1 := "podname1"
|
||||
addEndpointPodOrFail(c, ns, podname1, labels, containerPorts1)
|
||||
names = append(names, podname1)
|
||||
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{podname1: {port1}})
|
||||
|
||||
podname2 := "podname2"
|
||||
addEndpointPodOrFail(c, ns, podname2, labels, containerPorts2)
|
||||
names = append(names, podname2)
|
||||
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{podname1: {port1}, podname2: {port2}})
|
||||
|
||||
podname3 := "podname3"
|
||||
addEndpointPodOrFail(c, ns, podname3, labels, append(containerPorts1, containerPorts2...))
|
||||
names = append(names, podname3)
|
||||
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{podname1: {port1}, podname2: {port2}, podname3: {port1, port2}})
|
||||
|
||||
err = c.Pods(ns).Delete(podname1, nil)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
names = []string{podname2, podname3}
|
||||
|
||||
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{podname2: {port2}, podname3: {port1, port2}})
|
||||
|
||||
err = c.Pods(ns).Delete(podname2, nil)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
names = []string{podname3}
|
||||
|
||||
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{podname3: {port1, port2}})
|
||||
|
||||
err = c.Pods(ns).Delete(podname3, nil)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
names = []string{}
|
||||
|
||||
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{})
|
||||
|
||||
// We deferred Gingko pieces that may Fail, we aren't done.
|
||||
defer func() {
|
||||
|
|
@ -268,7 +382,7 @@ var _ = Describe("Services", func() {
|
|||
}
|
||||
|
||||
serviceName := "external-lb-test"
|
||||
ns := namespace0
|
||||
ns := namespaces[0]
|
||||
labels := map[string]string{
|
||||
"key0": "value0",
|
||||
}
|
||||
|
|
@ -363,8 +477,7 @@ var _ = Describe("Services", func() {
|
|||
return
|
||||
}
|
||||
|
||||
serviceNames := []string{"s0"} // Could add more here, but then it takes longer.
|
||||
namespaces := []string{namespace0, namespace1} // As above.
|
||||
serviceNames := []string{"s0"} // Could add more here, but then it takes longer.
|
||||
labels := map[string]string{
|
||||
"key0": "value0",
|
||||
"key1": "value1",
|
||||
|
|
@ -437,54 +550,85 @@ func validateUniqueOrFail(s []string) {
|
|||
}
|
||||
}
|
||||
|
||||
func flattenSubsets(subsets []api.EndpointSubset, expectedPort int) util.StringSet {
|
||||
ips := util.StringSet{}
|
||||
func getPortsByIp(subsets []api.EndpointSubset) map[string][]int {
|
||||
m := make(map[string][]int)
|
||||
for _, ss := range subsets {
|
||||
for _, port := range ss.Ports {
|
||||
if port.Port == expectedPort {
|
||||
for _, addr := range ss.Addresses {
|
||||
ips.Insert(addr.IP)
|
||||
for _, addr := range ss.Addresses {
|
||||
Logf("Found IP %v and port %v", addr.IP, port.Port)
|
||||
if _, ok := m[addr.IP]; !ok {
|
||||
m[addr.IP] = make([]int, 0)
|
||||
}
|
||||
m[addr.IP] = append(m[addr.IP], port.Port)
|
||||
}
|
||||
}
|
||||
}
|
||||
return ips
|
||||
return m
|
||||
}
|
||||
|
||||
func validateIPsOrFail(c *client.Client, ns string, expectedPods []string, ips util.StringSet) {
|
||||
for _, name := range expectedPods {
|
||||
func translatePodNameToIpOrFail(c *client.Client, ns string, expectedEndpoints map[string][]int) map[string][]int {
|
||||
portsByIp := make(map[string][]int)
|
||||
|
||||
for name, portList := range expectedEndpoints {
|
||||
pod, err := c.Pods(ns).Get(name)
|
||||
if err != nil {
|
||||
Failf("failed to get pod %s, that's pretty weird. validation failed: %s", name, err)
|
||||
}
|
||||
if !ips.Has(pod.Status.PodIP) {
|
||||
Failf("ip validation failed, expected: %v, saw: %v", ips, pod.Status.PodIP)
|
||||
}
|
||||
portsByIp[pod.Status.PodIP] = portList
|
||||
By(fmt.Sprintf(""))
|
||||
}
|
||||
By(fmt.Sprintf("successfully validated IPs %v against expected endpoints %v on namespace %s", ips, expectedPods, ns))
|
||||
By(fmt.Sprintf("successfully translated pod names to ips: %v -> %v on namespace %s", expectedEndpoints, portsByIp, ns))
|
||||
return portsByIp
|
||||
}
|
||||
|
||||
func validateEndpointsOrFail(c *client.Client, ns, serviceName string, expectedPort int, expectedPods []string) {
|
||||
func validatePortsOrFail(endpoints map[string][]int, expectedEndpoints map[string][]int) {
|
||||
if len(endpoints) != len(expectedEndpoints) {
|
||||
// should not happen because we check this condition before
|
||||
Failf("invalid number of endpoints got %v, expected %v", endpoints, expectedEndpoints)
|
||||
}
|
||||
for ip := range expectedEndpoints {
|
||||
if _, ok := endpoints[ip]; !ok {
|
||||
Failf("endpoint %v not found", ip)
|
||||
}
|
||||
if len(endpoints[ip]) != len(expectedEndpoints[ip]) {
|
||||
Failf("invalid list of ports for ip %v. Got %v, expected %v", ip, endpoints[ip], expectedEndpoints[ip])
|
||||
}
|
||||
sort.Ints(endpoints[ip])
|
||||
sort.Ints(expectedEndpoints[ip])
|
||||
for index := range endpoints[ip] {
|
||||
if endpoints[ip][index] != expectedEndpoints[ip][index] {
|
||||
Failf("invalid list of ports for ip %v. Got %v, expected %v", ip, endpoints[ip], expectedEndpoints[ip])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func validateEndpointsOrFail(c *client.Client, ns, serviceName string, expectedEndpoints map[string][]int) {
|
||||
By(fmt.Sprintf("Validating endpoints %v with on service %s/%s", expectedEndpoints, ns, serviceName))
|
||||
for {
|
||||
endpoints, err := c.Endpoints(ns).Get(serviceName)
|
||||
if err == nil {
|
||||
ips := flattenSubsets(endpoints.Subsets, expectedPort)
|
||||
if len(ips) == len(expectedPods) {
|
||||
validateIPsOrFail(c, ns, expectedPods, ips)
|
||||
By(fmt.Sprintf("Found endpoints %v", endpoints))
|
||||
|
||||
portsByIp := getPortsByIp(endpoints.Subsets)
|
||||
|
||||
By(fmt.Sprintf("Found ports by ip %v", portsByIp))
|
||||
if len(portsByIp) == len(expectedEndpoints) {
|
||||
expectedPortsByIp := translatePodNameToIpOrFail(c, ns, expectedEndpoints)
|
||||
validatePortsOrFail(portsByIp, expectedPortsByIp)
|
||||
break
|
||||
} else {
|
||||
By(fmt.Sprintf("Unexpected number of endpoints: found %v, expected %v (ignoring for 1 second)", ips, expectedPods))
|
||||
By(fmt.Sprintf("Unexpected number of endpoints: found %v, expected %v (ignoring for 1 second)", portsByIp, expectedEndpoints))
|
||||
}
|
||||
} else {
|
||||
By(fmt.Sprintf("Failed to get endpoints: %v (ignoring for 1 second)", err))
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
By(fmt.Sprintf("successfully validated endpoints %v port %d on service %s/%s", expectedPods, expectedPort, ns, serviceName))
|
||||
By(fmt.Sprintf("successfully validated endpoints %v with on service %s/%s", expectedEndpoints, ns, serviceName))
|
||||
}
|
||||
|
||||
func addEndpointPodOrFail(c *client.Client, ns, name string, labels map[string]string) {
|
||||
func addEndpointPodOrFail(c *client.Client, ns, name string, labels map[string]string, containerPorts []api.ContainerPort) {
|
||||
By(fmt.Sprintf("Adding pod %v in namespace %v", name, ns))
|
||||
pod := &api.Pod{
|
||||
ObjectMeta: api.ObjectMeta{
|
||||
|
|
@ -496,7 +640,7 @@ func addEndpointPodOrFail(c *client.Client, ns, name string, labels map[string]s
|
|||
{
|
||||
Name: "test",
|
||||
Image: "gcr.io/google_containers/pause",
|
||||
Ports: []api.ContainerPort{{ContainerPort: 80}},
|
||||
Ports: containerPorts,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -504,10 +648,3 @@ func addEndpointPodOrFail(c *client.Client, ns, name string, labels map[string]s
|
|||
_, err := c.Pods(ns).Create(pod)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
|
||||
// dateStamp returns the current time as a string "YYYY-MM-DDTHHMMSS"
|
||||
// Handy for unique names across test runs
|
||||
func dateStamp() string {
|
||||
now := time.Now()
|
||||
return fmt.Sprintf("%04d-%02d-%02dt%02d%02d%02d", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second())
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue