github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Validate endpoints in e2e tests by pod UID and port 

Before this patch endpoints were validated by container IP and port.
Depending on the endpoint controller logic neither of the two must match for a
valid endpoint (e.g. in a Mesos setup).

This patch checks that the endpoint targetRef points to the right pod by UID,
instead of comparing IPs.

A later patch will make sure the
compared port is the actual container port, not the host port.

/xref mesosphere/kubernetes-mesos#365
pull/6/head
Dr. Stefan Schimanski 2015-06-26 10:08:48 +02:00
parent fbb5ce6636
commit 8848e26154
2 changed files with 53 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
test/e2e/kubectl.go
View File

@ -348,12 +348,15 @@ var _ = Describe("Kubectl client", func() {
endpoints, err := c.Endpoints(ns).Get(name)
Expect(err).NotTo(HaveOccurred())
ipToPort := getPortsByIp(endpoints.Subsets)
if len(ipToPort) != 1 {
Logf("No IP found, retrying")
uidToPort := getPortsByPodUID(endpoints.Subsets)
if len(uidToPort) == 0 {
Logf("No endpoint found, retrying")
continue
}
for _, port := range ipToPort {
if len(uidToPort) > 1 {
Fail("To many endpoints found")
}
for _, port := range uidToPort {
if port[0] != redisPort {
Failf("Wrong endpoint port: %d", port[0])
}

87
test/e2e/service.go
View File

@ -32,6 +32,7 @@ import (
"k8s.io/kubernetes/pkg/client"
"k8s.io/kubernetes/pkg/fields"
"k8s.io/kubernetes/pkg/labels"
"k8s.io/kubernetes/pkg/types"
"k8s.io/kubernetes/pkg/util"
"k8s.io/kubernetes/pkg/util/wait"
)
@ -101,7 +102,7 @@ var _ = Describe("Services", func() {
_, err := c.Services(ns).Create(service)
Expect(err).NotTo(HaveOccurred())
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{})
validateEndpointsOrFail(c, ns, serviceName, PortsByPodName{})
var names []string
defer func() {
@ -115,25 +116,25 @@ var _ = Describe("Services", func() {
addEndpointPodOrFail(c, ns, name1, labels, []api.ContainerPort{{ContainerPort: 80}})
names = append(names, name1)
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{name1: {80}})
validateEndpointsOrFail(c, ns, serviceName, PortsByPodName{name1: {80}})
name2 := "test2"
addEndpointPodOrFail(c, ns, name2, labels, []api.ContainerPort{{ContainerPort: 80}})
names = append(names, name2)
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{name1: {80}, name2: {80}})
validateEndpointsOrFail(c, ns, serviceName, PortsByPodName{name1: {80}, name2: {80}})
err = c.Pods(ns).Delete(name1, nil)
Expect(err).NotTo(HaveOccurred())
names = []string{name2}
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{name2: {80}})
validateEndpointsOrFail(c, ns, serviceName, PortsByPodName{name2: {80}})
err = c.Pods(ns).Delete(name2, nil)
Expect(err).NotTo(HaveOccurred())
names = []string{}
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{})
validateEndpointsOrFail(c, ns, serviceName, PortsByPodName{})
})
It("should serve multiport endpoints from pods", func() {
@ -175,7 +176,7 @@ var _ = Describe("Services", func() {
Expect(err).NotTo(HaveOccurred())
port1 := 100
port2 := 101
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{})
validateEndpointsOrFail(c, ns, serviceName, PortsByPodName{})
var names []string
defer func() {
@ -201,35 +202,35 @@ var _ = Describe("Services", func() {
podname1 := "podname1"
addEndpointPodOrFail(c, ns, podname1, labels, containerPorts1)
names = append(names, podname1)
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{podname1: {port1}})
validateEndpointsOrFail(c, ns, serviceName, PortsByPodName{podname1: {port1}})
podname2 := "podname2"
addEndpointPodOrFail(c, ns, podname2, labels, containerPorts2)
names = append(names, podname2)
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{podname1: {port1}, podname2: {port2}})
validateEndpointsOrFail(c, ns, serviceName, PortsByPodName{podname1: {port1}, podname2: {port2}})
podname3 := "podname3"
addEndpointPodOrFail(c, ns, podname3, labels, append(containerPorts1, containerPorts2...))
names = append(names, podname3)
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{podname1: {port1}, podname2: {port2}, podname3: {port1, port2}})
validateEndpointsOrFail(c, ns, serviceName, PortsByPodName{podname1: {port1}, podname2: {port2}, podname3: {port1, port2}})
err = c.Pods(ns).Delete(podname1, nil)
Expect(err).NotTo(HaveOccurred())
names = []string{podname2, podname3}
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{podname2: {port2}, podname3: {port1, port2}})
validateEndpointsOrFail(c, ns, serviceName, PortsByPodName{podname2: {port2}, podname3: {port1, port2}})
err = c.Pods(ns).Delete(podname2, nil)
Expect(err).NotTo(HaveOccurred())
names = []string{podname3}
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{podname3: {port1, port2}})
validateEndpointsOrFail(c, ns, serviceName, PortsByPodName{podname3: {port1, port2}})
err = c.Pods(ns).Delete(podname3, nil)
Expect(err).NotTo(HaveOccurred())
names = []string{}
validateEndpointsOrFail(c, ns, serviceName, map[string][]int{})
validateEndpointsOrFail(c, ns, serviceName, PortsByPodName{})
})
It("should be able to up and down services", func() {
@ -1022,60 +1023,63 @@ func validateUniqueOrFail(s []string) {
}
}
func getPortsByIp(subsets []api.EndpointSubset) map[string][]int {
m := make(map[string][]int)
func getPortsByPodUID(subsets []api.EndpointSubset) PortsByPodUID {
m := PortsByPodUID{}
for _, ss := range subsets {
for _, port := range ss.Ports {
for _, addr := range ss.Addresses {
Logf("Found IP %v and port %v", addr.IP, port.Port)
if _, ok := m[addr.IP]; !ok {
m[addr.IP] = make([]int, 0)
Logf("Found pod %v and port %v", addr.TargetRef.UID, port.Port)
if _, ok := m[addr.TargetRef.UID]; !ok {
m[addr.TargetRef.UID] = make([]int, 0)
}
m[addr.IP] = append(m[addr.IP], port.Port)
m[addr.TargetRef.UID] = append(m[addr.TargetRef.UID], port.Port)
}
}
}
return m
}
func translatePodNameToIpOrFail(c *client.Client, ns string, expectedEndpoints map[string][]int) map[string][]int {
portsByIp := make(map[string][]int)
type PortsByPodName map[string][]int
type PortsByPodUID map[types.UID][]int
func translatePodNameToUIDOrFail(c *client.Client, ns string, expectedEndpoints PortsByPodName) PortsByPodUID {
portsByUID := make(PortsByPodUID)
for name, portList := range expectedEndpoints {
pod, err := c.Pods(ns).Get(name)
if err != nil {
Failf("failed to get pod %s, that's pretty weird. validation failed: %s", name, err)
}
portsByIp[pod.Status.PodIP] = portList
portsByUID[pod.ObjectMeta.UID] = portList
By(fmt.Sprintf(""))
}
By(fmt.Sprintf("successfully translated pod names to ips: %v -> %v on namespace %s", expectedEndpoints, portsByIp, ns))
return portsByIp
By(fmt.Sprintf("successfully translated pod names to UIDs: %v -> %v on namespace %s", expectedEndpoints, portsByUID, ns))
return portsByUID
}
func validatePortsOrFail(endpoints map[string][]int, expectedEndpoints map[string][]int) {
func validatePortsOrFail(endpoints PortsByPodUID, expectedEndpoints PortsByPodUID) {
if len(endpoints) != len(expectedEndpoints) {
// should not happen because we check this condition before
Failf("invalid number of endpoints got %v, expected %v", endpoints, expectedEndpoints)
}
for ip := range expectedEndpoints {
if _, ok := endpoints[ip]; !ok {
Failf("endpoint %v not found", ip)
for podUID := range expectedEndpoints {
if _, ok := endpoints[podUID]; !ok {
Failf("endpoint %v not found", podUID)
}
if len(endpoints[ip]) != len(expectedEndpoints[ip]) {
Failf("invalid list of ports for ip %v. Got %v, expected %v", ip, endpoints[ip], expectedEndpoints[ip])
if len(endpoints[podUID]) != len(expectedEndpoints[podUID]) {
Failf("invalid list of ports for uid %v. Got %v, expected %v", podUID, endpoints[podUID], expectedEndpoints[podUID])
}
sort.Ints(endpoints[ip])
sort.Ints(expectedEndpoints[ip])
for index := range endpoints[ip] {
if endpoints[ip][index] != expectedEndpoints[ip][index] {
Failf("invalid list of ports for ip %v. Got %v, expected %v", ip, endpoints[ip], expectedEndpoints[ip])
sort.Ints(endpoints[podUID])
sort.Ints(expectedEndpoints[podUID])
for index := range endpoints[podUID] {
if endpoints[podUID][index] != expectedEndpoints[podUID][index] {
Failf("invalid list of ports for uid %v. Got %v, expected %v", podUID, endpoints[podUID], expectedEndpoints[podUID])
}
}
}
}
func validateEndpointsOrFail(c *client.Client, namespace, serviceName string, expectedEndpoints map[string][]int) {
func validateEndpointsOrFail(c *client.Client, namespace, serviceName string, expectedEndpoints PortsByPodName) {
By(fmt.Sprintf("Waiting up to %v for service %s in namespace %s to expose endpoints %v", serviceStartTimeout, serviceName, namespace, expectedEndpoints))
for start := time.Now(); time.Since(start) < serviceStartTimeout; time.Sleep(5 * time.Second) {
endpoints, err := c.Endpoints(namespace).Get(serviceName)
@ -1085,16 +1089,17 @@ func validateEndpointsOrFail(c *client.Client, namespace, serviceName string, ex
}
Logf("Found endpoints %v", endpoints)
portsByIp := getPortsByIp(endpoints.Subsets)
Logf("Found ports by ip %v", portsByIp)
portsByPodUID := getPortsByPodUID(endpoints.Subsets)
Logf("Found port by pod UID %v", portsByPodUID)
if len(portsByIp) == len(expectedEndpoints) {
expectedPortsByIp := translatePodNameToIpOrFail(c, namespace, expectedEndpoints)
validatePortsOrFail(portsByIp, expectedPortsByIp)
expectedPortsByPodUID := translatePodNameToUIDOrFail(c, namespace, expectedEndpoints)
if len(portsByPodUID) == len(expectedEndpoints) {
validatePortsOrFail(portsByPodUID, expectedPortsByPodUID)
By(fmt.Sprintf("Successfully validated that service %s in namespace %s exposes endpoints %v (%v elapsed)", serviceName, namespace, expectedEndpoints, time.Since(start)))
return
}
Logf("Unexpected number of endpoints: found %v, expected %v (%v elapsed, ignoring for 5s)", portsByIp, expectedEndpoints, time.Since(start))
Logf("Unexpected number of endpoints: found %v, expected %v (%v elapsed, ignoring for 5s)", portsByPodUID, expectedEndpoints, time.Since(start))
}
Failf("Timed out waiting for service %s in namespace %s to expose endpoints %v (%v elapsed)", serviceName, namespace, expectedEndpoints, serviceStartTimeout)
}