github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #11077 from krousey/service_flake 

Adding retry logic around service updates
pull/6/head
Tim Hockin 2015-07-13 13:06:02 -07:00
parent f5cc7cd525 aca4559bbd
commit 3452748de9
1 changed files with 43 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
test/e2e/service.go
View File

@ -26,6 +26,7 @@ import (
"time"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api/errors"
"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
@ -370,8 +371,9 @@ var _ = Describe("Services", func() {
t.CreateWebserverRC(1)
By("changing service " + serviceName + " to type=NodePort")
service.Spec.Type = api.ServiceTypeNodePort
service, err = c.Services(ns).Update(service)
service, err = updateService(c, ns, serviceName, func(s *api.Service) {
s.Spec.Type = api.ServiceTypeNodePort
})
Expect(err).NotTo(HaveOccurred())
if service.Spec.Type != api.ServiceTypeNodePort {
@ -398,7 +400,9 @@ var _ = Describe("Services", func() {
By("changing service " + serviceName + " to type=LoadBalancer")
service.Spec.Type = api.ServiceTypeLoadBalancer
service, err = c.Services(ns).Update(service)
service, err = updateService(c, ns, serviceName, func(s *api.Service) {
s.Spec.Type = api.ServiceTypeLoadBalancer
})
Expect(err).NotTo(HaveOccurred())
// Wait for the load balancer to be created asynchronously
@ -434,8 +438,9 @@ var _ = Describe("Services", func() {
//Check for (unlikely) assignment at bottom of range
nodePort2 = nodePort1 + 1
}
service.Spec.Ports[0].NodePort = nodePort2
service, err = c.Services(ns).Update(service)
service, err = updateService(c, ns, serviceName, func(s *api.Service) {
s.Spec.Ports[0].NodePort = nodePort2
})
Expect(err).NotTo(HaveOccurred())
if service.Spec.Type != api.ServiceTypeLoadBalancer {
@ -463,9 +468,10 @@ var _ = Describe("Services", func() {
testNotReachable(ip, nodePort1)
By("changing service " + serviceName + " back to type=ClusterIP")
service.Spec.Type = api.ServiceTypeClusterIP
service.Spec.Ports[0].NodePort = 0
service, err = c.Services(ns).Update(service)
service, err = updateService(c, ns, serviceName, func(s *api.Service) {
s.Spec.Type = api.ServiceTypeClusterIP
s.Spec.Ports[0].NodePort = 0
})
Expect(err).NotTo(HaveOccurred())
if service.Spec.Type != api.ServiceTypeClusterIP {
@ -552,8 +558,9 @@ var _ = Describe("Services", func() {
testLoadBalancerReachable(ingress, 80)
By("changing service " + serviceName + " to type=NodePort")
service.Spec.Type = api.ServiceTypeNodePort
service, err = c.Services(ns).Update(service)
service, err = updateService(c, ns, serviceName, func(s *api.Service) {
s.Spec.Type = api.ServiceTypeNodePort
})
Expect(err).NotTo(HaveOccurred())
if service.Spec.Type != api.ServiceTypeNodePort {
@ -678,8 +685,9 @@ var _ = Describe("Services", func() {
}
}
By(fmt.Sprintf("changing service "+serviceName+" to out-of-range NodePort %d", outOfRangeNodePort))
service.Spec.Ports[0].NodePort = outOfRangeNodePort
result, err := t.Client.Services(t.Namespace).Update(service)
result, err := updateService(c, ns, serviceName, func(s *api.Service) {
s.Spec.Ports[0].NodePort = outOfRangeNodePort
})
if err == nil {
Failf("failed to prevent update of service with out-of-range NodePort: %v", result)
}
@ -801,6 +809,29 @@ var _ = Describe("Services", func() {
})
})
// updateService fetches a service, calls the update function on it,
// and then attempts to send the updated service. It retries up to 2
// times in the face of timeouts and conflicts.
func updateService(c *client.Client, namespace, serviceName string, update func(*api.Service)) (*api.Service, error) {
var service *api.Service
var err error
for i := 0; i < 3; i++ {
service, err = c.Services(namespace).Get(serviceName)
if err != nil {
return service, err
}
update(service)
service, err = c.Services(namespace).Update(service)
if !errors.IsConflict(err) && !errors.IsServerTimeout(err) {
return service, err
}
}
return service, err
}
func waitForLoadBalancerIngress(c *client.Client, serviceName, namespace string) (*api.Service, error) {
// TODO: once support ticket 21807001 is resolved, reduce this timeout back to something reasonable
const timeout = 20 * time.Minute