From 127e1b6115645dc7c58f408a8ae72503f1771661 Mon Sep 17 00:00:00 2001 From: mfanjie Date: Wed, 7 Sep 2016 10:48:04 +0800 Subject: [PATCH] always clean gce resources in service e2e --- pkg/cloudprovider/providers/gce/gce.go | 9 ++++ test/e2e/framework/util.go | 13 ++++++ test/e2e/service.go | 64 ++++++++++++++++++++++++++ 3 files changed, 86 insertions(+) diff --git a/pkg/cloudprovider/providers/gce/gce.go b/pkg/cloudprovider/providers/gce/gce.go index 372b268f70..8ebe9dde41 100644 --- a/pkg/cloudprovider/providers/gce/gce.go +++ b/pkg/cloudprovider/providers/gce/gce.go @@ -1446,6 +1446,15 @@ func (gce *GCECloud) deleteForwardingRule(name, region string) error { return nil } +// DeleteTargetPool deletes the given target pool. +func (gce *GCECloud) DeleteTargetPool(name string, hc *compute.HttpHealthCheck) error { + region, err := GetGCERegion(gce.localZone) + if err != nil { + return err + } + return gce.deleteTargetPool(name, region, hc) +} + func (gce *GCECloud) deleteTargetPool(name, region string, hc *compute.HttpHealthCheck) error { op, err := gce.service.TargetPools.Delete(gce.projectID, region, name).Do() if err != nil && isHTTPErrorCode(err, http.StatusNotFound) { diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index e12a5a606f..442d8009f0 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -4582,3 +4582,16 @@ func (p *E2ETestNodePreparer) CleanupNodes() error { } return encounteredError } + +func CleanupGCEResources(loadBalancerName string) (err error) { + gceCloud, ok := TestContext.CloudConfig.Provider.(*gcecloud.GCECloud) + if !ok { + return fmt.Errorf("failed to convert CloudConfig.Provider to GCECloud: %#v", TestContext.CloudConfig.Provider) + } + gceCloud.DeleteFirewall(loadBalancerName) + gceCloud.DeleteGlobalForwardingRule(loadBalancerName) + gceCloud.DeleteGlobalStaticIP(loadBalancerName) + hc, _ := gceCloud.GetHttpHealthCheck(loadBalancerName) + gceCloud.DeleteTargetPool(loadBalancerName, hc) + return nil +} diff --git a/test/e2e/service.go b/test/e2e/service.go index 2af254e2a7..ea615da131 100644 --- a/test/e2e/service.go +++ b/test/e2e/service.go @@ -80,11 +80,24 @@ var _ = framework.KubeDescribe("Services", func() { f := framework.NewDefaultFramework("services") var cs clientset.Interface + serviceLBNames := []string{} BeforeEach(func() { cs = f.ClientSet }) + AfterEach(func() { + if CurrentGinkgoTestDescription().Failed { + describeSvc(f.Namespace.Name) + } + for _, lb := range serviceLBNames { + framework.Logf("cleaning gce resource for %s", lb) + cleanupServiceGCEResources(lb) + } + //reset serviceLBNames + serviceLBNames = []string{} + }) + // TODO: We get coverage of TCP/UDP and multi-port services through the DNS test. We should have a simpler test for multi-port TCP here. It("should provide secure master service [Conformance]", func() { @@ -582,6 +595,10 @@ var _ = framework.KubeDescribe("Services", func() { s.Spec.Type = api.ServiceTypeLoadBalancer }) } + serviceLBNames = append(serviceLBNames, getLoadBalancerName(tcpService)) + if loadBalancerSupportsUDP { + serviceLBNames = append(serviceLBNames, getLoadBalancerName(udpService)) + } By("waiting for the TCP service to have a load balancer") // Wait for the load balancer to be created asynchronously @@ -1083,6 +1100,7 @@ var _ = framework.KubeDescribe("ESIPP [Slow][Feature:ExternalTrafficLocalOnly]", loadBalancerCreateTimeout := loadBalancerCreateTimeoutDefault var cs clientset.Interface + serviceLBNames := []string{} BeforeEach(func() { // requires cloud load-balancer support - this feature currently supported only on GCE/GKE @@ -1094,12 +1112,25 @@ var _ = framework.KubeDescribe("ESIPP [Slow][Feature:ExternalTrafficLocalOnly]", } }) + AfterEach(func() { + if CurrentGinkgoTestDescription().Failed { + describeSvc(f.Namespace.Name) + } + for _, lb := range serviceLBNames { + framework.Logf("cleaning gce resource for %s", lb) + cleanupServiceGCEResources(lb) + } + //reset serviceLBNames + serviceLBNames = []string{} + }) + It("should work for type=LoadBalancer [Slow][Feature:ExternalTrafficLocalOnly]", func() { namespace := f.Namespace.Name serviceName := "external-local" jig := NewServiceTestJig(cs, serviceName) svc := jig.createOnlyLocalLoadBalancerService(namespace, serviceName, loadBalancerCreateTimeout, true) + serviceLBNames = append(serviceLBNames, getLoadBalancerName(svc)) healthCheckNodePort := int(service.GetServiceHealthCheckNodePort(svc)) if healthCheckNodePort == 0 { framework.Failf("Service HealthCheck NodePort was not allocated") @@ -1165,6 +1196,7 @@ var _ = framework.KubeDescribe("ESIPP [Slow][Feature:ExternalTrafficLocalOnly]", nodes := jig.getNodes(maxNodesForEndpointsTests) svc := jig.createOnlyLocalLoadBalancerService(namespace, serviceName, loadBalancerCreateTimeout, false) + serviceLBNames = append(serviceLBNames, getLoadBalancerName(svc)) defer func() { jig.ChangeServiceType(svc.Namespace, svc.Name, api.ServiceTypeClusterIP, loadBalancerCreateTimeout) Expect(cs.Core().Services(svc.Namespace).Delete(svc.Name, nil)).NotTo(HaveOccurred()) @@ -1224,6 +1256,7 @@ var _ = framework.KubeDescribe("ESIPP [Slow][Feature:ExternalTrafficLocalOnly]", nodes := jig.getNodes(maxNodesForEndpointsTests) svc := jig.createOnlyLocalLoadBalancerService(namespace, serviceName, loadBalancerCreateTimeout, true) + serviceLBNames = append(serviceLBNames, getLoadBalancerName(svc)) defer func() { jig.ChangeServiceType(svc.Namespace, svc.Name, api.ServiceTypeClusterIP, loadBalancerCreateTimeout) Expect(cs.Core().Services(svc.Namespace).Delete(svc.Name, nil)).NotTo(HaveOccurred()) @@ -1272,6 +1305,7 @@ var _ = framework.KubeDescribe("ESIPP [Slow][Feature:ExternalTrafficLocalOnly]", } svc := jig.createOnlyLocalLoadBalancerService(namespace, serviceName, loadBalancerCreateTimeout, true) + serviceLBNames = append(serviceLBNames, getLoadBalancerName(svc)) defer func() { jig.ChangeServiceType(svc.Namespace, svc.Name, api.ServiceTypeClusterIP, loadBalancerCreateTimeout) Expect(cs.Core().Services(svc.Namespace).Delete(svc.Name, nil)).NotTo(HaveOccurred()) @@ -2697,3 +2731,33 @@ func execSourceipTest(f *framework.Framework, c clientset.Interface, ns, nodeNam } return execPod.Status.PodIP, outputs[1] } + +func getLoadBalancerName(service *api.Service) string { + //GCE requires that the name of a load balancer starts with a lower case letter. + ret := "a" + string(service.UID) + ret = strings.Replace(ret, "-", "", -1) + //AWS requires that the name of a load balancer is shorter than 32 bytes. + if len(ret) > 32 { + ret = ret[:32] + } + return ret +} + +func cleanupServiceGCEResources(loadBalancerName string) { + if pollErr := wait.Poll(5*time.Second, lbCleanupTimeout, func() (bool, error) { + if err := framework.CleanupGCEResources(loadBalancerName); err != nil { + framework.Logf("Still waiting for glbc to cleanup: %v", err) + return false, nil + } + return true, nil + }); pollErr != nil { + framework.Failf("Failed to cleanup service GCE resources.") + } +} + +func describeSvc(ns string) { + framework.Logf("\nOutput of kubectl describe svc:\n") + desc, _ := framework.RunKubectl( + "describe", "svc", fmt.Sprintf("--namespace=%v", ns)) + framework.Logf(desc) +}