mirror of https://github.com/k3s-io/k3s
Merge pull request #37418 from bprashanth/ing_cleanup
Automatic merge from submit-queue Cleanup old cloud resources after 48 hours With this pr the ingress e2e purges old leaked resources (>48h), so even if tests fail due to leaks, the entire queue won't close till someone bumps up quota through a manual request.pull/6/head
Kubernetes Submit Queue
GitHub
commit
251333971d
|
|
@ -66,6 +66,19 @@ const (
|
|||
|
||||
// Ingress class annotation defined in ingress repository.
|
||||
ingressClass = "kubernetes.io/ingress.class"
|
||||
|
||||
// all cloud resources created by the ingress controller start with this
|
||||
// prefix.
|
||||
k8sPrefix = "k8s-"
|
||||
|
||||
// clusterDelimiter is the delimiter used by the ingress controller
|
||||
// to split uid from other naming/metadata.
|
||||
clusterDelimiter = "--"
|
||||
|
||||
// Cloud resources created by the ingress controller older than this
|
||||
// are automatically purged to prevent running out of quota.
|
||||
// TODO(37335): write soak tests and bump this up to a week.
|
||||
maxAge = -48 * time.Hour
|
||||
)
|
||||
|
||||
type testJig struct {
|
||||
|
|
@ -311,32 +324,40 @@ func describeIng(ns string) {
|
|||
}
|
||||
|
||||
func cleanupGCE(gceController *GCEIngressController) {
|
||||
if pollErr := wait.Poll(5*time.Second, lbCleanupTimeout, func() (bool, error) {
|
||||
pollErr := wait.Poll(5*time.Second, lbCleanupTimeout, func() (bool, error) {
|
||||
if err := gceController.Cleanup(false); err != nil {
|
||||
framework.Logf("Still waiting for glbc to cleanup:\n%v", err)
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
}); pollErr != nil {
|
||||
warning := fmt.Sprintf("No reasources leaked.")
|
||||
if cleanupErr := gceController.Cleanup(true); cleanupErr != nil {
|
||||
warning = fmt.Sprintf("WARNING: Leaked resources: %v\n", cleanupErr)
|
||||
}
|
||||
framework.Failf("L7 controller failed to delete all cloud resources on time. %v", warning)
|
||||
})
|
||||
// Always try to cleanup even if pollErr == nil, because the cleanup
|
||||
// routine also purges old leaked resources based on creation timestamp.
|
||||
if cleanupErr := gceController.Cleanup(true); cleanupErr != nil {
|
||||
By(fmt.Sprintf("WARNING: possibly leaked resources: %v\n", cleanupErr))
|
||||
} else {
|
||||
By("No resources leaked.")
|
||||
}
|
||||
if pollErr != nil {
|
||||
framework.Failf("L7 controller failed to delete all cloud resources on time. %v", pollErr)
|
||||
}
|
||||
}
|
||||
|
||||
func (cont *GCEIngressController) deleteForwardingRule(del bool) string {
|
||||
msg := ""
|
||||
fwList := []compute.ForwardingRule{}
|
||||
for _, regex := range []string{fmt.Sprintf("k8s-fw-.*--%v", cont.UID), fmt.Sprintf("k8s-fws-.*--%v", cont.UID)} {
|
||||
for _, regex := range []string{fmt.Sprintf("%vfw-.*%v.*", k8sPrefix, clusterDelimiter), fmt.Sprintf("%vfws-.*%v.*", k8sPrefix, clusterDelimiter)} {
|
||||
gcloudList("forwarding-rules", regex, cont.cloud.ProjectID, &fwList)
|
||||
if len(fwList) != 0 {
|
||||
for _, f := range fwList {
|
||||
msg += fmt.Sprintf("%v (forwarding rule)\n", f.Name)
|
||||
if del {
|
||||
gcloudDelete("forwarding-rules", f.Name, cont.cloud.ProjectID, "--global")
|
||||
}
|
||||
if len(fwList) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, f := range fwList {
|
||||
if !cont.canDelete(f.Name, f.CreationTimestamp, del) {
|
||||
continue
|
||||
}
|
||||
msg += fmt.Sprintf("%v (forwarding rule)\n", f.Name)
|
||||
if del {
|
||||
gcloudDelete("forwarding-rules", f.Name, cont.cloud.ProjectID, "--global")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -346,12 +367,16 @@ func (cont *GCEIngressController) deleteForwardingRule(del bool) string {
|
|||
func (cont *GCEIngressController) deleteAddresses(del bool) string {
|
||||
msg := ""
|
||||
ipList := []compute.Address{}
|
||||
gcloudList("addresses", fmt.Sprintf("k8s-fw-.*--%v", cont.UID), cont.cloud.ProjectID, &ipList)
|
||||
regex := fmt.Sprintf("%vfw-.*%v.*", k8sPrefix, clusterDelimiter)
|
||||
gcloudList("addresses", regex, cont.cloud.ProjectID, &ipList)
|
||||
if len(ipList) != 0 {
|
||||
for _, ip := range ipList {
|
||||
if !cont.canDelete(ip.Name, ip.CreationTimestamp, del) {
|
||||
continue
|
||||
}
|
||||
msg += fmt.Sprintf("%v (static-ip)\n", ip.Name)
|
||||
if del {
|
||||
gcloudDelete("addresses", ip.Name, cont.cloud.ProjectID)
|
||||
gcloudDelete("addresses", ip.Name, cont.cloud.ProjectID, "--global")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -375,9 +400,13 @@ func (cont *GCEIngressController) deleteAddresses(del bool) string {
|
|||
func (cont *GCEIngressController) deleteTargetProxy(del bool) string {
|
||||
msg := ""
|
||||
tpList := []compute.TargetHttpProxy{}
|
||||
gcloudList("target-http-proxies", fmt.Sprintf("k8s-tp-.*--%v", cont.UID), cont.cloud.ProjectID, &tpList)
|
||||
regex := fmt.Sprintf("%vtp-.*%v.*", k8sPrefix, clusterDelimiter)
|
||||
gcloudList("target-http-proxies", regex, cont.cloud.ProjectID, &tpList)
|
||||
if len(tpList) != 0 {
|
||||
for _, t := range tpList {
|
||||
if !cont.canDelete(t.Name, t.CreationTimestamp, del) {
|
||||
continue
|
||||
}
|
||||
msg += fmt.Sprintf("%v (target-http-proxy)\n", t.Name)
|
||||
if del {
|
||||
gcloudDelete("target-http-proxies", t.Name, cont.cloud.ProjectID)
|
||||
|
|
@ -385,9 +414,13 @@ func (cont *GCEIngressController) deleteTargetProxy(del bool) string {
|
|||
}
|
||||
}
|
||||
tpsList := []compute.TargetHttpsProxy{}
|
||||
gcloudList("target-https-proxies", fmt.Sprintf("k8s-tps-.*--%v", cont.UID), cont.cloud.ProjectID, &tpsList)
|
||||
regex = fmt.Sprintf("%vtps-.*%v.*", k8sPrefix, clusterDelimiter)
|
||||
gcloudList("target-https-proxies", regex, cont.cloud.ProjectID, &tpsList)
|
||||
if len(tpsList) != 0 {
|
||||
for _, t := range tpsList {
|
||||
if !cont.canDelete(t.Name, t.CreationTimestamp, del) {
|
||||
continue
|
||||
}
|
||||
msg += fmt.Sprintf("%v (target-https-proxy)\n", t.Name)
|
||||
if del {
|
||||
gcloudDelete("target-https-proxies", t.Name, cont.cloud.ProjectID)
|
||||
|
|
@ -410,7 +443,7 @@ func (cont *GCEIngressController) deleteUrlMap(del bool) (msg string) {
|
|||
return msg
|
||||
}
|
||||
for _, um := range umList.Items {
|
||||
if !strings.HasSuffix(um.Name, cont.UID) {
|
||||
if !cont.canDelete(um.Name, um.CreationTimestamp, del) {
|
||||
continue
|
||||
}
|
||||
msg += fmt.Sprintf("%v (url-map)\n", um.Name)
|
||||
|
|
@ -434,10 +467,11 @@ func (cont *GCEIngressController) deleteBackendService(del bool) (msg string) {
|
|||
return fmt.Sprintf("Failed to list backend services: %v", err)
|
||||
}
|
||||
if len(beList.Items) == 0 {
|
||||
framework.Logf("No backend services found")
|
||||
return msg
|
||||
}
|
||||
for _, be := range beList.Items {
|
||||
if !strings.HasSuffix(be.Name, cont.UID) {
|
||||
if !cont.canDelete(be.Name, be.CreationTimestamp, del) {
|
||||
continue
|
||||
}
|
||||
msg += fmt.Sprintf("%v (backend-service)\n", be.Name)
|
||||
|
|
@ -464,12 +498,12 @@ func (cont *GCEIngressController) deleteHttpHealthCheck(del bool) (msg string) {
|
|||
return msg
|
||||
}
|
||||
for _, hc := range hcList.Items {
|
||||
if !strings.HasSuffix(hc.Name, cont.UID) {
|
||||
if !cont.canDelete(hc.Name, hc.CreationTimestamp, del) {
|
||||
continue
|
||||
}
|
||||
msg += fmt.Sprintf("%v (http-health-check)\n", hc.Name)
|
||||
if del {
|
||||
if err := gceCloud.DeleteBackendService(hc.Name); err != nil &&
|
||||
if err := gceCloud.DeleteHttpHealthCheck(hc.Name); err != nil &&
|
||||
!cont.isHTTPErrorCode(err, http.StatusNotFound) {
|
||||
msg += fmt.Sprintf("Failed to delete HTTP health check %v\n", hc.Name)
|
||||
}
|
||||
|
|
@ -489,7 +523,7 @@ func (cont *GCEIngressController) deleteSSLCertificate(del bool) (msg string) {
|
|||
}
|
||||
if len(sslList.Items) != 0 {
|
||||
for _, s := range sslList.Items {
|
||||
if !strings.HasSuffix(s.Name, cont.UID) {
|
||||
if !cont.canDelete(s.Name, s.CreationTimestamp, del) {
|
||||
continue
|
||||
}
|
||||
msg += fmt.Sprintf("%v (ssl-certificate)\n", s.Name)
|
||||
|
|
@ -519,7 +553,7 @@ func (cont *GCEIngressController) deleteInstanceGroup(del bool) (msg string) {
|
|||
return msg
|
||||
}
|
||||
for _, ig := range igList.Items {
|
||||
if !strings.HasSuffix(ig.Name, cont.UID) {
|
||||
if !cont.canDelete(ig.Name, ig.CreationTimestamp, del) {
|
||||
continue
|
||||
}
|
||||
msg += fmt.Sprintf("%v (instance-group)\n", ig.Name)
|
||||
|
|
@ -533,6 +567,34 @@ func (cont *GCEIngressController) deleteInstanceGroup(del bool) (msg string) {
|
|||
return msg
|
||||
}
|
||||
|
||||
// canDelete returns true if either the name ends in a suffix matching this
|
||||
// controller's UID, or the creationTimestamp exceeds the maxAge and del is set
|
||||
// to true. Always returns false if the name doesn't match that we expect for
|
||||
// Ingress cloud resources.
|
||||
func (cont *GCEIngressController) canDelete(resourceName, creationTimestamp string, delOldResources bool) bool {
|
||||
// ignore everything not created by an ingress controller.
|
||||
if !strings.HasPrefix(resourceName, k8sPrefix) || len(strings.Split(resourceName, clusterDelimiter)) != 2 {
|
||||
return false
|
||||
}
|
||||
// always delete things that are created by the current ingress controller.
|
||||
if strings.HasSuffix(resourceName, cont.UID) {
|
||||
return true
|
||||
}
|
||||
if !delOldResources {
|
||||
return false
|
||||
}
|
||||
createdTime, err := time.Parse(time.RFC3339, creationTimestamp)
|
||||
if err != nil {
|
||||
framework.Logf("WARNING: Failed to parse creation timestamp %v for %v: %v", creationTimestamp, resourceName, err)
|
||||
return false
|
||||
}
|
||||
if createdTime.Before(time.Now().Add(maxAge)) {
|
||||
framework.Logf("%v created on %v IS too old", resourceName, creationTimestamp)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (cont *GCEIngressController) deleteFirewallRule(del bool) (msg string) {
|
||||
gceCloud := cont.cloud.Provider.(*gcecloud.GCECloud)
|
||||
fwName := fmt.Sprintf("k8s-fw-l7--%v", cont.UID)
|
||||
|
|
@ -595,7 +657,7 @@ func (cont *GCEIngressController) init() {
|
|||
if len(testName) > nameLenLimit {
|
||||
framework.Logf("WARNING: test name including cluster UID: %v is over the GCE limit of %v", testName, nameLenLimit)
|
||||
} else {
|
||||
framework.Logf("Deteced cluster UID %v", cont.UID)
|
||||
framework.Logf("Detected cluster UID %v", cont.UID)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue