Merge pull request #60291 from hzxuzhonghu/cloud-cm-use-healthz

Automatic merge from submit-queue (batch tested with PRs 60376, 55584, 60358, 54631, 60291). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

cloud-controller-manager get /healthz to wait for apiserver to be healthy

**What this PR does / why we need it**:

currently cloud-controller-manager use `restclient.ServerAPIVersions()` to wait for apiserver to be healthy.
Remove ServerAPIVersions and make use of /healthz as all other components do.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #60288

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```

cmd/cloud-controller-manager/app/controllermanager.go

@@ -260,15 +260,9 @@ func startControllers(c *cloudcontrollerconfig.CompletedConfig, kubeconfig *rest
 
 	// If apiserver is not running we should wait for some time and fail only then. This is particularly
 	// important when we start apiserver and controller manager at the same time.
-	err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
-		if _, err = restclient.ServerAPIVersions(kubeconfig); err == nil {
-			return true, nil
-		}
-		glog.Errorf("Failed to get api versions from server: %v", err)
-		return false, nil
-	})
+	err = genericcontrollermanager.WaitForAPIServer(versionedClient, 10*time.Second)
 	if err != nil {
-		glog.Fatalf("Failed to get api versions from server: %v", err)
+		glog.Fatalf("Failed to wait for apiserver being healthy: %v", err)
 	}
 
 	sharedInformers.Start(stop)

cmd/controller-manager/app/BUILD

@@ -4,6 +4,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "config.go",
+        "helper.go",
         "insecure_serving.go",
         "serve.go",
     ],
@@ -15,6 +16,7 @@ go_library(
         "//pkg/util/configz:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/endpoints/filters:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server:go_default_library",

cmd/controller-manager/app/helper.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/golang/glog"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+)
+
+// WaitForAPIServer waits for the API Server's /healthz endpoint to report "ok" with timeout.
+func WaitForAPIServer(client clientset.Interface, timeout time.Duration) error {
+	var lastErr error
+
+	err := wait.PollImmediate(time.Second, timeout, func() (bool, error) {
+		healthStatus := 0
+		result := client.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus)
+		if result.Error() != nil {
+			lastErr = fmt.Errorf("failed to get apiserver /healthz status: %v", result.Error())
+			return false, nil
+		}
+		if healthStatus != http.StatusOK {
+			content, _ := result.Raw()
+			lastErr = fmt.Errorf("APIServer isn't healthy: %v", string(content))
+			glog.Warningf("APIServer isn't healthy yet: %v. Waiting a little while.", string(content))
+			return false, nil
+		}
+
+		return true, nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("%v: %v", err, lastErr)
+	}
+
+	return nil
+}

cmd/kube-controller-manager/app/controllermanager.go

@@ -24,7 +24,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
-	"net/http"
 	"os"
 	"time"
 
@@ -36,7 +35,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/informers"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/leaderelection"
@@ -350,34 +348,8 @@ func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc
 //  users don't have to restart their controller manager if they change the apiserver.
 // Until we get there, the structure here needs to be exposed for the construction of a proper ControllerContext.
 func GetAvailableResources(clientBuilder controller.ControllerClientBuilder) (map[schema.GroupVersionResource]bool, error) {
-	var discoveryClient discovery.DiscoveryInterface
-
-	var healthzContent string
-	// If apiserver is not running we should wait for some time and fail only then. This is particularly
-	// important when we start apiserver and controller manager at the same time.
-	err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
-		client, err := clientBuilder.Client("controller-discovery")
-		if err != nil {
-			glog.Errorf("Failed to get api versions from server: %v", err)
-			return false, nil
-		}
-
-		healthStatus := 0
-		resp := client.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus)
-		if healthStatus != http.StatusOK {
-			glog.Errorf("Server isn't healthy yet.  Waiting a little while.")
-			return false, nil
-		}
-		content, _ := resp.Raw()
-		healthzContent = string(content)
-
-		discoveryClient = client.Discovery()
-		return true, nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to get api versions from server: %v: %v", healthzContent, err)
-	}
-
+	client := clientBuilder.ClientOrDie("controller-discovery")
+	discoveryClient := client.Discovery()
 	resourceMap, err := discoveryClient.ServerResources()
 	if err != nil {
 		utilruntime.HandleError(fmt.Errorf("unable to get all supported resources from server: %v", err))
@@ -407,6 +379,12 @@ func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clien
 	versionedClient := rootClientBuilder.ClientOrDie("shared-informers")
 	sharedInformers := informers.NewSharedInformerFactory(versionedClient, ResyncPeriod(s)())
 
+	// If apiserver is not running we should wait for some time and fail only then. This is particularly
+	// important when we start apiserver and controller manager at the same time.
+	if err := genericcontrollerconfig.WaitForAPIServer(versionedClient, 10*time.Second); err != nil {
+		return ControllerContext{}, fmt.Errorf("failed to wait for apiserver being healthy: %v", err)
+	}
+
 	availableResources, err := GetAvailableResources(rootClientBuilder)
 	if err != nil {
 		return ControllerContext{}, err

staging/src/k8s.io/client-go/rest/BUILD

@@ -54,7 +54,6 @@ go_library(
         "transport.go",
         "url_utils.go",
         "urlbackoff.go",
-        "versions.go",
         "zz_generated.deepcopy.go",
     ],
     importpath = "k8s.io/client-go/rest",

staging/src/k8s.io/client-go/rest/versions.go

@@ -1,88 +0,0 @@
-/*
-Copyright 2014 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package rest
-
-import (
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"path"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-const (
-	legacyAPIPath  = "/api"
-	defaultAPIPath = "/apis"
-)
-
-// TODO: Is this obsoleted by the discovery client?
-
-// ServerAPIVersions returns the GroupVersions supported by the API server.
-// It creates a RESTClient based on the passed in config, but it doesn't rely
-// on the Version and Codec of the config, because it uses AbsPath and
-// takes the raw response.
-func ServerAPIVersions(c *Config) (groupVersions []string, err error) {
-	transport, err := TransportFor(c)
-	if err != nil {
-		return nil, err
-	}
-	client := http.Client{Transport: transport}
-
-	configCopy := *c
-	configCopy.GroupVersion = nil
-	configCopy.APIPath = ""
-	baseURL, _, err := defaultServerUrlFor(&configCopy)
-	if err != nil {
-		return nil, err
-	}
-	// Get the groupVersions exposed at /api
-	originalPath := baseURL.Path
-	baseURL.Path = path.Join(originalPath, legacyAPIPath)
-	resp, err := client.Get(baseURL.String())
-	if err != nil {
-		return nil, err
-	}
-	var v metav1.APIVersions
-	defer resp.Body.Close()
-	err = json.NewDecoder(resp.Body).Decode(&v)
-	if err != nil {
-		return nil, fmt.Errorf("unexpected error: %v", err)
-	}
-
-	groupVersions = append(groupVersions, v.Versions...)
-	// Get the groupVersions exposed at /apis
-	baseURL.Path = path.Join(originalPath, defaultAPIPath)
-	resp2, err := client.Get(baseURL.String())
-	if err != nil {
-		return nil, err
-	}
-	var apiGroupList metav1.APIGroupList
-	defer resp2.Body.Close()
-	err = json.NewDecoder(resp2.Body).Decode(&apiGroupList)
-	if err != nil {
-		return nil, fmt.Errorf("unexpected error: %v", err)
-	}
-
-	for _, g := range apiGroupList.Groups {
-		for _, gv := range g.Versions {
-			groupVersions = append(groupVersions, gv.GroupVersion)
-		}
-	}
-
-	return groupVersions, nil
-}