From 97be082fb1a267c8fa709149e3d8955266a86fee Mon Sep 17 00:00:00 2001 From: hzxuzhonghu Date: Mon, 26 Feb 2018 15:57:42 +0800 Subject: [PATCH] cloud-controller-manager get /healthz instead of calling restclient.ServerAPIVersions to wait for apiserver being healthy --- .../app/controllermanager.go | 10 +--- cmd/controller-manager/app/helper.go | 55 +++++++++++++++++++ .../app/controllermanager.go | 38 +++---------- 3 files changed, 65 insertions(+), 38 deletions(-) create mode 100644 cmd/controller-manager/app/helper.go diff --git a/cmd/cloud-controller-manager/app/controllermanager.go b/cmd/cloud-controller-manager/app/controllermanager.go index 5f7c4575cb..36a83fc86c 100644 --- a/cmd/cloud-controller-manager/app/controllermanager.go +++ b/cmd/cloud-controller-manager/app/controllermanager.go @@ -260,15 +260,9 @@ func startControllers(c *cloudcontrollerconfig.CompletedConfig, kubeconfig *rest // If apiserver is not running we should wait for some time and fail only then. This is particularly // important when we start apiserver and controller manager at the same time. - err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { - if _, err = restclient.ServerAPIVersions(kubeconfig); err == nil { - return true, nil - } - glog.Errorf("Failed to get api versions from server: %v", err) - return false, nil - }) + err = genericcontrollermanager.WaitForAPIServer(versionedClient, 10*time.Second) if err != nil { - glog.Fatalf("Failed to get api versions from server: %v", err) + glog.Fatalf("Failed to wait for apiserver being healthy: %v", err) } sharedInformers.Start(stop) diff --git a/cmd/controller-manager/app/helper.go b/cmd/controller-manager/app/helper.go new file mode 100644 index 0000000000..38ec45417d --- /dev/null +++ b/cmd/controller-manager/app/helper.go @@ -0,0 +1,55 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "net/http" + "time" + + "github.com/golang/glog" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" +) + +// WaitForAPIServer waits for the API Server's /healthz endpoint to report "ok" with timeout. +func WaitForAPIServer(client clientset.Interface, timeout time.Duration) error { + var lastErr error + + err := wait.PollImmediate(time.Second, timeout, func() (bool, error) { + healthStatus := 0 + result := client.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus) + if result.Error() != nil { + lastErr = fmt.Errorf("failed to get apiserver /healthz status: %v", result.Error()) + return false, nil + } + if healthStatus != http.StatusOK { + content, _ := result.Raw() + lastErr = fmt.Errorf("APIServer isn't healthy: %v", string(content)) + glog.Warningf("APIServer isn't healthy yet: %v. Waiting a little while.", string(content)) + return false, nil + } + + return true, nil + }) + + if err != nil { + return fmt.Errorf("%v: %v", err, lastErr) + } + + return nil +} diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index 00dbe3d1b0..7658f29e32 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -24,7 +24,6 @@ import ( "fmt" "io/ioutil" "math/rand" - "net/http" "os" "time" @@ -36,7 +35,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/discovery" "k8s.io/client-go/informers" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection" @@ -350,34 +348,8 @@ func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc // users don't have to restart their controller manager if they change the apiserver. // Until we get there, the structure here needs to be exposed for the construction of a proper ControllerContext. func GetAvailableResources(clientBuilder controller.ControllerClientBuilder) (map[schema.GroupVersionResource]bool, error) { - var discoveryClient discovery.DiscoveryInterface - - var healthzContent string - // If apiserver is not running we should wait for some time and fail only then. This is particularly - // important when we start apiserver and controller manager at the same time. - err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { - client, err := clientBuilder.Client("controller-discovery") - if err != nil { - glog.Errorf("Failed to get api versions from server: %v", err) - return false, nil - } - - healthStatus := 0 - resp := client.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus) - if healthStatus != http.StatusOK { - glog.Errorf("Server isn't healthy yet. Waiting a little while.") - return false, nil - } - content, _ := resp.Raw() - healthzContent = string(content) - - discoveryClient = client.Discovery() - return true, nil - }) - if err != nil { - return nil, fmt.Errorf("failed to get api versions from server: %v: %v", healthzContent, err) - } - + client := clientBuilder.ClientOrDie("controller-discovery") + discoveryClient := client.Discovery() resourceMap, err := discoveryClient.ServerResources() if err != nil { utilruntime.HandleError(fmt.Errorf("unable to get all supported resources from server: %v", err)) @@ -407,6 +379,12 @@ func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clien versionedClient := rootClientBuilder.ClientOrDie("shared-informers") sharedInformers := informers.NewSharedInformerFactory(versionedClient, ResyncPeriod(s)()) + // If apiserver is not running we should wait for some time and fail only then. This is particularly + // important when we start apiserver and controller manager at the same time. + if err := genericcontrollerconfig.WaitForAPIServer(versionedClient, 10*time.Second); err != nil { + return ControllerContext{}, fmt.Errorf("failed to wait for apiserver being healthy: %v", err) + } + availableResources, err := GetAvailableResources(rootClientBuilder) if err != nil { return ControllerContext{}, err