From 97be082fb1a267c8fa709149e3d8955266a86fee Mon Sep 17 00:00:00 2001 From: hzxuzhonghu Date: Mon, 26 Feb 2018 15:57:42 +0800 Subject: [PATCH 1/3] cloud-controller-manager get /healthz instead of calling restclient.ServerAPIVersions to wait for apiserver being healthy --- .../app/controllermanager.go | 10 +--- cmd/controller-manager/app/helper.go | 55 +++++++++++++++++++ .../app/controllermanager.go | 38 +++---------- 3 files changed, 65 insertions(+), 38 deletions(-) create mode 100644 cmd/controller-manager/app/helper.go diff --git a/cmd/cloud-controller-manager/app/controllermanager.go b/cmd/cloud-controller-manager/app/controllermanager.go index 5f7c4575cb..36a83fc86c 100644 --- a/cmd/cloud-controller-manager/app/controllermanager.go +++ b/cmd/cloud-controller-manager/app/controllermanager.go @@ -260,15 +260,9 @@ func startControllers(c *cloudcontrollerconfig.CompletedConfig, kubeconfig *rest // If apiserver is not running we should wait for some time and fail only then. This is particularly // important when we start apiserver and controller manager at the same time. - err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { - if _, err = restclient.ServerAPIVersions(kubeconfig); err == nil { - return true, nil - } - glog.Errorf("Failed to get api versions from server: %v", err) - return false, nil - }) + err = genericcontrollermanager.WaitForAPIServer(versionedClient, 10*time.Second) if err != nil { - glog.Fatalf("Failed to get api versions from server: %v", err) + glog.Fatalf("Failed to wait for apiserver being healthy: %v", err) } sharedInformers.Start(stop) diff --git a/cmd/controller-manager/app/helper.go b/cmd/controller-manager/app/helper.go new file mode 100644 index 0000000000..38ec45417d --- /dev/null +++ b/cmd/controller-manager/app/helper.go @@ -0,0 +1,55 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "net/http" + "time" + + "github.com/golang/glog" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" +) + +// WaitForAPIServer waits for the API Server's /healthz endpoint to report "ok" with timeout. +func WaitForAPIServer(client clientset.Interface, timeout time.Duration) error { + var lastErr error + + err := wait.PollImmediate(time.Second, timeout, func() (bool, error) { + healthStatus := 0 + result := client.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus) + if result.Error() != nil { + lastErr = fmt.Errorf("failed to get apiserver /healthz status: %v", result.Error()) + return false, nil + } + if healthStatus != http.StatusOK { + content, _ := result.Raw() + lastErr = fmt.Errorf("APIServer isn't healthy: %v", string(content)) + glog.Warningf("APIServer isn't healthy yet: %v. Waiting a little while.", string(content)) + return false, nil + } + + return true, nil + }) + + if err != nil { + return fmt.Errorf("%v: %v", err, lastErr) + } + + return nil +} diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index 00dbe3d1b0..7658f29e32 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -24,7 +24,6 @@ import ( "fmt" "io/ioutil" "math/rand" - "net/http" "os" "time" @@ -36,7 +35,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/discovery" "k8s.io/client-go/informers" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection" @@ -350,34 +348,8 @@ func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc // users don't have to restart their controller manager if they change the apiserver. // Until we get there, the structure here needs to be exposed for the construction of a proper ControllerContext. func GetAvailableResources(clientBuilder controller.ControllerClientBuilder) (map[schema.GroupVersionResource]bool, error) { - var discoveryClient discovery.DiscoveryInterface - - var healthzContent string - // If apiserver is not running we should wait for some time and fail only then. This is particularly - // important when we start apiserver and controller manager at the same time. - err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) { - client, err := clientBuilder.Client("controller-discovery") - if err != nil { - glog.Errorf("Failed to get api versions from server: %v", err) - return false, nil - } - - healthStatus := 0 - resp := client.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus) - if healthStatus != http.StatusOK { - glog.Errorf("Server isn't healthy yet. Waiting a little while.") - return false, nil - } - content, _ := resp.Raw() - healthzContent = string(content) - - discoveryClient = client.Discovery() - return true, nil - }) - if err != nil { - return nil, fmt.Errorf("failed to get api versions from server: %v: %v", healthzContent, err) - } - + client := clientBuilder.ClientOrDie("controller-discovery") + discoveryClient := client.Discovery() resourceMap, err := discoveryClient.ServerResources() if err != nil { utilruntime.HandleError(fmt.Errorf("unable to get all supported resources from server: %v", err)) @@ -407,6 +379,12 @@ func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clien versionedClient := rootClientBuilder.ClientOrDie("shared-informers") sharedInformers := informers.NewSharedInformerFactory(versionedClient, ResyncPeriod(s)()) + // If apiserver is not running we should wait for some time and fail only then. This is particularly + // important when we start apiserver and controller manager at the same time. + if err := genericcontrollerconfig.WaitForAPIServer(versionedClient, 10*time.Second); err != nil { + return ControllerContext{}, fmt.Errorf("failed to wait for apiserver being healthy: %v", err) + } + availableResources, err := GetAvailableResources(rootClientBuilder) if err != nil { return ControllerContext{}, err From 817176572dd86bc8d2c7f8741cd76ca32d685e86 Mon Sep 17 00:00:00 2001 From: hzxuzhonghu Date: Mon, 26 Feb 2018 15:58:36 +0800 Subject: [PATCH 2/3] remove unused rest/versions.go --- staging/src/k8s.io/client-go/rest/versions.go | 88 ------------------- 1 file changed, 88 deletions(-) delete mode 100644 staging/src/k8s.io/client-go/rest/versions.go diff --git a/staging/src/k8s.io/client-go/rest/versions.go b/staging/src/k8s.io/client-go/rest/versions.go deleted file mode 100644 index 9d41812f26..0000000000 --- a/staging/src/k8s.io/client-go/rest/versions.go +++ /dev/null @@ -1,88 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rest - -import ( - "encoding/json" - "fmt" - "net/http" - "path" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const ( - legacyAPIPath = "/api" - defaultAPIPath = "/apis" -) - -// TODO: Is this obsoleted by the discovery client? - -// ServerAPIVersions returns the GroupVersions supported by the API server. -// It creates a RESTClient based on the passed in config, but it doesn't rely -// on the Version and Codec of the config, because it uses AbsPath and -// takes the raw response. -func ServerAPIVersions(c *Config) (groupVersions []string, err error) { - transport, err := TransportFor(c) - if err != nil { - return nil, err - } - client := http.Client{Transport: transport} - - configCopy := *c - configCopy.GroupVersion = nil - configCopy.APIPath = "" - baseURL, _, err := defaultServerUrlFor(&configCopy) - if err != nil { - return nil, err - } - // Get the groupVersions exposed at /api - originalPath := baseURL.Path - baseURL.Path = path.Join(originalPath, legacyAPIPath) - resp, err := client.Get(baseURL.String()) - if err != nil { - return nil, err - } - var v metav1.APIVersions - defer resp.Body.Close() - err = json.NewDecoder(resp.Body).Decode(&v) - if err != nil { - return nil, fmt.Errorf("unexpected error: %v", err) - } - - groupVersions = append(groupVersions, v.Versions...) - // Get the groupVersions exposed at /apis - baseURL.Path = path.Join(originalPath, defaultAPIPath) - resp2, err := client.Get(baseURL.String()) - if err != nil { - return nil, err - } - var apiGroupList metav1.APIGroupList - defer resp2.Body.Close() - err = json.NewDecoder(resp2.Body).Decode(&apiGroupList) - if err != nil { - return nil, fmt.Errorf("unexpected error: %v", err) - } - - for _, g := range apiGroupList.Groups { - for _, gv := range g.Versions { - groupVersions = append(groupVersions, gv.GroupVersion) - } - } - - return groupVersions, nil -} From 8908c5d0a0538b473cbf1cd546967411da74ab75 Mon Sep 17 00:00:00 2001 From: hzxuzhonghu Date: Fri, 23 Feb 2018 17:28:49 +0800 Subject: [PATCH 3/3] run update bazel --- cmd/controller-manager/app/BUILD | 2 ++ staging/src/k8s.io/client-go/rest/BUILD | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/controller-manager/app/BUILD b/cmd/controller-manager/app/BUILD index 769db3a2bc..d36a9526ec 100644 --- a/cmd/controller-manager/app/BUILD +++ b/cmd/controller-manager/app/BUILD @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "config.go", + "helper.go", "insecure_serving.go", "serve.go", ], @@ -15,6 +16,7 @@ go_library( "//pkg/util/configz:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apiserver/pkg/endpoints/filters:go_default_library", "//vendor/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//vendor/k8s.io/apiserver/pkg/server:go_default_library", diff --git a/staging/src/k8s.io/client-go/rest/BUILD b/staging/src/k8s.io/client-go/rest/BUILD index 511329b5d9..ceab3512c0 100644 --- a/staging/src/k8s.io/client-go/rest/BUILD +++ b/staging/src/k8s.io/client-go/rest/BUILD @@ -54,7 +54,6 @@ go_library( "transport.go", "url_utils.go", "urlbackoff.go", - "versions.go", "zz_generated.deepcopy.go", ], importpath = "k8s.io/client-go/rest",