Merge pull request #66056 from mikedanese/fixhang

Automatic merge from submit-queue (batch tested with PRs 67349, 66056). If you want to cherry-pick this change to another branch, please follow the instructions here: https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md.

wait until apiserver connection before starting kubelet tls bootstrap

I wonder if this helps with sometimes slow network programming

cc @mwielgus @awly

pkg/kubelet/certificate/bootstrap/BUILD

@@ -21,8 +21,11 @@ go_library(
     srcs = ["bootstrap.go"],
     importpath = "k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap",
     deps = [
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
         "//staging/src/k8s.io/client-go/rest:go_default_library",
         "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",

pkg/kubelet/certificate/bootstrap/bootstrap.go

@@ -17,6 +17,8 @@ limitations under the License.
 package bootstrap
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -24,8 +26,11 @@ import (
 
 	"github.com/golang/glog"
 
+	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes/scheme"
 	certificates "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -59,6 +64,7 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string,
 	if err != nil {
 		return fmt.Errorf("unable to load bootstrap kubeconfig: %v", err)
 	}
+
 	bootstrapClient, err := certificates.NewForConfig(bootstrapClientConfig)
 	if err != nil {
 		return fmt.Errorf("unable to create certificates signing request client: %v", err)
@@ -92,6 +98,10 @@ func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string,
 		}
 	}
 
+	if err := waitForServer(*bootstrapClientConfig, 1*time.Minute); err != nil {
+		glog.Warningf("Error waiting for apiserver to come up: %v", err)
+	}
+
 	certData, err := csr.RequestNodeCertificate(bootstrapClient.CertificateSigningRequests(), keyData, nodeName)
 	if err != nil {
 		return err
@@ -207,3 +217,30 @@ func verifyKeyData(data []byte) bool {
 	_, err := certutil.ParsePrivateKeyPEM(data)
 	return err == nil
 }
+
+func waitForServer(cfg restclient.Config, deadline time.Duration) error {
+	cfg.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs}
+	cfg.Timeout = 1 * time.Second
+	cli, err := restclient.UnversionedRESTClientFor(&cfg)
+	if err != nil {
+		return fmt.Errorf("couldn't create client: %v", err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.TODO(), deadline)
+	defer cancel()
+
+	var connected bool
+	wait.JitterUntil(func() {
+		if _, err := cli.Get().AbsPath("/healthz").Do().Raw(); err != nil {
+			glog.Infof("Failed to connect to apiserver: %v", err)
+			return
+		}
+		cancel()
+		connected = true
+	}, 2*time.Second, 0.2, true, ctx.Done())
+
+	if !connected {
+		return errors.New("timed out waiting to connect to apiserver")
+	}
+	return nil
+}