Merge pull request #56014 from xiangpengzhao/join-kubelet-conf

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Re-engineer the kubeadm join logic.

**What this PR does / why we need it**:
- wait for the kubelet to create `/etc/kubernetes/kubelet.conf`
- use those credentials to PATCH the node

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
ref: https://github.com/kubernetes/kubeadm/issues/28#issuecomment-345502933

**Special notes for your reviewer**:
/cc @kubernetes/sig-cluster-lifecycle-pr-reviews

**Release note**:

```release-note
NONE
```

cmd/kubeadm/app/cmd/BUILD

@@ -70,6 +70,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/version:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",

cmd/kubeadm/app/cmd/join.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -28,6 +29,8 @@ import (
 	flag "github.com/spf13/pflag"
 
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
 	certutil "k8s.io/client-go/util/cert"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
@@ -237,9 +240,23 @@ func (j *Join) Run(out io.Writer) error {
 		return err
 	}
 
+	kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName)
+
+	// Write the bootstrap kubelet config file or the TLS-Boostrapped kubelet config file down to disk
+	if err := kubeconfigutil.WriteToDisk(kubeconfigFile, cfg); err != nil {
+		return fmt.Errorf("couldn't save bootstrap-kubelet.conf to disk: %v", err)
+	}
+
+	// Write the ca certificate to disk so kubelet can use it for authentication
+	cluster := cfg.Contexts[cfg.CurrentContext].Cluster
+	err = certutil.WriteCert(j.cfg.CACertPath, cfg.Clusters[cluster].CertificateAuthorityData)
+	if err != nil {
+		return fmt.Errorf("couldn't save the CA certificate to disk: %v", err)
+	}
+
 	// NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
 	if features.Enabled(j.cfg.FeatureGates, features.DynamicKubeletConfig) {
-		client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetAdminKubeConfigPath())
+		client, err := getTLSBootstrappedClient()
 		if err != nil {
 			return err
 		}
@@ -250,20 +267,25 @@ func (j *Join) Run(out io.Writer) error {
 		}
 	}
 
-	kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName)
-
-	// Write the bootstrap kubelet config file or the TLS-Boostrapped kubelet config file down to disk
-	if err := kubeconfigutil.WriteToDisk(kubeconfigFile, cfg); err != nil {
-		return err
-	}
-
-	// Write the ca certificate to disk so kubelet can use it for authentication
-	cluster := cfg.Contexts[cfg.CurrentContext].Cluster
-	err = certutil.WriteCert(j.cfg.CACertPath, cfg.Clusters[cluster].CertificateAuthorityData)
-	if err != nil {
-		return fmt.Errorf("couldn't save the CA certificate to disk: %v", err)
-	}
-
 	fmt.Fprintf(out, joinDoneMsgf)
 	return nil
 }
+
+// getTLSBootstrappedClient waits for the kubelet to perform the TLS bootstrap
+// and then creates a client from config file /etc/kubernetes/kubelet.conf
+func getTLSBootstrappedClient() (clientset.Interface, error) {
+	fmt.Println("[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...")
+
+	kubeletKubeConfig := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)
+
+	// Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned.
+	err := wait.PollImmediateInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
+		_, err := os.Stat(kubeletKubeConfig)
+		return (err == nil), nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return kubeconfigutil.ClientSetFromFile(kubeletKubeConfig)
+}

cmd/kubeadm/app/phases/kubelet/kubelet.go

@@ -39,6 +39,9 @@ import (
 
 // CreateBaseKubeletConfiguration creates base kubelet configuration for dynamic kubelet configuration feature.
 func CreateBaseKubeletConfiguration(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
+	fmt.Printf("[kubelet] Uploading a ConfigMap %q in namespace %s with base configuration for the kubelets in the cluster",
+		kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.NamespaceSystem)
+
 	_, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs()
 	if err != nil {
 		return err
@@ -69,6 +72,9 @@ func CreateBaseKubeletConfiguration(cfg *kubeadmapi.MasterConfiguration, client
 
 // UpdateNodeWithConfigMap updates node ConfigSource with KubeletBaseConfigurationConfigMap
 func UpdateNodeWithConfigMap(client clientset.Interface, nodeName string) error {
+	fmt.Printf("[kubelet] Using Dynamic Kubelet Config for node %q; config sourced from ConfigMap %q in namespace %s",
+		nodeName, kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.NamespaceSystem)
+
 	// Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned.
 	return wait.Poll(kubeadmconstants.APICallRetryInterval, kubeadmconstants.UpdateNodeTimeout, func() (bool, error) {
 		node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})