Merge pull request #57224 from xiangpengzhao/kubeadm-phase-kubelet

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add phase command for dynamic kubelet configuration in kubeadm.

**What this PR does / why we need it**:
As the title says.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
partially fixes this: https://github.com/kubernetes/kubeadm/issues/571

**Special notes for your reviewer**:
/cc @luxas @fabriziopandini 
@kubernetes/sig-cluster-lifecycle-pr-reviews 

**Release note**:

```release-note
Phase `kubeadm alpha phase kubelet` is added to support dynamic kubelet configuration in kubeadm.
```

cmd/kubeadm/app/cmd/phases/BUILD

@@ -1,10 +1,4 @@
-package(default_visibility = ["//visibility:public"])
-
-load(
-    "@io_bazel_rules_go//go:def.bzl",
-    "go_library",
-    "go_test",
-)
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -15,6 +9,7 @@ go_library(
         "controlplane.go",
         "etcd.go",
         "kubeconfig.go",
+        "kubelet.go",
         "markmaster.go",
         "phase.go",
         "preflight.go",
@@ -23,6 +18,7 @@ go_library(
         "util.go",
     ],
     importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases",
+    visibility = ["//visibility:public"],
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library",
@@ -38,6 +34,7 @@ go_library(
         "//cmd/kubeadm/app/phases/controlplane:go_default_library",
         "//cmd/kubeadm/app/phases/etcd:go_default_library",
         "//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
+        "//cmd/kubeadm/app/phases/kubelet:go_default_library",
         "//cmd/kubeadm/app/phases/markmaster:go_default_library",
         "//cmd/kubeadm/app/phases/selfhosting:go_default_library",
         "//cmd/kubeadm/app/phases/uploadconfig:go_default_library",
@@ -47,11 +44,13 @@ go_library(
         "//cmd/kubeadm/app/util/config:go_default_library",
         "//cmd/kubeadm/app/util/kubeconfig:go_default_library",
         "//pkg/api/legacyscheme:go_default_library",
+        "//pkg/util/node:go_default_library",
         "//pkg/util/normalizer:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
@@ -68,6 +67,7 @@ go_test(
         "controlplane_test.go",
         "etcd_test.go",
         "kubeconfig_test.go",
+        "kubelet_test.go",
     ],
     embed = [":go_default_library"],
     deps = [
@@ -79,6 +79,7 @@ go_test(
         "//cmd/kubeadm/test/cmd:go_default_library",
         "//cmd/kubeadm/test/kubeconfig:go_default_library",
         "//pkg/util/node:go_default_library",
+        "//vendor/github.com/spf13/cobra:go_default_library",
         "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
     ],
 )
@@ -94,4 +95,5 @@ filegroup(
     name = "all-srcs",
     srcs = [":package-srcs"],
     tags = ["automanaged"],
+    visibility = ["//visibility:public"],
 )

cmd/kubeadm/app/cmd/phases/kubelet.go

@@ -0,0 +1,201 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package phases
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"github.com/spf13/cobra"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
+	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
+	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
+	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	nodeutil "k8s.io/kubernetes/pkg/util/node"
+	"k8s.io/kubernetes/pkg/util/normalizer"
+)
+
+var (
+	kubeletWriteInitConfigLongDesc = normalizer.LongDesc(`
+		Writes init kubelet configuration to disk for dynamic kubelet configuration feature.
+		Please note that the kubelet configuration can be passed to kubeadm as a value into the master configuration file.
+		` + cmdutil.AlphaDisclaimer)
+
+	kubeletWriteInitConfigExample = normalizer.Examples(`
+		# Writes init kubelet configuration to disk.
+		kubeadm alpha phase kubelet init
+		`)
+
+	kubeletUploadDynamicConfigLongDesc = normalizer.LongDesc(`
+		Uploads dynamic kubelet configuration as ConfigMap and links it to the current node as ConfigMapRef.
+		Please note that the kubelet configuration can be passed to kubeadm as a value into the master configuration file.
+		` + cmdutil.AlphaDisclaimer)
+
+	kubeletUploadDynamicConfigExample = normalizer.Examples(`
+		# Uploads dynamic kubelet configuration as ConfigMap.
+		kubeadm alpha phase kubelet upload
+		`)
+
+	kubeletEnableDynamicConfigLongDesc = normalizer.LongDesc(`
+		Enables or updates dynamic kubelet configuration on node. This should be run on nodes.
+		Please note that the kubelet configuration can be passed to kubeadm as a value into the master configuration file.
+		` + cmdutil.AlphaDisclaimer)
+
+	kubeletEnableDynamicConfigExample = normalizer.Examples(`
+		# Enables dynamic kubelet configuration on node.
+		kubeadm alpha phase kubelet enable
+		`)
+)
+
+// NewCmdKubelet returns main command for Kubelet phase
+func NewCmdKubelet() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "kubelet",
+		Short: "Adopts dynamic kubelet configuration.",
+		Long:  cmdutil.MacroCommandLongDescription,
+	}
+
+	cmd.AddCommand(NewCmdKubeletWriteInitConfig())
+	cmd.AddCommand(NewCmdKubeletUploadDynamicConfig())
+	cmd.AddCommand(NewCmdKubeletEnableDynamicConfig())
+
+	return cmd
+}
+
+// NewCmdKubeletWriteInitConfig calls cobra.Command for writing init kubelet configuration
+func NewCmdKubeletWriteInitConfig() *cobra.Command {
+	var cfgPath string
+	cmd := &cobra.Command{
+		Use:     "init",
+		Short:   "Writes init kubelet configuration to disk",
+		Long:    kubeletWriteInitConfigLongDesc,
+		Example: kubeletWriteInitConfigExample,
+		Run: func(cmd *cobra.Command, args []string) {
+			cfg := &kubeadmapiext.MasterConfiguration{
+				// KubernetesVersion is not used by kubelet init, but we set this explicitly to avoid
+				// the lookup of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig
+				KubernetesVersion: "v1.9.0",
+			}
+			legacyscheme.Scheme.Default(cfg)
+
+			// This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags
+			internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg)
+			kubeadmutil.CheckErr(err)
+			if features.Enabled(internalcfg.FeatureGates, features.DynamicKubeletConfig) {
+				err = kubeletphase.WriteInitKubeletConfigToDiskOnMaster(internalcfg)
+				kubeadmutil.CheckErr(err)
+			} else {
+				fmt.Println("[kubelet] feature gate DynamicKubeletConfig is not enabled, do nothing.")
+			}
+		},
+	}
+
+	cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")
+
+	return cmd
+}
+
+// NewCmdKubeletUploadDynamicConfig calls cobra.Command for uploading dynamic kubelet configuration
+func NewCmdKubeletUploadDynamicConfig() *cobra.Command {
+	var cfgPath, kubeConfigFile string
+
+	cmd := &cobra.Command{
+		Use:     "upload",
+		Short:   "Uploads dynamic kubelet configuration as ConfigMap",
+		Long:    kubeletUploadDynamicConfigLongDesc,
+		Example: kubeletUploadDynamicConfigExample,
+		Run: func(cmd *cobra.Command, args []string) {
+			cfg := &kubeadmapiext.MasterConfiguration{
+				// KubernetesVersion is not used by kubelet upload, but we set this explicitly to avoid
+				// the lookup of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig
+				KubernetesVersion: "v1.9.0",
+			}
+			legacyscheme.Scheme.Default(cfg)
+
+			// This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags
+			internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg)
+			kubeadmutil.CheckErr(err)
+			if features.Enabled(internalcfg.FeatureGates, features.DynamicKubeletConfig) {
+				client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile)
+				kubeadmutil.CheckErr(err)
+				err = kubeletphase.CreateBaseKubeletConfiguration(internalcfg, client)
+				kubeadmutil.CheckErr(err)
+			} else {
+				fmt.Println("[kubelet] feature gate DynamicKubeletConfig is not enabled, do nothing.")
+			}
+		},
+	}
+
+	cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")
+	cmd.Flags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use when talking to the cluster")
+
+	return cmd
+}
+
+// NewCmdKubeletEnableDynamicConfig calls cobra.Command for enabling dynamic kubelet configuration on node
+func NewCmdKubeletEnableDynamicConfig() *cobra.Command {
+	cfg := &kubeadmapiext.NodeConfiguration{}
+	legacyscheme.Scheme.Default(cfg)
+
+	var cfgPath string
+	cmd := &cobra.Command{
+		Use:     "enable",
+		Aliases: []string{"update"},
+		Short:   "Enables or updates dynamic kubelet configuration on node",
+		Long:    kubeletEnableDynamicConfigLongDesc,
+		Example: kubeletEnableDynamicConfigExample,
+		Run: func(cmd *cobra.Command, args []string) {
+			nodeName, err := getNodeName(cfgPath, cfg)
+			kubeadmutil.CheckErr(err)
+			if features.Enabled(cfg.FeatureGates, features.DynamicKubeletConfig) {
+				err = kubeletphase.ConsumeBaseKubeletConfiguration(nodeName)
+				kubeadmutil.CheckErr(err)
+			} else {
+				fmt.Println("[kubelet] feature gate DynamicKubeletConfig is not enabled, do nothing.")
+			}
+		},
+	}
+
+	cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")
+	cmd.Flags().StringVar(&cfg.NodeName, "node-name", cfg.NodeName, "Name of the node that should enable the dynamic kubelet configuration")
+
+	return cmd
+}
+
+func getNodeName(cfgPath string, cfg *kubeadmapiext.NodeConfiguration) (string, error) {
+	if cfgPath != "" {
+		b, err := ioutil.ReadFile(cfgPath)
+		if err != nil {
+			return "", fmt.Errorf("unable to read config from %q [%v]", cfgPath, err)
+		}
+		if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), b, cfg); err != nil {
+			return "", fmt.Errorf("unable to decode config from %q [%v]", cfgPath, err)
+		}
+	}
+
+	if cfg.NodeName == "" {
+		cfg.NodeName = nodeutil.GetHostname("")
+	}
+
+	return cfg.NodeName, nil
+}

cmd/kubeadm/app/cmd/phases/kubelet_test.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package phases
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+
+	cmdtestutil "k8s.io/kubernetes/cmd/kubeadm/test/cmd"
+)
+
+func TestKubeletSubCommandsHasFlags(t *testing.T) {
+	subCmds := []*cobra.Command{
+		NewCmdKubeletWriteInitConfig(),
+		NewCmdKubeletUploadDynamicConfig(),
+		NewCmdKubeletEnableDynamicConfig(),
+	}
+
+	commonFlags := []string{
+		"config",
+	}
+
+	var tests = []struct {
+		command         string
+		additionalFlags []string
+	}{
+		{
+			command: "init",
+		},
+		{
+			command: "upload",
+			additionalFlags: []string{
+				"kubeconfig",
+			},
+		},
+		{
+			command: "enable",
+			additionalFlags: []string{
+				"node-name",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		expectedFlags := append(commonFlags, test.additionalFlags...)
+		cmdtestutil.AssertSubCommandHasFlags(t, subCmds, test.command, expectedFlags...)
+	}
+}

cmd/kubeadm/app/cmd/phases/phase.go

@@ -36,6 +36,7 @@ func NewCmdPhase(out io.Writer) *cobra.Command {
 	cmd.AddCommand(NewCmdCerts())
 	cmd.AddCommand(NewCmdControlplane())
 	cmd.AddCommand(NewCmdEtcd())
+	cmd.AddCommand(NewCmdKubelet())
 	cmd.AddCommand(NewCmdKubeConfig(out))
 	cmd.AddCommand(NewCmdMarkMaster())
 	cmd.AddCommand(NewCmdPreFlight())

docs/.generated_docs

@@ -45,6 +45,10 @@ docs/admin/kubeadm_alpha_phase_kubeconfig_controller-manager.md
 docs/admin/kubeadm_alpha_phase_kubeconfig_kubelet.md
 docs/admin/kubeadm_alpha_phase_kubeconfig_scheduler.md
 docs/admin/kubeadm_alpha_phase_kubeconfig_user.md
+docs/admin/kubeadm_alpha_phase_kubelet.md
+docs/admin/kubeadm_alpha_phase_kubelet_enable.md
+docs/admin/kubeadm_alpha_phase_kubelet_init.md
+docs/admin/kubeadm_alpha_phase_kubelet_upload.md
 docs/admin/kubeadm_alpha_phase_mark-master.md
 docs/admin/kubeadm_alpha_phase_preflight.md
 docs/admin/kubeadm_alpha_phase_preflight_master.md
@@ -114,6 +118,10 @@ docs/man/man1/kubeadm-alpha-phase-kubeconfig-kubelet.1
 docs/man/man1/kubeadm-alpha-phase-kubeconfig-scheduler.1
 docs/man/man1/kubeadm-alpha-phase-kubeconfig-user.1
 docs/man/man1/kubeadm-alpha-phase-kubeconfig.1
+docs/man/man1/kubeadm-alpha-phase-kubelet-enable.1
+docs/man/man1/kubeadm-alpha-phase-kubelet-init.1
+docs/man/man1/kubeadm-alpha-phase-kubelet-upload.1
+docs/man/man1/kubeadm-alpha-phase-kubelet.1
 docs/man/man1/kubeadm-alpha-phase-mark-master.1
 docs/man/man1/kubeadm-alpha-phase-preflight-master.1
 docs/man/man1/kubeadm-alpha-phase-preflight-node.1

docs/admin/kubeadm_alpha_phase_kubelet.md

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/admin/kubeadm_alpha_phase_kubelet_enable.md

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/admin/kubeadm_alpha_phase_kubelet_init.md

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/admin/kubeadm_alpha_phase_kubelet_upload.md

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/man/man1/kubeadm-alpha-phase-kubelet-enable.1

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/man/man1/kubeadm-alpha-phase-kubelet-init.1

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/man/man1/kubeadm-alpha-phase-kubelet-upload.1

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.

docs/man/man1/kubeadm-alpha-phase-kubelet.1

@@ -0,0 +1,3 @@
+This file is autogenerated, but we've stopped checking such files into the
+repository to reduce the need for rebases. Please run hack/generate-docs.sh to
+populate this file.