From 0aa0f3208a769a3c1c14568a7ca889532efcaeef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Tue, 22 May 2018 09:12:25 +0300 Subject: [PATCH] kubeadm: Write kubelet config file to disk and persist in-cluster. Also write runtime environment file and fixup the kubelet phases command --- cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go | 11 +- .../app/apis/kubeadm/v1alpha1/defaults.go | 36 ++- .../app/apis/kubeadm/v1alpha2/defaults.go | 36 ++- cmd/kubeadm/app/cmd/init.go | 62 +++-- cmd/kubeadm/app/cmd/join.go | 24 +- cmd/kubeadm/app/cmd/phases/kubelet.go | 210 ++++++++-------- cmd/kubeadm/app/cmd/phases/kubelet_test.go | 28 ++- cmd/kubeadm/app/cmd/upgrade/common.go | 18 +- cmd/kubeadm/app/constants/constants.go | 26 +- cmd/kubeadm/app/phases/kubelet/config.go | 187 ++++++++++++++ cmd/kubeadm/app/phases/kubelet/config_test.go | 78 ++++++ cmd/kubeadm/app/phases/kubelet/dynamic.go | 117 +++++++++ .../app/phases/kubelet/dynamic_test.go | 63 +++++ cmd/kubeadm/app/phases/kubelet/flags.go | 78 ++++++ cmd/kubeadm/app/phases/kubelet/kubelet.go | 235 ------------------ .../app/phases/kubelet/kubelet_test.go | 134 ---------- .../app/phases/upgrade/configuration.go | 77 ------ cmd/kubeadm/app/phases/upgrade/postupgrade.go | 6 + cmd/kubeadm/app/preflight/checks.go | 7 +- cmd/kubeadm/app/util/config/cluster.go | 65 +++++ pkg/util/initsystem/initsystem.go | 21 ++ 21 files changed, 885 insertions(+), 634 deletions(-) create mode 100644 cmd/kubeadm/app/phases/kubelet/config.go create mode 100644 cmd/kubeadm/app/phases/kubelet/config_test.go create mode 100644 cmd/kubeadm/app/phases/kubelet/dynamic.go create mode 100644 cmd/kubeadm/app/phases/kubelet/dynamic_test.go create mode 100644 cmd/kubeadm/app/phases/kubelet/flags.go delete mode 100644 cmd/kubeadm/app/phases/kubelet/kubelet.go delete mode 100644 cmd/kubeadm/app/phases/kubelet/kubelet_test.go delete mode 100644 cmd/kubeadm/app/phases/upgrade/configuration.go create mode 100644 cmd/kubeadm/app/util/config/cluster.go diff --git a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go index a8e8c8ce44..3bd46500aa 100644 --- a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go +++ b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go @@ -68,9 +68,16 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { StaticPodPath: "foo", ClusterDNS: []string{"foo"}, ClusterDomain: "foo", - Authorization: kubeletconfigv1beta1.KubeletAuthorization{Mode: "foo"}, + Authorization: kubeletconfigv1beta1.KubeletAuthorization{ + Mode: "Webhook", + }, Authentication: kubeletconfigv1beta1.KubeletAuthentication{ - X509: kubeletconfigv1beta1.KubeletX509Authentication{ClientCAFile: "foo"}, + X509: kubeletconfigv1beta1.KubeletX509Authentication{ + ClientCAFile: "/etc/kubernetes/pki/ca.crt", + }, + Anonymous: kubeletconfigv1beta1.KubeletAnonymousAuthentication{ + Enabled: utilpointer.BoolPtr(false), + }, }, }, } diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go index 46d0535506..9b2b499f6c 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go @@ -24,11 +24,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1" kubeproxyscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme" kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1" + utilpointer "k8s.io/kubernetes/pkg/util/pointer" ) const ( @@ -143,9 +143,7 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) { } SetDefaultsEtcdSelfHosted(obj) - if features.Enabled(obj.FeatureGates, features.DynamicKubeletConfig) { - SetDefaults_KubeletConfiguration(obj) - } + SetDefaults_KubeletConfiguration(obj) SetDefaults_ProxyConfiguration(obj) SetDefaults_AuditPolicyConfiguration(obj) } @@ -235,15 +233,31 @@ func SetDefaults_KubeletConfiguration(obj *MasterConfiguration) { } } if obj.KubeletConfiguration.BaseConfig.ClusterDomain == "" { - obj.KubeletConfiguration.BaseConfig.ClusterDomain = DefaultServiceDNSDomain - } - if obj.KubeletConfiguration.BaseConfig.Authorization.Mode == "" { - obj.KubeletConfiguration.BaseConfig.Authorization.Mode = kubeletconfigv1beta1.KubeletAuthorizationModeWebhook - } - if obj.KubeletConfiguration.BaseConfig.Authentication.X509.ClientCAFile == "" { - obj.KubeletConfiguration.BaseConfig.Authentication.X509.ClientCAFile = DefaultCACertPath + obj.KubeletConfiguration.BaseConfig.ClusterDomain = obj.Networking.DNSDomain } + // Enforce security-related kubelet options + + // Require all clients to the kubelet API to have client certs signed by the cluster CA + obj.KubeletConfiguration.BaseConfig.Authentication.X509.ClientCAFile = DefaultCACertPath + obj.KubeletConfiguration.BaseConfig.Authentication.Anonymous.Enabled = utilpointer.BoolPtr(false) + + // On every client request to the kubelet API, execute a webhook (SubjectAccessReview request) to the API server + // and ask it whether the client is authorized to access the kubelet API + obj.KubeletConfiguration.BaseConfig.Authorization.Mode = kubeletconfigv1beta1.KubeletAuthorizationModeWebhook + + // Let clients using other authentication methods like ServiceAccount tokens also access the kubelet API + // TODO: Enable in a future PR + // obj.KubeletConfiguration.BaseConfig.Authentication.Webhook.Enabled = utilpointer.BoolPtr(true) + + // Disable the readonly port of the kubelet, in order to not expose unnecessary information + // TODO: Enable in a future PR + // obj.KubeletConfiguration.BaseConfig.ReadOnlyPort = 0 + + // Serve a /healthz webserver on localhost:10248 that kubeadm can talk to + obj.KubeletConfiguration.BaseConfig.HealthzBindAddress = "127.0.0.1" + obj.KubeletConfiguration.BaseConfig.HealthzPort = utilpointer.Int32Ptr(10248) + scheme, _, _ := kubeletscheme.NewSchemeAndCodecs() if scheme != nil { scheme.Default(obj.KubeletConfiguration.BaseConfig) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/defaults.go index 266f0033a9..946d3f2e6e 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha2/defaults.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha2/defaults.go @@ -23,11 +23,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1" kubeproxyscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme" kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1" + utilpointer "k8s.io/kubernetes/pkg/util/pointer" ) const ( @@ -127,9 +127,7 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) { obj.ClusterName = DefaultClusterName } - if features.Enabled(obj.FeatureGates, features.DynamicKubeletConfig) { - SetDefaults_KubeletConfiguration(obj) - } + SetDefaults_KubeletConfiguration(obj) SetDefaults_ProxyConfiguration(obj) SetDefaults_AuditPolicyConfiguration(obj) } @@ -198,15 +196,31 @@ func SetDefaults_KubeletConfiguration(obj *MasterConfiguration) { } } if obj.KubeletConfiguration.BaseConfig.ClusterDomain == "" { - obj.KubeletConfiguration.BaseConfig.ClusterDomain = DefaultServiceDNSDomain - } - if obj.KubeletConfiguration.BaseConfig.Authorization.Mode == "" { - obj.KubeletConfiguration.BaseConfig.Authorization.Mode = kubeletconfigv1beta1.KubeletAuthorizationModeWebhook - } - if obj.KubeletConfiguration.BaseConfig.Authentication.X509.ClientCAFile == "" { - obj.KubeletConfiguration.BaseConfig.Authentication.X509.ClientCAFile = DefaultCACertPath + obj.KubeletConfiguration.BaseConfig.ClusterDomain = obj.Networking.DNSDomain } + // Enforce security-related kubelet options + + // Require all clients to the kubelet API to have client certs signed by the cluster CA + obj.KubeletConfiguration.BaseConfig.Authentication.X509.ClientCAFile = DefaultCACertPath + obj.KubeletConfiguration.BaseConfig.Authentication.Anonymous.Enabled = utilpointer.BoolPtr(false) + + // On every client request to the kubelet API, execute a webhook (SubjectAccessReview request) to the API server + // and ask it whether the client is authorized to access the kubelet API + obj.KubeletConfiguration.BaseConfig.Authorization.Mode = kubeletconfigv1beta1.KubeletAuthorizationModeWebhook + + // Let clients using other authentication methods like ServiceAccount tokens also access the kubelet API + // TODO: Enable in a future PR + // obj.KubeletConfiguration.BaseConfig.Authentication.Webhook.Enabled = utilpointer.BoolPtr(true) + + // Disable the readonly port of the kubelet, in order to not expose unnecessary information + // TODO: Enable in a future PR + // obj.KubeletConfiguration.BaseConfig.ReadOnlyPort = 0 + + // Serve a /healthz webserver on localhost:10248 that kubeadm can talk to + obj.KubeletConfiguration.BaseConfig.HealthzBindAddress = "127.0.0.1" + obj.KubeletConfiguration.BaseConfig.HealthzPort = utilpointer.Int32Ptr(10248) + scheme, _, _ := kubeletscheme.NewSchemeAndCodecs() if scheme != nil { scheme.Default(obj.KubeletConfiguration.BaseConfig) diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 9b5ab21bc7..7c03b5b4c4 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -259,22 +259,29 @@ func NewInit(cfgPath string, externalcfg *kubeadmapiv1alpha2.MasterConfiguration return nil, err } - // Try to start the kubelet service in case it's inactive - glog.V(1).Infof("Starting kubelet") - preflight.TryStartKubelet(ignorePreflightErrors) - - return &Init{cfg: cfg, skipTokenPrint: skipTokenPrint, dryRun: dryRun}, nil + return &Init{cfg: cfg, skipTokenPrint: skipTokenPrint, dryRun: dryRun, ignorePreflightErrors: ignorePreflightErrors}, nil } // Init defines struct used by "kubeadm init" command type Init struct { - cfg *kubeadmapi.MasterConfiguration - skipTokenPrint bool - dryRun bool + cfg *kubeadmapi.MasterConfiguration + skipTokenPrint bool + dryRun bool + ignorePreflightErrors sets.String } // Run executes master node provisioning, including certificates, needed static pod manifests, etc. func (i *Init) Run(out io.Writer) error { + + // Write env file with flags for the kubelet to use + if err := kubeletphase.WriteKubeletDynamicEnvFile(i.cfg); err != nil { + return err + } + + // Try to start the kubelet service in case it's inactive + glog.V(1).Infof("Starting kubelet") + preflight.TryStartKubelet(i.ignorePreflightErrors) + // Get directories to write files to; can be faked if we're dry-running glog.V(1).Infof("[init] Getting certificates directory from configuration") realCertsDir := i.cfg.CertificatesDir @@ -346,14 +353,14 @@ func (i *Init) Run(out io.Writer) error { return fmt.Errorf("error printing files on dryrun: %v", err) } - // NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) { - glog.V(1).Infof("[init] feature --dynamic-config-dir is enabled") - glog.V(1).Infof("[init] writing base kubelet configuration to disk on master") - // Write base kubelet configuration for dynamic kubelet configuration feature. - if err := kubeletphase.WriteInitKubeletConfigToDiskOnMaster(i.cfg); err != nil { - return fmt.Errorf("error writing base kubelet configuration to disk: %v", err) - } + kubeletVersion, err := preflight.GetKubeletVersion(utilsexec.New()) + if err != nil { + return err + } + + // Write the kubelet configuration to disk. + if err := kubeletphase.WriteConfigToDisk(i.cfg.KubeletConfiguration.BaseConfig, kubeletVersion); err != nil { + return fmt.Errorf("error writing kubelet configuration to disk: %v", err) } // Create a kubernetes client and wait for the API server to be healthy (if not dryrunning) @@ -381,15 +388,6 @@ func (i *Init) Run(out io.Writer) error { return fmt.Errorf("couldn't initialize a Kubernetes cluster") } - // NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) { - // Create base kubelet configuration for dynamic kubelet configuration feature. - glog.V(1).Infof("[init] creating base kubelet configuration") - if err := kubeletphase.CreateBaseKubeletConfiguration(i.cfg, client); err != nil { - return fmt.Errorf("error creating base kubelet configuration: %v", err) - } - } - // Upload currently used configuration to the cluster // Note: This is done right in the beginning of cluster initialization; as we might want to make other phases // depend on centralized information from this source in the future @@ -398,12 +396,26 @@ func (i *Init) Run(out io.Writer) error { return fmt.Errorf("error uploading configuration: %v", err) } + glog.V(1).Infof("[init] creating kubelet configuration configmap") + if err := kubeletphase.CreateConfigMap(i.cfg, client); err != nil { + return fmt.Errorf("error creating kubelet configuration ConfigMap: %v", err) + } + // PHASE 4: Mark the master with the right label/taint glog.V(1).Infof("[init] marking the master with right label") if err := markmasterphase.MarkMaster(client, i.cfg.NodeName, !i.cfg.NoTaintMaster); err != nil { return fmt.Errorf("error marking master: %v", err) } + // NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + // This feature is disabled by default, as it is alpha still + if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) { + // Enable dynamic kubelet configuration for the node. + if err := kubeletphase.EnableDynamicConfigForNode(client, i.cfg.NodeName, kubeletVersion); err != nil { + return fmt.Errorf("error enabling dynamic kubelet configuration: %v", err) + } + } + // PHASE 5: Set up the node bootstrap tokens if !i.skipTokenPrint { glog.Infof("[bootstraptoken] using token: %s\n", i.cfg.Token) diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index ce6566cd6e..77fcb6697c 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -252,10 +252,28 @@ func (j *Join) Run(out io.Writer) error { return fmt.Errorf("couldn't save the CA certificate to disk: %v", err) } - // NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - glog.V(1).Infoln("[join] consuming base kubelet configuration") + kubeletVersion, err := preflight.GetKubeletVersion(utilsexec.New()) + if err != nil { + return err + } + + // Write the configuration for the kubelet down to disk so the kubelet can start + if err := kubeletphase.DownloadConfig(kubeconfigFile, kubeletVersion); err != nil { + return err + } + + // Now the kubelet will perform the TLS Bootstrap, transforming bootstrap-kubeconfig.conf to kubeconfig.conf in /etc/kubernetes + + // NOTE: the "--dynamic-config-dir" flag should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf for this to work + // This feature is disabled by default, as it is alpha still + glog.V(1).Infoln("[join] enabling dynamic kubelet configuration") if features.Enabled(j.cfg.FeatureGates, features.DynamicKubeletConfig) { - if err := kubeletphase.ConsumeBaseKubeletConfiguration(j.cfg.NodeName); err != nil { + client, err := kubeletphase.GetLocalNodeTLSBootstrappedClient() + if err != nil { + return err + } + + if err := kubeletphase.EnableDynamicConfigForNode(client, j.cfg.NodeName, kubeletVersion); err != nil { return fmt.Errorf("error consuming base kubelet configuration: %v", err) } } diff --git a/cmd/kubeadm/app/cmd/phases/kubelet.go b/cmd/kubeadm/app/cmd/phases/kubelet.go index 3e6f681105..c07257d9b7 100644 --- a/cmd/kubeadm/app/cmd/phases/kubelet.go +++ b/cmd/kubeadm/app/cmd/phases/kubelet.go @@ -18,184 +18,172 @@ package phases import ( "fmt" - "io/ioutil" + "os" "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/runtime" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - "k8s.io/kubernetes/cmd/kubeadm/app/features" kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" - nodeutil "k8s.io/kubernetes/pkg/util/node" "k8s.io/kubernetes/pkg/util/normalizer" + "k8s.io/kubernetes/pkg/util/version" ) var ( - kubeletWriteInitConfigLongDesc = normalizer.LongDesc(` - Writes init kubelet configuration to disk for dynamic kubelet configuration feature. - Please note that the kubelet configuration can be passed to kubeadm as a value into the master configuration file. + kubeletWriteConfigToDiskLongDesc = normalizer.LongDesc(` + Writes kubelet configuration to disk, either based on the kubelet-config-1.X ConfigMap in the cluster, or from the + configuration passed to the command via "--config". ` + cmdutil.AlphaDisclaimer) - kubeletWriteInitConfigExample = normalizer.Examples(` - # Writes init kubelet configuration to disk. - kubeadm alpha phase kubelet init + kubeletWriteConfigToDiskExample = normalizer.Examples(` + # Writes kubelet configuration for a node to disk. The information is fetched from the cluster ConfigMap + kubeadm alpha phase kubelet write-config-to-disk --kubelet-version v1.11.0 --kubeconfig /etc/kubernetes/kubelet.conf + + # Writes kubelet configuration down to disk, based on the configuration flag passed to --config + kubeadm alpha phase kubelet write-config-to-disk --kubelet-version v1.11.0 --config kubeadm.yaml `) kubeletUploadDynamicConfigLongDesc = normalizer.LongDesc(` - Uploads dynamic kubelet configuration as ConfigMap and links it to the current node as ConfigMapRef. - Please note that the kubelet configuration can be passed to kubeadm as a value into the master configuration file. + Uploads kubelet configuration extracted from the kubeadm MasterConfiguration object to a ConfigMap + of the form kubelet-config-1.X in the cluster, where X is the minor version of the current Kubernetes version ` + cmdutil.AlphaDisclaimer) kubeletUploadDynamicConfigExample = normalizer.Examples(` - # Uploads dynamic kubelet configuration as ConfigMap. - kubeadm alpha phase kubelet upload + # Uploads the kubelet configuration from the kubeadm Config file to a ConfigMap in the cluster. + kubeadm alpha phase kubelet upload-config --config kubeadm.yaml `) kubeletEnableDynamicConfigLongDesc = normalizer.LongDesc(` - Enables or updates dynamic kubelet configuration on node. This should be run on nodes. - Please note that the kubelet configuration can be passed to kubeadm as a value into the master configuration file. + Enables or updates dynamic kubelet configuration for a Node, against the kubelet-config-1.X ConfigMap in the cluster, + where X is the minor version of the desired kubelet version. + + WARNING: This feature is still experimental, and disabled by default. Enable only if you know what you are doing, as it + may have surprising side-effects at this stage. + ` + cmdutil.AlphaDisclaimer) kubeletEnableDynamicConfigExample = normalizer.Examples(` - # Enables dynamic kubelet configuration on node. - kubeadm alpha phase kubelet enable + # Enables dynamic kubelet configuration for a Node. + kubeadm alpha phase kubelet enable-dynamic-config --node-name node-1 --kubelet-version v1.11.0 + + WARNING: This feature is still experimental, and disabled by default. Enable only if you know what you are doing, as it + may have surprising side-effects at this stage. `) ) // NewCmdKubelet returns main command for Kubelet phase func NewCmdKubelet() *cobra.Command { + var kubeConfigFile string cmd := &cobra.Command{ Use: "kubelet", - Short: "Adopts dynamic kubelet configuration.", + Short: "Handles kubelet configuration.", Long: cmdutil.MacroCommandLongDescription, } - cmd.AddCommand(NewCmdKubeletWriteInitConfig()) - cmd.AddCommand(NewCmdKubeletUploadDynamicConfig()) - cmd.AddCommand(NewCmdKubeletEnableDynamicConfig()) + cmd.PersistentFlags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use when talking to the cluster") + cmd.AddCommand(NewCmdKubeletWriteConfigToDisk(&kubeConfigFile)) + cmd.AddCommand(NewCmdKubeletUploadConfig(&kubeConfigFile)) + cmd.AddCommand(NewCmdKubeletEnableDynamicConfig(&kubeConfigFile)) return cmd } -// NewCmdKubeletWriteInitConfig calls cobra.Command for writing init kubelet configuration -func NewCmdKubeletWriteInitConfig() *cobra.Command { +// NewCmdKubeletUploadConfig calls cobra.Command for uploading dynamic kubelet configuration +func NewCmdKubeletUploadConfig(kubeConfigFile *string) *cobra.Command { var cfgPath string - cmd := &cobra.Command{ - Use: "init", - Short: "Writes init kubelet configuration to disk", - Long: kubeletWriteInitConfigLongDesc, - Example: kubeletWriteInitConfigExample, - Run: func(cmd *cobra.Command, args []string) { - cfg := &kubeadmapiv1alpha2.MasterConfiguration{ - // KubernetesVersion is not used by kubelet init, but we set this explicitly to avoid - // the lookup of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig - KubernetesVersion: "v1.9.0", - } - kubeadmscheme.Scheme.Default(cfg) - - // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags - internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg) - kubeadmutil.CheckErr(err) - if features.Enabled(internalcfg.FeatureGates, features.DynamicKubeletConfig) { - err = kubeletphase.WriteInitKubeletConfigToDiskOnMaster(internalcfg) - kubeadmutil.CheckErr(err) - } else { - fmt.Println("[kubelet] feature gate DynamicKubeletConfig is not enabled, do nothing.") - } - }, - } - - cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)") - - return cmd -} - -// NewCmdKubeletUploadDynamicConfig calls cobra.Command for uploading dynamic kubelet configuration -func NewCmdKubeletUploadDynamicConfig() *cobra.Command { - var cfgPath, kubeConfigFile string cmd := &cobra.Command{ - Use: "upload", - Short: "Uploads dynamic kubelet configuration as ConfigMap", + Use: "upload-config", + Short: "Uploads kubelet configuration to a ConfigMap", Long: kubeletUploadDynamicConfigLongDesc, Example: kubeletUploadDynamicConfigExample, Run: func(cmd *cobra.Command, args []string) { - cfg := &kubeadmapiv1alpha2.MasterConfiguration{ - // KubernetesVersion is not used by kubelet upload, but we set this explicitly to avoid - // the lookup of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig - KubernetesVersion: "v1.9.0", + if len(cfgPath) == 0 { + kubeadmutil.CheckErr(fmt.Errorf("The --config argument is required")) } - kubeadmscheme.Scheme.Default(cfg) - // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags - internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg) + // This call returns the ready-to-use configuration based on the configuration file + internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, &kubeadmapiv1alpha2.MasterConfiguration{}) + kubeadmutil.CheckErr(err) + + client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) + kubeadmutil.CheckErr(err) + + err = kubeletphase.CreateConfigMap(internalcfg, client) kubeadmutil.CheckErr(err) - if features.Enabled(internalcfg.FeatureGates, features.DynamicKubeletConfig) { - client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) - kubeadmutil.CheckErr(err) - err = kubeletphase.CreateBaseKubeletConfiguration(internalcfg, client) - kubeadmutil.CheckErr(err) - } else { - fmt.Println("[kubelet] feature gate DynamicKubeletConfig is not enabled, do nothing.") - } }, } cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)") - cmd.Flags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use when talking to the cluster") + return cmd +} +// NewCmdKubeletWriteConfigToDisk calls cobra.Command for writing init kubelet configuration +func NewCmdKubeletWriteConfigToDisk(kubeConfigFile *string) *cobra.Command { + var cfgPath, kubeletVersionStr string + cmd := &cobra.Command{ + Use: "write-config-to-disk", + Short: "Writes kubelet configuration to disk, either based on the --config argument or the kubeadm-config ConfigMap.", + Long: kubeletWriteConfigToDiskLongDesc, + Example: kubeletWriteConfigToDiskExample, + Run: func(cmd *cobra.Command, args []string) { + if len(kubeletVersionStr) == 0 { + kubeadmutil.CheckErr(fmt.Errorf("The --kubelet-version argument is required")) + } + + kubeletVersion, err := version.ParseSemantic(kubeletVersionStr) + kubeadmutil.CheckErr(err) + + client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) + kubeadmutil.CheckErr(err) + + // This call returns the ready-to-use configuration based on the configuration file + internalcfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "kubelet", cfgPath) + kubeadmutil.CheckErr(err) + + err = kubeletphase.WriteConfigToDisk(internalcfg.KubeletConfiguration.BaseConfig, kubeletVersion) + kubeadmutil.CheckErr(err) + }, + } + + cmd.Flags().StringVar(&kubeletVersionStr, "kubelet-version", kubeletVersionStr, "The desired version for the kubelet") + cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)") return cmd } // NewCmdKubeletEnableDynamicConfig calls cobra.Command for enabling dynamic kubelet configuration on node -func NewCmdKubeletEnableDynamicConfig() *cobra.Command { - cfg := &kubeadmapiv1alpha2.NodeConfiguration{} - kubeadmscheme.Scheme.Default(cfg) +// This feature is still in alpha and an experimental state +func NewCmdKubeletEnableDynamicConfig(kubeConfigFile *string) *cobra.Command { + var nodeName, kubeletVersionStr string - var cfgPath string cmd := &cobra.Command{ - Use: "enable", - Aliases: []string{"update"}, - Short: "Enables or updates dynamic kubelet configuration on node", + Use: "enable-dynamic-config", + Short: "EXPERIMENTAL: Enables or updates dynamic kubelet configuration for a Node", Long: kubeletEnableDynamicConfigLongDesc, Example: kubeletEnableDynamicConfigExample, Run: func(cmd *cobra.Command, args []string) { - nodeName, err := getNodeName(cfgPath, cfg) - kubeadmutil.CheckErr(err) - if features.Enabled(cfg.FeatureGates, features.DynamicKubeletConfig) { - err = kubeletphase.ConsumeBaseKubeletConfiguration(nodeName) - kubeadmutil.CheckErr(err) - } else { - fmt.Println("[kubelet] feature gate DynamicKubeletConfig is not enabled, do nothing.") + if len(nodeName) == 0 { + kubeadmutil.CheckErr(fmt.Errorf("The --node-name argument is required")) } + if len(kubeletVersionStr) == 0 { + kubeadmutil.CheckErr(fmt.Errorf("The --kubelet-version argument is required")) + } + + kubeletVersion, err := version.ParseSemantic(kubeletVersionStr) + kubeadmutil.CheckErr(err) + + client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) + kubeadmutil.CheckErr(err) + + err = kubeletphase.EnableDynamicConfigForNode(client, nodeName, kubeletVersion) + kubeadmutil.CheckErr(err) }, } - cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)") - cmd.Flags().StringVar(&cfg.NodeName, "node-name", cfg.NodeName, "Name of the node that should enable the dynamic kubelet configuration") - + cmd.Flags().StringVar(&nodeName, "node-name", nodeName, "Name of the node that should enable the dynamic kubelet configuration") + cmd.Flags().StringVar(&kubeletVersionStr, "kubelet-version", kubeletVersionStr, "The desired version for the kubelet") return cmd } - -func getNodeName(cfgPath string, cfg *kubeadmapiv1alpha2.NodeConfiguration) (string, error) { - if cfgPath != "" { - b, err := ioutil.ReadFile(cfgPath) - if err != nil { - return "", fmt.Errorf("unable to read config from %q [%v]", cfgPath, err) - } - if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), b, cfg); err != nil { - return "", fmt.Errorf("unable to decode config from %q [%v]", cfgPath, err) - } - } - - if cfg.NodeName == "" { - cfg.NodeName = nodeutil.GetHostname("") - } - - return cfg.NodeName, nil -} diff --git a/cmd/kubeadm/app/cmd/phases/kubelet_test.go b/cmd/kubeadm/app/cmd/phases/kubelet_test.go index 25ba6f8a44..89159d2610 100644 --- a/cmd/kubeadm/app/cmd/phases/kubelet_test.go +++ b/cmd/kubeadm/app/cmd/phases/kubelet_test.go @@ -25,33 +25,37 @@ import ( ) func TestKubeletSubCommandsHasFlags(t *testing.T) { + kubeConfigFile := "foo" subCmds := []*cobra.Command{ - NewCmdKubeletWriteInitConfig(), - NewCmdKubeletUploadDynamicConfig(), - NewCmdKubeletEnableDynamicConfig(), + NewCmdKubeletUploadConfig(&kubeConfigFile), + NewCmdKubeletWriteConfigToDisk(&kubeConfigFile), + NewCmdKubeletEnableDynamicConfig(&kubeConfigFile), } - commonFlags := []string{ - "config", - } + commonFlags := []string{} var tests = []struct { command string additionalFlags []string }{ { - command: "init", - }, - { - command: "upload", + command: "upload-config", additionalFlags: []string{ - "kubeconfig", + "config", }, }, { - command: "enable", + command: "write-config-to-disk", + additionalFlags: []string{ + "kubelet-version", + "config", + }, + }, + { + command: "enable-dynamic-config", additionalFlags: []string{ "node-name", + "kubelet-version", }, }, } diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go index e6cadbb8fb..1f5f2336ca 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -24,17 +24,21 @@ import ( "os" "strings" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" fakediscovery "k8s.io/client-go/discovery/fake" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" ) @@ -61,8 +65,20 @@ func enforceRequirements(flags *cmdUpgradeFlags, dryRun bool, newK8sVersion stri } // Fetch the configuration from a file or ConfigMap and validate it - cfg, err := upgrade.FetchConfiguration(client, os.Stdout, flags.cfgPath) + fmt.Println("[upgrade/config] Making sure the configuration is correct:") + cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "upgrade/config", flags.cfgPath) if err != nil { + if apierrors.IsNotFound(err) { + fmt.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %s namespace must exist.\n", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem) + fmt.Println("[upgrade/config] Without this information, 'kubeadm upgrade' won't know how to configure your upgraded cluster.") + fmt.Println("") + fmt.Println("[upgrade/config] Next steps:") + fmt.Printf("\t- OPTION 1: Run 'kubeadm config upload from-flags' and specify the same CLI arguments you passed to 'kubeadm init' when you created your master.\n") + fmt.Printf("\t- OPTION 2: Run 'kubeadm config upload from-file' and specify the same config file you passed to 'kubeadm init' when you created your master.\n") + fmt.Printf("\t- OPTION 3: Pass a config file to 'kubeadm upgrade' using the --config flag.\n") + fmt.Println("") + err = fmt.Errorf("the ConfigMap %q in the %s namespace used for getting configuration information was not found", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem) + } return nil, fmt.Errorf("[upgrade/config] FATAL: %v", err) } diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index ea53fade78..afd5ce5ba5 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -161,9 +161,6 @@ const ( // system:nodes group subject is removed if present. NodesClusterRoleBinding = "system:node" - // KubeletBaseConfigMapRoleName defines the base kubelet configuration ConfigMap. - KubeletBaseConfigMapRoleName = "kubeadm:kubelet-base-configmap" - // APICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation APICallRetryInterval = 500 * time.Millisecond // DiscoveryRetryInterval specifies how long kubeadm should wait before retrying to connect to the master when doing discovery @@ -191,17 +188,17 @@ const ( // MasterConfigurationConfigMapKey specifies in what ConfigMap key the master configuration should be stored MasterConfigurationConfigMapKey = "MasterConfiguration" - // KubeletBaseConfigurationConfigMap specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored - KubeletBaseConfigurationConfigMap = "kubelet-base-config-1.9" + // KubeletBaseConfigurationConfigMapPrefix specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored + KubeletBaseConfigurationConfigMapPrefix = "kubelet-config-" // KubeletBaseConfigurationConfigMapKey specifies in what ConfigMap key the initial remote configuration of kubelet should be stored KubeletBaseConfigurationConfigMapKey = "kubelet" - // KubeletBaseConfigurationDir specifies the directory on the node where stores the initial remote configuration of kubelet - KubeletBaseConfigurationDir = "/var/lib/kubelet/config/init" + // KubeletBaseConfigMapRolePrefix defines the base kubelet configuration ConfigMap. + KubeletBaseConfigMapRolePrefix = "kubeadm:kubelet-config-" - // KubeletBaseConfigurationFile specifies the file name on the node which stores initial remote configuration of kubelet - KubeletBaseConfigurationFile = "kubelet" + // KubeletConfigurationFile specifies the file name on the node which stores initial remote configuration of kubelet + KubeletConfigurationFile = "/var/lib/kubelet/config.yaml" // MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports MinExternalEtcdVersion = "3.2.17" @@ -260,6 +257,14 @@ const ( // Copied from pkg/master/reconcilers to avoid pulling extra dependencies // TODO: Import this constant from a consts only package, that does not pull any further dependencies. LeaseEndpointReconcilerType = "lease" + + // KubeletEnvFile is a file "kubeadm init" writes at runtime. Using that interface, kubeadm can customize certain + // kubelet flags conditionally based on the environment at runtime. Also, parameters given to the configuration file + // might be passed through this file. "kubeadm init" writes one variable, with the name ${KubeletEnvFileVariableName}. + KubeletEnvFile = "/var/lib/kubelet/kubeadm-flags.env" + + // KubeletEnvFileVariableName specifies the shell script variable name "kubeadm init" should write a value to in KubeletEnvFile + KubeletEnvFileVariableName = "KUBELET_KUBEADM_ARGS" ) var ( @@ -290,6 +295,9 @@ var ( // MinimumKubeletVersion specifies the minimum version of kubelet which kubeadm supports MinimumKubeletVersion = version.MustParseSemantic("v1.10.0") + // MinimumKubeletConfigVersion specifies the minimum version of Kubernetes where kubeadm supports specifying --config to the kubelet + MinimumKubeletConfigVersion = version.MustParseSemantic("v1.11.0-alpha.1") + // SupportedEtcdVersion lists officially supported etcd versions with corresponding kubernetes releases SupportedEtcdVersion = map[uint8]string{ 10: "3.1.12", diff --git a/cmd/kubeadm/app/phases/kubelet/config.go b/cmd/kubeadm/app/phases/kubelet/config.go new file mode 100644 index 0000000000..5503a16a7f --- /dev/null +++ b/cmd/kubeadm/app/phases/kubelet/config.go @@ -0,0 +1,187 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "fmt" + "io/ioutil" + + "k8s.io/api/core/v1" + rbac "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" + rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1" + kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" + kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1" + "k8s.io/kubernetes/pkg/util/version" +) + +// WriteConfigToDisk writes the kubelet config object down to a file +// Used at "kubeadm init" and "kubeadm upgrade" time +func WriteConfigToDisk(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration, kubeletVersion *version.Version) error { + + // If the kubelet version is v1.10.x, exit + if kubeletVersion.LessThan(kubeadmconstants.MinimumKubeletConfigVersion) { + return nil + } + + kubeletBytes, err := getConfigBytes(kubeletConfig) + if err != nil { + return err + } + return writeConfigBytesToDisk(kubeletBytes) +} + +// CreateConfigMap creates a ConfigMap with the generic kubelet configuration. +// Used at "kubeadm init" and "kubeadm upgrade" time +func CreateConfigMap(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error { + + k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion) + if err != nil { + return err + } + + // If Kubernetes version is v1.10.x, exit + if k8sVersion.LessThan(kubeadmconstants.MinimumKubeletConfigVersion) { + return nil + } + + configMapName := configMapName(k8sVersion) + fmt.Printf("[kubelet] Creating a ConfigMap %q in namespace %s with the configuration for the kubelets in the cluster\n", configMapName, metav1.NamespaceSystem) + + kubeletBytes, err := getConfigBytes(cfg.KubeletConfiguration.BaseConfig) + if err != nil { + return err + } + + if err := apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: metav1.NamespaceSystem, + }, + Data: map[string]string{ + kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(kubeletBytes), + }, + }); err != nil { + return err + } + + if err := createConfigMapRBACRules(client, k8sVersion); err != nil { + return fmt.Errorf("error creating kubelet configuration configmap RBAC rules: %v", err) + } + return nil +} + +// createConfigMapRBACRules creates the RBAC rules for exposing the base kubelet ConfigMap in the kube-system namespace to unauthenticated users +func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Version) error { + if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapRBACName(k8sVersion), + Namespace: metav1.NamespaceSystem, + }, + Rules: []rbac.PolicyRule{ + rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(configMapName(k8sVersion)).RuleOrDie(), + }, + }); err != nil { + return err + } + + return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapRBACName(k8sVersion), + Namespace: metav1.NamespaceSystem, + }, + RoleRef: rbac.RoleRef{ + APIGroup: rbac.GroupName, + Kind: "Role", + Name: configMapRBACName(k8sVersion), + }, + Subjects: []rbac.Subject{ + { + Kind: rbac.GroupKind, + Name: kubeadmconstants.NodesGroup, + }, + { + Kind: rbac.GroupKind, + Name: kubeadmconstants.NodeBootstrapTokenAuthGroup, + }, + }, + }) +} + +// DownloadConfig downloads the kubelet configuration from a ConfigMap and writes it to disk. +// Used at "kubeadm join" time +func DownloadConfig(kubeletKubeConfig string, kubeletVersion *version.Version) error { + + // If the kubelet version is v1.10.x, exit + if kubeletVersion.LessThan(kubeadmconstants.MinimumKubeletConfigVersion) { + return nil + } + + // Download the ConfigMap from the cluster based on what version the kubelet is + configMapName := configMapName(kubeletVersion) + + fmt.Printf("[kubelet] Downloading configuration for the kubelet from the %q ConfigMap in the %s namespace\n", + configMapName, metav1.NamespaceSystem) + + client, err := kubeconfigutil.ClientSetFromFile(kubeletKubeConfig) + if err != nil { + return fmt.Errorf("couldn't create client from kubeconfig file %q", kubeletKubeConfig) + } + + kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{}) + if err != nil { + return err + } + + return writeConfigBytesToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey])) +} + +// configMapName returns the right ConfigMap name for the right branch of k8s +func configMapName(k8sVersion *version.Version) string { + return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigurationConfigMapPrefix, k8sVersion.Major(), k8sVersion.Minor()) +} + +// configMapRBACName returns the name for the Role/RoleBinding for the kubelet config configmap for the right branch of k8s +func configMapRBACName(k8sVersion *version.Version) string { + return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigMapRolePrefix, k8sVersion.Major(), k8sVersion.Minor()) +} + +// getConfigBytes marshals a kubeletconfiguration object to bytes +func getConfigBytes(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) ([]byte, error) { + _, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs() + if err != nil { + return []byte{}, err + } + + return kubeadmutil.MarshalToYamlForCodecs(kubeletConfig, kubeletconfigv1beta1.SchemeGroupVersion, *kubeletCodecs) +} + +// writeConfigBytesToDisk writes a byte slice down to disk at the specific location of the kubelet config file +func writeConfigBytesToDisk(b []byte) error { + fmt.Printf("[kubelet] Writing kubelet configuration to file %q\n", kubeadmconstants.KubeletConfigurationFile) + + if err := ioutil.WriteFile(kubeadmconstants.KubeletConfigurationFile, b, 0644); err != nil { + return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", kubeadmconstants.KubeletConfigurationFile, err) + } + return nil +} diff --git a/cmd/kubeadm/app/phases/kubelet/config_test.go b/cmd/kubeadm/app/phases/kubelet/config_test.go new file mode 100644 index 0000000000..0e6d5b68c3 --- /dev/null +++ b/cmd/kubeadm/app/phases/kubelet/config_test.go @@ -0,0 +1,78 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "testing" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + core "k8s.io/client-go/testing" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1" + "k8s.io/kubernetes/pkg/util/version" +) + +func TestCreateConfigMap(t *testing.T) { + nodeName := "fake-node" + client := fake.NewSimpleClientset() + cfg := &kubeadmapi.MasterConfiguration{ + NodeName: nodeName, + KubernetesVersion: "v1.11.0", + KubeletConfiguration: kubeadmapi.KubeletConfiguration{ + BaseConfig: &kubeletconfigv1beta1.KubeletConfiguration{}, + }, + } + + client.PrependReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) { + return true, &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + }, + Spec: v1.NodeSpec{}, + }, nil + }) + client.PrependReactor("create", "roles", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, nil + }) + client.PrependReactor("create", "rolebindings", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, nil + }) + client.PrependReactor("create", "configmaps", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, nil + }) + + if err := CreateConfigMap(cfg, client); err != nil { + t.Errorf("CreateConfigMap: unexpected error %v", err) + } +} + +func TestCreateConfigMapRBACRules(t *testing.T) { + client := fake.NewSimpleClientset() + client.PrependReactor("create", "roles", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, nil + }) + client.PrependReactor("create", "rolebindings", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, nil + }) + + if err := createConfigMapRBACRules(client, version.MustParseSemantic("v1.11.0")); err != nil { + t.Errorf("createConfigMapRBACRules: unexpected error %v", err) + } +} diff --git a/cmd/kubeadm/app/phases/kubelet/dynamic.go b/cmd/kubeadm/app/phases/kubelet/dynamic.go new file mode 100644 index 0000000000..b65d746037 --- /dev/null +++ b/cmd/kubeadm/app/phases/kubelet/dynamic.go @@ -0,0 +1,117 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" + "k8s.io/kubernetes/pkg/util/version" +) + +// EnableDynamicConfigForNode updates the Node's ConfigSource to enable Dynamic Kubelet Configuration, depending on what version the kubelet is +// Used at "kubeadm init", "kubeadm join" and "kubeadm upgrade" time +// This func is ONLY run if the user enables the `DynamicKubeletConfig` feature gate, which is by default off +func EnableDynamicConfigForNode(client clientset.Interface, nodeName string, kubeletVersion *version.Version) error { + + // If the kubelet version is v1.10.x, exit + if kubeletVersion.LessThan(kubeadmconstants.MinimumKubeletConfigVersion) { + return nil + } + + configMapName := configMapName(kubeletVersion) + fmt.Printf("[kubelet] Enabling Dynamic Kubelet Config for Node %q; config sourced from ConfigMap %q in namespace %s\n", + nodeName, configMapName, metav1.NamespaceSystem) + fmt.Println("[kubelet] WARNING: The Dynamic Kubelet Config feature is alpha and off by default. It hasn't been well-tested yet at this stage, use with caution.") + + // Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned. + return wait.Poll(kubeadmconstants.APICallRetryInterval, kubeadmconstants.UpdateNodeTimeout, func() (bool, error) { + node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return false, nil + } + + oldData, err := json.Marshal(node) + if err != nil { + return false, err + } + + kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{}) + if err != nil { + return false, nil + } + + node.Spec.ConfigSource = &v1.NodeConfigSource{ + ConfigMap: &v1.ConfigMapNodeConfigSource{ + Name: configMapName, + Namespace: metav1.NamespaceSystem, + UID: kubeletCfg.UID, + KubeletConfigKey: kubeadmconstants.KubeletBaseConfigurationConfigMapKey, + }, + } + + newData, err := json.Marshal(node) + if err != nil { + return false, err + } + + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{}) + if err != nil { + return false, err + } + + if _, err := client.CoreV1().Nodes().Patch(node.Name, types.StrategicMergePatchType, patchBytes); err != nil { + if apierrs.IsConflict(err) { + fmt.Println("Temporarily unable to update node metadata due to conflict (will retry)") + return false, nil + } + return false, err + } + + return true, nil + }) +} + +// GetLocalNodeTLSBootstrappedClient waits for the kubelet to perform the TLS bootstrap +// and then creates a client from config file /etc/kubernetes/kubelet.conf +func GetLocalNodeTLSBootstrappedClient() (clientset.Interface, error) { + fmt.Println("[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...") + + kubeletKubeConfig := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName) + + // Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned. + err := wait.PollImmediateInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) { + _, err := os.Stat(kubeletKubeConfig) + return (err == nil), nil + }) + if err != nil { + return nil, err + } + + return kubeconfigutil.ClientSetFromFile(kubeletKubeConfig) +} diff --git a/cmd/kubeadm/app/phases/kubelet/dynamic_test.go b/cmd/kubeadm/app/phases/kubelet/dynamic_test.go new file mode 100644 index 0000000000..fe0ba35f41 --- /dev/null +++ b/cmd/kubeadm/app/phases/kubelet/dynamic_test.go @@ -0,0 +1,63 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "testing" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + core "k8s.io/client-go/testing" + "k8s.io/kubernetes/pkg/util/version" +) + +func TestEnableDynamicConfigForNode(t *testing.T) { + nodeName := "fake-node" + client := fake.NewSimpleClientset() + client.PrependReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) { + return true, &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + }, + Spec: v1.NodeSpec{ + ConfigSource: &v1.NodeConfigSource{ + ConfigMap: &v1.ConfigMapNodeConfigSource{ + UID: "", + }, + }, + }, + }, nil + }) + client.PrependReactor("get", "configmaps", func(action core.Action) (bool, runtime.Object, error) { + return true, &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubelet-config-1.11", + Namespace: metav1.NamespaceSystem, + UID: "fake-uid", + }, + }, nil + }) + client.PrependReactor("patch", "nodes", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, nil + }) + + if err := EnableDynamicConfigForNode(client, nodeName, version.MustParseSemantic("v1.11.0")); err != nil { + t.Errorf("UpdateNodeWithConfigMap: unexpected error %v", err) + } +} diff --git a/cmd/kubeadm/app/phases/kubelet/flags.go b/cmd/kubeadm/app/phases/kubelet/flags.go new file mode 100644 index 0000000000..167966793a --- /dev/null +++ b/cmd/kubeadm/app/phases/kubelet/flags.go @@ -0,0 +1,78 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" +) + +// WriteKubeletDynamicEnvFile writes a environment file with dynamic flags to the kubelet. +// Used at "kubeadm init" and "kubeadm join" time. +func WriteKubeletDynamicEnvFile(cfg *kubeadmapi.MasterConfiguration) error { + + // TODO: Pass through extra arguments from the config file here in the future + argList := kubeadmutil.BuildArgumentListFromMap(buildKubeletArgMap(cfg), map[string]string{}) + envFileContent := fmt.Sprintf("%s=%s\n", constants.KubeletEnvFileVariableName, strings.Join(argList, " ")) + + return writeKubeletFlagBytesToDisk([]byte(envFileContent)) +} + +// buildKubeletArgMap takes a MasterConfiguration object and builds based on that a string-string map with flags +// that should be given to the local kubelet daemon. +func buildKubeletArgMap(cfg *kubeadmapi.MasterConfiguration) map[string]string { + kubeletFlags := map[string]string{} + + if cfg.CRISocket == kubeadmapiv1alpha2.DefaultCRISocket { + // These flags should only be set when running docker + kubeletFlags["network-plugin"] = "cni" + kubeletFlags["cni-conf-dir"] = "/etc/cni/net.d" + kubeletFlags["cni-bin-dir"] = "/opt/cni/bin" + } else { + kubeletFlags["container-runtime"] = "remote" + kubeletFlags["container-runtime-endpoint"] = cfg.CRISocket + } + // TODO: Add support for registering custom Taints and Labels + // TODO: Add support for overriding flags with ExtraArgs + // TODO: Pass through --hostname-override if a custom name is used? + // TODO: Check if `systemd-resolved` is running, and set `--resolv-conf` based on that + // TODO: Conditionally set `--cgroup-driver` to either `systemd` or `cgroupfs` + + return kubeletFlags +} + +// writeKubeletFlagBytesToDisk writes a byte slice down to disk at the specific location of the kubelet flag overrides file +func writeKubeletFlagBytesToDisk(b []byte) error { + fmt.Printf("[kubelet] Writing kubelet environment file with flags to file %q\n", constants.KubeletEnvFile) + + // creates target folder if not already exists + if err := os.MkdirAll(filepath.Dir(constants.KubeletEnvFile), 0700); err != nil { + return fmt.Errorf("failed to create directory %q: %v", filepath.Dir(constants.KubeletEnvFile), err) + } + if err := ioutil.WriteFile(constants.KubeletEnvFile, b, 0644); err != nil { + return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", constants.KubeletEnvFile, err) + } + return nil +} diff --git a/cmd/kubeadm/app/phases/kubelet/kubelet.go b/cmd/kubeadm/app/phases/kubelet/kubelet.go deleted file mode 100644 index 4f0a51888a..0000000000 --- a/cmd/kubeadm/app/phases/kubelet/kubelet.go +++ /dev/null @@ -1,235 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubelet - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "path/filepath" - - "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/strategicpatch" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" - rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1" - kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme" - kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1" -) - -// CreateBaseKubeletConfiguration creates base kubelet configuration for dynamic kubelet configuration feature. -func CreateBaseKubeletConfiguration(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error { - fmt.Printf("[kubelet] Uploading a ConfigMap %q in namespace %s with base configuration for the kubelets in the cluster\n", - kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.NamespaceSystem) - - _, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs() - if err != nil { - return err - } - kubeletBytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg.KubeletConfiguration.BaseConfig, kubeletconfigv1beta1.SchemeGroupVersion, *kubeletCodecs) - if err != nil { - return err - } - - if err = apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.KubeletBaseConfigurationConfigMap, - Namespace: metav1.NamespaceSystem, - }, - Data: map[string]string{ - kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(kubeletBytes), - }, - }); err != nil { - return err - } - - if err := createKubeletBaseConfigMapRBACRules(client); err != nil { - return fmt.Errorf("error creating base kubelet configmap RBAC rules: %v", err) - } - - return updateNodeWithConfigMap(client, cfg.NodeName) -} - -// ConsumeBaseKubeletConfiguration consumes base kubelet configuration for dynamic kubelet configuration feature. -func ConsumeBaseKubeletConfiguration(nodeName string) error { - client, err := getLocalNodeTLSBootstrappedClient() - if err != nil { - return err - } - - kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.GetOptions{}) - if err != nil { - return err - } - - if err := writeInitKubeletConfigToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey])); err != nil { - return fmt.Errorf("failed to write initial remote configuration of kubelet to disk for node %s: %v", nodeName, err) - } - - return updateNodeWithConfigMap(client, nodeName) -} - -// updateNodeWithConfigMap updates node ConfigSource with KubeletBaseConfigurationConfigMap -func updateNodeWithConfigMap(client clientset.Interface, nodeName string) error { - fmt.Printf("[kubelet] Using Dynamic Kubelet Config for node %q; config sourced from ConfigMap %q in namespace %s\n", - nodeName, kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.NamespaceSystem) - - // Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned. - return wait.Poll(kubeadmconstants.APICallRetryInterval, kubeadmconstants.UpdateNodeTimeout, func() (bool, error) { - node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) - if err != nil { - return false, nil - } - - oldData, err := json.Marshal(node) - if err != nil { - return false, err - } - - kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.GetOptions{}) - if err != nil { - return false, nil - } - - node.Spec.ConfigSource = &v1.NodeConfigSource{ - ConfigMap: &v1.ConfigMapNodeConfigSource{ - Name: kubeadmconstants.KubeletBaseConfigurationConfigMap, - Namespace: metav1.NamespaceSystem, - UID: kubeletCfg.UID, - KubeletConfigKey: kubeadmconstants.KubeletBaseConfigurationConfigMapKey, - }, - } - - newData, err := json.Marshal(node) - if err != nil { - return false, err - } - - patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{}) - if err != nil { - return false, err - } - - if _, err := client.CoreV1().Nodes().Patch(node.Name, types.StrategicMergePatchType, patchBytes); err != nil { - if apierrs.IsConflict(err) { - fmt.Println("Temporarily unable to update node metadata due to conflict (will retry)") - return false, nil - } - return false, err - } - - return true, nil - }) -} - -// createKubeletBaseConfigMapRBACRules creates the RBAC rules for exposing the base kubelet ConfigMap in the kube-system namespace to unauthenticated users -func createKubeletBaseConfigMapRBACRules(client clientset.Interface) error { - if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{ - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.KubeletBaseConfigMapRoleName, - Namespace: metav1.NamespaceSystem, - }, - Rules: []rbac.PolicyRule{ - rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(kubeadmconstants.KubeletBaseConfigurationConfigMap).RuleOrDie(), - }, - }); err != nil { - return err - } - - return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.KubeletBaseConfigMapRoleName, - Namespace: metav1.NamespaceSystem, - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbac.GroupName, - Kind: "Role", - Name: kubeadmconstants.KubeletBaseConfigMapRoleName, - }, - Subjects: []rbac.Subject{ - { - Kind: rbac.GroupKind, - Name: kubeadmconstants.NodesGroup, - }, - { - Kind: rbac.GroupKind, - Name: kubeadmconstants.NodeBootstrapTokenAuthGroup, - }, - }, - }) -} - -// getLocalNodeTLSBootstrappedClient waits for the kubelet to perform the TLS bootstrap -// and then creates a client from config file /etc/kubernetes/kubelet.conf -func getLocalNodeTLSBootstrappedClient() (clientset.Interface, error) { - fmt.Println("[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...") - - kubeletKubeConfig := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName) - - // Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned. - err := wait.PollImmediateInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) { - _, err := os.Stat(kubeletKubeConfig) - return (err == nil), nil - }) - if err != nil { - return nil, err - } - - return kubeconfigutil.ClientSetFromFile(kubeletKubeConfig) -} - -// WriteInitKubeletConfigToDiskOnMaster writes base kubelet configuration to disk on master. -func WriteInitKubeletConfigToDiskOnMaster(cfg *kubeadmapi.MasterConfiguration) error { - fmt.Printf("[kubelet] Writing base configuration of kubelets to disk on master node %s\n", cfg.NodeName) - - _, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs() - if err != nil { - return err - } - - kubeletBytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg.KubeletConfiguration.BaseConfig, kubeletconfigv1beta1.SchemeGroupVersion, *kubeletCodecs) - if err != nil { - return err - } - - if err := writeInitKubeletConfigToDisk(kubeletBytes); err != nil { - return fmt.Errorf("failed to write base configuration of kubelet to disk on master node %s: %v", cfg.NodeName, err) - } - - return nil -} - -func writeInitKubeletConfigToDisk(kubeletConfig []byte) error { - if err := os.MkdirAll(kubeadmconstants.KubeletBaseConfigurationDir, 0644); err != nil { - return fmt.Errorf("failed to create directory %q: %v", kubeadmconstants.KubeletBaseConfigurationDir, err) - } - baseConfigFile := filepath.Join(kubeadmconstants.KubeletBaseConfigurationDir, kubeadmconstants.KubeletBaseConfigurationFile) - if err := ioutil.WriteFile(baseConfigFile, kubeletConfig, 0644); err != nil { - return fmt.Errorf("failed to write initial remote configuration of kubelet into file %q: %v", baseConfigFile, err) - } - return nil -} diff --git a/cmd/kubeadm/app/phases/kubelet/kubelet_test.go b/cmd/kubeadm/app/phases/kubelet/kubelet_test.go deleted file mode 100644 index 8d211b052a..0000000000 --- a/cmd/kubeadm/app/phases/kubelet/kubelet_test.go +++ /dev/null @@ -1,134 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubelet - -import ( - "testing" - - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes/fake" - core "k8s.io/client-go/testing" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1" -) - -func TestCreateBaseKubeletConfiguration(t *testing.T) { - nodeName := "fake-node" - client := fake.NewSimpleClientset() - cfg := &kubeadmapi.MasterConfiguration{ - NodeName: nodeName, - KubeletConfiguration: kubeadmapi.KubeletConfiguration{ - BaseConfig: &kubeletconfigv1beta1.KubeletConfiguration{ - TypeMeta: metav1.TypeMeta{ - Kind: "KubeletConfiguration", - }, - }, - }, - } - - client.PrependReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) { - return true, &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, - }, - Spec: v1.NodeSpec{ - ConfigSource: &v1.NodeConfigSource{ - ConfigMap: &v1.ConfigMapNodeConfigSource{ - UID: "", - }, - }, - }, - }, nil - }) - client.PrependReactor("get", "configmaps", func(action core.Action) (bool, runtime.Object, error) { - return true, &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.KubeletBaseConfigurationConfigMap, - Namespace: metav1.NamespaceSystem, - UID: "fake-uid", - }, - }, nil - }) - client.PrependReactor("patch", "nodes", func(action core.Action) (bool, runtime.Object, error) { - return true, nil, nil - }) - client.PrependReactor("create", "roles", func(action core.Action) (bool, runtime.Object, error) { - return true, nil, nil - }) - client.PrependReactor("create", "rolebindings", func(action core.Action) (bool, runtime.Object, error) { - return true, nil, nil - }) - client.PrependReactor("create", "configmaps", func(action core.Action) (bool, runtime.Object, error) { - return true, nil, nil - }) - - if err := CreateBaseKubeletConfiguration(cfg, client); err != nil { - t.Errorf("CreateBaseKubeletConfiguration: unexepected error %v", err) - } -} - -func TestUpdateNodeWithConfigMap(t *testing.T) { - nodeName := "fake-node" - client := fake.NewSimpleClientset() - client.PrependReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) { - return true, &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, - }, - Spec: v1.NodeSpec{ - ConfigSource: &v1.NodeConfigSource{ - ConfigMap: &v1.ConfigMapNodeConfigSource{ - UID: "", - }, - }, - }, - }, nil - }) - client.PrependReactor("get", "configmaps", func(action core.Action) (bool, runtime.Object, error) { - return true, &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.KubeletBaseConfigurationConfigMap, - Namespace: metav1.NamespaceSystem, - UID: "fake-uid", - }, - }, nil - }) - client.PrependReactor("patch", "nodes", func(action core.Action) (bool, runtime.Object, error) { - return true, nil, nil - }) - - if err := updateNodeWithConfigMap(client, nodeName); err != nil { - t.Errorf("UpdateNodeWithConfigMap: unexepected error %v", err) - } -} - -func TestCreateKubeletBaseConfigMapRBACRules(t *testing.T) { - client := fake.NewSimpleClientset() - client.PrependReactor("create", "roles", func(action core.Action) (bool, runtime.Object, error) { - return true, nil, nil - }) - client.PrependReactor("create", "rolebindings", func(action core.Action) (bool, runtime.Object, error) { - return true, nil, nil - }) - - if err := createKubeletBaseConfigMapRBACRules(client); err != nil { - t.Errorf("createKubeletBaseConfigMapRBACRules: unexepected error %v", err) - } -} diff --git a/cmd/kubeadm/app/phases/upgrade/configuration.go b/cmd/kubeadm/app/phases/upgrade/configuration.go deleted file mode 100644 index de798b2c3d..0000000000 --- a/cmd/kubeadm/app/phases/upgrade/configuration.go +++ /dev/null @@ -1,77 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "fmt" - "io" - "io/ioutil" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" -) - -// FetchConfiguration fetches configuration required for upgrading your cluster from a file (which has precedence) or a ConfigMap in the cluster -func FetchConfiguration(client clientset.Interface, w io.Writer, cfgPath string) (*kubeadmapi.MasterConfiguration, error) { - fmt.Println("[upgrade/config] Making sure the configuration is correct:") - - // Load the configuration from a file or the cluster - configBytes, err := loadConfigurationBytes(client, w, cfgPath) - if err != nil { - return nil, err - } - - // Take the versioned configuration populated from the file or configmap, convert it to internal, default and validate - versionedcfg, err := configutil.BytesToInternalConfig(configBytes) - if err != nil { - return nil, fmt.Errorf("could not decode configuration: %v", err) - } - return versionedcfg, nil -} - -// loadConfigurationBytes loads the configuration byte slice from either a file or the cluster ConfigMap -func loadConfigurationBytes(client clientset.Interface, w io.Writer, cfgPath string) ([]byte, error) { - // The config file has the highest priority - if cfgPath != "" { - fmt.Printf("[upgrade/config] Reading configuration options from a file: %s\n", cfgPath) - return ioutil.ReadFile(cfgPath) - } - - fmt.Println("[upgrade/config] Reading configuration from the cluster...") - - configMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(constants.MasterConfigurationConfigMap, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - fmt.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %s namespace must exist.\n", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem) - fmt.Println("[upgrade/config] Without this information, 'kubeadm upgrade' won't know how to configure your upgraded cluster.") - fmt.Println("") - fmt.Println("[upgrade/config] Next steps:") - fmt.Printf("\t- OPTION 1: Run 'kubeadm config upload from-flags' and specify the same CLI arguments you passed to 'kubeadm init' when you created your master.\n") - fmt.Printf("\t- OPTION 2: Run 'kubeadm config upload from-file' and specify the same config file you passed to 'kubeadm init' when you created your master.\n") - fmt.Printf("\t- OPTION 3: Pass a config file to 'kubeadm upgrade' using the --config flag.\n") - fmt.Println("") - return []byte{}, fmt.Errorf("the ConfigMap %q in the %s namespace used for getting configuration information was not found", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem) - } else if err != nil { - return []byte{}, fmt.Errorf("an unexpected error happened when trying to get the ConfigMap %q in the %s namespace: %v", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem, err) - } - - fmt.Printf("[upgrade/config] FYI: You can look at this config file with 'kubectl -n %s get cm %s -oyaml'\n", metav1.NamespaceSystem, constants.MasterConfigurationConfigMap) - return []byte(configMap.Data[constants.MasterConfigurationConfigMapKey]), nil -} diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go index 4672f22795..47a20ef70c 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go @@ -36,6 +36,7 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" + kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" @@ -102,6 +103,11 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterC } } + // Create the new, version-branched kubelet ComponentConfig ConfigMap + if err := kubeletphase.CreateConfigMap(cfg, client); err != nil { + errs = append(errs, fmt.Errorf("error creating kubelet configuration ConfigMap: %v", err)) + } + // Upgrade kube-dns/CoreDNS and kube-proxy if err := dns.EnsureDNSAddon(cfg, client); err != nil { errs = append(errs, err) diff --git a/cmd/kubeadm/app/preflight/checks.go b/cmd/kubeadm/app/preflight/checks.go index a7f9df241d..52dc8f9334 100644 --- a/cmd/kubeadm/app/preflight/checks.go +++ b/cmd/kubeadm/app/preflight/checks.go @@ -1043,10 +1043,11 @@ func TryStartKubelet(ignorePreflightErrors sets.String) { initSystem, err := initsystem.GetInitSystem() if err != nil { glog.Infoln("[preflight] no supported init system detected, won't ensure kubelet is running.") - } else if initSystem.ServiceExists("kubelet") && !initSystem.ServiceIsActive("kubelet") { + } else if initSystem.ServiceExists("kubelet") { - glog.Infoln("[preflight] starting the kubelet service") - if err := initSystem.ServiceStart("kubelet"); err != nil { + glog.Infoln("[preflight] Activating the kubelet service") + // This runs "systemctl daemon-reload && systemctl restart kubelet" + if err := initSystem.ServiceRestart("kubelet"); err != nil { glog.Warningf("[preflight] unable to start the kubelet service: [%v]\n", err) glog.Warningf("[preflight] please ensure kubelet is running manually.") } diff --git a/cmd/kubeadm/app/util/config/cluster.go b/cmd/kubeadm/app/util/config/cluster.go new file mode 100644 index 0000000000..f2ec1a777c --- /dev/null +++ b/cmd/kubeadm/app/util/config/cluster.go @@ -0,0 +1,65 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "fmt" + "io" + "io/ioutil" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" +) + +// TODO: Add unit tests for this file + +// FetchConfigFromFileOrCluster fetches configuration required for upgrading your cluster from a file (which has precedence) or a ConfigMap in the cluster +func FetchConfigFromFileOrCluster(client clientset.Interface, w io.Writer, logPrefix, cfgPath string) (*kubeadmapi.MasterConfiguration, error) { + // Load the configuration from a file or the cluster + configBytes, err := loadConfigurationBytes(client, w, logPrefix, cfgPath) + if err != nil { + return nil, err + } + + // Take the versioned configuration populated from the file or ConfigMap, convert it to internal, default and validate + return BytesToInternalConfig(configBytes) +} + +// loadConfigurationBytes loads the configuration byte slice from either a file or the cluster ConfigMap +func loadConfigurationBytes(client clientset.Interface, w io.Writer, logPrefix, cfgPath string) ([]byte, error) { + // The config file has the highest priority + if cfgPath != "" { + fmt.Fprintf(w, "[%s] Reading configuration options from a file: %s\n", logPrefix, cfgPath) + return ioutil.ReadFile(cfgPath) + } + + fmt.Fprintf(w, "[%s] Reading configuration from the cluster...\n", logPrefix) + + configMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(constants.MasterConfigurationConfigMap, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + // Return the apierror directly so the caller of this function can know what type of error occurred and act based on that + return []byte{}, err + } else if err != nil { + return []byte{}, fmt.Errorf("an unexpected error happened when trying to get the ConfigMap %q in the %s namespace: %v", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem, err) + } + + fmt.Fprintf(w, "[%s] FYI: You can look at this config file with 'kubectl -n %s get cm %s -oyaml'\n", logPrefix, metav1.NamespaceSystem, constants.MasterConfigurationConfigMap) + return []byte(configMap.Data[constants.MasterConfigurationConfigMapKey]), nil +} diff --git a/pkg/util/initsystem/initsystem.go b/pkg/util/initsystem/initsystem.go index e4f8870a36..aca7a41b86 100644 --- a/pkg/util/initsystem/initsystem.go +++ b/pkg/util/initsystem/initsystem.go @@ -29,6 +29,9 @@ type InitSystem interface { // ServiceStop tries to stop a specific service ServiceStop(service string) error + // ServiceRestart tries to reload the environment and restart the specific service + ServiceRestart(service string) error + // ServiceExists ensures the service is defined for this init system. ServiceExists(service string) bool @@ -47,6 +50,14 @@ func (sysd SystemdInitSystem) ServiceStart(service string) error { return err } +func (sysd SystemdInitSystem) ServiceRestart(service string) error { + if err := exec.Command("systemctl", "daemon-reload").Run(); err != nil { + return fmt.Errorf("failed to reload systemd: %v", err) + } + args := []string{"restart", service} + return exec.Command("systemctl", args...).Run() +} + func (sysd SystemdInitSystem) ServiceStop(service string) error { args := []string{"stop", service} err := exec.Command("systemctl", args...).Run() @@ -95,6 +106,16 @@ func (sysd WindowsInitSystem) ServiceStart(service string) error { return err } +func (sysd WindowsInitSystem) ServiceRestart(service string) error { + if err := sysd.ServiceStop(service); err != nil { + return fmt.Errorf("couldn't stop service: %v", err) + } + if err := sysd.ServiceStart(service); err != nil { + return fmt.Errorf("couldn't start service: %v", err) + } + return nil +} + func (sysd WindowsInitSystem) ServiceStop(service string) error { args := []string{"Stop-Service", service} err := exec.Command("powershell", args...).Run()