github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

kubeadm: Write kubelet config file to disk and persist in-cluster. Also write runtime environment file and fixup the kubelet phases command

pull/8/head
Lucas Käldström 2018-05-22 09:12:25 +03:00
parent a0036fcae1
commit 0aa0f3208a
No known key found for this signature in database
GPG Key ID: 3FA3783D77751514
21 changed files with 885 additions and 634 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go
View File

@ -68,9 +68,16 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
StaticPodPath: "foo",
ClusterDNS: []string{"foo"},
ClusterDomain: "foo",
Authorization: kubeletconfigv1beta1.KubeletAuthorization{Mode: "foo"},
Authorization: kubeletconfigv1beta1.KubeletAuthorization{
Mode: "Webhook",
},
Authentication: kubeletconfigv1beta1.KubeletAuthentication{
X509: kubeletconfigv1beta1.KubeletX509Authentication{ClientCAFile: "foo"},
X509: kubeletconfigv1beta1.KubeletX509Authentication{
ClientCAFile: "/etc/kubernetes/pki/ca.crt",
},
Anonymous: kubeletconfigv1beta1.KubeletAnonymousAuthentication{
Enabled: utilpointer.BoolPtr(false),
},
},
},
}

36
cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go
View File

@ -24,11 +24,11 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
kubeproxyscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"
kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
utilpointer "k8s.io/kubernetes/pkg/util/pointer"
)
const (
@ -143,9 +143,7 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) {
}
SetDefaultsEtcdSelfHosted(obj)
if features.Enabled(obj.FeatureGates, features.DynamicKubeletConfig) {
SetDefaults_KubeletConfiguration(obj)
}
SetDefaults_KubeletConfiguration(obj)
SetDefaults_ProxyConfiguration(obj)
SetDefaults_AuditPolicyConfiguration(obj)
}
@ -235,15 +233,31 @@ func SetDefaults_KubeletConfiguration(obj *MasterConfiguration) {
}
}
if obj.KubeletConfiguration.BaseConfig.ClusterDomain == "" {
obj.KubeletConfiguration.BaseConfig.ClusterDomain = DefaultServiceDNSDomain
}
if obj.KubeletConfiguration.BaseConfig.Authorization.Mode == "" {
obj.KubeletConfiguration.BaseConfig.Authorization.Mode = kubeletconfigv1beta1.KubeletAuthorizationModeWebhook
}
if obj.KubeletConfiguration.BaseConfig.Authentication.X509.ClientCAFile == "" {
obj.KubeletConfiguration.BaseConfig.Authentication.X509.ClientCAFile = DefaultCACertPath
obj.KubeletConfiguration.BaseConfig.ClusterDomain = obj.Networking.DNSDomain
}
// Enforce security-related kubelet options
// Require all clients to the kubelet API to have client certs signed by the cluster CA
obj.KubeletConfiguration.BaseConfig.Authentication.X509.ClientCAFile = DefaultCACertPath
obj.KubeletConfiguration.BaseConfig.Authentication.Anonymous.Enabled = utilpointer.BoolPtr(false)
// On every client request to the kubelet API, execute a webhook (SubjectAccessReview request) to the API server
// and ask it whether the client is authorized to access the kubelet API
obj.KubeletConfiguration.BaseConfig.Authorization.Mode = kubeletconfigv1beta1.KubeletAuthorizationModeWebhook
// Let clients using other authentication methods like ServiceAccount tokens also access the kubelet API
// TODO: Enable in a future PR
// obj.KubeletConfiguration.BaseConfig.Authentication.Webhook.Enabled = utilpointer.BoolPtr(true)
// Disable the readonly port of the kubelet, in order to not expose unnecessary information
// TODO: Enable in a future PR
// obj.KubeletConfiguration.BaseConfig.ReadOnlyPort = 0
// Serve a /healthz webserver on localhost:10248 that kubeadm can talk to
obj.KubeletConfiguration.BaseConfig.HealthzBindAddress = "127.0.0.1"
obj.KubeletConfiguration.BaseConfig.HealthzPort = utilpointer.Int32Ptr(10248)
scheme, _, _ := kubeletscheme.NewSchemeAndCodecs()
if scheme != nil {
scheme.Default(obj.KubeletConfiguration.BaseConfig)

36
cmd/kubeadm/app/apis/kubeadm/v1alpha2/defaults.go
View File

@ -23,11 +23,11 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
kubeproxyscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"
kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
utilpointer "k8s.io/kubernetes/pkg/util/pointer"
)
const (
@ -127,9 +127,7 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) {
obj.ClusterName = DefaultClusterName
}
if features.Enabled(obj.FeatureGates, features.DynamicKubeletConfig) {
SetDefaults_KubeletConfiguration(obj)
}
SetDefaults_KubeletConfiguration(obj)
SetDefaults_ProxyConfiguration(obj)
SetDefaults_AuditPolicyConfiguration(obj)
}
@ -198,15 +196,31 @@ func SetDefaults_KubeletConfiguration(obj *MasterConfiguration) {
}
}
if obj.KubeletConfiguration.BaseConfig.ClusterDomain == "" {
obj.KubeletConfiguration.BaseConfig.ClusterDomain = DefaultServiceDNSDomain
}
if obj.KubeletConfiguration.BaseConfig.Authorization.Mode == "" {
obj.KubeletConfiguration.BaseConfig.Authorization.Mode = kubeletconfigv1beta1.KubeletAuthorizationModeWebhook
}
if obj.KubeletConfiguration.BaseConfig.Authentication.X509.ClientCAFile == "" {
obj.KubeletConfiguration.BaseConfig.Authentication.X509.ClientCAFile = DefaultCACertPath
obj.KubeletConfiguration.BaseConfig.ClusterDomain = obj.Networking.DNSDomain
}
// Enforce security-related kubelet options
// Require all clients to the kubelet API to have client certs signed by the cluster CA
obj.KubeletConfiguration.BaseConfig.Authentication.X509.ClientCAFile = DefaultCACertPath
obj.KubeletConfiguration.BaseConfig.Authentication.Anonymous.Enabled = utilpointer.BoolPtr(false)
// On every client request to the kubelet API, execute a webhook (SubjectAccessReview request) to the API server
// and ask it whether the client is authorized to access the kubelet API
obj.KubeletConfiguration.BaseConfig.Authorization.Mode = kubeletconfigv1beta1.KubeletAuthorizationModeWebhook
// Let clients using other authentication methods like ServiceAccount tokens also access the kubelet API
// TODO: Enable in a future PR
// obj.KubeletConfiguration.BaseConfig.Authentication.Webhook.Enabled = utilpointer.BoolPtr(true)
// Disable the readonly port of the kubelet, in order to not expose unnecessary information
// TODO: Enable in a future PR
// obj.KubeletConfiguration.BaseConfig.ReadOnlyPort = 0
// Serve a /healthz webserver on localhost:10248 that kubeadm can talk to
obj.KubeletConfiguration.BaseConfig.HealthzBindAddress = "127.0.0.1"
obj.KubeletConfiguration.BaseConfig.HealthzPort = utilpointer.Int32Ptr(10248)
scheme, _, _ := kubeletscheme.NewSchemeAndCodecs()
if scheme != nil {
scheme.Default(obj.KubeletConfiguration.BaseConfig)

62
cmd/kubeadm/app/cmd/init.go
View File

@ -259,22 +259,29 @@ func NewInit(cfgPath string, externalcfg *kubeadmapiv1alpha2.MasterConfiguration
return nil, err
}
// Try to start the kubelet service in case it's inactive
glog.V(1).Infof("Starting kubelet")
preflight.TryStartKubelet(ignorePreflightErrors)
return &Init{cfg: cfg, skipTokenPrint: skipTokenPrint, dryRun: dryRun}, nil
return &Init{cfg: cfg, skipTokenPrint: skipTokenPrint, dryRun: dryRun, ignorePreflightErrors: ignorePreflightErrors}, nil
}
// Init defines struct used by "kubeadm init" command
type Init struct {
cfg *kubeadmapi.MasterConfiguration
skipTokenPrint bool
dryRun bool
cfg *kubeadmapi.MasterConfiguration
skipTokenPrint bool
dryRun bool
ignorePreflightErrors sets.String
}
// Run executes master node provisioning, including certificates, needed static pod manifests, etc.
func (i *Init) Run(out io.Writer) error {
// Write env file with flags for the kubelet to use
if err := kubeletphase.WriteKubeletDynamicEnvFile(i.cfg); err != nil {
return err
}
// Try to start the kubelet service in case it's inactive
glog.V(1).Infof("Starting kubelet")
preflight.TryStartKubelet(i.ignorePreflightErrors)
// Get directories to write files to; can be faked if we're dry-running
glog.V(1).Infof("[init] Getting certificates directory from configuration")
realCertsDir := i.cfg.CertificatesDir
@ -346,14 +353,14 @@ func (i *Init) Run(out io.Writer) error {
return fmt.Errorf("error printing files on dryrun: %v", err)
}
// NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) {
glog.V(1).Infof("[init] feature --dynamic-config-dir is enabled")
glog.V(1).Infof("[init] writing base kubelet configuration to disk on master")
// Write base kubelet configuration for dynamic kubelet configuration feature.
if err := kubeletphase.WriteInitKubeletConfigToDiskOnMaster(i.cfg); err != nil {
return fmt.Errorf("error writing base kubelet configuration to disk: %v", err)
}
kubeletVersion, err := preflight.GetKubeletVersion(utilsexec.New())
if err != nil {
return err
}
// Write the kubelet configuration to disk.
if err := kubeletphase.WriteConfigToDisk(i.cfg.KubeletConfiguration.BaseConfig, kubeletVersion); err != nil {
return fmt.Errorf("error writing kubelet configuration to disk: %v", err)
}
// Create a kubernetes client and wait for the API server to be healthy (if not dryrunning)
@ -381,15 +388,6 @@ func (i *Init) Run(out io.Writer) error {
return fmt.Errorf("couldn't initialize a Kubernetes cluster")
}
// NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) {
// Create base kubelet configuration for dynamic kubelet configuration feature.
glog.V(1).Infof("[init] creating base kubelet configuration")
if err := kubeletphase.CreateBaseKubeletConfiguration(i.cfg, client); err != nil {
return fmt.Errorf("error creating base kubelet configuration: %v", err)
}
}
// Upload currently used configuration to the cluster
// Note: This is done right in the beginning of cluster initialization; as we might want to make other phases
// depend on centralized information from this source in the future
@ -398,12 +396,26 @@ func (i *Init) Run(out io.Writer) error {
return fmt.Errorf("error uploading configuration: %v", err)
}
glog.V(1).Infof("[init] creating kubelet configuration configmap")
if err := kubeletphase.CreateConfigMap(i.cfg, client); err != nil {
return fmt.Errorf("error creating kubelet configuration ConfigMap: %v", err)
}
// PHASE 4: Mark the master with the right label/taint
glog.V(1).Infof("[init] marking the master with right label")
if err := markmasterphase.MarkMaster(client, i.cfg.NodeName, !i.cfg.NoTaintMaster); err != nil {
return fmt.Errorf("error marking master: %v", err)
}
// NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
// This feature is disabled by default, as it is alpha still
if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) {
// Enable dynamic kubelet configuration for the node.
if err := kubeletphase.EnableDynamicConfigForNode(client, i.cfg.NodeName, kubeletVersion); err != nil {
return fmt.Errorf("error enabling dynamic kubelet configuration: %v", err)
}
}
// PHASE 5: Set up the node bootstrap tokens
if !i.skipTokenPrint {
glog.Infof("[bootstraptoken] using token: %s\n", i.cfg.Token)

24
cmd/kubeadm/app/cmd/join.go
View File

@ -252,10 +252,28 @@ func (j *Join) Run(out io.Writer) error {
return fmt.Errorf("couldn't save the CA certificate to disk: %v", err)
}
// NOTE: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
glog.V(1).Infoln("[join] consuming base kubelet configuration")
kubeletVersion, err := preflight.GetKubeletVersion(utilsexec.New())
if err != nil {
return err
}
// Write the configuration for the kubelet down to disk so the kubelet can start
if err := kubeletphase.DownloadConfig(kubeconfigFile, kubeletVersion); err != nil {
return err
}
// Now the kubelet will perform the TLS Bootstrap, transforming bootstrap-kubeconfig.conf to kubeconfig.conf in /etc/kubernetes
// NOTE: the "--dynamic-config-dir" flag should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf for this to work
// This feature is disabled by default, as it is alpha still
glog.V(1).Infoln("[join] enabling dynamic kubelet configuration")
if features.Enabled(j.cfg.FeatureGates, features.DynamicKubeletConfig) {
if err := kubeletphase.ConsumeBaseKubeletConfiguration(j.cfg.NodeName); err != nil {
client, err := kubeletphase.GetLocalNodeTLSBootstrappedClient()
if err != nil {
return err
}
if err := kubeletphase.EnableDynamicConfigForNode(client, j.cfg.NodeName, kubeletVersion); err != nil {
return fmt.Errorf("error consuming base kubelet configuration: %v", err)
}
}

210
cmd/kubeadm/app/cmd/phases/kubelet.go
View File

@ -18,184 +18,172 @@ package phases
import (
"fmt"
"io/ioutil"
"os"
"github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/runtime"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
nodeutil "k8s.io/kubernetes/pkg/util/node"
"k8s.io/kubernetes/pkg/util/normalizer"
"k8s.io/kubernetes/pkg/util/version"
)
var (
kubeletWriteInitConfigLongDesc = normalizer.LongDesc(`
Writes init kubelet configuration to disk for dynamic kubelet configuration feature.
Please note that the kubelet configuration can be passed to kubeadm as a value into the master configuration file.
kubeletWriteConfigToDiskLongDesc = normalizer.LongDesc(`
Writes kubelet configuration to disk, either based on the kubelet-config-1.X ConfigMap in the cluster, or from the
configuration passed to the command via "--config".
` + cmdutil.AlphaDisclaimer)
kubeletWriteInitConfigExample = normalizer.Examples(`
# Writes init kubelet configuration to disk.
kubeadm alpha phase kubelet init
kubeletWriteConfigToDiskExample = normalizer.Examples(`
# Writes kubelet configuration for a node to disk. The information is fetched from the cluster ConfigMap
kubeadm alpha phase kubelet write-config-to-disk --kubelet-version v1.11.0 --kubeconfig /etc/kubernetes/kubelet.conf
# Writes kubelet configuration down to disk, based on the configuration flag passed to --config
kubeadm alpha phase kubelet write-config-to-disk --kubelet-version v1.11.0 --config kubeadm.yaml
`)
kubeletUploadDynamicConfigLongDesc = normalizer.LongDesc(`
Uploads dynamic kubelet configuration as ConfigMap and links it to the current node as ConfigMapRef.
Please note that the kubelet configuration can be passed to kubeadm as a value into the master configuration file.
Uploads kubelet configuration extracted from the kubeadm MasterConfiguration object to a ConfigMap
of the form kubelet-config-1.X in the cluster, where X is the minor version of the current Kubernetes version
` + cmdutil.AlphaDisclaimer)
kubeletUploadDynamicConfigExample = normalizer.Examples(`
# Uploads dynamic kubelet configuration as ConfigMap.
kubeadm alpha phase kubelet upload
# Uploads the kubelet configuration from the kubeadm Config file to a ConfigMap in the cluster.
kubeadm alpha phase kubelet upload-config --config kubeadm.yaml
`)
kubeletEnableDynamicConfigLongDesc = normalizer.LongDesc(`
Enables or updates dynamic kubelet configuration on node. This should be run on nodes.
Please note that the kubelet configuration can be passed to kubeadm as a value into the master configuration file.
Enables or updates dynamic kubelet configuration for a Node, against the kubelet-config-1.X ConfigMap in the cluster,
where X is the minor version of the desired kubelet version.
WARNING: This feature is still experimental, and disabled by default. Enable only if you know what you are doing, as it
may have surprising side-effects at this stage.
` + cmdutil.AlphaDisclaimer)
kubeletEnableDynamicConfigExample = normalizer.Examples(`
# Enables dynamic kubelet configuration on node.
kubeadm alpha phase kubelet enable
# Enables dynamic kubelet configuration for a Node.
kubeadm alpha phase kubelet enable-dynamic-config --node-name node-1 --kubelet-version v1.11.0
WARNING: This feature is still experimental, and disabled by default. Enable only if you know what you are doing, as it
may have surprising side-effects at this stage.
`)
)
// NewCmdKubelet returns main command for Kubelet phase
func NewCmdKubelet() *cobra.Command {
var kubeConfigFile string
cmd := &cobra.Command{
Use: "kubelet",
Short: "Adopts dynamic kubelet configuration.",
Short: "Handles kubelet configuration.",
Long: cmdutil.MacroCommandLongDescription,
}
cmd.AddCommand(NewCmdKubeletWriteInitConfig())
cmd.AddCommand(NewCmdKubeletUploadDynamicConfig())
cmd.AddCommand(NewCmdKubeletEnableDynamicConfig())
cmd.PersistentFlags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use when talking to the cluster")
cmd.AddCommand(NewCmdKubeletWriteConfigToDisk(&kubeConfigFile))
cmd.AddCommand(NewCmdKubeletUploadConfig(&kubeConfigFile))
cmd.AddCommand(NewCmdKubeletEnableDynamicConfig(&kubeConfigFile))
return cmd
}
// NewCmdKubeletWriteInitConfig calls cobra.Command for writing init kubelet configuration
func NewCmdKubeletWriteInitConfig() *cobra.Command {
// NewCmdKubeletUploadConfig calls cobra.Command for uploading dynamic kubelet configuration
func NewCmdKubeletUploadConfig(kubeConfigFile *string) *cobra.Command {
var cfgPath string
cmd := &cobra.Command{
Use: "init",
Short: "Writes init kubelet configuration to disk",
Long: kubeletWriteInitConfigLongDesc,
Example: kubeletWriteInitConfigExample,
Run: func(cmd *cobra.Command, args []string) {
cfg := &kubeadmapiv1alpha2.MasterConfiguration{
// KubernetesVersion is not used by kubelet init, but we set this explicitly to avoid
// the lookup of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig
KubernetesVersion: "v1.9.0",
}
kubeadmscheme.Scheme.Default(cfg)
// This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags
internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg)
kubeadmutil.CheckErr(err)
if features.Enabled(internalcfg.FeatureGates, features.DynamicKubeletConfig) {
err = kubeletphase.WriteInitKubeletConfigToDiskOnMaster(internalcfg)
kubeadmutil.CheckErr(err)
} else {
fmt.Println("[kubelet] feature gate DynamicKubeletConfig is not enabled, do nothing.")
}
},
}
cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")
return cmd
}
// NewCmdKubeletUploadDynamicConfig calls cobra.Command for uploading dynamic kubelet configuration
func NewCmdKubeletUploadDynamicConfig() *cobra.Command {
var cfgPath, kubeConfigFile string
cmd := &cobra.Command{
Use: "upload",
Short: "Uploads dynamic kubelet configuration as ConfigMap",
Use: "upload-config",
Short: "Uploads kubelet configuration to a ConfigMap",
Long: kubeletUploadDynamicConfigLongDesc,
Example: kubeletUploadDynamicConfigExample,
Run: func(cmd *cobra.Command, args []string) {
cfg := &kubeadmapiv1alpha2.MasterConfiguration{
// KubernetesVersion is not used by kubelet upload, but we set this explicitly to avoid
// the lookup of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig
KubernetesVersion: "v1.9.0",
if len(cfgPath) == 0 {
kubeadmutil.CheckErr(fmt.Errorf("The --config argument is required"))
}
kubeadmscheme.Scheme.Default(cfg)
// This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags
internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg)
// This call returns the ready-to-use configuration based on the configuration file
internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, &kubeadmapiv1alpha2.MasterConfiguration{})
kubeadmutil.CheckErr(err)
client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile)
kubeadmutil.CheckErr(err)
err = kubeletphase.CreateConfigMap(internalcfg, client)
kubeadmutil.CheckErr(err)
if features.Enabled(internalcfg.FeatureGates, features.DynamicKubeletConfig) {
client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile)
kubeadmutil.CheckErr(err)
err = kubeletphase.CreateBaseKubeletConfiguration(internalcfg, client)
kubeadmutil.CheckErr(err)
} else {
fmt.Println("[kubelet] feature gate DynamicKubeletConfig is not enabled, do nothing.")
}
},
}
cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")
cmd.Flags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use when talking to the cluster")
return cmd
}
// NewCmdKubeletWriteConfigToDisk calls cobra.Command for writing init kubelet configuration
func NewCmdKubeletWriteConfigToDisk(kubeConfigFile *string) *cobra.Command {
var cfgPath, kubeletVersionStr string
cmd := &cobra.Command{
Use: "write-config-to-disk",
Short: "Writes kubelet configuration to disk, either based on the --config argument or the kubeadm-config ConfigMap.",
Long: kubeletWriteConfigToDiskLongDesc,
Example: kubeletWriteConfigToDiskExample,
Run: func(cmd *cobra.Command, args []string) {
if len(kubeletVersionStr) == 0 {
kubeadmutil.CheckErr(fmt.Errorf("The --kubelet-version argument is required"))
}
kubeletVersion, err := version.ParseSemantic(kubeletVersionStr)
kubeadmutil.CheckErr(err)
client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile)
kubeadmutil.CheckErr(err)
// This call returns the ready-to-use configuration based on the configuration file
internalcfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "kubelet", cfgPath)
kubeadmutil.CheckErr(err)
err = kubeletphase.WriteConfigToDisk(internalcfg.KubeletConfiguration.BaseConfig, kubeletVersion)
kubeadmutil.CheckErr(err)
},
}
cmd.Flags().StringVar(&kubeletVersionStr, "kubelet-version", kubeletVersionStr, "The desired version for the kubelet")
cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")
return cmd
}
// NewCmdKubeletEnableDynamicConfig calls cobra.Command for enabling dynamic kubelet configuration on node
func NewCmdKubeletEnableDynamicConfig() *cobra.Command {
cfg := &kubeadmapiv1alpha2.NodeConfiguration{}
kubeadmscheme.Scheme.Default(cfg)
// This feature is still in alpha and an experimental state
func NewCmdKubeletEnableDynamicConfig(kubeConfigFile *string) *cobra.Command {
var nodeName, kubeletVersionStr string
var cfgPath string
cmd := &cobra.Command{
Use: "enable",
Aliases: []string{"update"},
Short: "Enables or updates dynamic kubelet configuration on node",
Use: "enable-dynamic-config",
Short: "EXPERIMENTAL: Enables or updates dynamic kubelet configuration for a Node",
Long: kubeletEnableDynamicConfigLongDesc,
Example: kubeletEnableDynamicConfigExample,
Run: func(cmd *cobra.Command, args []string) {
nodeName, err := getNodeName(cfgPath, cfg)
kubeadmutil.CheckErr(err)
if features.Enabled(cfg.FeatureGates, features.DynamicKubeletConfig) {
err = kubeletphase.ConsumeBaseKubeletConfiguration(nodeName)
kubeadmutil.CheckErr(err)
} else {
fmt.Println("[kubelet] feature gate DynamicKubeletConfig is not enabled, do nothing.")
if len(nodeName) == 0 {
kubeadmutil.CheckErr(fmt.Errorf("The --node-name argument is required"))
}
if len(kubeletVersionStr) == 0 {
kubeadmutil.CheckErr(fmt.Errorf("The --kubelet-version argument is required"))
}
kubeletVersion, err := version.ParseSemantic(kubeletVersionStr)
kubeadmutil.CheckErr(err)
client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile)
kubeadmutil.CheckErr(err)
err = kubeletphase.EnableDynamicConfigForNode(client, nodeName, kubeletVersion)
kubeadmutil.CheckErr(err)
},
}
cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")
cmd.Flags().StringVar(&cfg.NodeName, "node-name", cfg.NodeName, "Name of the node that should enable the dynamic kubelet configuration")
cmd.Flags().StringVar(&nodeName, "node-name", nodeName, "Name of the node that should enable the dynamic kubelet configuration")
cmd.Flags().StringVar(&kubeletVersionStr, "kubelet-version", kubeletVersionStr, "The desired version for the kubelet")
return cmd
}
func getNodeName(cfgPath string, cfg *kubeadmapiv1alpha2.NodeConfiguration) (string, error) {
if cfgPath != "" {
b, err := ioutil.ReadFile(cfgPath)
if err != nil {
return "", fmt.Errorf("unable to read config from %q [%v]", cfgPath, err)
}
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), b, cfg); err != nil {
return "", fmt.Errorf("unable to decode config from %q [%v]", cfgPath, err)
}
}
if cfg.NodeName == "" {
cfg.NodeName = nodeutil.GetHostname("")
}
return cfg.NodeName, nil
}

28
cmd/kubeadm/app/cmd/phases/kubelet_test.go
View File

@ -25,33 +25,37 @@ import (
)
func TestKubeletSubCommandsHasFlags(t *testing.T) {
kubeConfigFile := "foo"
subCmds := []*cobra.Command{
NewCmdKubeletWriteInitConfig(),
NewCmdKubeletUploadDynamicConfig(),
NewCmdKubeletEnableDynamicConfig(),
NewCmdKubeletUploadConfig(&kubeConfigFile),
NewCmdKubeletWriteConfigToDisk(&kubeConfigFile),
NewCmdKubeletEnableDynamicConfig(&kubeConfigFile),
}
commonFlags := []string{
"config",
}
commonFlags := []string{}
var tests = []struct {
command string
additionalFlags []string
}{
{
command: "init",
},
{
command: "upload",
command: "upload-config",
additionalFlags: []string{
"kubeconfig",
"config",
},
},
{
command: "enable",
command: "write-config-to-disk",
additionalFlags: []string{
"kubelet-version",
"config",
},
},
{
command: "enable-dynamic-config",
additionalFlags: []string{
"node-name",
"kubelet-version",
},
},
}

18
cmd/kubeadm/app/cmd/upgrade/common.go
View File

@ -24,17 +24,21 @@ import (
"os"
"strings"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
fakediscovery "k8s.io/client-go/discovery/fake"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
)
@ -61,8 +65,20 @@ func enforceRequirements(flags *cmdUpgradeFlags, dryRun bool, newK8sVersion stri
}
// Fetch the configuration from a file or ConfigMap and validate it
cfg, err := upgrade.FetchConfiguration(client, os.Stdout, flags.cfgPath)
fmt.Println("[upgrade/config] Making sure the configuration is correct:")
cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "upgrade/config", flags.cfgPath)
if err != nil {
if apierrors.IsNotFound(err) {
fmt.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %s namespace must exist.\n", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem)
fmt.Println("[upgrade/config] Without this information, 'kubeadm upgrade' won't know how to configure your upgraded cluster.")
fmt.Println("")
fmt.Println("[upgrade/config] Next steps:")
fmt.Printf("\t- OPTION 1: Run 'kubeadm config upload from-flags' and specify the same CLI arguments you passed to 'kubeadm init' when you created your master.\n")
fmt.Printf("\t- OPTION 2: Run 'kubeadm config upload from-file' and specify the same config file you passed to 'kubeadm init' when you created your master.\n")
fmt.Printf("\t- OPTION 3: Pass a config file to 'kubeadm upgrade' using the --config flag.\n")
fmt.Println("")
err = fmt.Errorf("the ConfigMap %q in the %s namespace used for getting configuration information was not found", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem)
}
return nil, fmt.Errorf("[upgrade/config] FATAL: %v", err)
}

26
cmd/kubeadm/app/constants/constants.go
View File

@ -161,9 +161,6 @@ const (
// system:nodes group subject is removed if present.
NodesClusterRoleBinding = "system:node"
// KubeletBaseConfigMapRoleName defines the base kubelet configuration ConfigMap.
KubeletBaseConfigMapRoleName = "kubeadm:kubelet-base-configmap"
// APICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation
APICallRetryInterval = 500 * time.Millisecond
// DiscoveryRetryInterval specifies how long kubeadm should wait before retrying to connect to the master when doing discovery
@ -191,17 +188,17 @@ const (
// MasterConfigurationConfigMapKey specifies in what ConfigMap key the master configuration should be stored
MasterConfigurationConfigMapKey = "MasterConfiguration"
// KubeletBaseConfigurationConfigMap specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored
KubeletBaseConfigurationConfigMap = "kubelet-base-config-1.9"
// KubeletBaseConfigurationConfigMapPrefix specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored
KubeletBaseConfigurationConfigMapPrefix = "kubelet-config-"
// KubeletBaseConfigurationConfigMapKey specifies in what ConfigMap key the initial remote configuration of kubelet should be stored
KubeletBaseConfigurationConfigMapKey = "kubelet"
// KubeletBaseConfigurationDir specifies the directory on the node where stores the initial remote configuration of kubelet
KubeletBaseConfigurationDir = "/var/lib/kubelet/config/init"
// KubeletBaseConfigMapRolePrefix defines the base kubelet configuration ConfigMap.
KubeletBaseConfigMapRolePrefix = "kubeadm:kubelet-config-"
// KubeletBaseConfigurationFile specifies the file name on the node which stores initial remote configuration of kubelet
KubeletBaseConfigurationFile = "kubelet"
// KubeletConfigurationFile specifies the file name on the node which stores initial remote configuration of kubelet
KubeletConfigurationFile = "/var/lib/kubelet/config.yaml"
// MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports
MinExternalEtcdVersion = "3.2.17"
@ -260,6 +257,14 @@ const (
// Copied from pkg/master/reconcilers to avoid pulling extra dependencies
// TODO: Import this constant from a consts only package, that does not pull any further dependencies.
LeaseEndpointReconcilerType = "lease"
// KubeletEnvFile is a file "kubeadm init" writes at runtime. Using that interface, kubeadm can customize certain
// kubelet flags conditionally based on the environment at runtime. Also, parameters given to the configuration file
// might be passed through this file. "kubeadm init" writes one variable, with the name ${KubeletEnvFileVariableName}.
KubeletEnvFile = "/var/lib/kubelet/kubeadm-flags.env"
// KubeletEnvFileVariableName specifies the shell script variable name "kubeadm init" should write a value to in KubeletEnvFile
KubeletEnvFileVariableName = "KUBELET_KUBEADM_ARGS"
)
var (
@ -290,6 +295,9 @@ var (
// MinimumKubeletVersion specifies the minimum version of kubelet which kubeadm supports
MinimumKubeletVersion = version.MustParseSemantic("v1.10.0")
// MinimumKubeletConfigVersion specifies the minimum version of Kubernetes where kubeadm supports specifying --config to the kubelet
MinimumKubeletConfigVersion = version.MustParseSemantic("v1.11.0-alpha.1")
// SupportedEtcdVersion lists officially supported etcd versions with corresponding kubernetes releases
SupportedEtcdVersion = map[uint8]string{
10: "3.1.12",

187
cmd/kubeadm/app/phases/kubelet/config.go Normal file
View File

@ -0,0 +1,187 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"fmt"
"io/ioutil"
"k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
"k8s.io/kubernetes/pkg/util/version"
)
// WriteConfigToDisk writes the kubelet config object down to a file
// Used at "kubeadm init" and "kubeadm upgrade" time
func WriteConfigToDisk(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration, kubeletVersion *version.Version) error {
// If the kubelet version is v1.10.x, exit
if kubeletVersion.LessThan(kubeadmconstants.MinimumKubeletConfigVersion) {
return nil
}
kubeletBytes, err := getConfigBytes(kubeletConfig)
if err != nil {
return err
}
return writeConfigBytesToDisk(kubeletBytes)
}
// CreateConfigMap creates a ConfigMap with the generic kubelet configuration.
// Used at "kubeadm init" and "kubeadm upgrade" time
func CreateConfigMap(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
if err != nil {
return err
}
// If Kubernetes version is v1.10.x, exit
if k8sVersion.LessThan(kubeadmconstants.MinimumKubeletConfigVersion) {
return nil
}
configMapName := configMapName(k8sVersion)
fmt.Printf("[kubelet] Creating a ConfigMap %q in namespace %s with the configuration for the kubelets in the cluster\n", configMapName, metav1.NamespaceSystem)
kubeletBytes, err := getConfigBytes(cfg.KubeletConfiguration.BaseConfig)
if err != nil {
return err
}
if err := apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: configMapName,
Namespace: metav1.NamespaceSystem,
},
Data: map[string]string{
kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(kubeletBytes),
},
}); err != nil {
return err
}
if err := createConfigMapRBACRules(client, k8sVersion); err != nil {
return fmt.Errorf("error creating kubelet configuration configmap RBAC rules: %v", err)
}
return nil
}
// createConfigMapRBACRules creates the RBAC rules for exposing the base kubelet ConfigMap in the kube-system namespace to unauthenticated users
func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Version) error {
if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{
ObjectMeta: metav1.ObjectMeta{
Name: configMapRBACName(k8sVersion),
Namespace: metav1.NamespaceSystem,
},
Rules: []rbac.PolicyRule{
rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(configMapName(k8sVersion)).RuleOrDie(),
},
}); err != nil {
return err
}
return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: configMapRBACName(k8sVersion),
Namespace: metav1.NamespaceSystem,
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "Role",
Name: configMapRBACName(k8sVersion),
},
Subjects: []rbac.Subject{
{
Kind: rbac.GroupKind,
Name: kubeadmconstants.NodesGroup,
},
{
Kind: rbac.GroupKind,
Name: kubeadmconstants.NodeBootstrapTokenAuthGroup,
},
},
})
}
// DownloadConfig downloads the kubelet configuration from a ConfigMap and writes it to disk.
// Used at "kubeadm join" time
func DownloadConfig(kubeletKubeConfig string, kubeletVersion *version.Version) error {
// If the kubelet version is v1.10.x, exit
if kubeletVersion.LessThan(kubeadmconstants.MinimumKubeletConfigVersion) {
return nil
}
// Download the ConfigMap from the cluster based on what version the kubelet is
configMapName := configMapName(kubeletVersion)
fmt.Printf("[kubelet] Downloading configuration for the kubelet from the %q ConfigMap in the %s namespace\n",
configMapName, metav1.NamespaceSystem)
client, err := kubeconfigutil.ClientSetFromFile(kubeletKubeConfig)
if err != nil {
return fmt.Errorf("couldn't create client from kubeconfig file %q", kubeletKubeConfig)
}
kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{})
if err != nil {
return err
}
return writeConfigBytesToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey]))
}
// configMapName returns the right ConfigMap name for the right branch of k8s
func configMapName(k8sVersion *version.Version) string {
return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigurationConfigMapPrefix, k8sVersion.Major(), k8sVersion.Minor())
}
// configMapRBACName returns the name for the Role/RoleBinding for the kubelet config configmap for the right branch of k8s
func configMapRBACName(k8sVersion *version.Version) string {
return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigMapRolePrefix, k8sVersion.Major(), k8sVersion.Minor())
}
// getConfigBytes marshals a kubeletconfiguration object to bytes
func getConfigBytes(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) ([]byte, error) {
_, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs()
if err != nil {
return []byte{}, err
}
return kubeadmutil.MarshalToYamlForCodecs(kubeletConfig, kubeletconfigv1beta1.SchemeGroupVersion, *kubeletCodecs)
}
// writeConfigBytesToDisk writes a byte slice down to disk at the specific location of the kubelet config file
func writeConfigBytesToDisk(b []byte) error {
fmt.Printf("[kubelet] Writing kubelet configuration to file %q\n", kubeadmconstants.KubeletConfigurationFile)
if err := ioutil.WriteFile(kubeadmconstants.KubeletConfigurationFile, b, 0644); err != nil {
return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", kubeadmconstants.KubeletConfigurationFile, err)
}
return nil
}

78
cmd/kubeadm/app/phases/kubelet/config_test.go Normal file
View File

@ -0,0 +1,78 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"testing"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
"k8s.io/kubernetes/pkg/util/version"
)
func TestCreateConfigMap(t *testing.T) {
nodeName := "fake-node"
client := fake.NewSimpleClientset()
cfg := &kubeadmapi.MasterConfiguration{
NodeName: nodeName,
KubernetesVersion: "v1.11.0",
KubeletConfiguration: kubeadmapi.KubeletConfiguration{
BaseConfig: &kubeletconfigv1beta1.KubeletConfiguration{},
},
}
client.PrependReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) {
return true, &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: nodeName,
},
Spec: v1.NodeSpec{},
}, nil
})
client.PrependReactor("create", "roles", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
client.PrependReactor("create", "rolebindings", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
client.PrependReactor("create", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
if err := CreateConfigMap(cfg, client); err != nil {
t.Errorf("CreateConfigMap: unexpected error %v", err)
}
}
func TestCreateConfigMapRBACRules(t *testing.T) {
client := fake.NewSimpleClientset()
client.PrependReactor("create", "roles", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
client.PrependReactor("create", "rolebindings", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
if err := createConfigMapRBACRules(client, version.MustParseSemantic("v1.11.0")); err != nil {
t.Errorf("createConfigMapRBACRules: unexpected error %v", err)
}
}

117
cmd/kubeadm/app/phases/kubelet/dynamic.go Normal file
View File

@ -0,0 +1,117 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"k8s.io/api/core/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/strategicpatch"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
"k8s.io/kubernetes/pkg/util/version"
)
// EnableDynamicConfigForNode updates the Node's ConfigSource to enable Dynamic Kubelet Configuration, depending on what version the kubelet is
// Used at "kubeadm init", "kubeadm join" and "kubeadm upgrade" time
// This func is ONLY run if the user enables the `DynamicKubeletConfig` feature gate, which is by default off
func EnableDynamicConfigForNode(client clientset.Interface, nodeName string, kubeletVersion *version.Version) error {
// If the kubelet version is v1.10.x, exit
if kubeletVersion.LessThan(kubeadmconstants.MinimumKubeletConfigVersion) {
return nil
}
configMapName := configMapName(kubeletVersion)
fmt.Printf("[kubelet] Enabling Dynamic Kubelet Config for Node %q; config sourced from ConfigMap %q in namespace %s\n",
nodeName, configMapName, metav1.NamespaceSystem)
fmt.Println("[kubelet] WARNING: The Dynamic Kubelet Config feature is alpha and off by default. It hasn't been well-tested yet at this stage, use with caution.")
// Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned.
return wait.Poll(kubeadmconstants.APICallRetryInterval, kubeadmconstants.UpdateNodeTimeout, func() (bool, error) {
node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
if err != nil {
return false, nil
}
oldData, err := json.Marshal(node)
if err != nil {
return false, err
}
kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{})
if err != nil {
return false, nil
}
node.Spec.ConfigSource = &v1.NodeConfigSource{
ConfigMap: &v1.ConfigMapNodeConfigSource{
Name: configMapName,
Namespace: metav1.NamespaceSystem,
UID: kubeletCfg.UID,
KubeletConfigKey: kubeadmconstants.KubeletBaseConfigurationConfigMapKey,
},
}
newData, err := json.Marshal(node)
if err != nil {
return false, err
}
patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{})
if err != nil {
return false, err
}
if _, err := client.CoreV1().Nodes().Patch(node.Name, types.StrategicMergePatchType, patchBytes); err != nil {
if apierrs.IsConflict(err) {
fmt.Println("Temporarily unable to update node metadata due to conflict (will retry)")
return false, nil
}
return false, err
}
return true, nil
})
}
// GetLocalNodeTLSBootstrappedClient waits for the kubelet to perform the TLS bootstrap
// and then creates a client from config file /etc/kubernetes/kubelet.conf
func GetLocalNodeTLSBootstrappedClient() (clientset.Interface, error) {
fmt.Println("[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...")
kubeletKubeConfig := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)
// Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned.
err := wait.PollImmediateInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
_, err := os.Stat(kubeletKubeConfig)
return (err == nil), nil
})
if err != nil {
return nil, err
}
return kubeconfigutil.ClientSetFromFile(kubeletKubeConfig)
}

63
cmd/kubeadm/app/phases/kubelet/dynamic_test.go Normal file
View File

@ -0,0 +1,63 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"testing"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
"k8s.io/kubernetes/pkg/util/version"
)
func TestEnableDynamicConfigForNode(t *testing.T) {
nodeName := "fake-node"
client := fake.NewSimpleClientset()
client.PrependReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) {
return true, &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: nodeName,
},
Spec: v1.NodeSpec{
ConfigSource: &v1.NodeConfigSource{
ConfigMap: &v1.ConfigMapNodeConfigSource{
UID: "",
},
},
},
}, nil
})
client.PrependReactor("get", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
return true, &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "kubelet-config-1.11",
Namespace: metav1.NamespaceSystem,
UID: "fake-uid",
},
}, nil
})
client.PrependReactor("patch", "nodes", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
if err := EnableDynamicConfigForNode(client, nodeName, version.MustParseSemantic("v1.11.0")); err != nil {
t.Errorf("UpdateNodeWithConfigMap: unexpected error %v", err)
}
}

78
cmd/kubeadm/app/phases/kubelet/flags.go Normal file
View File

@ -0,0 +1,78 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
)
// WriteKubeletDynamicEnvFile writes a environment file with dynamic flags to the kubelet.
// Used at "kubeadm init" and "kubeadm join" time.
func WriteKubeletDynamicEnvFile(cfg *kubeadmapi.MasterConfiguration) error {
// TODO: Pass through extra arguments from the config file here in the future
argList := kubeadmutil.BuildArgumentListFromMap(buildKubeletArgMap(cfg), map[string]string{})
envFileContent := fmt.Sprintf("%s=%s\n", constants.KubeletEnvFileVariableName, strings.Join(argList, " "))
return writeKubeletFlagBytesToDisk([]byte(envFileContent))
}
// buildKubeletArgMap takes a MasterConfiguration object and builds based on that a string-string map with flags
// that should be given to the local kubelet daemon.
func buildKubeletArgMap(cfg *kubeadmapi.MasterConfiguration) map[string]string {
kubeletFlags := map[string]string{}
if cfg.CRISocket == kubeadmapiv1alpha2.DefaultCRISocket {
// These flags should only be set when running docker
kubeletFlags["network-plugin"] = "cni"
kubeletFlags["cni-conf-dir"] = "/etc/cni/net.d"
kubeletFlags["cni-bin-dir"] = "/opt/cni/bin"
} else {
kubeletFlags["container-runtime"] = "remote"
kubeletFlags["container-runtime-endpoint"] = cfg.CRISocket
}
// TODO: Add support for registering custom Taints and Labels
// TODO: Add support for overriding flags with ExtraArgs
// TODO: Pass through --hostname-override if a custom name is used?
// TODO: Check if `systemd-resolved` is running, and set `--resolv-conf` based on that
// TODO: Conditionally set `--cgroup-driver` to either `systemd` or `cgroupfs`
return kubeletFlags
}
// writeKubeletFlagBytesToDisk writes a byte slice down to disk at the specific location of the kubelet flag overrides file
func writeKubeletFlagBytesToDisk(b []byte) error {
fmt.Printf("[kubelet] Writing kubelet environment file with flags to file %q\n", constants.KubeletEnvFile)
// creates target folder if not already exists
if err := os.MkdirAll(filepath.Dir(constants.KubeletEnvFile), 0700); err != nil {
return fmt.Errorf("failed to create directory %q: %v", filepath.Dir(constants.KubeletEnvFile), err)
}
if err := ioutil.WriteFile(constants.KubeletEnvFile, b, 0644); err != nil {
return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", constants.KubeletEnvFile, err)
}
return nil
}

235
cmd/kubeadm/app/phases/kubelet/kubelet.go
View File

@ -1,235 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/strategicpatch"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
)
// CreateBaseKubeletConfiguration creates base kubelet configuration for dynamic kubelet configuration feature.
func CreateBaseKubeletConfiguration(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
fmt.Printf("[kubelet] Uploading a ConfigMap %q in namespace %s with base configuration for the kubelets in the cluster\n",
kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.NamespaceSystem)
_, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs()
if err != nil {
return err
}
kubeletBytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg.KubeletConfiguration.BaseConfig, kubeletconfigv1beta1.SchemeGroupVersion, *kubeletCodecs)
if err != nil {
return err
}
if err = apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: kubeadmconstants.KubeletBaseConfigurationConfigMap,
Namespace: metav1.NamespaceSystem,
},
Data: map[string]string{
kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(kubeletBytes),
},
}); err != nil {
return err
}
if err := createKubeletBaseConfigMapRBACRules(client); err != nil {
return fmt.Errorf("error creating base kubelet configmap RBAC rules: %v", err)
}
return updateNodeWithConfigMap(client, cfg.NodeName)
}
// ConsumeBaseKubeletConfiguration consumes base kubelet configuration for dynamic kubelet configuration feature.
func ConsumeBaseKubeletConfiguration(nodeName string) error {
client, err := getLocalNodeTLSBootstrappedClient()
if err != nil {
return err
}
kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.GetOptions{})
if err != nil {
return err
}
if err := writeInitKubeletConfigToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey])); err != nil {
return fmt.Errorf("failed to write initial remote configuration of kubelet to disk for node %s: %v", nodeName, err)
}
return updateNodeWithConfigMap(client, nodeName)
}
// updateNodeWithConfigMap updates node ConfigSource with KubeletBaseConfigurationConfigMap
func updateNodeWithConfigMap(client clientset.Interface, nodeName string) error {
fmt.Printf("[kubelet] Using Dynamic Kubelet Config for node %q; config sourced from ConfigMap %q in namespace %s\n",
nodeName, kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.NamespaceSystem)
// Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned.
return wait.Poll(kubeadmconstants.APICallRetryInterval, kubeadmconstants.UpdateNodeTimeout, func() (bool, error) {
node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
if err != nil {
return false, nil
}
oldData, err := json.Marshal(node)
if err != nil {
return false, err
}
kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.GetOptions{})
if err != nil {
return false, nil
}
node.Spec.ConfigSource = &v1.NodeConfigSource{
ConfigMap: &v1.ConfigMapNodeConfigSource{
Name: kubeadmconstants.KubeletBaseConfigurationConfigMap,
Namespace: metav1.NamespaceSystem,
UID: kubeletCfg.UID,
KubeletConfigKey: kubeadmconstants.KubeletBaseConfigurationConfigMapKey,
},
}
newData, err := json.Marshal(node)
if err != nil {
return false, err
}
patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{})
if err != nil {
return false, err
}
if _, err := client.CoreV1().Nodes().Patch(node.Name, types.StrategicMergePatchType, patchBytes); err != nil {
if apierrs.IsConflict(err) {
fmt.Println("Temporarily unable to update node metadata due to conflict (will retry)")
return false, nil
}
return false, err
}
return true, nil
})
}
// createKubeletBaseConfigMapRBACRules creates the RBAC rules for exposing the base kubelet ConfigMap in the kube-system namespace to unauthenticated users
func createKubeletBaseConfigMapRBACRules(client clientset.Interface) error {
if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{
ObjectMeta: metav1.ObjectMeta{
Name: kubeadmconstants.KubeletBaseConfigMapRoleName,
Namespace: metav1.NamespaceSystem,
},
Rules: []rbac.PolicyRule{
rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(kubeadmconstants.KubeletBaseConfigurationConfigMap).RuleOrDie(),
},
}); err != nil {
return err
}
return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: kubeadmconstants.KubeletBaseConfigMapRoleName,
Namespace: metav1.NamespaceSystem,
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "Role",
Name: kubeadmconstants.KubeletBaseConfigMapRoleName,
},
Subjects: []rbac.Subject{
{
Kind: rbac.GroupKind,
Name: kubeadmconstants.NodesGroup,
},
{
Kind: rbac.GroupKind,
Name: kubeadmconstants.NodeBootstrapTokenAuthGroup,
},
},
})
}
// getLocalNodeTLSBootstrappedClient waits for the kubelet to perform the TLS bootstrap
// and then creates a client from config file /etc/kubernetes/kubelet.conf
func getLocalNodeTLSBootstrappedClient() (clientset.Interface, error) {
fmt.Println("[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...")
kubeletKubeConfig := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)
// Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned.
err := wait.PollImmediateInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
_, err := os.Stat(kubeletKubeConfig)
return (err == nil), nil
})
if err != nil {
return nil, err
}
return kubeconfigutil.ClientSetFromFile(kubeletKubeConfig)
}
// WriteInitKubeletConfigToDiskOnMaster writes base kubelet configuration to disk on master.
func WriteInitKubeletConfigToDiskOnMaster(cfg *kubeadmapi.MasterConfiguration) error {
fmt.Printf("[kubelet] Writing base configuration of kubelets to disk on master node %s\n", cfg.NodeName)
_, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs()
if err != nil {
return err
}
kubeletBytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg.KubeletConfiguration.BaseConfig, kubeletconfigv1beta1.SchemeGroupVersion, *kubeletCodecs)
if err != nil {
return err
}
if err := writeInitKubeletConfigToDisk(kubeletBytes); err != nil {
return fmt.Errorf("failed to write base configuration of kubelet to disk on master node %s: %v", cfg.NodeName, err)
}
return nil
}
func writeInitKubeletConfigToDisk(kubeletConfig []byte) error {
if err := os.MkdirAll(kubeadmconstants.KubeletBaseConfigurationDir, 0644); err != nil {
return fmt.Errorf("failed to create directory %q: %v", kubeadmconstants.KubeletBaseConfigurationDir, err)
}
baseConfigFile := filepath.Join(kubeadmconstants.KubeletBaseConfigurationDir, kubeadmconstants.KubeletBaseConfigurationFile)
if err := ioutil.WriteFile(baseConfigFile, kubeletConfig, 0644); err != nil {
return fmt.Errorf("failed to write initial remote configuration of kubelet into file %q: %v", baseConfigFile, err)
}
return nil
}

134
cmd/kubeadm/app/phases/kubelet/kubelet_test.go
View File

@ -1,134 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"testing"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
)
func TestCreateBaseKubeletConfiguration(t *testing.T) {
nodeName := "fake-node"
client := fake.NewSimpleClientset()
cfg := &kubeadmapi.MasterConfiguration{
NodeName: nodeName,
KubeletConfiguration: kubeadmapi.KubeletConfiguration{
BaseConfig: &kubeletconfigv1beta1.KubeletConfiguration{
TypeMeta: metav1.TypeMeta{
Kind: "KubeletConfiguration",
},
},
},
}
client.PrependReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) {
return true, &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: nodeName,
},
Spec: v1.NodeSpec{
ConfigSource: &v1.NodeConfigSource{
ConfigMap: &v1.ConfigMapNodeConfigSource{
UID: "",
},
},
},
}, nil
})
client.PrependReactor("get", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
return true, &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: kubeadmconstants.KubeletBaseConfigurationConfigMap,
Namespace: metav1.NamespaceSystem,
UID: "fake-uid",
},
}, nil
})
client.PrependReactor("patch", "nodes", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
client.PrependReactor("create", "roles", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
client.PrependReactor("create", "rolebindings", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
client.PrependReactor("create", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
if err := CreateBaseKubeletConfiguration(cfg, client); err != nil {
t.Errorf("CreateBaseKubeletConfiguration: unexepected error %v", err)
}
}
func TestUpdateNodeWithConfigMap(t *testing.T) {
nodeName := "fake-node"
client := fake.NewSimpleClientset()
client.PrependReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) {
return true, &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: nodeName,
},
Spec: v1.NodeSpec{
ConfigSource: &v1.NodeConfigSource{
ConfigMap: &v1.ConfigMapNodeConfigSource{
UID: "",
},
},
},
}, nil
})
client.PrependReactor("get", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
return true, &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: kubeadmconstants.KubeletBaseConfigurationConfigMap,
Namespace: metav1.NamespaceSystem,
UID: "fake-uid",
},
}, nil
})
client.PrependReactor("patch", "nodes", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
if err := updateNodeWithConfigMap(client, nodeName); err != nil {
t.Errorf("UpdateNodeWithConfigMap: unexepected error %v", err)
}
}
func TestCreateKubeletBaseConfigMapRBACRules(t *testing.T) {
client := fake.NewSimpleClientset()
client.PrependReactor("create", "roles", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
client.PrependReactor("create", "rolebindings", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
if err := createKubeletBaseConfigMapRBACRules(client); err != nil {
t.Errorf("createKubeletBaseConfigMapRBACRules: unexepected error %v", err)
}
}

77
cmd/kubeadm/app/phases/upgrade/configuration.go
View File

@ -1,77 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrade
import (
"fmt"
"io"
"io/ioutil"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
)
// FetchConfiguration fetches configuration required for upgrading your cluster from a file (which has precedence) or a ConfigMap in the cluster
func FetchConfiguration(client clientset.Interface, w io.Writer, cfgPath string) (*kubeadmapi.MasterConfiguration, error) {
fmt.Println("[upgrade/config] Making sure the configuration is correct:")
// Load the configuration from a file or the cluster
configBytes, err := loadConfigurationBytes(client, w, cfgPath)
if err != nil {
return nil, err
}
// Take the versioned configuration populated from the file or configmap, convert it to internal, default and validate
versionedcfg, err := configutil.BytesToInternalConfig(configBytes)
if err != nil {
return nil, fmt.Errorf("could not decode configuration: %v", err)
}
return versionedcfg, nil
}
// loadConfigurationBytes loads the configuration byte slice from either a file or the cluster ConfigMap
func loadConfigurationBytes(client clientset.Interface, w io.Writer, cfgPath string) ([]byte, error) {
// The config file has the highest priority
if cfgPath != "" {
fmt.Printf("[upgrade/config] Reading configuration options from a file: %s\n", cfgPath)
return ioutil.ReadFile(cfgPath)
}
fmt.Println("[upgrade/config] Reading configuration from the cluster...")
configMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(constants.MasterConfigurationConfigMap, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
fmt.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %s namespace must exist.\n", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem)
fmt.Println("[upgrade/config] Without this information, 'kubeadm upgrade' won't know how to configure your upgraded cluster.")
fmt.Println("")
fmt.Println("[upgrade/config] Next steps:")
fmt.Printf("\t- OPTION 1: Run 'kubeadm config upload from-flags' and specify the same CLI arguments you passed to 'kubeadm init' when you created your master.\n")
fmt.Printf("\t- OPTION 2: Run 'kubeadm config upload from-file' and specify the same config file you passed to 'kubeadm init' when you created your master.\n")
fmt.Printf("\t- OPTION 3: Pass a config file to 'kubeadm upgrade' using the --config flag.\n")
fmt.Println("")
return []byte{}, fmt.Errorf("the ConfigMap %q in the %s namespace used for getting configuration information was not found", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem)
} else if err != nil {
return []byte{}, fmt.Errorf("an unexpected error happened when trying to get the ConfigMap %q in the %s namespace: %v", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem, err)
}
fmt.Printf("[upgrade/config] FYI: You can look at this config file with 'kubectl -n %s get cm %s -oyaml'\n", metav1.NamespaceSystem, constants.MasterConfigurationConfigMap)
return []byte(configMap.Data[constants.MasterConfigurationConfigMapKey]), nil
}

6
cmd/kubeadm/app/phases/upgrade/postupgrade.go
View File

@ -36,6 +36,7 @@ import (
"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
@ -102,6 +103,11 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterC
}
}
// Create the new, version-branched kubelet ComponentConfig ConfigMap
if err := kubeletphase.CreateConfigMap(cfg, client); err != nil {
errs = append(errs, fmt.Errorf("error creating kubelet configuration ConfigMap: %v", err))
}
// Upgrade kube-dns/CoreDNS and kube-proxy
if err := dns.EnsureDNSAddon(cfg, client); err != nil {
errs = append(errs, err)

7
cmd/kubeadm/app/preflight/checks.go
View File

@ -1043,10 +1043,11 @@ func TryStartKubelet(ignorePreflightErrors sets.String) {
initSystem, err := initsystem.GetInitSystem()
if err != nil {
glog.Infoln("[preflight] no supported init system detected, won't ensure kubelet is running.")
} else if initSystem.ServiceExists("kubelet") && !initSystem.ServiceIsActive("kubelet") {
} else if initSystem.ServiceExists("kubelet") {
glog.Infoln("[preflight] starting the kubelet service")
if err := initSystem.ServiceStart("kubelet"); err != nil {
glog.Infoln("[preflight] Activating the kubelet service")
// This runs "systemctl daemon-reload && systemctl restart kubelet"
if err := initSystem.ServiceRestart("kubelet"); err != nil {
glog.Warningf("[preflight] unable to start the kubelet service: [%v]\n", err)
glog.Warningf("[preflight] please ensure kubelet is running manually.")
}

65
cmd/kubeadm/app/util/config/cluster.go Normal file
View File

@ -0,0 +1,65 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import (
"fmt"
"io"
"io/ioutil"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
)
// TODO: Add unit tests for this file
// FetchConfigFromFileOrCluster fetches configuration required for upgrading your cluster from a file (which has precedence) or a ConfigMap in the cluster
func FetchConfigFromFileOrCluster(client clientset.Interface, w io.Writer, logPrefix, cfgPath string) (*kubeadmapi.MasterConfiguration, error) {
// Load the configuration from a file or the cluster
configBytes, err := loadConfigurationBytes(client, w, logPrefix, cfgPath)
if err != nil {
return nil, err
}
// Take the versioned configuration populated from the file or ConfigMap, convert it to internal, default and validate
return BytesToInternalConfig(configBytes)
}
// loadConfigurationBytes loads the configuration byte slice from either a file or the cluster ConfigMap
func loadConfigurationBytes(client clientset.Interface, w io.Writer, logPrefix, cfgPath string) ([]byte, error) {
// The config file has the highest priority
if cfgPath != "" {
fmt.Fprintf(w, "[%s] Reading configuration options from a file: %s\n", logPrefix, cfgPath)
return ioutil.ReadFile(cfgPath)
}
fmt.Fprintf(w, "[%s] Reading configuration from the cluster...\n", logPrefix)
configMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(constants.MasterConfigurationConfigMap, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
// Return the apierror directly so the caller of this function can know what type of error occurred and act based on that
return []byte{}, err
} else if err != nil {
return []byte{}, fmt.Errorf("an unexpected error happened when trying to get the ConfigMap %q in the %s namespace: %v", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem, err)
}
fmt.Fprintf(w, "[%s] FYI: You can look at this config file with 'kubectl -n %s get cm %s -oyaml'\n", logPrefix, metav1.NamespaceSystem, constants.MasterConfigurationConfigMap)
return []byte(configMap.Data[constants.MasterConfigurationConfigMapKey]), nil
}

21
pkg/util/initsystem/initsystem.go
View File

@ -29,6 +29,9 @@ type InitSystem interface {
// ServiceStop tries to stop a specific service
ServiceStop(service string) error
// ServiceRestart tries to reload the environment and restart the specific service
ServiceRestart(service string) error
// ServiceExists ensures the service is defined for this init system.
ServiceExists(service string) bool
@ -47,6 +50,14 @@ func (sysd SystemdInitSystem) ServiceStart(service string) error {
return err
}
func (sysd SystemdInitSystem) ServiceRestart(service string) error {
if err := exec.Command("systemctl", "daemon-reload").Run(); err != nil {
return fmt.Errorf("failed to reload systemd: %v", err)
}
args := []string{"restart", service}
return exec.Command("systemctl", args...).Run()
}
func (sysd SystemdInitSystem) ServiceStop(service string) error {
args := []string{"stop", service}
err := exec.Command("systemctl", args...).Run()
@ -95,6 +106,16 @@ func (sysd WindowsInitSystem) ServiceStart(service string) error {
return err
}
func (sysd WindowsInitSystem) ServiceRestart(service string) error {
if err := sysd.ServiceStop(service); err != nil {
return fmt.Errorf("couldn't stop service: %v", err)
}
if err := sysd.ServiceStart(service); err != nil {
return fmt.Errorf("couldn't start service: %v", err)
}
return nil
}
func (sysd WindowsInitSystem) ServiceStop(service string) error {
args := []string{"Stop-Service", service}
err := exec.Command("powershell", args...).Run()