From 6bbed9fef00d9789abf041f3961c9543e7a4ff45 Mon Sep 17 00:00:00 2001 From: RA489 Date: Tue, 29 Jan 2019 12:07:42 +0530 Subject: [PATCH] kubeadm: graduate control plane join phase --- cmd/kubeadm/app/cmd/BUILD | 2 - cmd/kubeadm/app/cmd/join.go | 115 ++-------- cmd/kubeadm/app/cmd/phases/join/BUILD | 5 +- .../app/cmd/phases/join/controlplanejoin.go | 206 ++++++++++++++++++ ...controlplane.go => controlplaneprepare.go} | 0 5 files changed, 233 insertions(+), 95 deletions(-) create mode 100644 cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go rename cmd/kubeadm/app/cmd/phases/join/{controlplane.go => controlplaneprepare.go} (100%) diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index 8d69273226..772da2d909 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -40,9 +40,7 @@ go_library( "//cmd/kubeadm/app/images:go_default_library", "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library", "//cmd/kubeadm/app/phases/certs:go_default_library", - "//cmd/kubeadm/app/phases/etcd:go_default_library", "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", - "//cmd/kubeadm/app/phases/markcontrolplane:go_default_library", "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index db70b9e82f..2f6de07d9a 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -20,7 +20,6 @@ import ( "fmt" "io" "os" - "path/filepath" "text/template" "github.com/lithammer/dedent" @@ -41,9 +40,6 @@ import ( cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/discovery" - etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" - markcontrolplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markcontrolplane" - uploadconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" @@ -137,6 +133,7 @@ type joinOptions struct { // this data is shared across all the phases that are included in the workflow. type joinData struct { cfg *kubeadmapi.JoinConfiguration + skipTokenPrint bool initCfg *kubeadmapi.InitConfiguration tlsBootstrapCfg *clientcmdapi.Config clientSets map[string]*clientset.Clientset @@ -162,14 +159,30 @@ func NewCmdJoin(out io.Writer, joinOptions *joinOptions) *cobra.Command { c, err := joinRunner.InitData(args) kubeadmutil.CheckErr(err) + data := c.(*joinData) + err = joinRunner.Run(args) kubeadmutil.CheckErr(err) - // TODO: remove this once we have all phases in place. - // the method joinData.Run() itself should be removed too. - data := c.(*joinData) - err = data.Run() - kubeadmutil.CheckErr(err) + // if the node is hosting a new control plane instance + if data.cfg.ControlPlane != nil { + // outputs the join control plane done message and exit + etcdMessage := "" + if data.initCfg.Etcd.External == nil { + etcdMessage = "* A new etcd member was added to the local/stacked etcd cluster." + } + + ctx := map[string]string{ + "KubeConfigPath": kubeadmconstants.GetAdminKubeConfigPath(), + "etcdMessage": etcdMessage, + } + joinControPlaneDoneTemp.Execute(data.outputWriter, ctx) + + } else { + // otherwise, if the node joined as a worker node; + // outputs the join done message and exit + fmt.Fprintf(data.outputWriter, joinWorkerNodeDoneMsg) + } }, // We accept the control-plane location as an optional positional argument Args: cobra.MaximumNArgs(1), @@ -182,6 +195,7 @@ func NewCmdJoin(out io.Writer, joinOptions *joinOptions) *cobra.Command { joinRunner.AppendPhase(phases.NewControlPlanePreparePhase()) joinRunner.AppendPhase(phases.NewCheckEtcdPhase()) joinRunner.AppendPhase(phases.NewKubeletStartPhase()) + joinRunner.AppendPhase(phases.NewControlPlaneJoinPhase()) // sets the data builder function, that will be used by the runner // both when running the entire workflow or single phases @@ -413,89 +427,6 @@ func (j *joinData) OutputWriter() io.Writer { return j.outputWriter } -// Run executes worker node provisioning and tries to join an existing cluster. -func (j *joinData) Run() error { - // Fetch the init configuration based on the join configuration. - // TODO: individual phases should call these: - // - phases that need initCfg should call joinData.InitCfg(). - // - phases that need tlsBootstrapCfg should call joinData.TLSBootstrapCfg(). - initCfg, err := j.InitCfg() - if err != nil { - return err - } - - // if the node is hosting a new control plane instance - if j.cfg.ControlPlane != nil { - // Completes the control plane setup - if err := j.PostInstallControlPlane(initCfg); err != nil { - return err - } - - // outputs the join control plane done template and exits - etcdMessage := "" - // in case of local etcd - if initCfg.Etcd.External == nil { - etcdMessage = "* A new etcd member was added to the local/stacked etcd cluster." - } - - ctx := map[string]string{ - "KubeConfigPath": kubeadmconstants.GetAdminKubeConfigPath(), - "etcdMessage": etcdMessage, - } - joinControPlaneDoneTemp.Execute(j.outputWriter, ctx) - return nil - } - - // otherwise, if the node joined as a worker node; - // outputs the join done message and exits - fmt.Fprintf(j.outputWriter, joinWorkerNodeDoneMsg) - return nil -} - -// PostInstallControlPlane marks the new node as control-plane and update the cluster status with information about current node -func (j *joinData) PostInstallControlPlane(initConfiguration *kubeadmapi.InitConfiguration) error { - kubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName) - - client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) - if err != nil { - return errors.Wrap(err, "couldn't create Kubernetes client") - } - - // in case of local etcd - if initConfiguration.Etcd.External == nil { - // creates target folder if doesn't exist already - if err := os.MkdirAll(initConfiguration.Etcd.Local.DataDir, 0700); err != nil { - return errors.Wrapf(err, "failed to create etcd directory %q", initConfiguration.Etcd.Local.DataDir) - } - - // Adds a new etcd instance; in order to do this the new etcd instance should be "announced" to - // the existing etcd members before being created. - // This operation must be executed after kubelet is already started in order to minimize the time - // between the new etcd member is announced and the start of the static pod running the new etcd member, because during - // this time frame etcd gets temporary not available (only when moving from 1 to 2 members in the etcd cluster). - // From https://coreos.com/etcd/docs/latest/v2/runtime-configuration.html - // "If you add a new member to a 1-node cluster, the cluster cannot make progress before the new member starts - // because it needs two members as majority to agree on the consensus. You will only see this behavior between the time - // etcdctl member add informs the cluster about the new member and the new member successfully establishing a connection to the existing one." - klog.V(1).Info("[join] adding etcd") - if err := etcdphase.CreateStackedEtcdStaticPodManifestFile(client, kubeadmconstants.GetStaticPodDirectory(), initConfiguration.NodeRegistration.Name, &initConfiguration.ClusterConfiguration, &initConfiguration.LocalAPIEndpoint); err != nil { - return errors.Wrap(err, "error creating local etcd static pod manifest file") - } - } - - klog.V(1).Info("[join] uploading currently used configuration to the cluster") - if err := uploadconfigphase.UploadConfiguration(initConfiguration, client); err != nil { - return errors.Wrap(err, "error uploading configuration") - } - - klog.V(1).Info("[join] marking the control-plane with right label") - if err = markcontrolplanephase.MarkControlPlane(client, initConfiguration.NodeRegistration.Name, initConfiguration.NodeRegistration.Taints); err != nil { - return errors.Wrap(err, "error applying control-plane label and taints") - } - - return nil -} - // fetchInitConfigurationFromJoinConfiguration retrieves the init configuration from a join configuration, performing the discovery func fetchInitConfigurationFromJoinConfiguration(cfg *kubeadmapi.JoinConfiguration, tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) { // Retrieves the kubeadm configuration diff --git a/cmd/kubeadm/app/cmd/phases/join/BUILD b/cmd/kubeadm/app/cmd/phases/join/BUILD index 7fa558ea70..957d1ec4e0 100644 --- a/cmd/kubeadm/app/cmd/phases/join/BUILD +++ b/cmd/kubeadm/app/cmd/phases/join/BUILD @@ -4,7 +4,8 @@ go_library( name = "go_default_library", srcs = [ "checketcd.go", - "controlplane.go", + "controlplanejoin.go", + "controlplaneprepare.go", "kubelet.go", "preflight.go", ], @@ -21,7 +22,9 @@ go_library( "//cmd/kubeadm/app/phases/etcd:go_default_library", "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", "//cmd/kubeadm/app/phases/kubelet:go_default_library", + "//cmd/kubeadm/app/phases/markcontrolplane:go_default_library", "//cmd/kubeadm/app/phases/patchnode:go_default_library", + "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", diff --git a/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go b/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go new file mode 100644 index 0000000000..2c0166da75 --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go @@ -0,0 +1,206 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "fmt" + + "github.com/pkg/errors" + clientset "k8s.io/client-go/kubernetes" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" + markcontrolplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markcontrolplane" + uploadconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" + "k8s.io/kubernetes/pkg/util/normalizer" +) + +var ( + controlPlaneJoinExample = normalizer.Examples(` + # Joins a machine as a control plane instance + kubeadm join phase control-plane-join all + `) +) + +type controlPlaneJoinData interface { + Cfg() *kubeadmapi.JoinConfiguration + ClientSetFromFile(string) (*clientset.Clientset, error) + InitCfg() (*kubeadmapi.InitConfiguration, error) + KubeConfigPath() string +} + +func getControlPlaneJoinPhaseFlags() []string { + return []string{ + options.APIServerAdvertiseAddress, + options.APIServerBindPort, + options.CfgPath, + options.ControlPlane, + options.NodeName, + options.TokenDiscovery, + options.TokenDiscoveryCAHash, + options.TokenDiscoverySkipCAHash, + options.KubeconfigPath, + } +} + +// NewControlPlaneJoinPhase creates a kubeadm workflow phase that implements joining a machine as a control plane instance +func NewControlPlaneJoinPhase() workflow.Phase { + return workflow.Phase{ + Name: "control-plane-join", + Short: "Joins a machine as a control plane instance", + Long: cmdutil.MacroCommandLongDescription, + Phases: []workflow.Phase{ + { + Name: "all", + Short: "Joins a machine as a control plane instance", + InheritFlags: getControlPlaneJoinPhaseFlags(), + RunAllSiblings: true, + }, + newEtcdLocalSubphase(), + newUploadConfigSubphase(), + newMarkControlPlaneSubphase(), + }, + } +} + +func newEtcdLocalSubphase() workflow.Phase { + return workflow.Phase{ + Name: "etcd", + Short: "Generates the static Pod manifest file for a local etcd member", + Run: runEtcdPhase, + InheritFlags: getControlPlaneJoinPhaseFlags(), + } +} + +func newUploadConfigSubphase() workflow.Phase { + return workflow.Phase{ + Name: "upload-config", + Short: "Upload the currently used configuration to the cluster", + Run: runUploadConfigPhase, + InheritFlags: getControlPlaneJoinPhaseFlags(), + } +} + +func newMarkControlPlaneSubphase() workflow.Phase { + return workflow.Phase{ + Name: "mark-control-plane", + Short: "Mark a node as a control-plane", + Run: runMarkControlPlanePhase, + InheritFlags: getControlPlaneJoinPhaseFlags(), + } +} + +func runEtcdPhase(c workflow.RunData) error { + data, ok := c.(controlPlaneJoinData) + if !ok { + return errors.New("control-plane-join phase invoked with an invalid data struct") + } + + if data.Cfg().ControlPlane == nil { + return nil + } + + kubeConfigFile := data.KubeConfigPath() + + client, err := data.ClientSetFromFile(kubeConfigFile) + if err != nil { + return errors.Wrap(err, "couldn't create Kubernetes client") + } + cfg, err := data.InitCfg() + if err != nil { + return err + } + // in case of local etcd + if cfg.Etcd.External != nil { + fmt.Println("[control-plane-join] using external etcd - no local stacked instance added") + return nil + } + + // Adds a new etcd instance; in order to do this the new etcd instance should be "announced" to + // the existing etcd members before being created. + // This operation must be executed after kubelet is already started in order to minimize the time + // between the new etcd member is announced and the start of the static pod running the new etcd member, because during + // this time frame etcd gets temporary not available (only when moving from 1 to 2 members in the etcd cluster). + // From https://coreos.com/etcd/docs/latest/v2/runtime-configuration.html + // "If you add a new member to a 1-node cluster, the cluster cannot make progress before the new member starts + // because it needs two members as majority to agree on the consensus. You will only see this behavior between the time + // etcdctl member add informs the cluster about the new member and the new member successfully establishing a connection to the // existing one." + if err := etcdphase.CreateStackedEtcdStaticPodManifestFile(client, kubeadmconstants.GetStaticPodDirectory(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil { + return errors.Wrap(err, "error creating local etcd static pod manifest file") + } + + return nil +} + +func runUploadConfigPhase(c workflow.RunData) error { + data, ok := c.(controlPlaneJoinData) + if !ok { + return errors.New("control-plane-join phase invoked with an invalid data struct") + } + + if data.Cfg().ControlPlane == nil { + return nil + } + + kubeConfigFile := data.KubeConfigPath() + + client, err := data.ClientSetFromFile(kubeConfigFile) + if err != nil { + return errors.Wrap(err, "couldn't create Kubernetes client") + } + cfg, err := data.InitCfg() + if err != nil { + return err + } + + if err := uploadconfigphase.UploadConfiguration(cfg, client); err != nil { + return errors.Wrap(err, "error uploading configuration") + } + + return nil +} + +func runMarkControlPlanePhase(c workflow.RunData) error { + data, ok := c.(controlPlaneJoinData) + if !ok { + return errors.New("control-plane-join phase invoked with an invalid data struct") + } + + if data.Cfg().ControlPlane == nil { + return nil + } + + kubeConfigFile := data.KubeConfigPath() + + client, err := data.ClientSetFromFile(kubeConfigFile) + if err != nil { + return errors.Wrap(err, "couldn't create Kubernetes client") + } + cfg, err := data.InitCfg() + if err != nil { + return err + } + + if err := markcontrolplanephase.MarkControlPlane(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.Taints); err != nil { + return errors.Wrap(err, "error applying control-plane label and taints") + } + + return nil +} diff --git a/cmd/kubeadm/app/cmd/phases/join/controlplane.go b/cmd/kubeadm/app/cmd/phases/join/controlplaneprepare.go similarity index 100% rename from cmd/kubeadm/app/cmd/phases/join/controlplane.go rename to cmd/kubeadm/app/cmd/phases/join/controlplaneprepare.go