github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #73145 from ereslibre/graduate-control-plane-prepare-phase 

kubeadm: graduate control plane prepare phase
pull/564/head
Kubernetes Prow Robot 2019-02-04 08:23:30 -08:00 committed by GitHub
parent 618ec4cea4 031ce12dbd
commit 25afae9ac6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 257 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/kubeadm/app/cmd/BUILD
View File

@ -38,7 +38,6 @@ go_library(
"//cmd/kubeadm/app/images:go_default_library",
"//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library",
"//cmd/kubeadm/app/phases/certs:go_default_library",
"//cmd/kubeadm/app/phases/controlplane:go_default_library",
"//cmd/kubeadm/app/phases/etcd:go_default_library",
"//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
"//cmd/kubeadm/app/phases/kubelet:go_default_library",

68
cmd/kubeadm/app/cmd/join.go
View File

@ -29,6 +29,7 @@ import (
flag "github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
certutil "k8s.io/client-go/util/cert"
"k8s.io/klog"
@ -42,10 +43,7 @@ import (
cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/discovery"
certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
markcontrolplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markcontrolplane"
patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
@ -161,6 +159,7 @@ type joinData struct {
cfg *kubeadmapi.JoinConfiguration
initCfg *kubeadmapi.InitConfiguration
tlsBootstrapCfg *clientcmdapi.Config
clientSets map[string]*clientset.Clientset
ignorePreflightErrors sets.String
outputWriter io.Writer
}
@ -200,6 +199,8 @@ func NewCmdJoin(out io.Writer, joinOptions *joinOptions) *cobra.Command {
addJoinOtherFlags(cmd.Flags(), &joinOptions.cfgPath, &joinOptions.ignorePreflightErrors, &joinOptions.controlPlane, &joinOptions.token)
joinRunner.AppendPhase(phases.NewPreflightJoinPhase())
joinRunner.AppendPhase(phases.NewControlPlanePreparePhase())
joinRunner.AppendPhase(phases.NewCheckEtcdPhase())
// sets the data builder function, that will be used by the runner
// both when running the entire workflow or single phases
@ -373,6 +374,7 @@ func newJoinData(cmd *cobra.Command, args []string, options *joinOptions, out io
return &joinData{
cfg: cfg,
clientSets: map[string]*clientset.Clientset{},
ignorePreflightErrors: ignorePreflightErrorsSet,
outputWriter: out,
}, nil
@ -408,6 +410,18 @@ func (j *joinData) InitCfg() (*kubeadmapi.InitConfiguration, error) {
return initCfg, err
}
func (j *joinData) ClientSetFromFile(path string) (*clientset.Clientset, error) {
if client, ok := j.clientSets[path]; ok {
return client, nil
}
client, err := kubeconfigutil.ClientSetFromFile(path)
if err != nil {
return nil, errors.Wrap(err, "[join] couldn't create Kubernetes client")
}
j.clientSets[path] = client
return client, nil
}
// IgnorePreflightErrors returns the list of preflight errors to ignore.
func (j *joinData) IgnorePreflightErrors() sets.String {
return j.ignorePreflightErrors
@ -432,13 +446,6 @@ func (j *joinData) Run() error {
if err != nil {
return err
}
if j.cfg.ControlPlane != nil {
// Prepares the node for hosting a new control plane instance by writing necessary
// kubeconfig files, and static pod manifests
if err := j.PrepareForHostingControlPlane(initCfg); err != nil {
return err
}
}
// Executes the kubelet TLS bootstrap process, that completes with the node
// joining the cluster with a dedicates set of credentials as required by
@ -478,47 +485,6 @@ func (j *joinData) Run() error {
return nil
}
// PrepareForHostingControlPlane makes all preparation activities require for a node hosting a new control plane instance
func (j *joinData) PrepareForHostingControlPlane(initConfiguration *kubeadmapi.InitConfiguration) error {
// Generate missing certificates (if any)
if err := certsphase.CreatePKIAssets(initConfiguration); err != nil {
return err
}
// Generate kubeconfig files for controller manager, scheduler and for the admin/kubeadm itself
// NB. The kubeconfig file for kubelet will be generated by the TLS bootstrap process in
// following steps of the join --experimental-control plane workflow
if err := kubeconfigphase.CreateJoinControlPlaneKubeConfigFiles(kubeadmconstants.KubernetesDir, initConfiguration); err != nil {
return errors.Wrap(err, "error generating kubeconfig files")
}
// Creates static pod manifests file for the control plane components to be deployed on this node
// Static pods will be created and managed by the kubelet as soon as it starts
if err := controlplanephase.CreateInitStaticPodManifestFiles(kubeadmconstants.GetStaticPodDirectory(), initConfiguration); err != nil {
return errors.Wrap(err, "error creating static pod manifest files for the control plane components")
}
// in case of local etcd
if initConfiguration.Etcd.External == nil {
// Checks that the etcd cluster is healthy
// NB. this check cannot be implemented before because it requires the admin.conf and all the certificates
// for connecting to etcd already in place
kubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName)
client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile)
if err != nil {
return errors.Wrap(err, "couldn't create Kubernetes client")
}
if err := etcdphase.CheckLocalEtcdClusterStatus(client, &initConfiguration.ClusterConfiguration); err != nil {
return err
}
}
return nil
}
// BootstrapKubelet executes the kubelet TLS bootstrap process.
// This process is executed by the kubelet and completes with the node joining the cluster
// with a dedicates set of credentials as required by the node authorizer

1
cmd/kubeadm/app/cmd/phases/BUILD
View File

@ -6,6 +6,7 @@ go_library(
"addons.go",
"bootstraptoken.go",
"certs.go",
"checketcd.go",
"controlplane.go",
"etcd.go",
"kubeconfig.go",

78
cmd/kubeadm/app/cmd/phases/checketcd.go Normal file
View File

@ -0,0 +1,78 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package phases
import (
"fmt"
"github.com/pkg/errors"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
)
type checkEtcdData interface {
Cfg() *kubeadmapi.JoinConfiguration
ClientSetFromFile(string) (*clientset.Clientset, error)
InitCfg() (*kubeadmapi.InitConfiguration, error)
}
// NewCheckEtcdPhase is a hidden phase that runs after the control-plane-prepare and
// before the bootstrap-kubelet phase that ensures etcd is healthy
func NewCheckEtcdPhase() workflow.Phase {
return workflow.Phase{
Name: "check-etcd",
Run: runCheckEtcdPhase,
Hidden: true,
}
}
func runCheckEtcdPhase(c workflow.RunData) error {
data, ok := c.(checkEtcdData)
if !ok {
return errors.New("check-etcd phase invoked with an invalid data struct")
}
// Skip if this is not a control plane
if data.Cfg().ControlPlane == nil {
return nil
}
cfg, err := data.InitCfg()
if err != nil {
return err
}
if cfg.Etcd.External != nil {
fmt.Println("[check-etcd] Skipping etcd check in external mode")
return nil
}
fmt.Println("[check-etcd] Checking that the etcd cluster is healthy")
// Checks that the etcd cluster is healthy
// NB. this check cannot be implemented before because it requires the admin.conf and all the certificates
// for connecting to etcd already in place
client, err := data.ClientSetFromFile(kubeadmconstants.GetAdminKubeConfigPath())
if err != nil {
return err
}
return etcdphase.CheckLocalEtcdClusterStatus(client, &cfg.ClusterConfiguration)
}

168
cmd/kubeadm/app/cmd/phases/controlplane.go
View File

@ -21,18 +21,21 @@ import (
"github.com/pkg/errors"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
"k8s.io/kubernetes/pkg/util/normalizer"
)
var (
controlPlaneExample = normalizer.Examples(`
# Generates all static Pod manifest files for control plane components,
# Generates all static Pod manifest files for control plane components,
# functionally equivalent to what is generated by kubeadm init.
kubeadm init phase control-plane all
@ -65,6 +68,12 @@ type controlPlaneData interface {
ManifestDir() string
}
type controlPlanePrepareData interface {
Cfg() *kubeadmapi.JoinConfiguration
ClientSetFromFile(string) (*clientset.Clientset, error)
InitCfg() (*kubeadmapi.InitConfiguration, error)
}
func getPhaseDescription(component string) string {
return fmt.Sprintf("Generates the %s static Pod manifest", component)
}
@ -83,20 +92,34 @@ func NewControlPlanePhase() workflow.Phase {
Example: controlPlaneExample,
RunAllSiblings: true,
},
newControlPlaneSubPhase(kubeadmconstants.KubeAPIServer),
newControlPlaneSubPhase(kubeadmconstants.KubeControllerManager),
newControlPlaneSubPhase(kubeadmconstants.KubeScheduler),
newControlPlaneSubphase(kubeadmconstants.KubeAPIServer),
newControlPlaneSubphase(kubeadmconstants.KubeControllerManager),
newControlPlaneSubphase(kubeadmconstants.KubeScheduler),
},
Run: runControlPlanePhase,
}
return phase
}
func newControlPlaneSubPhase(component string) workflow.Phase {
// NewControlPlanePreparePhase creates a kubeadm workflow phase that implements the preparation of the node to serve a control plane
func NewControlPlanePreparePhase() workflow.Phase {
return workflow.Phase{
Name: "control-plane-prepare",
Short: "Prepares the machine for serving a control plane.",
Long: cmdutil.MacroCommandLongDescription,
Phases: []workflow.Phase{
newControlPlanePrepareCertsSubphase(),
newControlPlanePrepareKubeconfigSubphase(),
newControlPlanePrepareManifestsSubphases(),
},
}
}
func newControlPlaneSubphase(component string) workflow.Phase {
phase := workflow.Phase{
Name: controlPlanePhaseProperties[component].name,
Short: controlPlanePhaseProperties[component].short,
Run: runControlPlaneSubPhase(component),
Run: runControlPlaneSubphase(component),
InheritFlags: getControlPlanePhaseFlags(component),
}
return phase
@ -132,6 +155,65 @@ func getControlPlanePhaseFlags(name string) []string {
return flags
}
func getControlPlanePreparePhaseFlags() []string {
return []string{
options.APIServerAdvertiseAddress,
options.APIServerBindPort,
options.CfgPath,
options.ControlPlane,
options.NodeName,
options.TokenDiscovery,
options.TokenDiscoveryCAHash,
options.TokenDiscoverySkipCAHash,
}
}
func newControlPlanePrepareCertsSubphase() workflow.Phase {
return workflow.Phase{
Name: "certs",
Short: "Generates the certificates for the new control plane components",
Run: runControlPlanePrepareCertsPhaseLocal,
InheritFlags: getControlPlanePreparePhaseFlags(),
}
}
func newControlPlanePrepareKubeconfigSubphase() workflow.Phase {
return workflow.Phase{
Name: "kubeconfig",
Short: "Generates the kubeconfig for the new control plane components",
Run: runControlPlanePrepareKubeconfigPhaseLocal,
InheritFlags: getControlPlanePreparePhaseFlags(),
}
}
func newControlPlanePrepareManifestsSubphases() workflow.Phase {
return workflow.Phase{
Name: "manifests",
Short: "Generates the manifests for the new control plane components",
Phases: []workflow.Phase{
{
Name: "all",
Short: "Generates all static Pod manifest files",
InheritFlags: getControlPlanePreparePhaseFlags(),
RunAllSiblings: true,
},
newControlPlanePrepareSubphase(kubeadmconstants.KubeAPIServer),
newControlPlanePrepareSubphase(kubeadmconstants.KubeControllerManager),
newControlPlanePrepareSubphase(kubeadmconstants.KubeScheduler),
},
InheritFlags: getControlPlanePreparePhaseFlags(),
}
}
func newControlPlanePrepareSubphase(component string) workflow.Phase {
return workflow.Phase{
Name: controlPlanePhaseProperties[component].name,
Short: controlPlanePhaseProperties[component].short,
Run: runControlPlanePrepareJoinSubphase(component),
InheritFlags: getControlPlanePreparePhaseFlags(),
}
}
func runControlPlanePhase(c workflow.RunData) error {
data, ok := c.(controlPlaneData)
if !ok {
@ -142,7 +224,7 @@ func runControlPlanePhase(c workflow.RunData) error {
return nil
}
func runControlPlaneSubPhase(component string) func(c workflow.RunData) error {
func runControlPlaneSubphase(component string) func(c workflow.RunData) error {
return func(c workflow.RunData) error {
data, ok := c.(controlPlaneData)
if !ok {
@ -154,3 +236,75 @@ func runControlPlaneSubPhase(component string) func(c workflow.RunData) error {
return controlplane.CreateStaticPodFiles(data.ManifestDir(), &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component)
}
}
func runControlPlanePrepareCertsPhaseLocal(c workflow.RunData) error {
data, ok := c.(controlPlanePrepareData)
if !ok {
return errors.New("control-plane-prepare phase invoked with an invalid data struct")
}
// Skip if this is not a control plane
if data.Cfg().ControlPlane == nil {
return nil
}
cfg, err := data.InitCfg()
if err != nil {
return err
}
// Generate missing certificates (if any)
return certsphase.CreatePKIAssets(cfg)
}
func runControlPlanePrepareKubeconfigPhaseLocal(c workflow.RunData) error {
data, ok := c.(controlPlanePrepareData)
if !ok {
return errors.New("control-plane-prepare phase invoked with an invalid data struct")
}
// Skip if this is not a control plane
if data.Cfg().ControlPlane == nil {
return nil
}
cfg, err := data.InitCfg()
if err != nil {
return err
}
fmt.Println("[control-plane-prepare] Generating kubeconfig files")
// Generate kubeconfig files for controller manager, scheduler and for the admin/kubeadm itself
// NB. The kubeconfig file for kubelet will be generated by the TLS bootstrap process in
// following steps of the join --experimental-control plane workflow
if err := kubeconfigphase.CreateJoinControlPlaneKubeConfigFiles(kubeadmconstants.KubernetesDir, cfg); err != nil {
return errors.Wrap(err, "error generating kubeconfig files")
}
return nil
}
func runControlPlanePrepareJoinSubphase(component string) func(c workflow.RunData) error {
return func(c workflow.RunData) error {
data, ok := c.(controlPlanePrepareData)
if !ok {
return errors.New("control-plane-prepare phase invoked with an invalid data struct")
}
// Skip if this is not a control plane
if data.Cfg().ControlPlane == nil {
return nil
}
cfg, err := data.InitCfg()
if err != nil {
return err
}
// Creates static pod manifests file for the control plane components to be deployed on this node
// Static pods will be created and managed by the kubelet as soon as it starts
fmt.Printf("[control-plane-prepare] Creating static Pod manifest for %q\n", component)
return controlplane.CreateStaticPodFiles(kubeadmconstants.GetStaticPodDirectory(), &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component)
}
}