kubeadm refactor joinControlPlane config

pull/58/head
fabriziopandini 2018-11-10 17:36:49 +01:00
parent a3ccea9d87
commit d484c8c087
15 changed files with 206 additions and 61 deletions

View File

@ -290,12 +290,15 @@ type JoinConfiguration struct {
// Discovery specifies the options for the kubelet to use during the TLS Bootstrap process
Discovery Discovery
// ControlPlane flag specifies that the joining node should host an additional
// control plane instance.
ControlPlane bool
// ControlPlane defines the additional control plane instance to be deployed on the joining node.
// If nil, no additional control plane instance will be deployed.
ControlPlane *JoinControlPlane
}
// APIEndpoint represents the endpoint of the instance of the API server eventually to be deployed on this node.
APIEndpoint APIEndpoint
// JoinControlPlane contains elements describing an additional control plane instance to be deployed on the joining node.
type JoinControlPlane struct {
// LocalAPIEndpoint represents the endpoint of the API server instance to be deployed on this node.
LocalAPIEndpoint APIEndpoint
}
// Discovery specifies the options for the kubelet to use during the TLS Bootstrap process

View File

@ -81,6 +81,13 @@ func Convert_v1alpha3_JoinConfiguration_To_kubeadm_JoinConfiguration(in *JoinCon
}
}
if in.ControlPlane == true {
out.ControlPlane = &kubeadm.JoinControlPlane{}
if err := autoConvert_v1alpha3_APIEndpoint_To_kubeadm_APIEndpoint(&in.APIEndpoint, &out.ControlPlane.LocalAPIEndpoint, s); err != nil {
return err
}
}
return nil
}
@ -102,6 +109,13 @@ func Convert_kubeadm_JoinConfiguration_To_v1alpha3_JoinConfiguration(in *kubeadm
out.DiscoveryFile = in.Discovery.File.KubeConfigPath
}
if in.ControlPlane != nil {
out.ControlPlane = true
if err := autoConvert_kubeadm_APIEndpoint_To_v1alpha3_APIEndpoint(&in.ControlPlane.LocalAPIEndpoint, &out.APIEndpoint, s); err != nil {
return err
}
}
return nil
}

View File

@ -132,7 +132,7 @@ func SetDefaults_JoinConfiguration(obj *JoinConfiguration) {
}
SetDefaults_NodeRegistrationOptions(&obj.NodeRegistration)
SetDefaults_APIEndpoint(&obj.APIEndpoint)
SetDefaults_JoinControlPlane(obj.ControlPlane)
SetDefaults_Discovery(&obj.Discovery)
}
@ -142,6 +142,12 @@ func SetDefaults_NodeRegistrationOptions(obj *NodeRegistrationOptions) {
}
}
func SetDefaults_JoinControlPlane(obj *JoinControlPlane) {
if obj != nil {
SetDefaults_APIEndpoint(&obj.LocalAPIEndpoint)
}
}
// SetDefaults_Discovery assigns default values for the discovery process
func SetDefaults_Discovery(obj *Discovery) {
if len(obj.TLSBootstrapToken) == 0 && obj.BootstrapToken != nil {

View File

@ -255,7 +255,6 @@ type ExternalEtcd struct {
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// JoinConfiguration contains elements describing a particular node.
// TODO: This struct should be replaced by dynamic kubelet configuration.
type JoinConfiguration struct {
metav1.TypeMeta `json:",inline"`
@ -270,12 +269,15 @@ type JoinConfiguration struct {
// Discovery specifies the options for the kubelet to use during the TLS Bootstrap process
Discovery Discovery `json:"discovery"`
// ControlPlane flag specifies that the joining node should host an additional
// control plane instance.
ControlPlane bool `json:"controlPlane,omitempty"`
// ControlPlane defines the additional control plane instance to be deployed on the joining node.
// If nil, no additional control plane instance will be deployed.
ControlPlane *JoinControlPlane `json:"controlPlane,omitempty"`
}
// APIEndpoint represents the endpoint of the instance of the API server eventually to be deployed on this node.
APIEndpoint APIEndpoint `json:"apiEndpoint,omitempty"`
// JoinControlPlane contains elements describing an additional control plane instance to be deployed on the joining node.
type JoinControlPlane struct {
// LocalAPIEndpoint represents the endpoint of the API server instance to be deployed on this node.
LocalAPIEndpoint APIEndpoint `json:"localAPIEndpoint,omitempty"`
}
// Discovery specifies the options for the kubelet to use during the TLS Bootstrap process

View File

@ -27,7 +27,6 @@ import (
"github.com/pkg/errors"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation"
"k8s.io/apimachinery/pkg/util/validation/field"
@ -78,7 +77,7 @@ func ValidateJoinConfiguration(c *kubeadm.JoinConfiguration) field.ErrorList {
allErrs := field.ErrorList{}
allErrs = append(allErrs, ValidateDiscovery(&c.Discovery, field.NewPath("discovery"))...)
allErrs = append(allErrs, ValidateNodeRegistrationOptions(&c.NodeRegistration, field.NewPath("nodeRegistration"))...)
allErrs = append(allErrs, ValidateAPIEndpoint(&c.APIEndpoint, field.NewPath("apiEndpoint"))...)
allErrs = append(allErrs, ValidateJoinControlPlane(c.ControlPlane, field.NewPath("controlPlane"))...)
if !filepath.IsAbs(c.CACertPath) || !strings.HasSuffix(c.CACertPath, ".crt") {
allErrs = append(allErrs, field.Invalid(field.NewPath("caCertPath"), c.CACertPath, "the ca certificate path must be an absolute path"))
@ -86,6 +85,15 @@ func ValidateJoinConfiguration(c *kubeadm.JoinConfiguration) field.ErrorList {
return allErrs
}
// ValidateJoinControlPlane validates joining control plane configuration and collects all encountered errors
func ValidateJoinControlPlane(c *kubeadm.JoinControlPlane, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if c != nil {
allErrs = append(allErrs, ValidateAPIEndpoint(&c.LocalAPIEndpoint, fldPath.Child("localAPIEndpoint"))...)
}
return allErrs
}
// ValidateNodeRegistrationOptions validates the NodeRegistrationOptions object
func ValidateNodeRegistrationOptions(nro *kubeadm.NodeRegistrationOptions, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}

View File

@ -23,7 +23,6 @@ import (
"time"
"github.com/spf13/pflag"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
@ -541,6 +540,84 @@ func TestValidateJoinConfiguration(t *testing.T) {
},
},
}, false},
{&kubeadm.JoinConfiguration{ // Pass without JoinControlPlane
CACertPath: "/some/cert.crt",
Discovery: kubeadm.Discovery{
BootstrapToken: &kubeadm.BootstrapTokenDiscovery{
Token: "abcdef.1234567890123456",
APIServerEndpoint: "1.2.3.4:6443",
CACertHashes: []string{"aaaa"},
},
TLSBootstrapToken: "abcdef.1234567890123456",
},
NodeRegistration: kubeadm.NodeRegistrationOptions{
Name: "aaa",
CRISocket: "/var/run/dockershim.sock",
},
}, true},
{&kubeadm.JoinConfiguration{ // Pass with JoinControlPlane
CACertPath: "/some/cert.crt",
Discovery: kubeadm.Discovery{
BootstrapToken: &kubeadm.BootstrapTokenDiscovery{
Token: "abcdef.1234567890123456",
APIServerEndpoint: "1.2.3.4:6443",
CACertHashes: []string{"aaaa"},
},
TLSBootstrapToken: "abcdef.1234567890123456",
},
NodeRegistration: kubeadm.NodeRegistrationOptions{
Name: "aaa",
CRISocket: "/var/run/dockershim.sock",
},
ControlPlane: &kubeadm.JoinControlPlane{
LocalAPIEndpoint: kubeadm.APIEndpoint{
AdvertiseAddress: "1.2.3.4",
BindPort: 1234,
},
},
}, true},
{&kubeadm.JoinConfiguration{ // Fail JoinControlPlane.AdvertiseAddress validation
CACertPath: "/some/cert.crt",
Discovery: kubeadm.Discovery{
BootstrapToken: &kubeadm.BootstrapTokenDiscovery{
Token: "abcdef.1234567890123456",
APIServerEndpoint: "1.2.3.4:6443",
CACertHashes: []string{"aaaa"},
},
TLSBootstrapToken: "abcdef.1234567890123456",
},
NodeRegistration: kubeadm.NodeRegistrationOptions{
Name: "aaa",
CRISocket: "/var/run/dockershim.sock",
},
ControlPlane: &kubeadm.JoinControlPlane{
LocalAPIEndpoint: kubeadm.APIEndpoint{
AdvertiseAddress: "aaa",
BindPort: 1234,
},
},
}, false},
{&kubeadm.JoinConfiguration{ // Fail JoinControlPlane.BindPort validation
CACertPath: "/some/cert.crt",
Discovery: kubeadm.Discovery{
BootstrapToken: &kubeadm.BootstrapTokenDiscovery{
Token: "abcdef.1234567890123456",
APIServerEndpoint: "1.2.3.4:6443",
CACertHashes: []string{"aaaa"},
},
TLSBootstrapToken: "abcdef.1234567890123456",
},
NodeRegistration: kubeadm.NodeRegistrationOptions{
Name: "aaa",
CRISocket: "/var/run/dockershim.sock",
},
ControlPlane: &kubeadm.JoinControlPlane{
LocalAPIEndpoint: kubeadm.APIEndpoint{
AdvertiseAddress: "1.2.3.4",
BindPort: -1,
},
},
}, false},
}
for _, rt := range tests {
actual := ValidateJoinConfiguration(rt.s)

View File

@ -81,8 +81,11 @@ func newCmdPreFlightNode() *cobra.Command {
internalcfg, err := configutil.JoinConfigFileAndDefaultsToInternalConfig(cfgPath, cfg)
kubeadmutil.CheckErr(err)
err = configutil.VerifyAPIServerBindAddress(internalcfg.APIEndpoint.AdvertiseAddress)
if internalcfg.ControlPlane != nil {
err = configutil.VerifyAPIServerBindAddress(internalcfg.ControlPlane.LocalAPIEndpoint.AdvertiseAddress)
kubeadmutil.CheckErr(err)
}
fmt.Println("[preflight] running pre-flight checks")

View File

@ -164,6 +164,9 @@ func NewCmdJoin(out io.Writer) *cobra.Command {
var token string
var cfgPath string
var ignorePreflightErrors []string
var controlPlane bool
var advertiseAddress string
var bindPort int32 = kubeadmapiv1beta1.DefaultAPIBindPort
cmd := &cobra.Command{
Use: "join",
@ -190,6 +193,15 @@ func NewCmdJoin(out io.Writer) *cobra.Command {
cfg.Discovery.TLSBootstrapToken = token
}
if controlPlane {
cfg.ControlPlane = &kubeadmapiv1beta1.JoinControlPlane{
LocalAPIEndpoint: kubeadmapiv1beta1.APIEndpoint{
AdvertiseAddress: advertiseAddress,
BindPort: bindPort,
},
}
}
j, err := NewValidJoin(cmd.PersistentFlags(), cfg, cfgPath, ignorePreflightErrors)
kubeadmutil.CheckErr(err)
kubeadmutil.CheckErr(j.Run(out))
@ -199,7 +211,7 @@ func NewCmdJoin(out io.Writer) *cobra.Command {
AddJoinConfigFlags(cmd.PersistentFlags(), cfg, &token)
AddJoinBootstrapTokenDiscoveryFlags(cmd.PersistentFlags(), btd)
AddJoinFileDiscoveryFlags(cmd.PersistentFlags(), fd)
AddJoinOtherFlags(cmd.PersistentFlags(), &cfgPath, &ignorePreflightErrors)
AddJoinOtherFlags(cmd.PersistentFlags(), &cfgPath, &ignorePreflightErrors, &controlPlane, &advertiseAddress, &bindPort)
return cmd
}
@ -232,17 +244,6 @@ func AddJoinConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta1.JoinConfig
&cfg.NodeRegistration.CRISocket, "cri-socket", cfg.NodeRegistration.CRISocket,
`Specify the CRI socket to connect to.`,
)
flagSet.BoolVar(
&cfg.ControlPlane, "experimental-control-plane", cfg.ControlPlane,
"Create a new control plane instance on this node")
flagSet.StringVar(
&cfg.APIEndpoint.AdvertiseAddress, "apiserver-advertise-address", cfg.APIEndpoint.AdvertiseAddress,
"If the node should host a new control plane instance, the IP address the API Server will advertise it's listening on.",
)
flagSet.Int32Var(
&cfg.APIEndpoint.BindPort, "apiserver-bind-port", cfg.APIEndpoint.BindPort,
"If the node should host a new control plane instance, the port for the API Server to bind to.",
)
}
// AddJoinBootstrapTokenDiscoveryFlags adds bootstrap token specific discovery flags to the specified flagset
@ -266,15 +267,22 @@ func AddJoinFileDiscoveryFlags(flagSet *flag.FlagSet, fd *kubeadmapiv1beta1.File
}
// AddJoinOtherFlags adds join flags that are not bound to a configuration file to the given flagset
func AddJoinOtherFlags(flagSet *flag.FlagSet, cfgPath *string, ignorePreflightErrors *[]string) {
func AddJoinOtherFlags(flagSet *flag.FlagSet, cfgPath *string, ignorePreflightErrors *[]string, controlPlane *bool, advertiseAddress *string, bindPort *int32) {
flagSet.StringVar(
cfgPath, "config", *cfgPath,
"Path to kubeadm config file.")
flagSet.StringSliceVar(
ignorePreflightErrors, "ignore-preflight-errors", *ignorePreflightErrors,
"A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.",
)
"A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.")
flagSet.BoolVar(
controlPlane, "experimental-control-plane", *controlPlane,
"Create a new control plane instance on this node")
flagSet.StringVar(
advertiseAddress, "apiserver-advertise-address", *advertiseAddress,
"If the node should host a new control plane instance, the IP address the API Server will advertise it's listening on.")
flagSet.Int32Var(
bindPort, "apiserver-bind-port", *bindPort,
"If the node should host a new control plane instance, the port for the API Server to bind to.")
}
// Join defines struct used by kubeadm join command
@ -292,7 +300,7 @@ func NewJoin(cfgPath string, defaultcfg *kubeadmapiv1beta1.JoinConfiguration, ig
klog.V(1).Infoln("[join] found NodeName empty; using OS hostname as NodeName")
}
if defaultcfg.APIEndpoint.AdvertiseAddress == "" {
if defaultcfg.ControlPlane != nil && defaultcfg.ControlPlane.LocalAPIEndpoint.AdvertiseAddress == "" {
klog.V(1).Infoln("[join] found advertiseAddress empty; using default interface's IP address as advertiseAddress")
}
@ -300,9 +308,11 @@ func NewJoin(cfgPath string, defaultcfg *kubeadmapiv1beta1.JoinConfiguration, ig
if err != nil {
return nil, err
}
if err := configutil.VerifyAPIServerBindAddress(internalCfg.APIEndpoint.AdvertiseAddress); err != nil {
if defaultcfg.ControlPlane != nil {
if err := configutil.VerifyAPIServerBindAddress(internalCfg.ControlPlane.LocalAPIEndpoint.AdvertiseAddress); err != nil {
return nil, err
}
}
fmt.Println("[preflight] Running pre-flight checks")
@ -330,7 +340,7 @@ func NewJoin(cfgPath string, defaultcfg *kubeadmapiv1beta1.JoinConfiguration, ig
// Run executes worker node provisioning and tries to join an existing cluster.
func (j *Join) Run(out io.Writer) error {
if j.cfg.ControlPlane == true {
if j.cfg.ControlPlane != nil {
// Checks if the cluster configuration supports
// joining a new control plane instance and if all the necessary certificates are provided
if err := j.CheckIfReadyForAdditionalControlPlane(j.initCfg); err != nil {
@ -366,7 +376,7 @@ func (j *Join) Run(out io.Writer) error {
}
// if the node is hosting a new control plane instance
if j.cfg.ControlPlane == true {
if j.cfg.ControlPlane != nil {
// Completes the control plane setup
if err := j.PostInstallControlPlane(j.initCfg); err != nil {
return err
@ -493,7 +503,11 @@ func (j *Join) BootstrapKubelet(tlsBootstrapCfg *clientcmdapi.Config) error {
// Write env file with flags for the kubelet to use. We only want to
// register the joining node with the specified taints if the node
// is not a master. The markmaster phase will register the taints otherwise.
registerTaintsUsingFlags := !j.cfg.ControlPlane
registerTaintsUsingFlags := false
if j.cfg.ControlPlane == nil {
registerTaintsUsingFlags = true
}
if err := kubeletphase.WriteKubeletDynamicEnvFile(&j.cfg.NodeRegistration, j.initCfg.FeatureGates, registerTaintsUsingFlags, kubeadmconstants.KubeletRunDirectory); err != nil {
return err
}
@ -609,7 +623,9 @@ func fetchInitConfigurationFromJoinConfiguration(cfg *kubeadmapi.JoinConfigurati
// injects into the kubeadm configuration the information about the joining node
initConfiguration.NodeRegistration = cfg.NodeRegistration
initConfiguration.LocalAPIEndpoint = cfg.APIEndpoint
if cfg.ControlPlane != nil {
initConfiguration.LocalAPIEndpoint = cfg.ControlPlane.LocalAPIEndpoint
}
return initConfiguration, tlsBootstrapCfg, nil
}

View File

@ -37,11 +37,10 @@ import (
"github.com/PuerkitoBio/purell"
"github.com/blang/semver"
"github.com/pkg/errors"
"k8s.io/klog"
netutil "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/sets"
versionutil "k8s.io/apimachinery/pkg/util/version"
"k8s.io/klog"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/images"
@ -943,7 +942,7 @@ func RunJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.JoinConfigura
FileAvailableCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName)},
}
checks = addCommonChecks(execer, cfg, checks)
if !cfg.ControlPlane {
if cfg.ControlPlane == nil {
checks = append(checks, FileAvailableCheck{Path: cfg.CACertPath})
}

View File

@ -20,9 +20,8 @@ import (
"io/ioutil"
"github.com/pkg/errors"
"k8s.io/klog"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/klog"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
@ -33,18 +32,31 @@ import (
// SetJoinDynamicDefaults checks and sets configuration values for the JoinConfiguration object
func SetJoinDynamicDefaults(cfg *kubeadmapi.JoinConfiguration) error {
if err := SetNodeRegistrationDynamicDefaults(&cfg.NodeRegistration, cfg.ControlPlane); err != nil {
addMasterTaint := false
if cfg.ControlPlane != nil {
addMasterTaint = true
}
if err := SetNodeRegistrationDynamicDefaults(&cfg.NodeRegistration, addMasterTaint); err != nil {
return err
}
if err := SetAPIEndpointDynamicDefaults(&cfg.APIEndpoint); err != nil {
if err := SetJoinControlPlaneDefaults(cfg.ControlPlane); err != nil {
return err
}
return nil
}
// SetJoinControlPlaneDefaults checks and sets configuration values for the JoinControlPlane object
func SetJoinControlPlaneDefaults(cfg *kubeadmapi.JoinControlPlane) error {
if cfg != nil {
if err := SetAPIEndpointDynamicDefaults(&cfg.LocalAPIEndpoint); err != nil {
return err
}
}
return nil
}
// JoinConfigFileAndDefaultsToInternalConfig takes a path to a config file and a versioned configuration that can serve as the default config
// If cfgPath is specified, defaultversionedcfg will always get overridden. Otherwise, the default config (often populated by flags) will be used.
// Then the external, versioned configuration is defaulted and converted to the internal type.

View File

@ -1,8 +1,8 @@
APIEndpoint:
CACertPath: /etc/kubernetes/pki/ca.crt
ControlPlane:
LocalAPIEndpoint:
AdvertiseAddress: 192.168.2.2
BindPort: 6443
CACertPath: /etc/kubernetes/pki/ca.crt
ControlPlane: false
Discovery:
BootstrapToken:
APIServerEndpoint: kube-apiserver:6443
@ -16,4 +16,6 @@ NodeRegistration:
CRISocket: /var/run/dockershim.sock
KubeletExtraArgs: null
Name: master-1
Taints: null
Taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master

View File

@ -3,6 +3,7 @@ apiEndpoint:
bindPort: 6443
apiVersion: kubeadm.k8s.io/v1alpha3
caCertPath: /etc/kubernetes/pki/ca.crt
controlPlane: true
discoveryFile: ""
discoveryTimeout: 5m0s
discoveryToken: abcdef.0123456789abcdef
@ -13,5 +14,8 @@ kind: JoinConfiguration
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: master-1
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
tlsBootstrapToken: abcdef.0123456789abcdef
token: abcdef.0123456789abcdef

View File

@ -1,8 +1,9 @@
apiEndpoint:
advertiseAddress: 192.168.2.2
bindPort: 6443
apiVersion: kubeadm.k8s.io/v1beta1
caCertPath: /etc/kubernetes/pki/ca.crt
controlPlane:
localAPIEndpoint:
advertiseAddress: 192.168.2.2
bindPort: 6443
discovery:
bootstrapToken:
apiServerEndpoint: kube-apiserver:6443
@ -14,3 +15,6 @@ kind: JoinConfiguration
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: master-1
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master

View File

@ -1,6 +1,3 @@
apiEndpoint:
advertiseAddress: 192.168.2.2
bindPort: 6443
apiVersion: kubeadm.k8s.io/v1beta1
caCertPath: /etc/kubernetes/pki/ca.crt
discovery:

View File

@ -1,5 +1,3 @@
apiEndpoint:
advertiseAddress: 192.168.2.2
apiVersion: kubeadm.k8s.io/v1alpha3
discoveryToken: abcdef.0123456789abcdef
discoveryTokenAPIServers: