From f7c494fe5b1a07d243e778ba29e511eb53e9821d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Sat, 2 Dec 2017 00:27:07 +0200 Subject: [PATCH] kubeadm: Fix a couple of upgrade/downgrade-related bugs --- .../app/apis/kubeadm/v1alpha1/defaults.go | 30 +++++----- .../app/apis/kubeadm/v1alpha1/types.go | 2 +- cmd/kubeadm/app/cmd/init.go | 2 +- cmd/kubeadm/app/cmd/phases/selfhosting.go | 2 +- cmd/kubeadm/app/cmd/upgrade/apply.go | 12 ++-- cmd/kubeadm/app/cmd/upgrade/common.go | 22 ++++---- cmd/kubeadm/app/cmd/upgrade/common_test.go | 9 +-- cmd/kubeadm/app/cmd/upgrade/plan.go | 4 +- cmd/kubeadm/app/features/features.go | 18 ++++-- cmd/kubeadm/app/phases/kubelet/kubelet.go | 6 +- .../app/phases/selfhosting/selfhosting.go | 10 ++-- cmd/kubeadm/app/phases/upgrade/BUILD | 1 + cmd/kubeadm/app/phases/upgrade/policy.go | 43 ++++++++++---- cmd/kubeadm/app/phases/upgrade/policy_test.go | 20 ++++--- cmd/kubeadm/app/phases/upgrade/postupgrade.go | 56 ++++++++++++++----- .../app/phases/upgrade/postupgrade_v18_19.go | 2 + cmd/kubeadm/app/util/apiclient/idempotency.go | 9 +++ cmd/kubeadm/app/util/error.go | 2 +- cmd/kubeadm/app/util/error_test.go | 4 +- 19 files changed, 165 insertions(+), 89 deletions(-) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go index 948956e46d..54da3c2751 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go @@ -156,26 +156,24 @@ func SetDefaults_NodeConfiguration(obj *NodeConfiguration) { } } -// SetDefaultsEtcdSelfHosted sets defaults for self-hosted etcd +// SetDefaultsEtcdSelfHosted sets defaults for self-hosted etcd if used func SetDefaultsEtcdSelfHosted(obj *MasterConfiguration) { - if obj.Etcd.SelfHosted == nil { - obj.Etcd.SelfHosted = &SelfHostedEtcd{} - } + if obj.Etcd.SelfHosted != nil { + if obj.Etcd.SelfHosted.ClusterServiceName == "" { + obj.Etcd.SelfHosted.ClusterServiceName = DefaultEtcdClusterServiceName + } - if obj.Etcd.SelfHosted.ClusterServiceName == "" { - obj.Etcd.SelfHosted.ClusterServiceName = DefaultEtcdClusterServiceName - } + if obj.Etcd.SelfHosted.EtcdVersion == "" { + obj.Etcd.SelfHosted.EtcdVersion = constants.DefaultEtcdVersion + } - if obj.Etcd.SelfHosted.EtcdVersion == "" { - obj.Etcd.SelfHosted.EtcdVersion = constants.DefaultEtcdVersion - } + if obj.Etcd.SelfHosted.OperatorVersion == "" { + obj.Etcd.SelfHosted.OperatorVersion = DefaultEtcdOperatorVersion + } - if obj.Etcd.SelfHosted.OperatorVersion == "" { - obj.Etcd.SelfHosted.OperatorVersion = DefaultEtcdOperatorVersion - } - - if obj.Etcd.SelfHosted.CertificatesDir == "" { - obj.Etcd.SelfHosted.CertificatesDir = DefaultEtcdCertDir + if obj.Etcd.SelfHosted.CertificatesDir == "" { + obj.Etcd.SelfHosted.CertificatesDir = DefaultEtcdCertDir + } } } diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go index 349dde5510..ee7b84f1b1 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go @@ -147,7 +147,7 @@ type NodeConfiguration struct { // KubeletConfiguration contains elements describing initial remote configuration of kubelet type KubeletConfiguration struct { - BaseConfig *kubeletconfigv1alpha1.KubeletConfiguration `json:"baseConfig"` + BaseConfig *kubeletconfigv1alpha1.KubeletConfiguration `json:"baseConfig,omitempty"` } // HostPathMount contains elements describing volumes that are mounted from the diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 7088e1af0b..7d09adce95 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -433,7 +433,7 @@ func (i *Init) Run(out io.Writer) error { // Temporary control plane is up, now we create our self hosted control // plane components and remove the static manifests: fmt.Println("[self-hosted] Creating self-hosted control plane.") - if err := selfhostingphase.CreateSelfHostedControlPlane(manifestDir, kubeConfigDir, i.cfg, client, waiter); err != nil { + if err := selfhostingphase.CreateSelfHostedControlPlane(manifestDir, kubeConfigDir, i.cfg, client, waiter, i.dryRun); err != nil { return fmt.Errorf("error creating self hosted control plane: %v", err) } } diff --git a/cmd/kubeadm/app/cmd/phases/selfhosting.go b/cmd/kubeadm/app/cmd/phases/selfhosting.go index b9200ca57e..9bf6530ee1 100644 --- a/cmd/kubeadm/app/cmd/phases/selfhosting.go +++ b/cmd/kubeadm/app/cmd/phases/selfhosting.go @@ -103,7 +103,7 @@ func getSelfhostingSubCommand() *cobra.Command { // Converts the Static Pod-hosted control plane into a self-hosted one waiter := apiclient.NewKubeWaiter(client, 2*time.Minute, os.Stdout) - err = selfhosting.CreateSelfHostedControlPlane(constants.GetStaticPodDirectory(), constants.KubernetesDir, internalcfg, client, waiter) + err = selfhosting.CreateSelfHostedControlPlane(constants.GetStaticPodDirectory(), constants.KubernetesDir, internalcfg, client, waiter, false) kubeadmutil.CheckErr(err) }, } diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go index b4ea162c14..7313bffb6c 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -28,6 +28,7 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" @@ -119,14 +120,11 @@ func NewCmdApply(parentFlags *cmdUpgradeFlags) *cobra.Command { func RunApply(flags *applyFlags) error { // Start with the basics, verify that the cluster is healthy and get the configuration from the cluster (using the ConfigMap) - upgradeVars, err := enforceRequirements(flags.parent.featureGatesString, flags.parent.kubeConfigPath, flags.parent.cfgPath, flags.parent.printConfig, flags.dryRun, flags.parent.ignorePreflightErrorsSet) + upgradeVars, err := enforceRequirements(flags.parent, flags.dryRun, flags.newK8sVersionStr) if err != nil { return err } - // Set the upgraded version on the external config object now - upgradeVars.cfg.KubernetesVersion = flags.newK8sVersionStr - // Grab the external, versioned configuration and convert it to the internal type for usage here later internalcfg := &kubeadmapi.MasterConfiguration{} legacyscheme.Scheme.Convert(upgradeVars.cfg, internalcfg, nil) @@ -144,6 +142,10 @@ func RunApply(flags *applyFlags) error { } flags.newK8sVersion = k8sVer + if err := features.ValidateVersion(features.InitFeatureGates, internalcfg.FeatureGates, internalcfg.KubernetesVersion); err != nil { + return err + } + // Enforce the version skew policies if err := EnforceVersionPolicies(flags, upgradeVars.versionGetter); err != nil { return fmt.Errorf("[upgrade/version] FATAL: %v", err) @@ -167,7 +169,7 @@ func RunApply(flags *applyFlags) error { } // Upgrade RBAC rules and addons. - if err := upgrade.PerformPostUpgradeTasks(upgradeVars.client, internalcfg, flags.newK8sVersion); err != nil { + if err := upgrade.PerformPostUpgradeTasks(upgradeVars.client, internalcfg, flags.newK8sVersion, flags.dryRun); err != nil { return fmt.Errorf("[upgrade/postupgrade] FATAL post-upgrade error: %v", err) } diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go index 2fbbf08f51..254c9afbce 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -48,35 +48,37 @@ type upgradeVariables struct { } // enforceRequirements verifies that it's okay to upgrade and then returns the variables needed for the rest of the procedure -func enforceRequirements(featureGatesString, kubeConfigPath, cfgPath string, printConfig, dryRun bool, ignoreChecksErrors sets.String) (*upgradeVariables, error) { - client, err := getClient(kubeConfigPath, dryRun) +func enforceRequirements(flags *cmdUpgradeFlags, dryRun bool, newK8sVersion string) (*upgradeVariables, error) { + client, err := getClient(flags.kubeConfigPath, dryRun) if err != nil { - return nil, fmt.Errorf("couldn't create a Kubernetes client from file %q: %v", kubeConfigPath, err) + return nil, fmt.Errorf("couldn't create a Kubernetes client from file %q: %v", flags.kubeConfigPath, err) } // Run healthchecks against the cluster - if err := upgrade.CheckClusterHealth(client, ignoreChecksErrors); err != nil { + if err := upgrade.CheckClusterHealth(client, flags.ignorePreflightErrorsSet); err != nil { return nil, fmt.Errorf("[upgrade/health] FATAL: %v", err) } // Fetch the configuration from a file or ConfigMap and validate it - cfg, err := upgrade.FetchConfiguration(client, os.Stdout, cfgPath) + cfg, err := upgrade.FetchConfiguration(client, os.Stdout, flags.cfgPath) if err != nil { return nil, fmt.Errorf("[upgrade/config] FATAL: %v", err) } + // If a new k8s version should be set, apply the change before printing the config + if len(newK8sVersion) != 0 { + cfg.KubernetesVersion = newK8sVersion + } + // If the user told us to print this information out; do it! - if printConfig { + if flags.printConfig { printConfiguration(cfg, os.Stdout) } - cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString) + cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, flags.featureGatesString) if err != nil { return nil, fmt.Errorf("[upgrade/config] FATAL: %v", err) } - if err := features.ValidateVersion(features.InitFeatureGates, cfg.FeatureGates, cfg.KubernetesVersion); err != nil { - return nil, err - } return &upgradeVariables{ client: client, diff --git a/cmd/kubeadm/app/cmd/upgrade/common_test.go b/cmd/kubeadm/app/cmd/upgrade/common_test.go index e710579897..5ebff084e0 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common_test.go +++ b/cmd/kubeadm/app/cmd/upgrade/common_test.go @@ -52,8 +52,7 @@ func TestPrintConfiguration(t *testing.T) { keyFile: "" imageRepository: "" kubeProxy: {} - kubeletConfiguration: - baseConfig: null + kubeletConfiguration: {} kubernetesVersion: v1.7.1 networking: dnsDomain: "" @@ -86,8 +85,7 @@ func TestPrintConfiguration(t *testing.T) { keyFile: "" imageRepository: "" kubeProxy: {} - kubeletConfiguration: - baseConfig: null + kubeletConfiguration: {} kubernetesVersion: v1.7.1 networking: dnsDomain: "" @@ -130,8 +128,7 @@ func TestPrintConfiguration(t *testing.T) { operatorVersion: v0.1.0 imageRepository: "" kubeProxy: {} - kubeletConfiguration: - baseConfig: null + kubeletConfiguration: {} kubernetesVersion: v1.7.1 networking: dnsDomain: "" diff --git a/cmd/kubeadm/app/cmd/upgrade/plan.go b/cmd/kubeadm/app/cmd/upgrade/plan.go index 61e0b4376a..09031f8165 100644 --- a/cmd/kubeadm/app/cmd/upgrade/plan.go +++ b/cmd/kubeadm/app/cmd/upgrade/plan.go @@ -54,8 +54,8 @@ func NewCmdPlan(parentFlags *cmdUpgradeFlags) *cobra.Command { // RunPlan takes care of outputting available versions to upgrade to for the user func RunPlan(parentFlags *cmdUpgradeFlags) error { - // Start with the basics, verify that the cluster is healthy, build a client and a versionGetter. Never set dry-run for plan. - upgradeVars, err := enforceRequirements(parentFlags.featureGatesString, parentFlags.kubeConfigPath, parentFlags.cfgPath, parentFlags.printConfig, false, parentFlags.ignorePreflightErrorsSet) + // Start with the basics, verify that the cluster is healthy, build a client and a versionGetter. Never dry-run when planning. + upgradeVars, err := enforceRequirements(parentFlags, false, "") if err != nil { return err } diff --git a/cmd/kubeadm/app/features/features.go b/cmd/kubeadm/app/features/features.go index 2a9fa52ac7..36f5a495c4 100644 --- a/cmd/kubeadm/app/features/features.go +++ b/cmd/kubeadm/app/features/features.go @@ -33,10 +33,10 @@ const ( // CoreDNS is alpha in v1.9 CoreDNS = "CoreDNS" - // SelfHosting is beta in v1.8 + // SelfHosting is beta in v1.9 SelfHosting = "SelfHosting" - // StoreCertsInSecrets is alpha in v1.8 + // StoreCertsInSecrets is alpha in v1.8 and v1.9 StoreCertsInSecrets = "StoreCertsInSecrets" // DynamicKubeletConfig is alpha in v1.9 @@ -47,9 +47,10 @@ var v190 = version.MustParseSemantic("v1.9.0-alpha.1") // InitFeatureGates are the default feature gates for the init command var InitFeatureGates = FeatureList{ - SelfHosting: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Beta}}, - StoreCertsInSecrets: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}}, - HighAvailability: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190}, + SelfHosting: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Beta}}, + StoreCertsInSecrets: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}}, + // We don't want to advertise this feature gate exists in v1.9 to avoid confusion as it is not yet working + HighAvailability: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190, HiddenInHelpText: true}, CoreDNS: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190}, DynamicKubeletConfig: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190}, } @@ -57,7 +58,8 @@ var InitFeatureGates = FeatureList{ // Feature represents a feature being gated type Feature struct { utilfeature.FeatureSpec - MinimumVersion *version.Version + MinimumVersion *version.Version + HiddenInHelpText bool } // FeatureList represents a list of feature gates @@ -113,6 +115,10 @@ func Keys(featureList FeatureList) []string { func KnownFeatures(f *FeatureList) []string { var known []string for k, v := range *f { + if v.HiddenInHelpText { + continue + } + pre := "" if v.PreRelease != utilfeature.GA { pre = fmt.Sprintf("%s - ", v.PreRelease) diff --git a/cmd/kubeadm/app/phases/kubelet/kubelet.go b/cmd/kubeadm/app/phases/kubelet/kubelet.go index 7d2ac05515..aec20293b6 100644 --- a/cmd/kubeadm/app/phases/kubelet/kubelet.go +++ b/cmd/kubeadm/app/phases/kubelet/kubelet.go @@ -43,7 +43,7 @@ import ( // CreateBaseKubeletConfiguration creates base kubelet configuration for dynamic kubelet configuration feature. func CreateBaseKubeletConfiguration(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error { - fmt.Printf("[kubelet] Uploading a ConfigMap %q in namespace %s with base configuration for the kubelets in the cluster", + fmt.Printf("[kubelet] Uploading a ConfigMap %q in namespace %s with base configuration for the kubelets in the cluster\n", kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.NamespaceSystem) _, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs() @@ -95,7 +95,7 @@ func ConsumeBaseKubeletConfiguration(nodeName string) error { // updateNodeWithConfigMap updates node ConfigSource with KubeletBaseConfigurationConfigMap func updateNodeWithConfigMap(client clientset.Interface, nodeName string) error { - fmt.Printf("[kubelet] Using Dynamic Kubelet Config for node %q; config sourced from ConfigMap %q in namespace %s", + fmt.Printf("[kubelet] Using Dynamic Kubelet Config for node %q; config sourced from ConfigMap %q in namespace %s\n", nodeName, kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.NamespaceSystem) // Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned. @@ -203,7 +203,7 @@ func getLocalNodeTLSBootstrappedClient() (clientset.Interface, error) { // WriteInitKubeletConfigToDiskOnMaster writes base kubelet configuration to disk on master. func WriteInitKubeletConfigToDiskOnMaster(cfg *kubeadmapi.MasterConfiguration) error { - fmt.Printf("[kubelet] Writing base configuration of kubelets to disk on master node %s", cfg.NodeName) + fmt.Printf("[kubelet] Writing base configuration of kubelets to disk on master node %s\n", cfg.NodeName) _, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs() if err != nil { diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go index 4e8e83072a..18ae9563f8 100644 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go +++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go @@ -52,7 +52,7 @@ const ( // 8. In order to avoid race conditions, we have to make sure that static pod is deleted correctly before we continue // Otherwise, there is a race condition when we proceed without kubelet having restarted the API server correctly and the next .Create call flakes // 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop -func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubeadmapi.MasterConfiguration, client clientset.Interface, waiter apiclient.Waiter) error { +func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubeadmapi.MasterConfiguration, client clientset.Interface, waiter apiclient.Waiter, dryRun bool) error { // Adjust the timeout slightly to something self-hosting specific waiter.SetTimeout(selfHostingWaitTimeout) @@ -104,9 +104,11 @@ func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubea return err } - // Remove the old Static Pod manifest - if err := os.RemoveAll(manifestPath); err != nil { - return fmt.Errorf("unable to delete static pod manifest for %s [%v]", componentName, err) + // Remove the old Static Pod manifest if not dryrunning + if !dryRun { + if err := os.RemoveAll(manifestPath); err != nil { + return fmt.Errorf("unable to delete static pod manifest for %s [%v]", componentName, err) + } } // Wait for the mirror Pod hash to be removed; otherwise we'll run into race conditions here when the kubelet hasn't had time to diff --git a/cmd/kubeadm/app/phases/upgrade/BUILD b/cmd/kubeadm/app/phases/upgrade/BUILD index 907707d127..644794563b 100644 --- a/cmd/kubeadm/app/phases/upgrade/BUILD +++ b/cmd/kubeadm/app/phases/upgrade/BUILD @@ -36,6 +36,7 @@ go_library( "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", "//cmd/kubeadm/app/util/config:go_default_library", + "//cmd/kubeadm/app/util/dryrun:go_default_library", "//pkg/api/legacyscheme:go_default_library", "//pkg/util/version:go_default_library", "//pkg/version:go_default_library", diff --git a/cmd/kubeadm/app/phases/upgrade/policy.go b/cmd/kubeadm/app/phases/upgrade/policy.go index 594596ad8c..185aa6675a 100644 --- a/cmd/kubeadm/app/phases/upgrade/policy.go +++ b/cmd/kubeadm/app/phases/upgrade/policy.go @@ -28,6 +28,9 @@ const ( // MaximumAllowedMinorVersionUpgradeSkew describes how many minor versions kubeadm can upgrade the control plane version in one go MaximumAllowedMinorVersionUpgradeSkew = 1 + // MaximumAllowedMinorVersionDowngradeSkew describes how many minor versions kubeadm can upgrade the control plane version in one go + MaximumAllowedMinorVersionDowngradeSkew = 1 + // MaximumAllowedMinorVersionKubeletSkew describes how many minor versions the control plane version and the kubelet can skew in a kubeadm cluster MaximumAllowedMinorVersionKubeletSkew = 1 ) @@ -72,23 +75,41 @@ func EnforceVersionPolicies(versionGetter VersionGetter, newK8sVersionStr string skewErrors.Mandatory = append(skewErrors.Mandatory, fmt.Errorf("Specified version to upgrade to %q is equal to or lower than the minimum supported version %q. Please specify a higher version to upgrade to", newK8sVersionStr, clusterVersionStr)) } - // Make sure new version is higher than the current Kubernetes version - if clusterVersion.AtLeast(newK8sVersion) { - // Even though we don't officially support downgrades, it "should work", and if user(s) need it and are willing to try; they can do so with --force - skewErrors.Skippable = append(skewErrors.Skippable, fmt.Errorf("Specified version to upgrade to %q is equal to or lower than the cluster version %q. Downgrades are not supported yet", newK8sVersionStr, clusterVersionStr)) - } else { - // If this code path runs, it's an upgrade (this code will run most of the time) - // kubeadm doesn't support upgrades between two minor versions; e.g. a v1.7 -> v1.9 upgrade is not supported. Enforce that here - if newK8sVersion.Minor() > clusterVersion.Minor()+MaximumAllowedMinorVersionUpgradeSkew { - skewErrors.Mandatory = append(skewErrors.Mandatory, fmt.Errorf("Specified version to upgrade to %q is too high; kubeadm can upgrade only %d minor version at a time", newK8sVersionStr, MaximumAllowedMinorVersionUpgradeSkew)) + // kubeadm doesn't support upgrades between two minor versions; e.g. a v1.7 -> v1.9 upgrade is not supported right away + if newK8sVersion.Minor() > clusterVersion.Minor()+MaximumAllowedMinorVersionUpgradeSkew { + tooLargeUpgradeSkewErr := fmt.Errorf("Specified version to upgrade to %q is too high; kubeadm can upgrade only %d minor version at a time", newK8sVersionStr, MaximumAllowedMinorVersionUpgradeSkew) + // If the version that we're about to upgrade to is a released version, we should fully enforce this policy + // If the version is a CI/dev/experimental version, it's okay to jump two minor version steps, but then require the -f flag + if len(newK8sVersion.PreRelease()) == 0 { + skewErrors.Mandatory = append(skewErrors.Mandatory, tooLargeUpgradeSkewErr) + } else { + skewErrors.Skippable = append(skewErrors.Skippable, tooLargeUpgradeSkewErr) + } + } + + // kubeadm doesn't support downgrades between two minor versions; e.g. a v1.9 -> v1.7 downgrade is not supported right away + if newK8sVersion.Minor() < clusterVersion.Minor()-MaximumAllowedMinorVersionDowngradeSkew { + tooLargeDowngradeSkewErr := fmt.Errorf("Specified version to downgrade to %q is too low; kubeadm can downgrade only %d minor version at a time", newK8sVersionStr, MaximumAllowedMinorVersionDowngradeSkew) + // If the version that we're about to downgrade to is a released version, we should fully enforce this policy + // If the version is a CI/dev/experimental version, it's okay to jump two minor version steps, but then require the -f flag + if len(newK8sVersion.PreRelease()) == 0 { + skewErrors.Mandatory = append(skewErrors.Mandatory, tooLargeDowngradeSkewErr) + } else { + skewErrors.Skippable = append(skewErrors.Skippable, tooLargeDowngradeSkewErr) } } // If the kubeadm version is lower than what we want to upgrade to; error if kubeadmVersion.LessThan(newK8sVersion) { if newK8sVersion.Minor() > kubeadmVersion.Minor() { - // This is totally unsupported; kubeadm has no idea how it should handle a newer minor release than itself - skewErrors.Mandatory = append(skewErrors.Mandatory, fmt.Errorf("Specified version to upgrade to %q is one minor release higher than the kubeadm minor release (%d > %d). Such an upgrade is not supported", newK8sVersionStr, newK8sVersion.Minor(), kubeadmVersion.Minor())) + tooLargeKubeadmSkew := fmt.Errorf("Specified version to upgrade to %q is at least one minor release higher than the kubeadm minor release (%d > %d). Such an upgrade is not supported", newK8sVersionStr, newK8sVersion.Minor(), kubeadmVersion.Minor()) + // This is unsupported; kubeadm has no idea how it should handle a newer minor release than itself + // If the version is a CI/dev/experimental version though, lower the severity of this check, but then require the -f flag + if len(newK8sVersion.PreRelease()) == 0 { + skewErrors.Mandatory = append(skewErrors.Mandatory, tooLargeKubeadmSkew) + } else { + skewErrors.Skippable = append(skewErrors.Skippable, tooLargeKubeadmSkew) + } } else { // Upgrading to a higher patch version than kubeadm is ok if the user specifies --force. Not recommended, but possible. skewErrors.Skippable = append(skewErrors.Skippable, fmt.Errorf("Specified version to upgrade to %q is higher than the kubeadm version %q. Upgrade kubeadm first using the tool you used to install kubeadm", newK8sVersionStr, kubeadmVersionStr)) diff --git a/cmd/kubeadm/app/phases/upgrade/policy_test.go b/cmd/kubeadm/app/phases/upgrade/policy_test.go index 9b81609000..64f6d0e692 100644 --- a/cmd/kubeadm/app/phases/upgrade/policy_test.go +++ b/cmd/kubeadm/app/phases/upgrade/policy_test.go @@ -46,23 +46,21 @@ func TestEnforceVersionPolicies(t *testing.T) { }, newK8sVersion: "v1.9.0", }, - { // downgrades not supported + { // downgrades ok vg: &fakeVersionGetter{ clusterVersion: "v1.8.3", kubeletVersion: "v1.8.3", kubeadmVersion: "v1.8.3", }, - newK8sVersion: "v1.8.2", - expectedSkippableErrs: 1, + newK8sVersion: "v1.8.2", }, - { // upgrades without bumping the version number not supported yet. TODO: Change this? + { // upgrades without bumping the version number ok vg: &fakeVersionGetter{ clusterVersion: "v1.8.3", kubeletVersion: "v1.8.3", kubeadmVersion: "v1.8.3", }, - newK8sVersion: "v1.8.3", - expectedSkippableErrs: 1, + newK8sVersion: "v1.8.3", }, { // new version must be higher than v1.8.0 vg: &fakeVersionGetter{ @@ -72,7 +70,6 @@ func TestEnforceVersionPolicies(t *testing.T) { }, newK8sVersion: "v1.7.10", expectedMandatoryErrs: 1, // version must be higher than v1.8.0 - expectedSkippableErrs: 1, // version shouldn't be downgraded }, { // upgrading two minor versions in one go is not supported vg: &fakeVersionGetter{ @@ -84,6 +81,15 @@ func TestEnforceVersionPolicies(t *testing.T) { expectedMandatoryErrs: 1, // can't upgrade two minor versions expectedSkippableErrs: 1, // kubelet <-> apiserver skew too large }, + { // downgrading two minor versions in one go is not supported + vg: &fakeVersionGetter{ + clusterVersion: "v1.10.3", + kubeletVersion: "v1.10.3", + kubeadmVersion: "v1.10.0", + }, + newK8sVersion: "v1.8.3", + expectedMandatoryErrs: 1, // can't downgrade two minor versions + }, { // kubeadm version must be higher than the new kube version. However, patch version skews may be forced vg: &fakeVersionGetter{ clusterVersion: "v1.8.3", diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go index 2eae65ab1a..b16a2fbe6e 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go @@ -18,6 +18,8 @@ package upgrade import ( "fmt" + "os" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/errors" @@ -31,14 +33,16 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" "k8s.io/kubernetes/pkg/util/version" ) // PerformPostUpgradeTasks runs nearly the same functions as 'kubeadm init' would do // Note that the markmaster phase is left out, not needed, and no token is created as that doesn't belong to the upgrade -func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterConfiguration, newK8sVer *version.Version) error { +func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterConfiguration, newK8sVer *version.Version, dryRun bool) error { errs := []error{} // Upload currently used configuration to the cluster @@ -63,6 +67,11 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterC errs = append(errs, err) } + // Upgrade to a self-hosted control plane if possible + if err := upgradeToSelfHosting(client, cfg, newK8sVer, dryRun); err != nil { + errs = append(errs, err) + } + // TODO: Is this needed to do here? I think that updating cluster info should probably be separate from a normal upgrade // Create the cluster-info ConfigMap with the associated RBAC rules // if err := clusterinfo.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil { @@ -92,9 +101,11 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterC if err := dns.EnsureDNSAddon(cfg, client); err != nil { errs = append(errs, err) } - - if err := coreDNSDeployment(cfg, client); err != nil { - errs = append(errs, err) + // Remove the old kube-dns deployment if coredns is now used + if !dryRun { + if err := removeOldKubeDNSDeploymentIfCoreDNSIsUsed(cfg, client); err != nil { + errs = append(errs, err) + } } if err := proxy.EnsureProxyAddon(cfg, client); err != nil { @@ -103,22 +114,41 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterC return errors.NewAggregate(errs) } -func coreDNSDeployment(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error { +func removeOldKubeDNSDeploymentIfCoreDNSIsUsed(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error { if features.Enabled(cfg.FeatureGates, features.CoreDNS) { return apiclient.TryRunCommand(func() error { - getCoreDNS, err := client.AppsV1beta2().Deployments(metav1.NamespaceSystem).Get(kubeadmconstants.CoreDNS, metav1.GetOptions{}) + coreDNSDeployment, err := client.AppsV1beta2().Deployments(metav1.NamespaceSystem).Get(kubeadmconstants.CoreDNS, metav1.GetOptions{}) if err != nil { return err } - if getCoreDNS.Status.ReadyReplicas == 0 { + if coreDNSDeployment.Status.ReadyReplicas == 0 { return fmt.Errorf("the CodeDNS deployment isn't ready yet") } - err = client.AppsV1beta2().Deployments(metav1.NamespaceSystem).Delete(kubeadmconstants.KubeDNS, nil) - if err != nil { - return err - } - return nil - }, 5) + return apiclient.DeleteDeploymentForeground(client, metav1.NamespaceSystem, kubeadmconstants.KubeDNS) + }, 10) } return nil } + +func upgradeToSelfHosting(client clientset.Interface, cfg *kubeadmapi.MasterConfiguration, newK8sVer *version.Version, dryRun bool) error { + if features.Enabled(cfg.FeatureGates, features.SelfHosting) && !IsControlPlaneSelfHosted(client) && newK8sVer.AtLeast(v190alpha3) { + + waiter := getWaiter(dryRun, client) + + // kubeadm will now convert the static Pod-hosted control plane into a self-hosted one + fmt.Println("[self-hosted] Creating self-hosted control plane.") + if err := selfhosting.CreateSelfHostedControlPlane(kubeadmconstants.GetStaticPodDirectory(), kubeadmconstants.KubernetesDir, cfg, client, waiter, dryRun); err != nil { + return fmt.Errorf("error creating self hosted control plane: %v", err) + } + } + return nil +} + +// getWaiter gets the right waiter implementation for the right occasion +// TODO: Consolidate this with what's in init.go? +func getWaiter(dryRun bool, client clientset.Interface) apiclient.Waiter { + if dryRun { + return dryrunutil.NewWaiter() + } + return apiclient.NewKubeWaiter(client, 30*time.Minute, os.Stdout) +} diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade_v18_19.go b/cmd/kubeadm/app/phases/upgrade/postupgrade_v18_19.go index 8243a8100f..ef8f5d305d 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade_v18_19.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade_v18_19.go @@ -30,7 +30,9 @@ import ( "k8s.io/kubernetes/pkg/util/version" ) +// TODO: Maybe move these constants elsewhere in future releases var v190 = version.MustParseSemantic("v1.9.0") +var v190alpha3 = version.MustParseSemantic("v1.9.0-alpha.3") var expiry = 180 * 24 * time.Hour // backupAPIServerCertAndKey backups the old cert and key of kube-apiserver to a specified directory. diff --git a/cmd/kubeadm/app/util/apiclient/idempotency.go b/cmd/kubeadm/app/util/apiclient/idempotency.go index a2f11a7445..319ad5c1b9 100644 --- a/cmd/kubeadm/app/util/apiclient/idempotency.go +++ b/cmd/kubeadm/app/util/apiclient/idempotency.go @@ -107,6 +107,15 @@ func DeleteDaemonSetForeground(client clientset.Interface, namespace, name strin return client.AppsV1beta2().DaemonSets(namespace).Delete(name, deleteOptions) } +// DeleteDeploymentForeground deletes the specified Deployment in foreground mode; i.e. it blocks until/makes sure all the managed Pods are deleted +func DeleteDeploymentForeground(client clientset.Interface, namespace, name string) error { + foregroundDelete := metav1.DeletePropagationForeground + deleteOptions := &metav1.DeleteOptions{ + PropagationPolicy: &foregroundDelete, + } + return client.AppsV1beta2().Deployments(namespace).Delete(name, deleteOptions) +} + // CreateOrUpdateRole creates a Role if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. func CreateOrUpdateRole(client clientset.Interface, role *rbac.Role) error { if _, err := client.RbacV1().Roles(role.ObjectMeta.Namespace).Create(role); err != nil { diff --git a/cmd/kubeadm/app/util/error.go b/cmd/kubeadm/app/util/error.go index c27ab860d7..61327cf560 100644 --- a/cmd/kubeadm/app/util/error.go +++ b/cmd/kubeadm/app/util/error.go @@ -80,7 +80,7 @@ func checkErr(prefix string, err error, handleErr func(string, int)) { func FormatErrMsg(errs []error) string { var errMsg string for _, err := range errs { - errMsg = fmt.Sprintf("%s\t-%s\n", errMsg, err.Error()) + errMsg = fmt.Sprintf("%s\t- %s\n", errMsg, err.Error()) } return errMsg } diff --git a/cmd/kubeadm/app/util/error_test.go b/cmd/kubeadm/app/util/error_test.go index 831007550a..c28a6cc056 100644 --- a/cmd/kubeadm/app/util/error_test.go +++ b/cmd/kubeadm/app/util/error_test.go @@ -64,13 +64,13 @@ func TestFormatErrMsg(t *testing.T) { fmt.Errorf(errMsg1), fmt.Errorf(errMsg2), }, - expect: "\t-" + errMsg1 + "\n" + "\t-" + errMsg2 + "\n", + expect: "\t- " + errMsg1 + "\n" + "\t- " + errMsg2 + "\n", }, { errs: []error{ fmt.Errorf(errMsg1), }, - expect: "\t-" + errMsg1 + "\n", + expect: "\t- " + errMsg1 + "\n", }, }