Merge pull request #75956 from fabriziopandini/fix-kubeadm-upgrade-12-13-14

kubeadm : fix-kubeadm-upgrade-12-13-14

cmd/kubeadm/app/cmd/upgrade/apply.go

@@ -148,6 +148,13 @@ func runApply(flags *applyFlags, userVersion string) error {
 		return errors.Wrap(err, "[upgrade/version] FATAL")
 	}
 
+	// block if the local etcd manifest is listening on local host only and the user explicitly opted out from etcd upgrade.
+	// this is necessary because we want all the user to move to the new etcd manifest with v1.14.
+	// N.B. this code is necessary only in v1.14; starting from v1.15 all the etcd manifests should have 2 endpoints
+	if cfg.Etcd.External == nil && etcdutil.IsEtcdListeningOnLocalHostOnly() && !flags.etcdUpgrade {
+		return errors.New("kubeadm detected that the local etcd member is still listening only on localhost. Please upgrade etcd to avoid problems with new releases of kubeadm")
+	}
+
 	// If the current session is interactive, ask the user whether they really want to upgrade.
 	if flags.sessionIsInteractive() {
 		if err := InteractivelyConfirmUpgrade("Are you sure you want to proceed with the upgrade?"); err != nil {

cmd/kubeadm/app/phases/certs/renewal/renewal.go

@@ -18,6 +18,7 @@ package renewal
 
 import (
 	"crypto/x509"
+	"net"
 
 	"github.com/pkg/errors"
 	certutil "k8s.io/client-go/util/cert"
@@ -60,3 +61,43 @@ func certToConfig(cert *x509.Certificate) *certutil.Config {
 		Usages: cert.ExtKeyUsage,
 	}
 }
+
+// RenewAndMutateExistingEtcdServerCert loads a certificate file, uses the renew interface to renew it,
+// and saves the resulting certificate and key over the old one.
+// This method differs from usual RenewExistingCert because it checks if the etcd server certificate
+// includes the advertiseAddress in the SANS list; if not, the certificate is mutated in order to include it.
+// N.B. this code is necessary only in v1.14; starting from v1.15 all the etcd manifests should have 2 endpoints
+func RenewAndMutateExistingEtcdServerCert(certsDir, baseName string, advertiseAddress net.IP, impl Interface) error {
+	certificatePath, _ := pkiutil.PathsForCertAndKey(certsDir, baseName)
+	certs, err := certutil.CertsFromFile(certificatePath)
+	if err != nil {
+		return errors.Wrapf(err, "failed to load existing certificate %s", baseName)
+	}
+
+	if len(certs) != 1 {
+		return errors.Errorf("wanted exactly one certificate from %s, got %d", baseName, len(certs))
+	}
+
+	cfg := certToConfig(certs[0])
+
+	hasAdvertiseAddress := false
+	for _, val := range cfg.AltNames.IPs {
+		if val.Equal(advertiseAddress) {
+			hasAdvertiseAddress = true
+			break
+		}
+	}
+	if !hasAdvertiseAddress {
+		cfg.AltNames.IPs = append(cfg.AltNames.IPs, advertiseAddress)
+	}
+
+	newCert, newKey, err := impl.Renew(cfg)
+	if err != nil {
+		return errors.Wrapf(err, "failed to renew certificate %s", baseName)
+	}
+
+	if err := pkiutil.WriteCertAndKey(certsDir, baseName, newCert, newKey); err != nil {
+		return errors.Wrapf(err, "failed to write new certificate %s", baseName)
+	}
+	return nil
+}

cmd/kubeadm/app/phases/upgrade/staticpods.go

@@ -18,6 +18,7 @@ package upgrade
 
 import (
 	"fmt"
+	"net"
 	"os"
 	"strings"
 	"time"
@@ -263,15 +264,30 @@ func performEtcdStaticPodUpgrade(client clientset.Interface, waiter apiclient.Wa
 	}
 
 	// gets the etcd version of the local/stacked etcd member running on the current machine
+	// the version is read from che cluster; this should take into account that there are still
+	// around old etcd manifest with etcd listening on local host only
+	// N.B. taking care of old etcd manifests is necessary only in v1.14; starting from v1.15 all the etcd manifest should have 2 endpoints
 	currentEtcdVersions, err := oldEtcdClient.GetClusterVersions()
 	if err != nil {
 		return true, errors.Wrap(err, "failed to retrieve the current etcd version")
 	}
-	currentEtcdVersionStr, ok := currentEtcdVersions[etcdutil.GetClientURL(&cfg.LocalAPIEndpoint)]
-	if !ok {
-		return true, errors.Wrap(err, "failed to retrieve the current etcd version")
-	}
 
+	var ok bool
+	var currentEtcdVersionStr string
+	if etcdutil.IsEtcdListeningOnLocalHostOnly() {
+		// in case of etcd listening on local host only, there could be only etcd member in the cluster, and so
+		// also in the currentEtcdVersions map; we are using a for to take the value of the first element
+		for _, v := range currentEtcdVersions {
+			currentEtcdVersionStr = v
+			break
+		}
+	} else {
+		// otherwise take the etcd version of the etcd member hosted on the current machine
+		currentEtcdVersionStr, ok = currentEtcdVersions[etcdutil.GetClientURL(&cfg.LocalAPIEndpoint)]
+		if !ok {
+			return true, errors.Wrap(err, "failed to retrieve the current etcd version")
+		}
+	}
 	currentEtcdVersion, err := version.ParseSemantic(currentEtcdVersionStr)
 	if err != nil {
 		return true, errors.Wrapf(err, "failed to parse the current etcd version(%s)", currentEtcdVersionStr)
@@ -500,6 +516,21 @@ func renewCerts(cfg *kubeadmapi.InitConfiguration, component string) error {
 					&certsphase.KubeadmCertEtcdPeer,
 					&certsphase.KubeadmCertEtcdHealthcheck,
 				} {
+					if cert.BaseName == constants.EtcdServerCertAndKeyBaseName {
+						// When renewing the etcd server certificate it is necessary to mutate it from listening on
+						// localhost only to listening on localhost and API server advertise address (if not already the case)
+						// N.B. this code is necessary only in v1.14; starting from v1.15 all the etcd manifest should have 2 endpoints
+						advertiseAddress := net.ParseIP(cfg.LocalAPIEndpoint.AdvertiseAddress)
+						if advertiseAddress == nil {
+							return errors.Errorf("error parsing LocalAPIEndpoint AdvertiseAddress %q: is not a valid textual representation of an IP address", cfg.LocalAPIEndpoint.AdvertiseAddress)
+						}
+
+						if err := renewal.RenewAndMutateExistingEtcdServerCert(cfg.CertificatesDir, cert.BaseName, advertiseAddress, renewer); err != nil {
+							return errors.Wrapf(err, "failed to renew %s certificate and key", certsphase.KubeadmCertEtcdServer.Name)
+						}
+
+						continue
+					}
 					if err := renewal.RenewExistingCert(cfg.CertificatesDir, cert.BaseName, renewer); err != nil {
 						return errors.Wrapf(err, "failed to renew %s certificate and key", cert.Name)
 					}

cmd/kubeadm/app/util/etcd/BUILD

@@ -9,6 +9,7 @@ go_library(
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/util/config:go_default_library",
+        "//cmd/kubeadm/app/util/staticpod:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/github.com/coreos/etcd/clientv3:go_default_library",
         "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library",

cmd/kubeadm/app/util/etcd/etcd.go

@@ -19,8 +19,10 @@ package etcd
 import (
 	"context"
 	"crypto/tls"
+	"fmt"
 	"net"
 	"net/url"
+	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -34,6 +36,7 @@ import (
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/config"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
 )
 
 // ClusterInterrogator is an interface to get etcd cluster related information
@@ -75,10 +78,50 @@ func New(endpoints []string, ca, cert, key string) (*Client, error) {
 	return &client, nil
 }
 
+// IsEtcdListeningOnLocalHostOnly return true if the etcd manifest have etcd listening on localhost only.
+// Listening on local host only was the default in kubeadm <= v1.12, while starting from v1.13 etcd is listening
+// on localhost and API server advertise address (thus allowing add new member when doing join --control-plane).
+// N.B. this code is necessary only in v1.14; starting from v1.15 all the etcd manifest should have 2 endpoints
+func IsEtcdListeningOnLocalHostOnly() bool {
+	etcdManifestFile := constants.GetStaticPodFilepath(constants.Etcd, constants.GetStaticPodDirectory())
+	if _, err := os.Stat(etcdManifestFile); err == nil {
+		klog.V(1).Infoln("checking etcd manifest")
+		etcdPod, err := staticpod.ReadStaticPodFromDisk(etcdManifestFile)
+		if err == nil && len(etcdPod.Spec.Containers) > 0 {
+			etcdContainer := etcdPod.Spec.Containers[0]
+			for _, arg := range etcdContainer.Command {
+				if arg == "--listen-client-urls=https://127.0.0.1:2379" {
+					klog.V(1).Infoln("etcd manifest created by kubeadm v1.12 or older")
+					return true
+				}
+			}
+		}
+	}
+
+	return false
+}
+
 // NewFromCluster creates an etcd client for the etcd endpoints defined in the ClusterStatus value stored in
 // the kubeadm-config ConfigMap in kube-system namespace.
 // Once created, the client synchronizes client's endpoints with the known endpoints from the etcd membership API (reality check).
 func NewFromCluster(client clientset.Interface, certificatesDir string) (*Client, error) {
+	// if etcd is listening on localhost only, connect to it
+	if IsEtcdListeningOnLocalHostOnly() {
+		endpoints := []string{fmt.Sprintf("localhost:%d", constants.EtcdListenClientPort)}
+
+		etcdClient, err := New(
+			endpoints,
+			filepath.Join(certificatesDir, constants.EtcdCACertName),
+			filepath.Join(certificatesDir, constants.EtcdHealthcheckClientCertName),
+			filepath.Join(certificatesDir, constants.EtcdHealthcheckClientKeyName),
+		)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error creating etcd client for %v endpoint", endpoints)
+		}
+
+		return etcdClient, nil
+	}
+
 	// etcd is listening the API server advertise address on each control-plane node
 	// so it is necessary to get the list of endpoints from kubeadm cluster status before connecting