diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go index d49f08395e..30cbbbfb9b 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -148,6 +148,13 @@ func runApply(flags *applyFlags, userVersion string) error { return errors.Wrap(err, "[upgrade/version] FATAL") } + // block if the local etcd manifest is listening on local host only and the user explicitly opted out from etcd upgrade. + // this is necessary because we want all the user to move to the new etcd manifest with v1.14. + // N.B. this code is necessary only in v1.14; starting from v1.15 all the etcd manifests should have 2 endpoints + if cfg.Etcd.External == nil && etcdutil.IsEtcdListeningOnLocalHostOnly() && !flags.etcdUpgrade { + return errors.New("kubeadm detected that the local etcd member is still listening only on localhost. Please upgrade etcd to avoid problems with new releases of kubeadm") + } + // If the current session is interactive, ask the user whether they really want to upgrade. if flags.sessionIsInteractive() { if err := InteractivelyConfirmUpgrade("Are you sure you want to proceed with the upgrade?"); err != nil { diff --git a/cmd/kubeadm/app/phases/certs/renewal/renewal.go b/cmd/kubeadm/app/phases/certs/renewal/renewal.go index 7c8950c7e3..538c49379a 100644 --- a/cmd/kubeadm/app/phases/certs/renewal/renewal.go +++ b/cmd/kubeadm/app/phases/certs/renewal/renewal.go @@ -18,6 +18,7 @@ package renewal import ( "crypto/x509" + "net" "github.com/pkg/errors" certutil "k8s.io/client-go/util/cert" @@ -60,3 +61,43 @@ func certToConfig(cert *x509.Certificate) *certutil.Config { Usages: cert.ExtKeyUsage, } } + +// RenewAndMutateExistingEtcdServerCert loads a certificate file, uses the renew interface to renew it, +// and saves the resulting certificate and key over the old one. +// This method differs from usual RenewExistingCert because it checks if the etcd server certificate +// includes the advertiseAddress in the SANS list; if not, the certificate is mutated in order to include it. +// N.B. this code is necessary only in v1.14; starting from v1.15 all the etcd manifests should have 2 endpoints +func RenewAndMutateExistingEtcdServerCert(certsDir, baseName string, advertiseAddress net.IP, impl Interface) error { + certificatePath, _ := pkiutil.PathsForCertAndKey(certsDir, baseName) + certs, err := certutil.CertsFromFile(certificatePath) + if err != nil { + return errors.Wrapf(err, "failed to load existing certificate %s", baseName) + } + + if len(certs) != 1 { + return errors.Errorf("wanted exactly one certificate from %s, got %d", baseName, len(certs)) + } + + cfg := certToConfig(certs[0]) + + hasAdvertiseAddress := false + for _, val := range cfg.AltNames.IPs { + if val.Equal(advertiseAddress) { + hasAdvertiseAddress = true + break + } + } + if !hasAdvertiseAddress { + cfg.AltNames.IPs = append(cfg.AltNames.IPs, advertiseAddress) + } + + newCert, newKey, err := impl.Renew(cfg) + if err != nil { + return errors.Wrapf(err, "failed to renew certificate %s", baseName) + } + + if err := pkiutil.WriteCertAndKey(certsDir, baseName, newCert, newKey); err != nil { + return errors.Wrapf(err, "failed to write new certificate %s", baseName) + } + return nil +} diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods.go b/cmd/kubeadm/app/phases/upgrade/staticpods.go index ef1344cff2..7eb0970be3 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods.go @@ -18,6 +18,7 @@ package upgrade import ( "fmt" + "net" "os" "strings" "time" @@ -263,15 +264,30 @@ func performEtcdStaticPodUpgrade(client clientset.Interface, waiter apiclient.Wa } // gets the etcd version of the local/stacked etcd member running on the current machine + // the version is read from che cluster; this should take into account that there are still + // around old etcd manifest with etcd listening on local host only + // N.B. taking care of old etcd manifests is necessary only in v1.14; starting from v1.15 all the etcd manifest should have 2 endpoints currentEtcdVersions, err := oldEtcdClient.GetClusterVersions() if err != nil { return true, errors.Wrap(err, "failed to retrieve the current etcd version") } - currentEtcdVersionStr, ok := currentEtcdVersions[etcdutil.GetClientURL(&cfg.LocalAPIEndpoint)] - if !ok { - return true, errors.Wrap(err, "failed to retrieve the current etcd version") - } + var ok bool + var currentEtcdVersionStr string + if etcdutil.IsEtcdListeningOnLocalHostOnly() { + // in case of etcd listening on local host only, there could be only etcd member in the cluster, and so + // also in the currentEtcdVersions map; we are using a for to take the value of the first element + for _, v := range currentEtcdVersions { + currentEtcdVersionStr = v + break + } + } else { + // otherwise take the etcd version of the etcd member hosted on the current machine + currentEtcdVersionStr, ok = currentEtcdVersions[etcdutil.GetClientURL(&cfg.LocalAPIEndpoint)] + if !ok { + return true, errors.Wrap(err, "failed to retrieve the current etcd version") + } + } currentEtcdVersion, err := version.ParseSemantic(currentEtcdVersionStr) if err != nil { return true, errors.Wrapf(err, "failed to parse the current etcd version(%s)", currentEtcdVersionStr) @@ -500,6 +516,21 @@ func renewCerts(cfg *kubeadmapi.InitConfiguration, component string) error { &certsphase.KubeadmCertEtcdPeer, &certsphase.KubeadmCertEtcdHealthcheck, } { + if cert.BaseName == constants.EtcdServerCertAndKeyBaseName { + // When renewing the etcd server certificate it is necessary to mutate it from listening on + // localhost only to listening on localhost and API server advertise address (if not already the case) + // N.B. this code is necessary only in v1.14; starting from v1.15 all the etcd manifest should have 2 endpoints + advertiseAddress := net.ParseIP(cfg.LocalAPIEndpoint.AdvertiseAddress) + if advertiseAddress == nil { + return errors.Errorf("error parsing LocalAPIEndpoint AdvertiseAddress %q: is not a valid textual representation of an IP address", cfg.LocalAPIEndpoint.AdvertiseAddress) + } + + if err := renewal.RenewAndMutateExistingEtcdServerCert(cfg.CertificatesDir, cert.BaseName, advertiseAddress, renewer); err != nil { + return errors.Wrapf(err, "failed to renew %s certificate and key", certsphase.KubeadmCertEtcdServer.Name) + } + + continue + } if err := renewal.RenewExistingCert(cfg.CertificatesDir, cert.BaseName, renewer); err != nil { return errors.Wrapf(err, "failed to renew %s certificate and key", cert.Name) } diff --git a/cmd/kubeadm/app/util/etcd/BUILD b/cmd/kubeadm/app/util/etcd/BUILD index 3168a0c50e..60160dd9a7 100644 --- a/cmd/kubeadm/app/util/etcd/BUILD +++ b/cmd/kubeadm/app/util/etcd/BUILD @@ -9,6 +9,7 @@ go_library( "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/util/config:go_default_library", + "//cmd/kubeadm/app/util/staticpod:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//vendor/github.com/coreos/etcd/clientv3:go_default_library", "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library", diff --git a/cmd/kubeadm/app/util/etcd/etcd.go b/cmd/kubeadm/app/util/etcd/etcd.go index d6656d7a7a..e6ab6152aa 100644 --- a/cmd/kubeadm/app/util/etcd/etcd.go +++ b/cmd/kubeadm/app/util/etcd/etcd.go @@ -19,8 +19,10 @@ package etcd import ( "context" "crypto/tls" + "fmt" "net" "net/url" + "os" "path/filepath" "strconv" "strings" @@ -34,6 +36,7 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/util/config" + "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" ) // ClusterInterrogator is an interface to get etcd cluster related information @@ -75,10 +78,50 @@ func New(endpoints []string, ca, cert, key string) (*Client, error) { return &client, nil } +// IsEtcdListeningOnLocalHostOnly return true if the etcd manifest have etcd listening on localhost only. +// Listening on local host only was the default in kubeadm <= v1.12, while starting from v1.13 etcd is listening +// on localhost and API server advertise address (thus allowing add new member when doing join --control-plane). +// N.B. this code is necessary only in v1.14; starting from v1.15 all the etcd manifest should have 2 endpoints +func IsEtcdListeningOnLocalHostOnly() bool { + etcdManifestFile := constants.GetStaticPodFilepath(constants.Etcd, constants.GetStaticPodDirectory()) + if _, err := os.Stat(etcdManifestFile); err == nil { + klog.V(1).Infoln("checking etcd manifest") + etcdPod, err := staticpod.ReadStaticPodFromDisk(etcdManifestFile) + if err == nil && len(etcdPod.Spec.Containers) > 0 { + etcdContainer := etcdPod.Spec.Containers[0] + for _, arg := range etcdContainer.Command { + if arg == "--listen-client-urls=https://127.0.0.1:2379" { + klog.V(1).Infoln("etcd manifest created by kubeadm v1.12 or older") + return true + } + } + } + } + + return false +} + // NewFromCluster creates an etcd client for the etcd endpoints defined in the ClusterStatus value stored in // the kubeadm-config ConfigMap in kube-system namespace. // Once created, the client synchronizes client's endpoints with the known endpoints from the etcd membership API (reality check). func NewFromCluster(client clientset.Interface, certificatesDir string) (*Client, error) { + // if etcd is listening on localhost only, connect to it + if IsEtcdListeningOnLocalHostOnly() { + endpoints := []string{fmt.Sprintf("localhost:%d", constants.EtcdListenClientPort)} + + etcdClient, err := New( + endpoints, + filepath.Join(certificatesDir, constants.EtcdCACertName), + filepath.Join(certificatesDir, constants.EtcdHealthcheckClientCertName), + filepath.Join(certificatesDir, constants.EtcdHealthcheckClientKeyName), + ) + if err != nil { + return nil, errors.Wrapf(err, "error creating etcd client for %v endpoint", endpoints) + } + + return etcdClient, nil + } + // etcd is listening the API server advertise address on each control-plane node // so it is necessary to get the list of endpoints from kubeadm cluster status before connecting