From 493ebb95176cba7d1c38583821f6d9bafe2c3723 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Tue, 16 Jan 2024 22:43:08 +0000 Subject: [PATCH] Fix ipv6 endpoint address selection for on-demand snapshots Signed-off-by: Brad Davidson (cherry picked from commit 8224a3a7f6deb0ea75636d301396031e0a298456) Signed-off-by: Brad Davidson --- pkg/cli/etcdsnapshot/etcd_snapshot.go | 45 ++++++++++++++++++++++++++- pkg/cli/server/server.go | 1 + pkg/etcd/etcd.go | 11 ++++--- pkg/etcd/etcd_test.go | 2 +- 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/pkg/cli/etcdsnapshot/etcd_snapshot.go b/pkg/cli/etcdsnapshot/etcd_snapshot.go index 0c152bef59..d1d88af66a 100644 --- a/pkg/cli/etcdsnapshot/etcd_snapshot.go +++ b/pkg/cli/etcdsnapshot/etcd_snapshot.go @@ -3,8 +3,8 @@ package etcdsnapshot import ( "context" "encoding/json" - "errors" "fmt" + "net" "os" "path/filepath" "sort" @@ -17,7 +17,9 @@ import ( daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/etcd" "github.com/k3s-io/k3s/pkg/server" + "github.com/k3s-io/k3s/pkg/util" util2 "github.com/k3s-io/k3s/pkg/util" + "github.com/pkg/errors" "github.com/rancher/wrangler/pkg/signals" "github.com/urfave/cli" "gopkg.in/yaml.v2" @@ -52,6 +54,7 @@ func commandSetup(app *cli.Context, cfg *cmds.Server, config *server.Config) (*e config.DisableAgent = true config.ControlConfig.DataDir = dataDir + config.ControlConfig.BindAddress = cfg.BindAddress config.ControlConfig.EtcdSnapshotName = cfg.EtcdSnapshotName config.ControlConfig.EtcdSnapshotDir = cfg.EtcdSnapshotDir config.ControlConfig.EtcdSnapshotCompress = cfg.EtcdSnapshotCompress @@ -73,6 +76,46 @@ func commandSetup(app *cli.Context, cfg *cmds.Server, config *server.Config) (*e config.ControlConfig.Runtime.ClientETCDKey = filepath.Join(dataDir, "tls", "etcd", "client.key") config.ControlConfig.Runtime.KubeConfigAdmin = filepath.Join(dataDir, "cred", "admin.kubeconfig") + // We need to go through defaulting of cluster addresses to ensure that the etcd config for the standalone + // command uses the same endpoint selection logic as it does when starting up the full server. Specifically, + // we need to set an IPv6 service CIDR on IPv6-only or IPv6-first nodes, as the etcd default endpoints check + // the service CIDR primary addresss family to determine what loopback address to use. + _, nodeIPs, err := util.GetHostnameAndIPs(cmds.AgentConfig.NodeName, cmds.AgentConfig.NodeIP) + if err != nil { + return nil, err + } + + // configure ClusterIPRanges. Use default 10.42.0.0/16 or fd00:42::/56 if user did not set it + _, defaultClusterCIDR, defaultServiceCIDR, _ := util.GetDefaultAddresses(nodeIPs[0]) + if len(cfg.ClusterCIDR) == 0 { + cfg.ClusterCIDR.Set(defaultClusterCIDR) + } + for _, cidr := range util.SplitStringSlice(cfg.ClusterCIDR) { + _, parsed, err := net.ParseCIDR(cidr) + if err != nil { + return nil, errors.Wrapf(err, "invalid cluster-cidr %s", cidr) + } + config.ControlConfig.ClusterIPRanges = append(config.ControlConfig.ClusterIPRanges, parsed) + } + + // set ClusterIPRange to the first address (first defined IPFamily is preferred) + config.ControlConfig.ClusterIPRange = config.ControlConfig.ClusterIPRanges[0] + + // configure ServiceIPRanges. Use default 10.43.0.0/16 or fd00:43::/112 if user did not set it + if len(cfg.ServiceCIDR) == 0 { + cfg.ServiceCIDR.Set(defaultServiceCIDR) + } + for _, cidr := range util.SplitStringSlice(cfg.ServiceCIDR) { + _, parsed, err := net.ParseCIDR(cidr) + if err != nil { + return nil, errors.Wrapf(err, "invalid service-cidr %s", cidr) + } + config.ControlConfig.ServiceIPRanges = append(config.ControlConfig.ServiceIPRanges, parsed) + } + + // set ServiceIPRange to the first address (first defined IPFamily is preferred) + config.ControlConfig.ServiceIPRange = config.ControlConfig.ServiceIPRanges[0] + e := etcd.NewETCD() if err := e.SetControlConfig(&config.ControlConfig); err != nil { return nil, err diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 2d1a5ae2c6..bd62448f17 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -442,6 +442,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont serverConfig.ControlConfig.DisableControllerManager = true serverConfig.ControlConfig.DisableScheduler = true serverConfig.ControlConfig.DisableCCM = true + serverConfig.ControlConfig.DisableServiceLB = true // If the supervisor and apiserver are on the same port, everything is running embedded // and we don't need the kubelet or containerd up to perform a cluster reset. diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index 87541284a1..9a2e5fb010 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -400,6 +400,7 @@ func (e *ETCD) Reset(ctx context.Context, rebootstrap func() error) error { if err := os.WriteFile(e.ResetFile(), []byte{}, 0600); err != nil { return err } + return e.newCluster(ctx, true) } @@ -757,7 +758,7 @@ func getAdvertiseAddress(advertiseIP string) (string, error) { // newCluster returns options to set up etcd for a new cluster func (e *ETCD) newCluster(ctx context.Context, reset bool) error { - logrus.Infof("Starting etcd for new cluster") + logrus.Infof("Starting etcd for new cluster, cluster-reset=%v", reset) err := e.cluster(ctx, reset, executor.InitialOptions{ AdvertisePeerURL: e.peerURL(), Cluster: fmt.Sprintf("%s=%s", e.name, e.peerURL()), @@ -766,8 +767,10 @@ func (e *ETCD) newCluster(ctx context.Context, reset bool) error { if err != nil { return err } - if err := e.migrateFromSQLite(ctx); err != nil { - return fmt.Errorf("failed to migrate content from sqlite to etcd: %w", err) + if !reset { + if err := e.migrateFromSQLite(ctx); err != nil { + return fmt.Errorf("failed to migrate content from sqlite to etcd: %w", err) + } } return nil } @@ -848,7 +851,7 @@ func (e *ETCD) clientURL() string { // on other nodes connect mid-process. func (e *ETCD) advertiseClientURLs(reset bool) string { if reset { - return fmt.Sprintf("https://%s", net.JoinHostPort(e.config.Loopback(true), "2379")) + return fmt.Sprintf("https://%s:2379", e.config.Loopback(true)) } return e.clientURL() } diff --git a/pkg/etcd/etcd_test.go b/pkg/etcd/etcd_test.go index c3af06284a..7293d99c53 100644 --- a/pkg/etcd/etcd_test.go +++ b/pkg/etcd/etcd_test.go @@ -47,7 +47,7 @@ func generateTestConfig() *config.Control { EtcdSnapshotRetention: 5, EtcdS3Endpoint: "s3.amazonaws.com", EtcdS3Region: "us-east-1", - SANs: []string{"127.0.0.1"}, + SANs: []string{"127.0.0.1", mustGetAddress()}, CriticalControlArgs: criticalControlArgs, } }