From 7168f7282cd6cd56874e195b9fcb8ec00c4e1139 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Thu, 14 Apr 2022 17:31:49 -0700 Subject: [PATCH] Move IPv4/v6 selection into helpers Signed-off-by: Brad Davidson (cherry picked from commit b12cd6293575bbb0d415eaac4d4473b891f8731c) --- pkg/agent/loadbalancer/loadbalancer.go | 14 ++++-- pkg/cli/server/server.go | 13 +---- pkg/daemons/config/types.go | 32 +++++++++++++ pkg/daemons/control/deps/deps.go | 1 - pkg/etcd/etcd.go | 36 +++----------- pkg/server/server.go | 66 +++----------------------- 6 files changed, 55 insertions(+), 107 deletions(-) diff --git a/pkg/agent/loadbalancer/loadbalancer.go b/pkg/agent/loadbalancer/loadbalancer.go index 063785531c..efd64e445b 100644 --- a/pkg/agent/loadbalancer/loadbalancer.go +++ b/pkg/agent/loadbalancer/loadbalancer.go @@ -3,10 +3,10 @@ package loadbalancer import ( "context" "errors" + "fmt" "net" "os" "path/filepath" - "strconv" "sync" "github.com/k3s-io/k3s/pkg/version" @@ -42,11 +42,13 @@ var ( func New(ctx context.Context, dataDir, serviceName, serverURL string, lbServerPort int, isIPv6 bool) (_lb *LoadBalancer, _err error) { config := net.ListenConfig{Control: reusePort} - localhostAddress := "127.0.0.1" + var localAddress string if isIPv6 { - localhostAddress = "[::1]" + localAddress = fmt.Sprintf("[::1]:%d", lbServerPort) + } else { + localAddress = fmt.Sprintf("127.0.0.1:%d", lbServerPort) } - listener, err := config.Listen(ctx, "tcp", localhostAddress+":"+strconv.Itoa(lbServerPort)) + listener, err := config.Listen(ctx, "tcp", localAddress) defer func() { if _err != nil { logrus.Warnf("Error starting load balancer: %s", _err) @@ -58,7 +60,9 @@ func New(ctx context.Context, dataDir, serviceName, serverURL string, lbServerPo if err != nil { return nil, err } - localAddress := listener.Addr().String() + + // if lbServerPort was 0, the port was assigned by the OS when bound - see what we ended up with. + localAddress = listener.Addr().String() defaultServerAddress, localServerURL, err := parseURL(serverURL, localAddress) if err != nil { diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index ceb9902c4d..e738d6e024 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -464,18 +464,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont } }() - ip := serverConfig.ControlConfig.BindAddress - if ip == "" { - ip = "127.0.0.1" - if IPv6only { - ip = "::1" - } - } - if utilsnet.IsIPv6String(ip) { - ip = fmt.Sprintf("[%s]", ip) - } - - url := fmt.Sprintf("https://%s:%d", ip, serverConfig.ControlConfig.SupervisorPort) + url := fmt.Sprintf("https://%s:%d", serverConfig.ControlConfig.BindAddressOrLoopback(false), serverConfig.ControlConfig.SupervisorPort) token, err := clientaccess.FormatToken(serverConfig.ControlConfig.Runtime.AgentToken, serverConfig.ControlConfig.Runtime.ServerCA) if err != nil { return err diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 5d5fdb89ee..f1b122ed57 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -10,10 +10,12 @@ import ( "strings" "time" + "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/kine/pkg/endpoint" "github.com/rancher/wrangler/pkg/generated/controllers/core" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apiserver/pkg/authentication/authenticator" + utilsnet "k8s.io/utils/net" ) const ( @@ -195,6 +197,36 @@ type Control struct { Runtime *ControlRuntime `json:"-"` } +// BindAddressOrLoopback returns an IPv4 or IPv6 address suitable for embedding in server +// URLs. If a bind address was configured, that is returned. If the chooseHostInterface +// parameter is true, and a suitable default interface can be found, that interface's +// address is returned. If neither of the previous were used, the loopback address is +// returned. IPv6 addresses are enclosed in square brackets, as per RFC2732. +func (c *Control) BindAddressOrLoopback(chooseHostInterface bool) string { + ip := c.BindAddress + if ip == "" && chooseHostInterface { + if hostIP, _ := utilnet.ChooseHostInterface(); len(hostIP) > 0 { + ip = hostIP.String() + } + } + if utilsnet.IsIPv6String(ip) { + return fmt.Sprintf("[%s]", ip) + } else if ip != "" { + return ip + } + return c.Loopback() +} + +// Loopback returns an IPv4 or IPv6 loopback address, depending on whether the cluster +// service CIDRs indicate an IPv4/Dual-Stack or IPv6 only cluster. IPv6 addresses are +// enclosed in square brackets, as per RFC2732. +func (c *Control) Loopback() string { + if IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(c.ServiceIPRanges); IPv6OnlyService { + return "[::1]" + } + return "127.0.0.1" +} + type ControlRuntimeBootstrap struct { ETCDServerCA string ETCDServerCAKey string diff --git a/pkg/daemons/control/deps/deps.go b/pkg/daemons/control/deps/deps.go index 86fcfa4bbe..e48004af03 100644 --- a/pkg/daemons/control/deps/deps.go +++ b/pkg/daemons/control/deps/deps.go @@ -28,7 +28,6 @@ import ( "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apiserver/pkg/apis/apiserver" apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1" "k8s.io/apiserver/pkg/authentication/user" ) diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index 403e56daa4..32986336cf 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "io/ioutil" + "net" "net/http" "net/url" "os" @@ -47,12 +48,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/client-go/util/retry" - utilsnet "k8s.io/utils/net" ) const ( - defaultEndpoint = "https://127.0.0.1:2379" - defaultEndpointv6 = "https://[::1]:2379" testTimeout = time.Second * 10 manageTickerTime = time.Second * 15 learnerMaxStallTime = time.Minute * 5 @@ -138,13 +136,6 @@ func NewETCD() *ETCD { } } -func getLocalhostAddress(address string) string { - if utilsnet.IsIPv6String(address) { - return "[::1]" - } - return "127.0.0.1" -} - // EndpointName returns the name of the endpoint. func (e *ETCD) EndpointName() string { return "etcd" @@ -655,10 +646,7 @@ func getEndpoints(control *config.Control) []string { if len(runtime.EtcdConfig.Endpoints) > 0 { return runtime.EtcdConfig.Endpoints } - if utilsnet.IsIPv6String(control.PrivateIP) { - return []string{defaultEndpointv6} - } - return []string{defaultEndpoint} + return []string{fmt.Sprintf("https://%s:2379", control.Loopback())} } // toTLSConfig converts the ControlRuntime configuration to TLS configuration suitable @@ -767,29 +755,19 @@ func (e *ETCD) migrateFromSQLite(ctx context.Context) error { // peerURL returns the peer access address for the local node func (e *ETCD) peerURL() string { - if utilsnet.IsIPv6String(e.address) { - return fmt.Sprintf("https://[%s]:2380", e.address) - } - return fmt.Sprintf("https://%s:2380", e.address) + return fmt.Sprintf("https://%s", net.JoinHostPort(e.address, "2380")) } // clientURL returns the client access address for the local node func (e *ETCD) clientURL() string { - if utilsnet.IsIPv6String(e.address) { - return fmt.Sprintf("https://[%s]:2379", e.address) - } - return fmt.Sprintf("https://%s:2379", e.address) + return fmt.Sprintf("https://%s", net.JoinHostPort(e.address, "2379")) } // metricsURL returns the metrics access address func (e *ETCD) metricsURL(expose bool) string { - address := fmt.Sprintf("http://%s:2381", getLocalhostAddress(e.address)) + address := fmt.Sprintf("http://%s:2381", e.config.Loopback()) if expose { - if utilsnet.IsIPv6String(e.address) { - address = fmt.Sprintf("http://[%s]:2381,%s", e.address, address) - } else { - address = fmt.Sprintf("http://%s:2381,%s", e.address, address) - } + address = fmt.Sprintf("http://%s,%s", net.JoinHostPort(e.address, "2381"), address) } return address } @@ -801,7 +779,7 @@ func (e *ETCD) cluster(ctx context.Context, forceNew bool, options executor.Init Name: e.name, InitialOptions: options, ForceNewCluster: forceNew, - ListenClientURLs: e.clientURL() + "," + fmt.Sprintf("https://%s:2379", getLocalhostAddress(e.address)), + ListenClientURLs: e.clientURL() + "," + fmt.Sprintf("https://%s:2379", e.config.Loopback()), ListenMetricsURLs: e.metricsURL(e.config.EtcdExposeMetrics), ListenPeerURLs: e.peerURL(), AdvertiseClientURLs: e.clientURL(), diff --git a/pkg/server/server.go b/pkg/server/server.go index cc2b4f1eed..7504450cc3 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io/ioutil" - net2 "net" "os" "path" "path/filepath" @@ -35,8 +34,6 @@ import ( "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/net" - utilsnet "k8s.io/utils/net" ) const ( @@ -85,22 +82,7 @@ func StartServer(ctx context.Context, config *Config, cfg *cmds.Server) error { go startOnAPIServerReady(ctx, wg, config) } - ip := net2.ParseIP(config.ControlConfig.BindAddress) - if ip == nil { - hostIP, err := net.ChooseHostInterface() - if err == nil { - ip = hostIP - } else { - IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(config.ControlConfig.ServiceIPRanges) - if IPv6OnlyService { - ip = net2.ParseIP("::1") - } else { - ip = net2.ParseIP("127.0.0.1") - } - } - } - - if err := printTokens(ip.String(), &config.ControlConfig); err != nil { + if err := printTokens(&config.ControlConfig); err != nil { return err } @@ -328,20 +310,10 @@ func HomeKubeConfig(write, rootless bool) (string, error) { return resolvehome.Resolve(datadir.HomeConfig) } -func printTokens(advertiseIP string, config *config.Control) error { +func printTokens(config *config.Control) error { var ( nodeFile string ) - - if advertiseIP == "" { - IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(config.ServiceIPRanges) - if IPv6OnlyService { - advertiseIP = "::1" - } else { - advertiseIP = "127.0.0.1" - } - } - if len(config.Runtime.ServerToken) > 0 { p := filepath.Join(config.DataDir, "token") if err := writeToken(config.Runtime.ServerToken, p, config.Runtime.ServerCA); err == nil { @@ -362,33 +334,18 @@ func printTokens(advertiseIP string, config *config.Control) error { } if len(nodeFile) > 0 { - printToken(config.SupervisorPort, advertiseIP, "To join node to cluster:", "agent") + printToken(config.SupervisorPort, config.BindAddressOrLoopback(true), "To join node to cluster:", "agent") } return nil } func writeKubeConfig(certs string, config *Config) error { - ip := config.ControlConfig.BindAddress - if ip == "" { - IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(config.ControlConfig.ServiceIPRanges) - if IPv6OnlyService { - ip = "[::1]" - } else { - ip = "127.0.0.1" - } - } else if utilsnet.IsIPv6String(ip) { - ip = fmt.Sprintf("[%s]", ip) - } + ip := config.ControlConfig.BindAddressOrLoopback(false) port := config.ControlConfig.HTTPSPort // on servers without a local apiserver, tunnel access via the loadbalancer if config.ControlConfig.DisableAPIServer { - IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(config.ControlConfig.ServiceIPRanges) - if IPv6OnlyService { - ip = "[::1]" - } else { - ip = "127.0.0.1" - } + ip = config.ControlConfig.Loopback() port = config.ControlConfig.APIServerPort } url := fmt.Sprintf("https://%s:%d", ip, port) @@ -465,18 +422,7 @@ func setupDataDirAndChdir(config *config.Control) error { } func printToken(httpsPort int, advertiseIP, prefix, cmd string) { - ip := advertiseIP - if ip == "" { - hostIP, err := net.ChooseHostInterface() - if err != nil { - logrus.Errorf("Failed to choose interface: %v", err) - } - ip = hostIP.String() - } else if utilsnet.IsIPv6String(ip) { - ip = fmt.Sprintf("[%s]", ip) - } - - logrus.Infof("%s %s %s -s https://%s:%d -t ${NODE_TOKEN}", prefix, version.Program, cmd, ip, httpsPort) + logrus.Infof("%s %s %s -s https://%s:%d -t ${NODE_TOKEN}", prefix, version.Program, cmd, advertiseIP, httpsPort) } func writeToken(token, file, certs string) error {