Move IPv4/v6 selection into helpers

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
(cherry picked from commit b12cd62935)

pkg/agent/loadbalancer/loadbalancer.go

@@ -3,10 +3,10 @@ package loadbalancer
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net"
 	"os"
 	"path/filepath"
-	"strconv"
 	"sync"
 
 	"github.com/k3s-io/k3s/pkg/version"
@@ -42,11 +42,13 @@ var (
 
 func New(ctx context.Context, dataDir, serviceName, serverURL string, lbServerPort int, isIPv6 bool) (_lb *LoadBalancer, _err error) {
 	config := net.ListenConfig{Control: reusePort}
-	localhostAddress := "127.0.0.1"
+	var localAddress string
 	if isIPv6 {
-		localhostAddress = "[::1]"
+		localAddress = fmt.Sprintf("[::1]:%d", lbServerPort)
+	} else {
+		localAddress = fmt.Sprintf("127.0.0.1:%d", lbServerPort)
 	}
-	listener, err := config.Listen(ctx, "tcp", localhostAddress+":"+strconv.Itoa(lbServerPort))
+	listener, err := config.Listen(ctx, "tcp", localAddress)
 	defer func() {
 		if _err != nil {
 			logrus.Warnf("Error starting load balancer: %s", _err)
@@ -58,7 +60,9 @@ func New(ctx context.Context, dataDir, serviceName, serverURL string, lbServerPo
 	if err != nil {
 		return nil, err
 	}
-	localAddress := listener.Addr().String()
+
+	// if lbServerPort was 0, the port was assigned by the OS when bound - see what we ended up with.
+	localAddress = listener.Addr().String()
 
 	defaultServerAddress, localServerURL, err := parseURL(serverURL, localAddress)
 	if err != nil {

pkg/cli/server/server.go

@@ -464,18 +464,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 		}
 	}()
 
-	ip := serverConfig.ControlConfig.BindAddress
+	url := fmt.Sprintf("https://%s:%d", serverConfig.ControlConfig.BindAddressOrLoopback(false), serverConfig.ControlConfig.SupervisorPort)
-	if ip == "" {
-		ip = "127.0.0.1"
-		if IPv6only {
-			ip = "::1"
-		}
-	}
-	if utilsnet.IsIPv6String(ip) {
-		ip = fmt.Sprintf("[%s]", ip)
-	}
-
-	url := fmt.Sprintf("https://%s:%d", ip, serverConfig.ControlConfig.SupervisorPort)
 	token, err := clientaccess.FormatToken(serverConfig.ControlConfig.Runtime.AgentToken, serverConfig.ControlConfig.Runtime.ServerCA)
 	if err != nil {
 		return err

pkg/daemons/config/types.go

@@ -10,10 +10,12 @@ import (
 	"strings"
 	"time"
 
+	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/kine/pkg/endpoint"
 	"github.com/rancher/wrangler/pkg/generated/controllers/core"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
+	utilsnet "k8s.io/utils/net"
 )
 
 const (
@@ -195,6 +197,36 @@ type Control struct {
 	Runtime     *ControlRuntime `json:"-"`
 }
 
+// BindAddressOrLoopback returns an IPv4 or IPv6 address suitable for embedding in server
+// URLs. If a bind address was configured, that is returned. If the chooseHostInterface
+// parameter is true, and a suitable default interface can be found, that interface's
+// address is returned.  If neither of the previous were used, the loopback address is
+// returned. IPv6 addresses are enclosed in square brackets, as per RFC2732.
+func (c *Control) BindAddressOrLoopback(chooseHostInterface bool) string {
+	ip := c.BindAddress
+	if ip == "" && chooseHostInterface {
+		if hostIP, _ := utilnet.ChooseHostInterface(); len(hostIP) > 0 {
+			ip = hostIP.String()
+		}
+	}
+	if utilsnet.IsIPv6String(ip) {
+		return fmt.Sprintf("[%s]", ip)
+	} else if ip != "" {
+		return ip
+	}
+	return c.Loopback()
+}
+
+// Loopback returns an IPv4 or IPv6 loopback address, depending on whether the cluster
+// service CIDRs indicate an IPv4/Dual-Stack or IPv6 only cluster.  IPv6 addresses are
+// enclosed in square brackets, as per RFC2732.
+func (c *Control) Loopback() string {
+	if IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(c.ServiceIPRanges); IPv6OnlyService {
+		return "[::1]"
+	}
+	return "127.0.0.1"
+}
+
 type ControlRuntimeBootstrap struct {
 	ETCDServerCA       string
 	ETCDServerCAKey    string

pkg/daemons/control/deps/deps.go

@@ -28,7 +28,6 @@ import (
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/apiserver/pkg/apis/apiserver"
 	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
 	"k8s.io/apiserver/pkg/authentication/user"
 )

pkg/etcd/etcd.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -47,12 +48,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/client-go/util/retry"
-	utilsnet "k8s.io/utils/net"
 )
 
 const (
-	defaultEndpoint      = "https://127.0.0.1:2379"
-	defaultEndpointv6    = "https://[::1]:2379"
 	testTimeout          = time.Second * 10
 	manageTickerTime     = time.Second * 15
 	learnerMaxStallTime  = time.Minute * 5
@@ -138,13 +136,6 @@ func NewETCD() *ETCD {
 	}
 }
 
-func getLocalhostAddress(address string) string {
-	if utilsnet.IsIPv6String(address) {
-		return "[::1]"
-	}
-	return "127.0.0.1"
-}
-
 // EndpointName returns the name of the endpoint.
 func (e *ETCD) EndpointName() string {
 	return "etcd"
@@ -655,10 +646,7 @@ func getEndpoints(control *config.Control) []string {
 	if len(runtime.EtcdConfig.Endpoints) > 0 {
 		return runtime.EtcdConfig.Endpoints
 	}
-	if utilsnet.IsIPv6String(control.PrivateIP) {
+	return []string{fmt.Sprintf("https://%s:2379", control.Loopback())}
-		return []string{defaultEndpointv6}
-	}
-	return []string{defaultEndpoint}
 }
 
 // toTLSConfig converts the ControlRuntime configuration to TLS configuration suitable
@@ -767,29 +755,19 @@ func (e *ETCD) migrateFromSQLite(ctx context.Context) error {
 
 // peerURL returns the peer access address for the local node
 func (e *ETCD) peerURL() string {
-	if utilsnet.IsIPv6String(e.address) {
+	return fmt.Sprintf("https://%s", net.JoinHostPort(e.address, "2380"))
-		return fmt.Sprintf("https://[%s]:2380", e.address)
-	}
-	return fmt.Sprintf("https://%s:2380", e.address)
 }
 
 // clientURL returns the client access address for the local node
 func (e *ETCD) clientURL() string {
-	if utilsnet.IsIPv6String(e.address) {
+	return fmt.Sprintf("https://%s", net.JoinHostPort(e.address, "2379"))
-		return fmt.Sprintf("https://[%s]:2379", e.address)
-	}
-	return fmt.Sprintf("https://%s:2379", e.address)
 }
 
 // metricsURL returns the metrics access address
 func (e *ETCD) metricsURL(expose bool) string {
-	address := fmt.Sprintf("http://%s:2381", getLocalhostAddress(e.address))
+	address := fmt.Sprintf("http://%s:2381", e.config.Loopback())
 	if expose {
-		if utilsnet.IsIPv6String(e.address) {
+		address = fmt.Sprintf("http://%s,%s", net.JoinHostPort(e.address, "2381"), address)
-			address = fmt.Sprintf("http://[%s]:2381,%s", e.address, address)
-		} else {
-			address = fmt.Sprintf("http://%s:2381,%s", e.address, address)
-		}
 	}
 	return address
 }
@@ -801,7 +779,7 @@ func (e *ETCD) cluster(ctx context.Context, forceNew bool, options executor.Init
 		Name:                e.name,
 		InitialOptions:      options,
 		ForceNewCluster:     forceNew,
-		ListenClientURLs:    e.clientURL() + "," + fmt.Sprintf("https://%s:2379", getLocalhostAddress(e.address)),
+		ListenClientURLs:    e.clientURL() + "," + fmt.Sprintf("https://%s:2379", e.config.Loopback()),
 		ListenMetricsURLs:   e.metricsURL(e.config.EtcdExposeMetrics),
 		ListenPeerURLs:      e.peerURL(),
 		AdvertiseClientURLs: e.clientURL(),

pkg/server/server.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	net2 "net"
 	"os"
 	"path"
 	"path/filepath"
@@ -35,8 +34,6 @@ import (
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/net"
-	utilsnet "k8s.io/utils/net"
 )
 
 const (
@@ -85,22 +82,7 @@ func StartServer(ctx context.Context, config *Config, cfg *cmds.Server) error {
 		go startOnAPIServerReady(ctx, wg, config)
 	}
 
-	ip := net2.ParseIP(config.ControlConfig.BindAddress)
+	if err := printTokens(&config.ControlConfig); err != nil {
-	if ip == nil {
-		hostIP, err := net.ChooseHostInterface()
-		if err == nil {
-			ip = hostIP
-		} else {
-			IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(config.ControlConfig.ServiceIPRanges)
-			if IPv6OnlyService {
-				ip = net2.ParseIP("::1")
-			} else {
-				ip = net2.ParseIP("127.0.0.1")
-			}
-		}
-	}
-
-	if err := printTokens(ip.String(), &config.ControlConfig); err != nil {
 		return err
 	}
 
@@ -328,20 +310,10 @@ func HomeKubeConfig(write, rootless bool) (string, error) {
 	return resolvehome.Resolve(datadir.HomeConfig)
 }
 
-func printTokens(advertiseIP string, config *config.Control) error {
+func printTokens(config *config.Control) error {
 	var (
 		nodeFile string
 	)
-
-	if advertiseIP == "" {
-		IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(config.ServiceIPRanges)
-		if IPv6OnlyService {
-			advertiseIP = "::1"
-		} else {
-			advertiseIP = "127.0.0.1"
-		}
-	}
-
 	if len(config.Runtime.ServerToken) > 0 {
 		p := filepath.Join(config.DataDir, "token")
 		if err := writeToken(config.Runtime.ServerToken, p, config.Runtime.ServerCA); err == nil {
@@ -362,33 +334,18 @@ func printTokens(advertiseIP string, config *config.Control) error {
 	}
 
 	if len(nodeFile) > 0 {
-		printToken(config.SupervisorPort, advertiseIP, "To join node to cluster:", "agent")
+		printToken(config.SupervisorPort, config.BindAddressOrLoopback(true), "To join node to cluster:", "agent")
 	}
 
 	return nil
 }
 
 func writeKubeConfig(certs string, config *Config) error {
-	ip := config.ControlConfig.BindAddress
+	ip := config.ControlConfig.BindAddressOrLoopback(false)
-	if ip == "" {
-		IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(config.ControlConfig.ServiceIPRanges)
-		if IPv6OnlyService {
-			ip = "[::1]"
-		} else {
-			ip = "127.0.0.1"
-		}
-	} else if utilsnet.IsIPv6String(ip) {
-		ip = fmt.Sprintf("[%s]", ip)
-	}
 	port := config.ControlConfig.HTTPSPort
 	// on servers without a local apiserver, tunnel access via the loadbalancer
 	if config.ControlConfig.DisableAPIServer {
-		IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(config.ControlConfig.ServiceIPRanges)
+		ip = config.ControlConfig.Loopback()
-		if IPv6OnlyService {
-			ip = "[::1]"
-		} else {
-			ip = "127.0.0.1"
-		}
 		port = config.ControlConfig.APIServerPort
 	}
 	url := fmt.Sprintf("https://%s:%d", ip, port)
@@ -465,18 +422,7 @@ func setupDataDirAndChdir(config *config.Control) error {
 }
 
 func printToken(httpsPort int, advertiseIP, prefix, cmd string) {
-	ip := advertiseIP
+	logrus.Infof("%s %s %s -s https://%s:%d -t ${NODE_TOKEN}", prefix, version.Program, cmd, advertiseIP, httpsPort)
-	if ip == "" {
-		hostIP, err := net.ChooseHostInterface()
-		if err != nil {
-			logrus.Errorf("Failed to choose interface: %v", err)
-		}
-		ip = hostIP.String()
-	} else if utilsnet.IsIPv6String(ip) {
-		ip = fmt.Sprintf("[%s]", ip)
-	}
-
-	logrus.Infof("%s %s %s -s https://%s:%d -t ${NODE_TOKEN}", prefix, version.Program, cmd, ip, httpsPort)
 }
 
 func writeToken(token, file, certs string) error {