Add --flannel-backend flag

pkg/agent/config/config.go

@@ -330,8 +330,8 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 
 	nodeConfig := &config.Node{
 		Docker:                   envInfo.Docker,
-		NoFlannel:                envInfo.NoFlannel,
 		ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint,
+		FlannelBackend:           controlConfig.FlannelBackend,
 	}
 	nodeConfig.FlannelIface = flannelIface
 	nodeConfig.Images = filepath.Join(envInfo.DataDir, "images")
@@ -349,6 +349,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 	nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
 	nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet")
 	nodeConfig.AgentConfig.PauseImage = envInfo.PauseImage
+	nodeConfig.AgentConfig.IPSECPSK = controlConfig.IPSECPSK
 	nodeConfig.CACerts = info.CACerts
 	nodeConfig.Containerd.Config = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml")
 	nodeConfig.Containerd.Root = filepath.Join(envInfo.DataDir, "containerd")
@@ -361,6 +362,13 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 	nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml.tmpl")
 	nodeConfig.ServerAddress = serverURLParsed.Host
 	nodeConfig.Certificate = servingCert
+
+	if nodeConfig.FlannelBackend == config.FlannelBackendNone {
+		nodeConfig.NoFlannel = true
+	} else {
+		nodeConfig.NoFlannel = envInfo.NoFlannel
+	}
+
 	if !nodeConfig.NoFlannel {
 		if envInfo.FlannelConf == "" {
 			nodeConfig.FlannelConf = filepath.Join(envInfo.DataDir, "etc/flannel/net-conf.json")

pkg/agent/flannel/setup.go

@@ -2,6 +2,7 @@ package flannel
 
 import (
 	"context"
+	"fmt"
 	"path/filepath"
 	"strings"
 	"time"
@@ -36,27 +37,45 @@ const (
   ]
 }
 `
-	netJSON = `{
+
-    "Network": "%CIDR%",
+	flannelConf = `{
-    "Backend": {
+	"Network": "%CIDR%",
-    "Type": "vxlan"
+	"Backend": %backend%
-    }
 }
 `
+
+	vxlanBackend = `{
+	"Type": "vxlan"
+}`
+
+	ipsecBackend = `{
+	"Type": "ipsec",
+	"UDPEncap": true,
+	"PSK": "%psk%"
+}`
+
+	wireguardBackend = `{
+	"Type": "extension",
+	"PreStartupCommand": "wg genkey | tee privatekey | wg pubkey",
+	"PostStartupCommand": "export SUBNET_IP=$(echo $SUBNET | cut -d'/' -f 1); ip link del flannel.1 2>/dev/null; echo $PATH >&2; wg-add.sh flannel.1 && wg set flannel.1 listen-port 51820 private-key privatekey && ip addr add $SUBNET_IP/32 dev flannel.1 && ip link set flannel.1 up && ip route add $NETWORK dev flannel.1",
+	"ShutdownCommand": "ip link del flannel.1",
+	"SubnetAddCommand": "read PUBLICKEY; wg set flannel.1 peer $PUBLICKEY endpoint $PUBLIC_IP:51820 allowed-ips $SUBNET",
+	"SubnetRemoveCommand": "read PUBLICKEY; wg set flannel.1 peer $PUBLICKEY remove"
+}`
 )
 
-func Prepare(ctx context.Context, config *config.Node) error {
+func Prepare(ctx context.Context, nodeConfig *config.Node) error {
-	if err := createCNIConf(config.AgentConfig.CNIConfDir); err != nil {
+	if err := createCNIConf(nodeConfig.AgentConfig.CNIConfDir); err != nil {
 		return err
 	}
 
-	return createFlannelConf(config)
+	return createFlannelConf(nodeConfig)
 }
 
-func Run(ctx context.Context, config *config.Node) error {
+func Run(ctx context.Context, nodeConfig *config.Node) error {
-	nodeName := config.AgentConfig.NodeName
+	nodeName := nodeConfig.AgentConfig.NodeName
 
-	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode)
+	restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigNode)
 	if err != nil {
 		return err
 	}
@@ -80,7 +99,7 @@ func Run(ctx context.Context, config *config.Node) error {
 	}
 
 	go func() {
-		err := flannel(ctx, config.FlannelIface, config.FlannelConf, config.AgentConfig.KubeConfigNode)
+		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigNode)
 		logrus.Fatalf("flannel exited: %v", err)
 	}()
 
@@ -95,14 +114,29 @@ func createCNIConf(dir string) error {
 	return util.WriteFile(p, cniConf)
 }
 
-func createFlannelConf(config *config.Node) error {
+func createFlannelConf(nodeConfig *config.Node) error {
-	if config.FlannelConf == "" {
+	if nodeConfig.FlannelConf == "" {
 		return nil
 	}
-	if config.FlannelConfOverride {
+	if nodeConfig.FlannelConfOverride {
-		logrus.Infof("Using custom flannel conf defined at %s", config.FlannelConf)
+		logrus.Infof("Using custom flannel conf defined at %s", nodeConfig.FlannelConf)
 		return nil
 	}
-	return util.WriteFile(config.FlannelConf,
+	confJSON := strings.Replace(flannelConf, "%CIDR%", nodeConfig.AgentConfig.ClusterCIDR.String(), -1)
-		strings.Replace(netJSON, "%CIDR%", config.AgentConfig.ClusterCIDR.String(), -1))
+
+	var backendConf string
+
+	switch nodeConfig.FlannelBackend {
+	case config.FlannelBackendVXLAN:
+		backendConf = vxlanBackend
+	case config.FlannelBackendIPSEC:
+		backendConf = strings.Replace(ipsecBackend, "%psk%", nodeConfig.AgentConfig.IPSECPSK, -1)
+	case config.FlannelBackendWireguard:
+		backendConf = wireguardBackend
+	default:
+		return fmt.Errorf("Cannot configure unknown flannel backend '%s'", nodeConfig.FlannelBackend)
+	}
+	confJSON = strings.Replace(confJSON, "%backend%", backendConf, -1)
+
+	return util.WriteFile(nodeConfig.FlannelConf, confJSON)
 }

pkg/cli/cmds/server.go

@@ -1,6 +1,9 @@
 package cmds
 
 import (
+	"fmt"
+
+	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/urfave/cli"
 )
 
@@ -30,6 +33,7 @@ type Server struct {
 	AdvertiseIP         string
 	AdvertisePort       int
 	DisableScheduler    bool
+	FlannelBackend      string
 }
 
 var ServerConfig Server
@@ -189,6 +193,12 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				Usage:       "Disable Kubernetes default scheduler",
 				Destination: &ServerConfig.DisableScheduler,
 			},
+			cli.StringFlag{
+				Name:        "flannel-backend",
+				Usage:       fmt.Sprintf("One of '%s', '%s', '%s', or '%s'", config.FlannelBackendNone, config.FlannelBackendVXLAN, config.FlannelBackendIPSEC, config.FlannelBackendWireguard),
+				Destination: &ServerConfig.FlannelBackend,
+				Value:       config.FlannelBackendVXLAN,
+			},
 			NodeIPFlag,
 			NodeNameFlag,
 			DockerFlag,

pkg/cli/server/server.go

@@ -84,6 +84,7 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP
 	serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
 	serverConfig.ControlConfig.BootstrapReadOnly = !cfg.StoreBootstrap
+	serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend
 
 	if cmds.AgentConfig.FlannelIface != "" && cmds.AgentConfig.NodeIP == "" {
 		cmds.AgentConfig.NodeIP = netutil.GetIPFromInterface(cmds.AgentConfig.FlannelIface)

pkg/daemons/config/types.go

@@ -13,10 +13,18 @@ import (
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 )
 
+const (
+	FlannelBackendNone      = "none"
+	FlannelBackendVXLAN     = "vxlan"
+	FlannelBackendIPSEC     = "ipsec"
+	FlannelBackendWireguard = "wireguard"
+)
+
 type Node struct {
 	Docker                   bool
 	ContainerRuntimeEndpoint string
 	NoFlannel                bool
+	FlannelBackend           string
 	FlannelConf              string
 	FlannelConfOverride      bool
 	FlannelIface             *net.Interface
@@ -66,6 +74,7 @@ type Agent struct {
 	CNIPlugin           bool
 	NodeTaints          []string
 	NodeLabels          []string
+	IPSECPSK            string
 }
 
 type Control struct {
@@ -90,6 +99,8 @@ type Control struct {
 	ExtraControllerArgs   []string
 	ExtraSchedulerAPIArgs []string
 	NoLeaderElect         bool
+	FlannelBackend        string
+	IPSECPSK              string
 
 	Runtime *ControlRuntime `json:"-"`
 }
@@ -106,6 +117,7 @@ type ControlRuntimeBootstrap struct {
 	ClientKubeletKey   string
 	ClientKubeProxyKey string
 	ServingKubeletKey  string
+	IPSECKey           string
 }
 
 type ControlRuntime struct {

pkg/daemons/control/server.go

@@ -62,6 +62,11 @@ users:
 `))
 )
 
+const (
+	userTokenSize  = 16
+	ipsecTokenSize = 48
+)
+
 func Server(ctx context.Context, cfg *config.Control) error {
 	rand.Seed(time.Now().UTC().UnixNano())
 
@@ -252,6 +257,7 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
 	runtime.ServerCAKey = path.Join(config.DataDir, "tls", "server-ca.key")
 	runtime.RequestHeaderCA = path.Join(config.DataDir, "tls", "request-header-ca.crt")
 	runtime.RequestHeaderCAKey = path.Join(config.DataDir, "tls", "request-header-ca.key")
+	runtime.IPSECKey = path.Join(config.DataDir, "cred", "ipsec.psk")
 
 	runtime.ServiceKey = path.Join(config.DataDir, "tls", "service.key")
 	runtime.PasswdFile = path.Join(config.DataDir, "cred", "passwd")
@@ -304,6 +310,10 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
 		return err
 	}
 
+	if err := genEncryptedNetworkInfo(config, runtime); err != nil {
+		return err
+	}
+
 	if err := storeBootstrapData(ctx, config, etcdClient); err != nil {
 		return err
 	}
@@ -422,20 +432,43 @@ func WritePasswords(passwdFile string, records [][]string) error {
 	return os.Rename(passwdFile+".tmp", passwdFile)
 }
 
+func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.ControlRuntime) error {
+	if s, err := os.Stat(runtime.IPSECKey); err == nil && s.Size() > 0 {
+		psk, err := ioutil.ReadFile(runtime.IPSECKey)
+		if err != nil {
+			return err
+		}
+		controlConfig.IPSECPSK = strings.TrimSpace(string(psk))
+		return nil
+	}
+
+	psk, err := getToken(ipsecTokenSize)
+	if err != nil {
+		return err
+	}
+
+	controlConfig.IPSECPSK = psk
+	if err := ioutil.WriteFile(runtime.IPSECKey, []byte(psk+"\n"), 0600); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
 	if s, err := os.Stat(runtime.PasswdFile); err == nil && s.Size() > 0 {
 		return ensureNodeToken(config, runtime)
 	}
 
-	adminToken, err := getToken()
+	adminToken, err := getToken(userTokenSize)
 	if err != nil {
 		return err
 	}
-	systemToken, err := getToken()
+	systemToken, err := getToken(userTokenSize)
 	if err != nil {
 		return err
 	}
-	nodeToken, err := getToken()
+	nodeToken, err := getToken(userTokenSize)
 	if err != nil {
 		return err
 	}
@@ -451,8 +484,8 @@ func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
 	})
 }
 
-func getToken() (string, error) {
+func getToken(size int) (string, error) {
-	token := make([]byte, 16, 16)
+	token := make([]byte, size, size)
 	_, err := cryptorand.Read(token)
 	if err != nil {
 		return "", err

scripts/download

@@ -16,3 +16,5 @@ mkdir -p bin/aux && rm bin/mount && ln -sf ../busybox bin/aux/mount
 
 TRAEFIK_FILE=traefik-${TRAEFIK_VERSION}.tgz
 curl -sfL https://kubernetes-charts.storage.googleapis.com/${TRAEFIK_FILE} -o ${CHARTS_DIR}/${TRAEFIK_FILE}
+
+cp scripts/wg-add.sh bin/aux/

scripts/wg-add.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+auto-mtu() {
+	local mtu=0 endpoint output
+	while read -r _ endpoint; do
+		[[ $endpoint =~ ^\[?([a-z0-9:.]+)\]?:[0-9]+$ ]] || continue
+		output="$(ip route get "${BASH_REMATCH[1]}" || true)"
+		[[ ( $output =~ mtu\ ([0-9]+) || ( $output =~ dev\ ([^ ]+) && $(ip link show dev "${BASH_REMATCH[1]}") =~ mtu\ ([0-9]+) ) ) && ${BASH_REMATCH[1]} -gt $mtu ]] && mtu="${BASH_REMATCH[1]}"
+	done < <(wg show "$1" endpoints)
+	if [[ $mtu -eq 0 ]]; then
+		read -r output < <(ip route show default || true) || true
+		[[ ( $output =~ mtu\ ([0-9]+) || ( $output =~ dev\ ([^ ]+) && $(ip link show dev "${BASH_REMATCH[1]}") =~ mtu\ ([0-9]+) ) ) && ${BASH_REMATCH[1]} -gt $mtu ]] && mtu="${BASH_REMATCH[1]}"
+	fi
+	[[ $mtu -gt 0 ]] || mtu=1500
+	ip link set mtu $(( mtu - 80 )) up dev "$1"
+}
+
+# probe for any modules that may be needed
+modprobe wireguard 
+modprobe tun
+
+# try wireguard kernel module first
+ip link add "$1" type wireguard && exit
+
+# try boringtun and let it drop privileges
+boringtun "$1" && auto-mtu "$1" && exit
+
+# try boringtun w/o dropping privileges
+WG_SUDO=1 boringtun "$1" && auto-mtu "$1" && exit
+
+# try wireguard-go - p.s. should not use wireguard-go, it leaks memory
+WG_I_PREFER_BUGGY_USERSPACE_TO_POLISHED_KMOD=1 wireguard-go "$1" && auto-mtu "$1" && exit
+
+exit 1