github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add --flannel-backend flag

pull/719/head
Erik Wilson 2019-09-03 16:41:54 -07:00
parent 359a77939c
commit 959acf9c92
8 changed files with 158 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/agent/config/config.go
View File

@ -330,8 +330,8 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
nodeConfig := &config.Node{
Docker: envInfo.Docker,
NoFlannel: envInfo.NoFlannel,
ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint,
FlannelBackend: controlConfig.FlannelBackend,
}
nodeConfig.FlannelIface = flannelIface
nodeConfig.Images = filepath.Join(envInfo.DataDir, "images")
@ -349,6 +349,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet")
nodeConfig.AgentConfig.PauseImage = envInfo.PauseImage
nodeConfig.AgentConfig.IPSECPSK = controlConfig.IPSECPSK
nodeConfig.CACerts = info.CACerts
nodeConfig.Containerd.Config = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml")
nodeConfig.Containerd.Root = filepath.Join(envInfo.DataDir, "containerd")
@ -361,6 +362,13 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml.tmpl")
nodeConfig.ServerAddress = serverURLParsed.Host
nodeConfig.Certificate = servingCert
if nodeConfig.FlannelBackend == config.FlannelBackendNone {
nodeConfig.NoFlannel = true
} else {
nodeConfig.NoFlannel = envInfo.NoFlannel
}
if !nodeConfig.NoFlannel {
if envInfo.FlannelConf == "" {
nodeConfig.FlannelConf = filepath.Join(envInfo.DataDir, "etc/flannel/net-conf.json")

70
pkg/agent/flannel/setup.go
View File

@ -2,6 +2,7 @@ package flannel
import (
"context"
"fmt"
"path/filepath"
"strings"
"time"
@ -36,27 +37,45 @@ const (
]
}
`
netJSON = `{
"Network": "%CIDR%",
"Backend": {
"Type": "vxlan"
}
flannelConf = `{
"Network": "%CIDR%",
"Backend": %backend%
}
`
vxlanBackend = `{
"Type": "vxlan"
}`
ipsecBackend = `{
"Type": "ipsec",
"UDPEncap": true,
"PSK": "%psk%"
}`
wireguardBackend = `{
"Type": "extension",
"PreStartupCommand": "wg genkey | tee privatekey | wg pubkey",
"PostStartupCommand": "export SUBNET_IP=$(echo $SUBNET | cut -d'/' -f 1); ip link del flannel.1 2>/dev/null; echo $PATH >&2; wg-add.sh flannel.1 && wg set flannel.1 listen-port 51820 private-key privatekey && ip addr add $SUBNET_IP/32 dev flannel.1 && ip link set flannel.1 up && ip route add $NETWORK dev flannel.1",
"ShutdownCommand": "ip link del flannel.1",
"SubnetAddCommand": "read PUBLICKEY; wg set flannel.1 peer $PUBLICKEY endpoint $PUBLIC_IP:51820 allowed-ips $SUBNET",
"SubnetRemoveCommand": "read PUBLICKEY; wg set flannel.1 peer $PUBLICKEY remove"
}`
)
func Prepare(ctx context.Context, config *config.Node) error {
if err := createCNIConf(config.AgentConfig.CNIConfDir); err != nil {
func Prepare(ctx context.Context, nodeConfig *config.Node) error {
if err := createCNIConf(nodeConfig.AgentConfig.CNIConfDir); err != nil {
return err
}
return createFlannelConf(config)
return createFlannelConf(nodeConfig)
}
func Run(ctx context.Context, config *config.Node) error {
nodeName := config.AgentConfig.NodeName
func Run(ctx context.Context, nodeConfig *config.Node) error {
nodeName := nodeConfig.AgentConfig.NodeName
restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode)
restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigNode)
if err != nil {
return err
}
@ -80,7 +99,7 @@ func Run(ctx context.Context, config *config.Node) error {
}
go func() {
err := flannel(ctx, config.FlannelIface, config.FlannelConf, config.AgentConfig.KubeConfigNode)
err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigNode)
logrus.Fatalf("flannel exited: %v", err)
}()
@ -95,14 +114,29 @@ func createCNIConf(dir string) error {
return util.WriteFile(p, cniConf)
}
func createFlannelConf(config *config.Node) error {
if config.FlannelConf == "" {
func createFlannelConf(nodeConfig *config.Node) error {
if nodeConfig.FlannelConf == "" {
return nil
}
if config.FlannelConfOverride {
logrus.Infof("Using custom flannel conf defined at %s", config.FlannelConf)
if nodeConfig.FlannelConfOverride {
logrus.Infof("Using custom flannel conf defined at %s", nodeConfig.FlannelConf)
return nil
}
return util.WriteFile(config.FlannelConf,
strings.Replace(netJSON, "%CIDR%", config.AgentConfig.ClusterCIDR.String(), -1))
confJSON := strings.Replace(flannelConf, "%CIDR%", nodeConfig.AgentConfig.ClusterCIDR.String(), -1)
var backendConf string
switch nodeConfig.FlannelBackend {
case config.FlannelBackendVXLAN:
backendConf = vxlanBackend
case config.FlannelBackendIPSEC:
backendConf = strings.Replace(ipsecBackend, "%psk%", nodeConfig.AgentConfig.IPSECPSK, -1)
case config.FlannelBackendWireguard:
backendConf = wireguardBackend
default:
return fmt.Errorf("Cannot configure unknown flannel backend '%s'", nodeConfig.FlannelBackend)
}
confJSON = strings.Replace(confJSON, "%backend%", backendConf, -1)
return util.WriteFile(nodeConfig.FlannelConf, confJSON)
}

10
pkg/cli/cmds/server.go
View File

@ -1,6 +1,9 @@
package cmds
import (
"fmt"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/urfave/cli"
)
@ -30,6 +33,7 @@ type Server struct {
AdvertiseIP string
AdvertisePort int
DisableScheduler bool
FlannelBackend string
}
var ServerConfig Server
@ -189,6 +193,12 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
Usage: "Disable Kubernetes default scheduler",
Destination: &ServerConfig.DisableScheduler,
},
cli.StringFlag{
Name: "flannel-backend",
Usage: fmt.Sprintf("One of '%s', '%s', '%s', or '%s'", config.FlannelBackendNone, config.FlannelBackendVXLAN, config.FlannelBackendIPSEC, config.FlannelBackendWireguard),
Destination: &ServerConfig.FlannelBackend,
Value: config.FlannelBackendVXLAN,
},
NodeIPFlag,
NodeNameFlag,
DockerFlag,

1
pkg/cli/server/server.go
View File

@ -84,6 +84,7 @@ func run(app *cli.Context, cfg *cmds.Server) error {
serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP
serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
serverConfig.ControlConfig.BootstrapReadOnly = !cfg.StoreBootstrap
serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend
if cmds.AgentConfig.FlannelIface != "" && cmds.AgentConfig.NodeIP == "" {
cmds.AgentConfig.NodeIP = netutil.GetIPFromInterface(cmds.AgentConfig.FlannelIface)

12
pkg/daemons/config/types.go
View File

@ -13,10 +13,18 @@ import (
"k8s.io/apiserver/pkg/authentication/authenticator"
)
const (
FlannelBackendNone = "none"
FlannelBackendVXLAN = "vxlan"
FlannelBackendIPSEC = "ipsec"
FlannelBackendWireguard = "wireguard"
)
type Node struct {
Docker bool
ContainerRuntimeEndpoint string
NoFlannel bool
FlannelBackend string
FlannelConf string
FlannelConfOverride bool
FlannelIface *net.Interface
@ -66,6 +74,7 @@ type Agent struct {
CNIPlugin bool
NodeTaints []string
NodeLabels []string
IPSECPSK string
}
type Control struct {
@ -90,6 +99,8 @@ type Control struct {
ExtraControllerArgs []string
ExtraSchedulerAPIArgs []string
NoLeaderElect bool
FlannelBackend string
IPSECPSK string
Runtime *ControlRuntime `json:"-"`
}
@ -106,6 +117,7 @@ type ControlRuntimeBootstrap struct {
ClientKubeletKey string
ClientKubeProxyKey string
ServingKubeletKey string
IPSECKey string
}
type ControlRuntime struct {

43
pkg/daemons/control/server.go
View File

@ -62,6 +62,11 @@ users:
`))
)
const (
userTokenSize = 16
ipsecTokenSize = 48
)
func Server(ctx context.Context, cfg *config.Control) error {
rand.Seed(time.Now().UTC().UnixNano())
@ -252,6 +257,7 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
runtime.ServerCAKey = path.Join(config.DataDir, "tls", "server-ca.key")
runtime.RequestHeaderCA = path.Join(config.DataDir, "tls", "request-header-ca.crt")
runtime.RequestHeaderCAKey = path.Join(config.DataDir, "tls", "request-header-ca.key")
runtime.IPSECKey = path.Join(config.DataDir, "cred", "ipsec.psk")
runtime.ServiceKey = path.Join(config.DataDir, "tls", "service.key")
runtime.PasswdFile = path.Join(config.DataDir, "cred", "passwd")
@ -304,6 +310,10 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
return err
}
if err := genEncryptedNetworkInfo(config, runtime); err != nil {
return err
}
if err := storeBootstrapData(ctx, config, etcdClient); err != nil {
return err
}
@ -422,20 +432,43 @@ func WritePasswords(passwdFile string, records [][]string) error {
return os.Rename(passwdFile+".tmp", passwdFile)
}
func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.ControlRuntime) error {
if s, err := os.Stat(runtime.IPSECKey); err == nil && s.Size() > 0 {
psk, err := ioutil.ReadFile(runtime.IPSECKey)
if err != nil {
return err
}
controlConfig.IPSECPSK = strings.TrimSpace(string(psk))
return nil
}
psk, err := getToken(ipsecTokenSize)
if err != nil {
return err
}
controlConfig.IPSECPSK = psk
if err := ioutil.WriteFile(runtime.IPSECKey, []byte(psk+"\n"), 0600); err != nil {
return err
}
return nil
}
func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
if s, err := os.Stat(runtime.PasswdFile); err == nil && s.Size() > 0 {
return ensureNodeToken(config, runtime)
}
adminToken, err := getToken()
adminToken, err := getToken(userTokenSize)
if err != nil {
return err
}
systemToken, err := getToken()
systemToken, err := getToken(userTokenSize)
if err != nil {
return err
}
nodeToken, err := getToken()
nodeToken, err := getToken(userTokenSize)
if err != nil {
return err
}
@ -451,8 +484,8 @@ func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
})
}
func getToken() (string, error) {
token := make([]byte, 16, 16)
func getToken(size int) (string, error) {
token := make([]byte, size, size)
_, err := cryptorand.Read(token)
if err != nil {
return "", err

2
scripts/download
View File

@ -16,3 +16,5 @@ mkdir -p bin/aux && rm bin/mount && ln -sf ../busybox bin/aux/mount
TRAEFIK_FILE=traefik-${TRAEFIK_VERSION}.tgz
curl -sfL https://kubernetes-charts.storage.googleapis.com/${TRAEFIK_FILE} -o ${CHARTS_DIR}/${TRAEFIK_FILE}
cp scripts/wg-add.sh bin/aux/

34
scripts/wg-add.sh Executable file
View File

@ -0,0 +1,34 @@
#!/usr/bin/env bash
auto-mtu() {
local mtu=0 endpoint output
while read -r _ endpoint; do
[[ $endpoint =~ ^\[?([a-z0-9:.]+)\]?:[0-9]+$ ]] || continue
output="$(ip route get "${BASH_REMATCH[1]}" || true)"
[[ ( $output =~ mtu\ ([0-9]+) || ( $output =~ dev\ ([^ ]+) && $(ip link show dev "${BASH_REMATCH[1]}") =~ mtu\ ([0-9]+) ) ) && ${BASH_REMATCH[1]} -gt $mtu ]] && mtu="${BASH_REMATCH[1]}"
done < <(wg show "$1" endpoints)
if [[ $mtu -eq 0 ]]; then
read -r output < <(ip route show default || true) || true
[[ ( $output =~ mtu\ ([0-9]+) || ( $output =~ dev\ ([^ ]+) && $(ip link show dev "${BASH_REMATCH[1]}") =~ mtu\ ([0-9]+) ) ) && ${BASH_REMATCH[1]} -gt $mtu ]] && mtu="${BASH_REMATCH[1]}"
fi
[[ $mtu -gt 0 ]] || mtu=1500
ip link set mtu $(( mtu - 80 )) up dev "$1"
}
# probe for any modules that may be needed
modprobe wireguard
modprobe tun
# try wireguard kernel module first
ip link add "$1" type wireguard && exit
# try boringtun and let it drop privileges
boringtun "$1" && auto-mtu "$1" && exit
# try boringtun w/o dropping privileges
WG_SUDO=1 boringtun "$1" && auto-mtu "$1" && exit
# try wireguard-go - p.s. should not use wireguard-go, it leaks memory
WG_I_PREFER_BUGGY_USERSPACE_TO_POLISHED_KMOD=1 wireguard-go "$1" && auto-mtu "$1" && exit
exit 1