github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Add --flannel-backend flag

pull/719/head
Erik Wilson 5 years ago
parent
359a77939c
commit
959acf9c92
8 changed files with 158 additions and 24 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 10
      pkg/agent/config/config.go
  2. 70
      pkg/agent/flannel/setup.go
  3. 10
      pkg/cli/cmds/server.go
  4. 1
      pkg/cli/server/server.go
  5. 12
      pkg/daemons/config/types.go
  6. 43
      pkg/daemons/control/server.go
  7. 2
      scripts/download
  8. 34
      scripts/wg-add.sh

10
pkg/agent/config/config.go
Unescape View File

@ -330,8 +330,8 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
nodeConfig := &config.Node{
Docker: envInfo.Docker,
NoFlannel: envInfo.NoFlannel,
ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint,
FlannelBackend: controlConfig.FlannelBackend,
}
nodeConfig.FlannelIface = flannelIface
nodeConfig.Images = filepath.Join(envInfo.DataDir, "images")
@ -349,6 +349,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet")
nodeConfig.AgentConfig.PauseImage = envInfo.PauseImage
nodeConfig.AgentConfig.IPSECPSK = controlConfig.IPSECPSK
nodeConfig.CACerts = info.CACerts
nodeConfig.Containerd.Config = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml")
nodeConfig.Containerd.Root = filepath.Join(envInfo.DataDir, "containerd")
@ -361,6 +362,13 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml.tmpl")
nodeConfig.ServerAddress = serverURLParsed.Host
nodeConfig.Certificate = servingCert
if nodeConfig.FlannelBackend == config.FlannelBackendNone {
nodeConfig.NoFlannel = true
} else {
nodeConfig.NoFlannel = envInfo.NoFlannel
}
if !nodeConfig.NoFlannel {
if envInfo.FlannelConf == "" {
nodeConfig.FlannelConf = filepath.Join(envInfo.DataDir, "etc/flannel/net-conf.json")

70
pkg/agent/flannel/setup.go
Unescape View File

@ -2,6 +2,7 @@ package flannel
import (
"context"
"fmt"
"path/filepath"
"strings"
"time"
@ -36,27 +37,45 @@ const (
]
}
`
netJSON = `{
"Network": "%CIDR%",
"Backend": {
"Type": "vxlan"
}
flannelConf = `{
"Network": "%CIDR%",
"Backend": %backend%
}
`
vxlanBackend = `{
"Type": "vxlan"
}`
ipsecBackend = `{
"Type": "ipsec",
"UDPEncap": true,
"PSK": "%psk%"
}`
wireguardBackend = `{
"Type": "extension",
"PreStartupCommand": "wg genkey | tee privatekey | wg pubkey",
"PostStartupCommand": "export SUBNET_IP=$(echo $SUBNET | cut -d'/' -f 1); ip link del flannel.1 2>/dev/null; echo $PATH >&2; wg-add.sh flannel.1 && wg set flannel.1 listen-port 51820 private-key privatekey && ip addr add $SUBNET_IP/32 dev flannel.1 && ip link set flannel.1 up && ip route add $NETWORK dev flannel.1",
"ShutdownCommand": "ip link del flannel.1",
"SubnetAddCommand": "read PUBLICKEY; wg set flannel.1 peer $PUBLICKEY endpoint $PUBLIC_IP:51820 allowed-ips $SUBNET",
"SubnetRemoveCommand": "read PUBLICKEY; wg set flannel.1 peer $PUBLICKEY remove"
}`
)
func Prepare(ctx context.Context, config *config.Node) error {
if err := createCNIConf(config.AgentConfig.CNIConfDir); err != nil {
func Prepare(ctx context.Context, nodeConfig *config.Node) error {
if err := createCNIConf(nodeConfig.AgentConfig.CNIConfDir); err != nil {
return err
}
return createFlannelConf(config)
return createFlannelConf(nodeConfig)
}
func Run(ctx context.Context, config *config.Node) error {
nodeName := config.AgentConfig.NodeName
func Run(ctx context.Context, nodeConfig *config.Node) error {
nodeName := nodeConfig.AgentConfig.NodeName
restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode)
restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigNode)
if err != nil {
return err
}
@ -80,7 +99,7 @@ func Run(ctx context.Context, config *config.Node) error {
}
go func() {
err := flannel(ctx, config.FlannelIface, config.FlannelConf, config.AgentConfig.KubeConfigNode)
err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigNode)
logrus.Fatalf("flannel exited: %v", err)
}()
@ -95,14 +114,29 @@ func createCNIConf(dir string) error {
return util.WriteFile(p, cniConf)
}
func createFlannelConf(config *config.Node) error {
if config.FlannelConf == "" {
func createFlannelConf(nodeConfig *config.Node) error {
if nodeConfig.FlannelConf == "" {
return nil
}
if config.FlannelConfOverride {
logrus.Infof("Using custom flannel conf defined at %s", config.FlannelConf)
if nodeConfig.FlannelConfOverride {
logrus.Infof("Using custom flannel conf defined at %s", nodeConfig.FlannelConf)
return nil
}
return util.WriteFile(config.FlannelConf,
strings.Replace(netJSON, "%CIDR%", config.AgentConfig.ClusterCIDR.String(), -1))
confJSON := strings.Replace(flannelConf, "%CIDR%", nodeConfig.AgentConfig.ClusterCIDR.String(), -1)
var backendConf string
switch nodeConfig.FlannelBackend {
case config.FlannelBackendVXLAN:
backendConf = vxlanBackend
case config.FlannelBackendIPSEC:
backendConf = strings.Replace(ipsecBackend, "%psk%", nodeConfig.AgentConfig.IPSECPSK, -1)
case config.FlannelBackendWireguard:
backendConf = wireguardBackend
default:
return fmt.Errorf("Cannot configure unknown flannel backend '%s'", nodeConfig.FlannelBackend)
}
confJSON = strings.Replace(confJSON, "%backend%", backendConf, -1)
return util.WriteFile(nodeConfig.FlannelConf, confJSON)
}

10
pkg/cli/cmds/server.go
Unescape View File

@ -1,6 +1,9 @@
package cmds
import (
"fmt"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/urfave/cli"
)
@ -30,6 +33,7 @@ type Server struct {
AdvertiseIP string
AdvertisePort int
DisableScheduler bool
FlannelBackend string
}
var ServerConfig Server
@ -189,6 +193,12 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
Usage: "Disable Kubernetes default scheduler",
Destination: &ServerConfig.DisableScheduler,
},
cli.StringFlag{
Name: "flannel-backend",
Usage: fmt.Sprintf("One of '%s', '%s', '%s', or '%s'", config.FlannelBackendNone, config.FlannelBackendVXLAN, config.FlannelBackendIPSEC, config.FlannelBackendWireguard),
Destination: &ServerConfig.FlannelBackend,
Value: config.FlannelBackendVXLAN,
},
NodeIPFlag,
NodeNameFlag,
DockerFlag,

1
pkg/cli/server/server.go
Unescape View File

@ -84,6 +84,7 @@ func run(app *cli.Context, cfg *cmds.Server) error {
serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP
serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
serverConfig.ControlConfig.BootstrapReadOnly = !cfg.StoreBootstrap
serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend
if cmds.AgentConfig.FlannelIface != "" && cmds.AgentConfig.NodeIP == "" {
cmds.AgentConfig.NodeIP = netutil.GetIPFromInterface(cmds.AgentConfig.FlannelIface)

12
pkg/daemons/config/types.go
Unescape View File

@ -13,10 +13,18 @@ import (
"k8s.io/apiserver/pkg/authentication/authenticator"
)
const (
FlannelBackendNone = "none"
FlannelBackendVXLAN = "vxlan"
FlannelBackendIPSEC = "ipsec"
FlannelBackendWireguard = "wireguard"
)
type Node struct {
Docker bool
ContainerRuntimeEndpoint string
NoFlannel bool
FlannelBackend string
FlannelConf string
FlannelConfOverride bool
FlannelIface *net.Interface
@ -66,6 +74,7 @@ type Agent struct {
CNIPlugin bool
NodeTaints []string
NodeLabels []string
IPSECPSK string
}
type Control struct {
@ -90,6 +99,8 @@ type Control struct {
ExtraControllerArgs []string
ExtraSchedulerAPIArgs []string
NoLeaderElect bool
FlannelBackend string
IPSECPSK string
Runtime *ControlRuntime `json:"-"`
}
@ -106,6 +117,7 @@ type ControlRuntimeBootstrap struct {
ClientKubeletKey string
ClientKubeProxyKey string
ServingKubeletKey string
IPSECKey string
}
type ControlRuntime struct {

43
pkg/daemons/control/server.go
Unescape View File

@ -62,6 +62,11 @@ users:
`))
)
const (
userTokenSize = 16
ipsecTokenSize = 48
)
func Server(ctx context.Context, cfg *config.Control) error {
rand.Seed(time.Now().UTC().UnixNano())
@ -252,6 +257,7 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
runtime.ServerCAKey = path.Join(config.DataDir, "tls", "server-ca.key")
runtime.RequestHeaderCA = path.Join(config.DataDir, "tls", "request-header-ca.crt")
runtime.RequestHeaderCAKey = path.Join(config.DataDir, "tls", "request-header-ca.key")
runtime.IPSECKey = path.Join(config.DataDir, "cred", "ipsec.psk")
runtime.ServiceKey = path.Join(config.DataDir, "tls", "service.key")
runtime.PasswdFile = path.Join(config.DataDir, "cred", "passwd")
@ -304,6 +310,10 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
return err
}
if err := genEncryptedNetworkInfo(config, runtime); err != nil {
return err
}
if err := storeBootstrapData(ctx, config, etcdClient); err != nil {
return err
}
@ -422,20 +432,43 @@ func WritePasswords(passwdFile string, records [][]string) error {
return os.Rename(passwdFile+".tmp", passwdFile)
}
func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.ControlRuntime) error {
if s, err := os.Stat(runtime.IPSECKey); err == nil && s.Size() > 0 {
psk, err := ioutil.ReadFile(runtime.IPSECKey)
if err != nil {
return err
}
controlConfig.IPSECPSK = strings.TrimSpace(string(psk))
return nil
}
psk, err := getToken(ipsecTokenSize)
if err != nil {
return err
}
controlConfig.IPSECPSK = psk
if err := ioutil.WriteFile(runtime.IPSECKey, []byte(psk+"\n"), 0600); err != nil {
return err
}
return nil
}
func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
if s, err := os.Stat(runtime.PasswdFile); err == nil && s.Size() > 0 {
return ensureNodeToken(config, runtime)
}
adminToken, err := getToken()
adminToken, err := getToken(userTokenSize)
if err != nil {
return err
}
systemToken, err := getToken()
systemToken, err := getToken(userTokenSize)
if err != nil {
return err
}
nodeToken, err := getToken()
nodeToken, err := getToken(userTokenSize)
if err != nil {
return err
}
@ -451,8 +484,8 @@ func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
})
}
func getToken() (string, error) {
token := make([]byte, 16, 16)
func getToken(size int) (string, error) {
token := make([]byte, size, size)
_, err := cryptorand.Read(token)
if err != nil {
return "", err

2
scripts/download
Unescape View File

@ -16,3 +16,5 @@ mkdir -p bin/aux && rm bin/mount && ln -sf ../busybox bin/aux/mount
TRAEFIK_FILE=traefik-${TRAEFIK_VERSION}.tgz
curl -sfL https://kubernetes-charts.storage.googleapis.com/${TRAEFIK_FILE} -o ${CHARTS_DIR}/${TRAEFIK_FILE}
cp scripts/wg-add.sh bin/aux/

34
scripts/wg-add.sh
Unescape View File

@ -0,0 +1,34 @@
#!/usr/bin/env bash
auto-mtu() {
local mtu=0 endpoint output
while read -r _ endpoint; do
[[ $endpoint =~ ^\[?([a-z0-9:.]+)\]?:[0-9]+$ ]] || continue
output="$(ip route get "${BASH_REMATCH[1]}" || true)"
[[ ( $output =~ mtu\ ([0-9]+) || ( $output =~ dev\ ([^ ]+) && $(ip link show dev "${BASH_REMATCH[1]}") =~ mtu\ ([0-9]+) ) ) && ${BASH_REMATCH[1]} -gt $mtu ]] && mtu="${BASH_REMATCH[1]}"
done < <(wg show "$1" endpoints)
if [[ $mtu -eq 0 ]]; then
read -r output < <(ip route show default || true) || true
[[ ( $output =~ mtu\ ([0-9]+) || ( $output =~ dev\ ([^ ]+) && $(ip link show dev "${BASH_REMATCH[1]}") =~ mtu\ ([0-9]+) ) ) && ${BASH_REMATCH[1]} -gt $mtu ]] && mtu="${BASH_REMATCH[1]}"
fi
[[ $mtu -gt 0 ]] || mtu=1500
ip link set mtu $(( mtu - 80 )) up dev "$1"
}
# probe for any modules that may be needed
modprobe wireguard
modprobe tun
# try wireguard kernel module first
ip link add "$1" type wireguard && exit
# try boringtun and let it drop privileges
boringtun "$1" && auto-mtu "$1" && exit
# try boringtun w/o dropping privileges
WG_SUDO=1 boringtun "$1" && auto-mtu "$1" && exit
# try wireguard-go - p.s. should not use wireguard-go, it leaks memory
WG_I_PREFER_BUGGY_USERSPACE_TO_POLISHED_KMOD=1 wireguard-go "$1" && auto-mtu "$1" && exit
exit 1
Write Preview
Loading…
Cancel
Save