2019-01-01 08:23:01 +00:00
|
|
|
package flannel
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2019-09-03 23:41:54 +00:00
|
|
|
"fmt"
|
2021-08-17 09:27:54 +00:00
|
|
|
"net"
|
2019-01-09 16:54:15 +00:00
|
|
|
"path/filepath"
|
2023-10-16 18:53:09 +00:00
|
|
|
goruntime "runtime"
|
2019-01-01 08:23:01 +00:00
|
|
|
"strings"
|
|
|
|
|
2022-03-02 23:47:27 +00:00
|
|
|
"github.com/k3s-io/k3s/pkg/agent/util"
|
|
|
|
"github.com/k3s-io/k3s/pkg/daemons/config"
|
2021-10-29 08:59:03 +00:00
|
|
|
"github.com/pkg/errors"
|
2019-01-01 08:23:01 +00:00
|
|
|
"github.com/sirupsen/logrus"
|
2022-04-21 20:56:39 +00:00
|
|
|
v1 "k8s.io/api/core/v1"
|
2019-01-01 08:23:01 +00:00
|
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
2021-10-29 08:59:03 +00:00
|
|
|
"k8s.io/apimachinery/pkg/fields"
|
2022-04-21 20:56:39 +00:00
|
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
|
|
"k8s.io/apimachinery/pkg/watch"
|
2021-10-29 08:59:03 +00:00
|
|
|
typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
|
2022-04-21 20:56:39 +00:00
|
|
|
"k8s.io/client-go/tools/cache"
|
|
|
|
toolswatch "k8s.io/client-go/tools/watch"
|
2021-08-17 09:27:54 +00:00
|
|
|
utilsnet "k8s.io/utils/net"
|
2019-01-01 08:23:01 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
2019-09-03 23:41:54 +00:00
|
|
|
flannelConf = `{
|
|
|
|
"Network": "%CIDR%",
|
2022-03-09 11:30:33 +00:00
|
|
|
"EnableIPv6": %IPV6_ENABLED%,
|
2022-03-04 17:06:29 +00:00
|
|
|
"EnableIPv4": %IPV4_ENABLED%,
|
2021-08-17 09:27:54 +00:00
|
|
|
"IPv6Network": "%CIDR_IPV6%",
|
2019-09-03 23:41:54 +00:00
|
|
|
"Backend": %backend%
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
`
|
2019-09-03 23:41:54 +00:00
|
|
|
|
2019-11-27 17:25:55 +00:00
|
|
|
hostGWBackend = `{
|
|
|
|
"Type": "host-gw"
|
|
|
|
}`
|
|
|
|
|
2022-09-01 17:20:32 +00:00
|
|
|
tailscaledBackend = `{
|
|
|
|
"Type": "extension",
|
2023-07-07 07:49:01 +00:00
|
|
|
"PostStartupCommand": "tailscale set --accept-routes --advertise-routes=%Routes%",
|
2022-09-01 17:20:32 +00:00
|
|
|
"ShutdownCommand": "tailscale down"
|
|
|
|
}`
|
|
|
|
|
2022-04-04 14:10:37 +00:00
|
|
|
wireguardNativeBackend = `{
|
|
|
|
"Type": "wireguard",
|
2022-05-11 20:11:00 +00:00
|
|
|
"PersistentKeepaliveInterval": %PersistentKeepaliveInterval%,
|
|
|
|
"Mode": "%Mode%"
|
2022-04-04 14:10:37 +00:00
|
|
|
}`
|
|
|
|
|
2021-08-17 09:27:54 +00:00
|
|
|
emptyIPv6Network = "::/0"
|
|
|
|
|
|
|
|
ipv4 = iota
|
|
|
|
ipv6
|
2019-01-01 08:23:01 +00:00
|
|
|
)
|
|
|
|
|
2019-09-03 23:41:54 +00:00
|
|
|
func Prepare(ctx context.Context, nodeConfig *config.Node) error {
|
2022-06-08 08:38:07 +00:00
|
|
|
if err := createCNIConf(nodeConfig.AgentConfig.CNIConfDir, nodeConfig); err != nil {
|
2019-01-09 16:54:15 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2019-09-03 23:41:54 +00:00
|
|
|
return createFlannelConf(nodeConfig)
|
2019-01-09 16:54:15 +00:00
|
|
|
}
|
|
|
|
|
2021-10-29 08:59:03 +00:00
|
|
|
func Run(ctx context.Context, nodeConfig *config.Node, nodes typedcorev1.NodeInterface) error {
|
2021-11-20 19:12:03 +00:00
|
|
|
logrus.Infof("Starting flannel with backend %s", nodeConfig.FlannelBackend)
|
2021-10-29 08:59:03 +00:00
|
|
|
if err := waitForPodCIDR(ctx, nodeConfig.AgentConfig.NodeName, nodes); err != nil {
|
2021-11-20 19:12:03 +00:00
|
|
|
return errors.Wrap(err, "flannel failed to wait for PodCIDR assignment")
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2021-08-17 09:27:54 +00:00
|
|
|
netMode, err := findNetMode(nodeConfig.AgentConfig.ClusterCIDRs)
|
|
|
|
if err != nil {
|
2021-10-29 08:59:03 +00:00
|
|
|
return errors.Wrap(err, "failed to check netMode for flannel")
|
2021-08-17 09:27:54 +00:00
|
|
|
}
|
2019-01-09 16:54:15 +00:00
|
|
|
go func() {
|
2023-12-19 22:25:38 +00:00
|
|
|
err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConfFile, nodeConfig.AgentConfig.KubeConfigKubelet, nodeConfig.FlannelIPv6Masq, netMode)
|
2021-09-08 17:56:18 +00:00
|
|
|
if err != nil && !errors.Is(err, context.Canceled) {
|
|
|
|
logrus.Fatalf("flannel exited: %v", err)
|
|
|
|
}
|
2019-01-09 16:54:15 +00:00
|
|
|
}()
|
2019-01-01 08:23:01 +00:00
|
|
|
|
2019-10-27 05:53:25 +00:00
|
|
|
return nil
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2021-10-29 08:59:03 +00:00
|
|
|
// waitForPodCIDR watches nodes with this node's name, and returns when the PodCIDR has been set.
|
|
|
|
func waitForPodCIDR(ctx context.Context, nodeName string, nodes typedcorev1.NodeInterface) error {
|
|
|
|
fieldSelector := fields.Set{metav1.ObjectNameField: nodeName}.String()
|
2022-04-21 20:56:39 +00:00
|
|
|
lw := &cache.ListWatch{
|
|
|
|
ListFunc: func(options metav1.ListOptions) (object runtime.Object, e error) {
|
|
|
|
options.FieldSelector = fieldSelector
|
|
|
|
return nodes.List(ctx, options)
|
|
|
|
},
|
|
|
|
WatchFunc: func(options metav1.ListOptions) (i watch.Interface, e error) {
|
|
|
|
options.FieldSelector = fieldSelector
|
|
|
|
return nodes.Watch(ctx, options)
|
|
|
|
},
|
2021-10-29 08:59:03 +00:00
|
|
|
}
|
2022-04-21 20:56:39 +00:00
|
|
|
condition := func(ev watch.Event) (bool, error) {
|
|
|
|
if n, ok := ev.Object.(*v1.Node); ok {
|
|
|
|
return n.Spec.PodCIDR != "", nil
|
2021-10-29 08:59:03 +00:00
|
|
|
}
|
2022-04-21 20:56:39 +00:00
|
|
|
return false, errors.New("event object not of type v1.Node")
|
2021-10-29 08:59:03 +00:00
|
|
|
}
|
2022-04-21 20:56:39 +00:00
|
|
|
|
|
|
|
if _, err := toolswatch.UntilWithSync(ctx, lw, &v1.Node{}, nil, condition); err != nil {
|
|
|
|
return errors.Wrap(err, "failed to wait for PodCIDR assignment")
|
|
|
|
}
|
|
|
|
|
2021-11-20 19:12:03 +00:00
|
|
|
logrus.Info("Flannel found PodCIDR assigned for node " + nodeName)
|
2021-10-29 08:59:03 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2022-06-08 08:38:07 +00:00
|
|
|
func createCNIConf(dir string, nodeConfig *config.Node) error {
|
2021-11-20 19:12:03 +00:00
|
|
|
logrus.Debugf("Creating the CNI conf in directory %s", dir)
|
2019-01-09 16:54:15 +00:00
|
|
|
if dir == "" {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
p := filepath.Join(dir, "10-flannel.conflist")
|
2022-06-08 08:38:07 +00:00
|
|
|
|
|
|
|
if nodeConfig.AgentConfig.FlannelCniConfFile != "" {
|
|
|
|
logrus.Debugf("Using %s as the flannel CNI conf", nodeConfig.AgentConfig.FlannelCniConfFile)
|
2023-08-01 15:55:34 +00:00
|
|
|
return util.CopyFile(nodeConfig.AgentConfig.FlannelCniConfFile, p, false)
|
2022-06-08 08:38:07 +00:00
|
|
|
}
|
2023-10-16 18:53:09 +00:00
|
|
|
|
|
|
|
cniConfJSON := cniConf
|
|
|
|
if goruntime.GOOS == "windows" {
|
|
|
|
extIface, err := LookupExtInterface(nodeConfig.FlannelIface, ipv4)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
cniConfJSON = strings.ReplaceAll(cniConfJSON, "%IPV4_ADDRESS%", extIface.IfaceAddr.String())
|
|
|
|
cniConfJSON = strings.ReplaceAll(cniConfJSON, "%CLUSTER_CIDR%", nodeConfig.AgentConfig.ClusterCIDR.String())
|
|
|
|
cniConfJSON = strings.ReplaceAll(cniConfJSON, "%SERVICE_CIDR%", nodeConfig.AgentConfig.ServiceCIDR.String())
|
|
|
|
}
|
|
|
|
|
|
|
|
return util.WriteFile(p, cniConfJSON)
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2019-09-03 23:41:54 +00:00
|
|
|
func createFlannelConf(nodeConfig *config.Node) error {
|
2022-03-04 17:06:29 +00:00
|
|
|
var ipv4Enabled string
|
2021-11-20 19:12:03 +00:00
|
|
|
logrus.Debugf("Creating the flannel configuration for backend %s in file %s", nodeConfig.FlannelBackend, nodeConfig.FlannelConfFile)
|
|
|
|
if nodeConfig.FlannelConfFile == "" {
|
|
|
|
return errors.New("Flannel configuration not defined")
|
2019-01-09 16:54:15 +00:00
|
|
|
}
|
2019-09-03 23:41:54 +00:00
|
|
|
if nodeConfig.FlannelConfOverride {
|
2021-11-20 19:12:03 +00:00
|
|
|
logrus.Infof("Using custom flannel conf defined at %s", nodeConfig.FlannelConfFile)
|
2019-08-08 05:56:09 +00:00
|
|
|
return nil
|
|
|
|
}
|
2022-03-04 17:06:29 +00:00
|
|
|
netMode, err := findNetMode(nodeConfig.AgentConfig.ClusterCIDRs)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Fatalf("Flannel error checking netMode: %v", err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if netMode == ipv4 || netMode == (ipv4+ipv6) {
|
|
|
|
ipv4Enabled = "true"
|
|
|
|
} else {
|
|
|
|
ipv4Enabled = "false"
|
|
|
|
}
|
|
|
|
confJSON := strings.ReplaceAll(flannelConf, "%IPV4_ENABLED%", ipv4Enabled)
|
|
|
|
if netMode == ipv4 {
|
|
|
|
confJSON = strings.ReplaceAll(confJSON, "%CIDR%", nodeConfig.AgentConfig.ClusterCIDR.String())
|
2022-03-09 11:30:33 +00:00
|
|
|
confJSON = strings.ReplaceAll(confJSON, "%IPV6_ENABLED%", "false")
|
2022-03-04 17:06:29 +00:00
|
|
|
confJSON = strings.ReplaceAll(confJSON, "%CIDR_IPV6%", emptyIPv6Network)
|
|
|
|
} else if netMode == (ipv4 + ipv6) {
|
2022-03-09 11:30:33 +00:00
|
|
|
confJSON = strings.ReplaceAll(confJSON, "%IPV6_ENABLED%", "true")
|
2022-03-04 17:06:29 +00:00
|
|
|
for _, cidr := range nodeConfig.AgentConfig.ClusterCIDRs {
|
|
|
|
if utilsnet.IsIPv6(cidr.IP) {
|
|
|
|
// Only one ipv6 range available. This might change in future: https://github.com/kubernetes/enhancements/issues/2593
|
|
|
|
confJSON = strings.ReplaceAll(confJSON, "%CIDR_IPV6%", cidr.String())
|
2023-09-21 13:39:05 +00:00
|
|
|
} else {
|
|
|
|
confJSON = strings.ReplaceAll(confJSON, "%CIDR%", cidr.String())
|
2022-03-04 17:06:29 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
confJSON = strings.ReplaceAll(confJSON, "%CIDR%", "0.0.0.0/0")
|
2022-03-09 11:30:33 +00:00
|
|
|
confJSON = strings.ReplaceAll(confJSON, "%IPV6_ENABLED%", "true")
|
2022-03-04 17:06:29 +00:00
|
|
|
for _, cidr := range nodeConfig.AgentConfig.ClusterCIDRs {
|
|
|
|
if utilsnet.IsIPv6(cidr.IP) {
|
|
|
|
// Only one ipv6 range available. This might change in future: https://github.com/kubernetes/enhancements/issues/2593
|
|
|
|
confJSON = strings.ReplaceAll(confJSON, "%CIDR_IPV6%", cidr.String())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2019-09-03 23:41:54 +00:00
|
|
|
|
|
|
|
var backendConf string
|
2022-05-11 20:11:00 +00:00
|
|
|
backendOptions := make(map[string]string)
|
2019-09-03 23:41:54 +00:00
|
|
|
|
2023-10-16 18:53:09 +00:00
|
|
|
// precheck and error out unsupported flannel backends.
|
|
|
|
switch nodeConfig.FlannelBackend {
|
|
|
|
case config.FlannelBackendHostGW:
|
|
|
|
case config.FlannelBackendTailscale:
|
|
|
|
case config.FlannelBackendWireguardNative:
|
|
|
|
if goruntime.GOOS == "windows" {
|
|
|
|
return fmt.Errorf("unsupported flannel backend '%s' for Windows", nodeConfig.FlannelBackend)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-08-25 05:09:13 +00:00
|
|
|
switch nodeConfig.FlannelBackend {
|
2019-09-03 23:41:54 +00:00
|
|
|
case config.FlannelBackendVXLAN:
|
|
|
|
backendConf = vxlanBackend
|
2019-11-27 17:25:55 +00:00
|
|
|
case config.FlannelBackendHostGW:
|
|
|
|
backendConf = hostGWBackend
|
2022-09-01 17:20:32 +00:00
|
|
|
case config.FlannelBackendTailscale:
|
|
|
|
var routes string
|
|
|
|
switch netMode {
|
|
|
|
case ipv4:
|
|
|
|
routes = "$SUBNET"
|
|
|
|
case (ipv4 + ipv6):
|
|
|
|
routes = "$SUBNET,$IPV6SUBNET"
|
|
|
|
case ipv6:
|
|
|
|
routes = "$IPV6SUBNET"
|
|
|
|
default:
|
|
|
|
return fmt.Errorf("incorrect netMode for flannel tailscale backend")
|
|
|
|
}
|
|
|
|
backendConf = strings.ReplaceAll(tailscaledBackend, "%Routes%", routes)
|
2022-04-07 12:15:55 +00:00
|
|
|
case config.FlannelBackendWireguardNative:
|
2022-05-11 20:11:00 +00:00
|
|
|
mode, ok := backendOptions["Mode"]
|
|
|
|
if !ok {
|
|
|
|
mode = "separate"
|
|
|
|
}
|
|
|
|
keepalive, ok := backendOptions["PersistentKeepaliveInterval"]
|
|
|
|
if !ok {
|
|
|
|
keepalive = "25"
|
|
|
|
}
|
|
|
|
backendConf = strings.ReplaceAll(wireguardNativeBackend, "%Mode%", mode)
|
|
|
|
backendConf = strings.ReplaceAll(backendConf, "%PersistentKeepaliveInterval%", keepalive)
|
2019-09-03 23:41:54 +00:00
|
|
|
default:
|
|
|
|
return fmt.Errorf("Cannot configure unknown flannel backend '%s'", nodeConfig.FlannelBackend)
|
|
|
|
}
|
2021-06-07 07:52:26 +00:00
|
|
|
confJSON = strings.ReplaceAll(confJSON, "%backend%", backendConf)
|
2019-09-03 23:41:54 +00:00
|
|
|
|
2021-11-20 19:12:03 +00:00
|
|
|
logrus.Debugf("The flannel configuration is %s", confJSON)
|
|
|
|
return util.WriteFile(nodeConfig.FlannelConfFile, confJSON)
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
2019-09-06 00:39:18 +00:00
|
|
|
|
2021-08-17 09:27:54 +00:00
|
|
|
// fundNetMode returns the mode (ipv4, ipv6 or dual-stack) in which flannel is operating
|
|
|
|
func findNetMode(cidrs []*net.IPNet) (int, error) {
|
|
|
|
dualStack, err := utilsnet.IsDualStackCIDRs(cidrs)
|
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
if dualStack {
|
|
|
|
return ipv4 + ipv6, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, cidr := range cidrs {
|
|
|
|
if utilsnet.IsIPv4CIDR(cidr) {
|
|
|
|
return ipv4, nil
|
|
|
|
}
|
|
|
|
if utilsnet.IsIPv6CIDR(cidr) {
|
|
|
|
return ipv6, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return 0, errors.New("Failed checking netMode")
|
|
|
|
}
|