Added multiClusterCidr feature

Signed-off-by: Roberto Bonafiglia <roberto.bonafiglia@suse.com>

manifests/rolebindings.yaml

@@ -34,6 +34,7 @@ rules:
   - "networking.k8s.io"
   resources:
   - networkpolicies
+  - clustercidrs
   verbs:
   - list
   - watch
@@ -60,3 +61,31 @@ subjects:
   - apiGroup: rbac.authorization.k8s.io
     kind: User
     name: system:k3s-controller
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: clustercidrs-node
+rules:
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - clustercidrs
+  verbs:
+  - list
+  - watch
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: clustercidrs-node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: clustercidrs-node
+subjects:
+  - kind: Group
+    name: system:nodes
+    apiGroup: rbac.authorization.k8s.io

pkg/agent/config/config.go

@@ -438,6 +438,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 		Docker:                   envInfo.Docker,
 		SELinux:                  envInfo.EnableSELinux,
 		ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint,
+		MultiClusterCIDR:         controlConfig.MultiClusterCIDR,
 		FlannelBackend:           controlConfig.FlannelBackend,
 		FlannelIPv6Masq:          controlConfig.FlannelIPv6Masq,
 		FlannelExternalIP:        controlConfig.FlannelExternalIP,

pkg/agent/flannel/flannel.go

@@ -46,7 +46,7 @@ var (
 	FlannelExternalIPv6Annotation = FlannelBaseAnnotation + "/public-ipv6-overwrite"
 )
 
-func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kubeConfigFile string, flannelIPv6Masq bool, netMode int) error {
+func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kubeConfigFile string, flannelIPv6Masq bool, multiClusterCIDR bool, netMode int) error {
 	extIface, err := LookupExtInterface(flannelIface, netMode)
 	if err != nil {
 		return err
@@ -58,7 +58,7 @@ func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kube
 		FlannelBaseAnnotation,
 		flannelConf,
 		false,
-		false)
+		multiClusterCIDR)
 	if err != nil {
 		return err
 	}

pkg/agent/flannel/setup.go

@@ -100,7 +100,7 @@ func Run(ctx context.Context, nodeConfig *config.Node, nodes typedcorev1.NodeInt
 		return errors.Wrap(err, "failed to check netMode for flannel")
 	}
 	go func() {
-		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConfFile, nodeConfig.AgentConfig.KubeConfigKubelet, nodeConfig.FlannelIPv6Masq, netMode)
+		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConfFile, nodeConfig.AgentConfig.KubeConfigKubelet, nodeConfig.FlannelIPv6Masq, nodeConfig.MultiClusterCIDR, netMode)
 		if err != nil && !errors.Is(err, context.Canceled) {
 			logrus.Fatalf("flannel exited: %v", err)
 		}

pkg/cli/cmds/server.go

@@ -62,6 +62,7 @@ type Server struct {
 	AdvertisePort            int
 	DisableScheduler         bool
 	ServerURL                string
+	MultiClusterCIDR         bool
 	FlannelBackend           string
 	FlannelIPv6Masq          bool
 	FlannelExternalIP        bool
@@ -212,6 +213,11 @@ var ServerFlags = []cli.Flag{
 		Destination: &ServerConfig.FlannelBackend,
 		Value:       "vxlan",
 	},
+	&cli.BoolFlag{
+		Name:        "multi-cluster-cidr",
+		Usage:       "(experimental/networking) Enable multiClusterCIDR",
+		Destination: &ServerConfig.MultiClusterCIDR,
+	},
 	&cli.BoolFlag{
 		Name:        "flannel-ipv6-masq",
 		Usage:       "(networking) Enable IPv6 masquerading for pod",

pkg/cli/server/server.go

@@ -134,6 +134,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	serverConfig.ControlConfig.Datastore.BackendTLSConfig.KeyFile = cfg.DatastoreKeyFile
 	serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP
 	serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
+	serverConfig.ControlConfig.MultiClusterCIDR = cfg.MultiClusterCIDR
 	serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend
 	serverConfig.ControlConfig.FlannelIPv6Masq = cfg.FlannelIPv6Masq
 	serverConfig.ControlConfig.FlannelExternalIP = cfg.FlannelExternalIP

pkg/daemons/config/types.go

@@ -39,6 +39,7 @@ type Node struct {
 	ContainerRuntimeEndpoint string
 	NoFlannel                bool
 	SELinux                  bool
+	MultiClusterCIDR         bool
 	FlannelBackend           string
 	FlannelConfFile          string
 	FlannelConfOverride      bool
@@ -140,6 +141,7 @@ type CriticalControlArgs struct {
 	DisableNPC            bool         `cli:"disable-network-policy"`
 	DisableServiceLB      bool         `cli:"disable-service-lb"`
 	EncryptSecrets        bool         `cli:"secrets-encryption"`
+	MultiClusterCIDR      bool         `cli:"multi-cluster-cidr"`
 	FlannelBackend        string       `cli:"flannel-backend"`
 	FlannelIPv6Masq       bool         `cli:"flannel-ipv6-masq"`
 	FlannelExternalIP     bool         `cli:"flannel-external-ip"`

pkg/daemons/control/server.go

@@ -113,6 +113,10 @@ func controllerManager(ctx context.Context, cfg *config.Control) error {
 		"cluster-signing-legacy-unknown-cert-file":        runtime.SigningServerCA,
 		"cluster-signing-legacy-unknown-key-file":         runtime.ServerCAKey,
 	}
+	if cfg.MultiClusterCIDR {
+		argsMap["cidr-allocator-type"] = "MultiCIDRRangeAllocator"
+		argsMap["feature-gates"] = util.AddFeatureGate(argsMap["feature-gates"], "MultiCIDRRangeAllocator=true")
+	}
 	if cfg.NoLeaderElect {
 		argsMap["leader-elect"] = "false"
 	}
@@ -200,6 +204,10 @@ func apiServer(ctx context.Context, cfg *config.Control) error {
 	argsMap["enable-admission-plugins"] = "NodeRestriction"
 	argsMap["anonymous-auth"] = "false"
 	argsMap["profiling"] = "false"
+	if cfg.MultiClusterCIDR {
+		argsMap["feature-gates"] = util.AddFeatureGate(argsMap["feature-gates"], "MultiCIDRRangeAllocator=true")
+		argsMap["runtime-config"] = "networking.k8s.io/v1alpha1"
+	}
 	if cfg.EncryptSecrets {
 		argsMap["encryption-provider-config"] = runtime.EncryptionConfig
 	}
@@ -323,6 +331,10 @@ func cloudControllerManager(ctx context.Context, cfg *config.Control) error {
 		argsMap["controllers"] = argsMap["controllers"] + ",-cloud-node,-cloud-node-lifecycle"
 		argsMap["secure-port"] = "0"
 	}
+	if cfg.MultiClusterCIDR {
+		argsMap["cidr-allocator-type"] = "MultiCIDRRangeAllocator"
+		argsMap["feature-gates"] = util.AddFeatureGate(argsMap["feature-gates"], "MultiCIDRRangeAllocator=true")
+	}
 	if cfg.DisableServiceLB {
 		argsMap["controllers"] = argsMap["controllers"] + ",-service"
 	}

pkg/deploy/zz_generated_bindata.go

@@ -291,7 +291,7 @@ func metricsServerResourceReaderYaml() (*asset, error) {
 	return a, nil
 }
 
-var _rolebindingsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x92\x31\x6f\xe3\x30\x0c\x85\x77\xfd\x0a\x21\xbb\x72\x38\xdc\x72\xf0\xd8\x0e\xdd\x03\xb4\x3b\x6d\xb3\x09\x6b\x59\x14\x48\x2a\x41\xfb\xeb\x0b\xa7\x6e\x82\xa4\x76\x90\xb4\xdd\x24\x41\x7c\x1f\x1f\xf9\x20\xd3\x13\x8a\x12\xa7\xca\x4b\x0d\xcd\x12\x8a\x6d\x58\xe8\x0d\x8c\x38\x2d\xbb\xff\xba\x24\xfe\xb3\xfd\xeb\x3a\x4a\x6d\xe5\xef\x63\x51\x43\x59\x71\xc4\x3b\x4a\x2d\xa5\xb5\xeb\xd1\xa0\x05\x83\xca\x79\x9f\xa0\xc7\xca\x77\xa5\xc6\x00\x99\x14\x65\x8b\x12\x86\x6b\x44\x0b\xd0\xf6\x94\x9c\x70\xc4\x15\x3e\x0f\xbf\x21\xd3\x83\x70\xc9\x17\xc8\xce\xfb\x2f\xe0\x03\x47\x5f\xd5\xb0\xaf\x0e\xfa\x99\x46\x86\x96\xfa\x05\x1b\xd3\xca\x85\x9b\x20\x8f\x8a\x32\xe3\xc2\xb9\x10\x82\xfb\xfe\xb4\x26\xc6\xf4\xd9\xfe\x3f\x0d\x0d\x27\x13\x8e\x11\xc5\x49\x89\x78\xd2\xb8\x0e\x15\xc1\x2f\x16\xce\x7b\x41\xe5\x22\x0d\x8e\x6f\x89\x5b\x54\xe7\xfd\x16\xa5\x1e\x9f\xd6\x68\x57\xd6\x42\x8f\x9a\xa1\x39\x17\x88\xa4\xb6\x3f\xec\xc0\x9a\xcd\x84\x56\x42\xdb\xb1\x74\x94\xd6\xa3\xdf\x29\xf1\x8f\x3f\x99\x23\x35\x74\x33\x61\x42\x10\x53\x9b\x99\x92\xe9\xfe\x96\xb9\x9d\xd3\x1c\xfc\x1f\xb5\x7f\xb8\xb4\xf9\x88\xcf\xec\xee\xf7\xb3\x7d\x0a\x38\x06\x7b\xf0\x78\x1d\xe3\x2c\xdc\x97\x01\xef\x01\x00\x00\xff\xff\x46\xd3\x6d\x9d\x0f\x04\x00\x00")
+var _rolebindingsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x94\x31\x6f\xe3\x30\x0c\x85\x77\xfd\x0a\x21\xbb\x72\x38\xdc\x72\xf0\xd8\x0e\xdd\x03\xb4\xbb\x2c\xb1\x09\x6b\x59\x14\x48\x39\x41\xfb\xeb\x0b\xc7\x4e\xd2\xc4\x76\xe0\xb4\xe9\x66\x0b\xe2\xfb\x48\xbe\x07\xd9\x84\x2f\xc0\x82\x14\x0b\xcd\xa5\x75\x4b\xdb\xe4\x0d\x31\x7e\xd8\x8c\x14\x97\xd5\x7f\x59\x22\xfd\xd9\xfe\x55\x15\x46\x5f\xe8\xc7\xd0\x48\x06\x5e\x51\x80\x07\x8c\x1e\xe3\x5a\xd5\x90\xad\xb7\xd9\x16\x4a\xeb\x68\x6b\x28\x74\xd5\x94\x60\x6c\x42\x01\xde\x02\x9b\xf6\x37\x40\x36\xd6\xd7\x18\x15\x53\x80\x15\xbc\xb6\xb7\x6d\xc2\x27\xa6\x26\x5d\x21\x2b\xad\x07\xe0\x23\x47\xde\x25\x43\x5d\x1c\xf5\x13\xf6\x0c\x69\xca\x37\x70\x59\x0a\x65\x6e\x82\x3c\x0b\xf0\xc4\x14\x4a\x19\x63\xd4\xf7\xb7\x35\xb2\xa6\x43\xfb\xff\xc4\x38\x8a\x99\x29\x04\x60\xc5\x4d\x80\xb3\xc6\xa5\xad\x30\x7a\xb1\x50\x5a\x33\x08\x35\xec\xa0\x3f\x8b\xe4\x41\x94\xd6\x5b\xe0\xb2\x3f\x5a\x43\x9e\x59\x6b\x6b\x90\x64\xdd\xa5\x40\x40\xc9\xfb\x8f\x9d\xcd\x6e\x33\xa2\x15\x21\xef\x88\x2b\x8c\xeb\x7e\xde\x31\xf1\xee\x4e\xa2\x80\x0e\xf7\x04\xa3\x5d\xb7\x0c\x87\x9e\x6f\x45\x8e\x10\x20\xfa\x44\x18\x73\xa7\x9d\xc8\x4f\x69\xb6\x0b\x39\x69\xff\xd0\xc5\xe9\xcc\x4f\x98\x79\xff\xb0\x9f\x03\x4e\x49\x6f\x67\x9c\xc7\xb8\x48\xfb\x75\xc0\xfd\x63\xff\x35\x07\xa6\x4d\xf0\x64\xe4\x07\x49\x1b\xc6\x60\x76\xa8\x7e\xcd\xf8\x91\x71\xee\x67\xfa\x50\xfc\xdc\xf0\xae\x72\x8f\x18\x3a\x79\x78\x1d\xe6\xb5\xf1\x19\x00\x00\xff\xff\x20\xa2\xda\xb0\x09\x06\x00\x00")
 
 func rolebindingsYamlBytes() ([]byte, error) {
 	return bindataRead(