Add --flannel-external-ip flag

Using the node external IP address for all CNI traffic is a breaking change from previous versions; we should make it an opt-in for distributed clusters instead of default behavior.

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

pkg/agent/config/config.go

@@ -438,6 +438,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 		ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint,
 		FlannelBackend:           controlConfig.FlannelBackend,
 		FlannelIPv6Masq:          controlConfig.FlannelIPv6Masq,
+		FlannelExternalIP:        controlConfig.FlannelExternalIP,
 		EgressSelectorMode:       controlConfig.EgressSelectorMode,
 		ServerHTTPSPort:          controlConfig.HTTPSPort,
 		Token:                    info.String(),

pkg/agent/run.go

@@ -136,7 +136,7 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
 		return err
 	}
 
-	if err := configureNode(ctx, &nodeConfig.AgentConfig, coreClient.CoreV1().Nodes()); err != nil {
+	if err := configureNode(ctx, nodeConfig, coreClient.CoreV1().Nodes()); err != nil {
 		return err
 	}
 
@@ -296,7 +296,8 @@ func createProxyAndValidateToken(ctx context.Context, cfg *cmds.Agent) (proxy.Pr
 
 // configureNode waits for the node object to be created, and if/when it does,
 // ensures that the labels and annotations are up to date.
-func configureNode(ctx context.Context, agentConfig *daemonconfig.Agent, nodes typedcorev1.NodeInterface) error {
+func configureNode(ctx context.Context, nodeConfig *daemonconfig.Node, nodes typedcorev1.NodeInterface) error {
+	agentConfig := &nodeConfig.AgentConfig
 	fieldSelector := fields.Set{metav1.ObjectNameField: agentConfig.NodeName}.String()
 	lw := &cache.ListWatch{
 		ListFunc: func(options metav1.ListOptions) (object runtime.Object, e error) {
@@ -322,7 +323,7 @@ func configureNode(ctx context.Context, agentConfig *daemonconfig.Agent, nodes t
 		}
 
 		if !agentConfig.DisableCCM {
-			if annotations, changed := updateAddressAnnotations(agentConfig, node.Annotations); changed {
+			if annotations, changed := updateAddressAnnotations(nodeConfig, node.Annotations); changed {
 				node.Annotations = annotations
 				updateNode = true
 			}
@@ -400,7 +401,8 @@ func updateLegacyAddressLabels(agentConfig *daemonconfig.Agent, nodeLabels map[s
 }
 
 // updateAddressAnnotations updates the node annotations with important information about IP addresses of the node
-func updateAddressAnnotations(agentConfig *daemonconfig.Agent, nodeAnnotations map[string]string) (map[string]string, bool) {
+func updateAddressAnnotations(nodeConfig *daemonconfig.Node, nodeAnnotations map[string]string) (map[string]string, bool) {
+	agentConfig := &nodeConfig.AgentConfig
 	result := map[string]string{
 		cp.InternalIPKey: util.JoinIPs(agentConfig.NodeIPs),
 		cp.HostnameKey:   agentConfig.NodeName,
@@ -408,6 +410,7 @@ func updateAddressAnnotations(agentConfig *daemonconfig.Agent, nodeAnnotations m
 
 	if agentConfig.NodeExternalIP != "" {
 		result[cp.ExternalIPKey] = util.JoinIPs(agentConfig.NodeExternalIPs)
+		if nodeConfig.FlannelExternalIP {
 			for _, ipAddress := range agentConfig.NodeExternalIPs {
 				if utilsnet.IsIPv4(ipAddress) {
 					result[flannel.FlannelExternalIPv4Annotation] = ipAddress.String()
@@ -417,6 +420,7 @@ func updateAddressAnnotations(agentConfig *daemonconfig.Agent, nodeAnnotations m
 				}
 			}
 		}
+	}
 
 	result = labels.Merge(nodeAnnotations, result)
 	return result, !equality.Semantic.DeepEqual(nodeAnnotations, result)

pkg/cli/cmds/server.go

@@ -64,6 +64,7 @@ type Server struct {
 	ServerURL                string
 	FlannelBackend           string
 	FlannelIPv6Masq          bool
+	FlannelExternalIP        bool
 	EgressSelectorMode       string
 	DefaultLocalStoragePath  string
 	DisableCCM               bool
@@ -216,6 +217,11 @@ var ServerFlags = []cli.Flag{
 		Usage:       "(networking) Enable IPv6 masquerading for pod",
 		Destination: &ServerConfig.FlannelIPv6Masq,
 	},
+	cli.BoolFlag{
+		Name:        "flannel-external-ip",
+		Usage:       "(networking) Use node external IP addresses for Flannel traffic",
+		Destination: &ServerConfig.FlannelExternalIP,
+	},
 	cli.StringFlag{
 		Name:        "egress-selector-mode",
 		Usage:       "(networking) One of 'agent', 'cluster', 'pod', 'disabled'",

pkg/cli/server/server.go

@@ -137,6 +137,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
 	serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend
 	serverConfig.ControlConfig.FlannelIPv6Masq = cfg.FlannelIPv6Masq
+	serverConfig.ControlConfig.FlannelExternalIP = cfg.FlannelExternalIP
 	serverConfig.ControlConfig.EgressSelectorMode = cfg.EgressSelectorMode
 	serverConfig.ControlConfig.ExtraCloudControllerArgs = cfg.ExtraCloudControllerArgs
 	serverConfig.ControlConfig.DisableCCM = cfg.DisableCCM

pkg/daemons/config/types.go

@@ -44,6 +44,7 @@ type Node struct {
 	FlannelConfOverride      bool
 	FlannelIface             *net.Interface
 	FlannelIPv6Masq          bool
+	FlannelExternalIP        bool
 	EgressSelectorMode       string
 	Containerd               Containerd
 	CRIDockerd               CRIDockerd
@@ -137,6 +138,7 @@ type CriticalControlArgs struct {
 	DisableServiceLB      bool
 	FlannelBackend        string
 	FlannelIPv6Masq       bool
+	FlannelExternalIP     bool
 	EgressSelectorMode    string
 	NoCoreDNS             bool
 	ServiceIPRange        *net.IPNet