github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #8506 from manuelbuil/IPPrecedence125 

[Release 1.25] Take IPFamily precedence based on order
pull/8553/head
Manuel Buil 2023-10-03 18:41:02 +02:00 committed by GitHub
parent 857e01755b 7e1e1867d4
commit a873fba947
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 74 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
manifests/coredns.yaml
View File

@ -205,6 +205,7 @@ spec:
selector:
k8s-app: kube-dns
clusterIP: %{CLUSTER_DNS}%
clusterIPs: %{CLUSTER_DNS_LIST}%
ports:
- name: dns
port: 53
@ -215,3 +216,4 @@ spec:
- name: metrics
port: 9153
protocol: TCP
ipFamilyPolicy: %{CLUSTER_DNS_IPFAMILYPOLICY}%

1
manifests/metrics-server/metrics-server-service.yaml
View File

@ -15,3 +15,4 @@ spec:
name: https
protocol: TCP
targetPort: https
ipFamilyPolicy: PreferDualStack

10
pkg/agent/config/config.go
View File

@ -556,22 +556,18 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
nodeConfig.Certificate = servingCert
nodeConfig.AgentConfig.NodeIPs = nodeIPs
nodeIP, listenAddress, _, err := util.GetFirstIP(nodeIPs)
listenAddress, _, _, err := util.GetDefaultAddresses(nodeIPs[0])
if err != nil {
return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-ip")
}
nodeConfig.AgentConfig.NodeIP = nodeIP.String()
nodeConfig.AgentConfig.NodeIP = nodeIPs[0].String()
nodeConfig.AgentConfig.ListenAddress = listenAddress
nodeConfig.AgentConfig.NodeExternalIPs = nodeExternalIPs
// if configured, set NodeExternalIP to the first IPv4 address, for legacy clients
// unless only IPv6 address given
if len(nodeConfig.AgentConfig.NodeExternalIPs) > 0 {
nodeExternalIP, _, _, err := util.GetFirstIP(nodeConfig.AgentConfig.NodeExternalIPs)
if err != nil {
return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-external-ip")
}
nodeConfig.AgentConfig.NodeExternalIP = nodeExternalIP.String()
nodeConfig.AgentConfig.NodeExternalIP = nodeConfig.AgentConfig.NodeExternalIPs[0].String()
}
if nodeConfig.FlannelBackend == config.FlannelBackendNone {

3
pkg/agent/flannel/setup.go
View File

@ -197,12 +197,13 @@ func createFlannelConf(nodeConfig *config.Node) error {
confJSON = strings.ReplaceAll(confJSON, "%IPV6_ENABLED%", "false")
confJSON = strings.ReplaceAll(confJSON, "%CIDR_IPV6%", emptyIPv6Network)
} else if netMode == (ipv4 + ipv6) {
confJSON = strings.ReplaceAll(confJSON, "%CIDR%", nodeConfig.AgentConfig.ClusterCIDR.String())
confJSON = strings.ReplaceAll(confJSON, "%IPV6_ENABLED%", "true")
for _, cidr := range nodeConfig.AgentConfig.ClusterCIDRs {
if utilsnet.IsIPv6(cidr.IP) {
// Only one ipv6 range available. This might change in future: https://github.com/kubernetes/enhancements/issues/2593
confJSON = strings.ReplaceAll(confJSON, "%CIDR_IPV6%", cidr.String())
} else {
confJSON = strings.ReplaceAll(confJSON, "%CIDR%", cidr.String())
}
}
} else {

7
pkg/agent/run.go
View File

@ -70,12 +70,17 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
serviceIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ServiceCIDR)
clusterIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ClusterCIDR)
nodeIPv6 := utilsnet.IsIPv6String(nodeConfig.AgentConfig.NodeIP)
// check that cluster-cidr and service-cidr have the same IP versions
if (serviceIPv6 != clusterIPv6) || (dualCluster != dualService) || (serviceIPv4 != clusterIPv4) {
return fmt.Errorf("cluster-cidr: %v and service-cidr: %v, must share the same IP version (IPv4, IPv6 or dual-stack)", nodeConfig.AgentConfig.ClusterCIDRs, nodeConfig.AgentConfig.ServiceCIDRs)
}
if (clusterIPv6 && !nodeIPv6) || (dualCluster && !dualNode) || (clusterIPv4 && !nodeIPv4) {
// check that node-ip has the IP versions set in cluster-cidr
if (clusterIPv6 && !(nodeIPv6 || dualNode)) || (dualCluster && !dualNode) || (clusterIPv4 && !(nodeIPv4 || dualNode)) {
return fmt.Errorf("cluster-cidr: %v and node-ip: %v, must share the same IP version (IPv4, IPv6 or dual-stack)", nodeConfig.AgentConfig.ClusterCIDRs, nodeConfig.AgentConfig.NodeIPs)
}
enableIPv6 := dualCluster || clusterIPv6
enableIPv4 := dualCluster || clusterIPv4

68
pkg/cli/server/server.go
View File

@ -302,14 +302,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, serverConfig.ControlConfig.AdvertiseIP)
}
// configure ClusterIPRanges
_, _, IPv6only, _ := util.GetFirstIP(nodeIPs)
// configure ClusterIPRanges. Use default 10.42.0.0/16 or fd00:42::/56 if user did not set it
_, defaultClusterCIDR, defaultServiceCIDR, _ := util.GetDefaultAddresses(nodeIPs[0])
if len(cmds.ServerConfig.ClusterCIDR) == 0 {
clusterCIDR := "10.42.0.0/16"
if IPv6only {
clusterCIDR = "fd00:42::/56"
}
cmds.ServerConfig.ClusterCIDR.Set(clusterCIDR)
cmds.ServerConfig.ClusterCIDR.Set(defaultClusterCIDR)
}
for _, cidr := range util.SplitStringSlice(cmds.ServerConfig.ClusterCIDR) {
_, parsed, err := net.ParseCIDR(cidr)
@ -319,21 +315,12 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
serverConfig.ControlConfig.ClusterIPRanges = append(serverConfig.ControlConfig.ClusterIPRanges, parsed)
}
// set ClusterIPRange to the first IPv4 block, for legacy clients
// unless only IPv6 range given
clusterIPRange, err := util.GetFirstNet(serverConfig.ControlConfig.ClusterIPRanges)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-cidr")
}
serverConfig.ControlConfig.ClusterIPRange = clusterIPRange
// set ClusterIPRange to the first address (first defined IPFamily is preferred)
serverConfig.ControlConfig.ClusterIPRange = serverConfig.ControlConfig.ClusterIPRanges[0]
// configure ServiceIPRanges
// configure ServiceIPRanges. Use default 10.43.0.0/16 or fd00:43::/112 if user did not set it
if len(cmds.ServerConfig.ServiceCIDR) == 0 {
serviceCIDR := "10.43.0.0/16"
if IPv6only {
serviceCIDR = "fd00:43::/112"
}
cmds.ServerConfig.ServiceCIDR.Set(serviceCIDR)
cmds.ServerConfig.ServiceCIDR.Set(defaultServiceCIDR)
}
for _, cidr := range util.SplitStringSlice(cmds.ServerConfig.ServiceCIDR) {
_, parsed, err := net.ParseCIDR(cidr)
@ -343,13 +330,8 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
serverConfig.ControlConfig.ServiceIPRanges = append(serverConfig.ControlConfig.ServiceIPRanges, parsed)
}
// set ServiceIPRange to the first IPv4 block, for legacy clients
// unless only IPv6 range given
serviceIPRange, err := util.GetFirstNet(serverConfig.ControlConfig.ServiceIPRanges)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4/IPv6 service-cidr")
}
serverConfig.ControlConfig.ServiceIPRange = serviceIPRange
// set ServiceIPRange to the first address (first defined IPFamily is preferred)
serverConfig.ControlConfig.ServiceIPRange = serverConfig.ControlConfig.ServiceIPRanges[0]
serverConfig.ControlConfig.ServiceNodePortRange, err = utilnet.ParsePortRange(cfg.ServiceNodePortRange)
if err != nil {
@ -368,12 +350,13 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
// If there are no IPv4 ServiceCIDRs, an IPv6 ServiceCIDRs will be used.
// If neither of IPv4 or IPv6 are found an error is raised.
if len(cmds.ServerConfig.ClusterDNS) == 0 {
clusterDNS, err := utilsnet.GetIndexedIP(serverConfig.ControlConfig.ServiceIPRange, 10)
if err != nil {
return errors.Wrap(err, "cannot configure default cluster-dns address")
for _, svcCIDR := range serverConfig.ControlConfig.ServiceIPRanges {
clusterDNS, err := utilsnet.GetIndexedIP(svcCIDR, 10)
if err != nil {
return errors.Wrap(err, "cannot configure default cluster-dns address")
}
serverConfig.ControlConfig.ClusterDNSs = append(serverConfig.ControlConfig.ClusterDNSs, clusterDNS)
}
serverConfig.ControlConfig.ClusterDNS = clusterDNS
serverConfig.ControlConfig.ClusterDNSs = []net.IP{serverConfig.ControlConfig.ClusterDNS}
} else {
for _, ip := range util.SplitStringSlice(cmds.ServerConfig.ClusterDNS) {
parsed := net.ParseIP(ip)
@ -382,15 +365,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
}
serverConfig.ControlConfig.ClusterDNSs = append(serverConfig.ControlConfig.ClusterDNSs, parsed)
}
// Set ClusterDNS to the first IPv4 address, for legacy clients
// unless only IPv6 range given
clusterDNS, _, _, err := util.GetFirstIP(serverConfig.ControlConfig.ClusterDNSs)
if err != nil {
return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-dns address")
}
serverConfig.ControlConfig.ClusterDNS = clusterDNS
}
serverConfig.ControlConfig.ClusterDNS = serverConfig.ControlConfig.ClusterDNSs[0]
if err := validateNetworkConfiguration(serverConfig); err != nil {
return err
}
@ -582,18 +560,6 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
// validateNetworkConfig ensures that the network configuration values make sense.
func validateNetworkConfiguration(serverConfig server.Config) error {
// Dual-stack operation requires fairly extensive manual configuration at the moment - do some
// preflight checks to make sure that the user isn't trying to use flannel/npc, or trying to
// enable dual-stack DNS (which we don't currently support since it's not easy to template)
dualDNS, err := utilsnet.IsDualStackIPs(serverConfig.ControlConfig.ClusterDNSs)
if err != nil {
return errors.Wrap(err, "failed to validate cluster-dns")
}
if dualDNS == true {
return errors.New("dual-stack cluster-dns is not supported")
}
switch serverConfig.ControlConfig.EgressSelectorMode {
case config.EgressSelectorModeCluster, config.EgressSelectorModePod:
case config.EgressSelectorModeAgent, config.EgressSelectorModeDisabled:

5
pkg/cluster/cluster.go
View File

@ -10,10 +10,10 @@ import (
"github.com/k3s-io/k3s/pkg/cluster/managed"
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/etcd"
"github.com/k3s-io/k3s/pkg/util"
"github.com/k3s-io/kine/pkg/endpoint"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
utilsnet "k8s.io/utils/net"
)
type Cluster struct {
@ -54,8 +54,7 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
clientURL.Host = clientURL.Hostname() + ":2379"
clientURLs = append(clientURLs, clientURL.String())
}
IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(c.config.ServiceIPRanges)
etcdProxy, err := etcd.NewETCDProxy(ctx, true, c.config.DataDir, clientURLs[0], IPv6OnlyService)
etcdProxy, err := etcd.NewETCDProxy(ctx, true, c.config.DataDir, clientURLs[0], utilsnet.IsIPv6CIDR(c.config.ServiceIPRanges[0]))
if err != nil {
return nil, err
}

3
pkg/daemons/config/types.go
View File

@ -10,7 +10,6 @@ import (
"sync"
"time"
"github.com/k3s-io/k3s/pkg/util"
"github.com/k3s-io/kine/pkg/endpoint"
"github.com/rancher/wrangler/pkg/generated/controllers/core"
"github.com/rancher/wrangler/pkg/leader"
@ -250,7 +249,7 @@ func (c *Control) BindAddressOrLoopback(chooseHostInterface, urlSafe bool) strin
// service CIDRs indicate an IPv4/Dual-Stack or IPv6 only cluster. If the urlSafe
// parameter is true, IPv6 addresses are enclosed in square brackets, as per RFC2732.
func (c *Control) Loopback(urlSafe bool) string {
if IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(c.ServiceIPRanges); IPv6OnlyService {
if utilsnet.IsIPv6CIDR(c.ServiceIPRange) {
if urlSafe {
return "[::1]"
}

4
pkg/deploy/zz_generated_bindata.go
View File

File diff suppressed because one or more lines are too long

8
pkg/server/server.go
View File

@ -273,8 +273,16 @@ func stageFiles(ctx context.Context, sc *Context, controlConfig *config.Control)
return err
}
dataDir = filepath.Join(controlConfig.DataDir, "manifests")
dnsIPFamilyPolicy := "PreferDualStack"
if len(controlConfig.ClusterDNSs) == 1 {
dnsIPFamilyPolicy = "SingleStack"
}
templateVars := map[string]string{
"%{CLUSTER_DNS}%": controlConfig.ClusterDNS.String(),
"%{CLUSTER_DNS_LIST}%": fmt.Sprintf("[%s]", util.JoinIPs(controlConfig.ClusterDNSs)),
"%{CLUSTER_DNS_IPFAMILYPOLICY}%": dnsIPFamilyPolicy,
"%{CLUSTER_DOMAIN}%": controlConfig.ClusterDomain,
"%{DEFAULT_LOCAL_STORAGE_PATH}%": controlConfig.DefaultLocalStoragePath,
"%{SYSTEM_DEFAULT_REGISTRY}%": registryTemplate(controlConfig.SystemDefaultRegistry),

95
pkg/util/net.go
View File

@ -31,9 +31,9 @@ func JoinIPNets(elems []*net.IPNet) string {
return strings.Join(strs, ",")
}
// GetFirst4Net returns the first IPv4 network from the list of IP networks.
// getFirst4Net returns the first IPv4 network from the list of IP networks.
// If no IPv4 addresses are found, an error is raised.
func GetFirst4Net(elems []*net.IPNet) (*net.IPNet, error) {
func getFirst4Net(elems []*net.IPNet) (*net.IPNet, error) {
for _, elem := range elems {
if elem == nil || elem.IP.To4() == nil {
continue
@ -43,9 +43,9 @@ func GetFirst4Net(elems []*net.IPNet) (*net.IPNet, error) {
return nil, errors.New("no IPv4 CIDRs found")
}
// GetFirst4 returns the first IPv4 address from the list of IP addresses.
// getFirst4 returns the first IPv4 address from the list of IP addresses.
// If no IPv4 addresses are found, an error is raised.
func GetFirst4(elems []net.IP) (net.IP, error) {
func getFirst4(elems []net.IP) (net.IP, error) {
for _, elem := range elems {
if elem == nil || elem.To4() == nil {
continue
@ -64,7 +64,7 @@ func GetFirst4String(elems []string) (string, error) {
ips = append(ips, net.ParseIP(v))
}
}
ip, err := GetFirst4(ips)
ip, err := getFirst4(ips)
if err != nil {
return "", err
}
@ -82,9 +82,9 @@ func JoinIP4Nets(elems []*net.IPNet) string {
return strings.Join(strs, ",")
}
// GetFirst6 returns the first IPv6 address from the list of IP addresses.
// getFirst6 returns the first IPv6 address from the list of IP addresses.
// If no IPv6 addresses are found, an error is raised.
func GetFirst6(elems []net.IP) (net.IP, error) {
func getFirst6(elems []net.IP) (net.IP, error) {
for _, elem := range elems {
if elem != nil && netutils.IsIPv6(elem) {
return elem, nil
@ -93,9 +93,9 @@ func GetFirst6(elems []net.IP) (net.IP, error) {
return nil, errors.New("no IPv6 address found")
}
// GetFirst6Net returns the first IPv4 network from the list of IP networks.
// getFirst6Net returns the first IPv4 network from the list of IP networks.
// If no IPv6 addresses are found, an error is raised.
func GetFirst6Net(elems []*net.IPNet) (*net.IPNet, error) {
func getFirst6Net(elems []*net.IPNet) (*net.IPNet, error) {
for _, elem := range elems {
if elem != nil && netutils.IsIPv6(elem.IP) {
return elem, nil
@ -113,7 +113,7 @@ func GetFirst6String(elems []string) (string, error) {
ips = append(ips, net.ParseIP(v))
}
}
ip, err := GetFirst6(ips)
ip, err := getFirst6(ips)
if err != nil {
return "", err
}
@ -133,7 +133,7 @@ func JoinIP6Nets(elems []*net.IPNet) string {
// GetHostnameAndIPs takes a node name and list of IPs, usually from CLI args.
// If set, these are used to return the node's name and addresses. If not set,
// the system hostname and primary interface address are returned instead.
// the system hostname and primary interface addresses are returned instead.
func GetHostnameAndIPs(name string, nodeIPs cli.StringSlice) (string, []net.IP, error) {
ips := []net.IP{}
if len(nodeIPs) == 0 {
@ -202,37 +202,26 @@ func GetFirstValidIPString(s cli.StringSlice) string {
return ""
}
// GetFirstIP returns the first IPv4 address from the list of IP addresses.
// If no IPv4 addresses are found, returns the first IPv6 address
// if neither of IPv4 or IPv6 are found an error is raised.
// Additionally matching listen address and IP version is returned.
func GetFirstIP(nodeIPs []net.IP) (net.IP, string, bool, error) {
nodeIP, err := GetFirst4(nodeIPs)
ListenAddress := "0.0.0.0"
IPv6only := false
if err != nil {
nodeIP, err = GetFirst6(nodeIPs)
if err != nil {
return nil, "", false, err
}
ListenAddress = "::"
IPv6only = true
}
return nodeIP, ListenAddress, IPv6only, nil
}
// GetFirstIP checks what is the IPFamily of the first item. Based on that, returns a set of values
func GetDefaultAddresses(nodeIP net.IP) (string, string, string, error) {
// GetFirstNet returns the first IPv4 network from the list of IP networks.
// If no IPv4 addresses are found, returns the first IPv6 address
// if neither of IPv4 or IPv6 are found an error is raised.
func GetFirstNet(elems []*net.IPNet) (*net.IPNet, error) {
serviceIPRange, err := GetFirst4Net(elems)
if err != nil {
serviceIPRange, err = GetFirst6Net(elems)
if err != nil {
return nil, err
}
if netutils.IsIPv4(nodeIP) {
ListenAddress := "0.0.0.0"
clusterCIDR := "10.42.0.0/16"
serviceCIDR := "10.43.0.0/16"
return ListenAddress, clusterCIDR, serviceCIDR, nil
}
return serviceIPRange, nil
if netutils.IsIPv6(nodeIP) {
ListenAddress := "::"
clusterCIDR := "fd00:42::/56"
serviceCIDR := "fd00:43::/112"
return ListenAddress, clusterCIDR, serviceCIDR, nil
}
return "", "", "", fmt.Errorf("ip: %v is not ipv4 or ipv6", nodeIP)
}
// GetFirstString returns the first IP4 address from a list of IP address strings.
@ -251,32 +240,6 @@ func GetFirstString(elems []string) (string, bool, error) {
return ip, IPv6only, nil
}
// IsIPv6OnlyCIDRs returns if
// - all are valid cidrs
// - at least one cidr from v6 family is found
// - v4 family cidr is not found
func IsIPv6OnlyCIDRs(cidrs []*net.IPNet) (bool, error) {
v4Found := false
v6Found := false
for _, cidr := range cidrs {
if cidr == nil {
return false, fmt.Errorf("cidr %v is invalid", cidr)
}
if v4Found && v6Found {
continue
}
if cidr.IP != nil && cidr.IP.To4() == nil {
v6Found = true
continue
}
v4Found = true
}
return !v4Found && v6Found, nil
}
// IPToIPNet converts an IP to an IPNet, using a fully filled mask appropriate for the address family.
func IPToIPNet(ip net.IP) (*net.IPNet, error) {
address := ip.String()

2
tests/integration/longhorn/longhorn_int_test.go
View File

@ -107,7 +107,7 @@ var _ = Describe("longhorn", Ordered, func() {
return fmt.Errorf("pod volume-test \"%s\" reason: \"%s\" message \"%s\"", pod.Status.Phase, pod.Status.Reason, pod.Status.Message)
}
return nil
}, "60s", "5s").Should(Succeed())
}, "120s", "5s").Should(Succeed())
})
})