diff --git a/pkg/proxy/ipvs/netlink.go b/pkg/proxy/ipvs/netlink.go index 13be913f41..aa4dbc5b6d 100644 --- a/pkg/proxy/ipvs/netlink.go +++ b/pkg/proxy/ipvs/netlink.go @@ -32,7 +32,7 @@ type NetLinkHandle interface { DeleteDummyDevice(devName string) error // ListBindAddress will list all IP addresses which are bound in a given interface ListBindAddress(devName string) ([]string, error) - // GetLocalAddresses returns all unique local type IP addresses based on filter device interface. If filter device is not given, - // it will list all unique local type addresses. - GetLocalAddresses(filterDev string) (sets.String, error) + // GetLocalAddresses returns all unique local type IP addresses based on specified device and filter device + // If device is not specified, it will list all unique local type addresses except filter device addresses + GetLocalAddresses(dev, filterDev string) (sets.String, error) } diff --git a/pkg/proxy/ipvs/netlink_linux.go b/pkg/proxy/ipvs/netlink_linux.go index 1a47057952..95c4d17f33 100644 --- a/pkg/proxy/ipvs/netlink_linux.go +++ b/pkg/proxy/ipvs/netlink_linux.go @@ -123,7 +123,7 @@ func (h *netlinkHandle) ListBindAddress(devName string) ([]string, error) { } // GetLocalAddresses lists all LOCAL type IP addresses from host based on filter device. -// If filter device is not specified, it's equivalent to exec: +// If dev is not specified, it's equivalent to exec: // $ ip route show table local type local proto kernel // 10.0.0.1 dev kube-ipvs0 scope host src 10.0.0.1 // 10.0.0.10 dev kube-ipvs0 scope host src 10.0.0.10 @@ -136,20 +136,28 @@ func (h *netlinkHandle) ListBindAddress(devName string) ([]string, error) { // Then cut the unique src IP fields, // --> result set: [10.0.0.1, 10.0.0.10, 10.0.0.252, 100.106.89.164, 127.0.0.1, 192.168.122.1] -// If filter device is specified, it's equivalent to exec: +// If dev is specified, it's equivalent to exec: // $ ip route show table local type local proto kernel dev kube-ipvs0 // 10.0.0.1 scope host src 10.0.0.1 // 10.0.0.10 scope host src 10.0.0.10 // Then cut the unique src IP fields, // --> result set: [10.0.0.1, 10.0.0.10] -func (h *netlinkHandle) GetLocalAddresses(filterDev string) (sets.String, error) { - linkIndex := -1 - if len(filterDev) != 0 { + +// If filterDev is specified, the result will discard route of specified device and cut src from other routes. +func (h *netlinkHandle) GetLocalAddresses(dev, filterDev string) (sets.String, error) { + chosenLinkIndex, filterLinkIndex := -1, -1 + if dev != "" { + link, err := h.LinkByName(dev) + if err != nil { + return nil, fmt.Errorf("error get device %s, err: %v", filterDev, err) + } + chosenLinkIndex = link.Attrs().Index + } else if filterDev != "" { link, err := h.LinkByName(filterDev) if err != nil { return nil, fmt.Errorf("error get filter device %s, err: %v", filterDev, err) } - linkIndex = link.Attrs().Index + filterLinkIndex = link.Attrs().Index } routeFilter := &netlink.Route{ @@ -159,18 +167,20 @@ func (h *netlinkHandle) GetLocalAddresses(filterDev string) (sets.String, error) } filterMask := netlink.RT_FILTER_TABLE | netlink.RT_FILTER_TYPE | netlink.RT_FILTER_PROTOCOL - // find filter device - if linkIndex != -1 { - routeFilter.LinkIndex = linkIndex + // find chosen device + if chosenLinkIndex != -1 { + routeFilter.LinkIndex = chosenLinkIndex filterMask |= netlink.RT_FILTER_OIF } - routes, err := h.RouteListFiltered(netlink.FAMILY_ALL, routeFilter, filterMask) if err != nil { return nil, fmt.Errorf("error list route table, err: %v", err) } res := sets.NewString() for _, route := range routes { + if route.LinkIndex == filterLinkIndex { + continue + } if route.Src != nil { res.Insert(route.Src.String()) } diff --git a/pkg/proxy/ipvs/netlink_unsupported.go b/pkg/proxy/ipvs/netlink_unsupported.go index 27d5f4e91d..a83081f1fd 100644 --- a/pkg/proxy/ipvs/netlink_unsupported.go +++ b/pkg/proxy/ipvs/netlink_unsupported.go @@ -58,6 +58,6 @@ func (h *emptyHandle) ListBindAddress(devName string) ([]string, error) { } // GetLocalAddresses is part of interface. -func (h *emptyHandle) GetLocalAddresses(filterDev string) (sets.String, error) { +func (h *emptyHandle) GetLocalAddresses(dev, filterDev string) (sets.String, error) { return nil, fmt.Errorf("netlink is not supported in this platform") } diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index a0012c73bc..a14c586a49 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -241,7 +241,8 @@ type realIPGetter struct { } // NodeIPs returns all LOCAL type IP addresses from host which are taken as the Node IPs of NodePort service. -// Firstly, it will list source IP exists in local route table with `kernel` protocol type. For example, +// It will list source IP exists in local route table with `kernel` protocol type, and filter out IPVS proxier +// created dummy device `kube-ipvs0` For example, // $ ip route show table local type local proto kernel // 10.0.0.1 dev kube-ipvs0 scope host src 10.0.0.1 // 10.0.0.10 dev kube-ipvs0 scope host src 10.0.0.10 @@ -251,35 +252,14 @@ type realIPGetter struct { // 127.0.0.1 dev lo scope host src 127.0.0.1 // 172.17.0.1 dev docker0 scope host src 172.17.0.1 // 192.168.122.1 dev virbr0 scope host src 192.168.122.1 -// Then cut the unique src IP fields, -// --> result set1: [10.0.0.1, 10.0.0.10, 10.0.0.252, 100.106.89.164, 127.0.0.1, 192.168.122.1] - -// NOTE: For cases where an LB acts as a VIP (e.g. Google cloud), the VIP IP is considered LOCAL, but the protocol -// of the entry is 66, e.g. `10.128.0.6 dev ens4 proto 66 scope host`. Therefore, the rule mentioned above will -// filter these entries out. - -// Secondly, as we bind Cluster IPs to the dummy interface in IPVS proxier, we need to filter the them out so that -// we can eventually get the Node IPs. Fortunately, the dummy interface created by IPVS proxier is known as `kube-ipvs0`, -// so we just need to specify the `dev kube-ipvs0` argument in ip route command, for example, -// $ ip route show table local type local proto kernel dev kube-ipvs0 -// 10.0.0.1 scope host src 10.0.0.1 -// 10.0.0.10 scope host src 10.0.0.10 -// Then cut the unique src IP fields, -// --> result set2: [10.0.0.1, 10.0.0.10] - -// Finally, Node IP set = set1 - set2 +// Then filter out dev==kube-ipvs0, and cut the unique src IP fields, +// Node IP set: [100.106.89.164, 127.0.0.1, 192.168.122.1] func (r *realIPGetter) NodeIPs() (ips []net.IP, err error) { // Pass in empty filter device name for list all LOCAL type addresses. - allAddress, err := r.nl.GetLocalAddresses("") + nodeAddress, err := r.nl.GetLocalAddresses("", DefaultDummyDevice) if err != nil { return nil, fmt.Errorf("error listing LOCAL type addresses from host, error: %v", err) } - dummyAddress, err := r.nl.GetLocalAddresses(DefaultDummyDevice) - if err != nil { - return nil, fmt.Errorf("error listing LOCAL type addresses from device: %s, error: %v", DefaultDummyDevice, err) - } - // exclude ip address from dummy interface created by IPVS proxier - they are all Cluster IPs. - nodeAddress := allAddress.Difference(dummyAddress) // translate ip string to IP for _, ipStr := range nodeAddress.UnsortedList() { ips = append(ips, net.ParseIP(ipStr)) diff --git a/pkg/proxy/ipvs/testing/fake.go b/pkg/proxy/ipvs/testing/fake.go index dd12916dcd..39fe7dba2d 100644 --- a/pkg/proxy/ipvs/testing/fake.go +++ b/pkg/proxy/ipvs/testing/fake.go @@ -63,17 +63,20 @@ func (h *FakeNetlinkHandle) ListBindAddress(devName string) ([]string, error) { } // GetLocalAddresses is a mock implementation -func (h *FakeNetlinkHandle) GetLocalAddresses(filterDev string) (sets.String, error) { +func (h *FakeNetlinkHandle) GetLocalAddresses(dev, filterDev string) (sets.String, error) { res := sets.NewString() - if len(filterDev) != 0 { + if len(dev) != 0 { // list all addresses from a given network interface. - for _, addr := range h.localAddresses[filterDev] { + for _, addr := range h.localAddresses[dev] { res.Insert(addr) } return res, nil } // If filterDev is not given, will list all addresses from all available network interface. for linkName := range h.localAddresses { + if linkName == filterDev { + continue + } // list all addresses from a given network interface. for _, addr := range h.localAddresses[linkName] { res.Insert(addr) diff --git a/pkg/proxy/ipvs/testing/fake_test.go b/pkg/proxy/ipvs/testing/fake_test.go index fabc30a584..2c8a5265ba 100644 --- a/pkg/proxy/ipvs/testing/fake_test.go +++ b/pkg/proxy/ipvs/testing/fake_test.go @@ -27,21 +27,21 @@ func TestSetGetLocalAddresses(t *testing.T) { fake := NewFakeNetlinkHandle() fake.SetLocalAddresses("eth0", "1.2.3.4") expected := sets.NewString("1.2.3.4") - addr, _ := fake.GetLocalAddresses("eth0") + addr, _ := fake.GetLocalAddresses("eth0", "") if !reflect.DeepEqual(expected, addr) { t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, addr) } - list, _ := fake.GetLocalAddresses("") + list, _ := fake.GetLocalAddresses("", "") if !reflect.DeepEqual(expected, list) { t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, list) } fake.SetLocalAddresses("lo", "127.0.0.1") expected = sets.NewString("127.0.0.1") - addr, _ = fake.GetLocalAddresses("lo") + addr, _ = fake.GetLocalAddresses("lo", "") if !reflect.DeepEqual(expected, addr) { t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, addr) } - list, _ = fake.GetLocalAddresses("") + list, _ = fake.GetLocalAddresses("", "") expected = sets.NewString("1.2.3.4", "127.0.0.1") if !reflect.DeepEqual(expected, list) { t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, list)