Merge pull request #66491 from Lion-Wei/ipvs-nodeip

Automatic merge from submit-queue (batch tested with PRs 66491, 66587, 66856, 66657, 66923). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

optimize ipvs get nodeIP

**What this PR does / why we need it**:

Optimize ipvs get nodeIP.

The original ipvs `NodeIPs` need first get all local type address to set1, then get address of dummy device `kube-ipvs0` to set2, then do diff of set1 and set2 to get local addresses we need.
This work gonna result in unnecessary resource consumption, especially for large cluster, will have lots address in dummy device `kube-ipvs0`. 

This pr optimized the workaround.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:


**Release note**:
```release-note
NONE
```

pkg/proxy/ipvs/netlink.go

@@ -32,7 +32,7 @@ type NetLinkHandle interface {
 	DeleteDummyDevice(devName string) error
 	// ListBindAddress will list all IP addresses which are bound in a given interface
 	ListBindAddress(devName string) ([]string, error)
-	// GetLocalAddresses returns all unique local type IP addresses based on filter device interface.  If filter device is not given,
-	// it will list all unique local type addresses.
-	GetLocalAddresses(filterDev string) (sets.String, error)
+	// GetLocalAddresses returns all unique local type IP addresses based on specified device and filter device
+	// If device is not specified, it will list all unique local type addresses except filter device addresses
+	GetLocalAddresses(dev, filterDev string) (sets.String, error)
 }

pkg/proxy/ipvs/netlink_linux.go

@@ -123,7 +123,7 @@ func (h *netlinkHandle) ListBindAddress(devName string) ([]string, error) {
 }
 
 // GetLocalAddresses lists all LOCAL type IP addresses from host based on filter device.
-// If filter device is not specified, it's equivalent to exec:
+// If dev is not specified, it's equivalent to exec:
 // $ ip route show table local type local proto kernel
 // 10.0.0.1 dev kube-ipvs0  scope host  src 10.0.0.1
 // 10.0.0.10 dev kube-ipvs0  scope host  src 10.0.0.10
@@ -136,20 +136,28 @@ func (h *netlinkHandle) ListBindAddress(devName string) ([]string, error) {
 // Then cut the unique src IP fields,
 // --> result set: [10.0.0.1, 10.0.0.10, 10.0.0.252, 100.106.89.164, 127.0.0.1, 192.168.122.1]
 
-// If filter device is specified, it's equivalent to exec:
+// If dev is specified, it's equivalent to exec:
 // $ ip route show table local type local proto kernel dev kube-ipvs0
 // 10.0.0.1  scope host  src 10.0.0.1
 // 10.0.0.10  scope host  src 10.0.0.10
 // Then cut the unique src IP fields,
 // --> result set: [10.0.0.1, 10.0.0.10]
-func (h *netlinkHandle) GetLocalAddresses(filterDev string) (sets.String, error) {
-	linkIndex := -1
-	if len(filterDev) != 0 {
+
+// If filterDev is specified, the result will discard route of specified device and cut src from other routes.
+func (h *netlinkHandle) GetLocalAddresses(dev, filterDev string) (sets.String, error) {
+	chosenLinkIndex, filterLinkIndex := -1, -1
+	if dev != "" {
+		link, err := h.LinkByName(dev)
+		if err != nil {
+			return nil, fmt.Errorf("error get device %s, err: %v", filterDev, err)
+		}
+		chosenLinkIndex = link.Attrs().Index
+	} else if filterDev != "" {
 		link, err := h.LinkByName(filterDev)
 		if err != nil {
 			return nil, fmt.Errorf("error get filter device %s, err: %v", filterDev, err)
 		}
-		linkIndex = link.Attrs().Index
+		filterLinkIndex = link.Attrs().Index
 	}
 
 	routeFilter := &netlink.Route{
@@ -159,18 +167,20 @@ func (h *netlinkHandle) GetLocalAddresses(filterDev string) (sets.String, error)
 	}
 	filterMask := netlink.RT_FILTER_TABLE | netlink.RT_FILTER_TYPE | netlink.RT_FILTER_PROTOCOL
 
-	// find filter device
-	if linkIndex != -1 {
-		routeFilter.LinkIndex = linkIndex
+	// find chosen device
+	if chosenLinkIndex != -1 {
+		routeFilter.LinkIndex = chosenLinkIndex
 		filterMask |= netlink.RT_FILTER_OIF
 	}
-
 	routes, err := h.RouteListFiltered(netlink.FAMILY_ALL, routeFilter, filterMask)
 	if err != nil {
 		return nil, fmt.Errorf("error list route table, err: %v", err)
 	}
 	res := sets.NewString()
 	for _, route := range routes {
+		if route.LinkIndex == filterLinkIndex {
+			continue
+		}
 		if route.Src != nil {
 			res.Insert(route.Src.String())
 		}

pkg/proxy/ipvs/netlink_unsupported.go

@@ -58,6 +58,6 @@ func (h *emptyHandle) ListBindAddress(devName string) ([]string, error) {
 }
 
 // GetLocalAddresses is part of interface.
-func (h *emptyHandle) GetLocalAddresses(filterDev string) (sets.String, error) {
+func (h *emptyHandle) GetLocalAddresses(dev, filterDev string) (sets.String, error) {
 	return nil, fmt.Errorf("netlink is not supported in this platform")
 }

pkg/proxy/ipvs/proxier.go

@@ -242,7 +242,8 @@ type realIPGetter struct {
 }
 
 // NodeIPs returns all LOCAL type IP addresses from host which are taken as the Node IPs of NodePort service.
-// Firstly, it will list source IP exists in local route table with `kernel` protocol type.  For example,
+// It will list source IP exists in local route table with `kernel` protocol type, and filter out IPVS proxier
+// created dummy device `kube-ipvs0` For example,
 // $ ip route show table local type local proto kernel
 // 10.0.0.1 dev kube-ipvs0  scope host  src 10.0.0.1
 // 10.0.0.10 dev kube-ipvs0  scope host  src 10.0.0.10
@@ -252,35 +253,14 @@ type realIPGetter struct {
 // 127.0.0.1 dev lo  scope host  src 127.0.0.1
 // 172.17.0.1 dev docker0  scope host  src 172.17.0.1
 // 192.168.122.1 dev virbr0  scope host  src 192.168.122.1
-// Then cut the unique src IP fields,
-// --> result set1: [10.0.0.1, 10.0.0.10, 10.0.0.252, 100.106.89.164, 127.0.0.1, 192.168.122.1]
-
-// NOTE: For cases where an LB acts as a VIP (e.g. Google cloud), the VIP IP is considered LOCAL, but the protocol
-// of the entry is 66, e.g. `10.128.0.6 dev ens4  proto 66  scope host`.  Therefore, the rule mentioned above will
-// filter these entries out.
-
-// Secondly, as we bind Cluster IPs to the dummy interface in IPVS proxier, we need to filter the them out so that
-// we can eventually get the Node IPs.  Fortunately, the dummy interface created by IPVS proxier is known as `kube-ipvs0`,
-// so we just need to specify the `dev kube-ipvs0` argument in ip route command, for example,
-// $ ip route show table local type local proto kernel dev kube-ipvs0
-// 10.0.0.1  scope host  src 10.0.0.1
-// 10.0.0.10  scope host  src 10.0.0.10
-// Then cut the unique src IP fields,
-// --> result set2: [10.0.0.1, 10.0.0.10]
-
-// Finally, Node IP set = set1 - set2
+// Then filter out dev==kube-ipvs0, and cut the unique src IP fields,
+// Node IP set: [100.106.89.164, 127.0.0.1, 192.168.122.1]
 func (r *realIPGetter) NodeIPs() (ips []net.IP, err error) {
 	// Pass in empty filter device name for list all LOCAL type addresses.
-	allAddress, err := r.nl.GetLocalAddresses("")
+	nodeAddress, err := r.nl.GetLocalAddresses("", DefaultDummyDevice)
 	if err != nil {
 		return nil, fmt.Errorf("error listing LOCAL type addresses from host, error: %v", err)
 	}
-	dummyAddress, err := r.nl.GetLocalAddresses(DefaultDummyDevice)
-	if err != nil {
-		return nil, fmt.Errorf("error listing LOCAL type addresses from device: %s, error: %v", DefaultDummyDevice, err)
-	}
-	// exclude ip address from dummy interface created by IPVS proxier - they are all Cluster IPs.
-	nodeAddress := allAddress.Difference(dummyAddress)
 	// translate ip string to IP
 	for _, ipStr := range nodeAddress.UnsortedList() {
 		ips = append(ips, net.ParseIP(ipStr))

pkg/proxy/ipvs/testing/fake.go

@@ -63,17 +63,20 @@ func (h *FakeNetlinkHandle) ListBindAddress(devName string) ([]string, error) {
 }
 
 // GetLocalAddresses is a mock implementation
-func (h *FakeNetlinkHandle) GetLocalAddresses(filterDev string) (sets.String, error) {
+func (h *FakeNetlinkHandle) GetLocalAddresses(dev, filterDev string) (sets.String, error) {
 	res := sets.NewString()
-	if len(filterDev) != 0 {
+	if len(dev) != 0 {
 		// list all addresses from a given network interface.
-		for _, addr := range h.localAddresses[filterDev] {
+		for _, addr := range h.localAddresses[dev] {
 			res.Insert(addr)
 		}
 		return res, nil
 	}
 	// If filterDev is not given, will list all addresses from all available network interface.
 	for linkName := range h.localAddresses {
+		if linkName == filterDev {
+			continue
+		}
 		// list all addresses from a given network interface.
 		for _, addr := range h.localAddresses[linkName] {
 			res.Insert(addr)

pkg/proxy/ipvs/testing/fake_test.go

@@ -27,21 +27,21 @@ func TestSetGetLocalAddresses(t *testing.T) {
 	fake := NewFakeNetlinkHandle()
 	fake.SetLocalAddresses("eth0", "1.2.3.4")
 	expected := sets.NewString("1.2.3.4")
-	addr, _ := fake.GetLocalAddresses("eth0")
+	addr, _ := fake.GetLocalAddresses("eth0", "")
 	if !reflect.DeepEqual(expected, addr) {
 		t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, addr)
 	}
-	list, _ := fake.GetLocalAddresses("")
+	list, _ := fake.GetLocalAddresses("", "")
 	if !reflect.DeepEqual(expected, list) {
 		t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, list)
 	}
 	fake.SetLocalAddresses("lo", "127.0.0.1")
 	expected = sets.NewString("127.0.0.1")
-	addr, _ = fake.GetLocalAddresses("lo")
+	addr, _ = fake.GetLocalAddresses("lo", "")
 	if !reflect.DeepEqual(expected, addr) {
 		t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, addr)
 	}
-	list, _ = fake.GetLocalAddresses("")
+	list, _ = fake.GetLocalAddresses("", "")
 	expected = sets.NewString("1.2.3.4", "127.0.0.1")
 	if !reflect.DeepEqual(expected, list) {
 		t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, list)