github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

optimize ipvs get nodeIP

pull/8/head
liangwei 2018-07-23 14:36:45 +08:00
parent 4797c8df8f
commit 8a8b5845a9
6 changed files with 39 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/proxy/ipvs/netlink.go
View File

@ -32,7 +32,7 @@ type NetLinkHandle interface {
DeleteDummyDevice(devName string) error
// ListBindAddress will list all IP addresses which are bound in a given interface
ListBindAddress(devName string) ([]string, error)
// GetLocalAddresses returns all unique local type IP addresses based on filter device interface. If filter device is not given,
// it will list all unique local type addresses.
GetLocalAddresses(filterDev string) (sets.String, error)
// GetLocalAddresses returns all unique local type IP addresses based on specified device and filter device
// If device is not specified, it will list all unique local type addresses except filter device addresses
GetLocalAddresses(dev, filterDev string) (sets.String, error)
}

30
pkg/proxy/ipvs/netlink_linux.go
View File

@ -123,7 +123,7 @@ func (h *netlinkHandle) ListBindAddress(devName string) ([]string, error) {
}
// GetLocalAddresses lists all LOCAL type IP addresses from host based on filter device.
// If filter device is not specified, it's equivalent to exec:
// If dev is not specified, it's equivalent to exec:
// $ ip route show table local type local proto kernel
// 10.0.0.1 dev kube-ipvs0 scope host src 10.0.0.1
// 10.0.0.10 dev kube-ipvs0 scope host src 10.0.0.10
@ -136,20 +136,28 @@ func (h *netlinkHandle) ListBindAddress(devName string) ([]string, error) {
// Then cut the unique src IP fields,
// --> result set: [10.0.0.1, 10.0.0.10, 10.0.0.252, 100.106.89.164, 127.0.0.1, 192.168.122.1]
// If filter device is specified, it's equivalent to exec:
// If dev is specified, it's equivalent to exec:
// $ ip route show table local type local proto kernel dev kube-ipvs0
// 10.0.0.1 scope host src 10.0.0.1
// 10.0.0.10 scope host src 10.0.0.10
// Then cut the unique src IP fields,
// --> result set: [10.0.0.1, 10.0.0.10]
func (h *netlinkHandle) GetLocalAddresses(filterDev string) (sets.String, error) {
linkIndex := -1
if len(filterDev) != 0 {
// If filterDev is specified, the result will discard route of specified device and cut src from other routes.
func (h *netlinkHandle) GetLocalAddresses(dev, filterDev string) (sets.String, error) {
chosenLinkIndex, filterLinkIndex := -1, -1
if dev != "" {
link, err := h.LinkByName(dev)
if err != nil {
return nil, fmt.Errorf("error get device %s, err: %v", filterDev, err)
}
chosenLinkIndex = link.Attrs().Index
} else if filterDev != "" {
link, err := h.LinkByName(filterDev)
if err != nil {
return nil, fmt.Errorf("error get filter device %s, err: %v", filterDev, err)
}
linkIndex = link.Attrs().Index
filterLinkIndex = link.Attrs().Index
}
routeFilter := &netlink.Route{
@ -159,18 +167,20 @@ func (h *netlinkHandle) GetLocalAddresses(filterDev string) (sets.String, error)
}
filterMask := netlink.RT_FILTER_TABLE | netlink.RT_FILTER_TYPE | netlink.RT_FILTER_PROTOCOL
// find filter device
if linkIndex != -1 {
routeFilter.LinkIndex = linkIndex
// find chosen device
if chosenLinkIndex != -1 {
routeFilter.LinkIndex = chosenLinkIndex
filterMask |= netlink.RT_FILTER_OIF
}
routes, err := h.RouteListFiltered(netlink.FAMILY_ALL, routeFilter, filterMask)
if err != nil {
return nil, fmt.Errorf("error list route table, err: %v", err)
}
res := sets.NewString()
for _, route := range routes {
if route.LinkIndex == filterLinkIndex {
continue
}
if route.Src != nil {
res.Insert(route.Src.String())
}

2
pkg/proxy/ipvs/netlink_unsupported.go
View File

@ -58,6 +58,6 @@ func (h *emptyHandle) ListBindAddress(devName string) ([]string, error) {
}
// GetLocalAddresses is part of interface.
func (h *emptyHandle) GetLocalAddresses(filterDev string) (sets.String, error) {
func (h *emptyHandle) GetLocalAddresses(dev, filterDev string) (sets.String, error) {
return nil, fmt.Errorf("netlink is not supported in this platform")
}

30
pkg/proxy/ipvs/proxier.go
View File

@ -241,7 +241,8 @@ type realIPGetter struct {
}
// NodeIPs returns all LOCAL type IP addresses from host which are taken as the Node IPs of NodePort service.
// Firstly, it will list source IP exists in local route table with `kernel` protocol type. For example,
// It will list source IP exists in local route table with `kernel` protocol type, and filter out IPVS proxier
// created dummy device `kube-ipvs0` For example,
// $ ip route show table local type local proto kernel
// 10.0.0.1 dev kube-ipvs0 scope host src 10.0.0.1
// 10.0.0.10 dev kube-ipvs0 scope host src 10.0.0.10
@ -251,35 +252,14 @@ type realIPGetter struct {
// 127.0.0.1 dev lo scope host src 127.0.0.1
// 172.17.0.1 dev docker0 scope host src 172.17.0.1
// 192.168.122.1 dev virbr0 scope host src 192.168.122.1
// Then cut the unique src IP fields,
// --> result set1: [10.0.0.1, 10.0.0.10, 10.0.0.252, 100.106.89.164, 127.0.0.1, 192.168.122.1]
// NOTE: For cases where an LB acts as a VIP (e.g. Google cloud), the VIP IP is considered LOCAL, but the protocol
// of the entry is 66, e.g. `10.128.0.6 dev ens4 proto 66 scope host`. Therefore, the rule mentioned above will
// filter these entries out.
// Secondly, as we bind Cluster IPs to the dummy interface in IPVS proxier, we need to filter the them out so that
// we can eventually get the Node IPs. Fortunately, the dummy interface created by IPVS proxier is known as `kube-ipvs0`,
// so we just need to specify the `dev kube-ipvs0` argument in ip route command, for example,
// $ ip route show table local type local proto kernel dev kube-ipvs0
// 10.0.0.1 scope host src 10.0.0.1
// 10.0.0.10 scope host src 10.0.0.10
// Then cut the unique src IP fields,
// --> result set2: [10.0.0.1, 10.0.0.10]
// Finally, Node IP set = set1 - set2
// Then filter out dev==kube-ipvs0, and cut the unique src IP fields,
// Node IP set: [100.106.89.164, 127.0.0.1, 192.168.122.1]
func (r *realIPGetter) NodeIPs() (ips []net.IP, err error) {
// Pass in empty filter device name for list all LOCAL type addresses.
allAddress, err := r.nl.GetLocalAddresses("")
nodeAddress, err := r.nl.GetLocalAddresses("", DefaultDummyDevice)
if err != nil {
return nil, fmt.Errorf("error listing LOCAL type addresses from host, error: %v", err)
}
dummyAddress, err := r.nl.GetLocalAddresses(DefaultDummyDevice)
if err != nil {
return nil, fmt.Errorf("error listing LOCAL type addresses from device: %s, error: %v", DefaultDummyDevice, err)
}
// exclude ip address from dummy interface created by IPVS proxier - they are all Cluster IPs.
nodeAddress := allAddress.Difference(dummyAddress)
// translate ip string to IP
for _, ipStr := range nodeAddress.UnsortedList() {
ips = append(ips, net.ParseIP(ipStr))

9
pkg/proxy/ipvs/testing/fake.go
View File

@ -63,17 +63,20 @@ func (h *FakeNetlinkHandle) ListBindAddress(devName string) ([]string, error) {
}
// GetLocalAddresses is a mock implementation
func (h *FakeNetlinkHandle) GetLocalAddresses(filterDev string) (sets.String, error) {
func (h *FakeNetlinkHandle) GetLocalAddresses(dev, filterDev string) (sets.String, error) {
res := sets.NewString()
if len(filterDev) != 0 {
if len(dev) != 0 {
// list all addresses from a given network interface.
for _, addr := range h.localAddresses[filterDev] {
for _, addr := range h.localAddresses[dev] {
res.Insert(addr)
}
return res, nil
}
// If filterDev is not given, will list all addresses from all available network interface.
for linkName := range h.localAddresses {
if linkName == filterDev {
continue
}
// list all addresses from a given network interface.
for _, addr := range h.localAddresses[linkName] {
res.Insert(addr)

8
pkg/proxy/ipvs/testing/fake_test.go
View File

@ -27,21 +27,21 @@ func TestSetGetLocalAddresses(t *testing.T) {
fake := NewFakeNetlinkHandle()
fake.SetLocalAddresses("eth0", "1.2.3.4")
expected := sets.NewString("1.2.3.4")
addr, _ := fake.GetLocalAddresses("eth0")
addr, _ := fake.GetLocalAddresses("eth0", "")
if !reflect.DeepEqual(expected, addr) {
t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, addr)
}
list, _ := fake.GetLocalAddresses("")
list, _ := fake.GetLocalAddresses("", "")
if !reflect.DeepEqual(expected, list) {
t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, list)
}
fake.SetLocalAddresses("lo", "127.0.0.1")
expected = sets.NewString("127.0.0.1")
addr, _ = fake.GetLocalAddresses("lo")
addr, _ = fake.GetLocalAddresses("lo", "")
if !reflect.DeepEqual(expected, addr) {
t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, addr)
}
list, _ = fake.GetLocalAddresses("")
list, _ = fake.GetLocalAddresses("", "")
expected = sets.NewString("1.2.3.4", "127.0.0.1")
if !reflect.DeepEqual(expected, list) {
t.Errorf("Unexpected mismatch, expected: %v, got: %v", expected, list)