Merge pull request #54052 from m1093782566/netlink

Automatic merge from submit-queue (batch tested with PRs 55247, 55324, 55261, 55147, 54052). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Implement dummy device operation by netlink for ipvs proxier

**What this PR does / why we need it**:

Currently, we implement network interface operations, e.g. create a dummy device and delete a dummy interface via ip cmd and then collect the exist code. For example, we assume the following command:
```
ip link add kube-ipvs0 type dummy 
```
returns 2 means the dummy device `kube-ipvs0` exists, see https://github.com/kubernetes/kubernetes/blob/master/pkg/proxy/ipvs/proxier.go#L1529 .

However, from the man page of ip command, exit status 2 just means an error was reported by the kernel, see http://man7.org/linux/man-pages/man8/ip.8.html#EXIT_STATUS.

So, that's a bug.

This PR implements dummy device operation by netlink for ipvs proxier so that can get ride of ip command operations.

**Which issue this PR fixes**: 

fixes #54054

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```

/sig network
/area kube-proxy
/kind bug

cmd/kube-proxy/app/server.go

@@ -422,7 +422,7 @@ func (s *ProxyServer) Run() error {
 	if s.CleanupAndExit {
 		encounteredError := userspace.CleanupLeftovers(s.IptInterface)
 		encounteredError = iptables.CleanupLeftovers(s.IptInterface) || encounteredError
-		encounteredError = ipvs.CleanupLeftovers(s.execer, s.IpvsInterface, s.IptInterface) || encounteredError
+		encounteredError = ipvs.CleanupLeftovers(s.IpvsInterface, s.IptInterface) || encounteredError
 		if encounteredError {
 			return errors.New("encountered an error while tearing down rules.")
 		}

cmd/kube-proxy/app/server_others.go

@@ -159,7 +159,7 @@ func NewProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
 		userspace.CleanupLeftovers(iptInterface)
 		// IPVS Proxier will generate some iptables rules,
 		// need to clean them before switching to other proxy mode.
-		ipvs.CleanupLeftovers(execer, ipvsInterface, iptInterface)
+		ipvs.CleanupLeftovers(ipvsInterface, iptInterface)
 	} else if proxyMode == proxyModeIPVS {
 		glog.V(0).Info("Using ipvs Proxier.")
 		proxierIPVS, err := ipvs.NewProxier(
@@ -220,7 +220,7 @@ func NewProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExi
 		iptables.CleanupLeftovers(iptInterface)
 		// IPVS Proxier will generate some iptables rules,
 		// need to clean them before switching to other proxy mode.
-		ipvs.CleanupLeftovers(execer, ipvsInterface, iptInterface)
+		ipvs.CleanupLeftovers(ipvsInterface, iptInterface)
 	}
 
 	iptInterface.AddReloadFunc(proxier.Sync)

pkg/proxy/ipvs/BUILD

@@ -8,7 +8,9 @@ load(
 
 go_test(
     name = "go_default_test",
-    srcs = ["proxier_test.go"],
+    srcs = [
+        "proxier_test.go",
+    ],
     importpath = "k8s.io/kubernetes/pkg/proxy/ipvs",
     library = ":go_default_library",
     deps = [

pkg/proxy/ipvs/netlink.go

@@ -22,4 +22,8 @@ type NetLinkHandle interface {
 	EnsureAddressBind(address, devName string) (exist bool, err error)
 	// UnbindAddress unbind address from the interface
 	UnbindAddress(address, devName string) error
+	// EnsureDummyDevice checks if dummy device is exist and, if not, create one. If the dummy device is already exist, return true.
+	EnsureDummyDevice(devName string) (exist bool, err error)
+	// DeleteDummyDevice deletes the given dummy device by name.
+	DeleteDummyDevice(devName string) error
 }

pkg/proxy/ipvs/netlink_linux.go

@@ -70,3 +70,29 @@ func (h *netlinkHandle) UnbindAddress(address, devName string) error {
 	}
 	return nil
 }
+
+// EnsureDummyDevice is part of interface
+func (h *netlinkHandle) EnsureDummyDevice(devName string) (bool, error) {
+	_, err := h.LinkByName(devName)
+	if err == nil {
+		// found dummy device
+		return true, nil
+	}
+	dummy := &netlink.Dummy{
+		LinkAttrs: netlink.LinkAttrs{Name: devName},
+	}
+	return false, h.LinkAdd(dummy)
+}
+
+// DeleteDummyDevice is part of interface.
+func (h *netlinkHandle) DeleteDummyDevice(devName string) error {
+	link, err := h.LinkByName(devName)
+	if err != nil {
+		return fmt.Errorf("error deleting a non-exist dummy device: %s", devName)
+	}
+	dummy, ok := link.(*netlink.Dummy)
+	if !ok {
+		return fmt.Errorf("expect dummy device, got device type: %s", link.Type())
+	}
+	return h.LinkDel(dummy)
+}

pkg/proxy/ipvs/netlink_unsupported.go

@@ -39,3 +39,13 @@ func (h *emptyHandle) EnsureAddressBind(address, devName string) (exist bool, er
 func (h *emptyHandle) UnbindAddress(address, devName string) error {
 	return fmt.Errorf("netlink not supported for this platform")
 }
+
+// EnsureDummyDevice is part of interface
+func (h *emptyHandle) EnsureDummyDevice(devName string) (bool, error) {
+	return false, fmt.Errorf("netlink is not supported in this platform")
+}
+
+// DeleteDummyDevice is part of interface.
+func (h *emptyHandle) DeleteDummyDevice(devName string) error {
+	return fmt.Errorf("netlink is not supported in this platform")
+}

pkg/proxy/ipvs/proxier.go

@@ -748,7 +748,7 @@ func cleanupIptablesLeftovers(ipt utiliptables.Interface) (encounteredError bool
 }
 
 // CleanupLeftovers clean up all ipvs and iptables rules created by ipvs Proxier.
-func CleanupLeftovers(execer utilexec.Interface, ipvs utilipvs.Interface, ipt utiliptables.Interface) (encounteredError bool) {
+func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface) (encounteredError bool) {
 	// Return immediately when ipvs interface is nil - Probably initialization failed in somewhere.
 	if ipvs == nil {
 		return true
@@ -761,7 +761,8 @@ func CleanupLeftovers(execer utilexec.Interface, ipvs utilipvs.Interface, ipt ut
 		encounteredError = true
 	}
 	// Delete dummy interface created by ipvs Proxier.
-	err = deleteDummyDevice(execer, DefaultDummyDevice)
+	nl := NewNetLinkHandle()
+	err = nl.DeleteDummyDevice(DefaultDummyDevice)
 	if err != nil {
 		encounteredError = true
 	}
@@ -950,7 +951,7 @@ func (proxier *Proxier) syncProxyRules() {
 	// End install iptables
 
 	// make sure dummy interface exists in the system where ipvs Proxier will bind service address on it
-	_, err = ensureDummyDevice(proxier.exec, DefaultDummyDevice)
+	_, err = proxier.netlinkHandle.EnsureDummyDevice(DefaultDummyDevice)
 	if err != nil {
 		glog.Errorf("Failed to create dummy interface: %s, error: %v", DefaultDummyDevice, err)
 		return
@@ -1525,32 +1526,6 @@ func openLocalPort(lp *utilproxy.LocalPort) (utilproxy.Closeable, error) {
 	return socket, nil
 }
 
-const cmdIP = "ip"
-
-func ensureDummyDevice(execer utilexec.Interface, dummyDev string) (exist bool, err error) {
-	args := []string{"link", "add", dummyDev, "type", "dummy"}
-	out, err := execer.Command(cmdIP, args...).CombinedOutput()
-	if err != nil {
-		// "exit status code 2" will be returned if the device already exists
-		if ee, ok := err.(utilexec.ExitError); ok {
-			if ee.Exited() && ee.ExitStatus() == 2 {
-				return true, nil
-			}
-		}
-		return false, fmt.Errorf("error creating dummy interface %q: %v: %s", dummyDev, err, out)
-	}
-	return false, nil
-}
-
-func deleteDummyDevice(execer utilexec.Interface, dummyDev string) error {
-	args := []string{"link", "del", dummyDev}
-	out, err := execer.Command(cmdIP, args...).CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("error deleting dummy interface %q: %v: %s", dummyDev, err, out)
-	}
-	return nil
-}
-
 // ipvs Proxier fall back on iptables when it needs to do SNAT for engress packets
 // It will only operate iptables *nat table.
 // Create and link the kube postrouting chain for SNAT packets.

pkg/proxy/ipvs/proxier_test.go

@@ -2213,75 +2213,3 @@ func Test_endpointsToEndpointsMap(t *testing.T) {
 		}
 	}
 }
-
-func Test_ensureDummyDevice(t *testing.T) {
-	fcmd := fakeexec.FakeCmd{
-		CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
-			// Success.
-			func() ([]byte, error) { return []byte{}, nil },
-			// Exists.
-			func() ([]byte, error) { return nil, &fakeexec.FakeExitError{Status: 2} },
-		},
-	}
-	fexec := fakeexec.FakeExec{
-		CommandScript: []fakeexec.FakeCommandAction{
-			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
-			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
-		},
-	}
-	// Success.
-	exists, err := ensureDummyDevice(&fexec, DefaultDummyDevice)
-	if err != nil {
-		t.Errorf("expected success, got %v", err)
-	}
-	if exists {
-		t.Errorf("expected exists = false")
-	}
-	if fcmd.CombinedOutputCalls != 1 {
-		t.Errorf("expected 1 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls)
-	}
-	if !sets.NewString(fcmd.CombinedOutputLog[0]...).HasAll("ip", "link", "add", "kube-ipvs0", "type", "dummy") {
-		t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[0])
-	}
-	// Exists.
-	exists, err = ensureDummyDevice(&fexec, DefaultDummyDevice)
-	if err != nil {
-		t.Errorf("expected success, got %v", err)
-	}
-	if !exists {
-		t.Errorf("expected exists = true")
-	}
-}
-
-func Test_deleteDummyDevice(t *testing.T) {
-	fcmd := fakeexec.FakeCmd{
-		CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
-			// Success.
-			func() ([]byte, error) { return []byte{}, nil },
-			// Failure.
-			func() ([]byte, error) { return nil, &fakeexec.FakeExitError{Status: 1} },
-		},
-	}
-	fexec := fakeexec.FakeExec{
-		CommandScript: []fakeexec.FakeCommandAction{
-			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
-			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
-		},
-	}
-	// Success.
-	err := deleteDummyDevice(&fexec, DefaultDummyDevice)
-	if err != nil {
-		t.Errorf("expected success, got %v", err)
-	}
-	if fcmd.CombinedOutputCalls != 1 {
-		t.Errorf("expected 1 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls)
-	}
-	if !sets.NewString(fcmd.CombinedOutputLog[0]...).HasAll("ip", "link", "del", "kube-ipvs0") {
-		t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[0])
-	}
-	// Failure.
-	err = deleteDummyDevice(&fexec, DefaultDummyDevice)
-	if err == nil {
-		t.Errorf("expected failure")
-	}
-}

pkg/proxy/ipvs/testing/fake.go

@@ -34,3 +34,13 @@ func (h *FakeNetlinkHandle) EnsureAddressBind(address, devName string) (exist bo
 func (h *FakeNetlinkHandle) UnbindAddress(address, devName string) error {
 	return nil
 }
+
+// EnsureDummyDevice is a mock implementation
+func (h *FakeNetlinkHandle) EnsureDummyDevice(devName string) (bool, error) {
+	return false, nil
+}
+
+// DeleteDummyDevice is a mock implementation
+func (h *FakeNetlinkHandle) DeleteDummyDevice(devName string) error {
+	return nil
+}