From 4b90559369261f1892f81ecdd3cd808e2fe39d22 Mon Sep 17 00:00:00 2001 From: Weibin Lin Date: Fri, 2 Nov 2018 17:19:32 +0800 Subject: [PATCH] use 'nf_conntrack' instead of 'nf_conntrack_ipv4' for linux kernel >= 4.19 --- pkg/proxy/ipvs/proxier.go | 19 +++------ pkg/util/ipvs/ipvs.go | 54 +++++++++++++++++++++---- pkg/util/ipvs/kernelcheck_linux.go | 13 +++--- pkg/util/ipvs/kernelcheck_linux_test.go | 2 +- 4 files changed, 57 insertions(+), 31 deletions(-) diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index cfce9bb508..d9233a0801 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -157,15 +157,6 @@ var ipsetWithIptablesChain = []struct { {kubeNodePortLocalSetSCTP, string(KubeNodePortChain), "RETURN", "dst", "sctp"}, } -var ipvsModules = []string{ - "ip_vs", - "ip_vs_rr", - "ip_vs_wrr", - "ip_vs_sh", - "nf_conntrack_ipv4", - "nf_conntrack", -} - // In IPVS proxy mode, the following flags need to be set const sysctlRouteLocalnet = "net/ipv4/conf/all/route_localnet" const sysctlBridgeCallIPTables = "net/bridge/bridge-nf-call-iptables" @@ -455,14 +446,12 @@ func NewLinuxKernelHandler() *LinuxKernelHandler { // GetModules returns all installed kernel modules. func (handle *LinuxKernelHandler) GetModules() ([]string, error) { // Check whether IPVS required kernel modules are built-in - kernelVersionFile := "/proc/sys/kernel/osrelease" - b, err := ioutil.ReadFile(kernelVersionFile) + kernelVersion, ipvsModules, err := utilipvs.GetKernelVersionAndIPVSMods(handle.executor) if err != nil { - glog.Errorf("Failed to read file %s with error %v", kernelVersionFile, err) + return nil, err } - kernelVersion := strings.TrimSpace(string(b)) builtinModsFilePath := fmt.Sprintf("/lib/modules/%s/modules.builtin", kernelVersion) - b, err = ioutil.ReadFile(builtinModsFilePath) + b, err := ioutil.ReadFile(builtinModsFilePath) if err != nil { glog.Warningf("Failed to read file %s with error %v. You can ignore this message when kube-proxy is running inside container without mounting /lib/modules", builtinModsFilePath, err) } @@ -503,6 +492,8 @@ func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner) (bool, err } wantModules := sets.NewString() loadModules := sets.NewString() + linuxKernelHandler := NewLinuxKernelHandler() + _, ipvsModules, _ := utilipvs.GetKernelVersionAndIPVSMods(linuxKernelHandler.executor) wantModules.Insert(ipvsModules...) loadModules.Insert(mods...) modules := wantModules.Difference(loadModules).UnsortedList() diff --git a/pkg/util/ipvs/ipvs.go b/pkg/util/ipvs/ipvs.go index 5bfa3b1a21..6cd081851c 100644 --- a/pkg/util/ipvs/ipvs.go +++ b/pkg/util/ipvs/ipvs.go @@ -19,6 +19,11 @@ package ipvs import ( "net" "strconv" + "strings" + + "fmt" + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/utils/exec" ) // Interface is an injectable interface for running ipvs commands. Implementations must be goroutine-safe. @@ -67,14 +72,21 @@ const ( IPVSProxyMode = "ipvs" ) -// Sets of IPVS required kernel modules. -var ipvsModules = []string{ - "ip_vs", - "ip_vs_rr", - "ip_vs_wrr", - "ip_vs_sh", - "nf_conntrack_ipv4", -} +// IPVS required kernel modules. +const ( + // ModIPVS is the kernel module "ip_vs" + ModIPVS string = "ip_vs" + // ModIPVSRR is the kernel module "ip_vs_rr" + ModIPVSRR string = "ip_vs_rr" + // ModIPVSWRR is the kernel module "ip_vs_wrr" + ModIPVSWRR string = "ip_vs_wrr" + // ModIPVSSH is the kernel module "ip_vs_sh" + ModIPVSSH string = "ip_vs_sh" + // ModNfConntrackIPV4 is the module "nf_conntrack_ipv4" + ModNfConntrackIPV4 string = "nf_conntrack_ipv4" + // ModNfConntrack is the kernel module "nf_conntrack" + ModNfConntrack string = "nf_conntrack" +) // Equal check the equality of virtual server. // We don't use struct == since it doesn't work because of slice. @@ -110,3 +122,29 @@ func (rs *RealServer) Equal(other *RealServer) bool { return rs.Address.Equal(other.Address) && rs.Port == other.Port } + +// GetKernelVersionAndIPVSMods returns the linux kernel version and the required ipvs modules +func GetKernelVersionAndIPVSMods(Executor exec.Interface) (kernelVersion string, ipvsModules []string, err error) { + kernelVersionFile := "/proc/sys/kernel/osrelease" + out, err := Executor.Command("cut", "-f1", "-d", " ", kernelVersionFile).CombinedOutput() + if err != nil { + return "", nil, fmt.Errorf("error getting os release kernel version: %v(%s)", err, out) + } + kernelVersion = strings.TrimSpace(string(out)) + // parse kernel version + ver1, err := version.ParseGeneric(kernelVersion) + if err != nil { + return kernelVersion, nil, fmt.Errorf("error parsing kernel version: %v(%s)", err, kernelVersion) + } + // "nf_conntrack_ipv4" has been removed since v4.19 + // see https://github.com/torvalds/linux/commit/a0ae2562c6c4b2721d9fddba63b7286c13517d9f + ver2, _ := version.ParseGeneric("4.19") + // get required ipvs modules + if ver1.LessThan(ver2) { + ipvsModules = append(ipvsModules, ModIPVS, ModIPVSRR, ModIPVSWRR, ModIPVSSH, ModNfConntrackIPV4) + } else { + ipvsModules = append(ipvsModules, ModIPVS, ModIPVSRR, ModIPVSWRR, ModIPVSSH, ModNfConntrack) + } + + return kernelVersion, ipvsModules, nil +} diff --git a/pkg/util/ipvs/kernelcheck_linux.go b/pkg/util/ipvs/kernelcheck_linux.go index 83c3e6b0fe..726606b97e 100644 --- a/pkg/util/ipvs/kernelcheck_linux.go +++ b/pkg/util/ipvs/kernelcheck_linux.go @@ -44,6 +44,11 @@ func (r RequiredIPVSKernelModulesAvailableCheck) Name() string { func (r RequiredIPVSKernelModulesAvailableCheck) Check() (warnings, errors []error) { glog.V(1).Infoln("validating the kernel module IPVS required exists in machine or not") + kernelVersion, ipvsModules, err := GetKernelVersionAndIPVSMods(r.Executor) + if err != nil { + errors = append(errors, err) + } + // Find out loaded kernel modules out, err := r.Executor.Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput() if err != nil { @@ -60,14 +65,6 @@ func (r RequiredIPVSKernelModulesAvailableCheck) Check() (warnings, errors []err // Check builtin modules exist or not if len(modules) != 0 { - kernelVersionFile := "/proc/sys/kernel/osrelease" - b, err := r.Executor.Command("cut", "-f1", "-d", " ", kernelVersionFile).CombinedOutput() - if err != nil { - errors = append(errors, fmt.Errorf("error getting os release kernel version: %v(%s)", err, out)) - return nil, errors - } - - kernelVersion := strings.TrimSpace(string(b)) builtinModsFilePath := fmt.Sprintf("/lib/modules/%s/modules.builtin", kernelVersion) out, err := r.Executor.Command("cut", "-f1", "-d", " ", builtinModsFilePath).CombinedOutput() if err != nil { diff --git a/pkg/util/ipvs/kernelcheck_linux_test.go b/pkg/util/ipvs/kernelcheck_linux_test.go index 1f87a92390..8d2eb76465 100644 --- a/pkg/util/ipvs/kernelcheck_linux_test.go +++ b/pkg/util/ipvs/kernelcheck_linux_test.go @@ -97,8 +97,8 @@ func TestRequiredIPVSKernelModulesAvailableCheck(t *testing.T) { for i, tc := range cases { fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{ - func() ([]byte, error) { return []byte(cases[i].loadedKernel), nil }, func() ([]byte, error) { return []byte(cases[i].kernelVersion), nil }, + func() ([]byte, error) { return []byte(cases[i].loadedKernel), nil }, func() ([]byte, error) { return []byte(cases[i].builtinKernel), nil }, }, }