From 2f7941dd08077dc5be46e2249acaf3842c0b6067 Mon Sep 17 00:00:00 2001 From: stewart-yu Date: Mon, 21 May 2018 09:42:10 +0800 Subject: [PATCH] load kernel modules required by IPVS in kubeadm --- cmd/kubeadm/.import-restrictions | 1 + cmd/kubeadm/app/preflight/checks.go | 9 ++ pkg/util/ipvs/ipvs.go | 11 ++ pkg/util/ipvs/kernelcheck_linux.go | 94 ++++++++++++++++ pkg/util/ipvs/kernelcheck_linux_test.go | 130 +++++++++++++++++++++++ pkg/util/ipvs/kernelcheck_unsupported.go | 39 +++++++ 6 files changed, 284 insertions(+) create mode 100644 pkg/util/ipvs/kernelcheck_linux.go create mode 100644 pkg/util/ipvs/kernelcheck_linux_test.go create mode 100644 pkg/util/ipvs/kernelcheck_unsupported.go diff --git a/cmd/kubeadm/.import-restrictions b/cmd/kubeadm/.import-restrictions index b62e156460..3b77e68e8e 100644 --- a/cmd/kubeadm/.import-restrictions +++ b/cmd/kubeadm/.import-restrictions @@ -140,6 +140,7 @@ "k8s.io/kubernetes/pkg/util/slice", "k8s.io/kubernetes/pkg/util/taints", "k8s.io/kubernetes/pkg/util/version", + "k8s.io/kubernetes/pkg/util/ipvs", "k8s.io/kubernetes/pkg/version", "k8s.io/kubernetes/pkg/volume", "k8s.io/kubernetes/pkg/volume/util" diff --git a/cmd/kubeadm/app/preflight/checks.go b/cmd/kubeadm/app/preflight/checks.go index a7e7312c36..2519b04167 100644 --- a/cmd/kubeadm/app/preflight/checks.go +++ b/cmd/kubeadm/app/preflight/checks.go @@ -50,6 +50,7 @@ import ( authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" "k8s.io/kubernetes/pkg/util/initsystem" + ipvsutil "k8s.io/kubernetes/pkg/util/ipvs" "k8s.io/kubernetes/pkg/util/procfs" versionutil "k8s.io/kubernetes/pkg/util/version" kubeadmversion "k8s.io/kubernetes/pkg/version" @@ -867,6 +868,13 @@ func RunInitMasterChecks(execer utilsexec.Interface, cfg *kubeadmapi.MasterConfi } checks = addCommonChecks(execer, cfg, checks) + // Check ipvs required kernel module once we use ipvs kube-proxy mode + if cfg.KubeProxy.Config.Mode == ipvsutil.IPVSProxyMode { + checks = append(checks, + ipvsutil.RequiredIPVSKernelModulesAvailableCheck{Executor: execer}, + ) + } + if len(cfg.Etcd.Endpoints) == 0 { // Only do etcd related checks when no external endpoints were specified checks = append(checks, @@ -922,6 +930,7 @@ func RunJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.NodeConfigura FileAvailableCheck{Path: cfg.CACertPath}, FileAvailableCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)}, FileAvailableCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName)}, + ipvsutil.RequiredIPVSKernelModulesAvailableCheck{Executor: execer}, } checks = addCommonChecks(execer, cfg, checks) diff --git a/pkg/util/ipvs/ipvs.go b/pkg/util/ipvs/ipvs.go index 9d15beb0c6..58e76a56c9 100644 --- a/pkg/util/ipvs/ipvs.go +++ b/pkg/util/ipvs/ipvs.go @@ -61,8 +61,19 @@ const ( FlagPersistent = 0x1 // FlagHashed specify IPVS service hash flag FlagHashed = 0x2 + // IPVSProxyMode is match set up cluster with ipvs proxy model + IPVSProxyMode = "ipvs" ) +// Sets of IPVS required kernel modules. +var ipvsModules = []string{ + "ip_vs", + "ip_vs_rr", + "ip_vs_wrr", + "ip_vs_sh", + "nf_conntrack_ipv4", +} + // Equal check the equality of virtual server. // We don't use struct == since it doesn't work because of slice. func (svc *VirtualServer) Equal(other *VirtualServer) bool { diff --git a/pkg/util/ipvs/kernelcheck_linux.go b/pkg/util/ipvs/kernelcheck_linux.go new file mode 100644 index 0000000000..83c3e6b0fe --- /dev/null +++ b/pkg/util/ipvs/kernelcheck_linux.go @@ -0,0 +1,94 @@ +// +build linux + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ipvs + +import ( + "fmt" + "regexp" + "strings" + + "k8s.io/apimachinery/pkg/util/sets" + utilsexec "k8s.io/utils/exec" + + "github.com/golang/glog" +) + +// RequiredIPVSKernelModulesAvailableCheck tests IPVS required kernel modules. +type RequiredIPVSKernelModulesAvailableCheck struct { + Executor utilsexec.Interface +} + +// Name returns label for RequiredIPVSKernelModulesAvailableCheck +func (r RequiredIPVSKernelModulesAvailableCheck) Name() string { + return "RequiredIPVSKernelModulesAvailable" +} + +// Check try to validates IPVS required kernel modules exists or not. +// The name of function can not be changed. +func (r RequiredIPVSKernelModulesAvailableCheck) Check() (warnings, errors []error) { + glog.V(1).Infoln("validating the kernel module IPVS required exists in machine or not") + + // Find out loaded kernel modules + out, err := r.Executor.Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput() + if err != nil { + errors = append(errors, fmt.Errorf("error getting installed ipvs required kernel modules: %v(%s)", err, out)) + return nil, errors + } + mods := strings.Split(string(out), "\n") + + wantModules := sets.NewString() + loadModules := sets.NewString() + wantModules.Insert(ipvsModules...) + loadModules.Insert(mods...) + modules := wantModules.Difference(loadModules).UnsortedList() + + // Check builtin modules exist or not + if len(modules) != 0 { + kernelVersionFile := "/proc/sys/kernel/osrelease" + b, err := r.Executor.Command("cut", "-f1", "-d", " ", kernelVersionFile).CombinedOutput() + if err != nil { + errors = append(errors, fmt.Errorf("error getting os release kernel version: %v(%s)", err, out)) + return nil, errors + } + + kernelVersion := strings.TrimSpace(string(b)) + builtinModsFilePath := fmt.Sprintf("/lib/modules/%s/modules.builtin", kernelVersion) + out, err := r.Executor.Command("cut", "-f1", "-d", " ", builtinModsFilePath).CombinedOutput() + if err != nil { + errors = append(errors, fmt.Errorf("error getting required builtin kernel modules: %v(%s)", err, out)) + return nil, errors + } + + builtInModules := sets.NewString() + for _, builtInMode := range ipvsModules { + match, _ := regexp.Match(builtInMode+".ko", out) + if !match { + builtInModules.Insert(string(builtInMode)) + } + } + if len(builtInModules) != 0 { + warnings = append(warnings, fmt.Errorf( + "the IPVS proxier will not be used, because the following required kernel modules are not loaded: %v or no builtin kernel ipvs support: %v\n"+ + "you can solve this problem with following methods:\n 1. Run 'modprobe -- ' to load missing kernel modules;\n"+ + "2. Provide the missing builtin kernel ipvs support\n", modules, builtInModules)) + } + } + + return warnings, errors +} diff --git a/pkg/util/ipvs/kernelcheck_linux_test.go b/pkg/util/ipvs/kernelcheck_linux_test.go new file mode 100644 index 0000000000..1f87a92390 --- /dev/null +++ b/pkg/util/ipvs/kernelcheck_linux_test.go @@ -0,0 +1,130 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ipvs + +import ( + "testing" + + utilsexec "k8s.io/utils/exec" + fakeexec "k8s.io/utils/exec/testing" +) + +func TestRequiredIPVSKernelModulesAvailableCheck(t *testing.T) { + cases := []struct { + caseName string + + loadedKernel string + kernelVersion string + builtinKernel string + + expectErrors bool + expectWarnings bool + }{ + { + caseName: "no installed kernel modules and no builtin kernel modules", + loadedKernel: "", + kernelVersion: "3.13.0-24-generic", + builtinKernel: "", + expectErrors: false, + expectWarnings: true, + }, + { + caseName: "no installed kernel modules and missing builtin kernel modules", + loadedKernel: "", + kernelVersion: "3.13.0-24-generic", + builtinKernel: "kernel/net/netfilter/ipvs/ip_vs.ko\n" + + "kernel/net/ipv4/netfilter/nf_conntrack_ipv4.ko", + expectErrors: false, + expectWarnings: true, + }, + { + caseName: "no installed kernel modules and own all builtin kernel modules", + loadedKernel: "", + kernelVersion: "3.13.0-24-generic", + builtinKernel: "kernel/net/netfilter/ipvs/ip_vs.ko\n" + + "kernel/net/netfilter/ipvs/ip_vs_rr.ko\n" + + "kernel/net/netfilter/ipvs/ip_vs_wrr.ko\n" + + "kernel/net/netfilter/ipvs/ip_vs_sh.ko\n" + + "kernel/net/ipv4/netfilter/nf_conntrack_ipv4.ko", + expectErrors: false, + expectWarnings: false, + }, + { + caseName: "missing installed kernel modules and no builtin kernel modules", + loadedKernel: "ip_vs", + kernelVersion: "3.13.0-24-generic", + builtinKernel: "", + expectErrors: false, + expectWarnings: true, + }, + { + caseName: "own all installed kernel modules and no builtin kernel modules", + loadedKernel: "ip_vs\n" + "ip_vs_wrr\n" + "nf_conntrack_ipv4\n" + + "ip_vs_rr\n" + "ip_vs_sh", + kernelVersion: "3.13.0-24-generic", + builtinKernel: "", + expectErrors: false, + expectWarnings: false, + }, + { + caseName: "own all installed kernel modules and all builtin kernel modules", + loadedKernel: "ip_vs\n" + "ip_vs_wrr\n" + "nf_conntrack_ipv4\n" + "ip_vs_rr\n" + "ip_vs_sh", + kernelVersion: "3.13.0-24-generic", + builtinKernel: "kernel/net/netfilter/ipvs/ip_vs.ko\n" + + "kernel/net/netfilter/ipvs/ip_vs_rr.ko\n" + + "kernel/net/netfilter/ipvs/ip_vs_wrr.ko\n" + + "kernel/net/netfilter/ipvs/ip_vs_sh.ko\n" + + "kernel/net/ipv4/netfilter/nf_conntrack_ipv4.ko", + expectErrors: false, + expectWarnings: false, + }, + } + + for i, tc := range cases { + fcmd := fakeexec.FakeCmd{ + CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{ + func() ([]byte, error) { return []byte(cases[i].loadedKernel), nil }, + func() ([]byte, error) { return []byte(cases[i].kernelVersion), nil }, + func() ([]byte, error) { return []byte(cases[i].builtinKernel), nil }, + }, + } + + fexec := fakeexec.FakeExec{ + CommandScript: []fakeexec.FakeCommandAction{ + func(cmd string, args ...string) utilsexec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) utilsexec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) utilsexec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + }, + } + + check := RequiredIPVSKernelModulesAvailableCheck{ + Executor: &fexec, + } + warnings, errors := check.Check() + + switch { + case warnings != nil && !tc.expectWarnings: + t.Errorf("RequiredIPVSKernelModulesAvailableCheck: unexpected warnings for installed kernel modules %v and builtin kernel modules %v. Warnings: %v", tc.loadedKernel, tc.builtinKernel, warnings) + case warnings == nil && tc.expectWarnings: + t.Errorf("RequiredIPVSKernelModulesAvailableCheck: expected warnings for installed kernel modules %v and builtin kernel modules %v but got nothing", tc.loadedKernel, tc.builtinKernel) + case errors != nil && !tc.expectErrors: + t.Errorf("RequiredIPVSKernelModulesAvailableCheck: unexpected errors for installed kernel modules %v and builtin kernel modules %v. errors: %v", tc.loadedKernel, tc.builtinKernel, errors) + case errors == nil && tc.expectErrors: + t.Errorf("RequiredIPVSKernelModulesAvailableCheck: expected errors for installed kernel modules %v and builtin kernel modules %v but got nothing", tc.loadedKernel, tc.builtinKernel) + } + } +} diff --git a/pkg/util/ipvs/kernelcheck_unsupported.go b/pkg/util/ipvs/kernelcheck_unsupported.go new file mode 100644 index 0000000000..6247bff851 --- /dev/null +++ b/pkg/util/ipvs/kernelcheck_unsupported.go @@ -0,0 +1,39 @@ +// +build !linux + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ipvs + +import ( + utilsexec "k8s.io/utils/exec" +) + +// RequiredIPVSKernelModulesAvailableCheck tests IPVS required kernel modules. +type RequiredIPVSKernelModulesAvailableCheck struct { + Executor utilsexec.Interface +} + +// Name returns label for RequiredIPVSKernelModulesAvailableCheck +func (r RequiredIPVSKernelModulesAvailableCheck) Name() string { + return "RequiredIPVSKernelModulesAvailable" +} + +// Check try to validates IPVS required kernel modules exists or not. +func (r RequiredIPVSKernelModulesAvailableCheck) Check() (warnings, errors []error) { + + return nil, nil +}