github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor canUseIPVSMode and test it

pull/6/head
m1093782566 2017-11-26 21:47:50 +08:00
parent e7ad6e6008
commit dc2e57ba74
4 changed files with 139 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
cmd/kube-proxy/app/server_others.go
View File

@ -83,6 +83,7 @@ func newProxyServer(
var iptInterface utiliptables.Interface
var ipvsInterface utilipvs.Interface
var kernelHandler ipvs.KernelHandler
var ipsetInterface utilipset.Interface
var dbus utildbus.Interface
@ -92,6 +93,7 @@ func newProxyServer(
dbus = utildbus.New()
iptInterface = utiliptables.New(execer, dbus, protocol)
ipvsInterface = utilipvs.New(execer)
kernelHandler = ipvs.NewLinuxKernelHandler()
ipsetInterface = utilipset.New(execer)
// We omit creation of pretty much everything if we run in cleanup mode
@ -133,7 +135,7 @@ func newProxyServer(
var serviceEventHandler proxyconfig.ServiceHandler
var endpointsEventHandler proxyconfig.EndpointsHandler
proxyMode := getProxyMode(string(config.Mode), iptInterface, ipsetInterface, iptables.LinuxKernelCompatTester{})
proxyMode := getProxyMode(string(config.Mode), iptInterface, kernelHandler, ipsetInterface, iptables.LinuxKernelCompatTester{})
if proxyMode == proxyModeIPTables {
glog.V(0).Info("Using iptables Proxier.")
nodeIP := net.ParseIP(config.BindAddress)
@ -269,7 +271,7 @@ func newProxyServer(
}, nil
}
func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string {
func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, khandle ipvs.KernelHandler, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string {
if proxyMode == proxyModeUserspace {
return proxyModeUserspace
}
@ -280,7 +282,7 @@ func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, ipsetver
if utilfeature.DefaultFeatureGate.Enabled(features.SupportIPVSProxyMode) {
if proxyMode == proxyModeIPVS {
return tryIPVSProxy(iptver, ipsetver, kcompat)
return tryIPVSProxy(iptver, khandle, ipsetver, kcompat)
} else {
glog.Warningf("Can't use ipvs proxier, trying iptables proxier")
return tryIPTablesProxy(iptver, kcompat)
@ -290,10 +292,10 @@ func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, ipsetver
return tryIPTablesProxy(iptver, kcompat)
}
func tryIPVSProxy(iptver iptables.IPTablesVersioner, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string {
func tryIPVSProxy(iptver iptables.IPTablesVersioner, khandle ipvs.KernelHandler, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string {
// guaranteed false on error, error only necessary for debugging
// IPVS Proxier relies on ipset
useIPVSProxy, err := ipvs.CanUseIPVSProxier(ipsetver)
// IPVS Proxier relies on ip_vs_* kernel modules and ipset
useIPVSProxy, err := ipvs.CanUseIPVSProxier(khandle, ipsetver)
if err != nil {
// Try to fallback to iptables before falling back to userspace
utilruntime.HandleError(fmt.Errorf("can't determine whether to use ipvs proxy, error: %v", err))

13
cmd/kube-proxy/app/server_test.go
View File

@ -72,6 +72,15 @@ func (fake *fakeKernelCompatTester) IsCompatible() error {
return nil
}
// fakeKernelHandler implements KernelHandler.
type fakeKernelHandler struct {
modules []string
}
func (fake *fakeKernelHandler) GetModules() ([]string, error) {
return fake.modules, nil
}
func Test_getProxyMode(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("skipping on non-Linux")
@ -82,6 +91,7 @@ func Test_getProxyMode(t *testing.T) {
annotationVal string
iptablesVersion string
ipsetVersion string
kmods []string
kernelCompat bool
iptablesError error
ipsetError error
@ -140,7 +150,8 @@ func Test_getProxyMode(t *testing.T) {
versioner := &fakeIPTablesVersioner{c.iptablesVersion, c.iptablesError}
kcompater := &fakeKernelCompatTester{c.kernelCompat}
ipsetver := &fakeIPSetVersioner{c.ipsetVersion, c.ipsetError}
r := getProxyMode(c.flag, versioner, ipsetver, kcompater)
khandler := &fakeKernelHandler{c.kmods}
r := getProxyMode(c.flag, versioner, khandler, ipsetver, kcompater)
if r != c.expected {
t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r)
}

44
pkg/proxy/ipvs/proxier.go
View File

@ -688,14 +688,28 @@ func (em proxyEndpointsMap) unmerge(other proxyEndpointsMap) {
}
}
// CanUseIPVSProxier returns true if we can use the ipvs Proxier.
// This is determined by checking if all the required kernel modules can be loaded. It may
// return an error if it fails to get the kernel modules information without error, in which
// case it will also return false.
func CanUseIPVSProxier(ipsetver IPSetVersioner) (bool, error) {
// Try to load IPVS required kernel modules using modprobe
// KernelHandler can handle the current installed kernel modules.
type KernelHandler interface {
GetModules() ([]string, error)
}
// LinuxKernelHandler implements KernelHandler interface.
type LinuxKernelHandler struct {
executor utilexec.Interface
}
// NewLinuxKernelHandler initializes LinuxKernelHandler with exec.
func NewLinuxKernelHandler() *LinuxKernelHandler {
return &LinuxKernelHandler{
executor: utilexec.New(),
}
}
// GetModules returns all installed kernel modules.
func (handle *LinuxKernelHandler) GetModules() ([]string, error) {
// Try to load IPVS required kernel modules using modprobe first
for _, kmod := range ipvsModules {
err := utilexec.New().Command("modprobe", "--", kmod).Run()
err := handle.executor.Command("modprobe", "--", kmod).Run()
if err != nil {
glog.Warningf("Failed to load kernel module %v with modprobe. "+
"You can ignore this message when kube-proxy is running inside container without mounting /lib/modules", kmod)
@ -703,12 +717,24 @@ func CanUseIPVSProxier(ipsetver IPSetVersioner) (bool, error) {
}
// Find out loaded kernel modules
out, err := utilexec.New().Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput()
out, err := handle.executor.Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput()
if err != nil {
return false, err
return nil, err
}
mods := strings.Split(string(out), "\n")
return mods, nil
}
// CanUseIPVSProxier returns true if we can use the ipvs Proxier.
// This is determined by checking if all the required kernel modules can be loaded. It may
// return an error if it fails to get the kernel modules information without error, in which
// case it will also return false.
func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner) (bool, error) {
mods, err := handle.GetModules()
if err != nil {
return false, fmt.Errorf("error getting installed ipvs required kernel modules: %v", err)
}
wantModules := sets.NewString()
loadModules := sets.NewString()
wantModules.Insert(ipvsModules...)

84
pkg/proxy/ipvs/proxier_test.go
View File

@ -18,6 +18,7 @@ package ipvs
import (
"bytes"
"fmt"
"net"
"reflect"
"testing"
@ -87,6 +88,25 @@ func (fake *fakeHealthChecker) SyncEndpoints(newEndpoints map[types.NamespacedNa
return nil
}
// fakeKernelHandler implements KernelHandler.
type fakeKernelHandler struct {
modules []string
}
func (fake *fakeKernelHandler) GetModules() ([]string, error) {
return fake.modules, nil
}
// fakeKernelHandler implements KernelHandler.
type fakeIPSetVersioner struct {
version string
err error
}
func (fake *fakeIPSetVersioner) GetVersion() (string, error) {
return fake.version, fake.err
}
func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset utilipset.Interface, nodeIPs []net.IP) *Proxier {
fcmd := fakeexec.FakeCmd{
CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
@ -180,6 +200,70 @@ func makeTestEndpoints(namespace, name string, eptFunc func(*api.Endpoints)) *ap
return ept
}
func TestCanUseIPVSProxier(t *testing.T) {
testCases := []struct {
mods []string
kernelErr error
ipsetVersion string
ipsetErr error
ok bool
}{
// case 0, kernel error
{
mods: []string{"foo", "bar", "baz"},
kernelErr: fmt.Errorf("oops"),
ipsetVersion: "0.0",
ok: false,
},
// case 1, ipset error
{
mods: []string{"foo", "bar", "baz"},
ipsetVersion: MinIPSetCheckVersion,
ipsetErr: fmt.Errorf("oops"),
ok: false,
},
// case 2, missing required kernel modules and ipset version too low
{
mods: []string{"foo", "bar", "baz"},
ipsetVersion: "1.1",
ok: false,
},
// case 3, missing required ip_vs_* kernel modules
{
mods: []string{"ip_vs", "a", "bc", "def"},
ipsetVersion: MinIPSetCheckVersion,
ok: false,
},
// case 4, ipset version too low
{
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
ipsetVersion: "4.3.0",
ok: false,
},
// case 5
{
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
ipsetVersion: MinIPSetCheckVersion,
ok: true,
},
// case 6
{
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4", "foo", "bar"},
ipsetVersion: "6.19",
ok: true,
},
}
for i := range testCases {
handle := &fakeKernelHandler{modules: testCases[i].mods}
versioner := &fakeIPSetVersioner{version: testCases[i].ipsetVersion, err: testCases[i].ipsetErr}
ok, _ := CanUseIPVSProxier(handle, versioner)
if ok != testCases[i].ok {
t.Errorf("Case [%d], expect %v, got %v", i, testCases[i].ok, ok)
}
}
}
func TestNodePort(t *testing.T) {
ipt := iptablestest.NewFake()
ipvs := ipvstest.NewFake()