github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #56390 from m1093782566/test-mode 

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Test Proxy Mode

**What this PR does / why we need it**:

Currently, IPVS-based kube-proxy is protected by feature gateway and there is no UT for it. We should test ipvs proxy mode when feature gateway is set.

**Which issue(s) this PR fixes**:
Fixes #56404

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```
pull/6/head
Kubernetes Submit Queue 2017-12-15 21:59:32 -08:00 committed by GitHub
parent abc73363bd 10aea7e88c
commit bb94058b4f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 292 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/kube-proxy/app/BUILD
View File

@ -92,12 +92,14 @@ go_test(
deps = [
"//pkg/apis/core:go_default_library",
"//pkg/proxy/apis/kubeproxyconfig:go_default_library",
"//pkg/proxy/ipvs:go_default_library",
"//pkg/util/configz:go_default_library",
"//pkg/util/iptables:go_default_library",
"//pkg/util/pointer:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
],
)

14
cmd/kube-proxy/app/server_others.go
View File

@ -83,6 +83,7 @@ func newProxyServer(
var iptInterface utiliptables.Interface
var ipvsInterface utilipvs.Interface
var kernelHandler ipvs.KernelHandler
var ipsetInterface utilipset.Interface
var dbus utildbus.Interface
@ -92,6 +93,7 @@ func newProxyServer(
dbus = utildbus.New()
iptInterface = utiliptables.New(execer, dbus, protocol)
ipvsInterface = utilipvs.New(execer)
kernelHandler = ipvs.NewLinuxKernelHandler()
ipsetInterface = utilipset.New(execer)
// We omit creation of pretty much everything if we run in cleanup mode
@ -133,7 +135,7 @@ func newProxyServer(
var serviceEventHandler proxyconfig.ServiceHandler
var endpointsEventHandler proxyconfig.EndpointsHandler
proxyMode := getProxyMode(string(config.Mode), iptInterface, ipsetInterface, iptables.LinuxKernelCompatTester{})
proxyMode := getProxyMode(string(config.Mode), iptInterface, kernelHandler, ipsetInterface, iptables.LinuxKernelCompatTester{})
if proxyMode == proxyModeIPTables {
glog.V(0).Info("Using iptables Proxier.")
nodeIP := net.ParseIP(config.BindAddress)
@ -269,7 +271,7 @@ func newProxyServer(
}, nil
}
func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string {
func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, khandle ipvs.KernelHandler, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string {
if proxyMode == proxyModeUserspace {
return proxyModeUserspace
}
@ -280,7 +282,7 @@ func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, ipsetver
if utilfeature.DefaultFeatureGate.Enabled(features.SupportIPVSProxyMode) {
if proxyMode == proxyModeIPVS {
return tryIPVSProxy(iptver, ipsetver, kcompat)
return tryIPVSProxy(iptver, khandle, ipsetver, kcompat)
} else {
glog.Warningf("Can't use ipvs proxier, trying iptables proxier")
return tryIPTablesProxy(iptver, kcompat)
@ -290,10 +292,10 @@ func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, ipsetver
return tryIPTablesProxy(iptver, kcompat)
}
func tryIPVSProxy(iptver iptables.IPTablesVersioner, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string {
func tryIPVSProxy(iptver iptables.IPTablesVersioner, khandle ipvs.KernelHandler, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string {
// guaranteed false on error, error only necessary for debugging
// IPVS Proxier relies on ipset
useIPVSProxy, err := ipvs.CanUseIPVSProxier(ipsetver)
// IPVS Proxier relies on ip_vs_* kernel modules and ipset
useIPVSProxy, err := ipvs.CanUseIPVSProxier(khandle, ipsetver)
if err != nil {
// Try to fallback to iptables before falling back to userspace
utilruntime.HandleError(fmt.Errorf("can't determine whether to use ipvs proxy, error: %v", err))

164
cmd/kube-proxy/app/server_test.go
View File

@ -28,8 +28,10 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/diff"
utilfeature "k8s.io/apiserver/pkg/util/feature"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig"
"k8s.io/kubernetes/pkg/proxy/ipvs"
"k8s.io/kubernetes/pkg/util/configz"
"k8s.io/kubernetes/pkg/util/iptables"
utilpointer "k8s.io/kubernetes/pkg/util/pointer"
@ -72,6 +74,15 @@ func (fake *fakeKernelCompatTester) IsCompatible() error {
return nil
}
// fakeKernelHandler implements KernelHandler.
type fakeKernelHandler struct {
modules []string
}
func (fake *fakeKernelHandler) GetModules() ([]string, error) {
return fake.modules, nil
}
func Test_getProxyMode(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("skipping on non-Linux")
@ -82,6 +93,7 @@ func Test_getProxyMode(t *testing.T) {
annotationVal string
iptablesVersion string
ipsetVersion string
kmods []string
kernelCompat bool
iptablesError error
ipsetError error
@ -135,12 +147,162 @@ func Test_getProxyMode(t *testing.T) {
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // specify ipvs, feature gateway disabled, iptables version ok, kernel is compatible
flag: "ipvs",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // specify ipvs, feature gateway disabled, iptables version too low
flag: "ipvs",
iptablesVersion: "0.0.0",
expected: proxyModeUserspace,
},
{ // specify ipvs, feature gateway disabled, iptables version ok, kernel is not compatible
flag: "ipvs",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: false,
expected: proxyModeUserspace,
},
}
for i, c := range cases {
versioner := &fakeIPTablesVersioner{c.iptablesVersion, c.iptablesError}
kcompater := &fakeKernelCompatTester{c.kernelCompat}
ipsetver := &fakeIPSetVersioner{c.ipsetVersion, c.ipsetError}
r := getProxyMode(c.flag, versioner, ipsetver, kcompater)
khandler := &fakeKernelHandler{c.kmods}
r := getProxyMode(c.flag, versioner, khandler, ipsetver, kcompater)
if r != c.expected {
t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r)
}
}
}
// This is a coarse test, but it offers some modicum of confidence as the code is evolved.
func Test_getProxyModeEnableFeatureGateway(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("skipping on non-Linux")
}
// enable IPVS feature gateway
utilfeature.DefaultFeatureGate.Set("SupportIPVSProxyMode=true")
var cases = []struct {
flag string
iptablesVersion string
ipsetVersion string
kernelCompat bool
iptablesError error
ipsetError error
mods []string
expected string
}{
{ // flag says userspace
flag: "userspace",
expected: proxyModeUserspace,
},
{ // flag says iptables, error detecting version
flag: "iptables",
iptablesError: fmt.Errorf("oops!"),
expected: proxyModeUserspace,
},
{ // flag says iptables, version too low
flag: "iptables",
iptablesVersion: "0.0.0",
expected: proxyModeUserspace,
},
{ // flag says iptables, version ok, kernel not compatible
flag: "iptables",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: false,
expected: proxyModeUserspace,
},
{ // flag says iptables, version ok, kernel is compatible
flag: "iptables",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // detect, error
flag: "",
iptablesError: fmt.Errorf("oops!"),
expected: proxyModeUserspace,
},
{ // detect, version too low
flag: "",
iptablesVersion: "0.0.0",
expected: proxyModeUserspace,
},
{ // detect, version ok, kernel not compatible
flag: "",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: false,
expected: proxyModeUserspace,
},
{ // detect, version ok, kernel is compatible
flag: "",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // detect, version ok, kernel is compatible
flag: "",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // flag says ipvs, ipset version ok, kernel modules installed
flag: "ipvs",
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
ipsetVersion: ipvs.MinIPSetCheckVersion,
expected: proxyModeIPVS,
},
{ // flag says ipvs, ipset version too low, fallback on iptables mode
flag: "ipvs",
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
ipsetVersion: "0.0",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // flag says ipvs, bad ipset version, fallback on iptables mode
flag: "ipvs",
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
ipsetVersion: "a.b.c",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // flag says ipvs, required kernel modules are not installed, fallback on iptables mode
flag: "ipvs",
mods: []string{"foo", "bar", "baz"},
ipsetVersion: ipvs.MinIPSetCheckVersion,
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // flag says ipvs, required kernel modules are not installed, iptables version too old, fallback on userspace mode
flag: "ipvs",
mods: []string{"foo", "bar", "baz"},
ipsetVersion: ipvs.MinIPSetCheckVersion,
iptablesVersion: "0.0.0",
kernelCompat: true,
expected: proxyModeUserspace,
},
{ // flag says ipvs, ipset version too low, iptables version too old, kernel not compatible, fallback on userspace mode
flag: "ipvs",
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
ipsetVersion: "0.0",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: false,
expected: proxyModeUserspace,
},
}
for i, c := range cases {
versioner := &fakeIPTablesVersioner{c.iptablesVersion, c.iptablesError}
kcompater := &fakeKernelCompatTester{c.kernelCompat}
ipsetver := &fakeIPSetVersioner{c.ipsetVersion, c.ipsetError}
khandle := &fakeKernelHandler{c.mods}
r := getProxyMode(c.flag, versioner, khandle, ipsetver, kcompater)
if r != c.expected {
t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r)
}

44
pkg/proxy/ipvs/proxier.go
View File

@ -688,14 +688,28 @@ func (em proxyEndpointsMap) unmerge(other proxyEndpointsMap) {
}
}
// CanUseIPVSProxier returns true if we can use the ipvs Proxier.
// This is determined by checking if all the required kernel modules can be loaded. It may
// return an error if it fails to get the kernel modules information without error, in which
// case it will also return false.
func CanUseIPVSProxier(ipsetver IPSetVersioner) (bool, error) {
// Try to load IPVS required kernel modules using modprobe
// KernelHandler can handle the current installed kernel modules.
type KernelHandler interface {
GetModules() ([]string, error)
}
// LinuxKernelHandler implements KernelHandler interface.
type LinuxKernelHandler struct {
executor utilexec.Interface
}
// NewLinuxKernelHandler initializes LinuxKernelHandler with exec.
func NewLinuxKernelHandler() *LinuxKernelHandler {
return &LinuxKernelHandler{
executor: utilexec.New(),
}
}
// GetModules returns all installed kernel modules.
func (handle *LinuxKernelHandler) GetModules() ([]string, error) {
// Try to load IPVS required kernel modules using modprobe first
for _, kmod := range ipvsModules {
err := utilexec.New().Command("modprobe", "--", kmod).Run()
err := handle.executor.Command("modprobe", "--", kmod).Run()
if err != nil {
glog.Warningf("Failed to load kernel module %v with modprobe. "+
"You can ignore this message when kube-proxy is running inside container without mounting /lib/modules", kmod)
@ -703,12 +717,24 @@ func CanUseIPVSProxier(ipsetver IPSetVersioner) (bool, error) {
}
// Find out loaded kernel modules
out, err := utilexec.New().Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput()
out, err := handle.executor.Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput()
if err != nil {
return false, err
return nil, err
}
mods := strings.Split(string(out), "\n")
return mods, nil
}
// CanUseIPVSProxier returns true if we can use the ipvs Proxier.
// This is determined by checking if all the required kernel modules can be loaded. It may
// return an error if it fails to get the kernel modules information without error, in which
// case it will also return false.
func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner) (bool, error) {
mods, err := handle.GetModules()
if err != nil {
return false, fmt.Errorf("error getting installed ipvs required kernel modules: %v", err)
}
wantModules := sets.NewString()
loadModules := sets.NewString()
wantModules.Insert(ipvsModules...)

84
pkg/proxy/ipvs/proxier_test.go
View File

@ -18,6 +18,7 @@ package ipvs
import (
"bytes"
"fmt"
"net"
"reflect"
"testing"
@ -87,6 +88,25 @@ func (fake *fakeHealthChecker) SyncEndpoints(newEndpoints map[types.NamespacedNa
return nil
}
// fakeKernelHandler implements KernelHandler.
type fakeKernelHandler struct {
modules []string
}
func (fake *fakeKernelHandler) GetModules() ([]string, error) {
return fake.modules, nil
}
// fakeKernelHandler implements KernelHandler.
type fakeIPSetVersioner struct {
version string
err error
}
func (fake *fakeIPSetVersioner) GetVersion() (string, error) {
return fake.version, fake.err
}
func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset utilipset.Interface, nodeIPs []net.IP) *Proxier {
fcmd := fakeexec.FakeCmd{
CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
@ -180,6 +200,70 @@ func makeTestEndpoints(namespace, name string, eptFunc func(*api.Endpoints)) *ap
return ept
}
func TestCanUseIPVSProxier(t *testing.T) {
testCases := []struct {
mods []string
kernelErr error
ipsetVersion string
ipsetErr error
ok bool
}{
// case 0, kernel error
{
mods: []string{"foo", "bar", "baz"},
kernelErr: fmt.Errorf("oops"),
ipsetVersion: "0.0",
ok: false,
},
// case 1, ipset error
{
mods: []string{"foo", "bar", "baz"},
ipsetVersion: MinIPSetCheckVersion,
ipsetErr: fmt.Errorf("oops"),
ok: false,
},
// case 2, missing required kernel modules and ipset version too low
{
mods: []string{"foo", "bar", "baz"},
ipsetVersion: "1.1",
ok: false,
},
// case 3, missing required ip_vs_* kernel modules
{
mods: []string{"ip_vs", "a", "bc", "def"},
ipsetVersion: MinIPSetCheckVersion,
ok: false,
},
// case 4, ipset version too low
{
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
ipsetVersion: "4.3.0",
ok: false,
},
// case 5
{
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
ipsetVersion: MinIPSetCheckVersion,
ok: true,
},
// case 6
{
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4", "foo", "bar"},
ipsetVersion: "6.19",
ok: true,
},
}
for i := range testCases {
handle := &fakeKernelHandler{modules: testCases[i].mods}
versioner := &fakeIPSetVersioner{version: testCases[i].ipsetVersion, err: testCases[i].ipsetErr}
ok, _ := CanUseIPVSProxier(handle, versioner)
if ok != testCases[i].ok {
t.Errorf("Case [%d], expect %v, got %v", i, testCases[i].ok, ok)
}
}
}
func TestNodePort(t *testing.T) {
ipt := iptablestest.NewFake()
ipvs := ipvstest.NewFake()