github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
k3s/cmd/kube-proxy/app/server_test.go

594 lines
18 KiB
Go
Raw Blame History

/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package app
import (
"fmt"
"reflect"
"runtime"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/diff"
utilfeature "k8s.io/apiserver/pkg/util/feature"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig"
"k8s.io/kubernetes/pkg/proxy/ipvs"
"k8s.io/kubernetes/pkg/util/configz"
"k8s.io/kubernetes/pkg/util/iptables"
utilpointer "k8s.io/kubernetes/pkg/util/pointer"
)
type fakeNodeInterface struct {
node api.Node
}
func (fake *fakeNodeInterface) Get(hostname string, options metav1.GetOptions) (*api.Node, error) {
return &fake.node, nil
}
type fakeIPTablesVersioner struct {
version string // what to return
err error // what to return
}
func (fake *fakeIPTablesVersioner) GetVersion() (string, error) {
return fake.version, fake.err
}
type fakeIPSetVersioner struct {
version string // what to return
err error // what to return
}
func (fake *fakeIPSetVersioner) GetVersion() (string, error) {
return fake.version, fake.err
}
type fakeKernelCompatTester struct {
ok bool
}
func (fake *fakeKernelCompatTester) IsCompatible() error {
if !fake.ok {
return fmt.Errorf("error")
}
return nil
}
// fakeKernelHandler implements KernelHandler.
type fakeKernelHandler struct {
modules []string
}
func (fake *fakeKernelHandler) GetModules() ([]string, error) {
return fake.modules, nil
}
func Test_getProxyMode(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("skipping on non-Linux")
}
var cases = []struct {
flag string
annotationKey string
annotationVal string
iptablesVersion string
ipsetVersion string
kmods []string
kernelCompat bool
iptablesError error
ipsetError error
expected string
}{
{ // flag says userspace
flag: "userspace",
expected: proxyModeUserspace,
},
{ // flag says iptables, error detecting version
flag: "iptables",
iptablesError: fmt.Errorf("oops!"),
expected: proxyModeUserspace,
},
{ // flag says iptables, version too low
flag: "iptables",
iptablesVersion: "0.0.0",
expected: proxyModeUserspace,
},
{ // flag says iptables, version ok, kernel not compatible
flag: "iptables",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: false,
expected: proxyModeUserspace,
},
{ // flag says iptables, version ok, kernel is compatible
flag: "iptables",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // detect, error
flag: "",
iptablesError: fmt.Errorf("oops!"),
expected: proxyModeUserspace,
},
{ // detect, version too low
flag: "",
iptablesVersion: "0.0.0",
expected: proxyModeUserspace,
},
{ // detect, version ok, kernel not compatible
flag: "",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: false,
expected: proxyModeUserspace,
},
{ // detect, version ok, kernel is compatible
flag: "",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // specify ipvs, feature gateway disabled, iptables version ok, kernel is compatible
flag: "ipvs",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // specify ipvs, feature gateway disabled, iptables version too low
flag: "ipvs",
iptablesVersion: "0.0.0",
expected: proxyModeUserspace,
},
{ // specify ipvs, feature gateway disabled, iptables version ok, kernel is not compatible
flag: "ipvs",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: false,
expected: proxyModeUserspace,
},
}
for i, c := range cases {
versioner := &fakeIPTablesVersioner{c.iptablesVersion, c.iptablesError}
kcompater := &fakeKernelCompatTester{c.kernelCompat}
ipsetver := &fakeIPSetVersioner{c.ipsetVersion, c.ipsetError}
khandler := &fakeKernelHandler{c.kmods}
r := getProxyMode(c.flag, versioner, khandler, ipsetver, kcompater)
if r != c.expected {
t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r)
}
}
}
// This is a coarse test, but it offers some modicum of confidence as the code is evolved.
func Test_getProxyModeEnableFeatureGateway(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("skipping on non-Linux")
}
// enable IPVS feature gateway
utilfeature.DefaultFeatureGate.Set("SupportIPVSProxyMode=true")
var cases = []struct {
flag string
iptablesVersion string
ipsetVersion string
kernelCompat bool
iptablesError error
ipsetError error
mods []string
expected string
}{
{ // flag says userspace
flag: "userspace",
expected: proxyModeUserspace,
},
{ // flag says iptables, error detecting version
flag: "iptables",
iptablesError: fmt.Errorf("oops!"),
expected: proxyModeUserspace,
},
{ // flag says iptables, version too low
flag: "iptables",
iptablesVersion: "0.0.0",
expected: proxyModeUserspace,
},
{ // flag says iptables, version ok, kernel not compatible
flag: "iptables",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: false,
expected: proxyModeUserspace,
},
{ // flag says iptables, version ok, kernel is compatible
flag: "iptables",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // detect, error
flag: "",
iptablesError: fmt.Errorf("oops!"),
expected: proxyModeUserspace,
},
{ // detect, version too low
flag: "",
iptablesVersion: "0.0.0",
expected: proxyModeUserspace,
},
{ // detect, version ok, kernel not compatible
flag: "",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: false,
expected: proxyModeUserspace,
},
{ // detect, version ok, kernel is compatible
flag: "",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // detect, version ok, kernel is compatible
flag: "",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // flag says ipvs, ipset version ok, kernel modules installed
flag: "ipvs",
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
ipsetVersion: ipvs.MinIPSetCheckVersion,
expected: proxyModeIPVS,
},
{ // flag says ipvs, ipset version too low, fallback on iptables mode
flag: "ipvs",
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
ipsetVersion: "0.0",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // flag says ipvs, bad ipset version, fallback on iptables mode
flag: "ipvs",
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
ipsetVersion: "a.b.c",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // flag says ipvs, required kernel modules are not installed, fallback on iptables mode
flag: "ipvs",
mods: []string{"foo", "bar", "baz"},
ipsetVersion: ipvs.MinIPSetCheckVersion,
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // flag says ipvs, required kernel modules are not installed, iptables version too old, fallback on userspace mode
flag: "ipvs",
mods: []string{"foo", "bar", "baz"},
ipsetVersion: ipvs.MinIPSetCheckVersion,
iptablesVersion: "0.0.0",
kernelCompat: true,
expected: proxyModeUserspace,
},
{ // flag says ipvs, ipset version too low, iptables version too old, kernel not compatible, fallback on userspace mode
flag: "ipvs",
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
ipsetVersion: "0.0",
iptablesVersion: iptables.MinCheckVersion,
kernelCompat: false,
expected: proxyModeUserspace,
},
}
for i, c := range cases {
versioner := &fakeIPTablesVersioner{c.iptablesVersion, c.iptablesError}
kcompater := &fakeKernelCompatTester{c.kernelCompat}
ipsetver := &fakeIPSetVersioner{c.ipsetVersion, c.ipsetError}
khandle := &fakeKernelHandler{c.mods}
r := getProxyMode(c.flag, versioner, khandle, ipsetver, kcompater)
if r != c.expected {
t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r)
}
}
}
// This test verifies that NewProxyServer does not crash when CleanupAndExit is true.
func TestProxyServerWithCleanupAndExit(t *testing.T) {
// Each bind address below is a separate test case
bindAddresses := []string{
"0.0.0.0",
"::",
}
for _, addr := range bindAddresses {
options := NewOptions()
options.config = &kubeproxyconfig.KubeProxyConfiguration{
BindAddress: addr,
}
options.CleanupAndExit = true
proxyserver, err := NewProxyServer(options)
assert.Nil(t, err, "unexpected error in NewProxyServer, addr: %s", addr)
assert.NotNil(t, proxyserver, "nil proxy server obj, addr: %s", addr)
assert.NotNil(t, proxyserver.IptInterface, "nil iptables intf, addr: %s", addr)
assert.True(t, proxyserver.CleanupAndExit, "false CleanupAndExit, addr: %s", addr)
// Clean up config for next test case
configz.Delete(kubeproxyconfig.GroupName)
}
}
func TestGetConntrackMax(t *testing.T) {
ncores := runtime.NumCPU()
testCases := []struct {
min int32
max int32
maxPerCore int32
expected int
err string
}{
{
expected: 0,
},
{
max: 12345,
expected: 12345,
},
{
max: 12345,
maxPerCore: 67890,
expected: -1,
err: "mutually exclusive",
},
{
maxPerCore: 67890, // use this if Max is 0
min: 1, // avoid 0 default
expected: 67890 * ncores,
},
{
maxPerCore: 1, // ensure that Min is considered
min: 123456,
expected: 123456,
},
{
maxPerCore: 0, // leave system setting
min: 123456,
expected: 0,
},
}
for i, tc := range testCases {
cfg := kubeproxyconfig.KubeProxyConntrackConfiguration{
Min: utilpointer.Int32Ptr(tc.min),
Max: utilpointer.Int32Ptr(tc.max),
MaxPerCore: utilpointer.Int32Ptr(tc.maxPerCore),
}
x, e := getConntrackMax(cfg)
if e != nil {
if tc.err == "" {
t.Errorf("[%d] unexpected error: %v", i, e)
} else if !strings.Contains(e.Error(), tc.err) {
t.Errorf("[%d] expected an error containing %q: %v", i, tc.err, e)
}
} else if x != tc.expected {
t.Errorf("[%d] expected %d, got %d", i, tc.expected, x)
}
}
}
// TestLoadConfig tests proper operation of loadConfig()
func TestLoadConfig(t *testing.T) {
yamlTemplate := `apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: %s
clientConnection:
acceptContentTypes: "abc"
burst: 100
contentType: content-type
kubeconfig: "/path/to/kubeconfig"
qps: 7
clusterCIDR: "%s"
configSyncPeriod: 15s
conntrack:
max: 4
maxPerCore: 2
min: 1
tcpCloseWaitTimeout: 10s
tcpEstablishedTimeout: 20s
featureGates: "all"
healthzBindAddress: "%s"
hostnameOverride: "foo"
iptables:
masqueradeAll: true
masqueradeBit: 17
minSyncPeriod: 10s
syncPeriod: 60s
ipvs:
minSyncPeriod: 10s
syncPeriod: 60s
kind: KubeProxyConfiguration
metricsBindAddress: "%s"
mode: "%s"
oomScoreAdj: 17
portRange: "2-7"
resourceContainer: /foo
udpIdleTimeout: 123ms
`
testCases := []struct {
name string
mode string
bindAddress string
clusterCIDR string
healthzBindAddress string
metricsBindAddress string
}{
{
name: "iptables mode, IPv4 all-zeros bind address",
mode: "iptables",
bindAddress: "0.0.0.0",
clusterCIDR: "1.2.3.0/24",
healthzBindAddress: "1.2.3.4:12345",
metricsBindAddress: "2.3.4.5:23456",
},
{
name: "iptables mode, non-zeros IPv4 config",
mode: "iptables",
bindAddress: "9.8.7.6",
clusterCIDR: "1.2.3.0/24",
healthzBindAddress: "1.2.3.4:12345",
metricsBindAddress: "2.3.4.5:23456",
},
{
// Test for 'bindAddress: "::"' (IPv6 all-zeros) in kube-proxy
// config file. The user will need to put quotes around '::' since
// 'bindAddress: ::' is invalid yaml syntax.
name: "iptables mode, IPv6 \"::\" bind address",
mode: "iptables",
bindAddress: "\"::\"",
clusterCIDR: "fd00:1::0/64",
healthzBindAddress: "[fd00:1::5]:12345",
metricsBindAddress: "[fd00:2::5]:23456",
},
{
// Test for 'bindAddress: "[::]"' (IPv6 all-zeros in brackets)
// in kube-proxy config file. The user will need to use
// surrounding quotes here since 'bindAddress: [::]' is invalid
// yaml syntax.
name: "iptables mode, IPv6 \"[::]\" bind address",
mode: "iptables",
bindAddress: "\"[::]\"",
clusterCIDR: "fd00:1::0/64",
healthzBindAddress: "[fd00:1::5]:12345",
metricsBindAddress: "[fd00:2::5]:23456",
},
{
// Test for 'bindAddress: ::0' (another form of IPv6 all-zeros).
// No surrounding quotes are required around '::0'.
name: "iptables mode, IPv6 ::0 bind address",
mode: "iptables",
bindAddress: "::0",
clusterCIDR: "fd00:1::0/64",
healthzBindAddress: "[fd00:1::5]:12345",
metricsBindAddress: "[fd00:2::5]:23456",
},
{
name: "ipvs mode, IPv6 config",
mode: "ipvs",
bindAddress: "2001:db8::1",
clusterCIDR: "fd00:1::0/64",
healthzBindAddress: "[fd00:1::5]:12345",
metricsBindAddress: "[fd00:2::5]:23456",
},
}
for _, tc := range testCases {
expBindAddr := tc.bindAddress
if tc.bindAddress[0] == '"' {
// Surrounding double quotes will get stripped by the yaml parser.
expBindAddr = expBindAddr[1 : len(tc.bindAddress)-1]
}
expected := &kubeproxyconfig.KubeProxyConfiguration{
BindAddress: expBindAddr,
ClientConnection: kubeproxyconfig.ClientConnectionConfiguration{
AcceptContentTypes: "abc",
Burst: 100,
ContentType: "content-type",
KubeConfigFile: "/path/to/kubeconfig",
QPS: 7,
},
ClusterCIDR: tc.clusterCIDR,
ConfigSyncPeriod: metav1.Duration{Duration: 15 * time.Second},
Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{
Max: utilpointer.Int32Ptr(4),
MaxPerCore: utilpointer.Int32Ptr(2),
Min: utilpointer.Int32Ptr(1),
TCPCloseWaitTimeout: &metav1.Duration{Duration: 10 * time.Second},
TCPEstablishedTimeout: &metav1.Duration{Duration: 20 * time.Second},
},
FeatureGates: "all",
HealthzBindAddress: tc.healthzBindAddress,
HostnameOverride: "foo",
IPTables: kubeproxyconfig.KubeProxyIPTablesConfiguration{
MasqueradeAll: true,
MasqueradeBit: utilpointer.Int32Ptr(17),
MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second},
SyncPeriod: metav1.Duration{Duration: 60 * time.Second},
},
IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{
MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second},
SyncPeriod: metav1.Duration{Duration: 60 * time.Second},
},
MetricsBindAddress: tc.metricsBindAddress,
Mode: kubeproxyconfig.ProxyMode(tc.mode),
OOMScoreAdj: utilpointer.Int32Ptr(17),
PortRange: "2-7",
ResourceContainer: "/foo",
UDPIdleTimeout: metav1.Duration{Duration: 123 * time.Millisecond},
}
options := NewOptions()
yaml := fmt.Sprintf(
yamlTemplate, tc.bindAddress, tc.clusterCIDR,
tc.healthzBindAddress, tc.metricsBindAddress, tc.mode)
config, err := options.loadConfig([]byte(yaml))
assert.NoError(t, err, "unexpected error for %s: %v", tc.name, err)
if !reflect.DeepEqual(expected, config) {
t.Fatalf("unexpected config for %s, diff = %s", tc.name, diff.ObjectDiff(config, expected))
}
}
}
// TestLoadConfigFailures tests failure modes for loadConfig()
func TestLoadConfigFailures(t *testing.T) {
testCases := []struct {
name string
config string
expErr string
}{
{
name: "Decode error test",
config: "Twas bryllyg, and ye slythy toves",
expErr: "could not find expected ':'",
},
{
name: "Bad config type test",
config: "kind: KubeSchedulerConfiguration",
expErr: "no kind",
},
{
name: "Missing quotes around :: bindAddress",
config: "bindAddress: ::",
expErr: "mapping values are not allowed in this context",
},
}
version := "apiVersion: kubeproxy.config.k8s.io/v1alpha1"
for _, tc := range testCases {
options := NewOptions()
config := fmt.Sprintf("%s\n%s", version, tc.config)
_, err := options.loadConfig([]byte(config))
if assert.Error(t, err, tc.name) {
assert.Contains(t, err.Error(), tc.expErr, tc.name)
}
}
}
Reference in New Issue View Git Blame Copy Permalink