github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #23711 from dcbw/kubenet-promiscuous 

Automatic merge from submit-queue

kubenet: set bridge promiscuous mode based on hairpinMode

Closes: https://github.com/kubernetes/kubernetes/issues/23657
Closes: https://github.com/kubernetes/kubernetes/issues/20475

<!-- Reviewable:start -->
---
This change is [<img src="http://reviewable.k8s.io/review_button.svg" height="35" align="absmiddle" alt="Reviewable"/>](http://reviewable.k8s.io/reviews/kubernetes/kubernetes/23711)
<!-- Reviewable:end -->
pull/6/head
k8s-merge-robot 2016-05-12 10:33:18 -07:00
parent 4ac4e0f6a2 b749902b42
commit fd3ef234de
13 changed files with 65 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pkg/kubelet/dockertools/docker_test.go
View File

@ -32,6 +32,7 @@ import (
cadvisorapi "github.com/google/cadvisor/info/v1"
"k8s.io/kubernetes/cmd/kubelet/app/options"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/apis/componentconfig"
"k8s.io/kubernetes/pkg/client/record"
"k8s.io/kubernetes/pkg/credentialprovider"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
@ -650,7 +651,7 @@ func TestFindContainersByPod(t *testing.T) {
},
}
fakeClient := NewFakeDockerClient()
np, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil))
np, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
// image back-off is set to nil, this test should not pull images
containerManager := NewFakeDockerManager(fakeClient, &record.FakeRecorder{}, nil, nil, &cadvisorapi.MachineInfo{}, options.GetDefaultPodInfraContainerImage(), 0, 0, "", containertest.FakeOS{}, np, nil, nil, nil)
for i, test := range tests {

3
pkg/kubelet/dockertools/manager_test.go
View File

@ -37,6 +37,7 @@ import (
"k8s.io/kubernetes/cmd/kubelet/app/options"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/testapi"
"k8s.io/kubernetes/pkg/apis/componentconfig"
"k8s.io/kubernetes/pkg/client/record"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
@ -105,7 +106,7 @@ func createTestDockerManager(fakeHTTPClient *fakeHTTP, fakeDocker *FakeDockerCli
}
fakeRecorder := &record.FakeRecorder{}
containerRefManager := kubecontainer.NewRefManager()
networkPlugin, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil))
networkPlugin, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
dockerManager := NewFakeDockerManager(
fakeDocker,
fakeRecorder,

25
pkg/kubelet/kubelet.go
View File

@ -356,7 +356,17 @@ func NewMainKubelet(
}
glog.Infof("Using node IP: %q", klet.nodeIP.String())
}
if plug, err := network.InitNetworkPlugin(networkPlugins, networkPluginName, &networkHost{klet}); err != nil {
if mode, err := effectiveHairpinMode(componentconfig.HairpinMode(hairpinMode), containerRuntime, configureCBR0, networkPluginName); err != nil {
// This is a non-recoverable error. Returning it up the callstack will just
// lead to retries of the same failure, so just fail hard.
glog.Fatalf("Invalid hairpin mode: %v", err)
} else {
klet.hairpinMode = mode
}
glog.Infof("Hairpin mode set to %q", klet.hairpinMode)
if plug, err := network.InitNetworkPlugin(networkPlugins, networkPluginName, &networkHost{klet}, klet.hairpinMode); err != nil {
return nil, err
} else {
klet.networkPlugin = plug
@ -375,15 +385,6 @@ func NewMainKubelet(
klet.podCache = kubecontainer.NewCache()
klet.podManager = kubepod.NewBasicPodManager(kubepod.NewBasicMirrorClient(klet.kubeClient))
if mode, err := effectiveHairpinMode(componentconfig.HairpinMode(hairpinMode), containerRuntime, configureCBR0); err != nil {
// This is a non-recoverable error. Returning it up the callstack will just
// lead to retries of the same failure, so just fail hard.
glog.Fatalf("Invalid hairpin mode: %v", err)
} else {
klet.hairpinMode = mode
}
glog.Infof("Hairpin mode set to %q", klet.hairpinMode)
// Initialize the runtime.
switch containerRuntime {
case "docker":
@ -499,7 +500,7 @@ func NewMainKubelet(
// effectiveHairpinMode determines the effective hairpin mode given the
// configured mode, container runtime, and whether cbr0 should be configured.
func effectiveHairpinMode(hairpinMode componentconfig.HairpinMode, containerRuntime string, configureCBR0 bool) (componentconfig.HairpinMode, error) {
func effectiveHairpinMode(hairpinMode componentconfig.HairpinMode, containerRuntime string, configureCBR0 bool, networkPlugin string) (componentconfig.HairpinMode, error) {
// The hairpin mode setting doesn't matter if:
// - We're not using a bridge network. This is hard to check because we might
// be using a plugin. It matters if --configure-cbr0=true, and we currently
@ -514,7 +515,7 @@ func effectiveHairpinMode(hairpinMode componentconfig.HairpinMode, containerRunt
glog.Warningf("Hairpin mode set to %q but container runtime is %q, ignoring", hairpinMode, containerRuntime)
return componentconfig.HairpinNone, nil
}
if hairpinMode == componentconfig.PromiscuousBridge && !configureCBR0 {
if hairpinMode == componentconfig.PromiscuousBridge && !configureCBR0 && networkPlugin != "kubenet" {
// This is not a valid combination. Users might be using the
// default values (from before the hairpin-mode flag existed) and we
// should keep the old behavior.

3
pkg/kubelet/kubelet_test.go
View File

@ -37,6 +37,7 @@ import (
"k8s.io/kubernetes/pkg/api/resource"
"k8s.io/kubernetes/pkg/api/testapi"
"k8s.io/kubernetes/pkg/api/unversioned"
"k8s.io/kubernetes/pkg/apis/componentconfig"
"k8s.io/kubernetes/pkg/capabilities"
"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake"
"k8s.io/kubernetes/pkg/client/record"
@ -128,7 +129,7 @@ func newTestKubelet(t *testing.T) *TestKubelet {
kubelet.nodeName = testKubeletHostname
kubelet.runtimeState = newRuntimeState(maxWaitForContainerRuntime)
kubelet.runtimeState.setNetworkState(nil)
kubelet.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil))
kubelet.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
if tempDir, err := ioutil.TempDir("/tmp", "kubelet_test."); err != nil {
t.Fatalf("can't make a temp rootdir: %v", err)
} else {

3
pkg/kubelet/network/cni/cni.go
View File

@ -25,6 +25,7 @@ import (
"github.com/appc/cni/libcni"
cnitypes "github.com/appc/cni/pkg/types"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/apis/componentconfig"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/dockertools"
"k8s.io/kubernetes/pkg/kubelet/network"
@ -93,7 +94,7 @@ func getDefaultCNINetwork(pluginDir, vendorCNIDirPrefix string) (*cniNetwork, er
return nil, fmt.Errorf("No valid networks found in %s", pluginDir)
}
func (plugin *cniNetworkPlugin) Init(host network.Host) error {
func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode) error {
plugin.host = host
return nil
}

5
pkg/kubelet/network/cni/cni_test.go
View File

@ -34,6 +34,7 @@ import (
"k8s.io/kubernetes/cmd/kubelet/app/options"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/apis/componentconfig"
"k8s.io/kubernetes/pkg/client/record"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
@ -144,7 +145,7 @@ func newTestDockerManager() (*dockertools.DockerManager, *dockertools.FakeDocker
fakeDocker := dockertools.NewFakeDockerClient()
fakeRecorder := &record.FakeRecorder{}
containerRefManager := kubecontainer.NewRefManager()
networkPlugin, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil))
networkPlugin, _ := network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
dockerManager := dockertools.NewFakeDockerManager(
fakeDocker,
fakeRecorder,
@ -174,7 +175,7 @@ func TestCNIPlugin(t *testing.T) {
installPluginUnderTest(t, testVendorCNIDirPrefix, testNetworkConfigPath, vendorName, pluginName)
np := probeNetworkPluginsWithVendorCNIDirPrefix(path.Join(testNetworkConfigPath, pluginName), testVendorCNIDirPrefix)
plug, err := network.InitNetworkPlugin(np, "cni", NewFakeHost(nil))
plug, err := network.InitNetworkPlugin(np, "cni", NewFakeHost(nil), componentconfig.HairpinNone)
if err != nil {
t.Fatalf("Failed to select the desired plugin: %v", err)
}

3
pkg/kubelet/network/exec/exec.go
View File

@ -66,6 +66,7 @@ import (
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/api/unversioned"
"k8s.io/kubernetes/pkg/apis/componentconfig"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/network"
utilexec "k8s.io/kubernetes/pkg/util/exec"
@ -104,7 +105,7 @@ func ProbeNetworkPlugins(pluginDir string) []network.NetworkPlugin {
return execPlugins
}
func (plugin *execNetworkPlugin) Init(host network.Host) error {
func (plugin *execNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode) error {
err := plugin.validate()
if err != nil {
return err

17
pkg/kubelet/network/exec/exec_test.go
View File

@ -29,6 +29,7 @@ import (
"testing"
"text/template"
"k8s.io/kubernetes/pkg/apis/componentconfig"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/network"
nettest "k8s.io/kubernetes/pkg/kubelet/network/testing"
@ -134,7 +135,7 @@ func TestSelectPlugin(t *testing.T) {
installPluginUnderTest(t, "", testPluginPath, pluginName, nil)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil))
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
if err != nil {
t.Errorf("Failed to select the desired plugin: %v", err)
}
@ -156,7 +157,7 @@ func TestSelectVendoredPlugin(t *testing.T) {
installPluginUnderTest(t, vendor, testPluginPath, pluginName, nil)
vendoredPluginName := fmt.Sprintf("%s/%s", vendor, pluginName)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), vendoredPluginName, nettest.NewFakeHost(nil))
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), vendoredPluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
if err != nil {
t.Errorf("Failed to select the desired plugin: %v", err)
}
@ -177,7 +178,7 @@ func TestSelectWrongPlugin(t *testing.T) {
installPluginUnderTest(t, "", testPluginPath, pluginName, nil)
wrongPlugin := "abcd"
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), wrongPlugin, nettest.NewFakeHost(nil))
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), wrongPlugin, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
if plug != nil || err == nil {
t.Errorf("Expected to see an error. Wrong plugin selected.")
}
@ -205,7 +206,7 @@ func TestPluginValidation(t *testing.T) {
}
f.Close()
_, err = network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil))
_, err = network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
if err == nil {
// we expected an error here because validation would have failed
t.Errorf("Expected non-nil value.")
@ -223,7 +224,7 @@ func TestPluginSetupHook(t *testing.T) {
installPluginUnderTest(t, "", testPluginPath, pluginName, nil)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil))
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
err = plug.SetUpPod("podNamespace", "podName", kubecontainer.ContainerID{Type: "docker", ID: "dockerid2345"})
if err != nil {
@ -251,7 +252,7 @@ func TestPluginTearDownHook(t *testing.T) {
installPluginUnderTest(t, "", testPluginPath, pluginName, nil)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil))
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
err = plug.TearDownPod("podNamespace", "podName", kubecontainer.ContainerID{Type: "docker", ID: "dockerid2345"})
if err != nil {
@ -279,7 +280,7 @@ func TestPluginStatusHook(t *testing.T) {
installPluginUnderTest(t, "", testPluginPath, pluginName, nil)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil))
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
ip, err := plug.GetPodNetworkStatus("namespace", "name", kubecontainer.ContainerID{Type: "docker", ID: "dockerid2345"})
if err != nil {
@ -315,7 +316,7 @@ func TestPluginStatusHookIPv6(t *testing.T) {
}
installPluginUnderTest(t, "", testPluginPath, pluginName, execTemplate)
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil))
plug, err := network.InitNetworkPlugin(ProbeNetworkPlugins(testPluginPath), pluginName, nettest.NewFakeHost(nil), componentconfig.HairpinNone)
if err != nil {
t.Errorf("InitNetworkPlugin() failed: %v", err)
}

20
pkg/kubelet/network/kubenet/kubenet_linux.go
View File

@ -22,6 +22,7 @@ import (
"fmt"
"net"
"strings"
"sync"
"syscall"
"time"
@ -31,6 +32,7 @@ import (
"github.com/appc/cni/libcni"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/apis/componentconfig"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/dockertools"
"k8s.io/kubernetes/pkg/kubelet/network"
@ -38,7 +40,6 @@ import (
utilexec "k8s.io/kubernetes/pkg/util/exec"
utilsets "k8s.io/kubernetes/pkg/util/sets"
utilsysctl "k8s.io/kubernetes/pkg/util/sysctl"
"strings"
)
const (
@ -61,6 +62,7 @@ type kubenetNetworkPlugin struct {
mu sync.Mutex //Mutex for protecting podCIDRs map and netConfig
execer utilexec.Interface
nsenterPath string
hairpinMode componentconfig.HairpinMode
}
func NewPlugin() network.NetworkPlugin {
@ -71,8 +73,9 @@ func NewPlugin() network.NetworkPlugin {
}
}
func (plugin *kubenetNetworkPlugin) Init(host network.Host) error {
func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode) error {
plugin.host = host
plugin.hairpinMode = hairpinMode
plugin.cniConfig = &libcni.CNIConfig{
Path: []string{DefaultCNIDir},
}
@ -281,6 +284,19 @@ func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id k
return err
}
// Put the container bridge into promiscuous mode to force it to accept hairpin packets.
// TODO: Remove this once the kernel bug (#20096) is fixed.
// TODO: check and set promiscuous mode with netlink once vishvananda/netlink supports it
if plugin.hairpinMode == componentconfig.PromiscuousBridge {
output, err := plugin.execer.Command("ip", "link", "show", "dev", BridgeName).CombinedOutput()
if err != nil || strings.Index(string(output), "PROMISC") < 0 {
_, err := plugin.execer.Command("ip", "link", "set", BridgeName, "promisc", "on").CombinedOutput()
if err != nil {
return fmt.Errorf("Error setting promiscuous mode on %s: %v", BridgeName, err)
}
}
}
// The first SetUpPod call creates the bridge; ensure shaping is enabled
if plugin.shaper == nil {
plugin.shaper = bandwidth.NewTCShaper(BridgeName)

3
pkg/kubelet/network/kubenet/kubenet_unsupported.go
View File

@ -21,6 +21,7 @@ package kubenet
import (
"fmt"
"k8s.io/kubernetes/pkg/apis/componentconfig"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/network"
)
@ -33,7 +34,7 @@ func NewPlugin() network.NetworkPlugin {
return &kubenetNetworkPlugin{}
}
func (plugin *kubenetNetworkPlugin) Init(host network.Host) error {
func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode componentconfig.HairpinMode) error {
return fmt.Errorf("Kubenet is not supported in this build")
}

11
pkg/kubelet/network/plugins.go
View File

@ -26,6 +26,7 @@ import (
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/unversioned"
"k8s.io/kubernetes/pkg/apis/componentconfig"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
utilerrors "k8s.io/kubernetes/pkg/util/errors"
utilexec "k8s.io/kubernetes/pkg/util/exec"
@ -51,7 +52,7 @@ const (
type NetworkPlugin interface {
// Init initializes the plugin. This will be called exactly once
// before any other methods are called.
Init(host Host) error
Init(host Host, hairpinMode componentconfig.HairpinMode) error
// Called on various events like:
// NET_PLUGIN_EVENT_POD_CIDR_CHANGE
@ -104,11 +105,11 @@ type Host interface {
}
// InitNetworkPlugin inits the plugin that matches networkPluginName. Plugins must have unique names.
func InitNetworkPlugin(plugins []NetworkPlugin, networkPluginName string, host Host) (NetworkPlugin, error) {
func InitNetworkPlugin(plugins []NetworkPlugin, networkPluginName string, host Host, hairpinMode componentconfig.HairpinMode) (NetworkPlugin, error) {
if networkPluginName == "" {
// default to the no_op plugin
plug := &NoopNetworkPlugin{}
if err := plug.Init(host); err != nil {
if err := plug.Init(host, hairpinMode); err != nil {
return nil, err
}
return plug, nil
@ -133,7 +134,7 @@ func InitNetworkPlugin(plugins []NetworkPlugin, networkPluginName string, host H
chosenPlugin := pluginMap[networkPluginName]
if chosenPlugin != nil {
err := chosenPlugin.Init(host)
err := chosenPlugin.Init(host, hairpinMode)
if err != nil {
allErrs = append(allErrs, fmt.Errorf("Network plugin %q failed init: %v", networkPluginName, err))
} else {
@ -155,7 +156,7 @@ type NoopNetworkPlugin struct {
const sysctlBridgeCallIptables = "net/bridge/bridge-nf-call-iptables"
func (plugin *NoopNetworkPlugin) Init(host Host) error {
func (plugin *NoopNetworkPlugin) Init(host Host, hairpinMode componentconfig.HairpinMode) error {
// Set bridge-nf-call-iptables=1 to maintain compatibility with older
// kubernetes versions to ensure the iptables-based kube proxy functions
// correctly. Other plugins are responsible for setting this correctly

3
pkg/kubelet/network/plugins_test.go
View File

@ -19,12 +19,13 @@ package network
import (
"testing"
"k8s.io/kubernetes/pkg/apis/componentconfig"
nettest "k8s.io/kubernetes/pkg/kubelet/network/testing"
)
func TestSelectDefaultPlugin(t *testing.T) {
all_plugins := []NetworkPlugin{}
plug, err := InitNetworkPlugin(all_plugins, "", nettest.NewFakeHost(nil))
plug, err := InitNetworkPlugin(all_plugins, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
if err != nil {
t.Fatalf("Unexpected error in selecting default plugin: %v", err)
}

3
pkg/kubelet/runonce_test.go
View File

@ -24,6 +24,7 @@ import (
cadvisorapi "github.com/google/cadvisor/info/v1"
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/apis/componentconfig"
"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake"
"k8s.io/kubernetes/pkg/client/record"
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
@ -78,7 +79,7 @@ func TestRunOnce(t *testing.T) {
}
kb.containerManager = cm.NewStubContainerManager()
kb.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil))
kb.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
if err := kb.setupDataDirs(); err != nil {
t.Errorf("Failed to init data dirs: %v", err)
}