From 8778e50083b02c3ba0286221f9c8097593266bee Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 23 Jan 2018 14:04:38 -0600 Subject: [PATCH] kubelet: make --cni-bin-dir accept a comma-separated list of CNI plugin directories Allow CNI-related network plugin drivers (kubenet, cni) to search a list of directories for plugin binaries instead of just one. This allows using an administrator-provided path and fallbacks to others (like the previous default of /opt/cni/bin) for backwards compatibility. --- cmd/kubelet/app/plugins.go | 6 +++--- cmd/kubelet/app/server.go | 5 +++-- pkg/kubelet/BUILD | 1 + pkg/kubelet/config/flags.go | 2 +- pkg/kubelet/dockershim/docker_service.go | 12 ++++++------ pkg/kubelet/kubelet.go | 3 ++- pkg/kubelet/network/cni/cni.go | 5 +++++ pkg/kubelet/network/kubenet/kubenet_linux.go | 2 +- .../network/kubenet/kubenet_unsupported.go | 2 +- pkg/kubelet/rkt/rkt_test.go | 15 ++++++++------- 10 files changed, 31 insertions(+), 22 deletions(-) diff --git a/cmd/kubelet/app/plugins.go b/cmd/kubelet/app/plugins.go index 6fb144f1b6..cbb20ba93a 100644 --- a/cmd/kubelet/app/plugins.go +++ b/cmd/kubelet/app/plugins.go @@ -114,12 +114,12 @@ func GetDynamicPluginProber(pluginDir string) volume.DynamicPluginProber { } // ProbeNetworkPlugins collects all compiled-in plugins -func ProbeNetworkPlugins(cniConfDir, cniBinDir string) []network.NetworkPlugin { +func ProbeNetworkPlugins(cniConfDir string, cniBinDirs []string) []network.NetworkPlugin { allPlugins := []network.NetworkPlugin{} // for each existing plugin, add to the list - allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, []string{cniBinDir})...) - allPlugins = append(allPlugins, kubenet.NewPlugin([]string{cniBinDir})) + allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, cniBinDirs)...) + allPlugins = append(allPlugins, kubenet.NewPlugin(cniBinDirs)) return allPlugins } diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index dce5c4f213..501bda8d86 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -78,6 +78,7 @@ import ( evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" dynamickubeletconfig "k8s.io/kubernetes/pkg/kubelet/kubeletconfig" "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/configfiles" + "k8s.io/kubernetes/pkg/kubelet/network/cni" "k8s.io/kubernetes/pkg/kubelet/server" "k8s.io/kubernetes/pkg/kubelet/server/streaming" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" @@ -355,7 +356,7 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err ExternalKubeClient: nil, EventClient: nil, Mounter: mounter, - NetworkPlugins: ProbeNetworkPlugins(s.CNIConfDir, s.CNIBinDir), + NetworkPlugins: ProbeNetworkPlugins(s.CNIConfDir, cni.SplitDirs(s.CNIBinDir)), OOMAdjuster: oom.NewOOMAdjuster(), OSInterface: kubecontainer.RealOS{}, Writer: writer, @@ -1096,7 +1097,7 @@ func RunDockershim(f *options.KubeletFlags, c *kubeletconfiginternal.KubeletConf NonMasqueradeCIDR: f.NonMasqueradeCIDR, PluginName: r.NetworkPluginName, PluginConfDir: r.CNIConfDir, - PluginBinDir: r.CNIBinDir, + PluginBinDirs: cni.SplitDirs(r.CNIBinDir), MTU: int(r.NetworkPluginMTU), LegacyRuntimeHost: nh, } diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index f0230fbb11..6a8b302800 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -66,6 +66,7 @@ go_library( "//pkg/kubelet/metrics/collectors:go_default_library", "//pkg/kubelet/mountpod:go_default_library", "//pkg/kubelet/network:go_default_library", + "//pkg/kubelet/network/cni:go_default_library", "//pkg/kubelet/network/dns:go_default_library", "//pkg/kubelet/pleg:go_default_library", "//pkg/kubelet/pod:go_default_library", diff --git a/pkg/kubelet/config/flags.go b/pkg/kubelet/config/flags.go index f9bde6a6f7..9e1c94d59c 100644 --- a/pkg/kubelet/config/flags.go +++ b/pkg/kubelet/config/flags.go @@ -98,7 +98,7 @@ func (s *ContainerRuntimeOptions) AddFlags(fs *pflag.FlagSet) { // Network plugin settings. Shared by both docker and rkt. fs.StringVar(&s.NetworkPluginName, "network-plugin", s.NetworkPluginName, " The name of the network plugin to be invoked for various events in kubelet/pod lifecycle") fs.StringVar(&s.CNIConfDir, "cni-conf-dir", s.CNIConfDir, " The full path of the directory in which to search for CNI config files. Default: /etc/cni/net.d") - fs.StringVar(&s.CNIBinDir, "cni-bin-dir", s.CNIBinDir, " The full path of the directory in which to search for CNI plugin binaries. Default: /opt/cni/bin") + fs.StringVar(&s.CNIBinDir, "cni-bin-dir", s.CNIBinDir, " A comma-separated list of full paths of directories in which to search for CNI plugin binaries. Default: /opt/cni/bin") fs.Int32Var(&s.NetworkPluginMTU, "network-plugin-mtu", s.NetworkPluginMTU, " The MTU to be passed to the network plugin, to override the default. Set to 0 to use the default 1460 MTU.") // Rkt-specific settings. diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index bdf1dfc78b..82828d0df6 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -110,10 +110,10 @@ type NetworkPluginSettings struct { NonMasqueradeCIDR string // PluginName is the name of the plugin, runtime shim probes for PluginName string - // PluginBinDir is the directory in which the binaries for the plugin with - // PluginName is kept. The admin is responsible for provisioning these - // binaries before-hand. - PluginBinDir string + // PluginBinDirs is an array of directories in which the binaries for + // the plugin with PluginName may be found. The admin is responsible for + // provisioning these binaries before-hand. + PluginBinDirs []string // PluginConfDir is the directory in which the admin places a CNI conf. // Depending on the plugin, this may be an optional field, eg: kubenet // generates its own plugin conf. @@ -229,8 +229,8 @@ func NewDockerService(config *ClientConfig, podSandboxImage string, streamingCon } } // dockershim currently only supports CNI plugins. - cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, []string{pluginSettings.PluginBinDir}) - cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDir)) + cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginBinDirs) + cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDirs)) netHost := &dockerNetworkHost{ pluginSettings.LegacyRuntimeHost, &namespaceGetter{ds}, diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 5a58f20846..a45ef72f67 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -79,6 +79,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/pkg/kubelet/metrics/collectors" "k8s.io/kubernetes/pkg/kubelet/network" + "k8s.io/kubernetes/pkg/kubelet/network/cni" "k8s.io/kubernetes/pkg/kubelet/network/dns" "k8s.io/kubernetes/pkg/kubelet/pleg" kubepod "k8s.io/kubernetes/pkg/kubelet/pod" @@ -587,7 +588,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, NonMasqueradeCIDR: nonMasqueradeCIDR, PluginName: crOptions.NetworkPluginName, PluginConfDir: crOptions.CNIConfDir, - PluginBinDir: crOptions.CNIBinDir, + PluginBinDirs: cni.SplitDirs(crOptions.CNIBinDir), MTU: int(crOptions.NetworkPluginMTU), } diff --git a/pkg/kubelet/network/cni/cni.go b/pkg/kubelet/network/cni/cni.go index 0f6741442b..692e218dee 100644 --- a/pkg/kubelet/network/cni/cni.go +++ b/pkg/kubelet/network/cni/cni.go @@ -68,6 +68,11 @@ type cniPortMapping struct { HostIP string `json:"hostIP"` } +func SplitDirs(dirs string) []string { + // Use comma rather than colon to work better with Windows too + return strings.Split(dirs, ",") +} + func ProbeNetworkPlugins(confDir string, binDirs []string) []network.NetworkPlugin { old := binDirs binDirs = make([]string, len(binDirs)) diff --git a/pkg/kubelet/network/kubenet/kubenet_linux.go b/pkg/kubelet/network/kubenet/kubenet_linux.go index eff23ea219..8c8904034a 100644 --- a/pkg/kubelet/network/kubenet/kubenet_linux.go +++ b/pkg/kubelet/network/kubenet/kubenet_linux.go @@ -563,7 +563,7 @@ func (plugin *kubenetNetworkPlugin) Status() error { } if !plugin.checkRequiredCNIPlugins() { - return fmt.Errorf("could not locate kubenet required CNI plugins %v at %q or %q", requiredCNIPlugins, plugin.binDirs) + return fmt.Errorf("could not locate kubenet required CNI plugins %v at %q", requiredCNIPlugins, plugin.binDirs) } return nil } diff --git a/pkg/kubelet/network/kubenet/kubenet_unsupported.go b/pkg/kubelet/network/kubenet/kubenet_unsupported.go index 1cbc7ab725..ba2ee3f531 100644 --- a/pkg/kubelet/network/kubenet/kubenet_unsupported.go +++ b/pkg/kubelet/network/kubenet/kubenet_unsupported.go @@ -30,7 +30,7 @@ type kubenetNetworkPlugin struct { network.NoopNetworkPlugin } -func NewPlugin(networkPluginDir string) network.NetworkPlugin { +func NewPlugin(networkPluginDirs []string) network.NetworkPlugin { return &kubenetNetworkPlugin{} } diff --git a/pkg/kubelet/rkt/rkt_test.go b/pkg/kubelet/rkt/rkt_test.go index d235fe289c..8e5f6cdd81 100644 --- a/pkg/kubelet/rkt/rkt_test.go +++ b/pkg/kubelet/rkt/rkt_test.go @@ -1215,6 +1215,7 @@ func TestGenerateRunCommand(t *testing.T) { hostName := "test-hostname" boolTrue := true boolFalse := false + pluginDirs := []string{"/tmp"} tests := []struct { networkPlugin network.NetworkPlugin @@ -1231,7 +1232,7 @@ func TestGenerateRunCommand(t *testing.T) { }{ // Case #0, returns error. { - kubenet.NewPlugin("/tmp"), + kubenet.NewPlugin(pluginDirs), &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name-foo", @@ -1250,7 +1251,7 @@ func TestGenerateRunCommand(t *testing.T) { }, // Case #1, returns no dns, with private-net. { - kubenet.NewPlugin("/tmp"), + kubenet.NewPlugin(pluginDirs), &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name-foo", @@ -1269,7 +1270,7 @@ func TestGenerateRunCommand(t *testing.T) { }, // Case #2, returns no dns, with host-net. { - kubenet.NewPlugin("/tmp"), + kubenet.NewPlugin(pluginDirs), &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name-foo", @@ -1290,7 +1291,7 @@ func TestGenerateRunCommand(t *testing.T) { }, // Case #3, returns dns, dns searches, with private-net. { - kubenet.NewPlugin("/tmp"), + kubenet.NewPlugin(pluginDirs), &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name-foo", @@ -1311,7 +1312,7 @@ func TestGenerateRunCommand(t *testing.T) { }, // Case #4, returns no dns, dns searches, with host-network. { - kubenet.NewPlugin("/tmp"), + kubenet.NewPlugin(pluginDirs), &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name-foo", @@ -1351,7 +1352,7 @@ func TestGenerateRunCommand(t *testing.T) { }, // Case #6, if all containers are privileged, the result should have 'insecure-options=all-run' { - kubenet.NewPlugin("/tmp"), + kubenet.NewPlugin(pluginDirs), &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name-foo", @@ -1373,7 +1374,7 @@ func TestGenerateRunCommand(t *testing.T) { }, // Case #7, if not all containers are privileged, the result should not have 'insecure-options=all-run' { - kubenet.NewPlugin("/tmp"), + kubenet.NewPlugin(pluginDirs), &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name-foo",