kubelet: make --cni-bin-dir accept a comma-separated list of CNI plugin directories

Allow CNI-related network plugin drivers (kubenet, cni) to search a list of
directories for plugin binaries instead of just one.  This allows using an
administrator-provided path and fallbacks to others (like the previous default
of /opt/cni/bin) for backwards compatibility.

cmd/kubelet/app/plugins.go

@@ -114,12 +114,12 @@ func GetDynamicPluginProber(pluginDir string) volume.DynamicPluginProber {
 }
 
 // ProbeNetworkPlugins collects all compiled-in plugins
-func ProbeNetworkPlugins(cniConfDir, cniBinDir string) []network.NetworkPlugin {
+func ProbeNetworkPlugins(cniConfDir string, cniBinDirs []string) []network.NetworkPlugin {
 	allPlugins := []network.NetworkPlugin{}
 
 	// for each existing plugin, add to the list
-	allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, []string{cniBinDir})...)
+	allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, cniBinDirs)...)
-	allPlugins = append(allPlugins, kubenet.NewPlugin([]string{cniBinDir}))
+	allPlugins = append(allPlugins, kubenet.NewPlugin(cniBinDirs))
 
 	return allPlugins
 }

cmd/kubelet/app/server.go

@@ -78,6 +78,7 @@ import (
 	evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api"
 	dynamickubeletconfig "k8s.io/kubernetes/pkg/kubelet/kubeletconfig"
 	"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/configfiles"
+	"k8s.io/kubernetes/pkg/kubelet/network/cni"
 	"k8s.io/kubernetes/pkg/kubelet/server"
 	"k8s.io/kubernetes/pkg/kubelet/server/streaming"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
@@ -355,7 +356,7 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err
 		ExternalKubeClient:  nil,
 		EventClient:         nil,
 		Mounter:             mounter,
-		NetworkPlugins:      ProbeNetworkPlugins(s.CNIConfDir, s.CNIBinDir),
+		NetworkPlugins:      ProbeNetworkPlugins(s.CNIConfDir, cni.SplitDirs(s.CNIBinDir)),
 		OOMAdjuster:         oom.NewOOMAdjuster(),
 		OSInterface:         kubecontainer.RealOS{},
 		Writer:              writer,
@@ -1096,7 +1097,7 @@ func RunDockershim(f *options.KubeletFlags, c *kubeletconfiginternal.KubeletConf
 		NonMasqueradeCIDR: f.NonMasqueradeCIDR,
 		PluginName:        r.NetworkPluginName,
 		PluginConfDir:     r.CNIConfDir,
-		PluginBinDir:      r.CNIBinDir,
+		PluginBinDirs:     cni.SplitDirs(r.CNIBinDir),
 		MTU:               int(r.NetworkPluginMTU),
 		LegacyRuntimeHost: nh,
 	}

pkg/kubelet/BUILD

@@ -66,6 +66,7 @@ go_library(
         "//pkg/kubelet/metrics/collectors:go_default_library",
         "//pkg/kubelet/mountpod:go_default_library",
         "//pkg/kubelet/network:go_default_library",
+        "//pkg/kubelet/network/cni:go_default_library",
         "//pkg/kubelet/network/dns:go_default_library",
         "//pkg/kubelet/pleg:go_default_library",
         "//pkg/kubelet/pod:go_default_library",

pkg/kubelet/config/flags.go

@@ -98,7 +98,7 @@ func (s *ContainerRuntimeOptions) AddFlags(fs *pflag.FlagSet) {
 	// Network plugin settings. Shared by both docker and rkt.
 	fs.StringVar(&s.NetworkPluginName, "network-plugin", s.NetworkPluginName, "<Warning: Alpha feature> The name of the network plugin to be invoked for various events in kubelet/pod lifecycle")
 	fs.StringVar(&s.CNIConfDir, "cni-conf-dir", s.CNIConfDir, "<Warning: Alpha feature> The full path of the directory in which to search for CNI config files. Default: /etc/cni/net.d")
-	fs.StringVar(&s.CNIBinDir, "cni-bin-dir", s.CNIBinDir, "<Warning: Alpha feature> The full path of the directory in which to search for CNI plugin binaries. Default: /opt/cni/bin")
+	fs.StringVar(&s.CNIBinDir, "cni-bin-dir", s.CNIBinDir, "<Warning: Alpha feature> A comma-separated list of full paths of directories in which to search for CNI plugin binaries. Default: /opt/cni/bin")
 	fs.Int32Var(&s.NetworkPluginMTU, "network-plugin-mtu", s.NetworkPluginMTU, "<Warning: Alpha feature> The MTU to be passed to the network plugin, to override the default. Set to 0 to use the default 1460 MTU.")
 
 	// Rkt-specific settings.

pkg/kubelet/dockershim/docker_service.go

@@ -110,10 +110,10 @@ type NetworkPluginSettings struct {
 	NonMasqueradeCIDR string
 	// PluginName is the name of the plugin, runtime shim probes for
 	PluginName string
-	// PluginBinDir is the directory in which the binaries for the plugin with
+	// PluginBinDirs is an array of directories in which the binaries for
-	// PluginName is kept. The admin is responsible for provisioning these
+	// the plugin with PluginName may be found. The admin is responsible for
-	// binaries before-hand.
+	// provisioning these binaries before-hand.
-	PluginBinDir string
+	PluginBinDirs []string
 	// PluginConfDir is the directory in which the admin places a CNI conf.
 	// Depending on the plugin, this may be an optional field, eg: kubenet
 	// generates its own plugin conf.
@@ -229,8 +229,8 @@ func NewDockerService(config *ClientConfig, podSandboxImage string, streamingCon
 		}
 	}
 	// dockershim currently only supports CNI plugins.
-	cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, []string{pluginSettings.PluginBinDir})
+	cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginBinDirs)
-	cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDir))
+	cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDirs))
 	netHost := &dockerNetworkHost{
 		pluginSettings.LegacyRuntimeHost,
 		&namespaceGetter{ds},

pkg/kubelet/kubelet.go

@@ -79,6 +79,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/metrics"
 	"k8s.io/kubernetes/pkg/kubelet/metrics/collectors"
 	"k8s.io/kubernetes/pkg/kubelet/network"
+	"k8s.io/kubernetes/pkg/kubelet/network/cni"
 	"k8s.io/kubernetes/pkg/kubelet/network/dns"
 	"k8s.io/kubernetes/pkg/kubelet/pleg"
 	kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
@@ -587,7 +588,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 		NonMasqueradeCIDR: nonMasqueradeCIDR,
 		PluginName:        crOptions.NetworkPluginName,
 		PluginConfDir:     crOptions.CNIConfDir,
-		PluginBinDir:      crOptions.CNIBinDir,
+		PluginBinDirs:     cni.SplitDirs(crOptions.CNIBinDir),
 		MTU:               int(crOptions.NetworkPluginMTU),
 	}
 

pkg/kubelet/network/cni/cni.go

@@ -68,6 +68,11 @@ type cniPortMapping struct {
 	HostIP        string `json:"hostIP"`
 }
 
+func SplitDirs(dirs string) []string {
+	// Use comma rather than colon to work better with Windows too
+	return strings.Split(dirs, ",")
+}
+
 func ProbeNetworkPlugins(confDir string, binDirs []string) []network.NetworkPlugin {
 	old := binDirs
 	binDirs = make([]string, len(binDirs))

pkg/kubelet/network/kubenet/kubenet_linux.go

@@ -563,7 +563,7 @@ func (plugin *kubenetNetworkPlugin) Status() error {
 	}
 
 	if !plugin.checkRequiredCNIPlugins() {
-		return fmt.Errorf("could not locate kubenet required CNI plugins %v at %q or %q", requiredCNIPlugins, plugin.binDirs)
+		return fmt.Errorf("could not locate kubenet required CNI plugins %v at %q", requiredCNIPlugins, plugin.binDirs)
 	}
 	return nil
 }

pkg/kubelet/network/kubenet/kubenet_unsupported.go

@@ -30,7 +30,7 @@ type kubenetNetworkPlugin struct {
 	network.NoopNetworkPlugin
 }
 
-func NewPlugin(networkPluginDir string) network.NetworkPlugin {
+func NewPlugin(networkPluginDirs []string) network.NetworkPlugin {
 	return &kubenetNetworkPlugin{}
 }
 

pkg/kubelet/rkt/rkt_test.go

@@ -1215,6 +1215,7 @@ func TestGenerateRunCommand(t *testing.T) {
 	hostName := "test-hostname"
 	boolTrue := true
 	boolFalse := false
+	pluginDirs := []string{"/tmp"}
 
 	tests := []struct {
 		networkPlugin network.NetworkPlugin
@@ -1231,7 +1232,7 @@ func TestGenerateRunCommand(t *testing.T) {
 	}{
 		// Case #0, returns error.
 		{
-			kubenet.NewPlugin("/tmp"),
+			kubenet.NewPlugin(pluginDirs),
 			&v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "pod-name-foo",
@@ -1250,7 +1251,7 @@ func TestGenerateRunCommand(t *testing.T) {
 		},
 		// Case #1, returns no dns, with private-net.
 		{
-			kubenet.NewPlugin("/tmp"),
+			kubenet.NewPlugin(pluginDirs),
 			&v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "pod-name-foo",
@@ -1269,7 +1270,7 @@ func TestGenerateRunCommand(t *testing.T) {
 		},
 		// Case #2, returns no dns, with host-net.
 		{
-			kubenet.NewPlugin("/tmp"),
+			kubenet.NewPlugin(pluginDirs),
 			&v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "pod-name-foo",
@@ -1290,7 +1291,7 @@ func TestGenerateRunCommand(t *testing.T) {
 		},
 		// Case #3, returns dns, dns searches, with private-net.
 		{
-			kubenet.NewPlugin("/tmp"),
+			kubenet.NewPlugin(pluginDirs),
 			&v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "pod-name-foo",
@@ -1311,7 +1312,7 @@ func TestGenerateRunCommand(t *testing.T) {
 		},
 		// Case #4, returns no dns, dns searches, with host-network.
 		{
-			kubenet.NewPlugin("/tmp"),
+			kubenet.NewPlugin(pluginDirs),
 			&v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "pod-name-foo",
@@ -1351,7 +1352,7 @@ func TestGenerateRunCommand(t *testing.T) {
 		},
 		// Case #6, if all containers are privileged, the result should have 'insecure-options=all-run'
 		{
-			kubenet.NewPlugin("/tmp"),
+			kubenet.NewPlugin(pluginDirs),
 			&v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "pod-name-foo",
@@ -1373,7 +1374,7 @@ func TestGenerateRunCommand(t *testing.T) {
 		},
 		// Case #7, if not all containers are privileged, the result should not have 'insecure-options=all-run'
 		{
-			kubenet.NewPlugin("/tmp"),
+			kubenet.NewPlugin(pluginDirs),
 			&v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "pod-name-foo",