From 69ac723b78464b9474f9c069bbfd3dbbdbd19239 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 23 Jan 2018 13:27:38 -0600 Subject: [PATCH] cni: convert "vendor" option to multiple plugin binary search paths It's only used for the test code and after talking with Rajat, the vendor stuff was never really used anyway. So convert the vendor code into a plain array of plugin binary search paths, which is all the vendor code was doing anyway. --- cmd/kubelet/app/plugins.go | 2 +- pkg/kubelet/dockershim/docker_service.go | 2 +- pkg/kubelet/network/cni/cni.go | 72 +++++++++++------------- pkg/kubelet/network/cni/cni_others.go | 7 +-- pkg/kubelet/network/cni/cni_test.go | 64 ++++++++++----------- pkg/kubelet/network/cni/cni_windows.go | 2 +- 6 files changed, 71 insertions(+), 78 deletions(-) diff --git a/cmd/kubelet/app/plugins.go b/cmd/kubelet/app/plugins.go index ef41bb8e90..3194759754 100644 --- a/cmd/kubelet/app/plugins.go +++ b/cmd/kubelet/app/plugins.go @@ -118,7 +118,7 @@ func ProbeNetworkPlugins(cniConfDir, cniBinDir string) []network.NetworkPlugin { allPlugins := []network.NetworkPlugin{} // for each existing plugin, add to the list - allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, cniBinDir)...) + allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, []string{cniBinDir})...) allPlugins = append(allPlugins, kubenet.NewPlugin(cniBinDir)) return allPlugins diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index 3c88a0e4e0..bdf1dfc78b 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -229,7 +229,7 @@ func NewDockerService(config *ClientConfig, podSandboxImage string, streamingCon } } // dockershim currently only supports CNI plugins. - cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, pluginSettings.PluginBinDir) + cniPlugins := cni.ProbeNetworkPlugins(pluginSettings.PluginConfDir, []string{pluginSettings.PluginBinDir}) cniPlugins = append(cniPlugins, kubenet.NewPlugin(pluginSettings.PluginBinDir)) netHost := &dockerNetworkHost{ pluginSettings.LegacyRuntimeHost, diff --git a/pkg/kubelet/network/cni/cni.go b/pkg/kubelet/network/cni/cni.go index 40b1a85470..0f6741442b 100644 --- a/pkg/kubelet/network/cni/cni.go +++ b/pkg/kubelet/network/cni/cni.go @@ -33,10 +33,9 @@ import ( ) const ( - CNIPluginName = "cni" - DefaultConfDir = "/etc/cni/net.d" - DefaultBinDir = "/opt/cni/bin" - VendorCNIDirTemplate = "%s/opt/%s/bin" + CNIPluginName = "cni" + DefaultConfDir = "/etc/cni/net.d" + DefaultBinDir = "/opt/cni/bin" ) type cniNetworkPlugin struct { @@ -47,12 +46,11 @@ type cniNetworkPlugin struct { sync.RWMutex defaultNetwork *cniNetwork - host network.Host - execer utilexec.Interface - nsenterPath string - confDir string - binDir string - vendorCNIDirPrefix string + host network.Host + execer utilexec.Interface + nsenterPath string + confDir string + binDirs []string } type cniNetwork struct { @@ -70,17 +68,28 @@ type cniPortMapping struct { HostIP string `json:"hostIP"` } -func probeNetworkPluginsWithVendorCNIDirPrefix(confDir, binDir, vendorCNIDirPrefix string) []network.NetworkPlugin { - if binDir == "" { - binDir = DefaultBinDir +func ProbeNetworkPlugins(confDir string, binDirs []string) []network.NetworkPlugin { + old := binDirs + binDirs = make([]string, len(binDirs)) + for _, dir := range old { + if dir != "" { + binDirs = append(binDirs, dir) + } } + if len(binDirs) == 0 { + binDirs = []string{DefaultBinDir} + } + + if confDir == "" { + confDir = DefaultConfDir + } + plugin := &cniNetworkPlugin{ - defaultNetwork: nil, - loNetwork: getLoNetwork(binDir, vendorCNIDirPrefix), - execer: utilexec.New(), - confDir: confDir, - binDir: binDir, - vendorCNIDirPrefix: vendorCNIDirPrefix, + defaultNetwork: nil, + loNetwork: getLoNetwork(binDirs), + execer: utilexec.New(), + confDir: confDir, + binDirs: binDirs, } // sync NetworkConfig in best effort during probing. @@ -88,14 +97,7 @@ func probeNetworkPluginsWithVendorCNIDirPrefix(confDir, binDir, vendorCNIDirPref return []network.NetworkPlugin{plugin} } -func ProbeNetworkPlugins(confDir, binDir string) []network.NetworkPlugin { - return probeNetworkPluginsWithVendorCNIDirPrefix(confDir, binDir, "") -} - -func getDefaultCNINetwork(confDir, binDir, vendorCNIDirPrefix string) (*cniNetwork, error) { - if confDir == "" { - confDir = DefaultConfDir - } +func getDefaultCNINetwork(confDir string, binDirs []string) (*cniNetwork, error) { files, err := libcni.ConfFiles(confDir, []string{".conf", ".conflist", ".json"}) switch { case err != nil: @@ -136,23 +138,17 @@ func getDefaultCNINetwork(confDir, binDir, vendorCNIDirPrefix string) (*cniNetwo glog.Warningf("CNI config list %s has no networks, skipping", confFile) continue } - confType := confList.Plugins[0].Network.Type - // Search for vendor-specific plugins as well as default plugins in the CNI codebase. - vendorDir := vendorCNIDir(vendorCNIDirPrefix, confType) - cninet := &libcni.CNIConfig{ - Path: []string{vendorDir, binDir}, + network := &cniNetwork{ + name: confList.Name, + NetworkConfig: confList, + CNIConfig: &libcni.CNIConfig{Path: binDirs}, } - network := &cniNetwork{name: confList.Name, NetworkConfig: confList, CNIConfig: cninet} return network, nil } return nil, fmt.Errorf("No valid networks found in %s", confDir) } -func vendorCNIDir(prefix, pluginType string) string { - return fmt.Sprintf(VendorCNIDirTemplate, prefix, pluginType) -} - func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error { err := plugin.platformInit() if err != nil { @@ -166,7 +162,7 @@ func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfi } func (plugin *cniNetworkPlugin) syncNetworkConfig() { - network, err := getDefaultCNINetwork(plugin.confDir, plugin.binDir, plugin.vendorCNIDirPrefix) + network, err := getDefaultCNINetwork(plugin.confDir, plugin.binDirs) if err != nil { glog.Warningf("Unable to update cni config: %s", err) return diff --git a/pkg/kubelet/network/cni/cni_others.go b/pkg/kubelet/network/cni/cni_others.go index 735db9cc69..cdc0c1a11f 100644 --- a/pkg/kubelet/network/cni/cni_others.go +++ b/pkg/kubelet/network/cni/cni_others.go @@ -26,7 +26,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/network" ) -func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork { +func getLoNetwork(binDirs []string) *cniNetwork { loConfig, err := libcni.ConfListFromBytes([]byte(`{ "cniVersion": "0.2.0", "name": "cni-loopback", @@ -39,13 +39,10 @@ func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork { // catch this panic(err) } - cninet := &libcni.CNIConfig{ - Path: []string{vendorCNIDir(vendorDirPrefix, "loopback"), binDir}, - } loNetwork := &cniNetwork{ name: "lo", NetworkConfig: loConfig, - CNIConfig: cninet, + CNIConfig: &libcni.CNIConfig{Path: binDirs}, } return loNetwork diff --git a/pkg/kubelet/network/cni/cni_test.go b/pkg/kubelet/network/cni/cni_test.go index b34488ed4d..f1701a22d4 100644 --- a/pkg/kubelet/network/cni/cni_test.go +++ b/pkg/kubelet/network/cni/cni_test.go @@ -47,31 +47,28 @@ import ( fakeexec "k8s.io/utils/exec/testing" ) -func installPluginUnderTest(t *testing.T, testVendorCNIDirPrefix, testNetworkConfigPath, vendorName string, plugName string) { - pluginDir := path.Join(testNetworkConfigPath, plugName) - err := os.MkdirAll(pluginDir, 0777) - if err != nil { - t.Fatalf("Failed to create plugin config dir: %v", err) +// Returns .in file path, .out file path, and .env file path +func installPluginUnderTest(t *testing.T, testBinDir, testConfDir, testDataDir, binName string, confName string) (string, string, string) { + for _, dir := range []string{testBinDir, testConfDir, testDataDir} { + err := os.MkdirAll(dir, 0777) + if err != nil { + t.Fatalf("Failed to create test plugin dir %s: %v", dir, err) + } } - pluginConfig := path.Join(pluginDir, plugName+".conf") - f, err := os.Create(pluginConfig) - if err != nil { - t.Fatalf("Failed to install plugin") - } - networkConfig := fmt.Sprintf(`{ "name": "%s", "type": "%s", "capabilities": {"portMappings": true} }`, plugName, vendorName) + confFile := path.Join(testConfDir, confName+".conf") + f, err := os.Create(confFile) + if err != nil { + t.Fatalf("Failed to install plugin %s: %v", confFile, err) + } + networkConfig := fmt.Sprintf(`{ "name": "%s", "type": "%s", "capabilities": {"portMappings": true} }`, confName, binName) _, err = f.WriteString(networkConfig) if err != nil { t.Fatalf("Failed to write network config file (%v)", err) } f.Close() - vendorCNIDir := fmt.Sprintf(VendorCNIDirTemplate, testVendorCNIDirPrefix, vendorName) - err = os.MkdirAll(vendorCNIDir, 0777) - if err != nil { - t.Fatalf("Failed to create plugin dir: %v", err) - } - pluginExec := path.Join(vendorCNIDir, vendorName) + pluginExec := path.Join(testBinDir, binName) f, err = os.Create(pluginExec) const execScriptTempl = `#!/bin/bash @@ -83,11 +80,14 @@ mkdir -p {{.OutputDir}} &> /dev/null echo -n "$CNI_COMMAND $CNI_NETNS $K8S_POD_NAMESPACE $K8S_POD_NAME $K8S_POD_INFRA_CONTAINER_ID" >& {{.OutputFile}} echo -n "{ \"ip4\": { \"ip\": \"10.1.0.23/24\" } }" ` + inputFile := path.Join(testDataDir, binName+".in") + outputFile := path.Join(testDataDir, binName+".out") + envFile := path.Join(testDataDir, binName+".env") execTemplateData := &map[string]interface{}{ - "InputFile": path.Join(pluginDir, plugName+".in"), - "OutputFile": path.Join(pluginDir, plugName+".out"), - "OutputEnv": path.Join(pluginDir, plugName+".env"), - "OutputDir": pluginDir, + "InputFile": inputFile, + "OutputFile": outputFile, + "OutputEnv": envFile, + "OutputDir": testDataDir, } tObj := template.Must(template.New("test").Parse(execScriptTempl)) @@ -107,6 +107,8 @@ echo -n "{ \"ip4\": { \"ip\": \"10.1.0.23/24\" } }" } f.Close() + + return inputFile, outputFile, envFile } func tearDownPlugin(tmpDir string) { @@ -155,8 +157,8 @@ func (fnh *fakeNetworkHost) SupportsLegacyFeatures() bool { func TestCNIPlugin(t *testing.T) { // install some random plugin - pluginName := fmt.Sprintf("test%d", rand.Intn(1000)) - vendorName := fmt.Sprintf("test_vendor%d", rand.Intn(1000)) + netName := fmt.Sprintf("test%d", rand.Intn(1000)) + binName := fmt.Sprintf("test_vendor%d", rand.Intn(1000)) podIP := "10.0.0.2" podIPOutput := fmt.Sprintf("4: eth0 inet %s/24 scope global dynamic eth0\\ valid_lft forever preferred_lft forever", podIP) @@ -183,10 +185,11 @@ func TestCNIPlugin(t *testing.T) { // TODO mock for the test plugin too tmpDir := utiltesting.MkTmpdirOrDie("cni-test") - testNetworkConfigPath := path.Join(tmpDir, "plugins", "net", "cni") - testVendorCNIDirPrefix := tmpDir + testConfDir := path.Join(tmpDir, "etc", "cni", "net.d") + testBinDir := path.Join(tmpDir, "opt", "cni", "bin") + testDataDir := path.Join(tmpDir, "output") defer tearDownPlugin(tmpDir) - installPluginUnderTest(t, testVendorCNIDirPrefix, testNetworkConfigPath, vendorName, pluginName) + inputFile, outputFile, outputEnv := installPluginUnderTest(t, testBinDir, testConfDir, testDataDir, binName, netName) containerID := kubecontainer.ContainerID{Type: "test", ID: "test_infra_container"} pods := []*containertest.FakePod{{ @@ -198,7 +201,7 @@ func TestCNIPlugin(t *testing.T) { NetnsPath: "/proc/12345/ns/net", }} - plugins := probeNetworkPluginsWithVendorCNIDirPrefix(path.Join(testNetworkConfigPath, pluginName), "", testVendorCNIDirPrefix) + plugins := ProbeNetworkPlugins(testConfDir, []string{testBinDir}) if len(plugins) != 1 { t.Fatalf("Expected only one network plugin, got %d", len(plugins)) } @@ -238,9 +241,7 @@ func TestCNIPlugin(t *testing.T) { if err != nil { t.Errorf("Expected nil: %v", err) } - outputEnv := path.Join(testNetworkConfigPath, pluginName, pluginName+".env") eo, eerr := ioutil.ReadFile(outputEnv) - outputFile := path.Join(testNetworkConfigPath, pluginName, pluginName+".out") output, err := ioutil.ReadFile(outputFile) if err != nil || eerr != nil { t.Errorf("Failed to read output file %s: %v (env %s err %v)", outputFile, err, eo, eerr) @@ -257,7 +258,6 @@ func TestCNIPlugin(t *testing.T) { PortMappings []map[string]interface{} `json:"portMappings"` } `json:"runtimeConfig"` }{} - inputFile := path.Join(testNetworkConfigPath, pluginName, pluginName+".in") inputBytes, inerr := ioutil.ReadFile(inputFile) parseerr := json.Unmarshal(inputBytes, &inputConfig) if inerr != nil || parseerr != nil { @@ -285,7 +285,7 @@ func TestCNIPlugin(t *testing.T) { if err != nil { t.Errorf("Expected nil: %v", err) } - output, err = ioutil.ReadFile(path.Join(testNetworkConfigPath, pluginName, pluginName+".out")) + output, err = ioutil.ReadFile(outputFile) expectedOutput = "DEL /proc/12345/ns/net podNamespace podName test_infra_container" if string(output) != expectedOutput { t.Errorf("Mismatch in expected output for setup hook. Expected '%s', got '%s'", expectedOutput, string(output)) @@ -295,7 +295,7 @@ func TestCNIPlugin(t *testing.T) { } func TestLoNetNonNil(t *testing.T) { - if conf := getLoNetwork("", ""); conf == nil { + if conf := getLoNetwork(nil); conf == nil { t.Error("Expected non-nil lo network") } } diff --git a/pkg/kubelet/network/cni/cni_windows.go b/pkg/kubelet/network/cni/cni_windows.go index 9a0b17c8f1..81cc5ca1fe 100644 --- a/pkg/kubelet/network/cni/cni_windows.go +++ b/pkg/kubelet/network/cni/cni_windows.go @@ -27,7 +27,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/network" ) -func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork { +func getLoNetwork(binDirs []string) *cniNetwork { return nil }