diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 09deabd85d..1347a19eb0 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -147,7 +147,9 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.Int32Var(&s.LowDiskSpaceThresholdMB, "low-diskspace-threshold-mb", s.LowDiskSpaceThresholdMB, "The absolute free disk space, in MB, to maintain. When disk space falls below this threshold, new pods would be rejected. Default: 256") fs.DurationVar(&s.VolumeStatsAggPeriod.Duration, "volume-stats-agg-period", s.VolumeStatsAggPeriod.Duration, "Specifies interval for kubelet to calculate and cache the volume disk usage for all pods and volumes. To disable volume calculations, set to 0. Default: '1m'") fs.StringVar(&s.NetworkPluginName, "network-plugin", s.NetworkPluginName, " The name of the network plugin to be invoked for various events in kubelet/pod lifecycle") - fs.StringVar(&s.NetworkPluginDir, "network-plugin-dir", s.NetworkPluginDir, " The full path of the directory in which to search for network plugins") + fs.StringVar(&s.NetworkPluginDir, "network-plugin-dir", s.NetworkPluginDir, " The full path of the directory in which to search for network plugins or CNI config") + fs.StringVar(&s.CNIConfDir, "cni-conf-dir", s.CNIConfDir, " The full path of the directory in which to search for CNI config files. Default: /etc/cni/net.d") + fs.StringVar(&s.CNIBinDir, "cni-bin-dir", s.CNIBinDir, " The full path of the directory in which to search for CNI plugin binaries. Default: /opt/cni/bin") fs.Int32Var(&s.NetworkPluginMTU, "network-plugin-mtu", s.NetworkPluginMTU, " The MTU to be passed to the network plugin, to override the default. Set to 0 to use the default 1460 MTU.") fs.StringVar(&s.VolumePluginDir, "volume-plugin-dir", s.VolumePluginDir, " The full path of the directory in which to search for additional third party volume plugins") fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider for cloud services. By default, kubelet will attempt to auto-detect the cloud provider. Specify empty string for running with no cloud provider. [default=auto-detect]") diff --git a/cmd/kubelet/app/plugins.go b/cmd/kubelet/app/plugins.go index a3c6343f4c..833198118a 100644 --- a/cmd/kubelet/app/plugins.go +++ b/cmd/kubelet/app/plugins.go @@ -90,12 +90,16 @@ func ProbeVolumePlugins(pluginDir string) []volume.VolumePlugin { } // ProbeNetworkPlugins collects all compiled-in plugins -func ProbeNetworkPlugins(pluginDir string) []network.NetworkPlugin { +func ProbeNetworkPlugins(pluginDir, cniConfDir, cniBinDir string) []network.NetworkPlugin { allPlugins := []network.NetworkPlugin{} + // for backwards-compat, allow pluginDir as a source of CNI config files + if cniConfDir == "" { + cniConfDir = pluginDir + } // for each existing plugin, add to the list allPlugins = append(allPlugins, exec.ProbeNetworkPlugins(pluginDir)...) - allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(pluginDir)...) + allPlugins = append(allPlugins, cni.ProbeNetworkPlugins(cniConfDir, cniBinDir)...) allPlugins = append(allPlugins, kubenet.NewPlugin(pluginDir)) return allPlugins diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index f38d09204b..3cde7cb089 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -139,7 +139,7 @@ func UnsecuredKubeletDeps(s *options.KubeletServer) (*kubelet.KubeletDeps, error DockerClient: dockerClient, KubeClient: nil, Mounter: mounter, - NetworkPlugins: ProbeNetworkPlugins(s.NetworkPluginDir), + NetworkPlugins: ProbeNetworkPlugins(s.NetworkPluginDir, s.CNIConfDir, s.CNIBinDir), OOMAdjuster: oom.NewOOMAdjuster(), OSInterface: kubecontainer.RealOS{}, Writer: writer, diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index 3fd4cf7dd6..37aa77c6c0 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -257,8 +257,14 @@ type KubeletConfiguration struct { // computed (such as IPSEC). NetworkPluginMTU int32 `json:"networkPluginMTU"` // networkPluginDir is the full path of the directory in which to search - // for network plugins + // for network plugins (and, for backwards-compat, CNI config files) NetworkPluginDir string `json:"networkPluginDir"` + // CNIConfDir is the full path of the directory in which to search for + // CNI config files + CNIConfDir string `json:"cniConfDir"` + // CNIBinDir is the full path of the directory in which to search for + // CNI plugin binaries + CNIBinDir string `json:"cniBinDir"` // volumePluginDir is the full path of the directory in which to search // for additional third party volume plugins VolumePluginDir string `json:"volumePluginDir"` diff --git a/pkg/apis/componentconfig/v1alpha1/defaults.go b/pkg/apis/componentconfig/v1alpha1/defaults.go index 16031a1b75..b06b766c99 100644 --- a/pkg/apis/componentconfig/v1alpha1/defaults.go +++ b/pkg/apis/componentconfig/v1alpha1/defaults.go @@ -249,7 +249,9 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) { if obj.MinimumGCAge == zeroDuration { obj.MinimumGCAge = unversioned.Duration{Duration: 0} } - if obj.NetworkPluginDir == "" { + // For backwards-compat NetworkPluginDir is consulted as source of CNI config + // but CNIConfDir is preferred; don't set this default if CNI is in use. + if obj.NetworkPluginDir == "" && obj.NetworkPluginName != "cni" { obj.NetworkPluginDir = "/usr/libexec/kubernetes/kubelet-plugins/net/exec/" } if obj.NonMasqueradeCIDR == "" { diff --git a/pkg/apis/componentconfig/v1alpha1/types.go b/pkg/apis/componentconfig/v1alpha1/types.go index 75147cdacd..40001fb372 100644 --- a/pkg/apis/componentconfig/v1alpha1/types.go +++ b/pkg/apis/componentconfig/v1alpha1/types.go @@ -308,8 +308,14 @@ type KubeletConfiguration struct { // various events in kubelet/pod lifecycle NetworkPluginName string `json:"networkPluginName"` // networkPluginDir is the full path of the directory in which to search - // for network plugins + // for network plugins (and, for backwards-compat, CNI config files) NetworkPluginDir string `json:"networkPluginDir"` + // CNIConfDir is the full path of the directory in which to search for + // CNI config files + CNIConfDir string `json:"cniConfDir"` + // CNIBinDir is the full path of the directory in which to search for + // CNI plugin binaries + CNIBinDir string `json:"cniBinDir"` // networkPluginMTU is the MTU to be passed to the network plugin, // and overrides the default MTU for cases where it cannot be automatically // computed (such as IPSEC). diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go index d5dc36acec..c540752003 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go @@ -233,6 +233,8 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu out.VolumeStatsAggPeriod = in.VolumeStatsAggPeriod out.NetworkPluginName = in.NetworkPluginName out.NetworkPluginDir = in.NetworkPluginDir + out.CNIConfDir = in.CNIConfDir + out.CNIBinDir = in.CNIBinDir out.NetworkPluginMTU = in.NetworkPluginMTU out.VolumePluginDir = in.VolumePluginDir out.CloudProvider = in.CloudProvider @@ -412,6 +414,8 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu out.NetworkPluginName = in.NetworkPluginName out.NetworkPluginMTU = in.NetworkPluginMTU out.NetworkPluginDir = in.NetworkPluginDir + out.CNIConfDir = in.CNIConfDir + out.CNIBinDir = in.CNIBinDir out.VolumePluginDir = in.VolumePluginDir out.CloudProvider = in.CloudProvider out.CloudConfigFile = in.CloudConfigFile diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go index 952ce0de60..abc4cfc4a6 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go @@ -239,6 +239,8 @@ func DeepCopy_v1alpha1_KubeletConfiguration(in interface{}, out interface{}, c * out.VolumeStatsAggPeriod = in.VolumeStatsAggPeriod out.NetworkPluginName = in.NetworkPluginName out.NetworkPluginDir = in.NetworkPluginDir + out.CNIConfDir = in.CNIConfDir + out.CNIBinDir = in.CNIBinDir out.NetworkPluginMTU = in.NetworkPluginMTU out.VolumePluginDir = in.VolumePluginDir out.CloudProvider = in.CloudProvider diff --git a/pkg/apis/componentconfig/zz_generated.deepcopy.go b/pkg/apis/componentconfig/zz_generated.deepcopy.go index 7f3dd1b15f..b6ceb626c8 100644 --- a/pkg/apis/componentconfig/zz_generated.deepcopy.go +++ b/pkg/apis/componentconfig/zz_generated.deepcopy.go @@ -259,6 +259,8 @@ func DeepCopy_componentconfig_KubeletConfiguration(in interface{}, out interface out.NetworkPluginName = in.NetworkPluginName out.NetworkPluginMTU = in.NetworkPluginMTU out.NetworkPluginDir = in.NetworkPluginDir + out.CNIConfDir = in.CNIConfDir + out.CNIBinDir = in.CNIBinDir out.VolumePluginDir = in.VolumePluginDir out.CloudProvider = in.CloudProvider out.CloudConfigFile = in.CloudConfigFile diff --git a/pkg/kubelet/network/cni/cni.go b/pkg/kubelet/network/cni/cni.go index cc50fc45a4..67f1c47404 100644 --- a/pkg/kubelet/network/cni/cni.go +++ b/pkg/kubelet/network/cni/cni.go @@ -52,6 +52,7 @@ type cniNetworkPlugin struct { execer utilexec.Interface nsenterPath string pluginDir string + binDir string vendorCNIDirPrefix string } @@ -61,12 +62,16 @@ type cniNetwork struct { CNIConfig libcni.CNI } -func probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, vendorCNIDirPrefix string) []network.NetworkPlugin { +func probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, binDir, vendorCNIDirPrefix string) []network.NetworkPlugin { + if binDir == "" { + binDir = DefaultCNIDir + } plugin := &cniNetworkPlugin{ defaultNetwork: nil, - loNetwork: getLoNetwork(vendorCNIDirPrefix), + loNetwork: getLoNetwork(binDir, vendorCNIDirPrefix), execer: utilexec.New(), pluginDir: pluginDir, + binDir: binDir, vendorCNIDirPrefix: vendorCNIDirPrefix, } @@ -75,11 +80,11 @@ func probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, vendorCNIDirPrefix str return []network.NetworkPlugin{plugin} } -func ProbeNetworkPlugins(pluginDir string) []network.NetworkPlugin { - return probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, "") +func ProbeNetworkPlugins(pluginDir, binDir string) []network.NetworkPlugin { + return probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, binDir, "") } -func getDefaultCNINetwork(pluginDir, vendorCNIDirPrefix string) (*cniNetwork, error) { +func getDefaultCNINetwork(pluginDir, binDir, vendorCNIDirPrefix string) (*cniNetwork, error) { if pluginDir == "" { pluginDir = DefaultNetDir } @@ -101,7 +106,7 @@ func getDefaultCNINetwork(pluginDir, vendorCNIDirPrefix string) (*cniNetwork, er // Search for vendor-specific plugins as well as default plugins in the CNI codebase. vendorDir := vendorCNIDir(vendorCNIDirPrefix, conf.Network.Type) cninet := &libcni.CNIConfig{ - Path: []string{DefaultCNIDir, vendorDir}, + Path: []string{binDir, vendorDir}, } network := &cniNetwork{name: conf.Network.Name, NetworkConfig: conf, CNIConfig: cninet} return network, nil @@ -113,7 +118,7 @@ func vendorCNIDir(prefix, pluginType string) string { return fmt.Sprintf(VendorCNIDirTemplate, prefix, pluginType) } -func getLoNetwork(vendorDirPrefix string) *cniNetwork { +func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork { loConfig, err := libcni.ConfFromBytes([]byte(`{ "cniVersion": "0.1.0", "name": "cni-loopback", @@ -125,7 +130,7 @@ func getLoNetwork(vendorDirPrefix string) *cniNetwork { panic(err) } cninet := &libcni.CNIConfig{ - Path: []string{vendorCNIDir(vendorDirPrefix, loConfig.Network.Type), DefaultCNIDir}, + Path: []string{vendorCNIDir(vendorDirPrefix, loConfig.Network.Type), binDir}, } loNetwork := &cniNetwork{ name: "lo", @@ -153,7 +158,7 @@ func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode componentcon } func (plugin *cniNetworkPlugin) syncNetworkConfig() { - network, err := getDefaultCNINetwork(plugin.pluginDir, plugin.vendorCNIDirPrefix) + network, err := getDefaultCNINetwork(plugin.pluginDir, plugin.binDir, plugin.vendorCNIDirPrefix) if err != nil { glog.Errorf("error updating cni config: %s", err) return