Adding a root filesystem override for kubelet mounter

This is useful for supporting hostPath volumes via containerized
mounters in kubelet.

Signed-off-by: Vishnu kannan <vishnuk@google.com>
pull/6/head
Vishnu kannan 2016-10-26 12:57:00 -07:00
parent cee8645d36
commit e861a5761d
18 changed files with 3284 additions and 3191 deletions

View File

@ -67,7 +67,7 @@ func (realConntracker) SetTCPEstablishedTimeout(seconds int) error {
func isSysFSWritable() (bool, error) {
const permWritable = "rw"
const sysfsDevice = "sysfs"
m := mount.New("" /* default mount path */)
m := mount.New()
mountPoints, err := m.List()
if err != nil {
glog.Errorf("failed to list mount points: %v", err)

View File

@ -198,6 +198,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
fs.BoolVar(&s.ExitOnLockContention, "exit-on-lock-contention", s.ExitOnLockContention, "Whether kubelet should exit upon lock-file contention.")
fs.StringVar(&s.RktPath, "rkt-path", s.RktPath, "Path of rkt binary. Leave empty to use the first rkt in $PATH. Only used if --container-runtime='rkt'.")
fs.StringVar(&s.ExperimentalMounterPath, "experimental-mounter-path", s.ExperimentalMounterPath, "[Experimental] Path of mounter binary. Leave empty to use the default mount.")
fs.StringVar(&s.ExperimentalMounterRootfsPath, "experimental-mounter-rootfs-path", s.ExperimentalMounterRootfsPath, "[Experimental] Absolute path to the root filesystem for the mounter binary.")
fs.StringVar(&s.RktAPIEndpoint, "rkt-api-endpoint", s.RktAPIEndpoint, "The endpoint of the rkt API service to communicate with. Only used if --container-runtime='rkt'.")
fs.StringVar(&s.RktStage1Image, "rkt-stage1-image", s.RktStage1Image, "image to use as stage1. Local paths and http/https URLs are supported. If empty, the 'stage1.aci' in the same directory as '--rkt-path' will be used.")
fs.MarkDeprecated("rkt-stage1-image", "Will be removed in a future version. The default stage1 image will be specified by the rkt configurations, see https://github.com/coreos/rkt/blob/master/Documentation/configuration.md for more details.")

View File

@ -118,7 +118,7 @@ func UnsecuredKubeletDeps(s *options.KubeletServer) (*kubelet.KubeletDeps, error
return nil, err
}
mounter := mount.New(s.ExperimentalMounterPath)
mounter := mount.NewCustomMounter(s.ExperimentalMounterPath, s.ExperimentalMounterRootfsPath)
var writer kubeio.Writer = &kubeio.StdWriter{}
if s.Containerized {
glog.V(2).Info("Running kubelet in containerized mode (experimental)")

View File

@ -188,6 +188,7 @@ experimental-allowed-unsafe-sysctls
experimental-bootstrap-kubeconfig
experimental-keystone-url
experimental-mounter-path
experimental-mounter-rootfs-path
experimental-nvidia-gpus
experimental-prefix
experimental-runtime-integration-type

File diff suppressed because it is too large Load Diff

View File

@ -319,6 +319,8 @@ type KubeletConfiguration struct {
RktPath string `json:"rktPath,omitempty"`
// experimentalMounterPath is the path of mounter binary. Leave empty to use the default mount path
ExperimentalMounterPath string `json:"experimentalMounterPath,omitempty"`
// experimentalMounterRootfsPath is the absolute path to root filesystem for the mounter binary.
ExperimentalMounterRootfsPath string `json:"experimentalMounterRootfsPath,omitempty"`
// rktApiEndpoint is the endpoint of the rkt API service to communicate with.
// +optional
RktAPIEndpoint string `json:"rktAPIEndpoint,omitempty"`

View File

@ -367,6 +367,8 @@ type KubeletConfiguration struct {
// experimentalMounterPath is the path to mounter binary. If not set, kubelet will attempt to use mount
// binary that is available via $PATH,
ExperimentalMounterPath string `json:"experimentalMounterPath,omitempty"`
// experimentalMounterRootfsPath is the absolute path to root filesystem for the mounter binary.
ExperimentalMounterRootfsPath string `json:"experimentalMounterRootfsPath,omitempty"`
// rktApiEndpoint is the endpoint of the rkt API service to communicate with.
RktAPIEndpoint string `json:"rktAPIEndpoint"`
// rktStage1Image is the image to use as stage1. Local paths and

View File

@ -335,6 +335,7 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu
out.RuntimeRequestTimeout = in.RuntimeRequestTimeout
out.RktPath = in.RktPath
out.ExperimentalMounterPath = in.ExperimentalMounterPath
out.ExperimentalMounterRootfsPath = in.ExperimentalMounterRootfsPath
out.RktAPIEndpoint = in.RktAPIEndpoint
out.RktStage1Image = in.RktStage1Image
if err := api.Convert_Pointer_string_To_string(&in.LockFilePath, &out.LockFilePath, s); err != nil {
@ -517,6 +518,7 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu
out.RuntimeRequestTimeout = in.RuntimeRequestTimeout
out.RktPath = in.RktPath
out.ExperimentalMounterPath = in.ExperimentalMounterPath
out.ExperimentalMounterRootfsPath = in.ExperimentalMounterRootfsPath
out.RktAPIEndpoint = in.RktAPIEndpoint
out.RktStage1Image = in.RktStage1Image
if err := api.Convert_string_To_Pointer_string(&in.LockFilePath, &out.LockFilePath, s); err != nil {

View File

@ -314,6 +314,7 @@ func DeepCopy_v1alpha1_KubeletConfiguration(in interface{}, out interface{}, c *
out.RuntimeRequestTimeout = in.RuntimeRequestTimeout
out.RktPath = in.RktPath
out.ExperimentalMounterPath = in.ExperimentalMounterPath
out.ExperimentalMounterRootfsPath = in.ExperimentalMounterRootfsPath
out.RktAPIEndpoint = in.RktAPIEndpoint
out.RktStage1Image = in.RktStage1Image
if in.LockFilePath != nil {

View File

@ -316,6 +316,7 @@ func DeepCopy_componentconfig_KubeletConfiguration(in interface{}, out interface
out.RuntimeRequestTimeout = in.RuntimeRequestTimeout
out.RktPath = in.RktPath
out.ExperimentalMounterPath = in.ExperimentalMounterPath
out.ExperimentalMounterRootfsPath = in.ExperimentalMounterRootfsPath
out.RktAPIEndpoint = in.RktAPIEndpoint
out.RktStage1Image = in.RktStage1Image
out.LockFilePath = in.LockFilePath

View File

@ -95,7 +95,7 @@ func getMetadataFromConfigDrive() (*Metadata, error) {
glog.V(4).Infof("Attempting to mount configdrive %s on %s", dev, mntdir)
mounter := mount.New("" /* default mount path */)
mounter := mount.New()
err = mounter.Mount(dev, mntdir, "iso9660", []string{"ro"})
if err != nil {
err = mounter.Mount(dev, mntdir, "vfat", []string{"ro"})

View File

@ -2657,6 +2657,13 @@ var OpenAPIDefinitions *common.OpenAPIDefinitions = &common.OpenAPIDefinitions{
Format: "",
},
},
"experimentalMounterRootfsPath": {
SchemaProps: spec.SchemaProps{
Description: "experimentalMounterRootfsPath is the absolute path to root filesystem for the mounter binary.",
Type: []string{"string"},
Format: "",
},
},
"rktAPIEndpoint": {
SchemaProps: spec.SchemaProps{
Description: "rktApiEndpoint is the endpoint of the rkt API service to communicate with.",
@ -2971,7 +2978,7 @@ var OpenAPIDefinitions *common.OpenAPIDefinitions = &common.OpenAPIDefinitions{
},
},
},
Required: []string{"TypeMeta", "podManifestPath", "syncFrequency", "fileCheckFrequency", "httpCheckFrequency", "manifestURL", "manifestURLHeader", "enableServer", "address", "port", "readOnlyPort", "tlsCertFile", "tlsPrivateKeyFile", "certDirectory", "authentication", "authorization", "hostnameOverride", "podInfraContainerImage", "dockerEndpoint", "rootDirectory", "seccompProfileRoot", "allowPrivileged", "hostNetworkSources", "hostPIDSources", "hostIPCSources", "registryPullQPS", "registryBurst", "eventRecordQPS", "eventBurst", "enableDebuggingHandlers", "minimumGCAge", "maxPerPodContainerCount", "maxContainerCount", "cAdvisorPort", "healthzPort", "healthzBindAddress", "oomScoreAdj", "registerNode", "clusterDomain", "masterServiceNamespace", "clusterDNS", "streamingConnectionIdleTimeout", "nodeStatusUpdateFrequency", "imageMinimumGCAge", "imageGCHighThresholdPercent", "imageGCLowThresholdPercent", "lowDiskSpaceThresholdMB", "volumeStatsAggPeriod", "networkPluginName", "networkPluginMTU", "networkPluginDir", "cniConfDir", "cniBinDir", "volumePluginDir", "containerRuntime", "remoteRuntimeEndpoint", "remoteImageEndpoint", "experimentalMounterPath", "lockFilePath", "exitOnLockContention", "hairpinMode", "babysitDaemons", "maxPods", "nvidiaGPUs", "dockerExecHandlerName", "podCIDR", "resolvConf", "cpuCFSQuota", "containerized", "maxOpenFiles", "reconcileCIDR", "registerSchedulable", "contentType", "kubeAPIQPS", "kubeAPIBurst", "serializeImagePulls", "nodeLabels", "nonMasqueradeCIDR", "enableCustomMetrics", "podsPerCore", "enableControllerAttachDetach", "systemReserved", "kubeReserved", "protectKernelDefaults", "makeIPTablesUtilChains", "iptablesMasqueradeBit", "iptablesDropBit"},
Required: []string{"TypeMeta", "podManifestPath", "syncFrequency", "fileCheckFrequency", "httpCheckFrequency", "manifestURL", "manifestURLHeader", "enableServer", "address", "port", "readOnlyPort", "tlsCertFile", "tlsPrivateKeyFile", "certDirectory", "authentication", "authorization", "hostnameOverride", "podInfraContainerImage", "dockerEndpoint", "rootDirectory", "seccompProfileRoot", "allowPrivileged", "hostNetworkSources", "hostPIDSources", "hostIPCSources", "registryPullQPS", "registryBurst", "eventRecordQPS", "eventBurst", "enableDebuggingHandlers", "minimumGCAge", "maxPerPodContainerCount", "maxContainerCount", "cAdvisorPort", "healthzPort", "healthzBindAddress", "oomScoreAdj", "registerNode", "clusterDomain", "masterServiceNamespace", "clusterDNS", "streamingConnectionIdleTimeout", "nodeStatusUpdateFrequency", "imageMinimumGCAge", "imageGCHighThresholdPercent", "imageGCLowThresholdPercent", "lowDiskSpaceThresholdMB", "volumeStatsAggPeriod", "networkPluginName", "networkPluginMTU", "networkPluginDir", "cniConfDir", "cniBinDir", "volumePluginDir", "containerRuntime", "remoteRuntimeEndpoint", "remoteImageEndpoint", "experimentalMounterPath", "experimentalMounterRootfsPath", "lockFilePath", "exitOnLockContention", "hairpinMode", "babysitDaemons", "maxPods", "nvidiaGPUs", "dockerExecHandlerName", "podCIDR", "resolvConf", "cpuCFSQuota", "containerized", "maxOpenFiles", "reconcileCIDR", "registerSchedulable", "contentType", "kubeAPIQPS", "kubeAPIBurst", "serializeImagePulls", "nodeLabels", "nonMasqueradeCIDR", "enableCustomMetrics", "podsPerCore", "enableControllerAttachDetach", "systemReserved", "kubeReserved", "protectKernelDefaults", "makeIPTablesUtilChains", "iptablesMasqueradeBit", "iptablesDropBit"},
},
},
Dependencies: []string{
@ -14320,6 +14327,13 @@ var OpenAPIDefinitions *common.OpenAPIDefinitions = &common.OpenAPIDefinitions{
Format: "",
},
},
"experimentalMounterRootfsPath": {
SchemaProps: spec.SchemaProps{
Description: "experimentalMounterRootfsPath is the absolute path to root filesystem for the mounter binary.",
Type: []string{"string"},
Format: "",
},
},
"rktAPIEndpoint": {
SchemaProps: spec.SchemaProps{
Description: "rktApiEndpoint is the endpoint of the rkt API service to communicate with.",
@ -14634,7 +14648,7 @@ var OpenAPIDefinitions *common.OpenAPIDefinitions = &common.OpenAPIDefinitions{
},
},
},
Required: []string{"TypeMeta", "podManifestPath", "syncFrequency", "fileCheckFrequency", "httpCheckFrequency", "manifestURL", "manifestURLHeader", "enableServer", "address", "port", "readOnlyPort", "tlsCertFile", "tlsPrivateKeyFile", "certDirectory", "authentication", "authorization", "hostnameOverride", "podInfraContainerImage", "dockerEndpoint", "rootDirectory", "seccompProfileRoot", "allowPrivileged", "hostNetworkSources", "hostPIDSources", "hostIPCSources", "registryPullQPS", "registryBurst", "eventRecordQPS", "eventBurst", "enableDebuggingHandlers", "minimumGCAge", "maxPerPodContainerCount", "maxContainerCount", "cAdvisorPort", "healthzPort", "healthzBindAddress", "oomScoreAdj", "registerNode", "clusterDomain", "masterServiceNamespace", "clusterDNS", "streamingConnectionIdleTimeout", "nodeStatusUpdateFrequency", "imageMinimumGCAge", "imageGCHighThresholdPercent", "imageGCLowThresholdPercent", "lowDiskSpaceThresholdMB", "volumeStatsAggPeriod", "networkPluginName", "networkPluginDir", "cniConfDir", "cniBinDir", "networkPluginMTU", "volumePluginDir", "cloudProvider", "cloudConfigFile", "kubeletCgroups", "runtimeCgroups", "systemCgroups", "cgroupRoot", "containerRuntime", "remoteRuntimeEndpoint", "remoteImageEndpoint", "runtimeRequestTimeout", "rktPath", "experimentalMounterPath", "rktAPIEndpoint", "rktStage1Image", "lockFilePath", "exitOnLockContention", "hairpinMode", "babysitDaemons", "maxPods", "nvidiaGPUs", "dockerExecHandlerName", "podCIDR", "resolvConf", "cpuCFSQuota", "containerized", "maxOpenFiles", "reconcileCIDR", "registerSchedulable", "contentType", "kubeAPIQPS", "kubeAPIBurst", "serializeImagePulls", "outOfDiskTransitionFrequency", "nodeIP", "nodeLabels", "nonMasqueradeCIDR", "enableCustomMetrics", "evictionHard", "evictionSoft", "evictionSoftGracePeriod", "evictionPressureTransitionPeriod", "evictionMaxPodGracePeriod", "evictionMinimumReclaim", "podsPerCore", "enableControllerAttachDetach", "systemReserved", "kubeReserved", "protectKernelDefaults", "makeIPTablesUtilChains", "iptablesMasqueradeBit", "iptablesDropBit"},
Required: []string{"TypeMeta", "podManifestPath", "syncFrequency", "fileCheckFrequency", "httpCheckFrequency", "manifestURL", "manifestURLHeader", "enableServer", "address", "port", "readOnlyPort", "tlsCertFile", "tlsPrivateKeyFile", "certDirectory", "authentication", "authorization", "hostnameOverride", "podInfraContainerImage", "dockerEndpoint", "rootDirectory", "seccompProfileRoot", "allowPrivileged", "hostNetworkSources", "hostPIDSources", "hostIPCSources", "registryPullQPS", "registryBurst", "eventRecordQPS", "eventBurst", "enableDebuggingHandlers", "minimumGCAge", "maxPerPodContainerCount", "maxContainerCount", "cAdvisorPort", "healthzPort", "healthzBindAddress", "oomScoreAdj", "registerNode", "clusterDomain", "masterServiceNamespace", "clusterDNS", "streamingConnectionIdleTimeout", "nodeStatusUpdateFrequency", "imageMinimumGCAge", "imageGCHighThresholdPercent", "imageGCLowThresholdPercent", "lowDiskSpaceThresholdMB", "volumeStatsAggPeriod", "networkPluginName", "networkPluginDir", "cniConfDir", "cniBinDir", "networkPluginMTU", "volumePluginDir", "cloudProvider", "cloudConfigFile", "kubeletCgroups", "runtimeCgroups", "systemCgroups", "cgroupRoot", "containerRuntime", "remoteRuntimeEndpoint", "remoteImageEndpoint", "runtimeRequestTimeout", "rktPath", "experimentalMounterPath", "experimentalMounterRootfsPath", "rktAPIEndpoint", "rktStage1Image", "lockFilePath", "exitOnLockContention", "hairpinMode", "babysitDaemons", "maxPods", "nvidiaGPUs", "dockerExecHandlerName", "podCIDR", "resolvConf", "cpuCFSQuota", "containerized", "maxOpenFiles", "reconcileCIDR", "registerSchedulable", "contentType", "kubeAPIQPS", "kubeAPIBurst", "serializeImagePulls", "outOfDiskTransitionFrequency", "nodeIP", "nodeLabels", "nonMasqueradeCIDR", "enableCustomMetrics", "evictionHard", "evictionSoft", "evictionSoftGracePeriod", "evictionPressureTransitionPeriod", "evictionMaxPodGracePeriod", "evictionMinimumReclaim", "podsPerCore", "enableControllerAttachDetach", "systemReserved", "kubeReserved", "protectKernelDefaults", "makeIPTablesUtilChains", "iptablesMasqueradeBit", "iptablesDropBit"},
},
},
Dependencies: []string{

View File

@ -72,7 +72,7 @@ func NewHollowKubelet(
TLSOptions: nil,
OOMAdjuster: oom.NewFakeOOMAdjuster(),
Writer: &kubeio.StdWriter{},
Mounter: mount.New("" /* default mount path */),
Mounter: mount.New(),
}
return &HollowKubelet{

View File

@ -94,13 +94,25 @@ func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string,
}
// New returns a mount.Interface for the current system.
func New(mounterPath string) Interface {
func New() Interface {
return &Mounter{}
}
// NewCustomMounter returns a mount.Interface for the current system.
// It provides options to override the default mounter behavior.
// mounterPath allows using an alternative to `/bin/mount` for mounting.
// mounterRootfsPath allows specifying a custom root filesystem path for non default `mounterPath`.
func NewCustomMounter(mounterPath, mounterRootfsPath string) Interface {
// If mounter-path flag is not set, use default mount path
if len(mounterPath) == 0 {
if mounterPath == "" {
mounterPath = mount
}
if mounterRootfsPath == "" {
mounterRootfsPath = "/"
}
return &Mounter{
mounterPath: mounterPath,
mounterPath: mounterPath,
mounterRootfsPath: mounterRootfsPath,
}
}

View File

@ -25,6 +25,7 @@ import (
"io"
"os"
"os/exec"
"path"
"strconv"
"strings"
"syscall"
@ -53,7 +54,8 @@ const (
// for the linux platform. This implementation assumes that the
// kubelet is running in the host's root mount namespace.
type Mounter struct {
mounterPath string
mounterPath string
mounterRootfsPath string
}
// Mount mounts source to target as fstype with given options. 'source' and 'fstype' must
@ -61,16 +63,17 @@ type Mounter struct {
// type, where kernel handles fs type for you. The mount 'options' is a list of options,
// currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is
// required, call Mount with an empty string list or nil.
// Update source path to include a root filesystem override to make a containerized mounter (specified via `mounterPath`) work.
func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
bind, bindRemountOpts := isBind(options)
if bind {
err := doMount(mounter.mounterPath, source, target, fstype, []string{"bind"})
err := doMount(mounter.mounterPath, path.Join(mounter.mounterRootfsPath, source), target, fstype, []string{"bind"})
if err != nil {
return err
}
return doMount(mounter.mounterPath, source, target, fstype, bindRemountOpts)
return doMount(mounter.mounterPath, path.Join(mounter.mounterRootfsPath, source), target, fstype, bindRemountOpts)
} else {
return doMount(mounter.mounterPath, source, target, fstype, options)
return doMount(mounter.mounterPath, path.Join(mounter.mounterRootfsPath, source), target, fstype, options)
}
}

View File

@ -119,6 +119,8 @@ type NodeTestContextType struct {
ContainerRuntimeEndpoint string
// MounterPath is the path to the program to run to perform a mount
MounterPath string
// MounterRootfsPath is the path to the root filesystem for the program used to perform a mount in kubelet
MounterRootfsPath string
}
type CloudConfig struct {
@ -216,6 +218,7 @@ func RegisterNodeFlags() {
flag.StringVar(&TestContext.RuntimeIntegrationType, "runtime-integration-type", "", "Choose the integration path for the container runtime, mainly used for CRI validation.")
flag.StringVar(&TestContext.ContainerRuntimeEndpoint, "container-runtime-endpoint", "", "The endpoint of remote container runtime grpc server, mainly used for Remote CRI validation.")
flag.StringVar(&TestContext.MounterPath, "experimental-mounter-path", "", "Path of mounter binary. Leave empty to use the default mount.")
flag.StringVar(&TestContext.MounterRootfsPath, "experimental-mounter-rootfs-path", "", "Absolute path to root filesystem for the mounter binary.")
}
// overwriteFlagsWithViperConfig finds and writes values to flags using viper as input.

View File

@ -42,9 +42,10 @@ var resultsDir = flag.String("results-dir", "/tmp/", "Directory to scp test resu
var sshOptionsMap map[string]string
const (
archiveName = "e2e_node_test.tar.gz"
CNIRelease = "07a8a28637e97b22eb8dfe710eeae1344f69d16e"
CNIDirectory = "cni"
archiveName = "e2e_node_test.tar.gz"
CNIRelease = "07a8a28637e97b22eb8dfe710eeae1344f69d16e"
CNIDirectory = "cni"
mounterRootfsPath string = "/"
)
var CNIURL = fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/network-plugins/cni-%s.tar.gz", CNIRelease)
@ -263,7 +264,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
return "", false, fmt.Errorf("Issue detecting node's OS via node's /etc/os-release. Err: %v, Output:\n%s", err, output)
}
if strings.Contains(output, "ID=gci") {
glog.Infof("GCI node and GCI mounter both detected, modifying --experimental-mounter-path accordingly")
glog.Infof("GCI node and GCI mounter both detected, modifying --mounter-path & --experimental-mounter-rootfs-path accordingly")
// Note this implicitly requires the script to be where we expect in the tarball, so if that location changes the error
// here will tell us to update the remote test runner.
@ -274,6 +275,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
return "", false, err
}
// Insert args at beginning of testArgs, so any values from command line take precedence
testArgs = fmt.Sprintf("--experimental-mounter-rootfs-path=%s ", mounterRootfsPath) + testArgs
testArgs = fmt.Sprintf("--experimental-mounter-path=%s ", mounterPath) + testArgs
}

View File

@ -212,6 +212,7 @@ func (e *E2EServices) startKubelet() (*server, error) {
"--feature-gates", framework.TestContext.FeatureGates,
"--v", LOG_VERBOSITY_LEVEL, "--logtostderr",
"--experimental-mounter-path", framework.TestContext.MounterPath,
"--experimental-mounter-rootfs-path", framework.TestContext.MounterRootfsPath,
)
if framework.TestContext.RuntimeIntegrationType != "" {