github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Mulligan: Remove deprecated and experimental fields from KubeletConfiguration 

Revert "Merge pull request #51857 from kubernetes/revert-51307-kc-type-refactor"

This reverts commit 9d27d92420, reversing
changes made to 2e69d4e625.

See original: #51307

We punted this from 1.8 so it could go through an API review. The point
of this PR is that we are trying to stabilize the kubeletconfig API so
that we can move it out of alpha, and unblock features like Dynamic
Kubelet Config, Kubelet loading its initial config from a file instead
of flags, kubeadm and other install tools having a versioned API to rely
on, etc.

We shouldn't rev the version without both removing all the deprecated
junk from the KubeletConfiguration struct, and without (at least
temporarily) removing all of the fields that have "Experimental" in
their names. It wouldn't make sense to lock in to deprecated fields.
"Experimental" fields can be audited on a 1-by-1 basis after this PR,
and if found to be stable (or sufficiently alpha-gated), can be restored
to the KubeletConfiguration without the "Experimental" prefix.
pull/6/head
Michael Taufen 2017-09-26 15:01:09 -07:00
parent 34ceb5bc97
commit 8180536bed
16 changed files with 275 additions and 350 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/kubelet/app/BUILD
View File

@ -102,6 +102,7 @@ go_library(
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",

143
cmd/kubelet/app/options/options.go
View File

@ -20,10 +20,12 @@ package options
import (
"fmt"
_ "net/http/pprof"
"runtime"
"strings"
"github.com/spf13/pflag"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/pkg/util/flag"
"k8s.io/kubernetes/pkg/apis/componentconfig"
@ -53,9 +55,6 @@ type KubeletFlags struct {
BootstrapKubeconfig string
RotateCertificates bool
// RequireKubeConfig is deprecated! A valid KubeConfig is now required if --kubeconfig is provided.
RequireKubeConfig bool
// Insert a probability of random errors during calls to the master.
ChaosChance float64
// Crash immediately, rather than eating panics.
@ -111,18 +110,86 @@ type KubeletFlags struct {
// Omit this flag to use the combination of built-in default configuration values and flags.
// To use this flag, the DynamicKubeletConfig feature gate must be enabled.
InitConfigDir flag.StringFlag
// EXPERIMENTAL FLAGS
// Whitelist of unsafe sysctls or sysctl patterns (ending in *).
// +optional
AllowedUnsafeSysctls []string
// containerized should be set to true if kubelet is running in a container.
Containerized bool
// remoteRuntimeEndpoint is the endpoint of remote runtime service
RemoteRuntimeEndpoint string
// remoteImageEndpoint is the endpoint of remote image service
RemoteImageEndpoint string
// experimentalMounterPath is the path of mounter binary. Leave empty to use the default mount path
ExperimentalMounterPath string
// If enabled, the kubelet will integrate with the kernel memcg notification to determine if memory eviction thresholds are crossed rather than polling.
// +optional
ExperimentalKernelMemcgNotification bool
// This flag, if set, enables a check prior to mount operations to verify that the required components
// (binaries, etc.) to mount the volume are available on the underlying node. If the check is enabled
// and fails the mount operation fails.
ExperimentalCheckNodeCapabilitiesBeforeMount bool
// This flag, if set, will avoid including `EvictionHard` limits while computing Node Allocatable.
// Refer to [Node Allocatable](https://git.k8s.io/community/contributors/design-proposals/node-allocatable.md) doc for more information.
ExperimentalNodeAllocatableIgnoreEvictionThreshold bool
// A set of ResourceName=Percentage (e.g. memory=50%) pairs that describe
// how pod resource requests are reserved at the QoS level.
// Currently only memory is supported. [default=none]"
ExperimentalQOSReserved kubeletconfig.ConfigurationMap
// DEPRECATED FLAGS
// minimumGCAge is the minimum age for a finished container before it is
// garbage collected.
MinimumGCAge metav1.Duration
// maxPerPodContainerCount is the maximum number of old instances to
// retain per container. Each container takes up some disk space.
MaxPerPodContainerCount int32
// maxContainerCount is the maximum number of old instances of containers
// to retain globally. Each container takes up some disk space.
MaxContainerCount int32
// masterServiceNamespace is The namespace from which the kubernetes
// master services should be injected into pods.
MasterServiceNamespace string
// registerSchedulable tells the kubelet to register the node as
// schedulable. Won't have any effect if register-node is false.
// DEPRECATED: use registerWithTaints instead
RegisterSchedulable bool
// RequireKubeConfig is deprecated! A valid KubeConfig is now required if --kubeconfig is provided.
RequireKubeConfig bool
// nonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.
NonMasqueradeCIDR string
// This flag, if set, instructs the kubelet to keep volumes from terminated pods mounted to the node.
// This can be useful for debugging volume related issues.
KeepTerminatedPodVolumes bool
}
// NewKubeletFlags will create a new KubeletFlags with default values
func NewKubeletFlags() *KubeletFlags {
remoteRuntimeEndpoint := ""
if runtime.GOOS == "linux" {
remoteRuntimeEndpoint = "unix:///var/run/dockershim.sock"
} else if runtime.GOOS == "windows" {
remoteRuntimeEndpoint = "tcp://localhost:3735"
}
return &KubeletFlags{
// TODO(#41161:v1.10.0): Remove the default kubeconfig path and --require-kubeconfig.
RequireKubeConfig: false,
KubeConfig: flag.NewStringFlag("/var/lib/kubelet/kubeconfig"),
ContainerRuntimeOptions: *NewContainerRuntimeOptions(),
CertDirectory: "/var/lib/kubelet/pki",
RootDirectory: v1alpha1.DefaultRootDir,
RotateCertificates: false,
RequireKubeConfig: false,
KubeConfig: flag.NewStringFlag("/var/lib/kubelet/kubeconfig"),
ContainerRuntimeOptions: *NewContainerRuntimeOptions(),
CertDirectory: "/var/lib/kubelet/pki",
RootDirectory: v1alpha1.DefaultRootDir,
MasterServiceNamespace: metav1.NamespaceDefault,
MaxContainerCount: -1,
MaxPerPodContainerCount: 1,
MinimumGCAge: metav1.Duration{Duration: 0},
NonMasqueradeCIDR: "10.0.0.0/8",
RegisterSchedulable: true,
ExperimentalKernelMemcgNotification: false,
ExperimentalQOSReserved: make(map[string]string),
RemoteRuntimeEndpoint: remoteRuntimeEndpoint,
RotateCertificates: false,
}
}
@ -228,6 +295,34 @@ func (f *KubeletFlags) AddFlags(fs *pflag.FlagSet) {
fs.Var(&f.DynamicConfigDir, "dynamic-config-dir", "The Kubelet will use this directory for checkpointing downloaded configurations and tracking configuration health. The Kubelet will create this directory if it does not already exist. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Providing this flag enables dynamic Kubelet configuration. Presently, you must also enable the DynamicKubeletConfig feature gate to pass this flag.")
fs.Var(&f.InitConfigDir, "init-config-dir", "The Kubelet will look in this directory for the init configuration. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Omit this argument to use the built-in default configuration values. Presently, you must also enable the DynamicKubeletConfig feature gate to pass this flag.")
// EXPERIMENTAL FLAGS
fs.StringVar(&f.ExperimentalMounterPath, "experimental-mounter-path", f.ExperimentalMounterPath, "[Experimental] Path of mounter binary. Leave empty to use the default mount.")
fs.StringSliceVar(&f.AllowedUnsafeSysctls, "experimental-allowed-unsafe-sysctls", f.AllowedUnsafeSysctls, "Comma-separated whitelist of unsafe sysctls or unsafe sysctl patterns (ending in *). Use these at your own risk.")
fs.BoolVar(&f.Containerized, "containerized", f.Containerized, "Experimental support for running kubelet in a container. Intended for testing.")
fs.BoolVar(&f.ExperimentalKernelMemcgNotification, "experimental-kernel-memcg-notification", f.ExperimentalKernelMemcgNotification, "If enabled, the kubelet will integrate with the kernel memcg notification to determine if memory eviction thresholds are crossed rather than polling.")
fs.StringVar(&f.RemoteRuntimeEndpoint, "container-runtime-endpoint", f.RemoteRuntimeEndpoint, "[Experimental] The endpoint of remote runtime service. Currently unix socket is supported on Linux, and tcp is supported on windows. Examples:'unix:///var/run/dockershim.sock', 'tcp://localhost:3735'")
fs.StringVar(&f.RemoteImageEndpoint, "image-service-endpoint", f.RemoteImageEndpoint, "[Experimental] The endpoint of remote image service. If not specified, it will be the same with container-runtime-endpoint by default. Currently unix socket is supported on Linux, and tcp is supported on windows. Examples:'unix:///var/run/dockershim.sock', 'tcp://localhost:3735'")
fs.BoolVar(&f.ExperimentalCheckNodeCapabilitiesBeforeMount, "experimental-check-node-capabilities-before-mount", f.ExperimentalCheckNodeCapabilitiesBeforeMount, "[Experimental] if set true, the kubelet will check the underlying node for required componenets (binaries, etc.) before performing the mount")
fs.BoolVar(&f.ExperimentalNodeAllocatableIgnoreEvictionThreshold, "experimental-allocatable-ignore-eviction", f.ExperimentalNodeAllocatableIgnoreEvictionThreshold, "When set to 'true', Hard Eviction Thresholds will be ignored while calculating Node Allocatable. See https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/ for more details. [default=false]")
fs.Var(&f.ExperimentalQOSReserved, "experimental-qos-reserved", "A set of ResourceName=Percentage (e.g. memory=50%) pairs that describe how pod resource requests are reserved at the QoS level. Currently only memory is supported. [default=none]")
// DEPRECATED FLAGS
fs.DurationVar(&f.MinimumGCAge.Duration, "minimum-container-ttl-duration", f.MinimumGCAge.Duration, "Minimum age for a finished container before it is garbage collected. Examples: '300ms', '10s' or '2h45m'")
fs.MarkDeprecated("minimum-container-ttl-duration", "Use --eviction-hard or --eviction-soft instead. Will be removed in a future version.")
fs.Int32Var(&f.MaxPerPodContainerCount, "maximum-dead-containers-per-container", f.MaxPerPodContainerCount, "Maximum number of old instances to retain per container. Each container takes up some disk space.")
fs.MarkDeprecated("maximum-dead-containers-per-container", "Use --eviction-hard or --eviction-soft instead. Will be removed in a future version.")
fs.Int32Var(&f.MaxContainerCount, "maximum-dead-containers", f.MaxContainerCount, "Maximum number of old instances of containers to retain globally. Each container takes up some disk space. To disable, set to a negative number.")
fs.MarkDeprecated("maximum-dead-containers", "Use --eviction-hard or --eviction-soft instead. Will be removed in a future version.")
fs.StringVar(&f.MasterServiceNamespace, "master-service-namespace", f.MasterServiceNamespace, "The namespace from which the kubernetes master services should be injected into pods")
fs.MarkDeprecated("master-service-namespace", "This flag will be removed in a future version.")
fs.BoolVar(&f.RegisterSchedulable, "register-schedulable", f.RegisterSchedulable, "Register the node as schedulable. Won't have any effect if register-node is false.")
fs.MarkDeprecated("register-schedulable", "will be removed in a future version")
fs.StringVar(&f.NonMasqueradeCIDR, "non-masquerade-cidr", f.NonMasqueradeCIDR, "Traffic to IPs outside this range will use IP masquerade. Set to '0.0.0.0/0' to never masquerade.")
fs.MarkDeprecated("non-masquerade-cidr", "will be removed in a future version")
fs.BoolVar(&f.KeepTerminatedPodVolumes, "keep-terminated-pod-volumes", f.KeepTerminatedPodVolumes, "Keep terminated pod volumes mounted to the node after the pod terminates. Can be useful for debugging volume related issues.")
fs.MarkDeprecated("keep-terminated-pod-volumes", "will be removed in a future version")
}
// AddKubeletConfigFlags adds flags for a specific kubeletconfig.KubeletConfiguration to the specified FlagSet
@ -287,20 +382,13 @@ func AddKubeletConfigFlags(fs *pflag.FlagSet, c *kubeletconfig.KubeletConfigurat
fs.BoolVar(&c.EnableDebuggingHandlers, "enable-debugging-handlers", c.EnableDebuggingHandlers, "Enables server endpoints for log collection and local running of containers and commands")
fs.BoolVar(&c.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled")
fs.DurationVar(&c.MinimumGCAge.Duration, "minimum-container-ttl-duration", c.MinimumGCAge.Duration, "Minimum age for a finished container before it is garbage collected. Examples: '300ms', '10s' or '2h45m'")
fs.MarkDeprecated("minimum-container-ttl-duration", "Use --eviction-hard or --eviction-soft instead. Will be removed in a future version.")
fs.Int32Var(&c.MaxPerPodContainerCount, "maximum-dead-containers-per-container", c.MaxPerPodContainerCount, "Maximum number of old instances to retain per container. Each container takes up some disk space.")
fs.MarkDeprecated("maximum-dead-containers-per-container", "Use --eviction-hard or --eviction-soft instead. Will be removed in a future version.")
fs.Int32Var(&c.MaxContainerCount, "maximum-dead-containers", c.MaxContainerCount, "Maximum number of old instances of containers to retain globally. Each container takes up some disk space. To disable, set to a negative number.")
fs.MarkDeprecated("maximum-dead-containers", "Use --eviction-hard or --eviction-soft instead. Will be removed in a future version.")
fs.Int32Var(&c.CAdvisorPort, "cadvisor-port", c.CAdvisorPort, "The port of the localhost cAdvisor endpoint (set to 0 to disable)")
fs.Int32Var(&c.HealthzPort, "healthz-port", c.HealthzPort, "The port of the localhost healthz endpoint (set to 0 to disable)")
fs.Var(componentconfig.IPVar{Val: &c.HealthzBindAddress}, "healthz-bind-address", "The IP address for the healthz server to serve on. (set to 0.0.0.0 for all interfaces)")
fs.Int32Var(&c.OOMScoreAdj, "oom-score-adj", c.OOMScoreAdj, "The oom-score-adj value for kubelet process. Values must be within the range [-1000, 1000]")
fs.BoolVar(&c.RegisterNode, "register-node", c.RegisterNode, "Register the node with the apiserver. If --kubeconfig is not provided, this flag is irrelevant, as the Kubelet won't have an apiserver to register with. Default=true.")
fs.StringVar(&c.ClusterDomain, "cluster-domain", c.ClusterDomain, "Domain for this cluster. If set, kubelet will configure all containers to search this domain in addition to the host's search domains")
fs.StringVar(&c.MasterServiceNamespace, "master-service-namespace", c.MasterServiceNamespace, "The namespace from which the kubernetes master services should be injected into pods")
fs.MarkDeprecated("master-service-namespace", "This flag will be removed in a future version.")
fs.StringSliceVar(&c.ClusterDNS, "cluster-dns", c.ClusterDNS, "Comma-separated list of DNS server IP address. This value is used for containers DNS server in case of Pods with \"dnsPolicy=ClusterFirst\". Note: all DNS servers appearing in the list MUST serve the same set of records otherwise name resolution within the cluster may not work correctly. There is no guarantee as to which DNS server may be contacted for name resolution.")
fs.DurationVar(&c.StreamingConnectionIdleTimeout.Duration, "streaming-connection-idle-timeout", c.StreamingConnectionIdleTimeout.Duration, "Maximum time a streaming connection can be idle before the connection is automatically closed. 0 indicates no timeout. Example: '5m'")
fs.DurationVar(&c.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", c.NodeStatusUpdateFrequency.Duration, "Specifies how often kubelet posts node status to master. Note: be cautious when changing the constant, it must work with nodeMonitorGracePeriod in nodecontroller.")
@ -326,11 +414,9 @@ func AddKubeletConfigFlags(fs *pflag.FlagSet, c *kubeletconfig.KubeletConfigurat
fs.DurationVar(&c.RuntimeRequestTimeout.Duration, "runtime-request-timeout", c.RuntimeRequestTimeout.Duration, "Timeout of all runtime requests except long running request - pull, logs, exec and attach. When timeout exceeded, kubelet will cancel the request, throw out an error and retry later.")
fs.StringVar(&c.LockFilePath, "lock-file", c.LockFilePath, "<Warning: Alpha feature> The path to file for kubelet to use as a lock file.")
fs.BoolVar(&c.ExitOnLockContention, "exit-on-lock-contention", c.ExitOnLockContention, "Whether kubelet should exit upon lock-file contention.")
fs.StringVar(&c.ExperimentalMounterPath, "experimental-mounter-path", c.ExperimentalMounterPath, "[Experimental] Path of mounter binary. Leave empty to use the default mount.")
fs.StringVar(&c.HairpinMode, "hairpin-mode", c.HairpinMode, "How should the kubelet setup hairpin NAT. This allows endpoints of a Service to loadbalance back to themselves if they should try to access their own Service. Valid values are \"promiscuous-bridge\", \"hairpin-veth\" and \"none\".")
fs.Int32Var(&c.MaxPods, "max-pods", c.MaxPods, "Number of Pods that can run on this Kubelet.")
fs.StringVar(&c.NonMasqueradeCIDR, "non-masquerade-cidr", c.NonMasqueradeCIDR, "Traffic to IPs outside this range will use IP masquerade. Set to '0.0.0.0/0' to never masquerade.")
fs.MarkDeprecated("non-masquerade-cidr", "will be removed in a future version")
fs.StringVar(&c.PodCIDR, "pod-cidr", "", "The CIDR to use for pod IP addresses, only used in standalone mode. In cluster mode, this is obtained from the master.")
fs.StringVar(&c.ResolverConfig, "resolv-conf", c.ResolverConfig, "Resolver configuration file used as the basis for the container DNS resolution configuration.")
fs.BoolVar(&c.CPUCFSQuota, "cpu-cfs-quota", c.CPUCFSQuota, "Enable CPU CFS quota enforcement for containers that specify CPU limits")
@ -338,13 +424,10 @@ func AddKubeletConfigFlags(fs *pflag.FlagSet, c *kubeletconfig.KubeletConfigurat
fs.BoolVar(&c.MakeIPTablesUtilChains, "make-iptables-util-chains", c.MakeIPTablesUtilChains, "If true, kubelet will ensure iptables utility rules are present on host.")
fs.Int32Var(&c.IPTablesMasqueradeBit, "iptables-masquerade-bit", c.IPTablesMasqueradeBit, "The bit of the fwmark space to mark packets for SNAT. Must be within the range [0, 31]. Please match this parameter with corresponding parameter in kube-proxy.")
fs.Int32Var(&c.IPTablesDropBit, "iptables-drop-bit", c.IPTablesDropBit, "The bit of the fwmark space to mark packets for dropping. Must be within the range [0, 31].")
fs.StringSliceVar(&c.AllowedUnsafeSysctls, "experimental-allowed-unsafe-sysctls", c.AllowedUnsafeSysctls, "Comma-separated whitelist of unsafe sysctls or unsafe sysctl patterns (ending in *). Use these at your own risk.")
// Flags intended for testing, not recommended used in production environments.
fs.BoolVar(&c.Containerized, "containerized", c.Containerized, "Experimental support for running kubelet in a container. Intended for testing.")
fs.Int64Var(&c.MaxOpenFiles, "max-open-files", c.MaxOpenFiles, "Number of files that can be opened by Kubelet process.")
fs.BoolVar(&c.RegisterSchedulable, "register-schedulable", c.RegisterSchedulable, "Register the node as schedulable. Won't have any effect if register-node is false.")
fs.MarkDeprecated("register-schedulable", "will be removed in a future version")
fs.Var(utiltaints.NewTaintsVar(&c.RegisterWithTaints), "register-with-taints", "Register the node with the given list of taints (comma separated \"<key>=<value>:<effect>\"). No-op if register-node is false.")
fs.StringVar(&c.ContentType, "kube-api-content-type", c.ContentType, "Content type of requests sent to apiserver.")
fs.Int32Var(&c.KubeAPIQPS, "kube-api-qps", c.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver")
@ -359,17 +442,8 @@ func AddKubeletConfigFlags(fs *pflag.FlagSet, c *kubeletconfig.KubeletConfigurat
fs.DurationVar(&c.EvictionPressureTransitionPeriod.Duration, "eviction-pressure-transition-period", c.EvictionPressureTransitionPeriod.Duration, "Duration for which the kubelet has to wait before transitioning out of an eviction pressure condition.")
fs.Int32Var(&c.EvictionMaxPodGracePeriod, "eviction-max-pod-grace-period", c.EvictionMaxPodGracePeriod, "Maximum allowed grace period (in seconds) to use when terminating pods in response to a soft eviction threshold being met. If negative, defer to pod specified value.")
fs.StringVar(&c.EvictionMinimumReclaim, "eviction-minimum-reclaim", c.EvictionMinimumReclaim, "A set of minimum reclaims (e.g. imagefs.available=2Gi) that describes the minimum amount of resource the kubelet will reclaim when performing a pod eviction if that resource is under pressure.")
fs.BoolVar(&c.ExperimentalKernelMemcgNotification, "experimental-kernel-memcg-notification", c.ExperimentalKernelMemcgNotification, "If enabled, the kubelet will integrate with the kernel memcg notification to determine if memory eviction thresholds are crossed rather than polling.")
fs.Int32Var(&c.PodsPerCore, "pods-per-core", c.PodsPerCore, "Number of Pods per core that can run on this Kubelet. The total number of Pods on this Kubelet cannot exceed max-pods, so max-pods will be used if this calculation results in a larger number of Pods allowed on the Kubelet. A value of 0 disables this limit.")
fs.BoolVar(&c.ProtectKernelDefaults, "protect-kernel-defaults", c.ProtectKernelDefaults, "Default kubelet behaviour for kernel tuning. If set, kubelet errors if any of kernel tunables is different than kubelet defaults.")
fs.BoolVar(&c.KeepTerminatedPodVolumes, "keep-terminated-pod-volumes", c.KeepTerminatedPodVolumes, "Keep terminated pod volumes mounted to the node after the pod terminates. Can be useful for debugging volume related issues.")
fs.MarkDeprecated("keep-terminated-pod-volumes", "will be removed in a future version")
// CRI flags.
fs.StringVar(&c.RemoteRuntimeEndpoint, "container-runtime-endpoint", c.RemoteRuntimeEndpoint, "[Experimental] The endpoint of remote runtime service. Currently unix socket is supported on Linux, and tcp is supported on windows. Examples:'unix:///var/run/dockershim.sock', 'tcp://localhost:3735'")
fs.StringVar(&c.RemoteImageEndpoint, "image-service-endpoint", c.RemoteImageEndpoint, "[Experimental] The endpoint of remote image service. If not specified, it will be the same with container-runtime-endpoint by default. Currently unix socket is supported on Linux, and tcp is supported on windows. Examples:'unix:///var/run/dockershim.sock', 'tcp://localhost:3735'")
fs.BoolVar(&c.ExperimentalCheckNodeCapabilitiesBeforeMount, "experimental-check-node-capabilities-before-mount", c.ExperimentalCheckNodeCapabilitiesBeforeMount, "[Experimental] if set true, the kubelet will check the underlying node for required componenets (binaries, etc.) before performing the mount")
// Node Allocatable Flags
fs.Var(&c.SystemReserved, "system-reserved", "A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=500Mi) pairs that describe resources reserved for non-kubernetes components. Currently only cpu and memory are supported. See http://kubernetes.io/docs/user-guide/compute-resources for more detail. [default=none]")
@ -377,7 +451,4 @@ func AddKubeletConfigFlags(fs *pflag.FlagSet, c *kubeletconfig.KubeletConfigurat
fs.StringSliceVar(&c.EnforceNodeAllocatable, "enforce-node-allocatable", c.EnforceNodeAllocatable, "A comma separated list of levels of node allocatable enforcement to be enforced by kubelet. Acceptible options are 'pods', 'system-reserved' & 'kube-reserved'. If the latter two options are specified, '--system-reserved-cgroup' & '--kube-reserved-cgroup' must also be set respectively. See https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/ for more details.")
fs.StringVar(&c.SystemReservedCgroup, "system-reserved-cgroup", c.SystemReservedCgroup, "Absolute name of the top level cgroup that is used to manage non-kubernetes components for which compute resources were reserved via '--system-reserved' flag. Ex. '/system-reserved'. [default='']")
fs.StringVar(&c.KubeReservedCgroup, "kube-reserved-cgroup", c.KubeReservedCgroup, "Absolute name of the top level cgroup that is used to manage kubernetes components for which compute resources were reserved via '--kube-reserved' flag. Ex. '/kube-reserved'. [default='']")
fs.BoolVar(&c.ExperimentalNodeAllocatableIgnoreEvictionThreshold, "experimental-allocatable-ignore-eviction", c.ExperimentalNodeAllocatableIgnoreEvictionThreshold, "When set to 'true', Hard Eviction Thresholds will be ignored while calculating Node Allocatable. See https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/ for more details. [default=false]")
fs.Var(&c.ExperimentalQOSReserved, "experimental-qos-reserved", "A set of ResourceName=Percentage (e.g. memory=50%) pairs that describe how pod resource requests are reserved at the QoS level. Currently only memory is supported. [default=none]")
}

85
cmd/kubelet/app/server.go
View File

@ -38,6 +38,7 @@ import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
@ -696,7 +697,30 @@ func RunKubelet(kubeFlags *options.KubeletFlags, kubeCfg *kubeletconfiginternal.
kubeDeps.OSInterface = kubecontainer.RealOS{}
}
k, err := builder(kubeCfg, kubeDeps, &kubeFlags.ContainerRuntimeOptions, kubeFlags.HostnameOverride, kubeFlags.NodeIP, kubeFlags.ProviderID, kubeFlags.CloudProvider, kubeFlags.CertDirectory, kubeFlags.RootDirectory)
k, err := builder(kubeCfg,
kubeDeps,
&kubeFlags.ContainerRuntimeOptions,
kubeFlags.HostnameOverride,
kubeFlags.NodeIP,
kubeFlags.ProviderID,
kubeFlags.CloudProvider,
kubeFlags.CertDirectory,
kubeFlags.RootDirectory,
kubeFlags.AllowedUnsafeSysctls,
kubeFlags.Containerized,
kubeFlags.RemoteRuntimeEndpoint,
kubeFlags.RemoteImageEndpoint,
kubeFlags.ExperimentalMounterPath,
kubeFlags.ExperimentalKernelMemcgNotification,
kubeFlags.ExperimentalCheckNodeCapabilitiesBeforeMount,
kubeFlags.ExperimentalNodeAllocatableIgnoreEvictionThreshold,
kubeFlags.MinimumGCAge,
kubeFlags.MaxPerPodContainerCount,
kubeFlags.MaxContainerCount,
kubeFlags.MasterServiceNamespace,
kubeFlags.RegisterSchedulable,
kubeFlags.NonMasqueradeCIDR,
kubeFlags.KeepTerminatedPodVolumes)
if err != nil {
return fmt.Errorf("failed to create kubelet: %v", err)
}
@ -743,16 +767,54 @@ func startKubelet(k kubelet.Bootstrap, podCfg *config.PodConfig, kubeCfg *kubele
func CreateAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
kubeDeps *kubelet.Dependencies,
crOptions *options.ContainerRuntimeOptions,
hostnameOverride,
nodeIP,
providerID,
cloudProvider,
certDirectory,
rootDirectory string) (k kubelet.Bootstrap, err error) {
hostnameOverride string,
nodeIP string,
providerID string,
cloudProvider string,
certDirectory string,
rootDirectory string,
allowedUnsafeSysctls []string,
containerized bool,
remoteRuntimeEndpoint string,
remoteImageEndpoint string,
experimentalMounterPath string,
experimentalKernelMemcgNotification bool,
experimentalCheckNodeCapabilitiesBeforeMount bool,
experimentalNodeAllocatableIgnoreEvictionThreshold bool,
minimumGCAge metav1.Duration,
maxPerPodContainerCount int32,
maxContainerCount int32,
masterServiceNamespace string,
registerSchedulable bool,
nonMasqueradeCIDR string,
keepTerminatedPodVolumes bool) (k kubelet.Bootstrap, err error) {
// TODO: block until all sources have delivered at least one update to the channel, or break the sync loop
// up into "per source" synchronizations
k, err = kubelet.NewMainKubelet(kubeCfg, kubeDeps, crOptions, hostnameOverride, nodeIP, providerID, cloudProvider, certDirectory, rootDirectory)
k, err = kubelet.NewMainKubelet(kubeCfg,
kubeDeps,
crOptions,
hostnameOverride,
nodeIP,
providerID,
cloudProvider,
certDirectory,
rootDirectory,
allowedUnsafeSysctls,
containerized,
remoteRuntimeEndpoint,
remoteImageEndpoint,
experimentalMounterPath,
experimentalKernelMemcgNotification,
experimentalCheckNodeCapabilitiesBeforeMount,
experimentalNodeAllocatableIgnoreEvictionThreshold,
minimumGCAge,
maxPerPodContainerCount,
maxContainerCount,
masterServiceNamespace,
registerSchedulable,
nonMasqueradeCIDR,
keepTerminatedPodVolumes)
if err != nil {
return nil, err
}
@ -826,7 +888,8 @@ func BootstrapKubeletConfigController(defaultConfig *kubeletconfiginternal.Kubel
// RunDockershim only starts the dockershim in current process. This is only used for cri validate testing purpose
// TODO(random-liu): Move this to a separate binary.
func RunDockershim(c *kubeletconfiginternal.KubeletConfiguration, r *options.ContainerRuntimeOptions) error {
func RunDockershim(f *options.KubeletFlags, c *kubeletconfiginternal.KubeletConfiguration) error {
r := &f.ContainerRuntimeOptions
// Create docker client.
dockerClient := libdocker.ConnectToDockerOrDie(r.DockerEndpoint, c.RuntimeRequestTimeout.Duration,
r.ImagePullProgressDeadline.Duration)
@ -839,7 +902,7 @@ func RunDockershim(c *kubeletconfiginternal.KubeletConfiguration, r *options.Con
nh := &kubelet.NoOpLegacyHost{}
pluginSettings := dockershim.NetworkPluginSettings{
HairpinMode: kubeletconfiginternal.HairpinMode(c.HairpinMode),
NonMasqueradeCIDR: c.NonMasqueradeCIDR,
NonMasqueradeCIDR: f.NonMasqueradeCIDR,
PluginName: r.NetworkPluginName,
PluginConfDir: r.CNIConfDir,
PluginBinDir: binDir,
@ -867,7 +930,7 @@ func RunDockershim(c *kubeletconfiginternal.KubeletConfiguration, r *options.Con
}
glog.V(2).Infof("Starting the GRPC server for the docker CRI shim.")
server := dockerremote.NewDockerServer(c.RemoteRuntimeEndpoint, ds)
server := dockerremote.NewDockerServer(f.RemoteRuntimeEndpoint, ds)
if err := server.Start(); err != nil {
return err
}

2
cmd/kubelet/kubelet.go
View File

@ -97,7 +97,7 @@ func main() {
// start the experimental docker shim, if enabled
if kubeletFlags.ExperimentalDockershim {
if err := app.RunDockershim(kubeletConfig, &kubeletFlags.ContainerRuntimeOptions); err != nil {
if err := app.RunDockershim(kubeletFlags, kubeletConfig); err != nil {
die(err)
}
}

53
pkg/kubelet/apis/kubeletconfig/types.go
View File

@ -45,13 +45,10 @@ const (
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// A configuration field should go in KubeletFlags instead of KubeletConfiguration if any of these are true:
// - its value will never, or cannot safely be changed during the lifetime of a node
// - its value cannot be safely shared between nodes at the same time (e.g. a hostname)
// KubeletConfiguration is intended to be shared between nodes
// A configuration field should go in KubeletFlags instead of KubeletConfiguration if
// its value cannot be safely shared between nodes at the same time (e.g. a hostname)
// In general, please try to avoid adding flags or configuration fields,
// we already have a confusingly large amount of them.
// TODO: curate the ordering and structure of this config object
type KubeletConfiguration struct {
metav1.TypeMeta
@ -131,15 +128,6 @@ type KubeletConfiguration struct {
EnableDebuggingHandlers bool
// enableContentionProfiling enables lock contention profiling, if enableDebuggingHandlers is true.
EnableContentionProfiling bool
// minimumGCAge is the minimum age for a finished container before it is
// garbage collected.
MinimumGCAge metav1.Duration
// maxPerPodContainerCount is the maximum number of old instances to
// retain per container. Each container takes up some disk space.
MaxPerPodContainerCount int32
// maxContainerCount is the maximum number of old instances of containers
// to retain globally. Each container takes up some disk space.
MaxContainerCount int32
// cAdvisorPort is the port of the localhost cAdvisor endpoint (set to 0 to disable)
CAdvisorPort int32
// healthzPort is the port of the localhost healthz endpoint (set to 0 to disable)
@ -156,9 +144,6 @@ type KubeletConfiguration struct {
// configure all containers to search this domain in addition to the
// host's search domains.
ClusterDomain string
// masterServiceNamespace is The namespace from which the kubernetes
// master services should be injected into pods.
MasterServiceNamespace string
// clusterDNS is a list of IP address for a cluster DNS server. If set,
// kubelet will configure all containers to use this for DNS resolution
// instead of the host's DNS servers
@ -210,10 +195,6 @@ type KubeletConfiguration struct {
CgroupRoot string
// containerRuntime is the container runtime to use.
ContainerRuntime string
// remoteRuntimeEndpoint is the endpoint of remote runtime service
RemoteRuntimeEndpoint string
// remoteImageEndpoint is the endpoint of remote image service
RemoteImageEndpoint string
// CPUManagerPolicy is the name of the policy to use.
CPUManagerPolicy string
// CPU Manager reconciliation period.
@ -222,8 +203,6 @@ type KubeletConfiguration struct {
// requests - pull, logs, exec and attach.
// +optional
RuntimeRequestTimeout metav1.Duration
// experimentalMounterPath is the path of mounter binary. Leave empty to use the default mount path
ExperimentalMounterPath string
// lockFilePath is the path that kubelet will use to as a lock file.
// It uses this file as a lock to synchronize with other kubelet processes
// that may be running.
@ -253,14 +232,8 @@ type KubeletConfiguration struct {
// cpuCFSQuota is Enable CPU CFS quota enforcement for containers that
// specify CPU limits
CPUCFSQuota bool
// containerized should be set to true if kubelet is running in a container.
Containerized bool
// maxOpenFiles is Number of files that can be opened by Kubelet process.
MaxOpenFiles int64
// registerSchedulable tells the kubelet to register the node as
// schedulable. Won't have any effect if register-node is false.
// DEPRECATED: use registerWithTaints instead
RegisterSchedulable bool
// registerWithTaints are an array of taints to add to a node object when
// the kubelet registers itself. This only takes effect when registerNode
// is true and upon the initial registration of the node.
@ -279,8 +252,6 @@ type KubeletConfiguration struct {
SerializeImagePulls bool
// nodeLabels to add when registering the node in the cluster.
NodeLabels map[string]string
// nonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.
NonMasqueradeCIDR string
// enable gathering custom metrics.
EnableCustomMetrics bool
// Comma-delimited list of hard eviction expressions. For example, 'memory.available<300Mi'.
@ -301,19 +272,12 @@ type KubeletConfiguration struct {
// Comma-delimited list of minimum reclaims (e.g. imagefs.available=2Gi) that describes the minimum amount of resource the kubelet will reclaim when performing a pod eviction if that resource is under pressure.
// +optional
EvictionMinimumReclaim string
// If enabled, the kubelet will integrate with the kernel memcg notification to determine if memory eviction thresholds are crossed rather than polling.
// +optional
ExperimentalKernelMemcgNotification bool
// Maximum number of pods per core. Cannot exceed MaxPods
PodsPerCore int32
// enableControllerAttachDetach enables the Attach/Detach controller to
// manage attachment/detachment of volumes scheduled to this node, and
// disables kubelet from executing any attach/detach operations
EnableControllerAttachDetach bool
// A set of ResourceName=Percentage (e.g. memory=50%) pairs that describe
// how pod resource requests are reserved at the QoS level.
// Currently only memory is supported. [default=none]"
ExperimentalQOSReserved ConfigurationMap
// Default behaviour for kernel tuning
ProtectKernelDefaults bool
// If true, Kubelet ensures a set of iptables rules are present on host.
@ -328,20 +292,10 @@ type KubeletConfiguration struct {
// iptablesDropBit is the bit of the iptables fwmark space to use for dropping packets. Kubelet will ensure iptables mark and drop rules.
// Values must be within the range [0, 31]. Must be different from IPTablesMasqueradeBit
IPTablesDropBit int32
// Whitelist of unsafe sysctls or sysctl patterns (ending in *).
// +optional
AllowedUnsafeSysctls []string
// featureGates is a map of feature names to bools that enable or disable alpha/experimental features.
FeatureGates map[string]bool
// Tells the Kubelet to fail to start if swap is enabled on the node.
FailSwapOn bool
// This flag, if set, enables a check prior to mount operations to verify that the required components
// (binaries, etc.) to mount the volume are available on the underlying node. If the check is enabled
// and fails the mount operation fails.
ExperimentalCheckNodeCapabilitiesBeforeMount bool
// This flag, if set, instructs the kubelet to keep volumes from terminated pods mounted to the node.
// This can be useful for debugging volume related issues.
KeepTerminatedPodVolumes bool
/* following flags are meant for Node Allocatable */
@ -365,9 +319,6 @@ type KubeletConfiguration struct {
// This flag accepts a list of options. Acceptable options are `pods`, `system-reserved` & `kube-reserved`.
// Refer to [Node Allocatable](https://git.k8s.io/community/contributors/design-proposals/node/node-allocatable.md) doc for more information.
EnforceNodeAllocatable []string
// This flag, if set, will avoid including `EvictionHard` limits while computing Node Allocatable.
// Refer to [Node Allocatable](https://git.k8s.io/community/contributors/design-proposals/node/node-allocatable.md) doc for more information.
ExperimentalNodeAllocatableIgnoreEvictionThreshold bool
}
type KubeletAuthorizationMode string

33
pkg/kubelet/apis/kubeletconfig/v1alpha1/defaults.go
View File

@ -18,7 +18,6 @@ package v1alpha1
import (
"path/filepath"
"runtime"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -143,28 +142,12 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
temp := int32(80)
obj.ImageGCLowThresholdPercent = &temp
}
if obj.MasterServiceNamespace == "" {
obj.MasterServiceNamespace = metav1.NamespaceDefault
}
if obj.MaxContainerCount == nil {
temp := int32(-1)
obj.MaxContainerCount = &temp
}
if obj.MaxPerPodContainerCount == 0 {
obj.MaxPerPodContainerCount = 1
}
if obj.MaxOpenFiles == 0 {
obj.MaxOpenFiles = 1000000
}
if obj.MaxPods == 0 {
obj.MaxPods = 110
}
if obj.MinimumGCAge == zeroDuration {
obj.MinimumGCAge = metav1.Duration{Duration: 0}
}
if obj.NonMasqueradeCIDR == "" {
obj.NonMasqueradeCIDR = "10.0.0.0/8"
}
if obj.VolumePluginDir == "" {
obj.VolumePluginDir = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/"
}
@ -190,9 +173,6 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
if obj.RegisterNode == nil {
obj.RegisterNode = boolVar(true)
}
if obj.RegisterSchedulable == nil {
obj.RegisterSchedulable = boolVar(true)
}
if obj.RegistryBurst == 0 {
obj.RegistryBurst = 10
}
@ -235,18 +215,12 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
if obj.EvictionPressureTransitionPeriod == zeroDuration {
obj.EvictionPressureTransitionPeriod = metav1.Duration{Duration: 5 * time.Minute}
}
if obj.ExperimentalKernelMemcgNotification == nil {
obj.ExperimentalKernelMemcgNotification = boolVar(false)
}
if obj.SystemReserved == nil {
obj.SystemReserved = make(map[string]string)
}
if obj.KubeReserved == nil {
obj.KubeReserved = make(map[string]string)
}
if obj.ExperimentalQOSReserved == nil {
obj.ExperimentalQOSReserved = make(map[string]string)
}
if obj.MakeIPTablesUtilChains == nil {
obj.MakeIPTablesUtilChains = boolVar(true)
}
@ -268,13 +242,6 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
if obj.EnforceNodeAllocatable == nil {
obj.EnforceNodeAllocatable = defaultNodeAllocatableEnforcement
}
if obj.RemoteRuntimeEndpoint == "" {
if runtime.GOOS == "linux" {
obj.RemoteRuntimeEndpoint = "unix:///var/run/dockershim.sock"
} else if runtime.GOOS == "windows" {
obj.RemoteRuntimeEndpoint = "tcp://localhost:3735"
}
}
if obj.FeatureGates == nil {
obj.FeatureGates = make(map[string]bool)
}

53
pkg/kubelet/apis/kubeletconfig/v1alpha1/types.go
View File

@ -41,10 +41,8 @@ const (
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// A configuration field should go in KubeletFlags instead of KubeletConfiguration if any of these are true:
// - its value will never, or cannot safely be changed during the lifetime of a node
// - its value cannot be safely shared between nodes at the same time (e.g. a hostname)
// KubeletConfiguration is intended to be shared between nodes
// A configuration field should go in KubeletFlags instead of KubeletConfiguration if
// its value cannot be safely shared between nodes at the same time (e.g. a hostname)
// In general, please try to avoid adding flags or configuration fields,
// we already have a confusingly large amount of them.
type KubeletConfiguration struct {
@ -126,15 +124,6 @@ type KubeletConfiguration struct {
EnableDebuggingHandlers *bool `json:"enableDebuggingHandlers"`
// enableContentionProfiling enables lock contention profiling, if enableDebuggingHandlers is true.
EnableContentionProfiling bool `json:"enableContentionProfiling"`
// minimumGCAge is the minimum age for a finished container before it is
// garbage collected.
MinimumGCAge metav1.Duration `json:"minimumGCAge"`
// maxPerPodContainerCount is the maximum number of old instances to
// retain per container. Each container takes up some disk space.
MaxPerPodContainerCount int32 `json:"maxPerPodContainerCount"`
// maxContainerCount is the maximum number of old instances of containers
// to retain globally. Each container takes up some disk space.
MaxContainerCount *int32 `json:"maxContainerCount"`
// cAdvisorPort is the port of the localhost cAdvisor endpoint (set to 0 to disable)
CAdvisorPort *int32 `json:"cAdvisorPort"`
// healthzPort is the port of the localhost healthz endpoint (set to 0 to disable)
@ -151,9 +140,6 @@ type KubeletConfiguration struct {
// configure all containers to search this domain in addition to the
// host's search domains.
ClusterDomain string `json:"clusterDomain"`
// masterServiceNamespace is The namespace from which the kubernetes
// master services should be injected into pods.
MasterServiceNamespace string `json:"masterServiceNamespace"`
// clusterDNS is a list of IP address for the cluster DNS server. If set,
// kubelet will configure all containers to use this for DNS resolution
// instead of the host's DNS servers
@ -202,10 +188,6 @@ type KubeletConfiguration struct {
CgroupDriver string `json:"cgroupDriver,omitempty"`
// containerRuntime is the container runtime to use.
ContainerRuntime string `json:"containerRuntime"`
// remoteRuntimeEndpoint is the endpoint of remote runtime service
RemoteRuntimeEndpoint string `json:"remoteRuntimeEndpoint"`
// remoteImageEndpoint is the endpoint of remote image service
RemoteImageEndpoint string `json:"remoteImageEndpoint"`
// CPUManagerPolicy is the name of the policy to use.
CPUManagerPolicy string `json:"cpuManagerPolicy"`
// CPU Manager reconciliation period.
@ -213,9 +195,6 @@ type KubeletConfiguration struct {
// runtimeRequestTimeout is the timeout for all runtime requests except long running
// requests - pull, logs, exec and attach.
RuntimeRequestTimeout metav1.Duration `json:"runtimeRequestTimeout"`
// experimentalMounterPath is the path to mounter binary. If not set, kubelet will attempt to use mount
// binary that is available via $PATH,
ExperimentalMounterPath string `json:"experimentalMounterPath,omitempty"`
// lockFilePath is the path that kubelet will use to as a lock file.
// It uses this file as a lock to synchronize with other kubelet processes
// that may be running.
@ -245,14 +224,8 @@ type KubeletConfiguration struct {
// cpuCFSQuota is Enable CPU CFS quota enforcement for containers that
// specify CPU limits
CPUCFSQuota *bool `json:"cpuCFSQuota"`
// containerized should be set to true if kubelet is running in a container.
Containerized *bool `json:"containerized"`
// maxOpenFiles is Number of files that can be opened by Kubelet process.
MaxOpenFiles int64 `json:"maxOpenFiles"`
// registerSchedulable tells the kubelet to register the node as
// schedulable. Won't have any effect if register-node is false.
// DEPRECATED: use registerWithTaints instead
RegisterSchedulable *bool `json:"registerSchedulable"`
// registerWithTaints are an array of taints to add to a node object when
// the kubelet registers itself. This only takes effect when registerNode
// is true and upon the initial registration of the node.
@ -271,8 +244,6 @@ type KubeletConfiguration struct {
SerializeImagePulls *bool `json:"serializeImagePulls"`
// nodeLabels to add when registering the node in the cluster.
NodeLabels map[string]string `json:"nodeLabels"`
// nonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.
NonMasqueradeCIDR string `json:"nonMasqueradeCIDR"`
// enable gathering custom metrics.
EnableCustomMetrics bool `json:"enableCustomMetrics"`
// Comma-delimited list of hard eviction expressions. For example, 'memory.available<300Mi'.
@ -287,18 +258,12 @@ type KubeletConfiguration struct {
EvictionMaxPodGracePeriod int32 `json:"evictionMaxPodGracePeriod"`
// Comma-delimited list of minimum reclaims (e.g. imagefs.available=2Gi) that describes the minimum amount of resource the kubelet will reclaim when performing a pod eviction if that resource is under pressure.
EvictionMinimumReclaim string `json:"evictionMinimumReclaim"`
// If enabled, the kubelet will integrate with the kernel memcg notification to determine if memory eviction thresholds are crossed rather than polling.
ExperimentalKernelMemcgNotification *bool `json:"experimentalKernelMemcgNotification"`
// Maximum number of pods per core. Cannot exceed MaxPods
PodsPerCore int32 `json:"podsPerCore"`
// enableControllerAttachDetach enables the Attach/Detach controller to
// manage attachment/detachment of volumes scheduled to this node, and
// disables kubelet from executing any attach/detach operations
EnableControllerAttachDetach *bool `json:"enableControllerAttachDetach"`
// A set of ResourceName=Percentage (e.g. memory=50%) pairs that describe
// how pod resource requests are reserved at the QoS level.
// Currently only memory is supported. [default=none]"
ExperimentalQOSReserved map[string]string `json:"experimentalQOSReserved"`
// Default behaviour for kernel tuning
ProtectKernelDefaults bool `json:"protectKernelDefaults"`
// If true, Kubelet ensures a set of iptables rules are present on host.
@ -313,21 +278,10 @@ type KubeletConfiguration struct {
// iptablesDropBit is the bit of the iptables fwmark space to mark for dropping packets.
// Values must be within the range [0, 31]. Must be different from other mark bits.
IPTablesDropBit *int32 `json:"iptablesDropBit"`
// Whitelist of unsafe sysctls or sysctl patterns (ending in *). Use these at your own risk.
// Resource isolation might be lacking and pod might influence each other on the same node.
// +optional
AllowedUnsafeSysctls []string `json:"allowedUnsafeSysctls,omitempty"`
// featureGates is a map of feature names to bools that enable or disable alpha/experimental features.
FeatureGates map[string]bool `json:"featureGates,omitempty"`
// Tells the Kubelet to fail to start if swap is enabled on the node.
FailSwapOn bool `json:"failSwapOn,omitempty"`
// This flag, if set, enables a check prior to mount operations to verify that the required components
// (binaries, etc.) to mount the volume are available on the underlying node. If the check is enabled
// and fails the mount operation fails.
ExperimentalCheckNodeCapabilitiesBeforeMount bool `json:"experimentalCheckNodeCapabilitiesBeforeMount,omitempty"`
// This flag, if set, instructs the kubelet to keep volumes from terminated pods mounted to the node.
// This can be useful for debugging volume related issues.
KeepTerminatedPodVolumes bool `json:"keepTerminatedPodVolumes,omitempty"`
/* following flags are meant for Node Allocatable */
@ -352,9 +306,6 @@ type KubeletConfiguration struct {
// This flag accepts a list of options. Acceptible options are `pods`, `system-reserved` & `kube-reserved`.
// Refer to [Node Allocatable](https://git.k8s.io/community/contributors/design-proposals/node/node-allocatable.md) doc for more information.
EnforceNodeAllocatable []string `json:"enforceNodeAllocatable"`
// This flag, if set, will avoid including `EvictionHard` limits while computing Node Allocatable.
// Refer to [Node Allocatable](https://git.k8s.io/community/contributors/design-proposals/node/node-allocatable.md) doc for more information.
ExperimentalNodeAllocatableIgnoreEvictionThreshold bool `json:"experimentalNodeAllocatableIgnoreEvictionThreshold,omitempty"`
}
type KubeletAuthorizationMode string

48
pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.conversion.go
View File

@ -184,11 +184,6 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_kubeletconfig_KubeletConfigura
return err
}
out.EnableContentionProfiling = in.EnableContentionProfiling
out.MinimumGCAge = in.MinimumGCAge
out.MaxPerPodContainerCount = in.MaxPerPodContainerCount
if err := v1.Convert_Pointer_int32_To_int32(&in.MaxContainerCount, &out.MaxContainerCount, s); err != nil {
return err
}
if err := v1.Convert_Pointer_int32_To_int32(&in.CAdvisorPort, &out.CAdvisorPort, s); err != nil {
return err
}
@ -203,7 +198,6 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_kubeletconfig_KubeletConfigura
return err
}
out.ClusterDomain = in.ClusterDomain
out.MasterServiceNamespace = in.MasterServiceNamespace
out.ClusterDNS = *(*[]string)(unsafe.Pointer(&in.ClusterDNS))
out.StreamingConnectionIdleTimeout = in.StreamingConnectionIdleTimeout
out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency
@ -225,12 +219,9 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_kubeletconfig_KubeletConfigura
}
out.CgroupDriver = in.CgroupDriver
out.ContainerRuntime = in.ContainerRuntime
out.RemoteRuntimeEndpoint = in.RemoteRuntimeEndpoint
out.RemoteImageEndpoint = in.RemoteImageEndpoint
out.CPUManagerPolicy = in.CPUManagerPolicy
out.CPUManagerReconcilePeriod = in.CPUManagerReconcilePeriod
out.RuntimeRequestTimeout = in.RuntimeRequestTimeout
out.ExperimentalMounterPath = in.ExperimentalMounterPath
if err := v1.Convert_Pointer_string_To_string(&in.LockFilePath, &out.LockFilePath, s); err != nil {
return err
}
@ -242,13 +233,7 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_kubeletconfig_KubeletConfigura
if err := v1.Convert_Pointer_bool_To_bool(&in.CPUCFSQuota, &out.CPUCFSQuota, s); err != nil {
return err
}
if err := v1.Convert_Pointer_bool_To_bool(&in.Containerized, &out.Containerized, s); err != nil {
return err
}
out.MaxOpenFiles = in.MaxOpenFiles
if err := v1.Convert_Pointer_bool_To_bool(&in.RegisterSchedulable, &out.RegisterSchedulable, s); err != nil {
return err
}
out.RegisterWithTaints = *(*[]api.Taint)(unsafe.Pointer(&in.RegisterWithTaints))
out.ContentType = in.ContentType
if err := v1.Convert_Pointer_int32_To_int32(&in.KubeAPIQPS, &out.KubeAPIQPS, s); err != nil {
@ -259,7 +244,6 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_kubeletconfig_KubeletConfigura
return err
}
out.NodeLabels = *(*map[string]string)(unsafe.Pointer(&in.NodeLabels))
out.NonMasqueradeCIDR = in.NonMasqueradeCIDR
out.EnableCustomMetrics = in.EnableCustomMetrics
if err := v1.Convert_Pointer_string_To_string(&in.EvictionHard, &out.EvictionHard, s); err != nil {
return err
@ -269,14 +253,10 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_kubeletconfig_KubeletConfigura
out.EvictionPressureTransitionPeriod = in.EvictionPressureTransitionPeriod
out.EvictionMaxPodGracePeriod = in.EvictionMaxPodGracePeriod
out.EvictionMinimumReclaim = in.EvictionMinimumReclaim
if err := v1.Convert_Pointer_bool_To_bool(&in.ExperimentalKernelMemcgNotification, &out.ExperimentalKernelMemcgNotification, s); err != nil {
return err
}
out.PodsPerCore = in.PodsPerCore
if err := v1.Convert_Pointer_bool_To_bool(&in.EnableControllerAttachDetach, &out.EnableControllerAttachDetach, s); err != nil {
return err
}
out.ExperimentalQOSReserved = *(*kubeletconfig.ConfigurationMap)(unsafe.Pointer(&in.ExperimentalQOSReserved))
out.ProtectKernelDefaults = in.ProtectKernelDefaults
if err := v1.Convert_Pointer_bool_To_bool(&in.MakeIPTablesUtilChains, &out.MakeIPTablesUtilChains, s); err != nil {
return err
@ -287,17 +267,13 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_kubeletconfig_KubeletConfigura
if err := v1.Convert_Pointer_int32_To_int32(&in.IPTablesDropBit, &out.IPTablesDropBit, s); err != nil {
return err
}
out.AllowedUnsafeSysctls = *(*[]string)(unsafe.Pointer(&in.AllowedUnsafeSysctls))
out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates))
out.FailSwapOn = in.FailSwapOn
out.ExperimentalCheckNodeCapabilitiesBeforeMount = in.ExperimentalCheckNodeCapabilitiesBeforeMount
out.KeepTerminatedPodVolumes = in.KeepTerminatedPodVolumes
out.SystemReserved = *(*kubeletconfig.ConfigurationMap)(unsafe.Pointer(&in.SystemReserved))
out.KubeReserved = *(*kubeletconfig.ConfigurationMap)(unsafe.Pointer(&in.KubeReserved))
out.SystemReservedCgroup = in.SystemReservedCgroup
out.KubeReservedCgroup = in.KubeReservedCgroup
out.EnforceNodeAllocatable = *(*[]string)(unsafe.Pointer(&in.EnforceNodeAllocatable))
out.ExperimentalNodeAllocatableIgnoreEvictionThreshold = in.ExperimentalNodeAllocatableIgnoreEvictionThreshold
return nil
}
@ -349,11 +325,6 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigura
return err
}
out.EnableContentionProfiling = in.EnableContentionProfiling
out.MinimumGCAge = in.MinimumGCAge
out.MaxPerPodContainerCount = in.MaxPerPodContainerCount
if err := v1.Convert_int32_To_Pointer_int32(&in.MaxContainerCount, &out.MaxContainerCount, s); err != nil {
return err
}
if err := v1.Convert_int32_To_Pointer_int32(&in.CAdvisorPort, &out.CAdvisorPort, s); err != nil {
return err
}
@ -368,7 +339,6 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigura
return err
}
out.ClusterDomain = in.ClusterDomain
out.MasterServiceNamespace = in.MasterServiceNamespace
out.ClusterDNS = *(*[]string)(unsafe.Pointer(&in.ClusterDNS))
out.StreamingConnectionIdleTimeout = in.StreamingConnectionIdleTimeout
out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency
@ -390,12 +360,9 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigura
out.SystemCgroups = in.SystemCgroups
out.CgroupRoot = in.CgroupRoot
out.ContainerRuntime = in.ContainerRuntime
out.RemoteRuntimeEndpoint = in.RemoteRuntimeEndpoint
out.RemoteImageEndpoint = in.RemoteImageEndpoint
out.CPUManagerPolicy = in.CPUManagerPolicy
out.CPUManagerReconcilePeriod = in.CPUManagerReconcilePeriod
out.RuntimeRequestTimeout = in.RuntimeRequestTimeout
out.ExperimentalMounterPath = in.ExperimentalMounterPath
if err := v1.Convert_string_To_Pointer_string(&in.LockFilePath, &out.LockFilePath, s); err != nil {
return err
}
@ -407,13 +374,7 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigura
if err := v1.Convert_bool_To_Pointer_bool(&in.CPUCFSQuota, &out.CPUCFSQuota, s); err != nil {
return err
}
if err := v1.Convert_bool_To_Pointer_bool(&in.Containerized, &out.Containerized, s); err != nil {
return err
}
out.MaxOpenFiles = in.MaxOpenFiles
if err := v1.Convert_bool_To_Pointer_bool(&in.RegisterSchedulable, &out.RegisterSchedulable, s); err != nil {
return err
}
out.RegisterWithTaints = *(*[]core_v1.Taint)(unsafe.Pointer(&in.RegisterWithTaints))
out.ContentType = in.ContentType
if err := v1.Convert_int32_To_Pointer_int32(&in.KubeAPIQPS, &out.KubeAPIQPS, s); err != nil {
@ -424,7 +385,6 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigura
return err
}
out.NodeLabels = *(*map[string]string)(unsafe.Pointer(&in.NodeLabels))
out.NonMasqueradeCIDR = in.NonMasqueradeCIDR
out.EnableCustomMetrics = in.EnableCustomMetrics
if err := v1.Convert_string_To_Pointer_string(&in.EvictionHard, &out.EvictionHard, s); err != nil {
return err
@ -434,14 +394,10 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigura
out.EvictionPressureTransitionPeriod = in.EvictionPressureTransitionPeriod
out.EvictionMaxPodGracePeriod = in.EvictionMaxPodGracePeriod
out.EvictionMinimumReclaim = in.EvictionMinimumReclaim
if err := v1.Convert_bool_To_Pointer_bool(&in.ExperimentalKernelMemcgNotification, &out.ExperimentalKernelMemcgNotification, s); err != nil {
return err
}
out.PodsPerCore = in.PodsPerCore
if err := v1.Convert_bool_To_Pointer_bool(&in.EnableControllerAttachDetach, &out.EnableControllerAttachDetach, s); err != nil {
return err
}
out.ExperimentalQOSReserved = *(*map[string]string)(unsafe.Pointer(&in.ExperimentalQOSReserved))
out.ProtectKernelDefaults = in.ProtectKernelDefaults
if err := v1.Convert_bool_To_Pointer_bool(&in.MakeIPTablesUtilChains, &out.MakeIPTablesUtilChains, s); err != nil {
return err
@ -452,17 +408,13 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigura
if err := v1.Convert_int32_To_Pointer_int32(&in.IPTablesDropBit, &out.IPTablesDropBit, s); err != nil {
return err
}
out.AllowedUnsafeSysctls = *(*[]string)(unsafe.Pointer(&in.AllowedUnsafeSysctls))
out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates))
out.FailSwapOn = in.FailSwapOn
out.ExperimentalCheckNodeCapabilitiesBeforeMount = in.ExperimentalCheckNodeCapabilitiesBeforeMount
out.KeepTerminatedPodVolumes = in.KeepTerminatedPodVolumes
out.SystemReserved = *(*map[string]string)(unsafe.Pointer(&in.SystemReserved))
out.KubeReserved = *(*map[string]string)(unsafe.Pointer(&in.KubeReserved))
out.SystemReservedCgroup = in.SystemReservedCgroup
out.KubeReservedCgroup = in.KubeReservedCgroup
out.EnforceNodeAllocatable = *(*[]string)(unsafe.Pointer(&in.EnforceNodeAllocatable))
out.ExperimentalNodeAllocatableIgnoreEvictionThreshold = in.ExperimentalNodeAllocatableIgnoreEvictionThreshold
return nil
}

49
pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.deepcopy.go
View File

@ -217,16 +217,6 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
**out = **in
}
}
out.MinimumGCAge = in.MinimumGCAge
if in.MaxContainerCount != nil {
in, out := &in.MaxContainerCount, &out.MaxContainerCount
if *in == nil {
*out = nil
} else {
*out = new(int32)
**out = **in
}
}
if in.CAdvisorPort != nil {
in, out := &in.CAdvisorPort, &out.CAdvisorPort
if *in == nil {
@ -319,24 +309,6 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
**out = **in
}
}
if in.Containerized != nil {
in, out := &in.Containerized, &out.Containerized
if *in == nil {
*out = nil
} else {
*out = new(bool)
**out = **in
}
}
if in.RegisterSchedulable != nil {
in, out := &in.RegisterSchedulable, &out.RegisterSchedulable
if *in == nil {
*out = nil
} else {
*out = new(bool)
**out = **in
}
}
if in.RegisterWithTaints != nil {
in, out := &in.RegisterWithTaints, &out.RegisterWithTaints
*out = make([]core_v1.Taint, len(*in))
@ -379,15 +351,6 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
}
}
out.EvictionPressureTransitionPeriod = in.EvictionPressureTransitionPeriod
if in.ExperimentalKernelMemcgNotification != nil {
in, out := &in.ExperimentalKernelMemcgNotification, &out.ExperimentalKernelMemcgNotification
if *in == nil {
*out = nil
} else {
*out = new(bool)
**out = **in
}
}
if in.EnableControllerAttachDetach != nil {
in, out := &in.EnableControllerAttachDetach, &out.EnableControllerAttachDetach
if *in == nil {
@ -397,13 +360,6 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
**out = **in
}
}
if in.ExperimentalQOSReserved != nil {
in, out := &in.ExperimentalQOSReserved, &out.ExperimentalQOSReserved
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.MakeIPTablesUtilChains != nil {
in, out := &in.MakeIPTablesUtilChains, &out.MakeIPTablesUtilChains
if *in == nil {
@ -431,11 +387,6 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
**out = **in
}
}
if in.AllowedUnsafeSysctls != nil {
in, out := &in.AllowedUnsafeSysctls, &out.AllowedUnsafeSysctls
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.FeatureGates != nil {
in, out := &in.FeatureGates, &out.FeatureGates
*out = make(map[string]bool, len(*in))

13
pkg/kubelet/apis/kubeletconfig/zz_generated.deepcopy.go
View File

@ -154,7 +154,6 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
*out = make([]string, len(*in))
copy(*out, *in)
}
out.MinimumGCAge = in.MinimumGCAge
if in.ClusterDNS != nil {
in, out := &in.ClusterDNS, &out.ClusterDNS
*out = make([]string, len(*in))
@ -181,18 +180,6 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
}
}
out.EvictionPressureTransitionPeriod = in.EvictionPressureTransitionPeriod
if in.ExperimentalQOSReserved != nil {
in, out := &in.ExperimentalQOSReserved, &out.ExperimentalQOSReserved
*out = make(ConfigurationMap, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.AllowedUnsafeSysctls != nil {
in, out := &in.AllowedUnsafeSysctls, &out.AllowedUnsafeSysctls
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.FeatureGates != nil {
in, out := &in.FeatureGates, &out.FeatureGates
*out = make(map[string]bool, len(*in))

116
pkg/kubelet/kubelet.go
View File

@ -198,12 +198,27 @@ type Bootstrap interface {
type Builder func(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
kubeDeps *Dependencies,
crOptions *options.ContainerRuntimeOptions,
hostnameOverride,
nodeIP,
providerID,
cloudProvider,
certDirectory,
rootDirectory string) (Bootstrap, error)
hostnameOverride string,
nodeIP string,
providerID string,
cloudProvider string,
certDirectory string,
rootDirectory string,
allowedUnsafeSysctls []string,
containerized bool,
remoteRuntimeEndpoint string,
remoteImageEndpoint string,
experimentalMounterPath string,
experimentalKernelMemcgNotification bool,
experimentalCheckNodeCapabilitiesBeforeMount bool,
experimentalNodeAllocatableIgnoreEvictionThreshold bool,
minimumGCAge metav1.Duration,
maxPerPodContainerCount int32,
maxContainerCount int32,
masterServiceNamespace string,
registerSchedulable bool,
nonMasqueradeCIDR string,
keepTerminatedPodVolumes bool) (Bootstrap, error)
// Dependencies is a bin for things we might consider "injected dependencies" -- objects constructed
// at runtime that are necessary for running the Kubelet. This is a temporary solution for grouping
@ -287,12 +302,12 @@ func makePodSourceConfig(kubeCfg *kubeletconfiginternal.KubeletConfiguration, ku
return cfg, nil
}
func getRuntimeAndImageServices(config *kubeletconfiginternal.KubeletConfiguration) (internalapi.RuntimeService, internalapi.ImageManagerService, error) {
rs, err := remote.NewRemoteRuntimeService(config.RemoteRuntimeEndpoint, config.RuntimeRequestTimeout.Duration)
func getRuntimeAndImageServices(remoteRuntimeEndpoint string, remoteImageEndpoint string, runtimeRequestTimeout metav1.Duration) (internalapi.RuntimeService, internalapi.ImageManagerService, error) {
rs, err := remote.NewRemoteRuntimeService(remoteRuntimeEndpoint, runtimeRequestTimeout.Duration)
if err != nil {
return nil, nil, err
}
is, err := remote.NewRemoteImageService(config.RemoteImageEndpoint, config.RuntimeRequestTimeout.Duration)
is, err := remote.NewRemoteImageService(remoteImageEndpoint, runtimeRequestTimeout.Duration)
if err != nil {
return nil, nil, err
}
@ -304,12 +319,27 @@ func getRuntimeAndImageServices(config *kubeletconfiginternal.KubeletConfigurati
func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
kubeDeps *Dependencies,
crOptions *options.ContainerRuntimeOptions,
hostnameOverride,
nodeIP,
providerID,
cloudProvider,
certDirectory,
rootDirectory string) (*Kubelet, error) {
hostnameOverride string,
nodeIP string,
providerID string,
cloudProvider string,
certDirectory string,
rootDirectory string,
allowedUnsafeSysctls []string,
containerized bool,
remoteRuntimeEndpoint string,
remoteImageEndpoint string,
experimentalMounterPath string,
experimentalKernelMemcgNotification bool,
experimentalCheckNodeCapabilitiesBeforeMount bool,
experimentalNodeAllocatableIgnoreEvictionThreshold bool,
minimumGCAge metav1.Duration,
maxPerPodContainerCount int32,
maxContainerCount int32,
masterServiceNamespace string,
registerSchedulable bool,
nonMasqueradeCIDR string,
keepTerminatedPodVolumes bool) (*Kubelet, error) {
if rootDirectory == "" {
return nil, fmt.Errorf("invalid root directory %q", rootDirectory)
}
@ -377,9 +407,9 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
}
containerGCPolicy := kubecontainer.ContainerGCPolicy{
MinAge: kubeCfg.MinimumGCAge.Duration,
MaxPerPodContainer: int(kubeCfg.MaxPerPodContainerCount),
MaxContainers: int(kubeCfg.MaxContainerCount),
MinAge: minimumGCAge.Duration,
MaxPerPodContainer: int(maxPerPodContainerCount),
MaxContainers: int(maxContainerCount),
}
daemonEndpoints := &v1.NodeDaemonEndpoints{
@ -393,7 +423,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
}
enforceNodeAllocatable := kubeCfg.EnforceNodeAllocatable
if kubeCfg.ExperimentalNodeAllocatableIgnoreEvictionThreshold {
if experimentalNodeAllocatableIgnoreEvictionThreshold {
// Do not provide kubeCfg.EnforceNodeAllocatable to eviction threshold parsing if we are not enforcing Evictions
enforceNodeAllocatable = []string{}
}
@ -405,7 +435,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
PressureTransitionPeriod: kubeCfg.EvictionPressureTransitionPeriod.Duration,
MaxPodGracePeriodSeconds: int64(kubeCfg.EvictionMaxPodGracePeriod),
Thresholds: thresholds,
KernelMemcgNotification: kubeCfg.ExperimentalKernelMemcgNotification,
KernelMemcgNotification: experimentalKernelMemcgNotification,
}
serviceIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
@ -459,12 +489,12 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
resyncInterval: kubeCfg.SyncFrequency.Duration,
sourcesReady: config.NewSourcesReady(kubeDeps.PodConfig.SeenAllSources),
registerNode: kubeCfg.RegisterNode,
registerSchedulable: kubeCfg.RegisterSchedulable,
registerSchedulable: registerSchedulable,
clusterDomain: kubeCfg.ClusterDomain,
clusterDNS: clusterDNS,
serviceLister: serviceLister,
nodeInfo: nodeInfo,
masterServiceNamespace: kubeCfg.MasterServiceNamespace,
masterServiceNamespace: masterServiceNamespace,
streamingConnectionIdleTimeout: kubeCfg.StreamingConnectionIdleTimeout.Duration,
recorder: kubeDeps.Recorder,
cadvisor: kubeDeps.CAdvisorInterface,
@ -495,6 +525,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
iptablesMasqueradeBit: int(kubeCfg.IPTablesMasqueradeBit),
iptablesDropBit: int(kubeCfg.IPTablesDropBit),
experimentalHostUserNamespaceDefaulting: utilfeature.DefaultFeatureGate.Enabled(features.ExperimentalHostUserNamespaceDefaultingGate),
keepTerminatedPodVolumes: keepTerminatedPodVolumes,
}
secretManager := secret.NewCachingSecretManager(
@ -517,7 +548,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
}
glog.Infof("Hairpin mode set to %q", hairpinMode)
plug, err := network.InitNetworkPlugin(kubeDeps.NetworkPlugins, crOptions.NetworkPluginName, &criNetworkHost{&networkHost{klet}, &network.NoopPortMappingGetter{}}, hairpinMode, kubeCfg.NonMasqueradeCIDR, int(crOptions.NetworkPluginMTU))
plug, err := network.InitNetworkPlugin(kubeDeps.NetworkPlugins, crOptions.NetworkPluginName, &criNetworkHost{&networkHost{klet}, &network.NoopPortMappingGetter{}}, hairpinMode, nonMasqueradeCIDR, int(crOptions.NetworkPluginMTU))
if err != nil {
return nil, err
}
@ -536,10 +567,10 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
// podManager is also responsible for keeping secretManager and configMapManager contents up-to-date.
klet.podManager = kubepod.NewBasicPodManager(kubepod.NewBasicMirrorClient(klet.kubeClient), secretManager, configMapManager)
if kubeCfg.RemoteRuntimeEndpoint != "" {
// kubeCfg.RemoteImageEndpoint is same as kubeCfg.RemoteRuntimeEndpoint if not explicitly specified
if kubeCfg.RemoteImageEndpoint == "" {
kubeCfg.RemoteImageEndpoint = kubeCfg.RemoteRuntimeEndpoint
if remoteRuntimeEndpoint != "" {
// remoteImageEndpoint is same as remoteRuntimeEndpoint if not explicitly specified
if remoteImageEndpoint == "" {
remoteImageEndpoint = remoteRuntimeEndpoint
}
}
@ -550,7 +581,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
}
pluginSettings := dockershim.NetworkPluginSettings{
HairpinMode: hairpinMode,
NonMasqueradeCIDR: kubeCfg.NonMasqueradeCIDR,
NonMasqueradeCIDR: nonMasqueradeCIDR,
PluginName: crOptions.NetworkPluginName,
PluginConfDir: crOptions.CNIConfDir,
PluginBinDir: binDir,
@ -592,10 +623,10 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
// The unix socket for kubelet <-> dockershim communication.
glog.V(5).Infof("RemoteRuntimeEndpoint: %q, RemoteImageEndpoint: %q",
kubeCfg.RemoteRuntimeEndpoint,
kubeCfg.RemoteImageEndpoint)
remoteRuntimeEndpoint,
remoteImageEndpoint)
glog.V(2).Infof("Starting the GRPC server for the docker CRI shim.")
server := dockerremote.NewDockerServer(kubeCfg.RemoteRuntimeEndpoint, ds)
server := dockerremote.NewDockerServer(remoteRuntimeEndpoint, ds)
if err := server.Start(); err != nil {
return nil, err
}
@ -614,7 +645,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
default:
return nil, fmt.Errorf("unsupported CRI runtime: %q", kubeCfg.ContainerRuntime)
}
runtimeService, imageService, err := getRuntimeAndImageServices(kubeCfg)
runtimeService, imageService, err := getRuntimeAndImageServices(remoteRuntimeEndpoint, remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout)
if err != nil {
return nil, err
}
@ -651,7 +682,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
// be removed. Related issue:
// https://github.com/kubernetes/kubernetes/issues/51798
if (kubeCfg.ContainerRuntime == kubetypes.DockerContainerRuntime &&
goruntime.GOOS == "linux") || kubeCfg.RemoteRuntimeEndpoint == "/var/run/crio.sock" {
goruntime.GOOS == "linux") || remoteRuntimeEndpoint == "/var/run/crio.sock" {
klet.StatsProvider = stats.NewCadvisorStatsProvider(
klet.cadvisor,
klet.resourceAnalyzer,
@ -772,11 +803,11 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
// If the experimentalMounterPathFlag is set, we do not want to
// check node capabilities since the mount path is not the default
if len(kubeCfg.ExperimentalMounterPath) != 0 {
kubeCfg.ExperimentalCheckNodeCapabilitiesBeforeMount = false
if len(experimentalMounterPath) != 0 {
experimentalCheckNodeCapabilitiesBeforeMount = false
// Replace the nameserver in containerized-mounter's rootfs/etc/resolve.conf with kubelet.ClusterDNS
// so that service name could be resolved
klet.setupDNSinContainerizedMounter(kubeCfg.ExperimentalMounterPath)
klet.setupDNSinContainerizedMounter(experimentalMounterPath)
}
// setup volumeManager
@ -791,8 +822,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
kubeDeps.Mounter,
klet.getPodsDir(),
kubeDeps.Recorder,
kubeCfg.ExperimentalCheckNodeCapabilitiesBeforeMount,
kubeCfg.KeepTerminatedPodVolumes)
experimentalCheckNodeCapabilitiesBeforeMount,
keepTerminatedPodVolumes)
runtimeCache, err := kubecontainer.NewRuntimeCache(klet.containerRuntime)
if err != nil {
@ -824,7 +855,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
}
// Safe, whitelisted sysctls can always be used as unsafe sysctls in the spec
// Hence, we concatenate those two lists.
safeAndUnsafeSysctls := append(sysctl.SafeSysctlWhitelist(), kubeCfg.AllowedUnsafeSysctls...)
safeAndUnsafeSysctls := append(sysctl.SafeSysctlWhitelist(), allowedUnsafeSysctls...)
unsafeWhitelist, err := sysctl.NewWhitelist(safeAndUnsafeSysctls, v1.UnsafeSysctlsPodAnnotationKey)
if err != nil {
return nil, err
@ -1149,6 +1180,13 @@ type Kubelet struct {
// StatsProvider provides the node and the container stats.
*stats.StatsProvider
// containerized should be set to true if the kubelet is running in a container
containerized bool
// This flag, if set, instructs the kubelet to keep volumes from terminated pods mounted to the node.
// This can be useful for debugging volume related issues.
keepTerminatedPodVolumes bool // DEPRECATED
}
func allLocalIPsWithoutLoopback() ([]net.IP, error) {

2
pkg/kubelet/kubelet_node_status.go
View File

@ -272,7 +272,7 @@ func (kl *Kubelet) initialNode() (*v1.Node, error) {
glog.Infof("Controller attach/detach is disabled for this node; Kubelet will attach and detach volumes")
}
if kl.kubeletConfiguration.KeepTerminatedPodVolumes {
if kl.keepTerminatedPodVolumes {
if node.Annotations == nil {
node.Annotations = make(map[string]string)
}

4
pkg/kubelet/kubelet_pods.go
View File

@ -864,7 +864,7 @@ func (kl *Kubelet) PodResourcesAreReclaimed(pod *v1.Pod, status v1.PodStatus) bo
glog.V(3).Infof("Pod %q is terminated, but some containers have not been cleaned up: %+v", format.Pod(pod), runtimeStatus.ContainerStatuses)
return false
}
if kl.podVolumesExist(pod.UID) && !kl.kubeletConfiguration.KeepTerminatedPodVolumes {
if kl.podVolumesExist(pod.UID) && !kl.keepTerminatedPodVolumes {
// We shouldnt delete pods whose volumes have not been cleaned up if we are not keeping terminated pod volumes
glog.V(3).Infof("Pod %q is terminated, but some volumes have not been cleaned up", format.Pod(pod))
return false
@ -1704,7 +1704,7 @@ func (kl *Kubelet) cleanupOrphanedPodCgroups(cgroupPods map[types.UID]cm.CgroupN
// parent croup. If the volumes still exist, reduce the cpu shares for any
// process in the cgroup to the minimum value while we wait. if the kubelet
// is configured to keep terminated volumes, we will delete the cgroup and not block.
if podVolumesExist := kl.podVolumesExist(uid); podVolumesExist && !kl.kubeletConfiguration.KeepTerminatedPodVolumes {
if podVolumesExist := kl.podVolumesExist(uid); podVolumesExist && !kl.keepTerminatedPodVolumes {
glog.V(3).Infof("Orphaned pod %q found, but volumes not yet removed. Reducing cpu to minimum", uid)
if err := pcm.ReduceCPULimits(val); err != nil {
glog.Warningf("Failed to reduce cpu time for pod %q pending volume cleanup due to %v", uid, err)

1
pkg/kubemark/BUILD
View File

@ -21,7 +21,6 @@ go_library(
"//pkg/controller:go_default_library",
"//pkg/kubelet:go_default_library",
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library",
"//pkg/kubelet/cadvisor:go_default_library",
"//pkg/kubelet/cm:go_default_library",
"//pkg/kubelet/container/testing:go_default_library",

20
pkg/kubemark/hollow_kubelet.go
View File

@ -19,12 +19,12 @@ package kubemark
import (
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
kubeletapp "k8s.io/kubernetes/cmd/kubelet/app"
"k8s.io/kubernetes/cmd/kubelet/app/options"
"k8s.io/kubernetes/pkg/kubelet"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubeletv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
"k8s.io/kubernetes/pkg/kubelet/cm"
containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
@ -109,13 +109,13 @@ func GetHollowKubeletConfig(
glog.Infof("Using %s as root dir for hollow-kubelet", testRootDir)
// Flags struct
f := &options.KubeletFlags{
RootDirectory: testRootDir,
HostnameOverride: nodeName,
CloudProvider: kubeletv1alpha1.AutoDetectCloudProvider,
// Use the default runtime options.
ContainerRuntimeOptions: *options.NewContainerRuntimeOptions(),
}
f := options.NewKubeletFlags()
f.RootDirectory = testRootDir
f.HostnameOverride = nodeName
f.MinimumGCAge = metav1.Duration{Duration: 1 * time.Minute}
f.MaxContainerCount = 100
f.MaxPerPodContainerCount = 2
f.RegisterSchedulable = true
// Config struct
c, err := options.NewKubeletConfiguration()
@ -130,7 +130,6 @@ func GetHollowKubeletConfig(
c.PodManifestPath = manifestFilePath
c.FileCheckFrequency.Duration = 20 * time.Second
c.HTTPCheckFrequency.Duration = 20 * time.Second
c.MinimumGCAge.Duration = 1 * time.Minute
c.NodeStatusUpdateFrequency.Duration = 10 * time.Second
c.SyncFrequency.Duration = 10 * time.Second
c.EvictionPressureTransitionPeriod.Duration = 5 * time.Minute
@ -153,11 +152,8 @@ func GetHollowKubeletConfig(
// what the "real" kubelet currently does, because there's no way to
// set promiscuous mode on docker0.
c.HairpinMode = kubeletconfig.HairpinVeth
c.MaxContainerCount = 100
c.MaxOpenFiles = 1024
c.MaxPerPodContainerCount = 2
c.RegisterNode = true
c.RegisterSchedulable = true
c.RegistryBurst = 10
c.RegistryPullQPS = 5.0
c.ResolverConfig = kubetypes.ResolvConfDefault

2
test/e2e_node/eviction_test.go
View File

@ -102,7 +102,6 @@ var _ = framework.KubeDescribe("MemoryAllocatableEviction [Slow] [Serial] [Disru
kubeReserved.Sub(resource.MustParse("300Mi"))
initialConfig.KubeReserved = kubeletconfig.ConfigurationMap(map[string]string{string(v1.ResourceMemory): kubeReserved.String()})
initialConfig.EnforceNodeAllocatable = []string{cm.NodeAllocatableEnforcementKey}
initialConfig.ExperimentalNodeAllocatableIgnoreEvictionThreshold = false
initialConfig.CgroupsPerQOS = true
})
runEvictionTest(f, pressureTimeout, expectedNodeCondition, logMemoryMetrics, []podEvictSpec{
@ -133,7 +132,6 @@ var _ = framework.KubeDescribe("LocalStorageAllocatableEviction [Slow] [Serial]
initialConfig.KubeReserved = kubeletconfig.ConfigurationMap(map[string]string{string(v1.ResourceEphemeralStorage): fmt.Sprintf("%d", availableBytes-diskConsumed)})
initialConfig.EnforceNodeAllocatable = []string{cm.NodeAllocatableEnforcementKey}
initialConfig.CgroupsPerQOS = true
initialConfig.ExperimentalNodeAllocatableIgnoreEvictionThreshold = false
initialConfig.FeatureGates[string(features.LocalStorageCapacityIsolation)] = true
// set evictionHard to be very small, so that only the allocatable eviction threshold triggers
initialConfig.EvictionHard = "nodefs.available<1"