k3s/vendor/k8s.io/kubernetes/cmd/kubelet/app/server.go

1291 lines
47 KiB
Go
Raw Normal View History

2019-01-12 04:58:27 +00:00
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package app makes it easy to create a kubelet server for various contexts.
package app
import (
2019-08-30 18:33:25 +00:00
"context"
2019-01-12 04:58:27 +00:00
"crypto/tls"
"errors"
"fmt"
2020-08-10 17:43:49 +00:00
"math"
2019-01-12 04:58:27 +00:00
"net"
"net/http"
"os"
"path"
"path/filepath"
"strconv"
2019-08-30 18:33:25 +00:00
"strings"
2019-01-12 04:58:27 +00:00
"time"
"github.com/coreos/go-systemd/daemon"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
2020-08-10 17:43:49 +00:00
"k8s.io/klog/v2"
2019-12-12 01:27:03 +00:00
"k8s.io/utils/mount"
2019-01-12 04:58:27 +00:00
2019-04-07 17:07:55 +00:00
v1 "k8s.io/api/core/v1"
2019-01-12 04:58:27 +00:00
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2019-08-30 18:33:25 +00:00
"k8s.io/apimachinery/pkg/runtime"
2019-01-12 04:58:27 +00:00
"k8s.io/apimachinery/pkg/types"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/server/healthz"
utilfeature "k8s.io/apiserver/pkg/util/feature"
clientset "k8s.io/client-go/kubernetes"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/record"
certutil "k8s.io/client-go/util/cert"
"k8s.io/client-go/util/certificate"
2019-06-12 21:00:25 +00:00
"k8s.io/client-go/util/connrotation"
2019-04-07 17:07:55 +00:00
"k8s.io/client-go/util/keyutil"
2019-08-30 18:33:25 +00:00
cloudprovider "k8s.io/cloud-provider"
2020-08-10 17:43:49 +00:00
"k8s.io/component-base/cli/flag"
2019-04-07 17:07:55 +00:00
cliflag "k8s.io/component-base/cli/flag"
2020-08-10 17:43:49 +00:00
"k8s.io/component-base/configz"
2019-12-12 01:27:03 +00:00
"k8s.io/component-base/featuregate"
2020-08-10 17:43:49 +00:00
"k8s.io/component-base/logs"
2020-03-26 21:07:15 +00:00
"k8s.io/component-base/metrics"
2020-08-10 17:43:49 +00:00
"k8s.io/component-base/metrics/legacyregistry"
2019-12-12 01:27:03 +00:00
"k8s.io/component-base/version"
"k8s.io/component-base/version/verflag"
2019-01-12 04:58:27 +00:00
kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
"k8s.io/kubernetes/cmd/kubelet/app/options"
"k8s.io/kubernetes/pkg/api/legacyscheme"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/capabilities"
"k8s.io/kubernetes/pkg/credentialprovider"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet"
kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config"
kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/config/scheme"
kubeletconfigvalidation "k8s.io/kubernetes/pkg/kubelet/apis/config/validation"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
kubeletcertificate "k8s.io/kubernetes/pkg/kubelet/certificate"
"k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap"
"k8s.io/kubernetes/pkg/kubelet/cm"
2019-12-12 01:27:03 +00:00
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
2019-01-12 04:58:27 +00:00
"k8s.io/kubernetes/pkg/kubelet/config"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/eviction"
evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api"
dynamickubeletconfig "k8s.io/kubernetes/pkg/kubelet/kubeletconfig"
"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/configfiles"
2020-08-10 17:43:49 +00:00
kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
2019-01-12 04:58:27 +00:00
"k8s.io/kubernetes/pkg/kubelet/server"
2019-04-07 17:07:55 +00:00
"k8s.io/kubernetes/pkg/kubelet/stats/pidlimit"
2019-01-12 04:58:27 +00:00
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
utilfs "k8s.io/kubernetes/pkg/util/filesystem"
"k8s.io/kubernetes/pkg/util/flock"
nodeutil "k8s.io/kubernetes/pkg/util/node"
"k8s.io/kubernetes/pkg/util/oom"
"k8s.io/kubernetes/pkg/util/rlimit"
2019-09-27 21:51:53 +00:00
"k8s.io/kubernetes/pkg/volume/util/hostutil"
2019-04-07 17:07:55 +00:00
"k8s.io/kubernetes/pkg/volume/util/subpath"
2019-01-12 04:58:27 +00:00
"k8s.io/utils/exec"
)
const (
// Kubelet component name
componentKubelet = "kubelet"
)
// NewKubeletCommand creates a *cobra.Command object with default parameters
2020-08-10 17:43:49 +00:00
func NewKubeletCommand(ctx context.Context) *cobra.Command {
2019-01-12 04:58:27 +00:00
cleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError)
2019-04-07 17:07:55 +00:00
cleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
2019-01-12 04:58:27 +00:00
kubeletFlags := options.NewKubeletFlags()
kubeletConfig, err := options.NewKubeletConfiguration()
// programmer error
if err != nil {
klog.Fatal(err)
}
cmd := &cobra.Command{
Use: componentKubelet,
Long: `The kubelet is the primary "node agent" that runs on each
2019-04-07 17:07:55 +00:00
node. It can register the node with the apiserver using one of: the hostname; a flag to
override the hostname; or specific logic for a cloud provider.
The kubelet works in terms of a PodSpec. A PodSpec is a YAML or JSON object
2019-01-12 04:58:27 +00:00
that describes a pod. The kubelet takes a set of PodSpecs that are provided through
various mechanisms (primarily through the apiserver) and ensures that the containers
described in those PodSpecs are running and healthy. The kubelet doesn't manage
containers which were not created by Kubernetes.
Other than from an PodSpec from the apiserver, there are three ways that a container
manifest can be provided to the Kubelet.
File: Path passed as a flag on the command line. Files under this path will be monitored
periodically for updates. The monitoring period is 20s by default and is configurable
via a flag.
HTTP endpoint: HTTP endpoint passed as a parameter on the command line. This endpoint
is checked every 20 seconds (also configurable with a flag).
HTTP server: The kubelet can also listen for HTTP and respond to a simple API
(underspec'd currently) to submit a new manifest.`,
// The Kubelet has special flag parsing requirements to enforce flag precedence rules,
// so we do all our parsing manually in Run, below.
// DisableFlagParsing=true provides the full set of flags passed to the kubelet in the
// `args` arg to Run, without Cobra's interference.
DisableFlagParsing: true,
Run: func(cmd *cobra.Command, args []string) {
// initial flag parse, since we disable cobra's flag parsing
if err := cleanFlagSet.Parse(args); err != nil {
cmd.Usage()
klog.Fatal(err)
}
// check if there are non-flag arguments in the command line
cmds := cleanFlagSet.Args()
if len(cmds) > 0 {
cmd.Usage()
klog.Fatalf("unknown command: %s", cmds[0])
}
// short-circuit on help
help, err := cleanFlagSet.GetBool("help")
if err != nil {
klog.Fatal(`"help" flag is non-bool, programmer error, please correct`)
}
if help {
cmd.Help()
return
}
// short-circuit on verflag
verflag.PrintAndExitIfRequested()
2020-08-10 17:43:49 +00:00
cliflag.PrintFlags(cleanFlagSet)
2019-01-12 04:58:27 +00:00
// set feature gates from initial flags-based config
2019-04-07 17:07:55 +00:00
if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil {
2019-01-12 04:58:27 +00:00
klog.Fatal(err)
}
// validate the initial KubeletFlags
if err := options.ValidateKubeletFlags(kubeletFlags); err != nil {
klog.Fatal(err)
}
if kubeletFlags.ContainerRuntime == "remote" && cleanFlagSet.Changed("pod-infra-container-image") {
klog.Warning("Warning: For remote container runtime, --pod-infra-container-image is ignored in kubelet, which should be set in that remote runtime instead")
}
// load kubelet config file, if provided
if configFile := kubeletFlags.KubeletConfigFile; len(configFile) > 0 {
kubeletConfig, err = loadConfigFile(configFile)
if err != nil {
klog.Fatal(err)
}
// We must enforce flag precedence by re-parsing the command line into the new object.
// This is necessary to preserve backwards-compatibility across binary upgrades.
// See issue #56171 for more details.
if err := kubeletConfigFlagPrecedence(kubeletConfig, args); err != nil {
klog.Fatal(err)
}
// update feature gates based on new config
2019-04-07 17:07:55 +00:00
if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil {
2019-01-12 04:58:27 +00:00
klog.Fatal(err)
}
}
// We always validate the local configuration (command line + config file).
// This is the default "last-known-good" config for dynamic config, and must always remain valid.
if err := kubeletconfigvalidation.ValidateKubeletConfiguration(kubeletConfig); err != nil {
klog.Fatal(err)
}
// use dynamic kubelet config, if enabled
var kubeletConfigController *dynamickubeletconfig.Controller
if dynamicConfigDir := kubeletFlags.DynamicConfigDir.Value(); len(dynamicConfigDir) > 0 {
var dynamicKubeletConfig *kubeletconfiginternal.KubeletConfiguration
dynamicKubeletConfig, kubeletConfigController, err = BootstrapKubeletConfigController(dynamicConfigDir,
func(kc *kubeletconfiginternal.KubeletConfiguration) error {
// Here, we enforce flag precedence inside the controller, prior to the controller's validation sequence,
// so that we get a complete validation at the same point where we can decide to reject dynamic config.
// This fixes the flag-precedence component of issue #63305.
// See issue #56171 for general details on flag precedence.
return kubeletConfigFlagPrecedence(kc, args)
})
if err != nil {
klog.Fatal(err)
}
// If we should just use our existing, local config, the controller will return a nil config
if dynamicKubeletConfig != nil {
kubeletConfig = dynamicKubeletConfig
// Note: flag precedence was already enforced in the controller, prior to validation,
// by our above transform function. Now we simply update feature gates from the new config.
2019-04-07 17:07:55 +00:00
if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil {
2019-01-12 04:58:27 +00:00
klog.Fatal(err)
}
}
}
// construct a KubeletServer from kubeletFlags and kubeletConfig
kubeletServer := &options.KubeletServer{
KubeletFlags: *kubeletFlags,
KubeletConfiguration: *kubeletConfig,
}
// use kubeletServer to construct the default KubeletDeps
2019-12-12 01:27:03 +00:00
kubeletDeps, err := UnsecuredDependencies(kubeletServer, utilfeature.DefaultFeatureGate)
2019-01-12 04:58:27 +00:00
if err != nil {
klog.Fatal(err)
}
// add the kubelet config controller to kubeletDeps
kubeletDeps.KubeletConfigController = kubeletConfigController
// run the kubelet
klog.V(5).Infof("KubeletConfiguration: %#v", kubeletServer.KubeletConfiguration)
2020-08-10 17:43:49 +00:00
if err := Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate); err != nil {
2019-01-12 04:58:27 +00:00
klog.Fatal(err)
}
},
}
// keep cleanFlagSet separate, so Cobra doesn't pollute it with the global flags
kubeletFlags.AddFlags(cleanFlagSet)
options.AddKubeletConfigFlags(cleanFlagSet, kubeletConfig)
options.AddGlobalFlags(cleanFlagSet)
cleanFlagSet.BoolP("help", "h", false, fmt.Sprintf("help for %s", cmd.Name()))
// ugly, but necessary, because Cobra's default UsageFunc and HelpFunc pollute the flagset with global flags
const usageFmt = "Usage:\n %s\n\nFlags:\n%s"
cmd.SetUsageFunc(func(cmd *cobra.Command) error {
fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine(), cleanFlagSet.FlagUsagesWrapped(2))
return nil
})
cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine(), cleanFlagSet.FlagUsagesWrapped(2))
})
return cmd
}
// newFlagSetWithGlobals constructs a new pflag.FlagSet with global flags registered
// on it.
func newFlagSetWithGlobals() *pflag.FlagSet {
fs := pflag.NewFlagSet("", pflag.ExitOnError)
2019-04-07 17:07:55 +00:00
// set the normalize func, similar to k8s.io/component-base/cli//flags.go:InitFlags
fs.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
2019-01-12 04:58:27 +00:00
// explicitly add flags from libs that register global flags
options.AddGlobalFlags(fs)
return fs
}
// newFakeFlagSet constructs a pflag.FlagSet with the same flags as fs, but where
// all values have noop Set implementations
func newFakeFlagSet(fs *pflag.FlagSet) *pflag.FlagSet {
ret := pflag.NewFlagSet("", pflag.ExitOnError)
ret.SetNormalizeFunc(fs.GetNormalizeFunc())
fs.VisitAll(func(f *pflag.Flag) {
2019-04-07 17:07:55 +00:00
ret.VarP(cliflag.NoOp{}, f.Name, f.Shorthand, f.Usage)
2019-01-12 04:58:27 +00:00
})
return ret
}
// kubeletConfigFlagPrecedence re-parses flags over the KubeletConfiguration object.
// We must enforce flag precedence by re-parsing the command line into the new object.
// This is necessary to preserve backwards-compatibility across binary upgrades.
// See issue #56171 for more details.
func kubeletConfigFlagPrecedence(kc *kubeletconfiginternal.KubeletConfiguration, args []string) error {
// We use a throwaway kubeletFlags and a fake global flagset to avoid double-parses,
// as some Set implementations accumulate values from multiple flag invocations.
fs := newFakeFlagSet(newFlagSetWithGlobals())
// register throwaway KubeletFlags
options.NewKubeletFlags().AddFlags(fs)
// register new KubeletConfiguration
options.AddKubeletConfigFlags(fs, kc)
// Remember original feature gates, so we can merge with flag gates later
original := kc.FeatureGates
// re-parse flags
if err := fs.Parse(args); err != nil {
return err
}
// Add back feature gates that were set in the original kc, but not in flags
for k, v := range original {
if _, ok := kc.FeatureGates[k]; !ok {
kc.FeatureGates[k] = v
}
}
return nil
}
func loadConfigFile(name string) (*kubeletconfiginternal.KubeletConfiguration, error) {
const errFmt = "failed to load Kubelet config file %s, error %v"
// compute absolute path based on current working dir
kubeletConfigFile, err := filepath.Abs(name)
if err != nil {
return nil, fmt.Errorf(errFmt, name, err)
}
loader, err := configfiles.NewFsLoader(utilfs.DefaultFs{}, kubeletConfigFile)
if err != nil {
return nil, fmt.Errorf(errFmt, name, err)
}
kc, err := loader.Load()
if err != nil {
return nil, fmt.Errorf(errFmt, name, err)
}
return kc, err
}
// UnsecuredDependencies returns a Dependencies suitable for being run, or an error if the server setup
// is not valid. It will not start any background processes, and does not include authentication/authorization
2019-12-12 01:27:03 +00:00
func UnsecuredDependencies(s *options.KubeletServer, featureGate featuregate.FeatureGate) (*kubelet.Dependencies, error) {
2019-01-12 04:58:27 +00:00
// Initialize the TLS Options
tlsOptions, err := InitializeTLS(&s.KubeletFlags, &s.KubeletConfiguration)
if err != nil {
return nil, err
}
mounter := mount.New(s.ExperimentalMounterPath)
2019-04-07 17:07:55 +00:00
subpather := subpath.New(mounter)
2019-09-27 21:51:53 +00:00
hu := hostutil.NewHostUtil()
2019-01-12 04:58:27 +00:00
var pluginRunner = exec.New()
2020-08-10 17:43:49 +00:00
var dockerOptions *kubelet.DockerOptions
2019-01-12 04:58:27 +00:00
if s.ContainerRuntime == kubetypes.DockerContainerRuntime {
2020-08-10 17:43:49 +00:00
dockerOptions = &kubelet.DockerOptions{
2019-01-12 04:58:27 +00:00
DockerEndpoint: s.DockerEndpoint,
RuntimeRequestTimeout: s.RuntimeRequestTimeout.Duration,
ImagePullProgressDeadline: s.ImagePullProgressDeadline.Duration,
}
}
2019-12-12 01:27:03 +00:00
plugins, err := ProbeVolumePlugins(featureGate)
if err != nil {
return nil, err
}
2019-01-12 04:58:27 +00:00
return &kubelet.Dependencies{
Auth: nil, // default does not enforce auth[nz]
CAdvisorInterface: nil, // cadvisor.New launches background processes (bg http.ListenAndServe, and some bg cleaners), not set here
2019-08-30 18:33:25 +00:00
Cloud: nil, // cloud provider might start background processes
2019-01-12 04:58:27 +00:00
ContainerManager: nil,
2020-08-10 17:43:49 +00:00
DockerOptions: dockerOptions,
2019-01-12 04:58:27 +00:00
KubeClient: nil,
HeartbeatClient: nil,
EventClient: nil,
2019-09-27 21:51:53 +00:00
HostUtil: hu,
2019-01-12 04:58:27 +00:00
Mounter: mounter,
2019-04-07 17:07:55 +00:00
Subpather: subpather,
2019-01-12 04:58:27 +00:00
OOMAdjuster: oom.NewOOMAdjuster(),
OSInterface: kubecontainer.RealOS{},
2019-12-12 01:27:03 +00:00
VolumePlugins: plugins,
2019-01-12 04:58:27 +00:00
DynamicPluginProber: GetDynamicPluginProber(s.VolumePluginDir, pluginRunner),
TLSOptions: tlsOptions}, nil
}
// Run runs the specified KubeletServer with the given Dependencies. This should never exit.
// The kubeDeps argument may be nil - if so, it is initialized from the settings on KubeletServer.
// Otherwise, the caller is assumed to have set up the Dependencies object and a default one will
// not be generated.
2020-08-10 17:43:49 +00:00
func Run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate featuregate.FeatureGate) error {
logOption := logs.NewOptions()
logOption.LogFormat = s.Logging.Format
logOption.Apply()
2019-01-12 04:58:27 +00:00
// To help debugging, immediately log version
klog.Infof("Version: %+v", version.Get())
if err := initForOS(s.KubeletFlags.WindowsService); err != nil {
return fmt.Errorf("failed OS init: %v", err)
}
2020-08-10 17:43:49 +00:00
if err := run(ctx, s, kubeDeps, featureGate); err != nil {
2019-01-12 04:58:27 +00:00
return fmt.Errorf("failed to run Kubelet: %v", err)
}
return nil
}
func checkPermissions() error {
if uid := os.Getuid(); uid != 0 {
2019-09-27 21:51:53 +00:00
return fmt.Errorf("kubelet needs to run as uid `0`. It is being run as %d", uid)
2019-01-12 04:58:27 +00:00
}
// TODO: Check if kubelet is running in the `initial` user namespace.
// http://man7.org/linux/man-pages/man7/user_namespaces.7.html
return nil
}
func setConfigz(cz *configz.Config, kc *kubeletconfiginternal.KubeletConfiguration) error {
scheme, _, err := kubeletscheme.NewSchemeAndCodecs()
if err != nil {
return err
}
versioned := kubeletconfigv1beta1.KubeletConfiguration{}
if err := scheme.Convert(kc, &versioned, nil); err != nil {
return err
}
cz.Set(versioned)
return nil
}
func initConfigz(kc *kubeletconfiginternal.KubeletConfiguration) error {
cz, err := configz.New("kubeletconfig")
if err != nil {
klog.Errorf("unable to register configz: %s", err)
return err
}
if err := setConfigz(cz, kc); err != nil {
klog.Errorf("unable to register config: %s", err)
return err
}
return nil
}
// makeEventRecorder sets up kubeDeps.Recorder if it's nil. It's a no-op otherwise.
func makeEventRecorder(kubeDeps *kubelet.Dependencies, nodeName types.NodeName) {
if kubeDeps.Recorder != nil {
return
}
eventBroadcaster := record.NewBroadcaster()
kubeDeps.Recorder = eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: componentKubelet, Host: string(nodeName)})
2020-08-10 17:43:49 +00:00
eventBroadcaster.StartStructuredLogging(3)
2019-01-12 04:58:27 +00:00
if kubeDeps.EventClient != nil {
klog.V(4).Infof("Sending events to api server.")
eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeDeps.EventClient.Events("")})
} else {
klog.Warning("No api server defined - no events will be sent to API server.")
}
}
2020-08-10 17:43:49 +00:00
func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate featuregate.FeatureGate) (err error) {
2019-01-12 04:58:27 +00:00
// Set global feature gates based on the value on the initial KubeletServer
2019-04-07 17:07:55 +00:00
err = utilfeature.DefaultMutableFeatureGate.SetFromMap(s.KubeletConfiguration.FeatureGates)
2019-01-12 04:58:27 +00:00
if err != nil {
return err
}
// validate the initial KubeletServer (we set feature gates first, because this validation depends on feature gates)
if err := options.ValidateKubeletServer(s); err != nil {
return err
}
// Obtain Kubelet Lock File
if s.ExitOnLockContention && s.LockFilePath == "" {
return errors.New("cannot exit on lock file contention: no lock file specified")
}
done := make(chan struct{})
if s.LockFilePath != "" {
klog.Infof("acquiring file lock on %q", s.LockFilePath)
if err := flock.Acquire(s.LockFilePath); err != nil {
return fmt.Errorf("unable to acquire file lock on %q: %v", s.LockFilePath, err)
}
if s.ExitOnLockContention {
klog.Infof("watching for inotify events for: %v", s.LockFilePath)
if err := watchForLockfileContention(s.LockFilePath, done); err != nil {
return err
}
}
}
// Register current configuration with /configz endpoint
err = initConfigz(&s.KubeletConfiguration)
if err != nil {
klog.Errorf("unable to register KubeletConfiguration with configz, error: %v", err)
}
2020-03-26 21:07:15 +00:00
if len(s.ShowHiddenMetricsForVersion) > 0 {
metrics.SetShowHidden()
}
2019-01-12 04:58:27 +00:00
// About to get clients and such, detect standaloneMode
standaloneMode := true
if len(s.KubeConfig) > 0 {
standaloneMode = false
}
if kubeDeps == nil {
2019-12-12 01:27:03 +00:00
kubeDeps, err = UnsecuredDependencies(s, featureGate)
2019-01-12 04:58:27 +00:00
if err != nil {
return err
}
}
2019-08-30 18:33:25 +00:00
if kubeDeps.Cloud == nil {
if !cloudprovider.IsExternal(s.CloudProvider) {
2020-08-10 17:43:49 +00:00
cloudprovider.DeprecationWarningForProvider(s.CloudProvider)
2019-08-30 18:33:25 +00:00
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
if err != nil {
return err
}
2020-08-10 17:43:49 +00:00
if cloud != nil {
2019-08-30 18:33:25 +00:00
klog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile)
}
kubeDeps.Cloud = cloud
}
}
2019-01-12 04:58:27 +00:00
hostName, err := nodeutil.GetHostname(s.HostnameOverride)
if err != nil {
return err
}
2019-08-30 18:33:25 +00:00
nodeName, err := getNodeName(kubeDeps.Cloud, hostName)
2019-01-12 04:58:27 +00:00
if err != nil {
return err
}
// if in standalone mode, indicate as much by setting all clients to nil
2019-04-07 17:07:55 +00:00
switch {
case standaloneMode:
2019-01-12 04:58:27 +00:00
kubeDeps.KubeClient = nil
kubeDeps.EventClient = nil
kubeDeps.HeartbeatClient = nil
klog.Warningf("standalone mode, no API client")
2019-04-07 17:07:55 +00:00
case kubeDeps.KubeClient == nil, kubeDeps.EventClient == nil, kubeDeps.HeartbeatClient == nil:
2020-08-10 17:43:49 +00:00
clientConfig, closeAllConns, err := buildKubeletClientConfig(ctx, s, nodeName)
2019-01-12 04:58:27 +00:00
if err != nil {
return err
}
2019-06-12 21:00:25 +00:00
if closeAllConns == nil {
return errors.New("closeAllConns must be a valid function other than nil")
}
2019-04-07 17:07:55 +00:00
kubeDeps.OnHeartbeatFailure = closeAllConns
2019-01-12 04:58:27 +00:00
2019-04-07 17:07:55 +00:00
kubeDeps.KubeClient, err = clientset.NewForConfig(clientConfig)
2019-01-12 04:58:27 +00:00
if err != nil {
2019-04-07 17:07:55 +00:00
return fmt.Errorf("failed to initialize kubelet client: %v", err)
2019-01-12 04:58:27 +00:00
}
// make a separate client for events
eventClientConfig := *clientConfig
eventClientConfig.QPS = float32(s.EventRecordQPS)
eventClientConfig.Burst = int(s.EventBurst)
2019-04-07 17:07:55 +00:00
kubeDeps.EventClient, err = v1core.NewForConfig(&eventClientConfig)
2019-01-12 04:58:27 +00:00
if err != nil {
2019-04-07 17:07:55 +00:00
return fmt.Errorf("failed to initialize kubelet event client: %v", err)
2019-01-12 04:58:27 +00:00
}
// make a separate client for heartbeat with throttling disabled and a timeout attached
heartbeatClientConfig := *clientConfig
heartbeatClientConfig.Timeout = s.KubeletConfiguration.NodeStatusUpdateFrequency.Duration
2019-12-12 01:27:03 +00:00
// The timeout is the minimum of the lease duration and status update frequency
leaseTimeout := time.Duration(s.KubeletConfiguration.NodeLeaseDurationSeconds) * time.Second
if heartbeatClientConfig.Timeout > leaseTimeout {
heartbeatClientConfig.Timeout = leaseTimeout
2019-01-12 04:58:27 +00:00
}
2019-12-12 01:27:03 +00:00
2019-04-07 17:07:55 +00:00
heartbeatClientConfig.QPS = float32(-1)
kubeDeps.HeartbeatClient, err = clientset.NewForConfig(&heartbeatClientConfig)
2019-01-12 04:58:27 +00:00
if err != nil {
2019-04-07 17:07:55 +00:00
return fmt.Errorf("failed to initialize kubelet heartbeat client: %v", err)
2019-01-12 04:58:27 +00:00
}
}
if kubeDeps.Auth == nil {
2020-06-23 22:12:14 +00:00
auth, runAuthenticatorCAReload, err := BuildAuth(nodeName, kubeDeps.KubeClient, s.KubeletConfiguration)
2019-01-12 04:58:27 +00:00
if err != nil {
return err
}
kubeDeps.Auth = auth
2020-08-10 17:43:49 +00:00
runAuthenticatorCAReload(ctx.Done())
2019-01-12 04:58:27 +00:00
}
2019-08-30 18:33:25 +00:00
var cgroupRoots []string
if s.CgroupDriver == "none" {
cgroupRoots = []string{"/"}
} else {
2020-08-10 17:43:49 +00:00
nodeAllocatableRoot := cm.NodeAllocatableRoot(s.CgroupRoot, s.CgroupsPerQOS, s.CgroupDriver)
cgroupRoots = append(cgroupRoots, nodeAllocatableRoot)
2019-08-30 18:33:25 +00:00
kubeletCgroup, err := cm.GetKubeletContainer(s.KubeletCgroups)
if err != nil {
2019-09-19 23:17:14 +00:00
klog.Warningf("failed to get the kubelet's cgroup: %v. Kubelet system container metrics may be missing.", err)
2019-12-12 01:27:03 +00:00
} else if kubeletCgroup != "" {
2019-08-30 18:33:25 +00:00
cgroupRoots = append(cgroupRoots, kubeletCgroup)
}
runtimeCgroup, err := cm.GetRuntimeContainer(s.ContainerRuntime, s.RuntimeCgroups)
if err != nil {
2019-09-19 23:17:14 +00:00
klog.Warningf("failed to get the container runtime's cgroup: %v. Runtime system container metrics may be missing.", err)
2019-12-12 01:27:03 +00:00
} else if runtimeCgroup != "" {
2019-08-30 18:33:25 +00:00
// RuntimeCgroups is optional, so ignore if it isn't specified
cgroupRoots = append(cgroupRoots, runtimeCgroup)
}
2020-08-10 17:43:49 +00:00
2019-08-30 18:33:25 +00:00
if s.SystemCgroups != "" {
// SystemCgroups is optional, so ignore if it isn't specified
cgroupRoots = append(cgroupRoots, s.SystemCgroups)
}
}
2019-01-12 04:58:27 +00:00
if kubeDeps.CAdvisorInterface == nil {
imageFsInfoProvider := cadvisor.NewImageFsInfoProvider(s.ContainerRuntime, s.RemoteRuntimeEndpoint)
2019-08-30 18:33:25 +00:00
kubeDeps.CAdvisorInterface, err = cadvisor.New(imageFsInfoProvider, s.RootDirectory, cgroupRoots, cadvisor.UsingLegacyCadvisorStats(s.ContainerRuntime, s.RemoteRuntimeEndpoint))
2019-01-12 04:58:27 +00:00
if err != nil {
return err
}
}
// Setup event recorder if required.
makeEventRecorder(kubeDeps, nodeName)
if kubeDeps.ContainerManager == nil {
if s.CgroupsPerQOS && s.CgroupRoot == "" {
2019-04-07 17:07:55 +00:00
klog.Info("--cgroups-per-qos enabled, but --cgroup-root was not specified. defaulting to /")
2019-01-12 04:58:27 +00:00
s.CgroupRoot = "/"
}
2019-12-12 01:27:03 +00:00
var reservedSystemCPUs cpuset.CPUSet
var errParse error
if s.ReservedSystemCPUs != "" {
reservedSystemCPUs, errParse = cpuset.Parse(s.ReservedSystemCPUs)
if errParse != nil {
// invalid cpu list is provided, set reservedSystemCPUs to empty, so it won't overwrite kubeReserved/systemReserved
klog.Infof("Invalid ReservedSystemCPUs \"%s\"", s.ReservedSystemCPUs)
return errParse
}
// is it safe do use CAdvisor here ??
machineInfo, err := kubeDeps.CAdvisorInterface.MachineInfo()
if err != nil {
// if can't use CAdvisor here, fall back to non-explicit cpu list behavor
klog.Warning("Failed to get MachineInfo, set reservedSystemCPUs to empty")
reservedSystemCPUs = cpuset.NewCPUSet()
} else {
reservedList := reservedSystemCPUs.ToSlice()
first := reservedList[0]
last := reservedList[len(reservedList)-1]
if first < 0 || last >= machineInfo.NumCores {
// the specified cpuset is outside of the range of what the machine has
klog.Infof("Invalid cpuset specified by --reserved-cpus")
return fmt.Errorf("Invalid cpuset %q specified by --reserved-cpus", s.ReservedSystemCPUs)
}
}
} else {
reservedSystemCPUs = cpuset.NewCPUSet()
}
if reservedSystemCPUs.Size() > 0 {
// at cmd option valication phase it is tested either --system-reserved-cgroup or --kube-reserved-cgroup is specified, so overwrite should be ok
klog.Infof("Option --reserved-cpus is specified, it will overwrite the cpu setting in KubeReserved=\"%v\", SystemReserved=\"%v\".", s.KubeReserved, s.SystemReserved)
if s.KubeReserved != nil {
delete(s.KubeReserved, "cpu")
}
if s.SystemReserved == nil {
s.SystemReserved = make(map[string]string)
}
s.SystemReserved["cpu"] = strconv.Itoa(reservedSystemCPUs.Size())
klog.Infof("After cpu setting is overwritten, KubeReserved=\"%v\", SystemReserved=\"%v\"", s.KubeReserved, s.SystemReserved)
}
2019-01-12 04:58:27 +00:00
kubeReserved, err := parseResourceList(s.KubeReserved)
if err != nil {
return err
}
systemReserved, err := parseResourceList(s.SystemReserved)
if err != nil {
return err
}
var hardEvictionThresholds []evictionapi.Threshold
// If the user requested to ignore eviction thresholds, then do not set valid values for hardEvictionThresholds here.
if !s.ExperimentalNodeAllocatableIgnoreEvictionThreshold {
hardEvictionThresholds, err = eviction.ParseThresholdConfig([]string{}, s.EvictionHard, nil, nil, nil)
if err != nil {
return err
}
}
experimentalQOSReserved, err := cm.ParseQOSReserved(s.QOSReserved)
if err != nil {
return err
}
devicePluginEnabled := utilfeature.DefaultFeatureGate.Enabled(features.DevicePlugins)
kubeDeps.ContainerManager, err = cm.NewContainerManager(
kubeDeps.Mounter,
kubeDeps.CAdvisorInterface,
cm.NodeConfig{
RuntimeCgroupsName: s.RuntimeCgroups,
SystemCgroupsName: s.SystemCgroups,
KubeletCgroupsName: s.KubeletCgroups,
ContainerRuntime: s.ContainerRuntime,
CgroupsPerQOS: s.CgroupsPerQOS,
CgroupRoot: s.CgroupRoot,
CgroupDriver: s.CgroupDriver,
KubeletRootDir: s.RootDirectory,
ProtectKernelDefaults: s.ProtectKernelDefaults,
NodeAllocatableConfig: cm.NodeAllocatableConfig{
KubeReservedCgroupName: s.KubeReservedCgroup,
SystemReservedCgroupName: s.SystemReservedCgroup,
EnforceNodeAllocatable: sets.NewString(s.EnforceNodeAllocatable...),
KubeReserved: kubeReserved,
SystemReserved: systemReserved,
2019-12-12 01:27:03 +00:00
ReservedSystemCPUs: reservedSystemCPUs,
2019-01-12 04:58:27 +00:00
HardEvictionThresholds: hardEvictionThresholds,
},
QOSReserved: *experimentalQOSReserved,
ExperimentalCPUManagerPolicy: s.CPUManagerPolicy,
ExperimentalCPUManagerReconcilePeriod: s.CPUManagerReconcilePeriod.Duration,
ExperimentalPodPidsLimit: s.PodPidsLimit,
EnforceCPULimits: s.CPUCFSQuota,
CPUCFSQuotaPeriod: s.CPUCFSQuotaPeriod.Duration,
2019-09-27 21:51:53 +00:00
ExperimentalTopologyManagerPolicy: s.TopologyManagerPolicy,
2020-08-10 17:43:49 +00:00
Rootless: s.Rootless,
2019-01-12 04:58:27 +00:00
},
s.FailSwapOn,
devicePluginEnabled,
kubeDeps.Recorder)
if err != nil {
return err
}
}
if err := checkPermissions(); err != nil {
klog.Error(err)
}
utilruntime.ReallyCrash = s.ReallyCrashForTesting
// TODO(vmarmol): Do this through container config.
oomAdjuster := kubeDeps.OOMAdjuster
if err := oomAdjuster.ApplyOOMScoreAdj(0, int(s.OOMScoreAdj)); err != nil {
klog.Warning(err)
}
2020-03-26 21:07:15 +00:00
err = kubelet.PreInitRuntimeService(&s.KubeletConfiguration,
kubeDeps, &s.ContainerRuntimeOptions,
s.ContainerRuntime,
s.RuntimeCgroups,
s.RemoteRuntimeEndpoint,
s.RemoteImageEndpoint,
s.NonMasqueradeCIDR)
if err != nil {
return err
}
2019-01-12 04:58:27 +00:00
if err := RunKubelet(s, kubeDeps, s.RunOnce); err != nil {
return err
}
2019-10-16 05:42:28 +00:00
// If the kubelet config controller is available, and dynamic config is enabled, start the config and status sync loops
if utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) && len(s.DynamicConfigDir.Value()) > 0 &&
kubeDeps.KubeletConfigController != nil && !standaloneMode && !s.RunOnce {
if err := kubeDeps.KubeletConfigController.StartSync(kubeDeps.KubeClient, kubeDeps.EventClient, string(nodeName)); err != nil {
return err
}
}
2019-01-12 04:58:27 +00:00
if s.HealthzPort > 0 {
2019-07-14 07:58:54 +00:00
mux := http.NewServeMux()
healthz.InstallHandler(mux)
2019-01-12 04:58:27 +00:00
go wait.Until(func() {
2019-07-14 07:58:54 +00:00
err := http.ListenAndServe(net.JoinHostPort(s.HealthzBindAddress, strconv.Itoa(int(s.HealthzPort))), mux)
2019-01-12 04:58:27 +00:00
if err != nil {
2019-08-30 18:33:25 +00:00
klog.Errorf("Starting healthz server failed: %v", err)
2019-01-12 04:58:27 +00:00
}
}, 5*time.Second, wait.NeverStop)
}
if s.RunOnce {
return nil
}
// If systemd is used, notify it that we have started
go daemon.SdNotify(false, "READY=1")
select {
case <-done:
break
2020-08-10 17:43:49 +00:00
case <-ctx.Done():
2019-01-12 04:58:27 +00:00
break
}
return nil
}
2019-04-07 17:07:55 +00:00
// buildKubeletClientConfig constructs the appropriate client config for the kubelet depending on whether
// bootstrapping is enabled or client certificate rotation is enabled.
2020-08-10 17:43:49 +00:00
func buildKubeletClientConfig(ctx context.Context, s *options.KubeletServer, nodeName types.NodeName) (*restclient.Config, func(), error) {
if s.RotateCertificates {
2019-04-07 17:07:55 +00:00
// Rules for client rotation and the handling of kube config files:
//
// 1. If the client provides only a kubeconfig file, we must use that as the initial client
// kubeadm needs the initial data in the kubeconfig to be placed into the cert store
// 2. If the client provides only an initial bootstrap kubeconfig file, we must create a
// kubeconfig file at the target location that points to the cert store, but until
// the file is present the client config will have no certs
// 3. If the client provides both and the kubeconfig is valid, we must ignore the bootstrap
// kubeconfig.
// 4. If the client provides both and the kubeconfig is expired or otherwise invalid, we must
// replace the kubeconfig with a new file that points to the cert dir
//
// The desired configuration for bootstrapping is to use a bootstrap kubeconfig and to have
// the kubeconfig file be managed by this process. For backwards compatibility with kubeadm,
// which provides a high powered kubeconfig on the master with cert/key data, we must
// bootstrap the cert manager with the contents of the initial client config.
klog.Infof("Client rotation is on, will bootstrap in background")
certConfig, clientConfig, err := bootstrap.LoadClientConfig(s.KubeConfig, s.BootstrapKubeconfig, s.CertDirectory)
if err != nil {
return nil, nil, err
}
2019-08-30 18:33:25 +00:00
// use the correct content type for cert rotation, but don't set QPS
setContentTypeForClient(certConfig, s.ContentType)
kubeClientConfigOverrides(s, clientConfig)
2019-04-07 17:07:55 +00:00
clientCertificateManager, err := buildClientCertificateManager(certConfig, clientConfig, s.CertDirectory, nodeName)
if err != nil {
return nil, nil, err
}
2020-08-10 17:43:49 +00:00
legacyregistry.RawMustRegister(metrics.NewGaugeFunc(
metrics.GaugeOpts{
Subsystem: kubeletmetrics.KubeletSubsystem,
Name: "certificate_manager_client_ttl_seconds",
Help: "Gauge of the TTL (time-to-live) of the Kubelet's client certificate. " +
"The value is in seconds until certificate expiry (negative if already expired). " +
"If client certificate is invalid or unused, the value will be +INF.",
StabilityLevel: metrics.ALPHA,
},
func() float64 {
if c := clientCertificateManager.Current(); c != nil && c.Leaf != nil {
return math.Trunc(c.Leaf.NotAfter.Sub(time.Now()).Seconds())
}
return math.Inf(1)
},
))
2019-04-07 17:07:55 +00:00
// the rotating transport will use the cert from the cert manager instead of these files
transportConfig := restclient.AnonymousClientConfig(clientConfig)
// we set exitAfter to five minutes because we use this client configuration to request new certs - if we are unable
// to request new certs, we will be unable to continue normal operation. Exiting the process allows a wrapper
// or the bootstrapping credentials to potentially lay down new initial config.
closeAllConns, err := kubeletcertificate.UpdateTransport(wait.NeverStop, transportConfig, clientCertificateManager, 5*time.Minute)
if err != nil {
return nil, nil, err
}
klog.V(2).Info("Starting client certificate rotation.")
clientCertificateManager.Start()
return transportConfig, closeAllConns, nil
}
if len(s.BootstrapKubeconfig) > 0 {
2020-08-10 17:43:49 +00:00
if err := bootstrap.LoadClientCert(ctx, s.KubeConfig, s.BootstrapKubeconfig, s.CertDirectory, nodeName); err != nil {
2019-04-07 17:07:55 +00:00
return nil, nil, err
}
}
clientConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
&clientcmd.ClientConfigLoadingRules{ExplicitPath: s.KubeConfig},
&clientcmd.ConfigOverrides{},
).ClientConfig()
if err != nil {
return nil, nil, fmt.Errorf("invalid kubeconfig: %v", err)
}
kubeClientConfigOverrides(s, clientConfig)
2019-06-12 21:00:25 +00:00
closeAllConns, err := updateDialer(clientConfig)
if err != nil {
return nil, nil, err
}
return clientConfig, closeAllConns, nil
}
2019-04-07 17:07:55 +00:00
2019-06-12 21:00:25 +00:00
// updateDialer instruments a restconfig with a dial. the returned function allows forcefully closing all active connections.
func updateDialer(clientConfig *restclient.Config) (func(), error) {
if clientConfig.Transport != nil || clientConfig.Dial != nil {
return nil, fmt.Errorf("there is already a transport or dialer configured")
}
d := connrotation.NewDialer((&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext)
clientConfig.Dial = d.DialContext
return d.CloseAll, nil
2019-04-07 17:07:55 +00:00
}
// buildClientCertificateManager creates a certificate manager that will use certConfig to request a client certificate
// if no certificate is available, or the most recent clientConfig (which is assumed to point to the cert that the manager will
// write out).
func buildClientCertificateManager(certConfig, clientConfig *restclient.Config, certDir string, nodeName types.NodeName) (certificate.Manager, error) {
2020-08-10 17:43:49 +00:00
newClientsetFn := func(current *tls.Certificate) (clientset.Interface, error) {
2019-04-07 17:07:55 +00:00
// If we have a valid certificate, use that to fetch CSRs. Otherwise use the bootstrap
// credentials. In the future it would be desirable to change the behavior of bootstrap
// to always fall back to the external bootstrap credentials when such credentials are
// provided by a fundamental trust system like cloud VM identity or an HSM module.
config := certConfig
if current != nil {
config = clientConfig
}
2020-08-10 17:43:49 +00:00
return clientset.NewForConfig(config)
2019-04-07 17:07:55 +00:00
}
return kubeletcertificate.NewKubeletClientCertificateManager(
certDir,
nodeName,
// this preserves backwards compatibility with kubeadm which passes
// a high powered certificate to the kubelet as --kubeconfig and expects
// it to be rotated out immediately
clientConfig.CertData,
clientConfig.KeyData,
clientConfig.CertFile,
clientConfig.KeyFile,
2020-08-10 17:43:49 +00:00
newClientsetFn,
2019-04-07 17:07:55 +00:00
)
}
func kubeClientConfigOverrides(s *options.KubeletServer, clientConfig *restclient.Config) {
2019-08-30 18:33:25 +00:00
setContentTypeForClient(clientConfig, s.ContentType)
2019-04-07 17:07:55 +00:00
// Override kubeconfig qps/burst settings from flags
clientConfig.QPS = float32(s.KubeAPIQPS)
clientConfig.Burst = int(s.KubeAPIBurst)
}
2019-01-12 04:58:27 +00:00
// getNodeName returns the node name according to the cloud provider
// if cloud provider is specified. Otherwise, returns the hostname of the node.
2019-08-30 18:33:25 +00:00
func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName, error) {
if cloud == nil {
return types.NodeName(hostname), nil
}
instances, ok := cloud.Instances()
if !ok {
return "", fmt.Errorf("failed to get instances from cloud provider")
}
nodeName, err := instances.CurrentNodeName(context.TODO(), hostname)
if err != nil {
return "", fmt.Errorf("error fetching current node name from cloud provider: %v", err)
}
klog.V(2).Infof("cloud provider determined current node name to be %s", nodeName)
return nodeName, nil
2019-01-12 04:58:27 +00:00
}
// InitializeTLS checks for a configured TLSCertFile and TLSPrivateKeyFile: if unspecified a new self-signed
// certificate and key file are generated. Returns a configured server.TLSOptions object.
func InitializeTLS(kf *options.KubeletFlags, kc *kubeletconfiginternal.KubeletConfiguration) (*server.TLSOptions, error) {
if !kc.ServerTLSBootstrap && kc.TLSCertFile == "" && kc.TLSPrivateKeyFile == "" {
kc.TLSCertFile = path.Join(kf.CertDirectory, "kubelet.crt")
kc.TLSPrivateKeyFile = path.Join(kf.CertDirectory, "kubelet.key")
canReadCertAndKey, err := certutil.CanReadCertAndKey(kc.TLSCertFile, kc.TLSPrivateKeyFile)
if err != nil {
return nil, err
}
if !canReadCertAndKey {
hostName, err := nodeutil.GetHostname(kf.HostnameOverride)
if err != nil {
return nil, err
}
cert, key, err := certutil.GenerateSelfSignedCertKey(hostName, nil, nil)
if err != nil {
return nil, fmt.Errorf("unable to generate self signed cert: %v", err)
}
if err := certutil.WriteCert(kc.TLSCertFile, cert); err != nil {
return nil, err
}
2019-04-07 17:07:55 +00:00
if err := keyutil.WriteKey(kc.TLSPrivateKeyFile, key); err != nil {
2019-01-12 04:58:27 +00:00
return nil, err
}
klog.V(4).Infof("Using self-signed cert (%s, %s)", kc.TLSCertFile, kc.TLSPrivateKeyFile)
}
}
2019-04-07 17:07:55 +00:00
tlsCipherSuites, err := cliflag.TLSCipherSuites(kc.TLSCipherSuites)
2019-01-12 04:58:27 +00:00
if err != nil {
return nil, err
}
2020-08-10 17:43:49 +00:00
if len(tlsCipherSuites) > 0 {
insecureCiphers := flag.InsecureTLSCiphers()
for i := 0; i < len(tlsCipherSuites); i++ {
for cipherName, cipherID := range insecureCiphers {
if tlsCipherSuites[i] == cipherID {
klog.Warningf("Use of insecure cipher '%s' detected.", cipherName)
}
}
}
}
2019-04-07 17:07:55 +00:00
minTLSVersion, err := cliflag.TLSVersion(kc.TLSMinVersion)
2019-01-12 04:58:27 +00:00
if err != nil {
return nil, err
}
tlsOptions := &server.TLSOptions{
Config: &tls.Config{
MinVersion: minTLSVersion,
CipherSuites: tlsCipherSuites,
},
CertFile: kc.TLSCertFile,
KeyFile: kc.TLSPrivateKeyFile,
}
if len(kc.Authentication.X509.ClientCAFile) > 0 {
clientCAs, err := certutil.NewPool(kc.Authentication.X509.ClientCAFile)
if err != nil {
return nil, fmt.Errorf("unable to load client CA file %s: %v", kc.Authentication.X509.ClientCAFile, err)
}
// Specify allowed CAs for client certificates
tlsOptions.Config.ClientCAs = clientCAs
// Populate PeerCertificates in requests, but don't reject connections without verified certificates
tlsOptions.Config.ClientAuth = tls.RequestClientCert
}
return tlsOptions, nil
}
2019-08-30 18:33:25 +00:00
// setContentTypeForClient sets the appropritae content type into the rest config
// and handles defaulting AcceptContentTypes based on that input.
func setContentTypeForClient(cfg *restclient.Config, contentType string) {
if len(contentType) == 0 {
return
}
cfg.ContentType = contentType
switch contentType {
case runtime.ContentTypeProtobuf:
cfg.AcceptContentTypes = strings.Join([]string{runtime.ContentTypeProtobuf, runtime.ContentTypeJSON}, ",")
default:
// otherwise let the rest client perform defaulting
}
}
2019-01-12 04:58:27 +00:00
// RunKubelet is responsible for setting up and running a kubelet. It is used in three different applications:
// 1 Integration tests
// 2 Kubelet binary
// 3 Standalone 'kubernetes' binary
// Eventually, #2 will be replaced with instances of #3
func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencies, runOnce bool) error {
hostname, err := nodeutil.GetHostname(kubeServer.HostnameOverride)
if err != nil {
return err
}
// Query the cloud provider for our node name, default to hostname if kubeDeps.Cloud == nil
2019-08-30 18:33:25 +00:00
nodeName, err := getNodeName(kubeDeps.Cloud, hostname)
2019-01-12 04:58:27 +00:00
if err != nil {
return err
}
2020-08-10 17:43:49 +00:00
hostnameOverridden := len(kubeServer.HostnameOverride) > 0
2019-01-12 04:58:27 +00:00
// Setup event recorder if required.
makeEventRecorder(kubeDeps, nodeName)
2019-08-30 18:33:25 +00:00
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: true,
})
2019-01-12 04:58:27 +00:00
credentialprovider.SetPreferredDockercfgPath(kubeServer.RootDirectory)
klog.V(2).Infof("Using root directory: %v", kubeServer.RootDirectory)
if kubeDeps.OSInterface == nil {
kubeDeps.OSInterface = kubecontainer.RealOS{}
}
2019-04-07 17:07:55 +00:00
k, err := createAndInitKubelet(&kubeServer.KubeletConfiguration,
2019-01-12 04:58:27 +00:00
kubeDeps,
&kubeServer.ContainerRuntimeOptions,
kubeServer.ContainerRuntime,
2020-08-10 17:43:49 +00:00
hostname,
hostnameOverridden,
nodeName,
2019-01-12 04:58:27 +00:00
kubeServer.NodeIP,
kubeServer.ProviderID,
2019-04-07 17:07:55 +00:00
kubeServer.CloudProvider,
2019-01-12 04:58:27 +00:00
kubeServer.CertDirectory,
kubeServer.RootDirectory,
kubeServer.RegisterNode,
kubeServer.RegisterWithTaints,
kubeServer.AllowedUnsafeSysctls,
kubeServer.ExperimentalMounterPath,
2020-08-10 17:43:49 +00:00
kubeServer.KernelMemcgNotification,
2019-01-12 04:58:27 +00:00
kubeServer.ExperimentalCheckNodeCapabilitiesBeforeMount,
kubeServer.ExperimentalNodeAllocatableIgnoreEvictionThreshold,
kubeServer.MinimumGCAge,
kubeServer.MaxPerPodContainerCount,
kubeServer.MaxContainerCount,
kubeServer.MasterServiceNamespace,
kubeServer.RegisterSchedulable,
kubeServer.KeepTerminatedPodVolumes,
kubeServer.NodeLabels,
kubeServer.SeccompProfileRoot,
kubeServer.NodeStatusMaxImages)
if err != nil {
return fmt.Errorf("failed to create kubelet: %v", err)
}
// NewMainKubelet should have set up a pod source config if one didn't exist
// when the builder was run. This is just a precaution.
if kubeDeps.PodConfig == nil {
return fmt.Errorf("failed to create kubelet, pod source config was nil")
}
podCfg := kubeDeps.PodConfig
2020-08-10 17:43:49 +00:00
if err := rlimit.SetNumFiles(uint64(kubeServer.MaxOpenFiles)); err != nil {
klog.Errorf("Failed to set rlimit on max file handles: %v", err)
}
2019-01-12 04:58:27 +00:00
// process pods and exit.
if runOnce {
if _, err := k.RunOnce(podCfg.Updates()); err != nil {
return fmt.Errorf("runonce failed: %v", err)
}
2019-04-07 17:07:55 +00:00
klog.Info("Started kubelet as runonce")
2019-01-12 04:58:27 +00:00
} else {
2019-08-30 18:33:25 +00:00
startKubelet(k, podCfg, &kubeServer.KubeletConfiguration, kubeDeps, kubeServer.EnableCAdvisorJSONEndpoints, kubeServer.EnableServer)
2019-04-07 17:07:55 +00:00
klog.Info("Started kubelet")
2019-01-12 04:58:27 +00:00
}
return nil
}
2019-08-30 18:33:25 +00:00
func startKubelet(k kubelet.Bootstrap, podCfg *config.PodConfig, kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps *kubelet.Dependencies, enableCAdvisorJSONEndpoints, enableServer bool) {
2019-01-12 04:58:27 +00:00
// start the kubelet
2020-03-26 21:07:15 +00:00
go k.Run(podCfg.Updates())
2019-01-12 04:58:27 +00:00
// start the kubelet server
if enableServer {
2020-08-10 17:43:49 +00:00
go k.ListenAndServe(net.ParseIP(kubeCfg.Address), uint(kubeCfg.Port), kubeDeps.TLSOptions, kubeDeps.Auth,
enableCAdvisorJSONEndpoints, kubeCfg.EnableDebuggingHandlers, kubeCfg.EnableContentionProfiling, kubeCfg.EnableSystemLogHandler)
2019-01-12 04:58:27 +00:00
}
if kubeCfg.ReadOnlyPort > 0 {
2019-08-30 18:33:25 +00:00
go k.ListenAndServeReadOnly(net.ParseIP(kubeCfg.Address), uint(kubeCfg.ReadOnlyPort), enableCAdvisorJSONEndpoints)
}
if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPodResources) {
go k.ListenAndServePodResources()
2019-01-12 04:58:27 +00:00
}
}
2019-04-07 17:07:55 +00:00
func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
2019-01-12 04:58:27 +00:00
kubeDeps *kubelet.Dependencies,
crOptions *config.ContainerRuntimeOptions,
containerRuntime string,
2020-08-10 17:43:49 +00:00
hostname string,
hostnameOverridden bool,
nodeName types.NodeName,
2019-01-12 04:58:27 +00:00
nodeIP string,
providerID string,
2019-08-30 18:33:25 +00:00
cloudProvider string,
2019-01-12 04:58:27 +00:00
certDirectory string,
rootDirectory string,
registerNode bool,
registerWithTaints []api.Taint,
allowedUnsafeSysctls []string,
experimentalMounterPath string,
2020-08-10 17:43:49 +00:00
kernelMemcgNotification bool,
2019-01-12 04:58:27 +00:00
experimentalCheckNodeCapabilitiesBeforeMount bool,
experimentalNodeAllocatableIgnoreEvictionThreshold bool,
minimumGCAge metav1.Duration,
maxPerPodContainerCount int32,
maxContainerCount int32,
masterServiceNamespace string,
registerSchedulable bool,
keepTerminatedPodVolumes bool,
nodeLabels map[string]string,
seccompProfileRoot string,
nodeStatusMaxImages int32) (k kubelet.Bootstrap, err error) {
// TODO: block until all sources have delivered at least one update to the channel, or break the sync loop
// up into "per source" synchronizations
k, err = kubelet.NewMainKubelet(kubeCfg,
kubeDeps,
crOptions,
containerRuntime,
2020-08-10 17:43:49 +00:00
hostname,
hostnameOverridden,
nodeName,
2019-01-12 04:58:27 +00:00
nodeIP,
providerID,
2019-04-07 17:07:55 +00:00
cloudProvider,
2019-01-12 04:58:27 +00:00
certDirectory,
rootDirectory,
registerNode,
registerWithTaints,
allowedUnsafeSysctls,
experimentalMounterPath,
2020-08-10 17:43:49 +00:00
kernelMemcgNotification,
2019-01-12 04:58:27 +00:00
experimentalCheckNodeCapabilitiesBeforeMount,
experimentalNodeAllocatableIgnoreEvictionThreshold,
minimumGCAge,
maxPerPodContainerCount,
maxContainerCount,
masterServiceNamespace,
registerSchedulable,
keepTerminatedPodVolumes,
nodeLabels,
seccompProfileRoot,
nodeStatusMaxImages)
if err != nil {
return nil, err
}
k.BirthCry()
k.StartGarbageCollection()
return k, nil
}
// parseResourceList parses the given configuration map into an API
// ResourceList or returns an error.
func parseResourceList(m map[string]string) (v1.ResourceList, error) {
if len(m) == 0 {
return nil, nil
}
rl := make(v1.ResourceList)
for k, v := range m {
switch v1.ResourceName(k) {
2019-04-07 17:07:55 +00:00
// CPU, memory, local storage, and PID resources are supported.
case v1.ResourceCPU, v1.ResourceMemory, v1.ResourceEphemeralStorage, pidlimit.PIDs:
2019-08-30 18:33:25 +00:00
if v1.ResourceName(k) != pidlimit.PIDs || utilfeature.DefaultFeatureGate.Enabled(features.SupportNodePidsLimit) {
2019-04-07 17:07:55 +00:00
q, err := resource.ParseQuantity(v)
if err != nil {
return nil, err
}
if q.Sign() == -1 {
return nil, fmt.Errorf("resource quantity for %q cannot be negative: %v", k, v)
}
rl[v1.ResourceName(k)] = q
2019-01-12 04:58:27 +00:00
}
default:
return nil, fmt.Errorf("cannot reserve %q resource", k)
}
}
return rl, nil
}
// BootstrapKubeletConfigController constructs and bootstrap a configuration controller
func BootstrapKubeletConfigController(dynamicConfigDir string, transform dynamickubeletconfig.TransformFunc) (*kubeletconfiginternal.KubeletConfiguration, *dynamickubeletconfig.Controller, error) {
if !utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
return nil, nil, fmt.Errorf("failed to bootstrap Kubelet config controller, you must enable the DynamicKubeletConfig feature gate")
}
if len(dynamicConfigDir) == 0 {
return nil, nil, fmt.Errorf("cannot bootstrap Kubelet config controller, --dynamic-config-dir was not provided")
}
// compute absolute path and bootstrap controller
dir, err := filepath.Abs(dynamicConfigDir)
if err != nil {
return nil, nil, fmt.Errorf("failed to get absolute path for --dynamic-config-dir=%s", dynamicConfigDir)
}
// get the latest KubeletConfiguration checkpoint from disk, or return the default config if no valid checkpoints exist
c := dynamickubeletconfig.NewController(dir, transform)
kc, err := c.Bootstrap()
if err != nil {
return nil, nil, fmt.Errorf("failed to determine a valid configuration, error: %v", err)
}
return kc, c, nil
}