github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Switch to using kubelet config files instead of CLI args 

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
pull/11498/head
Brad Davidson 2024-07-01 17:47:08 +00:00 committed by Brad Davidson
parent 93e548326a
commit 6381ae93e7
7 changed files with 270 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -164,6 +164,7 @@ require (
k8s.io/klog/v2 v2.130.1
k8s.io/kube-proxy v0.0.0
k8s.io/kubectl v0.32.0
k8s.io/kubelet v0.32.0
k8s.io/kubernetes v1.32.0
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738
sigs.k8s.io/cri-tools v0.0.0-00010101000000-000000000000
@ -478,7 +479,6 @@ require (
k8s.io/kube-controller-manager v0.0.0 // indirect
k8s.io/kube-openapi v0.0.0-20241127205056-99599406b04f // indirect
k8s.io/kube-scheduler v0.0.0 // indirect
k8s.io/kubelet v0.32.0 // indirect
k8s.io/metrics v0.0.0 // indirect
k8s.io/mount-utils v0.32.0 // indirect
k8s.io/pod-security-admission v0.0.0 // indirect

7
pkg/agent/config/config.go
View File

@ -543,6 +543,12 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
return nil, err
}
// Ensure kubelet config dir exists
kubeletConfigDir := filepath.Join(envInfo.DataDir, "agent", "etc", "kubelet.conf.d")
if err := os.MkdirAll(kubeletConfigDir, 0700); err != nil {
return nil, err
}
nodeConfig := &config.Node{
Docker: envInfo.Docker,
SELinux: envInfo.EnableSELinux,
@ -571,6 +577,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain
nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
nodeConfig.AgentConfig.ClientCA = clientCAFile
nodeConfig.AgentConfig.KubeletConfigDir = kubeletConfigDir
nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
nodeConfig.AgentConfig.KubeConfigK3sController = kubeconfigK3sController

200
pkg/daemons/agent/agent.go
View File

@ -2,19 +2,34 @@ package agent
import (
"context"
"fmt"
"math/rand"
"net"
"os"
"path/filepath"
"strings"
"time"
"github.com/k3s-io/k3s/pkg/agent/config"
"github.com/k3s-io/k3s/pkg/agent/proxy"
"github.com/k3s-io/k3s/pkg/agent/util"
daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/daemons/executor"
"github.com/k3s-io/k3s/pkg/version"
"github.com/otiai10/copy"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/component-base/logs"
logsapi "k8s.io/component-base/logs/api/v1"
logsv1 "k8s.io/component-base/logs/api/v1"
_ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration
_ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration
kubeletconfig "k8s.io/kubelet/config/v1beta1"
"k8s.io/kubernetes/pkg/util/taints"
utilsnet "k8s.io/utils/net"
utilsptr "k8s.io/utils/ptr"
"sigs.k8s.io/yaml"
)
func Agent(ctx context.Context, nodeConfig *daemonconfig.Node, proxy proxy.Proxy) error {
@ -24,7 +39,7 @@ func Agent(ctx context.Context, nodeConfig *daemonconfig.Node, proxy proxy.Proxy
defer logs.FlushLogs()
if err := startKubelet(ctx, &nodeConfig.AgentConfig); err != nil {
return err
return errors.Wrap(err, "failed to start kubelet")
}
go func() {
@ -46,9 +61,21 @@ func startKubeProxy(ctx context.Context, cfg *daemonconfig.Agent) error {
}
func startKubelet(ctx context.Context, cfg *daemonconfig.Agent) error {
argsMap := kubeletArgs(cfg)
argsMap, defaultConfig, err := kubeletArgsAndConfig(cfg)
if err != nil {
return errors.Wrap(err, "prepare default configuration drop-in")
}
args := daemonconfig.GetArgs(argsMap, cfg.ExtraKubeletArgs)
extraArgs, err := extractConfigArgs(cfg.KubeletConfigDir, cfg.ExtraKubeletArgs, defaultConfig)
if err != nil {
return errors.Wrap(err, "prepare user configuration drop-ins")
}
if err := writeKubeletConfig(cfg.KubeletConfigDir, defaultConfig); err != nil {
return errors.Wrap(err, "generate default kubelet configuration drop-in")
}
args := daemonconfig.GetArgs(argsMap, extraArgs)
logrus.Infof("Running kubelet %s", daemonconfig.ArgString(args))
return executor.Kubelet(ctx, args)
@ -67,3 +94,170 @@ func ImageCredProvAvailable(cfg *daemonconfig.Agent) bool {
}
return true
}
// extractConfigArgs strips out any --config or --config-dir flags from the
// provided args list, and if set, copies the content of the file or dir into
// the target drop-in directory.
func extractConfigArgs(path string, extraArgs []string, config *kubeletconfig.KubeletConfiguration) ([]string, error) {
args := make([]string, 0, len(extraArgs))
strippedArgs := map[string]string{}
var skipVal bool
for i := range extraArgs {
if skipVal {
skipVal = false
continue
}
var val string
key := strings.TrimPrefix(extraArgs[i], "--")
if k, v, ok := strings.Cut(key, "="); ok {
// key=val pair
key = k
val = v
} else if len(extraArgs) > i+1 {
// key in this arg, value in next arg
val = extraArgs[i+1]
skipVal = true
}
switch key {
case "config", "config-dir":
if val == "" {
return nil, fmt.Errorf("value required for kubelet-arg --%s", key)
}
strippedArgs[key] = val
default:
args = append(args, extraArgs[i])
}
}
// copy the config file into our managed config dir, unless its already in there
if strippedArgs["config"] != "" && !strings.HasPrefix(strippedArgs["config"], path) {
src := strippedArgs["config"]
dest := filepath.Join(path, "10-cli-config.conf")
if err := util.CopyFile(src, dest, false); err != nil {
return nil, errors.Wrapf(err, "copy config %q into managed drop-in dir %q", src, dest)
}
}
// copy the config-dir into our managed config dir, unless its already in there
if strippedArgs["config-dir"] != "" && !strings.HasPrefix(strippedArgs["config-dir"], path) {
src := strippedArgs["config-dir"]
dest := filepath.Join(path, "20-cli-config-dir")
if err := copy.Copy(src, dest, copy.Options{PreserveOwner: true}); err != nil {
return nil, errors.Wrapf(err, "copy config-dir %q into managed drop-in dir %q", src, dest)
}
}
return args, nil
}
// writeKubeletConfig marshals the provided KubeletConfiguration object into a
// drop-in config file in the target drop-in directory.
func writeKubeletConfig(path string, config *kubeletconfig.KubeletConfiguration) error {
b, err := yaml.Marshal(config)
if err != nil {
return err
}
return os.WriteFile(filepath.Join(path, "00-"+version.Program+"-defaults.conf"), b, 0600)
}
func defaultKubeletConfig(cfg *daemonconfig.Agent) (*kubeletconfig.KubeletConfiguration, error) {
bindAddress := "127.0.0.1"
isIPv6 := utilsnet.IsIPv6(net.ParseIP([]string{cfg.NodeIP}[0]))
if isIPv6 {
bindAddress = "::1"
}
defaultConfig := &kubeletconfig.KubeletConfiguration{
TypeMeta: metav1.TypeMeta{
APIVersion: "kubelet.config.k8s.io/v1beta1",
Kind: "KubeletConfiguration",
},
CPUManagerReconcilePeriod: metav1.Duration{Duration: time.Second * 10},
CgroupDriver: "cgroupfs",
ClusterDomain: cfg.ClusterDomain,
EvictionPressureTransitionPeriod: metav1.Duration{Duration: time.Minute * 5},
FailSwapOn: utilsptr.To(false),
FileCheckFrequency: metav1.Duration{Duration: time.Second * 20},
HTTPCheckFrequency: metav1.Duration{Duration: time.Second * 20},
HealthzBindAddress: bindAddress,
ImageMinimumGCAge: metav1.Duration{Duration: time.Minute * 2},
NodeStatusReportFrequency: metav1.Duration{Duration: time.Minute * 5},
NodeStatusUpdateFrequency: metav1.Duration{Duration: time.Second * 10},
ProtectKernelDefaults: cfg.ProtectKernelDefaults,
ReadOnlyPort: 0,
RuntimeRequestTimeout: metav1.Duration{Duration: time.Minute * 2},
StreamingConnectionIdleTimeout: metav1.Duration{Duration: time.Hour * 4},
SyncFrequency: metav1.Duration{Duration: time.Minute},
VolumeStatsAggPeriod: metav1.Duration{Duration: time.Minute},
EvictionHard: map[string]string{
"imagefs.available": "5%",
"nodefs.available": "5%",
},
EvictionMinimumReclaim: map[string]string{
"imagefs.available": "10%",
"nodefs.available": "10%",
},
Authentication: kubeletconfig.KubeletAuthentication{
Anonymous: kubeletconfig.KubeletAnonymousAuthentication{
Enabled: utilsptr.To(false),
},
Webhook: kubeletconfig.KubeletWebhookAuthentication{
Enabled: utilsptr.To(true),
CacheTTL: metav1.Duration{Duration: time.Minute * 2},
},
},
Authorization: kubeletconfig.KubeletAuthorization{
Mode: kubeletconfig.KubeletAuthorizationModeWebhook,
Webhook: kubeletconfig.KubeletWebhookAuthorization{
CacheAuthorizedTTL: metav1.Duration{Duration: time.Minute * 5},
CacheUnauthorizedTTL: metav1.Duration{Duration: time.Second * 30},
},
},
Logging: logsv1.LoggingConfiguration{
Format: "text",
Verbosity: logsv1.VerbosityLevel(cfg.VLevel),
FlushFrequency: logsv1.TimeOrMetaDuration{
Duration: metav1.Duration{Duration: time.Second * 5},
SerializeAsString: true,
},
},
}
if cfg.ListenAddress != "" {
defaultConfig.Address = cfg.ListenAddress
}
if cfg.ClientCA != "" {
defaultConfig.Authentication.X509.ClientCAFile = cfg.ClientCA
}
if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" {
defaultConfig.TLSCertFile = cfg.ServingKubeletCert
defaultConfig.TLSPrivateKeyFile = cfg.ServingKubeletKey
}
for _, addr := range cfg.ClusterDNSs {
defaultConfig.ClusterDNS = append(defaultConfig.ClusterDNS, addr.String())
}
if cfg.ResolvConf != "" {
defaultConfig.ResolverConfig = utilsptr.To(cfg.ResolvConf)
}
if cfg.PodManifests != "" && defaultConfig.StaticPodPath == "" {
defaultConfig.StaticPodPath = cfg.PodManifests
}
if err := os.MkdirAll(defaultConfig.StaticPodPath, 0750); err != nil {
return nil, errors.Wrapf(err, "failed to create static pod manifest dir %s", defaultConfig.StaticPodPath)
}
if t, _, err := taints.ParseTaints(cfg.NodeTaints); err != nil {
return nil, errors.Wrap(err, "failed to parse node taints")
} else {
defaultConfig.RegisterWithTaints = t
}
logsv1.VModuleConfigurationPflag(&defaultConfig.Logging.VModule).Set(cfg.VModule)
return defaultConfig, nil
}

113
pkg/daemons/agent/agent_linux.go
View File

@ -5,7 +5,6 @@ package agent
import (
"net"
"os"
"path/filepath"
"strconv"
"strings"
@ -13,23 +12,25 @@ import (
"github.com/k3s-io/k3s/pkg/cgroups"
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/util"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
kubeletconfig "k8s.io/kubelet/config/v1beta1"
utilsnet "k8s.io/utils/net"
utilsptr "k8s.io/utils/ptr"
)
const socketPrefix = "unix://"
func createRootlessConfig(argsMap map[string]string, controllers map[string]bool) {
func createRootlessConfig(argsMap map[string]string, controllers map[string]bool) error {
argsMap["feature-gates=KubeletInUserNamespace"] = "true"
// "/sys/fs/cgroup" is namespaced
cgroupfsWritable := unix.Access("/sys/fs/cgroup", unix.W_OK) == nil
if controllers["cpu"] && controllers["pids"] && cgroupfsWritable {
logrus.Info("cgroup v2 controllers are delegated for rootless.")
return
return nil
}
logrus.Fatal("delegated cgroup v2 controllers are required for rootless.")
return errors.New("delegated cgroup v2 controllers are required for rootless")
}
func kubeProxyArgs(cfg *config.Agent) map[string]string {
@ -64,73 +65,49 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string {
return argsMap
}
func kubeletArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
if utilsnet.IsIPv6(net.ParseIP(cfg.NodeIP)) {
bindAddress = "::1"
// kubeletArgsAndConfig generates default kubelet args and configuration.
// Kubelet config is frustratingly split across deprecated CLI flags that raise warnings if you use them,
// and a structured configuration file that upstream does not provide a convienent way to initailize with default values.
// The defaults and our desired config also vary by OS.
func kubeletArgsAndConfig(cfg *config.Agent) (map[string]string, *kubeletconfig.KubeletConfiguration, error) {
defaultConfig, err := defaultKubeletConfig(cfg)
if err != nil {
return nil, nil, err
}
argsMap := map[string]string{
"healthz-bind-address": bindAddress,
"read-only-port": "0",
"cluster-domain": cfg.ClusterDomain,
"kubeconfig": cfg.KubeConfigKubelet,
"eviction-hard": "imagefs.available<5%,nodefs.available<5%",
"eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%",
"fail-swap-on": "false",
"cgroup-driver": "cgroupfs",
"authentication-token-webhook": "true",
"anonymous-auth": "false",
"authorization-mode": modes.ModeWebhook,
}
if cfg.PodManifests != "" && argsMap["pod-manifest-path"] == "" {
argsMap["pod-manifest-path"] = cfg.PodManifests
}
if err := os.MkdirAll(argsMap["pod-manifest-path"], 0755); err != nil {
logrus.Errorf("Failed to mkdir %s: %v", argsMap["pod-manifest-path"], err)
"config-dir": cfg.KubeletConfigDir,
"kubeconfig": cfg.KubeConfigKubelet,
}
if cfg.RootDir != "" {
argsMap["root-dir"] = cfg.RootDir
argsMap["cert-dir"] = filepath.Join(cfg.RootDir, "pki")
}
if len(cfg.ClusterDNS) > 0 {
argsMap["cluster-dns"] = util.JoinIPs(cfg.ClusterDNSs)
}
if cfg.ResolvConf != "" {
argsMap["resolv-conf"] = cfg.ResolvConf
}
if cfg.RuntimeSocket != "" {
argsMap["serialize-image-pulls"] = "false"
defaultConfig.SerializeImagePulls = utilsptr.To(false)
// note: this is a legacy cadvisor flag that the kubelet still exposes, but
// it must be set in order for cadvisor to pull stats properly.
if strings.Contains(cfg.RuntimeSocket, "containerd") {
argsMap["containerd"] = cfg.RuntimeSocket
}
// cadvisor wants the containerd CRI socket without the prefix, but kubelet wants it with the prefix
if strings.HasPrefix(cfg.RuntimeSocket, socketPrefix) {
argsMap["container-runtime-endpoint"] = cfg.RuntimeSocket
defaultConfig.ContainerRuntimeEndpoint = cfg.RuntimeSocket
} else {
argsMap["container-runtime-endpoint"] = socketPrefix + cfg.RuntimeSocket
defaultConfig.ContainerRuntimeEndpoint = socketPrefix + cfg.RuntimeSocket
}
}
if cfg.ImageServiceSocket != "" {
if strings.HasPrefix(cfg.ImageServiceSocket, socketPrefix) {
argsMap["image-service-endpoint"] = cfg.ImageServiceSocket
defaultConfig.ImageServiceEndpoint = cfg.ImageServiceSocket
} else {
argsMap["image-service-endpoint"] = socketPrefix + cfg.ImageServiceSocket
defaultConfig.ImageServiceEndpoint = socketPrefix + cfg.ImageServiceSocket
}
}
if cfg.ListenAddress != "" {
argsMap["address"] = cfg.ListenAddress
}
if cfg.ClientCA != "" {
argsMap["anonymous-auth"] = "false"
argsMap["client-ca-file"] = cfg.ClientCA
}
if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" {
argsMap["tls-cert-file"] = cfg.ServingKubeletCert
argsMap["tls-private-key-file"] = cfg.ServingKubeletKey
}
if cfg.NodeName != "" {
argsMap["hostname-override"] = cfg.NodeName
}
// If the embedded CCM is disabled, don't assume that dual-stack node IPs are safe.
// When using an external CCM, the user wants dual-stack node IPs, they will need to set the node-ip kubelet arg directly.
// This should be fine since most cloud providers have their own way of finding node IPs that doesn't depend on the kubelet
@ -141,33 +118,28 @@ func kubeletArgs(cfg *config.Agent) map[string]string {
argsMap["node-ip"] = cfg.NodeIP
}
} else {
argsMap["cloud-provider"] = "external"
if nodeIPs := util.JoinIPs(cfg.NodeIPs); nodeIPs != "" {
argsMap["node-ip"] = util.JoinIPs(cfg.NodeIPs)
}
}
kubeletRoot, runtimeRoot, controllers := cgroups.CheckCgroups()
if !controllers["pids"] {
return nil, nil, errors.New("pids cgroup controller not found")
}
if !controllers["cpu"] {
logrus.Warn("Disabling CPU quotas due to missing cpu controller or cpu.cfs_period_us")
argsMap["cpu-cfs-quota"] = "false"
}
if !controllers["pids"] {
logrus.Fatal("pids cgroup controller not found")
defaultConfig.CPUCFSQuota = utilsptr.To(false)
}
if kubeletRoot != "" {
argsMap["kubelet-cgroups"] = kubeletRoot
defaultConfig.KubeletCgroups = kubeletRoot
}
if runtimeRoot != "" {
argsMap["runtime-cgroups"] = runtimeRoot
}
argsMap["node-labels"] = strings.Join(cfg.NodeLabels, ",")
if len(cfg.NodeTaints) > 0 {
argsMap["register-with-taints"] = strings.Join(cfg.NodeTaints, ",")
}
if !cfg.DisableCCM {
argsMap["cloud-provider"] = "external"
}
if ImageCredProvAvailable(cfg) {
logrus.Infof("Kubelet image credential provider bin dir and configuration file found.")
@ -176,25 +148,18 @@ func kubeletArgs(cfg *config.Agent) map[string]string {
}
if cfg.Rootless {
createRootlessConfig(argsMap, controllers)
if err := createRootlessConfig(argsMap, controllers); err != nil {
return nil, nil, err
}
}
if cfg.Systemd {
argsMap["cgroup-driver"] = "systemd"
}
if cfg.ProtectKernelDefaults {
argsMap["protect-kernel-defaults"] = "true"
defaultConfig.CgroupDriver = "systemd"
}
if !cfg.DisableServiceLB {
argsMap["allowed-unsafe-sysctls"] = "net.ipv4.ip_forward,net.ipv6.conf.all.forwarding"
defaultConfig.AllowedUnsafeSysctls = []string{"net.ipv4.ip_forward", "net.ipv6.conf.all.forwarding"}
}
if cfg.VLevel != 0 {
argsMap["v"] = strconv.Itoa(cfg.VLevel)
}
if cfg.VModule != "" {
argsMap["vmodule"] = cfg.VModule
}
return argsMap
return argsMap, defaultConfig, nil
}

78
pkg/daemons/agent/agent_windows.go
View File

@ -5,15 +5,15 @@ package agent
import (
"net"
"os"
"path/filepath"
"strings"
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/util"
"github.com/sirupsen/logrus"
"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
kubeletconfig "k8s.io/kubelet/config/v1beta1"
utilsnet "k8s.io/utils/net"
utilsptr "k8s.io/utils/ptr"
)
const (
@ -38,64 +38,42 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string {
return argsMap
}
func kubeletArgs(cfg *config.Agent) map[string]string {
bindAddress := "127.0.0.1"
_, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP})
if IPv6only {
bindAddress = "::1"
// kubeletArgsAndConfig generates default kubelet args and configuration.
// Kubelet config is frustratingly split across deprecated CLI flags that raise warnings if you use them,
// and a structured configuration file that upstream does not provide a convienent way to initailize with default values.
// The defaults and our desired config also vary by OS.
func kubeletArgsAndConfig(cfg *config.Agent) (map[string]string, *kubeletconfig.KubeletConfiguration, error) {
defaultConfig, err := defaultKubeletConfig(cfg)
if err != nil {
return nil, nil, err
}
argsMap := map[string]string{
"healthz-bind-address": bindAddress,
"read-only-port": "0",
"cluster-domain": cfg.ClusterDomain,
"kubeconfig": cfg.KubeConfigKubelet,
"eviction-hard": "imagefs.available<5%,nodefs.available<5%",
"eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%",
"fail-swap-on": "false",
"authentication-token-webhook": "true",
"anonymous-auth": "false",
"authorization-mode": modes.ModeWebhook,
}
if cfg.PodManifests != "" && argsMap["pod-manifest-path"] == "" {
argsMap["pod-manifest-path"] = cfg.PodManifests
}
if err := os.MkdirAll(argsMap["pod-manifest-path"], 0755); err != nil {
logrus.Errorf("Failed to mkdir %s: %v", argsMap["pod-manifest-path"], err)
"config-dir": cfg.KubeletConfigDir,
"kubeconfig": cfg.KubeConfigKubelet,
}
if cfg.RootDir != "" {
argsMap["root-dir"] = cfg.RootDir
argsMap["cert-dir"] = filepath.Join(cfg.RootDir, "pki")
}
if len(cfg.ClusterDNS) > 0 {
argsMap["cluster-dns"] = util.JoinIPs(cfg.ClusterDNSs)
}
if cfg.ResolvConf != "" {
argsMap["resolv-conf"] = cfg.ResolvConf
}
if cfg.RuntimeSocket != "" {
argsMap["serialize-image-pulls"] = "false"
defaultConfig.SerializeImagePulls = utilsptr.To(false)
// cadvisor wants the containerd CRI socket without the prefix, but kubelet wants it with the prefix
if strings.HasPrefix(cfg.RuntimeSocket, socketPrefix) {
argsMap["container-runtime-endpoint"] = cfg.RuntimeSocket
defaultConfig.ContainerRuntimeEndpoint = cfg.RuntimeSocket
} else {
argsMap["container-runtime-endpoint"] = socketPrefix + cfg.RuntimeSocket
defaultConfig.ContainerRuntimeEndpoint = socketPrefix + cfg.RuntimeSocket
}
}
if cfg.ListenAddress != "" {
argsMap["address"] = cfg.ListenAddress
}
if cfg.ClientCA != "" {
argsMap["anonymous-auth"] = "false"
argsMap["client-ca-file"] = cfg.ClientCA
}
if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" {
argsMap["tls-cert-file"] = cfg.ServingKubeletCert
argsMap["tls-private-key-file"] = cfg.ServingKubeletKey
if cfg.ImageServiceSocket != "" {
if strings.HasPrefix(cfg.ImageServiceSocket, socketPrefix) {
defaultConfig.ImageServiceEndpoint = cfg.ImageServiceSocket
} else {
defaultConfig.ImageServiceEndpoint = socketPrefix + cfg.ImageServiceSocket
}
}
if cfg.NodeName != "" {
argsMap["hostname-override"] = cfg.NodeName
}
// If the embedded CCM is disabled, don't assume that dual-stack node IPs are safe.
// When using an external CCM, the user wants dual-stack node IPs, they will need to set the node-ip kubelet arg directly.
// This should be fine since most cloud providers have their own way of finding node IPs that doesn't depend on the kubelet
@ -106,19 +84,13 @@ func kubeletArgs(cfg *config.Agent) map[string]string {
argsMap["node-ip"] = cfg.NodeIP
}
} else {
argsMap["cloud-provider"] = "external"
if nodeIPs := util.JoinIPs(cfg.NodeIPs); nodeIPs != "" {
argsMap["node-ip"] = util.JoinIPs(cfg.NodeIPs)
}
}
argsMap["node-labels"] = strings.Join(cfg.NodeLabels, ",")
if len(cfg.NodeTaints) > 0 {
argsMap["register-with-taints"] = strings.Join(cfg.NodeTaints, ",")
}
if !cfg.DisableCCM {
argsMap["cloud-provider"] = "external"
}
if ImageCredProvAvailable(cfg) {
logrus.Infof("Kubelet image credential provider bin dir and configuration file found.")
@ -126,9 +98,5 @@ func kubeletArgs(cfg *config.Agent) map[string]string {
argsMap["image-credential-provider-config"] = cfg.ImageCredProvConfig
}
if cfg.ProtectKernelDefaults {
argsMap["protect-kernel-defaults"] = "true"
}
return argsMap
return argsMap, defaultConfig, nil
}

1
pkg/daemons/config/types.go
View File

@ -118,6 +118,7 @@ type Agent struct {
ClusterDomain string
ResolvConf string
RootDir string
KubeletConfigDir string
KubeConfigKubelet string
KubeConfigKubeProxy string
KubeConfigK3sController string

4
pkg/etcd/etcd.go
View File

@ -32,7 +32,7 @@ import (
"github.com/k3s-io/k3s/pkg/version"
"github.com/k3s-io/kine/pkg/client"
endpoint2 "github.com/k3s-io/kine/pkg/endpoint"
cp "github.com/otiai10/copy"
"github.com/otiai10/copy"
"github.com/pkg/errors"
certutil "github.com/rancher/dynamiclistener/cert"
controllerv1 "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
@ -1050,7 +1050,7 @@ func (e *ETCD) StartEmbeddedTemporary(ctx context.Context) error {
conn.Close()
}()
if err := cp.Copy(etcdDataDir, tmpDataDir, cp.Options{PreserveOwner: true}); err != nil {
if err := copy.Copy(etcdDataDir, tmpDataDir, copy.Options{PreserveOwner: true}); err != nil {
return err
}