2022-03-02 23:47:27 +00:00
|
|
|
//go:build linux
|
2021-06-01 19:29:46 +00:00
|
|
|
// +build linux
|
|
|
|
|
|
|
|
package agent
|
|
|
|
|
|
|
|
import (
|
2021-06-10 19:27:00 +00:00
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
"strings"
|
|
|
|
|
2022-03-02 23:47:27 +00:00
|
|
|
"github.com/k3s-io/k3s/pkg/cgroups"
|
|
|
|
"github.com/k3s-io/k3s/pkg/daemons/config"
|
|
|
|
"github.com/k3s-io/k3s/pkg/util"
|
2021-06-01 19:29:46 +00:00
|
|
|
"github.com/sirupsen/logrus"
|
|
|
|
"golang.org/x/sys/unix"
|
2021-06-10 19:27:00 +00:00
|
|
|
"k8s.io/apimachinery/pkg/util/net"
|
|
|
|
"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
|
2021-06-01 19:29:46 +00:00
|
|
|
)
|
|
|
|
|
2021-12-16 20:00:40 +00:00
|
|
|
const socketPrefix = "unix://"
|
|
|
|
|
2022-04-18 23:06:50 +00:00
|
|
|
func createRootlessConfig(argsMap map[string]string, controllers map[string]bool) {
|
2021-08-24 15:27:17 +00:00
|
|
|
argsMap["feature-gates=KubeletInUserNamespace"] = "true"
|
2021-06-01 19:29:46 +00:00
|
|
|
// "/sys/fs/cgroup" is namespaced
|
|
|
|
cgroupfsWritable := unix.Access("/sys/fs/cgroup", unix.W_OK) == nil
|
2022-04-18 23:06:50 +00:00
|
|
|
if controllers["cpu"] && controllers["pids"] && cgroupfsWritable {
|
2021-06-01 19:29:46 +00:00
|
|
|
logrus.Info("cgroup v2 controllers are delegated for rootless.")
|
2022-04-18 23:06:50 +00:00
|
|
|
return
|
2021-06-01 19:29:46 +00:00
|
|
|
}
|
2022-04-18 23:06:50 +00:00
|
|
|
logrus.Fatal("delegated cgroup v2 controllers are required for rootless.")
|
2021-06-01 19:29:46 +00:00
|
|
|
}
|
2021-06-10 19:27:00 +00:00
|
|
|
|
|
|
|
func kubeProxyArgs(cfg *config.Agent) map[string]string {
|
2021-11-10 19:23:05 +00:00
|
|
|
bindAddress := "127.0.0.1"
|
|
|
|
_, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP})
|
|
|
|
if IPv6only {
|
|
|
|
bindAddress = "::1"
|
|
|
|
}
|
2021-06-10 19:27:00 +00:00
|
|
|
argsMap := map[string]string{
|
|
|
|
"proxy-mode": "iptables",
|
2021-11-10 19:23:05 +00:00
|
|
|
"healthz-bind-address": bindAddress,
|
2021-06-10 19:27:00 +00:00
|
|
|
"kubeconfig": cfg.KubeConfigKubeProxy,
|
|
|
|
"cluster-cidr": util.JoinIPNets(cfg.ClusterCIDRs),
|
|
|
|
"conntrack-max-per-core": "0",
|
|
|
|
"conntrack-tcp-timeout-established": "0s",
|
|
|
|
"conntrack-tcp-timeout-close-wait": "0s",
|
|
|
|
}
|
|
|
|
if cfg.NodeName != "" {
|
|
|
|
argsMap["hostname-override"] = cfg.NodeName
|
|
|
|
}
|
|
|
|
return argsMap
|
|
|
|
}
|
|
|
|
|
|
|
|
func kubeletArgs(cfg *config.Agent) map[string]string {
|
2021-11-10 19:23:05 +00:00
|
|
|
bindAddress := "127.0.0.1"
|
|
|
|
_, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP})
|
|
|
|
if IPv6only {
|
|
|
|
bindAddress = "::1"
|
|
|
|
}
|
2021-06-10 19:27:00 +00:00
|
|
|
argsMap := map[string]string{
|
2022-04-18 23:06:50 +00:00
|
|
|
"healthz-bind-address": bindAddress,
|
|
|
|
"read-only-port": "0",
|
|
|
|
"cluster-domain": cfg.ClusterDomain,
|
|
|
|
"kubeconfig": cfg.KubeConfigKubelet,
|
|
|
|
"eviction-hard": "imagefs.available<5%,nodefs.available<5%",
|
|
|
|
"eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%",
|
|
|
|
"fail-swap-on": "false",
|
2021-06-10 19:27:00 +00:00
|
|
|
"cgroup-driver": "cgroupfs",
|
|
|
|
"authentication-token-webhook": "true",
|
|
|
|
"anonymous-auth": "false",
|
|
|
|
"authorization-mode": modes.ModeWebhook,
|
|
|
|
}
|
|
|
|
if cfg.PodManifests != "" && argsMap["pod-manifest-path"] == "" {
|
|
|
|
argsMap["pod-manifest-path"] = cfg.PodManifests
|
|
|
|
}
|
|
|
|
if err := os.MkdirAll(argsMap["pod-manifest-path"], 0755); err != nil {
|
|
|
|
logrus.Errorf("Failed to mkdir %s: %v", argsMap["pod-manifest-path"], err)
|
|
|
|
}
|
|
|
|
if cfg.RootDir != "" {
|
|
|
|
argsMap["root-dir"] = cfg.RootDir
|
|
|
|
argsMap["cert-dir"] = filepath.Join(cfg.RootDir, "pki")
|
|
|
|
}
|
|
|
|
if len(cfg.ClusterDNS) > 0 {
|
|
|
|
argsMap["cluster-dns"] = util.JoinIPs(cfg.ClusterDNSs)
|
|
|
|
}
|
|
|
|
if cfg.ResolvConf != "" {
|
|
|
|
argsMap["resolv-conf"] = cfg.ResolvConf
|
|
|
|
}
|
|
|
|
if cfg.RuntimeSocket != "" {
|
|
|
|
argsMap["serialize-image-pulls"] = "false"
|
2021-12-16 20:00:40 +00:00
|
|
|
if strings.Contains(cfg.RuntimeSocket, "containerd") {
|
|
|
|
argsMap["containerd"] = cfg.RuntimeSocket
|
|
|
|
}
|
|
|
|
// cadvisor wants the containerd CRI socket without the prefix, but kubelet wants it with the prefix
|
|
|
|
if strings.HasPrefix(cfg.RuntimeSocket, socketPrefix) {
|
|
|
|
argsMap["container-runtime-endpoint"] = cfg.RuntimeSocket
|
|
|
|
} else {
|
|
|
|
argsMap["container-runtime-endpoint"] = socketPrefix + cfg.RuntimeSocket
|
|
|
|
}
|
2022-03-11 01:01:53 +00:00
|
|
|
}
|
|
|
|
if cfg.PauseImage != "" {
|
2021-06-10 19:27:00 +00:00
|
|
|
argsMap["pod-infra-container-image"] = cfg.PauseImage
|
|
|
|
}
|
2021-09-01 23:27:42 +00:00
|
|
|
if cfg.ImageServiceSocket != "" {
|
2021-12-16 20:00:40 +00:00
|
|
|
if strings.HasPrefix(cfg.ImageServiceSocket, socketPrefix) {
|
2021-09-01 23:27:42 +00:00
|
|
|
argsMap["image-service-endpoint"] = cfg.ImageServiceSocket
|
|
|
|
} else {
|
2021-12-16 20:00:40 +00:00
|
|
|
argsMap["image-service-endpoint"] = socketPrefix + cfg.ImageServiceSocket
|
2021-09-01 23:27:42 +00:00
|
|
|
}
|
|
|
|
}
|
2021-06-10 19:27:00 +00:00
|
|
|
if cfg.ListenAddress != "" {
|
|
|
|
argsMap["address"] = cfg.ListenAddress
|
|
|
|
}
|
|
|
|
if cfg.ClientCA != "" {
|
|
|
|
argsMap["anonymous-auth"] = "false"
|
|
|
|
argsMap["client-ca-file"] = cfg.ClientCA
|
|
|
|
}
|
|
|
|
if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" {
|
|
|
|
argsMap["tls-cert-file"] = cfg.ServingKubeletCert
|
|
|
|
argsMap["tls-private-key-file"] = cfg.ServingKubeletKey
|
|
|
|
}
|
|
|
|
if cfg.NodeName != "" {
|
|
|
|
argsMap["hostname-override"] = cfg.NodeName
|
|
|
|
}
|
2022-06-02 22:21:46 +00:00
|
|
|
defaultIP, err := net.ChooseHostInterface()
|
|
|
|
if err != nil || defaultIP.String() != cfg.NodeIP {
|
|
|
|
argsMap["node-ip"] = cfg.NodeIP
|
2021-06-10 19:27:00 +00:00
|
|
|
}
|
2022-04-18 23:06:50 +00:00
|
|
|
kubeletRoot, runtimeRoot, controllers := cgroups.CheckCgroups()
|
|
|
|
if !controllers["cpu"] {
|
|
|
|
logrus.Warn("Disabling CPU quotas due to missing cpu controller or cpu.cfs_period_us")
|
2021-06-10 19:27:00 +00:00
|
|
|
argsMap["cpu-cfs-quota"] = "false"
|
|
|
|
}
|
2022-04-18 23:06:50 +00:00
|
|
|
if !controllers["pids"] {
|
|
|
|
logrus.Fatal("pids cgroup controller not found")
|
2021-06-10 19:27:00 +00:00
|
|
|
}
|
|
|
|
if kubeletRoot != "" {
|
|
|
|
argsMap["kubelet-cgroups"] = kubeletRoot
|
|
|
|
}
|
|
|
|
if runtimeRoot != "" {
|
|
|
|
argsMap["runtime-cgroups"] = runtimeRoot
|
|
|
|
}
|
|
|
|
|
|
|
|
argsMap["node-labels"] = strings.Join(cfg.NodeLabels, ",")
|
|
|
|
if len(cfg.NodeTaints) > 0 {
|
|
|
|
argsMap["register-with-taints"] = strings.Join(cfg.NodeTaints, ",")
|
|
|
|
}
|
|
|
|
|
2022-06-01 19:54:07 +00:00
|
|
|
if !cfg.DisableCCM {
|
|
|
|
argsMap["cloud-provider"] = "external"
|
|
|
|
}
|
|
|
|
|
2021-06-10 19:27:00 +00:00
|
|
|
if ImageCredProvAvailable(cfg) {
|
|
|
|
logrus.Infof("Kubelet image credential provider bin dir and configuration file found.")
|
2021-07-28 20:04:42 +00:00
|
|
|
argsMap["feature-gates"] = util.AddFeatureGate(argsMap["feature-gates"], "KubeletCredentialProviders=true")
|
2021-06-10 19:27:00 +00:00
|
|
|
argsMap["image-credential-provider-bin-dir"] = cfg.ImageCredProvBinDir
|
|
|
|
argsMap["image-credential-provider-config"] = cfg.ImageCredProvConfig
|
|
|
|
}
|
|
|
|
|
|
|
|
if cfg.Rootless {
|
2022-04-18 23:06:50 +00:00
|
|
|
createRootlessConfig(argsMap, controllers)
|
|
|
|
}
|
|
|
|
|
|
|
|
if cfg.Systemd {
|
|
|
|
argsMap["cgroup-driver"] = "systemd"
|
2021-06-10 19:27:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if cfg.ProtectKernelDefaults {
|
|
|
|
argsMap["protect-kernel-defaults"] = "true"
|
|
|
|
}
|
2021-11-09 15:44:34 +00:00
|
|
|
|
2022-09-29 20:37:50 +00:00
|
|
|
if !cfg.DisableServiceLB {
|
|
|
|
argsMap["allowed-unsafe-sysctls"] = "net.ipv4.ip_forward,net.ipv6.conf.all.forwarding"
|
2021-11-09 15:44:34 +00:00
|
|
|
}
|
|
|
|
|
2021-06-10 19:27:00 +00:00
|
|
|
return argsMap
|
|
|
|
}
|