mirror of https://github.com/k3s-io/k3s
211 lines
6.0 KiB
Go
211 lines
6.0 KiB
Go
package agent
|
|
|
|
import (
|
|
"bufio"
|
|
"math/rand"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/opencontainers/runc/libcontainer/system"
|
|
"github.com/rancher/k3s/pkg/daemons/config"
|
|
"github.com/rancher/k3s/pkg/daemons/executor"
|
|
"github.com/sirupsen/logrus"
|
|
"k8s.io/apimachinery/pkg/util/net"
|
|
"k8s.io/component-base/logs"
|
|
"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
|
|
|
|
_ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration
|
|
_ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration
|
|
)
|
|
|
|
func Agent(config *config.Agent) error {
|
|
rand.Seed(time.Now().UTC().UnixNano())
|
|
|
|
logs.InitLogs()
|
|
defer logs.FlushLogs()
|
|
|
|
if err := startKubelet(config); err != nil {
|
|
return err
|
|
}
|
|
|
|
if !config.DisableKubeProxy {
|
|
return startKubeProxy(config)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func startKubeProxy(cfg *config.Agent) error {
|
|
argsMap := map[string]string{
|
|
"proxy-mode": "iptables",
|
|
"healthz-bind-address": "127.0.0.1",
|
|
"kubeconfig": cfg.KubeConfigKubeProxy,
|
|
"cluster-cidr": cfg.ClusterCIDR.String(),
|
|
}
|
|
if cfg.NodeName != "" {
|
|
argsMap["hostname-override"] = cfg.NodeName
|
|
}
|
|
|
|
args := config.GetArgsList(argsMap, cfg.ExtraKubeProxyArgs)
|
|
logrus.Infof("Running kube-proxy %s", config.ArgString(args))
|
|
return executor.KubeProxy(args)
|
|
}
|
|
|
|
func startKubelet(cfg *config.Agent) error {
|
|
argsMap := map[string]string{
|
|
"healthz-bind-address": "127.0.0.1",
|
|
"read-only-port": "0",
|
|
"cluster-domain": cfg.ClusterDomain,
|
|
"kubeconfig": cfg.KubeConfigKubelet,
|
|
"eviction-hard": "imagefs.available<5%,nodefs.available<5%",
|
|
"eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%",
|
|
"fail-swap-on": "false",
|
|
//"cgroup-root": "/k3s",
|
|
"cgroup-driver": "cgroupfs",
|
|
"authentication-token-webhook": "true",
|
|
"anonymous-auth": "false",
|
|
"authorization-mode": modes.ModeWebhook,
|
|
}
|
|
if cfg.PodManifests != "" && argsMap["pod-manifest-path"] == "" {
|
|
argsMap["pod-manifest-path"] = cfg.PodManifests
|
|
}
|
|
if err := os.MkdirAll(argsMap["pod-manifest-path"], 0755); err != nil {
|
|
logrus.Errorf("Failed to mkdir %s: %v", argsMap["pod-manifest-path"], err)
|
|
}
|
|
if cfg.RootDir != "" {
|
|
argsMap["root-dir"] = cfg.RootDir
|
|
argsMap["cert-dir"] = filepath.Join(cfg.RootDir, "pki")
|
|
argsMap["seccomp-profile-root"] = filepath.Join(cfg.RootDir, "seccomp")
|
|
}
|
|
if cfg.CNIConfDir != "" {
|
|
argsMap["cni-conf-dir"] = cfg.CNIConfDir
|
|
}
|
|
if cfg.CNIBinDir != "" {
|
|
argsMap["cni-bin-dir"] = cfg.CNIBinDir
|
|
}
|
|
if cfg.CNIPlugin {
|
|
argsMap["network-plugin"] = "cni"
|
|
}
|
|
if len(cfg.ClusterDNS) > 0 {
|
|
argsMap["cluster-dns"] = cfg.ClusterDNS.String()
|
|
}
|
|
if cfg.ResolvConf != "" {
|
|
argsMap["resolv-conf"] = cfg.ResolvConf
|
|
}
|
|
if cfg.RuntimeSocket != "" {
|
|
argsMap["container-runtime"] = "remote"
|
|
argsMap["container-runtime-endpoint"] = cfg.RuntimeSocket
|
|
argsMap["containerd"] = cfg.RuntimeSocket
|
|
argsMap["serialize-image-pulls"] = "false"
|
|
} else if cfg.PauseImage != "" {
|
|
argsMap["pod-infra-container-image"] = cfg.PauseImage
|
|
}
|
|
if cfg.ListenAddress != "" {
|
|
argsMap["address"] = cfg.ListenAddress
|
|
}
|
|
if cfg.ClientCA != "" {
|
|
argsMap["anonymous-auth"] = "false"
|
|
argsMap["client-ca-file"] = cfg.ClientCA
|
|
}
|
|
if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" {
|
|
argsMap["tls-cert-file"] = cfg.ServingKubeletCert
|
|
argsMap["tls-private-key-file"] = cfg.ServingKubeletKey
|
|
}
|
|
if cfg.NodeName != "" {
|
|
argsMap["hostname-override"] = cfg.NodeName
|
|
}
|
|
defaultIP, err := net.ChooseHostInterface()
|
|
if err != nil || defaultIP.String() != cfg.NodeIP {
|
|
argsMap["node-ip"] = cfg.NodeIP
|
|
}
|
|
root, hasCFS, hasPIDs := checkCgroups()
|
|
if !hasCFS {
|
|
logrus.Warn("Disabling CPU quotas due to missing cpu.cfs_period_us")
|
|
argsMap["cpu-cfs-quota"] = "false"
|
|
}
|
|
if !hasPIDs {
|
|
logrus.Warn("Disabling pod PIDs limit feature due to missing cgroup pids support")
|
|
argsMap["cgroups-per-qos"] = "false"
|
|
argsMap["enforce-node-allocatable"] = ""
|
|
argsMap["feature-gates"] = addFeatureGate(argsMap["feature-gates"], "SupportPodPidsLimit=false")
|
|
}
|
|
if root != "" {
|
|
argsMap["runtime-cgroups"] = root
|
|
argsMap["kubelet-cgroups"] = root
|
|
}
|
|
if system.RunningInUserNS() {
|
|
argsMap["feature-gates"] = addFeatureGate(argsMap["feature-gates"], "DevicePlugins=false")
|
|
}
|
|
|
|
argsMap["node-labels"] = strings.Join(cfg.NodeLabels, ",")
|
|
if len(cfg.NodeTaints) > 0 {
|
|
argsMap["register-with-taints"] = strings.Join(cfg.NodeTaints, ",")
|
|
}
|
|
if !cfg.DisableCCM {
|
|
argsMap["cloud-provider"] = "external"
|
|
}
|
|
|
|
if cfg.Rootless {
|
|
// flags are from https://github.com/rootless-containers/usernetes/blob/v20190826.0/boot/kubelet.sh
|
|
argsMap["cgroup-driver"] = "none"
|
|
argsMap["feature-gates=SupportNoneCgroupDriver"] = "true"
|
|
argsMap["cgroups-per-qos"] = "false"
|
|
argsMap["enforce-node-allocatable"] = ""
|
|
}
|
|
|
|
if cfg.ProtectKernelDefaults {
|
|
argsMap["protect-kernel-defaults"] = "true"
|
|
}
|
|
|
|
args := config.GetArgsList(argsMap, cfg.ExtraKubeletArgs)
|
|
logrus.Infof("Running kubelet %s", config.ArgString(args))
|
|
|
|
return executor.Kubelet(args)
|
|
}
|
|
|
|
func addFeatureGate(current, new string) string {
|
|
if current == "" {
|
|
return new
|
|
}
|
|
return current + "," + new
|
|
}
|
|
|
|
func checkCgroups() (root string, hasCFS bool, hasPIDs bool) {
|
|
f, err := os.Open("/proc/self/cgroup")
|
|
if err != nil {
|
|
return "", false, false
|
|
}
|
|
defer f.Close()
|
|
|
|
scan := bufio.NewScanner(f)
|
|
for scan.Scan() {
|
|
parts := strings.Split(scan.Text(), ":")
|
|
if len(parts) < 3 {
|
|
continue
|
|
}
|
|
systems := strings.Split(parts[1], ",")
|
|
for _, system := range systems {
|
|
if system == "pids" {
|
|
hasPIDs = true
|
|
} else if system == "cpu" {
|
|
p := filepath.Join("/sys/fs/cgroup", parts[1], parts[2], "cpu.cfs_period_us")
|
|
if _, err := os.Stat(p); err == nil {
|
|
hasCFS = true
|
|
}
|
|
} else if system == "name=systemd" {
|
|
last := parts[len(parts)-1]
|
|
i := strings.LastIndex(last, ".slice")
|
|
if i > 0 {
|
|
root = "/systemd" + last[:i+len(".slice")]
|
|
} else {
|
|
root = "/systemd"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return root, hasCFS, hasPIDs
|
|
}
|