diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 9a5a16232f..2ef55a3228 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -249,6 +249,9 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { os.Setenv("NODE_NAME", nodeConfig.AgentConfig.NodeName) v1beta1.KubeletSocket = filepath.Join(envInfo.DataDir, "kubelet/device-plugins/kubelet.sock") + nodeConfig.AgentConfig.ExtraKubeletArgs = envInfo.ExtraKubeletArgs + nodeConfig.AgentConfig.ExtraKubeProxyArgs = envInfo.ExtraKubeProxyArgs + return nodeConfig, nil } diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go index 1c1f11e56d..3e56051ad6 100644 --- a/pkg/cli/cmds/agent.go +++ b/pkg/cli/cmds/agent.go @@ -21,6 +21,8 @@ type Agent struct { NoFlannel bool Debug bool AgentShared + ExtraKubeletArgs cli.StringSlice + ExtraKubeProxyArgs cli.StringSlice } type AgentShared struct { @@ -62,6 +64,16 @@ var ( EnvVar: "K3S_RESOLV_CONF", Destination: &AgentConfig.ResolvConf, } + ExtraKubeletArgs = cli.StringSliceFlag{ + Name: "kubelet-arg", + Usage: "(agent) Customized flag for kubelet process", + Value: &AgentConfig.ExtraKubeletArgs, + } + ExtraKubeProxyArgs = cli.StringSliceFlag{ + Name: "kube-proxy-arg", + Usage: "(agent) Customized flag for kube-proxy process", + Value: &AgentConfig.ExtraKubeProxyArgs, + } ) func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { @@ -107,6 +119,8 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { NodeIPFlag, CRIEndpointFlag, ResolvConfFlag, + ExtraKubeletArgs, + ExtraKubeProxyArgs, }, } } diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index 7a9ac82fa7..5453ab9014 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -5,19 +5,22 @@ import ( ) type Server struct { - Log string - ClusterCIDR string - ClusterSecret string - ServiceCIDR string - ClusterDNS string - HTTPSPort int - HTTPPort int - DataDir string - DisableAgent bool - KubeConfigOutput string - KubeConfigMode string - KnownIPs cli.StringSlice - BindAddress string + Log string + ClusterCIDR string + ClusterSecret string + ServiceCIDR string + ClusterDNS string + HTTPSPort int + HTTPPort int + DataDir string + DisableAgent bool + KubeConfigOutput string + KubeConfigMode string + KnownIPs cli.StringSlice + BindAddress string + ExtraAPIArgs cli.StringSlice + ExtraSchedulerArgs cli.StringSlice + ExtraControllerArgs cli.StringSlice } var ServerConfig Server @@ -106,12 +109,29 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { Usage: "Add additional hostname or IP as a Subject Alternative Name in the TLS cert", Value: &ServerConfig.KnownIPs, }, + cli.StringSliceFlag{ + Name: "kube-apiserver-arg", + Usage: "Customized flag for kube-apiserver process", + Value: &ServerConfig.ExtraAPIArgs, + }, + cli.StringSliceFlag{ + Name: "kube-scheduler-arg", + Usage: "Customized flag for kube-scheduler process", + Value: &ServerConfig.ExtraSchedulerArgs, + }, + cli.StringSliceFlag{ + Name: "kube-controller-arg", + Usage: "Customized flag for kube-controller-manager process", + Value: &ServerConfig.ExtraControllerArgs, + }, NodeIPFlag, NodeNameFlag, DockerFlag, FlannelFlag, CRIEndpointFlag, ResolvConfFlag, + ExtraKubeletArgs, + ExtraKubeProxyArgs, }, } } diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index fe0b086b63..8961532df4 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -83,6 +83,9 @@ func run(app *cli.Context, cfg *cmds.Server) error { serverConfig.TLSConfig.HTTPPort = cfg.HTTPPort serverConfig.TLSConfig.KnownIPs = knownIPs(cfg.KnownIPs) serverConfig.TLSConfig.BindAddress = cfg.BindAddress + serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs + serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs + serverConfig.ControlConfig.ExtraSchedulerAPIArgs = cfg.ExtraSchedulerArgs _, serverConfig.ControlConfig.ClusterIPRange, err = net2.ParseCIDR(cfg.ClusterCIDR) if err != nil { diff --git a/pkg/daemons/agent/agent.go b/pkg/daemons/agent/agent.go index 7d29118653..862ea28e47 100644 --- a/pkg/daemons/agent/agent.go +++ b/pkg/daemons/agent/agent.go @@ -28,14 +28,14 @@ func Agent(config *config.Agent) error { return nil } -func kubeProxy(config *config.Agent) { - args := []string{ - "--proxy-mode", "iptables", - "--healthz-bind-address", "127.0.0.1", - "--kubeconfig", config.KubeConfig, - "--cluster-cidr", config.ClusterCIDR.String(), +func kubeProxy(cfg *config.Agent) { + argsMap := map[string]string{ + "proxy-mode": "iptables", + "healthz-bind-address": "127.0.0.1", + "kubeconfig": cfg.KubeConfig, + "cluster-cidr": cfg.ClusterCIDR.String(), } - args = append(args, config.ExtraKubeletArgs...) + args := config.GetArgsList(argsMap, cfg.ExtraKubeProxyArgs) command := app2.NewProxyCommand() command.SetArgs(args) @@ -50,63 +50,64 @@ func kubelet(cfg *config.Agent) { logs.InitLogs() defer logs.FlushLogs() - args := []string{ - "--healthz-bind-address", "127.0.0.1", - "--read-only-port", "0", - "--allow-privileged=true", - "--cluster-domain", "cluster.local", - "--kubeconfig", cfg.KubeConfig, - "--eviction-hard", "imagefs.available<5%,nodefs.available<5%", - "--eviction-minimum-reclaim", "imagefs.available=10%,nodefs.available=10%", - "--fail-swap-on=false", - //"--cgroup-root", "/k3s", - "--cgroup-driver", "cgroupfs", + argsMap := map[string]string{ + "healthz-bind-address": "127.0.0.1", + "read-only-port": "0", + "allow-privileged": "true", + "cluster-domain": "cluster.local", + "kubeconfig": cfg.KubeConfig, + "eviction-hard": "imagefs.available<5%,nodefs.available<5%", + "eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%", + "fail-swap-on": "false", + //"cgroup-root": "/k3s", + "cgroup-driver": "cgroupfs", } if cfg.RootDir != "" { - args = append(args, "--root-dir", cfg.RootDir) - args = append(args, "--cert-dir", filepath.Join(cfg.RootDir, "pki")) - args = append(args, "--seccomp-profile-root", filepath.Join(cfg.RootDir, "seccomp")) + argsMap["root-dir"] = cfg.RootDir + argsMap["cert-dir"] = filepath.Join(cfg.RootDir, "pki") + argsMap["seccomp-profile-root"] = filepath.Join(cfg.RootDir, "seccomp") } if cfg.CNIConfDir != "" { - args = append(args, "--cni-conf-dir", cfg.CNIConfDir) + argsMap["cni-conf-dir"] = cfg.CNIConfDir } if cfg.CNIBinDir != "" { - args = append(args, "--cni-bin-dir", cfg.CNIBinDir) + argsMap["cni-bin-dir"] = cfg.CNIBinDir } if len(cfg.ClusterDNS) > 0 { - args = append(args, "--cluster-dns", cfg.ClusterDNS.String()) + argsMap["cluster-dns"] = cfg.ClusterDNS.String() } if cfg.ResolvConf != "" { - args = append(args, "--resolv-conf", cfg.ResolvConf) + argsMap["resolv-conf"] = cfg.ResolvConf } if cfg.RuntimeSocket != "" { - args = append(args, "--container-runtime", "remote") - args = append(args, "--container-runtime-endpoint", cfg.RuntimeSocket) - args = append(args, "--serialize-image-pulls=false") + argsMap["container-runtime"] = "remote" + argsMap["container-runtime-endpoint"] = cfg.RuntimeSocket + argsMap["serialize-image-pulls"] = "false" } if cfg.ListenAddress != "" { - args = append(args, "--address", cfg.ListenAddress) + argsMap["address"] = cfg.ListenAddress } if cfg.CACertPath != "" { - args = append(args, "--anonymous-auth=false", "--client-ca-file", cfg.CACertPath) + argsMap["anonymous-auth"] = "false" + argsMap["client-ca-file"] = cfg.CACertPath } if cfg.NodeName != "" { - args = append(args, "--hostname-override", cfg.NodeName) + argsMap["hostname-override"] = cfg.NodeName } defaultIP, err := net.ChooseHostInterface() if err != nil || defaultIP.String() != cfg.NodeIP { - args = append(args, "--node-ip", cfg.NodeIP) + argsMap["node-ip"] = cfg.NodeIP } root, hasCFS := checkCgroups() if !hasCFS { logrus.Warn("Disabling CPU quotas due to missing cpu.cfs_period_us") - args = append(args, "--cpu-cfs-quota=false") + argsMap["cpu-cfs-quota"] = "false" } if root != "" { - args = append(args, "--runtime-cgroups", root) - args = append(args, "--kubelet-cgroups", root) + argsMap["runtime-cgroups"] = root + argsMap["kubelet-cgroups"] = root } - args = append(args, cfg.ExtraKubeletArgs...) + args := config.GetArgsList(argsMap, cfg.ExtraKubeletArgs) command.SetArgs(args) diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 26cd4ccec2..f7ad7aae5c 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -2,6 +2,7 @@ package config import ( "crypto/tls" + "fmt" "net" "net/http" "strings" @@ -106,3 +107,21 @@ func (a ArgString) String() string { } return b.String() } + +func GetArgsList(argsMap map[string]string, extraArgs []string) []string { + // add extra args to args map to override any default option + for _, arg := range extraArgs { + splitArg := strings.Split(arg, "=") + if len(splitArg) < 2 { + argsMap[splitArg[0]] = "true" + continue + } + argsMap[splitArg[0]] = splitArg[1] + } + var args []string + for arg, value := range argsMap { + cmd := fmt.Sprintf("--%s=%s", arg, value) + args = append(args, cmd) + } + return args +} diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 83dae6ea3f..80ca9e1fc5 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -95,20 +95,22 @@ func Server(ctx context.Context, cfg *config.Control) error { } func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) { - args := []string{ - "--kubeconfig", runtime.KubeConfigSystem, - "--service-account-private-key-file", runtime.ServiceKey, - "--allocate-node-cidrs", - "--cluster-cidr", cfg.ClusterIPRange.String(), - "--root-ca-file", runtime.TokenCA, - "--port", "10252", - "--bind-address", "127.0.0.1", - "--secure-port", "0", + argsMap := map[string]string{ + "kubeconfig": runtime.KubeConfigSystem, + "service-account-private-key-file": runtime.ServiceKey, + "allocate-node-cidrs": "true", + "cluster-cidr": cfg.ClusterIPRange.String(), + "root-ca-file": runtime.TokenCA, + "port": "10252", + "bind-address": "127.0.0.1", + "secure-port": "0", } if cfg.NoLeaderElect { - args = append(args, "--leader-elect=false") + argsMap["leader-elect"] = "false" } - args = append(args, cfg.ExtraControllerArgs...) + + args := config.GetArgsList(argsMap, cfg.ExtraControllerArgs) + command := cmapp.NewControllerManagerCommand() command.SetArgs(args) @@ -119,16 +121,17 @@ func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) { } func scheduler(cfg *config.Control, runtime *config.ControlRuntime) { - args := []string{ - "--kubeconfig", runtime.KubeConfigSystem, - "--port", "10251", - "--bind-address", "127.0.0.1", - "--secure-port", "0", + argsMap := map[string]string{ + "kubeconfig": runtime.KubeConfigSystem, + "port": "10251", + "bind-address": "127.0.0.1", + "secure-port": "0", } if cfg.NoLeaderElect { - args = append(args, "--leader-elect=false") + argsMap["leader-elect"] = "false" } - args = append(args, cfg.ExtraSchedulerAPIArgs...) + args := config.GetArgsList(argsMap, cfg.ExtraSchedulerAPIArgs) + command := sapp.NewSchedulerCommand() command.SetArgs(args) @@ -139,19 +142,18 @@ func scheduler(cfg *config.Control, runtime *config.ControlRuntime) { } func apiServer(ctx context.Context, cfg *config.Control, runtime *config.ControlRuntime) (authenticator.Request, http.Handler, error) { - var args []string - + argsMap := make(map[string]string) if len(cfg.ETCDEndpoints) > 0 { - args = append(args, "--storage-backend", "etcd3") - args = append(args, "--etcd-servers", strings.Join(cfg.ETCDEndpoints, ",")) + argsMap["storage-backend"] = "etcd3" + argsMap["etcd-servers"] = strings.Join(cfg.ETCDEndpoints, ",") if cfg.ETCDKeyFile != "" { - args = append(args, "--etcd-keyfile", cfg.ETCDKeyFile) + argsMap["etcd-keyfile"] = cfg.ETCDKeyFile } if cfg.ETCDCAFile != "" { - args = append(args, "--etcd-cafile", cfg.ETCDCAFile) + argsMap["etcd-cafile"] = cfg.ETCDCAFile } if cfg.ETCDCertFile != "" { - args = append(args, "--etcd-certfile", cfg.ETCDCertFile) + argsMap["etcd-certfile"] = cfg.ETCDCertFile } } @@ -159,27 +161,28 @@ func apiServer(ctx context.Context, cfg *config.Control, runtime *config.Control os.MkdirAll(certDir, 0700) // TODO: sqlite doesn't need the watch cache, but etcd does, so make this dynamic - args = append(args, "--watch-cache=false") - args = append(args, "--cert-dir", certDir) - args = append(args, "--allow-privileged=true") - args = append(args, "--authorization-mode", strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ",")) - args = append(args, "--service-account-signing-key-file", runtime.ServiceKey) - args = append(args, "--service-cluster-ip-range", cfg.ServiceIPRange.String()) - args = append(args, "--advertise-port", strconv.Itoa(cfg.AdvertisePort)) - args = append(args, "--advertise-address", localhostIP.String()) - args = append(args, "--insecure-port", "0") - args = append(args, "--secure-port", strconv.Itoa(cfg.ListenPort)) - args = append(args, "--bind-address", localhostIP.String()) - args = append(args, "--tls-cert-file", runtime.TLSCert) - args = append(args, "--tls-private-key-file", runtime.TLSKey) - args = append(args, "--service-account-key-file", runtime.ServiceKey) - args = append(args, "--service-account-issuer", "k3s") - args = append(args, "--api-audiences", "unknown") - args = append(args, "--basic-auth-file", runtime.PasswdFile) - args = append(args, "--kubelet-client-certificate", runtime.NodeCert) - args = append(args, "--kubelet-client-key", runtime.NodeKey) + argsMap["watch-cache"] = "false" + argsMap["cert-dir"] = certDir + argsMap["allow-privileged"] = "true" + argsMap["authorization-mode"] = strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ",") + argsMap["service-account-signing-key-file"] = runtime.ServiceKey + argsMap["service-cluster-ip-range"] = cfg.ServiceIPRange.String() + argsMap["advertise-port"] = strconv.Itoa(cfg.AdvertisePort) + argsMap["advertise-address"] = localhostIP.String() + argsMap["insecure-port"] = "0" + argsMap["secure-port"] = strconv.Itoa(cfg.ListenPort) + argsMap["bind-address"] = localhostIP.String() + argsMap["tls-cert-file"] = runtime.TLSCert + argsMap["tls-private-key-file"] = runtime.TLSKey + argsMap["service-account-key-file"] = runtime.ServiceKey + argsMap["service-account-issuer"] = "k3s" + argsMap["api-audiences"] = "unknown" + argsMap["basic-auth-file"] = runtime.PasswdFile + argsMap["kubelet-client-certificate"] = runtime.NodeCert + argsMap["kubelet-client-key"] = runtime.NodeKey + + args := config.GetArgsList(argsMap, cfg.ExtraAPIArgs) - args = append(args, cfg.ExtraAPIArgs...) command := app.NewAPIServerCommand(ctx.Done()) command.SetArgs(args)