2019-01-01 08:23:01 +00:00
|
|
|
package control
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"math/rand"
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
"strconv"
|
|
|
|
"strings"
|
|
|
|
"time"
|
|
|
|
|
2022-03-28 20:45:39 +00:00
|
|
|
"github.com/k3s-io/k3s/pkg/authenticator"
|
2022-03-02 23:47:27 +00:00
|
|
|
"github.com/k3s-io/k3s/pkg/cluster"
|
|
|
|
"github.com/k3s-io/k3s/pkg/daemons/config"
|
|
|
|
"github.com/k3s-io/k3s/pkg/daemons/control/deps"
|
|
|
|
"github.com/k3s-io/k3s/pkg/daemons/executor"
|
|
|
|
"github.com/k3s-io/k3s/pkg/util"
|
|
|
|
"github.com/k3s-io/k3s/pkg/version"
|
2019-12-16 18:44:13 +00:00
|
|
|
"github.com/pkg/errors"
|
2019-01-01 08:23:01 +00:00
|
|
|
"github.com/sirupsen/logrus"
|
2021-09-13 22:24:00 +00:00
|
|
|
authorizationv1 "k8s.io/api/authorization/v1"
|
2019-10-15 21:17:26 +00:00
|
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
2023-08-22 21:09:31 +00:00
|
|
|
logsapi "k8s.io/component-base/logs/api/v1"
|
2019-01-01 08:23:01 +00:00
|
|
|
"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
|
2023-08-22 21:09:31 +00:00
|
|
|
"k8s.io/kubernetes/pkg/registry/core/node"
|
2020-04-27 17:09:58 +00:00
|
|
|
|
|
|
|
// for client metric registration
|
|
|
|
_ "k8s.io/component-base/metrics/prometheus/restclient"
|
2019-01-01 08:23:01 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
func Server(ctx context.Context, cfg *config.Control) error {
|
|
|
|
rand.Seed(time.Now().UTC().UnixNano())
|
|
|
|
|
2023-08-22 21:09:31 +00:00
|
|
|
logsapi.ReapplyHandling = logsapi.ReapplyHandlingIgnoreUnchanged
|
2022-02-24 19:01:14 +00:00
|
|
|
if err := prepare(ctx, cfg); err != nil {
|
2019-12-16 18:44:13 +00:00
|
|
|
return errors.Wrap(err, "preparing server")
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2022-04-04 21:54:50 +00:00
|
|
|
tunnel, err := setupTunnel(ctx, cfg)
|
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "setup tunnel server")
|
|
|
|
}
|
|
|
|
cfg.Runtime.Tunnel = tunnel
|
|
|
|
|
2023-08-22 21:09:31 +00:00
|
|
|
node.DisableProxyHostnameCheck = true
|
2019-01-01 08:23:01 +00:00
|
|
|
|
2022-03-28 20:45:39 +00:00
|
|
|
authArgs := []string{
|
|
|
|
"--basic-auth-file=" + cfg.Runtime.PasswdFile,
|
|
|
|
"--client-ca-file=" + cfg.Runtime.ClientCA,
|
|
|
|
}
|
|
|
|
auth, err := authenticator.FromArgs(authArgs)
|
2021-10-12 06:13:10 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2022-03-28 20:45:39 +00:00
|
|
|
cfg.Runtime.Authenticator = auth
|
2019-01-01 08:23:01 +00:00
|
|
|
|
2021-02-12 15:35:57 +00:00
|
|
|
if !cfg.DisableAPIServer {
|
2022-02-24 19:01:14 +00:00
|
|
|
go waitForAPIServerHandlers(ctx, cfg.Runtime)
|
2021-10-12 06:13:10 +00:00
|
|
|
|
2022-02-24 19:01:14 +00:00
|
|
|
if err := apiServer(ctx, cfg); err != nil {
|
2021-02-12 15:35:57 +00:00
|
|
|
return err
|
|
|
|
}
|
2022-02-16 22:19:58 +00:00
|
|
|
}
|
2019-11-04 11:08:47 +00:00
|
|
|
|
2022-02-16 22:19:58 +00:00
|
|
|
// Wait for an apiserver to become available before starting additional controllers,
|
|
|
|
// even if we're not running an apiserver locally.
|
|
|
|
if err := waitForAPIServerInBackground(ctx, cfg.Runtime); err != nil {
|
|
|
|
return err
|
2021-02-12 15:35:57 +00:00
|
|
|
}
|
2019-01-01 08:23:01 +00:00
|
|
|
|
2021-02-12 15:35:57 +00:00
|
|
|
if !cfg.DisableScheduler {
|
2022-02-24 19:01:14 +00:00
|
|
|
if err := scheduler(ctx, cfg); err != nil {
|
2020-04-27 17:09:58 +00:00
|
|
|
return err
|
|
|
|
}
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
2021-02-12 15:35:57 +00:00
|
|
|
if !cfg.DisableControllerManager {
|
2022-02-24 19:01:14 +00:00
|
|
|
if err := controllerManager(ctx, cfg); err != nil {
|
2021-02-12 15:35:57 +00:00
|
|
|
return err
|
|
|
|
}
|
2020-04-27 17:09:58 +00:00
|
|
|
}
|
2019-01-01 08:23:01 +00:00
|
|
|
|
2022-09-29 20:50:05 +00:00
|
|
|
if !cfg.DisableCCM || !cfg.DisableServiceLB {
|
2022-02-24 19:01:14 +00:00
|
|
|
if err := cloudControllerManager(ctx, cfg); err != nil {
|
2021-06-29 14:28:38 +00:00
|
|
|
return err
|
|
|
|
}
|
2019-10-15 21:17:26 +00:00
|
|
|
}
|
|
|
|
|
2019-01-01 08:23:01 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2022-02-24 19:01:14 +00:00
|
|
|
func controllerManager(ctx context.Context, cfg *config.Control) error {
|
|
|
|
runtime := cfg.Runtime
|
2019-04-05 00:43:00 +00:00
|
|
|
argsMap := map[string]string{
|
2022-12-07 01:44:53 +00:00
|
|
|
"controllers": "*,tokencleaner",
|
2019-05-29 18:53:51 +00:00
|
|
|
"kubeconfig": runtime.KubeConfigController,
|
2021-09-13 22:23:00 +00:00
|
|
|
"authorization-kubeconfig": runtime.KubeConfigController,
|
|
|
|
"authentication-kubeconfig": runtime.KubeConfigController,
|
2022-12-06 23:45:58 +00:00
|
|
|
"service-account-private-key-file": runtime.ServiceCurrentKey,
|
2019-04-05 00:43:00 +00:00
|
|
|
"allocate-node-cidrs": "true",
|
2022-07-21 21:38:49 +00:00
|
|
|
"service-cluster-ip-range": util.JoinIPNets(cfg.ServiceIPRanges),
|
2021-09-10 21:02:51 +00:00
|
|
|
"cluster-cidr": util.JoinIPNets(cfg.ClusterIPRanges),
|
2019-05-29 18:53:51 +00:00
|
|
|
"root-ca-file": runtime.ServerCA,
|
2020-07-10 20:08:04 +00:00
|
|
|
"profiling": "false",
|
2022-07-21 21:40:09 +00:00
|
|
|
"bind-address": cfg.Loopback(false),
|
2021-07-02 23:42:09 +00:00
|
|
|
"secure-port": "10257",
|
2019-05-29 18:53:51 +00:00
|
|
|
"use-service-account-credentials": "true",
|
2022-12-06 23:45:58 +00:00
|
|
|
"cluster-signing-kube-apiserver-client-cert-file": runtime.SigningClientCA,
|
2021-05-01 12:58:06 +00:00
|
|
|
"cluster-signing-kube-apiserver-client-key-file": runtime.ClientCAKey,
|
2022-12-06 23:45:58 +00:00
|
|
|
"cluster-signing-kubelet-client-cert-file": runtime.SigningClientCA,
|
2021-05-01 12:58:06 +00:00
|
|
|
"cluster-signing-kubelet-client-key-file": runtime.ClientCAKey,
|
2022-12-06 23:45:58 +00:00
|
|
|
"cluster-signing-kubelet-serving-cert-file": runtime.SigningServerCA,
|
2021-05-01 12:58:06 +00:00
|
|
|
"cluster-signing-kubelet-serving-key-file": runtime.ServerCAKey,
|
2022-12-06 23:45:58 +00:00
|
|
|
"cluster-signing-legacy-unknown-cert-file": runtime.SigningServerCA,
|
2022-02-02 17:15:41 +00:00
|
|
|
"cluster-signing-legacy-unknown-key-file": runtime.ServerCAKey,
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
2019-01-22 21:14:58 +00:00
|
|
|
if cfg.NoLeaderElect {
|
2019-04-05 00:43:00 +00:00
|
|
|
argsMap["leader-elect"] = "false"
|
2019-01-22 21:14:58 +00:00
|
|
|
}
|
2020-11-03 17:27:56 +00:00
|
|
|
if !cfg.DisableCCM {
|
|
|
|
argsMap["configure-cloud-routes"] = "false"
|
2022-12-07 01:44:53 +00:00
|
|
|
argsMap["controllers"] = argsMap["controllers"] + ",-service,-route,-cloud-node-lifecycle"
|
2020-11-03 17:27:56 +00:00
|
|
|
}
|
2019-04-05 00:43:00 +00:00
|
|
|
|
2023-12-19 03:14:02 +00:00
|
|
|
if cfg.VLevel != 0 {
|
|
|
|
argsMap["v"] = strconv.Itoa(cfg.VLevel)
|
|
|
|
}
|
|
|
|
if cfg.VModule != "" {
|
|
|
|
argsMap["vmodule"] = cfg.VModule
|
|
|
|
}
|
|
|
|
|
2021-07-28 20:04:19 +00:00
|
|
|
args := config.GetArgs(argsMap, cfg.ExtraControllerArgs)
|
2020-04-27 17:09:58 +00:00
|
|
|
logrus.Infof("Running kube-controller-manager %s", config.ArgString(args))
|
2019-04-05 00:43:00 +00:00
|
|
|
|
2022-02-24 19:01:14 +00:00
|
|
|
return executor.ControllerManager(ctx, cfg.Runtime.APIServerReady, args)
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2022-02-24 19:01:14 +00:00
|
|
|
func scheduler(ctx context.Context, cfg *config.Control) error {
|
|
|
|
runtime := cfg.Runtime
|
2019-04-05 00:43:00 +00:00
|
|
|
argsMap := map[string]string{
|
2021-09-13 22:23:00 +00:00
|
|
|
"kubeconfig": runtime.KubeConfigScheduler,
|
|
|
|
"authorization-kubeconfig": runtime.KubeConfigScheduler,
|
|
|
|
"authentication-kubeconfig": runtime.KubeConfigScheduler,
|
2022-07-21 21:40:09 +00:00
|
|
|
"bind-address": cfg.Loopback(false),
|
2021-09-13 22:23:00 +00:00
|
|
|
"secure-port": "10259",
|
|
|
|
"profiling": "false",
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
2019-01-22 21:14:58 +00:00
|
|
|
if cfg.NoLeaderElect {
|
2019-04-05 00:43:00 +00:00
|
|
|
argsMap["leader-elect"] = "false"
|
2019-01-22 21:14:58 +00:00
|
|
|
}
|
2023-12-19 03:14:02 +00:00
|
|
|
|
|
|
|
if cfg.VLevel != 0 {
|
|
|
|
argsMap["v"] = strconv.Itoa(cfg.VLevel)
|
|
|
|
}
|
|
|
|
if cfg.VModule != "" {
|
|
|
|
argsMap["vmodule"] = cfg.VModule
|
|
|
|
}
|
|
|
|
|
2021-07-28 20:04:19 +00:00
|
|
|
args := config.GetArgs(argsMap, cfg.ExtraSchedulerAPIArgs)
|
2019-04-05 00:43:00 +00:00
|
|
|
|
2020-04-27 17:09:58 +00:00
|
|
|
logrus.Infof("Running kube-scheduler %s", config.ArgString(args))
|
2022-02-24 19:01:14 +00:00
|
|
|
return executor.Scheduler(ctx, cfg.Runtime.APIServerReady, args)
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2022-02-24 19:01:14 +00:00
|
|
|
func apiServer(ctx context.Context, cfg *config.Control) error {
|
|
|
|
runtime := cfg.Runtime
|
2023-12-14 20:57:24 +00:00
|
|
|
argsMap := map[string]string{}
|
2019-06-11 22:48:47 +00:00
|
|
|
|
|
|
|
setupStorageBackend(argsMap, cfg)
|
2019-01-01 08:23:01 +00:00
|
|
|
|
2020-04-25 07:22:48 +00:00
|
|
|
certDir := filepath.Join(cfg.DataDir, "tls", "temporary-certs")
|
2019-02-08 04:14:20 +00:00
|
|
|
os.MkdirAll(certDir, 0700)
|
|
|
|
|
2019-04-05 00:43:00 +00:00
|
|
|
argsMap["cert-dir"] = certDir
|
|
|
|
argsMap["allow-privileged"] = "true"
|
2022-12-07 01:44:53 +00:00
|
|
|
argsMap["enable-bootstrap-token-auth"] = "true"
|
2019-04-05 00:43:00 +00:00
|
|
|
argsMap["authorization-mode"] = strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ",")
|
2022-12-06 23:45:58 +00:00
|
|
|
argsMap["service-account-signing-key-file"] = runtime.ServiceCurrentKey
|
2021-09-10 21:02:51 +00:00
|
|
|
argsMap["service-cluster-ip-range"] = util.JoinIPNets(cfg.ServiceIPRanges)
|
2021-02-01 19:11:17 +00:00
|
|
|
argsMap["service-node-port-range"] = cfg.ServiceNodePortRange.String()
|
2019-04-05 00:43:00 +00:00
|
|
|
argsMap["advertise-port"] = strconv.Itoa(cfg.AdvertisePort)
|
2019-05-29 18:53:51 +00:00
|
|
|
if cfg.AdvertiseIP != "" {
|
|
|
|
argsMap["advertise-address"] = cfg.AdvertiseIP
|
|
|
|
}
|
2020-04-28 22:00:30 +00:00
|
|
|
argsMap["secure-port"] = strconv.Itoa(cfg.APIServerPort)
|
|
|
|
if cfg.APIServerBindAddress == "" {
|
2022-07-21 21:40:09 +00:00
|
|
|
argsMap["bind-address"] = cfg.Loopback(false)
|
2020-04-28 22:00:30 +00:00
|
|
|
} else {
|
|
|
|
argsMap["bind-address"] = cfg.APIServerBindAddress
|
|
|
|
}
|
2023-07-13 23:33:37 +00:00
|
|
|
if cfg.EgressSelectorMode != config.EgressSelectorModeDisabled {
|
|
|
|
argsMap["enable-aggregator-routing"] = "true"
|
|
|
|
argsMap["egress-selector-config-file"] = runtime.EgressSelectorConfig
|
|
|
|
}
|
2019-05-29 18:53:51 +00:00
|
|
|
argsMap["tls-cert-file"] = runtime.ServingKubeAPICert
|
|
|
|
argsMap["tls-private-key-file"] = runtime.ServingKubeAPIKey
|
2019-04-05 00:43:00 +00:00
|
|
|
argsMap["service-account-key-file"] = runtime.ServiceKey
|
2021-04-28 05:45:33 +00:00
|
|
|
argsMap["service-account-issuer"] = "https://kubernetes.default.svc." + cfg.ClusterDomain
|
|
|
|
argsMap["api-audiences"] = "https://kubernetes.default.svc." + cfg.ClusterDomain + "," + version.Program
|
2019-09-02 17:03:17 +00:00
|
|
|
argsMap["kubelet-certificate-authority"] = runtime.ServerCA
|
2019-05-29 18:53:51 +00:00
|
|
|
argsMap["kubelet-client-certificate"] = runtime.ClientKubeAPICert
|
|
|
|
argsMap["kubelet-client-key"] = runtime.ClientKubeAPIKey
|
2022-10-31 08:05:04 +00:00
|
|
|
if cfg.FlannelExternalIP {
|
|
|
|
argsMap["kubelet-preferred-address-types"] = "ExternalIP,InternalIP,Hostname"
|
|
|
|
} else {
|
|
|
|
argsMap["kubelet-preferred-address-types"] = "InternalIP,ExternalIP,Hostname"
|
|
|
|
}
|
2019-04-10 18:09:38 +00:00
|
|
|
argsMap["requestheader-client-ca-file"] = runtime.RequestHeaderCA
|
2021-03-03 18:14:12 +00:00
|
|
|
argsMap["requestheader-allowed-names"] = deps.RequestHeaderCN
|
2019-04-10 18:09:38 +00:00
|
|
|
argsMap["proxy-client-cert-file"] = runtime.ClientAuthProxyCert
|
|
|
|
argsMap["proxy-client-key-file"] = runtime.ClientAuthProxyKey
|
|
|
|
argsMap["requestheader-extra-headers-prefix"] = "X-Remote-Extra-"
|
|
|
|
argsMap["requestheader-group-headers"] = "X-Remote-Group"
|
|
|
|
argsMap["requestheader-username-headers"] = "X-Remote-User"
|
2019-05-29 18:53:51 +00:00
|
|
|
argsMap["client-ca-file"] = runtime.ClientCA
|
2019-09-05 06:34:47 +00:00
|
|
|
argsMap["enable-admission-plugins"] = "NodeRestriction"
|
2019-08-27 04:36:56 +00:00
|
|
|
argsMap["anonymous-auth"] = "false"
|
2020-06-12 19:09:41 +00:00
|
|
|
argsMap["profiling"] = "false"
|
2019-12-12 22:41:10 +00:00
|
|
|
if cfg.EncryptSecrets {
|
|
|
|
argsMap["encryption-provider-config"] = runtime.EncryptionConfig
|
2023-08-01 17:20:21 +00:00
|
|
|
argsMap["encryption-provider-config-automatic-reload"] = "true"
|
2019-12-12 22:41:10 +00:00
|
|
|
}
|
2023-12-19 03:14:02 +00:00
|
|
|
if cfg.VLevel != 0 {
|
|
|
|
argsMap["v"] = strconv.Itoa(cfg.VLevel)
|
|
|
|
}
|
|
|
|
if cfg.VModule != "" {
|
|
|
|
argsMap["vmodule"] = cfg.VModule
|
|
|
|
}
|
|
|
|
|
2021-07-28 20:04:19 +00:00
|
|
|
args := config.GetArgs(argsMap, cfg.ExtraAPIArgs)
|
2019-04-05 00:43:00 +00:00
|
|
|
|
2020-04-27 17:09:58 +00:00
|
|
|
logrus.Infof("Running kube-apiserver %s", config.ArgString(args))
|
2021-03-03 18:14:12 +00:00
|
|
|
|
2020-05-05 22:02:16 +00:00
|
|
|
return executor.APIServer(ctx, runtime.ETCDReady, args)
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func defaults(config *config.Control) {
|
|
|
|
if config.AdvertisePort == 0 {
|
2019-05-29 18:53:51 +00:00
|
|
|
config.AdvertisePort = config.HTTPSPort
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2020-04-28 22:00:30 +00:00
|
|
|
if config.APIServerPort == 0 {
|
2019-08-27 04:37:47 +00:00
|
|
|
if config.HTTPSPort != 0 {
|
2020-04-28 22:00:30 +00:00
|
|
|
config.APIServerPort = config.HTTPSPort + 1
|
2019-08-27 04:37:47 +00:00
|
|
|
} else {
|
2020-04-28 22:00:30 +00:00
|
|
|
config.APIServerPort = 6444
|
2019-08-27 04:37:47 +00:00
|
|
|
}
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if config.DataDir == "" {
|
|
|
|
config.DataDir = "./management-state"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-02-24 19:01:14 +00:00
|
|
|
func prepare(ctx context.Context, config *config.Control) error {
|
2019-01-01 08:23:01 +00:00
|
|
|
var err error
|
|
|
|
|
|
|
|
defaults(config)
|
|
|
|
|
2019-07-26 21:54:44 +00:00
|
|
|
if err := os.MkdirAll(config.DataDir, 0700); err != nil {
|
2019-01-01 08:23:01 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
config.DataDir, err = filepath.Abs(config.DataDir)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2022-04-04 21:54:50 +00:00
|
|
|
os.MkdirAll(filepath.Join(config.DataDir, "etc"), 0700)
|
2020-04-25 07:22:48 +00:00
|
|
|
os.MkdirAll(filepath.Join(config.DataDir, "tls"), 0700)
|
|
|
|
os.MkdirAll(filepath.Join(config.DataDir, "cred"), 0700)
|
|
|
|
|
2022-02-24 19:01:14 +00:00
|
|
|
deps.CreateRuntimeCertFiles(config)
|
2019-12-12 22:41:10 +00:00
|
|
|
|
2019-11-11 22:18:26 +00:00
|
|
|
cluster := cluster.New(config)
|
|
|
|
|
2023-11-15 00:31:15 +00:00
|
|
|
if err := cluster.Bootstrap(ctx, config.ClusterReset); err != nil {
|
2019-06-27 19:00:43 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2022-02-24 19:01:14 +00:00
|
|
|
if err := deps.GenServerDeps(config); err != nil {
|
2019-10-27 05:53:25 +00:00
|
|
|
return err
|
2019-07-26 21:54:44 +00:00
|
|
|
}
|
|
|
|
|
2020-05-05 21:59:15 +00:00
|
|
|
ready, err := cluster.Start(ctx)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2022-02-24 19:01:14 +00:00
|
|
|
config.Runtime.ETCDReady = ready
|
2020-05-05 21:59:15 +00:00
|
|
|
return nil
|
2019-05-29 18:53:51 +00:00
|
|
|
}
|
2019-01-01 08:23:01 +00:00
|
|
|
|
2019-06-11 22:48:47 +00:00
|
|
|
func setupStorageBackend(argsMap map[string]string, cfg *config.Control) {
|
2019-07-26 21:54:44 +00:00
|
|
|
argsMap["storage-backend"] = "etcd3"
|
2019-06-11 22:48:47 +00:00
|
|
|
// specify the endpoints
|
2019-11-16 00:12:27 +00:00
|
|
|
if len(cfg.Datastore.Endpoint) > 0 {
|
|
|
|
argsMap["etcd-servers"] = cfg.Datastore.Endpoint
|
2019-06-11 22:48:47 +00:00
|
|
|
}
|
|
|
|
// storage backend tls configuration
|
2021-08-30 20:43:25 +00:00
|
|
|
if len(cfg.Datastore.BackendTLSConfig.CAFile) > 0 {
|
|
|
|
argsMap["etcd-cafile"] = cfg.Datastore.BackendTLSConfig.CAFile
|
2019-06-11 22:48:47 +00:00
|
|
|
}
|
2021-08-30 20:43:25 +00:00
|
|
|
if len(cfg.Datastore.BackendTLSConfig.CertFile) > 0 {
|
|
|
|
argsMap["etcd-certfile"] = cfg.Datastore.BackendTLSConfig.CertFile
|
2019-06-11 22:48:47 +00:00
|
|
|
}
|
2021-08-30 20:43:25 +00:00
|
|
|
if len(cfg.Datastore.BackendTLSConfig.KeyFile) > 0 {
|
|
|
|
argsMap["etcd-keyfile"] = cfg.Datastore.BackendTLSConfig.KeyFile
|
2019-06-11 22:48:47 +00:00
|
|
|
}
|
|
|
|
}
|
2019-09-26 22:38:04 +00:00
|
|
|
|
2022-02-24 19:01:14 +00:00
|
|
|
func cloudControllerManager(ctx context.Context, cfg *config.Control) error {
|
|
|
|
runtime := cfg.Runtime
|
2019-10-15 21:17:26 +00:00
|
|
|
argsMap := map[string]string{
|
2021-06-29 14:28:38 +00:00
|
|
|
"profiling": "false",
|
2021-09-10 21:02:51 +00:00
|
|
|
"allocate-node-cidrs": "true",
|
2022-09-29 20:50:05 +00:00
|
|
|
"leader-elect-resource-name": version.Program + "-cloud-controller-manager",
|
2021-09-10 21:02:51 +00:00
|
|
|
"cloud-provider": version.Program,
|
2022-09-29 20:37:50 +00:00
|
|
|
"cloud-config": runtime.CloudControllerConfig,
|
2021-09-10 21:02:51 +00:00
|
|
|
"cluster-cidr": util.JoinIPNets(cfg.ClusterIPRanges),
|
|
|
|
"configure-cloud-routes": "false",
|
2022-09-29 20:50:05 +00:00
|
|
|
"controllers": "*,-route",
|
2021-09-10 21:02:51 +00:00
|
|
|
"kubeconfig": runtime.KubeConfigCloudController,
|
2021-09-13 22:23:00 +00:00
|
|
|
"authorization-kubeconfig": runtime.KubeConfigCloudController,
|
|
|
|
"authentication-kubeconfig": runtime.KubeConfigCloudController,
|
2021-09-10 21:02:51 +00:00
|
|
|
"node-status-update-frequency": "1m0s",
|
2022-07-21 21:40:09 +00:00
|
|
|
"bind-address": cfg.Loopback(false),
|
2023-10-17 05:43:20 +00:00
|
|
|
"feature-gates": "CloudDualStackNodeIPs=true",
|
2019-10-15 21:17:26 +00:00
|
|
|
}
|
|
|
|
if cfg.NoLeaderElect {
|
2021-06-29 14:28:38 +00:00
|
|
|
argsMap["leader-elect"] = "false"
|
2019-10-15 21:17:26 +00:00
|
|
|
}
|
2022-09-29 20:50:05 +00:00
|
|
|
if cfg.DisableCCM {
|
|
|
|
argsMap["controllers"] = argsMap["controllers"] + ",-cloud-node,-cloud-node-lifecycle"
|
2022-11-28 22:49:33 +00:00
|
|
|
argsMap["secure-port"] = "0"
|
2022-09-29 20:50:05 +00:00
|
|
|
}
|
|
|
|
if cfg.DisableServiceLB {
|
|
|
|
argsMap["controllers"] = argsMap["controllers"] + ",-service"
|
|
|
|
}
|
2023-12-19 03:14:02 +00:00
|
|
|
if cfg.VLevel != 0 {
|
|
|
|
argsMap["v"] = strconv.Itoa(cfg.VLevel)
|
|
|
|
}
|
|
|
|
if cfg.VModule != "" {
|
|
|
|
argsMap["vmodule"] = cfg.VModule
|
|
|
|
}
|
|
|
|
|
2021-07-28 20:04:19 +00:00
|
|
|
args := config.GetArgs(argsMap, cfg.ExtraCloudControllerArgs)
|
2019-10-15 21:17:26 +00:00
|
|
|
|
2021-06-29 14:28:38 +00:00
|
|
|
logrus.Infof("Running cloud-controller-manager %s", config.ArgString(args))
|
2020-12-01 01:06:26 +00:00
|
|
|
|
2021-06-29 14:28:38 +00:00
|
|
|
ccmRBACReady := make(chan struct{})
|
2020-12-01 01:06:26 +00:00
|
|
|
|
2021-06-29 14:28:38 +00:00
|
|
|
go func() {
|
|
|
|
defer close(ccmRBACReady)
|
2019-10-15 21:17:26 +00:00
|
|
|
|
2021-06-29 14:28:38 +00:00
|
|
|
apiReadyLoop:
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
2022-02-24 19:01:14 +00:00
|
|
|
case <-cfg.Runtime.APIServerReady:
|
2021-06-29 14:28:38 +00:00
|
|
|
break apiReadyLoop
|
|
|
|
case <-time.After(30 * time.Second):
|
|
|
|
logrus.Infof("Waiting for API server to become available")
|
|
|
|
}
|
|
|
|
}
|
2019-10-15 21:17:26 +00:00
|
|
|
|
2021-06-29 14:28:38 +00:00
|
|
|
logrus.Infof("Waiting for cloud-controller-manager privileges to become available")
|
2019-10-15 21:17:26 +00:00
|
|
|
for {
|
2021-06-29 14:28:38 +00:00
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
2022-02-24 19:01:14 +00:00
|
|
|
case err := <-promise(func() error { return checkForCloudControllerPrivileges(ctx, cfg.Runtime, 5*time.Second) }):
|
2021-06-29 14:28:38 +00:00
|
|
|
if err != nil {
|
2021-09-13 23:20:53 +00:00
|
|
|
logrus.Infof("Waiting for cloud-controller-manager privileges to become available: %v", err)
|
2019-11-05 11:44:35 +00:00
|
|
|
continue
|
|
|
|
}
|
2021-06-29 14:28:38 +00:00
|
|
|
return
|
2019-10-15 21:17:26 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
2021-06-29 14:28:38 +00:00
|
|
|
|
2021-09-13 22:20:03 +00:00
|
|
|
return executor.CloudControllerManager(ctx, ccmRBACReady, args)
|
2019-10-15 21:17:26 +00:00
|
|
|
}
|
|
|
|
|
2021-09-13 22:20:03 +00:00
|
|
|
// checkForCloudControllerPrivileges makes a SubjectAccessReview request to the apiserver
|
|
|
|
// to validate that the embedded cloud controller manager has the required privileges,
|
|
|
|
// and does not return until the requested access is granted.
|
|
|
|
// If the CCM RBAC changes, the ResourceAttributes checked for by this function should
|
|
|
|
// be modified to check for the most recently added privilege.
|
|
|
|
func checkForCloudControllerPrivileges(ctx context.Context, runtime *config.ControlRuntime, timeout time.Duration) error {
|
2023-05-26 20:35:17 +00:00
|
|
|
return util.WaitForRBACReady(ctx, runtime.KubeConfigSupervisor, timeout, authorizationv1.ResourceAttributes{
|
2022-10-20 19:34:41 +00:00
|
|
|
Namespace: metav1.NamespaceSystem,
|
2023-01-26 23:48:12 +00:00
|
|
|
Verb: "watch",
|
|
|
|
Resource: "endpointslices",
|
|
|
|
Group: "discovery.k8s.io",
|
2022-10-20 19:34:41 +00:00
|
|
|
}, version.Program+"-cloud-controller-manager")
|
2019-10-15 21:17:26 +00:00
|
|
|
}
|
2019-11-04 11:08:47 +00:00
|
|
|
|
2021-10-12 06:13:10 +00:00
|
|
|
func waitForAPIServerHandlers(ctx context.Context, runtime *config.ControlRuntime) {
|
|
|
|
auth, handler, err := executor.APIServerHandlers(ctx)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Fatalf("Failed to get request handlers from apiserver: %v", err)
|
|
|
|
}
|
2022-03-28 20:45:39 +00:00
|
|
|
runtime.Authenticator = authenticator.Combine(runtime.Authenticator, auth)
|
2021-10-12 06:13:10 +00:00
|
|
|
runtime.APIServer = handler
|
|
|
|
}
|
|
|
|
|
2020-04-28 22:44:05 +00:00
|
|
|
func waitForAPIServerInBackground(ctx context.Context, runtime *config.ControlRuntime) error {
|
|
|
|
done := make(chan struct{})
|
|
|
|
runtime.APIServerReady = done
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
defer close(done)
|
2020-05-05 22:02:16 +00:00
|
|
|
|
|
|
|
etcdLoop:
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
case <-runtime.ETCDReady:
|
|
|
|
break etcdLoop
|
|
|
|
case <-time.After(30 * time.Second):
|
|
|
|
logrus.Infof("Waiting for etcd server to become available")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-04-28 22:44:05 +00:00
|
|
|
logrus.Infof("Waiting for API server to become available")
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
2023-05-26 20:35:17 +00:00
|
|
|
case err := <-promise(func() error { return util.WaitForAPIServerReady(ctx, runtime.KubeConfigSupervisor, 30*time.Second) }):
|
2020-04-28 22:44:05 +00:00
|
|
|
if err != nil {
|
|
|
|
logrus.Infof("Waiting for API server to become available")
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
return nil
|
2019-11-09 06:07:12 +00:00
|
|
|
}
|
2019-11-04 11:08:47 +00:00
|
|
|
|
2019-11-09 06:07:12 +00:00
|
|
|
func promise(f func() error) <-chan error {
|
|
|
|
c := make(chan error, 1)
|
|
|
|
go func() {
|
|
|
|
c <- f()
|
|
|
|
close(c)
|
|
|
|
}()
|
|
|
|
return c
|
2019-11-04 11:08:47 +00:00
|
|
|
}
|