2019-01-01 08:23:01 +00:00
|
|
|
package server
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"fmt"
|
|
|
|
"os"
|
2020-12-23 01:28:19 +00:00
|
|
|
"path"
|
2019-01-01 08:23:01 +00:00
|
|
|
"path/filepath"
|
2023-02-08 00:37:10 +00:00
|
|
|
"runtime/debug"
|
2019-01-22 21:14:58 +00:00
|
|
|
"strconv"
|
2019-05-03 00:55:05 +00:00
|
|
|
"strings"
|
2021-07-16 02:28:47 +00:00
|
|
|
"sync"
|
2019-09-26 18:10:19 +00:00
|
|
|
"time"
|
2019-01-01 08:23:01 +00:00
|
|
|
|
2022-10-18 19:35:47 +00:00
|
|
|
helm "github.com/k3s-io/helm-controller/pkg/controllers/chart"
|
|
|
|
helmcommon "github.com/k3s-io/helm-controller/pkg/controllers/common"
|
2022-03-02 23:47:27 +00:00
|
|
|
"github.com/k3s-io/k3s/pkg/cli/cmds"
|
|
|
|
"github.com/k3s-io/k3s/pkg/clientaccess"
|
|
|
|
"github.com/k3s-io/k3s/pkg/daemons/config"
|
|
|
|
"github.com/k3s-io/k3s/pkg/daemons/control"
|
|
|
|
"github.com/k3s-io/k3s/pkg/datadir"
|
|
|
|
"github.com/k3s-io/k3s/pkg/deploy"
|
|
|
|
"github.com/k3s-io/k3s/pkg/node"
|
|
|
|
"github.com/k3s-io/k3s/pkg/nodepassword"
|
|
|
|
"github.com/k3s-io/k3s/pkg/rootlessports"
|
|
|
|
"github.com/k3s-io/k3s/pkg/secretsencrypt"
|
|
|
|
"github.com/k3s-io/k3s/pkg/static"
|
|
|
|
"github.com/k3s-io/k3s/pkg/util"
|
|
|
|
"github.com/k3s-io/k3s/pkg/version"
|
2019-01-01 08:23:01 +00:00
|
|
|
"github.com/pkg/errors"
|
2021-07-03 08:37:19 +00:00
|
|
|
v1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
|
2019-05-09 22:05:51 +00:00
|
|
|
"github.com/rancher/wrangler/pkg/leader"
|
|
|
|
"github.com/rancher/wrangler/pkg/resolvehome"
|
2019-01-01 08:23:01 +00:00
|
|
|
"github.com/sirupsen/logrus"
|
2021-07-20 03:24:52 +00:00
|
|
|
corev1 "k8s.io/api/core/v1"
|
2019-10-27 05:53:25 +00:00
|
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
2019-01-01 08:23:01 +00:00
|
|
|
)
|
|
|
|
|
2020-12-02 19:54:13 +00:00
|
|
|
const (
|
|
|
|
MasterRoleLabelKey = "node-role.kubernetes.io/master"
|
|
|
|
ControlPlaneRoleLabelKey = "node-role.kubernetes.io/control-plane"
|
2021-02-12 15:35:57 +00:00
|
|
|
ETCDRoleLabelKey = "node-role.kubernetes.io/etcd"
|
2020-12-02 19:54:13 +00:00
|
|
|
)
|
2019-09-26 18:10:19 +00:00
|
|
|
|
2021-01-21 21:09:15 +00:00
|
|
|
func ResolveDataDir(dataDir string) (string, error) {
|
2019-03-08 22:47:44 +00:00
|
|
|
dataDir, err := datadir.Resolve(dataDir)
|
|
|
|
return filepath.Join(dataDir, "server"), err
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2021-07-28 20:56:59 +00:00
|
|
|
func StartServer(ctx context.Context, config *Config, cfg *cmds.Server) error {
|
2019-01-01 08:23:01 +00:00
|
|
|
if err := setupDataDirAndChdir(&config.ControlConfig); err != nil {
|
2019-10-27 05:53:25 +00:00
|
|
|
return err
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2019-05-03 00:55:05 +00:00
|
|
|
if err := setNoProxyEnv(&config.ControlConfig); err != nil {
|
2019-10-27 05:53:25 +00:00
|
|
|
return err
|
2019-05-03 00:55:05 +00:00
|
|
|
}
|
|
|
|
|
2019-01-01 08:23:01 +00:00
|
|
|
if err := control.Server(ctx, &config.ControlConfig); err != nil {
|
2019-10-27 05:53:25 +00:00
|
|
|
return errors.Wrap(err, "starting kubernetes")
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2021-07-20 21:25:00 +00:00
|
|
|
wg := &sync.WaitGroup{}
|
|
|
|
wg.Add(len(config.StartupHooks))
|
2019-01-01 08:23:01 +00:00
|
|
|
|
2021-07-28 20:56:59 +00:00
|
|
|
config.ControlConfig.Runtime.Handler = router(ctx, config, cfg)
|
2022-06-15 16:00:52 +00:00
|
|
|
config.ControlConfig.Runtime.StartupHooksWg = wg
|
|
|
|
|
2021-07-20 03:24:52 +00:00
|
|
|
shArgs := cmds.StartupHookArgs{
|
|
|
|
APIServerReady: config.ControlConfig.Runtime.APIServerReady,
|
|
|
|
KubeConfigAdmin: config.ControlConfig.Runtime.KubeConfigAdmin,
|
|
|
|
Skips: config.ControlConfig.Skips,
|
|
|
|
Disables: config.ControlConfig.Disables,
|
|
|
|
}
|
2020-08-19 21:30:53 +00:00
|
|
|
for _, hook := range config.StartupHooks {
|
2021-07-20 21:25:00 +00:00
|
|
|
if err := hook(ctx, wg, shArgs); err != nil {
|
2020-08-19 23:54:58 +00:00
|
|
|
return errors.Wrap(err, "startup hook")
|
|
|
|
}
|
2020-08-19 20:30:51 +00:00
|
|
|
}
|
|
|
|
|
2023-02-13 20:00:52 +00:00
|
|
|
go startOnAPIServerReady(ctx, config)
|
2021-07-20 21:25:00 +00:00
|
|
|
|
2022-04-15 00:31:49 +00:00
|
|
|
if err := printTokens(&config.ControlConfig); err != nil {
|
2019-10-27 05:53:25 +00:00
|
|
|
return err
|
|
|
|
}
|
2019-01-01 08:23:01 +00:00
|
|
|
|
2019-10-27 05:53:25 +00:00
|
|
|
return writeKubeConfig(config.ControlConfig.Runtime.ServerCA, config)
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2022-06-15 16:00:52 +00:00
|
|
|
func startOnAPIServerReady(ctx context.Context, config *Config) {
|
2020-10-15 19:06:25 +00:00
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
case <-config.ControlConfig.Runtime.APIServerReady:
|
2022-06-15 16:00:52 +00:00
|
|
|
if err := runControllers(ctx, config); err != nil {
|
2020-10-15 19:06:25 +00:00
|
|
|
logrus.Fatalf("failed to start controllers: %v", err)
|
2020-04-28 22:44:05 +00:00
|
|
|
}
|
2020-10-15 19:06:25 +00:00
|
|
|
}
|
2020-04-28 22:44:05 +00:00
|
|
|
}
|
|
|
|
|
2022-06-15 16:00:52 +00:00
|
|
|
func runControllers(ctx context.Context, config *Config) error {
|
2020-04-28 22:44:05 +00:00
|
|
|
controlConfig := &config.ControlConfig
|
|
|
|
|
2021-05-01 01:26:39 +00:00
|
|
|
sc, err := NewContext(ctx, controlConfig.Runtime.KubeConfigAdmin)
|
2019-05-09 22:05:51 +00:00
|
|
|
if err != nil {
|
2021-07-03 00:09:02 +00:00
|
|
|
return errors.Wrap(err, "failed to create new server context")
|
2019-05-09 22:05:51 +00:00
|
|
|
}
|
|
|
|
|
2022-06-15 16:00:52 +00:00
|
|
|
controlConfig.Runtime.StartupHooksWg.Wait()
|
2019-05-09 22:05:51 +00:00
|
|
|
if err := stageFiles(ctx, sc, controlConfig); err != nil {
|
2021-07-03 00:09:02 +00:00
|
|
|
return errors.Wrap(err, "failed to stage files")
|
2019-05-09 22:05:51 +00:00
|
|
|
}
|
|
|
|
|
2020-12-10 18:26:16 +00:00
|
|
|
// run migration before we set controlConfig.Runtime.Core
|
|
|
|
if err := nodepassword.MigrateFile(
|
|
|
|
sc.Core.Core().V1().Secret(),
|
|
|
|
sc.Core.Core().V1().Node(),
|
|
|
|
controlConfig.Runtime.NodePasswdFile); err != nil {
|
2021-07-03 00:09:02 +00:00
|
|
|
logrus.Warn(errors.Wrap(err, "error migrating node-password file"))
|
2020-12-10 18:26:16 +00:00
|
|
|
}
|
2020-05-05 21:59:15 +00:00
|
|
|
controlConfig.Runtime.Core = sc.Core
|
2020-12-10 18:26:16 +00:00
|
|
|
|
2023-02-08 00:37:10 +00:00
|
|
|
for name, cb := range controlConfig.Runtime.ClusterControllerStarts {
|
|
|
|
go runOrDie(ctx, name, cb)
|
2020-05-05 21:59:15 +00:00
|
|
|
}
|
|
|
|
|
2021-03-11 18:39:00 +00:00
|
|
|
for _, controller := range config.Controllers {
|
|
|
|
if err := controller(ctx, sc); err != nil {
|
2023-02-08 00:37:10 +00:00
|
|
|
return errors.Wrapf(err, "failed to start %s controller", util.GetFunctionName(controller))
|
2021-03-11 18:39:00 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-10-27 05:53:25 +00:00
|
|
|
if err := sc.Start(ctx); err != nil {
|
2021-07-03 00:09:02 +00:00
|
|
|
return errors.Wrap(err, "failed to start wranger controllers")
|
2019-05-09 22:05:51 +00:00
|
|
|
}
|
|
|
|
|
2023-02-13 20:00:52 +00:00
|
|
|
if !controlConfig.DisableAPIServer {
|
|
|
|
controlConfig.Runtime.LeaderElectedClusterControllerStarts[version.Program] = func(ctx context.Context) {
|
|
|
|
apiserverControllers(ctx, sc, config)
|
2019-05-09 22:05:51 +00:00
|
|
|
}
|
2019-08-27 04:56:01 +00:00
|
|
|
}
|
2021-02-12 15:35:57 +00:00
|
|
|
|
2021-12-07 22:31:32 +00:00
|
|
|
go setNodeLabelsAndAnnotations(ctx, sc.Core.Core().V1().Node(), config)
|
2020-06-22 21:06:01 +00:00
|
|
|
|
|
|
|
go setClusterDNSConfig(ctx, config, sc.Core.Core().V1().ConfigMap())
|
|
|
|
|
2019-08-27 04:56:01 +00:00
|
|
|
if controlConfig.NoLeaderElect {
|
2023-02-08 00:37:10 +00:00
|
|
|
for name, cb := range controlConfig.Runtime.LeaderElectedClusterControllerStarts {
|
|
|
|
go runOrDie(ctx, name, cb)
|
|
|
|
}
|
2019-08-27 04:56:01 +00:00
|
|
|
} else {
|
2023-02-08 00:37:10 +00:00
|
|
|
for name, cb := range controlConfig.Runtime.LeaderElectedClusterControllerStarts {
|
|
|
|
go leader.RunOrDie(ctx, "", name, sc.K8s, cb)
|
|
|
|
}
|
2019-08-27 04:56:01 +00:00
|
|
|
}
|
2019-05-09 22:05:51 +00:00
|
|
|
|
2019-10-27 05:53:25 +00:00
|
|
|
return nil
|
2019-05-09 22:05:51 +00:00
|
|
|
}
|
|
|
|
|
2023-02-13 20:00:52 +00:00
|
|
|
// apiServerControllers starts the core controllers, as well as the leader-elected controllers
|
|
|
|
// that should only run on a control-plane node.
|
|
|
|
func apiserverControllers(ctx context.Context, sc *Context, config *Config) {
|
|
|
|
if err := coreControllers(ctx, sc, config); err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
for _, controller := range config.LeaderControllers {
|
|
|
|
if err := controller(ctx, sc); err != nil {
|
|
|
|
panic(errors.Wrapf(err, "failed to start %s leader controller", util.GetFunctionName(controller)))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if err := sc.Start(ctx); err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-02-08 00:37:10 +00:00
|
|
|
// runOrDie is similar to leader.RunOrDie, except that it runs the callback
|
|
|
|
// immediately, without performing leader election.
|
|
|
|
func runOrDie(ctx context.Context, name string, cb leader.Callback) {
|
|
|
|
defer func() {
|
|
|
|
if err := recover(); err != nil {
|
|
|
|
logrus.WithField("stack", debug.Stack()).Fatalf("%s controller panic: %v", name, err)
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
cb(ctx)
|
|
|
|
<-ctx.Done()
|
|
|
|
}
|
|
|
|
|
2023-02-13 20:00:52 +00:00
|
|
|
// coreControllers starts the following controllers, if they are enabled:
|
|
|
|
// * Node controller (manages nodes passwords and coredns hosts file)
|
|
|
|
// * Helm controller
|
|
|
|
// * Secrets encryption
|
|
|
|
// * Rootless ports
|
|
|
|
// These controllers should only be run on nodes with a local apiserver
|
2020-12-02 19:54:13 +00:00
|
|
|
func coreControllers(ctx context.Context, sc *Context, config *Config) error {
|
2020-10-15 19:06:25 +00:00
|
|
|
if err := node.Register(ctx,
|
|
|
|
!config.ControlConfig.Skips["coredns"],
|
|
|
|
sc.Core.Core().V1().Secret(),
|
|
|
|
sc.Core.Core().V1().ConfigMap(),
|
|
|
|
sc.Core.Core().V1().Node()); err != nil {
|
|
|
|
return err
|
2019-05-09 22:05:51 +00:00
|
|
|
}
|
|
|
|
|
2022-09-29 20:37:50 +00:00
|
|
|
// apply SystemDefaultRegistry setting to Helm before starting controllers
|
2021-05-10 22:58:41 +00:00
|
|
|
if config.ControlConfig.SystemDefaultRegistry != "" {
|
|
|
|
helm.DefaultJobImage = config.ControlConfig.SystemDefaultRegistry + "/" + helm.DefaultJobImage
|
|
|
|
}
|
|
|
|
|
2021-06-25 18:54:36 +00:00
|
|
|
if !config.ControlConfig.DisableHelmController {
|
|
|
|
helm.Register(ctx,
|
2022-10-18 19:35:47 +00:00
|
|
|
metav1.NamespaceAll,
|
|
|
|
helmcommon.Name,
|
2022-03-15 19:01:55 +00:00
|
|
|
sc.K8s,
|
2021-06-25 18:54:36 +00:00
|
|
|
sc.Apply,
|
2022-10-18 19:35:47 +00:00
|
|
|
util.BuildControllerEventRecorder(sc.K8s, helmcommon.Name, metav1.NamespaceAll),
|
2021-06-25 18:54:36 +00:00
|
|
|
sc.Helm.Helm().V1().HelmChart(),
|
2022-10-18 19:35:47 +00:00
|
|
|
sc.Helm.Helm().V1().HelmChart().Cache(),
|
2021-06-25 18:54:36 +00:00
|
|
|
sc.Helm.Helm().V1().HelmChartConfig(),
|
2022-10-18 19:35:47 +00:00
|
|
|
sc.Helm.Helm().V1().HelmChartConfig().Cache(),
|
2021-06-25 18:54:36 +00:00
|
|
|
sc.Batch.Batch().V1().Job(),
|
2022-10-18 19:35:47 +00:00
|
|
|
sc.Batch.Batch().V1().Job().Cache(),
|
2021-06-25 18:54:36 +00:00
|
|
|
sc.Auth.Rbac().V1().ClusterRoleBinding(),
|
|
|
|
sc.Core.Core().V1().ServiceAccount(),
|
|
|
|
sc.Core.Core().V1().ConfigMap())
|
|
|
|
}
|
|
|
|
|
2021-12-07 22:31:32 +00:00
|
|
|
if config.ControlConfig.EncryptSecrets {
|
|
|
|
if err := secretsencrypt.Register(ctx,
|
|
|
|
sc.K8s,
|
|
|
|
&config.ControlConfig,
|
|
|
|
sc.Core.Core().V1().Node(),
|
|
|
|
sc.Core.Core().V1().Secret()); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-09-29 20:16:33 +00:00
|
|
|
if config.ControlConfig.Rootless {
|
2020-11-07 01:14:30 +00:00
|
|
|
return rootlessports.Register(ctx,
|
|
|
|
sc.Core.Core().V1().Service(),
|
2022-09-29 20:16:33 +00:00
|
|
|
!config.ControlConfig.DisableServiceLB,
|
2020-11-07 01:14:30 +00:00
|
|
|
config.ControlConfig.HTTPSPort)
|
2019-05-09 22:05:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func stageFiles(ctx context.Context, sc *Context, controlConfig *config.Control) error {
|
2023-02-13 20:00:52 +00:00
|
|
|
if controlConfig.DisableAPIServer {
|
|
|
|
return nil
|
|
|
|
}
|
2019-05-09 22:05:51 +00:00
|
|
|
dataDir := filepath.Join(controlConfig.DataDir, "static")
|
2021-01-11 01:59:37 +00:00
|
|
|
if err := static.Stage(dataDir); err != nil {
|
2019-05-09 22:05:51 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
dataDir = filepath.Join(controlConfig.DataDir, "manifests")
|
|
|
|
templateVars := map[string]string{
|
2021-05-10 22:58:41 +00:00
|
|
|
"%{CLUSTER_DNS}%": controlConfig.ClusterDNS.String(),
|
|
|
|
"%{CLUSTER_DOMAIN}%": controlConfig.ClusterDomain,
|
|
|
|
"%{DEFAULT_LOCAL_STORAGE_PATH}%": controlConfig.DefaultLocalStoragePath,
|
|
|
|
"%{SYSTEM_DEFAULT_REGISTRY}%": registryTemplate(controlConfig.SystemDefaultRegistry),
|
|
|
|
"%{SYSTEM_DEFAULT_REGISTRY_RAW}%": controlConfig.SystemDefaultRegistry,
|
2022-10-31 08:05:04 +00:00
|
|
|
"%{PREFERRED_ADDRESS_TYPES}%": addrTypesPrioTemplate(controlConfig.FlannelExternalIP),
|
2019-05-09 22:05:51 +00:00
|
|
|
}
|
|
|
|
|
2020-12-23 01:28:19 +00:00
|
|
|
skip := controlConfig.Skips
|
2021-02-11 00:22:16 +00:00
|
|
|
if !skip["traefik"] && isHelmChartTraefikV1(sc) {
|
|
|
|
logrus.Warn("Skipping Traefik v2 deployment due to existing Traefik v1 installation")
|
2020-12-23 01:28:19 +00:00
|
|
|
skip["traefik"] = true
|
|
|
|
}
|
|
|
|
if err := deploy.Stage(dataDir, templateVars, skip); err != nil {
|
2019-05-09 22:05:51 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2020-11-07 01:14:30 +00:00
|
|
|
return deploy.WatchFiles(ctx,
|
2021-06-10 21:10:38 +00:00
|
|
|
sc.K8s,
|
2020-11-07 01:14:30 +00:00
|
|
|
sc.Apply,
|
|
|
|
sc.K3s.K3s().V1().Addon(),
|
|
|
|
controlConfig.Disables,
|
|
|
|
dataDir)
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2021-05-10 22:58:41 +00:00
|
|
|
// registryTemplate behaves like the system_default_registry template in Rancher helm charts,
|
|
|
|
// and returns the registry value with a trailing forward slash if the registry string is not empty.
|
|
|
|
// If it is empty, it is passed through as a no-op.
|
|
|
|
func registryTemplate(registry string) string {
|
|
|
|
if registry == "" {
|
|
|
|
return registry
|
|
|
|
}
|
|
|
|
return registry + "/"
|
|
|
|
}
|
|
|
|
|
2022-10-31 08:05:04 +00:00
|
|
|
// addressTypesTemplate prioritizes ExternalIP addresses if we are in the multi-cloud env where
|
|
|
|
// cluster traffic flows over the external IPs only
|
|
|
|
func addrTypesPrioTemplate(flannelExternal bool) string {
|
|
|
|
if flannelExternal {
|
|
|
|
return "ExternalIP,InternalIP,Hostname"
|
|
|
|
}
|
|
|
|
|
|
|
|
return "InternalIP,ExternalIP,Hostname"
|
|
|
|
}
|
|
|
|
|
2021-02-24 17:31:21 +00:00
|
|
|
// isHelmChartTraefikV1 checks for an existing HelmChart resource with spec.chart containing traefik-1,
|
|
|
|
// as deployed by the legacy chart (https://%{KUBERNETES_API}%/static/charts/traefik-1.81.0.tgz)
|
2020-12-23 01:28:19 +00:00
|
|
|
func isHelmChartTraefikV1(sc *Context) bool {
|
|
|
|
prefix := "traefik-1."
|
|
|
|
helmChart, err := sc.Helm.Helm().V1().HelmChart().Get(metav1.NamespaceSystem, "traefik", metav1.GetOptions{})
|
|
|
|
if err != nil {
|
2021-02-24 17:31:21 +00:00
|
|
|
logrus.WithError(err).Info("Failed to get existing traefik HelmChart")
|
2020-12-23 01:28:19 +00:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
chart := path.Base(helmChart.Spec.Chart)
|
|
|
|
if strings.HasPrefix(chart, prefix) {
|
2021-02-24 17:31:21 +00:00
|
|
|
logrus.WithField("chart", chart).Info("Found existing traefik v1 HelmChart")
|
2020-12-23 01:28:19 +00:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2019-03-08 22:47:44 +00:00
|
|
|
func HomeKubeConfig(write, rootless bool) (string, error) {
|
2019-02-19 17:46:35 +00:00
|
|
|
if write {
|
2019-03-08 22:47:44 +00:00
|
|
|
if os.Getuid() == 0 && !rootless {
|
2019-02-19 17:46:35 +00:00
|
|
|
return datadir.GlobalConfig, nil
|
|
|
|
}
|
|
|
|
return resolvehome.Resolve(datadir.HomeConfig)
|
|
|
|
}
|
|
|
|
|
|
|
|
if _, err := os.Stat(datadir.GlobalConfig); err == nil {
|
|
|
|
return datadir.GlobalConfig, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
return resolvehome.Resolve(datadir.HomeConfig)
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2022-04-15 00:31:49 +00:00
|
|
|
func printTokens(config *config.Control) error {
|
2022-07-26 20:47:40 +00:00
|
|
|
var serverTokenFile string
|
|
|
|
if config.Runtime.ServerToken != "" {
|
|
|
|
serverTokenFile = filepath.Join(config.DataDir, "token")
|
|
|
|
if err := writeToken(config.Runtime.ServerToken, serverTokenFile, config.Runtime.ServerCA); err != nil {
|
|
|
|
return err
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
2019-10-27 05:53:25 +00:00
|
|
|
|
|
|
|
// backwards compatibility
|
|
|
|
np := filepath.Join(config.DataDir, "node-token")
|
|
|
|
if !isSymlink(np) {
|
|
|
|
if err := os.RemoveAll(np); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2022-07-26 20:47:40 +00:00
|
|
|
if err := os.Symlink(serverTokenFile, np); err != nil {
|
2019-10-27 05:53:25 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
2022-07-26 20:47:40 +00:00
|
|
|
|
|
|
|
logrus.Infof("Server node token is available at %s", serverTokenFile)
|
|
|
|
printToken(config.SupervisorPort, config.BindAddressOrLoopback(true, true), "To join server node to cluster:", "server", "SERVER_NODE_TOKEN")
|
|
|
|
}
|
|
|
|
|
|
|
|
var agentTokenFile string
|
|
|
|
if config.Runtime.AgentToken != "" {
|
|
|
|
if config.AgentToken != "" {
|
|
|
|
agentTokenFile = filepath.Join(config.DataDir, "agent-token")
|
|
|
|
if isSymlink(agentTokenFile) {
|
|
|
|
if err := os.RemoveAll(agentTokenFile); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if err := writeToken(config.Runtime.AgentToken, agentTokenFile, config.Runtime.ServerCA); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
} else if serverTokenFile != "" {
|
|
|
|
agentTokenFile = filepath.Join(config.DataDir, "agent-token")
|
|
|
|
if !isSymlink(agentTokenFile) {
|
|
|
|
if err := os.RemoveAll(agentTokenFile); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err := os.Symlink(serverTokenFile, agentTokenFile); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2022-07-26 20:47:40 +00:00
|
|
|
if agentTokenFile != "" {
|
|
|
|
logrus.Infof("Agent node token is available at %s", agentTokenFile)
|
|
|
|
printToken(config.SupervisorPort, config.BindAddressOrLoopback(true, true), "To join agent node to cluster:", "agent", "AGENT_NODE_TOKEN")
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
2019-10-27 05:53:25 +00:00
|
|
|
|
|
|
|
return nil
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2019-10-27 05:53:25 +00:00
|
|
|
func writeKubeConfig(certs string, config *Config) error {
|
2022-07-21 21:40:09 +00:00
|
|
|
ip := config.ControlConfig.BindAddressOrLoopback(false, true)
|
2022-02-16 22:19:58 +00:00
|
|
|
port := config.ControlConfig.HTTPSPort
|
|
|
|
// on servers without a local apiserver, tunnel access via the loadbalancer
|
|
|
|
if config.ControlConfig.DisableAPIServer {
|
2022-07-21 21:40:09 +00:00
|
|
|
ip = config.ControlConfig.Loopback(true)
|
2022-02-16 22:19:58 +00:00
|
|
|
port = config.ControlConfig.APIServerPort
|
|
|
|
}
|
|
|
|
url := fmt.Sprintf("https://%s:%d", ip, port)
|
2022-09-29 20:16:33 +00:00
|
|
|
kubeConfig, err := HomeKubeConfig(true, config.ControlConfig.Rootless)
|
2019-01-01 08:23:01 +00:00
|
|
|
def := true
|
|
|
|
if err != nil {
|
2020-05-05 22:09:04 +00:00
|
|
|
kubeConfig = filepath.Join(config.ControlConfig.DataDir, "kubeconfig-"+version.Program+".yaml")
|
2019-01-01 08:23:01 +00:00
|
|
|
def = false
|
|
|
|
}
|
2019-05-09 00:53:33 +00:00
|
|
|
kubeConfigSymlink := kubeConfig
|
2019-03-08 22:47:44 +00:00
|
|
|
if config.ControlConfig.KubeConfigOutput != "" {
|
|
|
|
kubeConfig = config.ControlConfig.KubeConfigOutput
|
2019-01-22 21:14:58 +00:00
|
|
|
}
|
|
|
|
|
2019-05-09 00:53:33 +00:00
|
|
|
if isSymlink(kubeConfigSymlink) {
|
|
|
|
if err := os.Remove(kubeConfigSymlink); err != nil {
|
2020-09-21 16:56:03 +00:00
|
|
|
logrus.Errorf("Failed to remove kubeconfig symlink")
|
2019-05-09 00:53:33 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-04-28 23:01:33 +00:00
|
|
|
if err = clientaccess.WriteClientKubeConfig(kubeConfig, url, config.ControlConfig.Runtime.ServerCA, config.ControlConfig.Runtime.ClientAdminCert,
|
2020-07-24 19:07:32 +00:00
|
|
|
config.ControlConfig.Runtime.ClientAdminKey); err == nil {
|
|
|
|
logrus.Infof("Wrote kubeconfig %s", kubeConfig)
|
|
|
|
} else {
|
2019-01-01 08:23:01 +00:00
|
|
|
logrus.Errorf("Failed to generate kubeconfig: %v", err)
|
2020-07-24 19:07:32 +00:00
|
|
|
return err
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2019-03-08 22:47:44 +00:00
|
|
|
if config.ControlConfig.KubeConfigMode != "" {
|
|
|
|
mode, err := strconv.ParseInt(config.ControlConfig.KubeConfigMode, 8, 0)
|
2019-01-22 21:14:58 +00:00
|
|
|
if err == nil {
|
2020-02-22 18:39:33 +00:00
|
|
|
util.SetFileModeForPath(kubeConfig, os.FileMode(mode))
|
2019-01-22 21:14:58 +00:00
|
|
|
} else {
|
2020-09-21 16:56:03 +00:00
|
|
|
logrus.Errorf("Failed to set %s to mode %s: %v", kubeConfig, os.FileMode(mode), err)
|
2019-01-22 21:14:58 +00:00
|
|
|
}
|
2019-02-19 17:46:35 +00:00
|
|
|
} else {
|
2020-02-22 18:39:33 +00:00
|
|
|
util.SetFileModeForPath(kubeConfig, os.FileMode(0600))
|
2019-01-22 21:14:58 +00:00
|
|
|
}
|
|
|
|
|
2019-05-09 00:53:33 +00:00
|
|
|
if kubeConfigSymlink != kubeConfig {
|
|
|
|
if err := writeConfigSymlink(kubeConfig, kubeConfigSymlink); err != nil {
|
2020-09-21 16:56:03 +00:00
|
|
|
logrus.Errorf("Failed to write kubeconfig symlink: %v", err)
|
2019-05-09 00:53:33 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-01-01 08:23:01 +00:00
|
|
|
if def {
|
|
|
|
logrus.Infof("Run: %s kubectl", filepath.Base(os.Args[0]))
|
|
|
|
}
|
2019-10-27 05:53:25 +00:00
|
|
|
|
|
|
|
return nil
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func setupDataDirAndChdir(config *config.Control) error {
|
|
|
|
var (
|
|
|
|
err error
|
|
|
|
)
|
|
|
|
|
2021-01-21 21:09:15 +00:00
|
|
|
config.DataDir, err = ResolveDataDir(config.DataDir)
|
2019-01-01 08:23:01 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
dataDir := config.DataDir
|
|
|
|
|
|
|
|
if err := os.MkdirAll(dataDir, 0700); err != nil {
|
|
|
|
return errors.Wrapf(err, "can not mkdir %s", dataDir)
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := os.Chdir(dataDir); err != nil {
|
|
|
|
return errors.Wrapf(err, "can not chdir %s", dataDir)
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2022-07-26 20:47:40 +00:00
|
|
|
func printToken(httpsPort int, advertiseIP, prefix, cmd, varName string) {
|
|
|
|
logrus.Infof("%s %s %s -s https://%s:%d -t ${%s}", prefix, version.Program, cmd, advertiseIP, httpsPort, varName)
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func writeToken(token, file, certs string) error {
|
|
|
|
if len(token) == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2021-06-22 20:42:34 +00:00
|
|
|
token, err := clientaccess.FormatToken(token, certs)
|
2019-10-27 05:53:25 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2022-10-08 00:36:57 +00:00
|
|
|
return os.WriteFile(file, []byte(token+"\n"), 0600)
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
2019-05-03 00:55:05 +00:00
|
|
|
|
|
|
|
func setNoProxyEnv(config *config.Control) error {
|
2020-10-12 18:02:07 +00:00
|
|
|
splitter := func(c rune) bool {
|
|
|
|
return c == ','
|
|
|
|
}
|
|
|
|
envList := []string{}
|
|
|
|
envList = append(envList, strings.FieldsFunc(os.Getenv("NO_PROXY"), splitter)...)
|
|
|
|
envList = append(envList, strings.FieldsFunc(os.Getenv("no_proxy"), splitter)...)
|
|
|
|
envList = append(envList,
|
|
|
|
".svc",
|
|
|
|
"."+config.ClusterDomain,
|
2021-04-21 22:56:20 +00:00
|
|
|
util.JoinIPNets(config.ClusterIPRanges),
|
|
|
|
util.JoinIPNets(config.ServiceIPRanges),
|
2020-10-12 18:02:07 +00:00
|
|
|
)
|
|
|
|
os.Unsetenv("no_proxy")
|
|
|
|
return os.Setenv("NO_PROXY", strings.Join(envList, ","))
|
2019-05-03 00:55:05 +00:00
|
|
|
}
|
2019-05-09 00:53:33 +00:00
|
|
|
|
|
|
|
func writeConfigSymlink(kubeconfig, kubeconfigSymlink string) error {
|
|
|
|
if err := os.Remove(kubeconfigSymlink); err != nil && !os.IsNotExist(err) {
|
|
|
|
return fmt.Errorf("failed to remove %s file: %v", kubeconfigSymlink, err)
|
|
|
|
}
|
|
|
|
if err := os.MkdirAll(filepath.Dir(kubeconfigSymlink), 0755); err != nil {
|
|
|
|
return fmt.Errorf("failed to create path for symlink: %v", err)
|
|
|
|
}
|
|
|
|
if err := os.Symlink(kubeconfig, kubeconfigSymlink); err != nil {
|
|
|
|
return fmt.Errorf("failed to create symlink: %v", err)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func isSymlink(config string) bool {
|
|
|
|
if fi, err := os.Lstat(config); err == nil && (fi.Mode()&os.ModeSymlink == os.ModeSymlink) {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
2019-09-26 18:10:19 +00:00
|
|
|
|
2021-12-07 22:31:32 +00:00
|
|
|
func setNodeLabelsAndAnnotations(ctx context.Context, nodes v1.NodeClient, config *Config) error {
|
2021-02-12 15:35:57 +00:00
|
|
|
if config.DisableAgent || config.ControlConfig.DisableAPIServer {
|
|
|
|
return nil
|
|
|
|
}
|
2019-09-26 18:10:19 +00:00
|
|
|
for {
|
|
|
|
nodeName := os.Getenv("NODE_NAME")
|
2020-12-10 18:26:16 +00:00
|
|
|
if nodeName == "" {
|
|
|
|
logrus.Info("Waiting for control-plane node agent startup")
|
|
|
|
time.Sleep(1 * time.Second)
|
|
|
|
continue
|
|
|
|
}
|
2019-10-27 05:53:25 +00:00
|
|
|
node, err := nodes.Get(nodeName, metav1.GetOptions{})
|
2019-09-26 18:10:19 +00:00
|
|
|
if err != nil {
|
2020-12-02 19:54:13 +00:00
|
|
|
logrus.Infof("Waiting for control-plane node %s startup: %v", nodeName, err)
|
2019-09-26 18:10:19 +00:00
|
|
|
time.Sleep(1 * time.Second)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if node.Labels == nil {
|
|
|
|
node.Labels = make(map[string]string)
|
|
|
|
}
|
2021-12-07 22:31:32 +00:00
|
|
|
v, ok := node.Labels[ControlPlaneRoleLabelKey]
|
2023-02-13 20:00:52 +00:00
|
|
|
if !ok || v != "true" {
|
2021-12-07 22:31:32 +00:00
|
|
|
node.Labels[ControlPlaneRoleLabelKey] = "true"
|
|
|
|
node.Labels[MasterRoleLabelKey] = "true"
|
|
|
|
}
|
|
|
|
|
|
|
|
if config.ControlConfig.EncryptSecrets {
|
|
|
|
if err = secretsencrypt.BootstrapEncryptionHashAnnotation(node, config.ControlConfig.Runtime); err != nil {
|
|
|
|
logrus.Infof("Unable to set encryption hash annotation %s", err.Error())
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
2021-02-12 15:35:57 +00:00
|
|
|
|
2019-10-27 05:53:25 +00:00
|
|
|
_, err = nodes.Update(node)
|
2019-09-26 18:10:19 +00:00
|
|
|
if err == nil {
|
2021-12-07 22:31:32 +00:00
|
|
|
logrus.Infof("Labels and annotations have been set successfully on node: %s", nodeName)
|
2019-09-26 18:10:19 +00:00
|
|
|
break
|
|
|
|
}
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return ctx.Err()
|
|
|
|
case <-time.After(time.Second):
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
2020-06-22 21:06:01 +00:00
|
|
|
|
2023-02-13 20:00:52 +00:00
|
|
|
func setClusterDNSConfig(ctx context.Context, config *Config, configMap v1.ConfigMapClient) error {
|
|
|
|
if config.ControlConfig.DisableAPIServer {
|
|
|
|
return nil
|
|
|
|
}
|
2020-06-22 21:06:01 +00:00
|
|
|
// check if configmap already exists
|
|
|
|
_, err := configMap.Get("kube-system", "cluster-dns", metav1.GetOptions{})
|
|
|
|
if err == nil {
|
2020-09-21 16:56:03 +00:00
|
|
|
logrus.Infof("Cluster dns configmap already exists")
|
2020-06-22 21:06:01 +00:00
|
|
|
return nil
|
|
|
|
}
|
2023-02-13 20:00:52 +00:00
|
|
|
clusterDNS := config.ControlConfig.ClusterDNS
|
|
|
|
clusterDomain := config.ControlConfig.ClusterDomain
|
2020-06-22 21:06:01 +00:00
|
|
|
c := &corev1.ConfigMap{
|
|
|
|
TypeMeta: metav1.TypeMeta{
|
|
|
|
Kind: "ConfigMap",
|
|
|
|
APIVersion: "v1",
|
|
|
|
},
|
|
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
|
|
Name: "cluster-dns",
|
|
|
|
Namespace: "kube-system",
|
|
|
|
},
|
|
|
|
Data: map[string]string{
|
|
|
|
"clusterDNS": clusterDNS.String(),
|
|
|
|
"clusterDomain": clusterDomain,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
for {
|
|
|
|
_, err = configMap.Create(c)
|
|
|
|
if err == nil {
|
2020-09-21 16:56:03 +00:00
|
|
|
logrus.Infof("Cluster dns configmap has been set successfully")
|
2020-06-22 21:06:01 +00:00
|
|
|
break
|
|
|
|
}
|
2020-12-10 18:26:16 +00:00
|
|
|
logrus.Infof("Waiting for control-plane dns startup: %v", err)
|
2020-06-22 21:06:01 +00:00
|
|
|
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return ctx.Err()
|
|
|
|
case <-time.After(time.Second):
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|