mirror of https://github.com/k3s-io/k3s
627 lines
26 KiB
Go
627 lines
26 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
systemd "github.com/coreos/go-systemd/daemon"
|
|
"github.com/erikdubbelboer/gspt"
|
|
"github.com/gorilla/mux"
|
|
"github.com/k3s-io/k3s/pkg/agent"
|
|
"github.com/k3s-io/k3s/pkg/agent/loadbalancer"
|
|
"github.com/k3s-io/k3s/pkg/cli/cmds"
|
|
"github.com/k3s-io/k3s/pkg/clientaccess"
|
|
"github.com/k3s-io/k3s/pkg/daemons/config"
|
|
"github.com/k3s-io/k3s/pkg/datadir"
|
|
"github.com/k3s-io/k3s/pkg/etcd"
|
|
"github.com/k3s-io/k3s/pkg/rootless"
|
|
"github.com/k3s-io/k3s/pkg/server"
|
|
"github.com/k3s-io/k3s/pkg/spegel"
|
|
"github.com/k3s-io/k3s/pkg/util"
|
|
"github.com/k3s-io/k3s/pkg/version"
|
|
"github.com/k3s-io/k3s/pkg/vpn"
|
|
"github.com/pkg/errors"
|
|
"github.com/rancher/wrangler/pkg/signals"
|
|
"github.com/sirupsen/logrus"
|
|
"github.com/urfave/cli"
|
|
utilnet "k8s.io/apimachinery/pkg/util/net"
|
|
"k8s.io/apiserver/pkg/authentication/authenticator"
|
|
kubeapiserverflag "k8s.io/component-base/cli/flag"
|
|
"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
|
|
utilsnet "k8s.io/utils/net"
|
|
|
|
_ "github.com/go-sql-driver/mysql" // ensure we have mysql
|
|
_ "github.com/lib/pq" // ensure we have postgres
|
|
_ "github.com/mattn/go-sqlite3" // ensure we have sqlite
|
|
)
|
|
|
|
func Run(app *cli.Context) error {
|
|
return run(app, &cmds.ServerConfig, server.CustomControllers{}, server.CustomControllers{})
|
|
}
|
|
|
|
func RunWithControllers(app *cli.Context, leaderControllers server.CustomControllers, controllers server.CustomControllers) error {
|
|
return run(app, &cmds.ServerConfig, leaderControllers, controllers)
|
|
}
|
|
|
|
func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomControllers, controllers server.CustomControllers) error {
|
|
var err error
|
|
// Validate build env
|
|
cmds.MustValidateGolang()
|
|
|
|
// hide process arguments from ps output, since they may contain
|
|
// database credentials or other secrets.
|
|
gspt.SetProcTitle(os.Args[0] + " server")
|
|
|
|
// If the agent is enabled, evacuate cgroup v2 before doing anything else that may fork.
|
|
// If the agent is disabled, we don't need to bother doing this as it is only the kubelet
|
|
// that cares about cgroups.
|
|
if !cfg.DisableAgent {
|
|
if err := cmds.EvacuateCgroup2(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// Initialize logging, and subprocess reaping if necessary.
|
|
// Log output redirection and subprocess reaping both require forking.
|
|
if err := cmds.InitLogging(); err != nil {
|
|
return err
|
|
}
|
|
|
|
if !cfg.DisableAgent && os.Getuid() != 0 && !cfg.Rootless {
|
|
return fmt.Errorf("server must run as root, or with --rootless and/or --disable-agent")
|
|
}
|
|
|
|
if cfg.Rootless {
|
|
dataDir, err := datadir.LocalHome(cfg.DataDir, true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cfg.DataDir = dataDir
|
|
if !cfg.DisableAgent {
|
|
dualNode, err := utilsnet.IsDualStackIPStrings(cmds.AgentConfig.NodeIP)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := rootless.Rootless(dataDir, dualNode); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
if cmds.AgentConfig.VPNAuthFile != "" {
|
|
cmds.AgentConfig.VPNAuth, err = util.ReadFile(cmds.AgentConfig.VPNAuthFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// Starts the VPN in the server if config was set up
|
|
if cmds.AgentConfig.VPNAuth != "" {
|
|
err := vpn.StartVPN(cmds.AgentConfig.VPNAuth)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
containerRuntimeReady := make(chan struct{})
|
|
|
|
serverConfig := server.Config{}
|
|
serverConfig.DisableAgent = cfg.DisableAgent
|
|
serverConfig.ControlConfig.Runtime = config.NewRuntime(containerRuntimeReady)
|
|
serverConfig.ControlConfig.Token = cfg.Token
|
|
serverConfig.ControlConfig.AgentToken = cfg.AgentToken
|
|
serverConfig.ControlConfig.JoinURL = cfg.ServerURL
|
|
if cfg.AgentTokenFile != "" {
|
|
serverConfig.ControlConfig.AgentToken, err = util.ReadFile(cfg.AgentTokenFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if cfg.TokenFile != "" {
|
|
serverConfig.ControlConfig.Token, err = util.ReadFile(cfg.TokenFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
serverConfig.ControlConfig.DataDir = cfg.DataDir
|
|
serverConfig.ControlConfig.KubeConfigOutput = cfg.KubeConfigOutput
|
|
serverConfig.ControlConfig.KubeConfigMode = cfg.KubeConfigMode
|
|
serverConfig.ControlConfig.HelmJobImage = cfg.HelmJobImage
|
|
serverConfig.ControlConfig.Rootless = cfg.Rootless
|
|
serverConfig.ControlConfig.ServiceLBNamespace = cfg.ServiceLBNamespace
|
|
serverConfig.ControlConfig.SANs = util.SplitStringSlice(cfg.TLSSan)
|
|
serverConfig.ControlConfig.SANSecurity = cfg.TLSSanSecurity
|
|
serverConfig.ControlConfig.BindAddress = cfg.BindAddress
|
|
serverConfig.ControlConfig.SupervisorPort = cfg.SupervisorPort
|
|
serverConfig.ControlConfig.HTTPSPort = cfg.HTTPSPort
|
|
serverConfig.ControlConfig.APIServerPort = cfg.APIServerPort
|
|
serverConfig.ControlConfig.APIServerBindAddress = cfg.APIServerBindAddress
|
|
serverConfig.ControlConfig.EnablePProf = cfg.EnablePProf
|
|
serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs
|
|
serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs
|
|
serverConfig.ControlConfig.ExtraEtcdArgs = cfg.ExtraEtcdArgs
|
|
serverConfig.ControlConfig.ExtraSchedulerAPIArgs = cfg.ExtraSchedulerArgs
|
|
serverConfig.ControlConfig.ClusterDomain = cfg.ClusterDomain
|
|
serverConfig.ControlConfig.Datastore.NotifyInterval = 5 * time.Second
|
|
serverConfig.ControlConfig.Datastore.Endpoint = cfg.DatastoreEndpoint
|
|
serverConfig.ControlConfig.Datastore.BackendTLSConfig.CAFile = cfg.DatastoreCAFile
|
|
serverConfig.ControlConfig.Datastore.BackendTLSConfig.CertFile = cfg.DatastoreCertFile
|
|
serverConfig.ControlConfig.Datastore.BackendTLSConfig.KeyFile = cfg.DatastoreKeyFile
|
|
serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP
|
|
serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
|
|
serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend
|
|
serverConfig.ControlConfig.FlannelIPv6Masq = cfg.FlannelIPv6Masq
|
|
serverConfig.ControlConfig.FlannelExternalIP = cfg.FlannelExternalIP
|
|
serverConfig.ControlConfig.EgressSelectorMode = cfg.EgressSelectorMode
|
|
serverConfig.ControlConfig.ExtraCloudControllerArgs = cfg.ExtraCloudControllerArgs
|
|
serverConfig.ControlConfig.DisableCCM = cfg.DisableCCM
|
|
serverConfig.ControlConfig.DisableNPC = cfg.DisableNPC
|
|
serverConfig.ControlConfig.DisableHelmController = cfg.DisableHelmController
|
|
serverConfig.ControlConfig.DisableKubeProxy = cfg.DisableKubeProxy
|
|
serverConfig.ControlConfig.DisableETCD = cfg.DisableETCD
|
|
serverConfig.ControlConfig.DisableAPIServer = cfg.DisableAPIServer
|
|
serverConfig.ControlConfig.DisableScheduler = cfg.DisableScheduler
|
|
serverConfig.ControlConfig.DisableControllerManager = cfg.DisableControllerManager
|
|
serverConfig.ControlConfig.DisableAgent = cfg.DisableAgent
|
|
serverConfig.ControlConfig.EmbeddedRegistry = cfg.EmbeddedRegistry
|
|
serverConfig.ControlConfig.ClusterInit = cfg.ClusterInit
|
|
serverConfig.ControlConfig.EncryptSecrets = cfg.EncryptSecrets
|
|
serverConfig.ControlConfig.EtcdExposeMetrics = cfg.EtcdExposeMetrics
|
|
serverConfig.ControlConfig.EtcdDisableSnapshots = cfg.EtcdDisableSnapshots
|
|
serverConfig.ControlConfig.VLevel = cmds.LogConfig.VLevel
|
|
serverConfig.ControlConfig.VModule = cmds.LogConfig.VModule
|
|
|
|
if !cfg.EtcdDisableSnapshots || cfg.ClusterReset {
|
|
serverConfig.ControlConfig.EtcdSnapshotCompress = cfg.EtcdSnapshotCompress
|
|
serverConfig.ControlConfig.EtcdSnapshotName = cfg.EtcdSnapshotName
|
|
serverConfig.ControlConfig.EtcdSnapshotCron = cfg.EtcdSnapshotCron
|
|
serverConfig.ControlConfig.EtcdSnapshotDir = cfg.EtcdSnapshotDir
|
|
serverConfig.ControlConfig.EtcdSnapshotRetention = cfg.EtcdSnapshotRetention
|
|
serverConfig.ControlConfig.EtcdS3 = cfg.EtcdS3
|
|
serverConfig.ControlConfig.EtcdS3Endpoint = cfg.EtcdS3Endpoint
|
|
serverConfig.ControlConfig.EtcdS3EndpointCA = cfg.EtcdS3EndpointCA
|
|
serverConfig.ControlConfig.EtcdS3SkipSSLVerify = cfg.EtcdS3SkipSSLVerify
|
|
serverConfig.ControlConfig.EtcdS3AccessKey = cfg.EtcdS3AccessKey
|
|
serverConfig.ControlConfig.EtcdS3SecretKey = cfg.EtcdS3SecretKey
|
|
serverConfig.ControlConfig.EtcdS3BucketName = cfg.EtcdS3BucketName
|
|
serverConfig.ControlConfig.EtcdS3Region = cfg.EtcdS3Region
|
|
serverConfig.ControlConfig.EtcdS3Folder = cfg.EtcdS3Folder
|
|
serverConfig.ControlConfig.EtcdS3Insecure = cfg.EtcdS3Insecure
|
|
serverConfig.ControlConfig.EtcdS3Timeout = cfg.EtcdS3Timeout
|
|
} else {
|
|
logrus.Info("ETCD snapshots are disabled")
|
|
}
|
|
|
|
if cfg.ClusterResetRestorePath != "" && !cfg.ClusterReset {
|
|
return errors.New("invalid flag use; --cluster-reset required with --cluster-reset-restore-path")
|
|
}
|
|
|
|
serverConfig.ControlConfig.ClusterReset = cfg.ClusterReset
|
|
serverConfig.ControlConfig.ClusterResetRestorePath = cfg.ClusterResetRestorePath
|
|
serverConfig.ControlConfig.SystemDefaultRegistry = cfg.SystemDefaultRegistry
|
|
|
|
if serverConfig.ControlConfig.SupervisorPort == 0 {
|
|
serverConfig.ControlConfig.SupervisorPort = serverConfig.ControlConfig.HTTPSPort
|
|
}
|
|
|
|
if serverConfig.ControlConfig.DisableETCD && serverConfig.ControlConfig.JoinURL == "" {
|
|
return errors.New("invalid flag use; --server is required with --disable-etcd")
|
|
}
|
|
|
|
if serverConfig.ControlConfig.Datastore.Endpoint != "" && serverConfig.ControlConfig.DisableAPIServer {
|
|
return errors.New("invalid flag use; cannot use --disable-apiserver with --datastore-endpoint")
|
|
}
|
|
|
|
if serverConfig.ControlConfig.Datastore.Endpoint != "" && serverConfig.ControlConfig.DisableETCD {
|
|
return errors.New("invalid flag use; cannot use --disable-etcd with --datastore-endpoint")
|
|
}
|
|
|
|
if serverConfig.ControlConfig.DisableAPIServer {
|
|
// Servers without a local apiserver need to connect to the apiserver via the proxy load-balancer.
|
|
serverConfig.ControlConfig.APIServerPort = cmds.AgentConfig.LBServerPort
|
|
// If the supervisor and externally-facing apiserver are not on the same port, the proxy will
|
|
// have a separate load-balancer for the apiserver that we need to use instead.
|
|
if serverConfig.ControlConfig.SupervisorPort != serverConfig.ControlConfig.HTTPSPort {
|
|
serverConfig.ControlConfig.APIServerPort = cmds.AgentConfig.LBServerPort - 1
|
|
}
|
|
}
|
|
|
|
if cmds.AgentConfig.FlannelIface != "" && len(cmds.AgentConfig.NodeIP) == 0 {
|
|
ip, err := util.GetIPFromInterface(cmds.AgentConfig.FlannelIface)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cmds.AgentConfig.NodeIP.Set(ip)
|
|
}
|
|
|
|
if serverConfig.ControlConfig.PrivateIP == "" && len(cmds.AgentConfig.NodeIP) != 0 {
|
|
serverConfig.ControlConfig.PrivateIP = util.GetFirstValidIPString(cmds.AgentConfig.NodeIP)
|
|
}
|
|
|
|
// Ensure that we add the localhost name/ip and node name/ip to the SAN list. This list is shared by the
|
|
// certs for the supervisor, kube-apiserver cert, and etcd. DNS entries for the in-cluster kubernetes
|
|
// service endpoint are added later when the certificates are created.
|
|
nodeName, nodeIPs, err := util.GetHostnameAndIPs(cmds.AgentConfig.NodeName, cmds.AgentConfig.NodeIP)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
serverConfig.ControlConfig.ServerNodeName = nodeName
|
|
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "::1", "localhost", nodeName)
|
|
for _, ip := range nodeIPs {
|
|
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, ip.String())
|
|
}
|
|
|
|
// if not set, try setting advertise-ip from agent VPN
|
|
if cmds.AgentConfig.VPNAuth != "" {
|
|
vpnInfo, err := vpn.GetVPNInfo(cmds.AgentConfig.VPNAuth)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// If we are in ipv6-only mode, we should pass the ipv6 address. Otherwise, ipv4
|
|
if utilsnet.IsIPv6(nodeIPs[0]) {
|
|
if vpnInfo.IPv6Address != nil {
|
|
logrus.Infof("Changed advertise-address to %v due to VPN", vpnInfo.IPv6Address)
|
|
if serverConfig.ControlConfig.AdvertiseIP != "" {
|
|
logrus.Warn("Conflict in the config detected. VPN integration overwrites advertise-address but the config is setting the advertise-address parameter")
|
|
}
|
|
serverConfig.ControlConfig.AdvertiseIP = vpnInfo.IPv6Address.String()
|
|
} else {
|
|
return errors.New("tailscale does not provide an ipv6 address")
|
|
}
|
|
} else {
|
|
// We are in dual-stack or ipv4-only mode
|
|
if vpnInfo.IPv4Address != nil {
|
|
logrus.Infof("Changed advertise-address to %v due to VPN", vpnInfo.IPv4Address)
|
|
if serverConfig.ControlConfig.AdvertiseIP != "" {
|
|
logrus.Warn("Conflict in the config detected. VPN integration overwrites advertise-address but the config is setting the advertise-address parameter")
|
|
}
|
|
serverConfig.ControlConfig.AdvertiseIP = vpnInfo.IPv4Address.String()
|
|
} else {
|
|
return errors.New("tailscale does not provide an ipv4 address")
|
|
}
|
|
}
|
|
logrus.Warn("Etcd IP (PrivateIP) remains the local IP. Running etcd traffic over VPN is not recommended due to performance issues")
|
|
} else {
|
|
|
|
// if not set, try setting advertise-ip from agent node-external-ip
|
|
if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeExternalIP) != 0 {
|
|
serverConfig.ControlConfig.AdvertiseIP = util.GetFirstValidIPString(cmds.AgentConfig.NodeExternalIP)
|
|
}
|
|
|
|
// if not set, try setting advertise-ip from agent node-ip
|
|
if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeIP) != 0 {
|
|
serverConfig.ControlConfig.AdvertiseIP = util.GetFirstValidIPString(cmds.AgentConfig.NodeIP)
|
|
}
|
|
}
|
|
|
|
// if we ended up with any advertise-ips, ensure they're added to the SAN list;
|
|
// note that kube-apiserver does not support dual-stack advertise-ip as of 1.21.0:
|
|
/// https://github.com/kubernetes/kubeadm/issues/1612#issuecomment-772583989
|
|
if serverConfig.ControlConfig.AdvertiseIP != "" {
|
|
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, serverConfig.ControlConfig.AdvertiseIP)
|
|
}
|
|
|
|
// configure ClusterIPRanges. Use default 10.42.0.0/16 or fd00:42::/56 if user did not set it
|
|
_, defaultClusterCIDR, defaultServiceCIDR, _ := util.GetDefaultAddresses(nodeIPs[0])
|
|
if len(cmds.ServerConfig.ClusterCIDR) == 0 {
|
|
cmds.ServerConfig.ClusterCIDR.Set(defaultClusterCIDR)
|
|
}
|
|
for _, cidr := range util.SplitStringSlice(cmds.ServerConfig.ClusterCIDR) {
|
|
_, parsed, err := net.ParseCIDR(cidr)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "invalid cluster-cidr %s", cidr)
|
|
}
|
|
serverConfig.ControlConfig.ClusterIPRanges = append(serverConfig.ControlConfig.ClusterIPRanges, parsed)
|
|
}
|
|
|
|
// set ClusterIPRange to the first address (first defined IPFamily is preferred)
|
|
serverConfig.ControlConfig.ClusterIPRange = serverConfig.ControlConfig.ClusterIPRanges[0]
|
|
|
|
// configure ServiceIPRanges. Use default 10.43.0.0/16 or fd00:43::/112 if user did not set it
|
|
if len(cmds.ServerConfig.ServiceCIDR) == 0 {
|
|
cmds.ServerConfig.ServiceCIDR.Set(defaultServiceCIDR)
|
|
}
|
|
for _, cidr := range util.SplitStringSlice(cmds.ServerConfig.ServiceCIDR) {
|
|
_, parsed, err := net.ParseCIDR(cidr)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "invalid service-cidr %s", cidr)
|
|
}
|
|
serverConfig.ControlConfig.ServiceIPRanges = append(serverConfig.ControlConfig.ServiceIPRanges, parsed)
|
|
}
|
|
|
|
// set ServiceIPRange to the first address (first defined IPFamily is preferred)
|
|
serverConfig.ControlConfig.ServiceIPRange = serverConfig.ControlConfig.ServiceIPRanges[0]
|
|
|
|
serverConfig.ControlConfig.ServiceNodePortRange, err = utilnet.ParsePortRange(cfg.ServiceNodePortRange)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "invalid port range %s", cfg.ServiceNodePortRange)
|
|
}
|
|
|
|
// the apiserver service does not yet support dual-stack operation
|
|
_, apiServerServiceIP, err := options.ServiceIPRange(*serverConfig.ControlConfig.ServiceIPRanges[0])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, apiServerServiceIP.String())
|
|
|
|
// If cluster-dns CLI arg is not set, we set ClusterDNS address to be the first IPv4 ServiceCIDR network + 10,
|
|
// i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10
|
|
// If there are no IPv4 ServiceCIDRs, an IPv6 ServiceCIDRs will be used.
|
|
// If neither of IPv4 or IPv6 are found an error is raised.
|
|
if len(cmds.ServerConfig.ClusterDNS) == 0 {
|
|
for _, svcCIDR := range serverConfig.ControlConfig.ServiceIPRanges {
|
|
clusterDNS, err := utilsnet.GetIndexedIP(svcCIDR, 10)
|
|
if err != nil {
|
|
return errors.Wrap(err, "cannot configure default cluster-dns address")
|
|
}
|
|
serverConfig.ControlConfig.ClusterDNSs = append(serverConfig.ControlConfig.ClusterDNSs, clusterDNS)
|
|
}
|
|
} else {
|
|
for _, ip := range util.SplitStringSlice(cmds.ServerConfig.ClusterDNS) {
|
|
parsed := net.ParseIP(ip)
|
|
if parsed == nil {
|
|
return fmt.Errorf("invalid cluster-dns address %s", ip)
|
|
}
|
|
serverConfig.ControlConfig.ClusterDNSs = append(serverConfig.ControlConfig.ClusterDNSs, parsed)
|
|
}
|
|
}
|
|
|
|
serverConfig.ControlConfig.ClusterDNS = serverConfig.ControlConfig.ClusterDNSs[0]
|
|
|
|
if err := validateNetworkConfiguration(serverConfig); err != nil {
|
|
return err
|
|
}
|
|
|
|
if cfg.DefaultLocalStoragePath == "" {
|
|
dataDir, err := datadir.LocalHome(cfg.DataDir, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
serverConfig.ControlConfig.DefaultLocalStoragePath = filepath.Join(dataDir, "/storage")
|
|
} else {
|
|
serverConfig.ControlConfig.DefaultLocalStoragePath = cfg.DefaultLocalStoragePath
|
|
}
|
|
|
|
serverConfig.ControlConfig.Skips = map[string]bool{}
|
|
serverConfig.ControlConfig.Disables = map[string]bool{}
|
|
for _, disable := range util.SplitStringSlice(app.StringSlice("disable")) {
|
|
disable = strings.TrimSpace(disable)
|
|
serverConfig.ControlConfig.Skips[disable] = true
|
|
serverConfig.ControlConfig.Disables[disable] = true
|
|
}
|
|
if serverConfig.ControlConfig.Skips["servicelb"] {
|
|
serverConfig.ControlConfig.DisableServiceLB = true
|
|
}
|
|
|
|
if serverConfig.ControlConfig.DisableCCM && serverConfig.ControlConfig.DisableServiceLB {
|
|
serverConfig.ControlConfig.Skips["ccm"] = true
|
|
serverConfig.ControlConfig.Disables["ccm"] = true
|
|
}
|
|
|
|
tlsMinVersionArg := getArgValueFromList("tls-min-version", serverConfig.ControlConfig.ExtraAPIArgs)
|
|
serverConfig.ControlConfig.TLSMinVersion, err = kubeapiserverflag.TLSVersion(tlsMinVersionArg)
|
|
if err != nil {
|
|
return errors.Wrap(err, "invalid tls-min-version")
|
|
}
|
|
|
|
serverConfig.StartupHooks = append(serverConfig.StartupHooks, cfg.StartupHooks...)
|
|
|
|
serverConfig.LeaderControllers = append(serverConfig.LeaderControllers, leaderControllers...)
|
|
serverConfig.Controllers = append(serverConfig.Controllers, controllers...)
|
|
|
|
// TLS config based on mozilla ssl-config generator
|
|
// https://ssl-config.mozilla.org/#server=golang&version=1.13.6&config=intermediate&guideline=5.4
|
|
// Need to disable the TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 Cipher for TLS1.2
|
|
tlsCipherSuitesArg := getArgValueFromList("tls-cipher-suites", serverConfig.ControlConfig.ExtraAPIArgs)
|
|
tlsCipherSuites := strings.Split(tlsCipherSuitesArg, ",")
|
|
for i := range tlsCipherSuites {
|
|
tlsCipherSuites[i] = strings.TrimSpace(tlsCipherSuites[i])
|
|
}
|
|
if len(tlsCipherSuites) == 0 || tlsCipherSuites[0] == "" {
|
|
tlsCipherSuites = []string{
|
|
"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
|
|
"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
|
|
"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
|
|
"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
|
|
"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
|
|
"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
|
|
}
|
|
serverConfig.ControlConfig.ExtraAPIArgs = append(serverConfig.ControlConfig.ExtraAPIArgs, "tls-cipher-suites="+strings.Join(tlsCipherSuites, ","))
|
|
}
|
|
serverConfig.ControlConfig.TLSCipherSuites, err = kubeapiserverflag.TLSCipherSuites(tlsCipherSuites)
|
|
if err != nil {
|
|
return errors.Wrap(err, "invalid tls-cipher-suites")
|
|
}
|
|
|
|
// If performing a cluster reset, make sure control-plane components are
|
|
// disabled so we only perform a reset or restore and bail out.
|
|
if cfg.ClusterReset {
|
|
serverConfig.ControlConfig.ClusterInit = true
|
|
serverConfig.ControlConfig.DisableAPIServer = true
|
|
serverConfig.ControlConfig.DisableControllerManager = true
|
|
serverConfig.ControlConfig.DisableScheduler = true
|
|
serverConfig.ControlConfig.DisableCCM = true
|
|
serverConfig.ControlConfig.DisableServiceLB = true
|
|
|
|
// If the supervisor and apiserver are on the same port, everything is running embedded
|
|
// and we don't need the kubelet or containerd up to perform a cluster reset.
|
|
if serverConfig.ControlConfig.SupervisorPort == serverConfig.ControlConfig.HTTPSPort {
|
|
cfg.DisableAgent = true
|
|
}
|
|
|
|
// If the user uses the cluster-reset argument in a cluster that has a ServerURL, we must return an error
|
|
// to remove the server flag on the configuration or in the cli
|
|
if serverConfig.ControlConfig.JoinURL != "" {
|
|
return errors.New("cannot perform cluster-reset while server URL is set - remove server from configuration before resetting")
|
|
}
|
|
|
|
dataDir, err := datadir.LocalHome(cfg.DataDir, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// delete local loadbalancers state for apiserver and supervisor servers
|
|
loadbalancer.ResetLoadBalancer(filepath.Join(dataDir, "agent"), loadbalancer.SupervisorServiceName)
|
|
loadbalancer.ResetLoadBalancer(filepath.Join(dataDir, "agent"), loadbalancer.APIServerServiceName)
|
|
|
|
if cfg.ClusterResetRestorePath != "" {
|
|
// at this point we're doing a restore. Check to see if we've
|
|
// passed in a token and if not, check if the token file exists.
|
|
// If it doesn't, return an error indicating the token is necessary.
|
|
if cfg.Token == "" {
|
|
tokenFile := filepath.Join(dataDir, "server", "token")
|
|
if _, err := os.Stat(tokenFile); err != nil {
|
|
if os.IsNotExist(err) {
|
|
return errors.New(tokenFile + " does not exist, please pass --token to complete the restoration")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
logrus.Info("Starting " + version.Program + " " + app.App.Version)
|
|
|
|
notifySocket := os.Getenv("NOTIFY_SOCKET")
|
|
os.Unsetenv("NOTIFY_SOCKET")
|
|
|
|
ctx := signals.SetupSignalContext()
|
|
|
|
if err := server.StartServer(ctx, &serverConfig, cfg); err != nil {
|
|
return err
|
|
}
|
|
|
|
go cmds.WriteCoverage(ctx)
|
|
|
|
go func() {
|
|
if !serverConfig.ControlConfig.DisableAPIServer {
|
|
<-serverConfig.ControlConfig.Runtime.APIServerReady
|
|
logrus.Info("Kube API server is now running")
|
|
serverConfig.ControlConfig.Runtime.StartupHooksWg.Wait()
|
|
}
|
|
if !serverConfig.ControlConfig.DisableETCD {
|
|
<-serverConfig.ControlConfig.Runtime.ETCDReady
|
|
logrus.Info("ETCD server is now running")
|
|
}
|
|
|
|
logrus.Info(version.Program + " is up and running")
|
|
os.Setenv("NOTIFY_SOCKET", notifySocket)
|
|
systemd.SdNotify(true, "READY=1\n")
|
|
}()
|
|
|
|
url := fmt.Sprintf("https://%s:%d", serverConfig.ControlConfig.BindAddressOrLoopback(false, true), serverConfig.ControlConfig.SupervisorPort)
|
|
token, err := clientaccess.FormatToken(serverConfig.ControlConfig.Runtime.AgentToken, serverConfig.ControlConfig.Runtime.ServerCA)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
agentConfig := cmds.AgentConfig
|
|
agentConfig.ContainerRuntimeReady = containerRuntimeReady
|
|
agentConfig.Debug = app.GlobalBool("debug")
|
|
agentConfig.DataDir = filepath.Dir(serverConfig.ControlConfig.DataDir)
|
|
agentConfig.ServerURL = url
|
|
agentConfig.Token = token
|
|
agentConfig.DisableLoadBalancer = !serverConfig.ControlConfig.DisableAPIServer
|
|
agentConfig.DisableServiceLB = serverConfig.ControlConfig.DisableServiceLB
|
|
agentConfig.ETCDAgent = serverConfig.ControlConfig.DisableAPIServer
|
|
agentConfig.ClusterReset = serverConfig.ControlConfig.ClusterReset
|
|
agentConfig.Rootless = cfg.Rootless
|
|
|
|
if agentConfig.Rootless {
|
|
// let agent specify Rootless kubelet flags, but not unshare twice
|
|
agentConfig.RootlessAlreadyUnshared = true
|
|
}
|
|
|
|
if serverConfig.ControlConfig.DisableAPIServer {
|
|
if cfg.ServerURL == "" {
|
|
// If this node is the initial member of the cluster and is not hosting an apiserver,
|
|
// always bootstrap the agent off local supervisor, and go through the process of reading
|
|
// apiserver endpoints from etcd and blocking further startup until one is available.
|
|
// This ensures that we don't end up in a chicken-and-egg situation on cluster restarts,
|
|
// where the loadbalancer is routing traffic to existing apiservers, but the apiservers
|
|
// are non-functional because they're waiting for us to start etcd.
|
|
loadbalancer.ResetLoadBalancer(filepath.Join(agentConfig.DataDir, "agent"), loadbalancer.SupervisorServiceName)
|
|
} else {
|
|
// If this is a secondary member of the cluster and is not hosting an apiserver,
|
|
// bootstrap the agent off the existing supervisor, instead of bootstrapping locally.
|
|
agentConfig.ServerURL = cfg.ServerURL
|
|
}
|
|
// initialize the apiAddress Channel for receiving the api address from etcd
|
|
agentConfig.APIAddressCh = make(chan []string)
|
|
go getAPIAddressFromEtcd(ctx, serverConfig, agentConfig)
|
|
}
|
|
|
|
if cfg.DisableAgent {
|
|
agentConfig.ContainerRuntimeEndpoint = "/dev/null"
|
|
return agent.RunStandalone(ctx, agentConfig)
|
|
}
|
|
|
|
if cfg.EmbeddedRegistry {
|
|
conf := spegel.DefaultRegistry
|
|
conf.Bootstrapper = spegel.NewChainingBootstrapper(
|
|
spegel.NewServerBootstrapper(&serverConfig.ControlConfig),
|
|
spegel.NewAgentBootstrapper(cfg.ServerURL, token, agentConfig.DataDir),
|
|
spegel.NewSelfBootstrapper(),
|
|
)
|
|
conf.HandlerFunc = func(_ *spegel.Config, router *mux.Router) error {
|
|
router.NotFoundHandler = serverConfig.ControlConfig.Runtime.Handler
|
|
serverConfig.ControlConfig.Runtime.Handler = router
|
|
return nil
|
|
}
|
|
conf.AuthFunc = func() authenticator.Request {
|
|
return serverConfig.ControlConfig.Runtime.Authenticator
|
|
}
|
|
}
|
|
|
|
return agent.Run(ctx, agentConfig)
|
|
}
|
|
|
|
// validateNetworkConfig ensures that the network configuration values make sense.
|
|
func validateNetworkConfiguration(serverConfig server.Config) error {
|
|
switch serverConfig.ControlConfig.EgressSelectorMode {
|
|
case config.EgressSelectorModeCluster, config.EgressSelectorModePod:
|
|
case config.EgressSelectorModeAgent, config.EgressSelectorModeDisabled:
|
|
if serverConfig.DisableAgent {
|
|
logrus.Warn("Webhooks and apiserver aggregation may not function properly without an agent; please set egress-selector-mode to 'cluster' or 'pod'")
|
|
}
|
|
default:
|
|
return fmt.Errorf("invalid egress-selector-mode %s", serverConfig.ControlConfig.EgressSelectorMode)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func getArgValueFromList(searchArg string, argList []string) string {
|
|
var value string
|
|
for _, arg := range argList {
|
|
splitArg := strings.SplitN(arg, "=", 2)
|
|
if splitArg[0] == searchArg {
|
|
value = splitArg[1]
|
|
// break if we found our value
|
|
break
|
|
}
|
|
}
|
|
return value
|
|
}
|
|
|
|
func getAPIAddressFromEtcd(ctx context.Context, serverConfig server.Config, agentConfig cmds.Agent) {
|
|
defer close(agentConfig.APIAddressCh)
|
|
for {
|
|
toCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
serverAddresses, err := etcd.GetAPIServerURLsFromETCD(toCtx, &serverConfig.ControlConfig)
|
|
if err == nil && len(serverAddresses) > 0 {
|
|
agentConfig.APIAddressCh <- serverAddresses
|
|
break
|
|
}
|
|
if !errors.Is(err, etcd.ErrAddressNotSet) {
|
|
logrus.Warnf("Failed to get apiserver address from etcd: %v", err)
|
|
}
|
|
<-toCtx.Done()
|
|
}
|
|
}
|