mirror of https://github.com/k3s-io/k3s
1105 lines
32 KiB
Go
1105 lines
32 KiB
Go
package control
|
|
|
|
import (
|
|
"context"
|
|
"crypto"
|
|
cryptorand "crypto/rand"
|
|
"crypto/x509"
|
|
b64 "encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"math/rand"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"text/template"
|
|
"time"
|
|
|
|
"k8s.io/apimachinery/pkg/util/sets"
|
|
|
|
"github.com/pkg/errors"
|
|
certutil "github.com/rancher/dynamiclistener/cert"
|
|
"github.com/rancher/k3s/pkg/clientaccess"
|
|
"github.com/rancher/k3s/pkg/cluster"
|
|
"github.com/rancher/k3s/pkg/daemons/config"
|
|
"github.com/rancher/k3s/pkg/daemons/executor"
|
|
"github.com/rancher/k3s/pkg/passwd"
|
|
"github.com/rancher/k3s/pkg/token"
|
|
"github.com/rancher/k3s/pkg/version"
|
|
"github.com/rancher/wrangler-api/pkg/generated/controllers/rbac"
|
|
"github.com/sirupsen/logrus"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
|
|
"k8s.io/apiserver/pkg/authentication/authenticator"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/client-go/tools/clientcmd"
|
|
ccm "k8s.io/cloud-provider"
|
|
ccmapp "k8s.io/cloud-provider/app"
|
|
ccmopt "k8s.io/cloud-provider/options"
|
|
app2 "k8s.io/controller-manager/app"
|
|
"k8s.io/kubernetes/pkg/controlplane"
|
|
"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
|
|
"k8s.io/kubernetes/pkg/proxy/util"
|
|
|
|
// registering k3s cloud provider
|
|
_ "github.com/rancher/k3s/pkg/cloudprovider"
|
|
// for client metric registration
|
|
_ "k8s.io/component-base/metrics/prometheus/restclient"
|
|
)
|
|
|
|
var (
|
|
localhostIP = net.ParseIP("127.0.0.1")
|
|
requestHeaderCN = "system:auth-proxy"
|
|
kubeconfigTemplate = template.Must(template.New("kubeconfig").Parse(`apiVersion: v1
|
|
clusters:
|
|
- cluster:
|
|
server: {{.URL}}
|
|
certificate-authority: {{.CACert}}
|
|
name: local
|
|
contexts:
|
|
- context:
|
|
cluster: local
|
|
namespace: default
|
|
user: user
|
|
name: Default
|
|
current-context: Default
|
|
kind: Config
|
|
preferences: {}
|
|
users:
|
|
- name: user
|
|
user:
|
|
client-certificate: {{.ClientCert}}
|
|
client-key: {{.ClientKey}}
|
|
`))
|
|
)
|
|
|
|
const (
|
|
ipsecTokenSize = 48
|
|
aescbcKeySize = 32
|
|
)
|
|
|
|
func Server(ctx context.Context, cfg *config.Control) error {
|
|
rand.Seed(time.Now().UTC().UnixNano())
|
|
|
|
runtime := &config.ControlRuntime{}
|
|
cfg.Runtime = runtime
|
|
|
|
if err := prepare(ctx, cfg, runtime); err != nil {
|
|
return errors.Wrap(err, "preparing server")
|
|
}
|
|
|
|
cfg.Runtime.Tunnel = setupTunnel()
|
|
util.DisableProxyHostnameCheck = true
|
|
|
|
auth, handler, err := apiServer(ctx, cfg, runtime)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := waitForAPIServerInBackground(ctx, runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
basicAuth, err := basicAuthenticator(runtime.PasswdFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
runtime.Authenticator = combineAuthenticators(basicAuth, auth)
|
|
runtime.Handler = handler
|
|
|
|
if !cfg.NoScheduler {
|
|
if err := scheduler(cfg, runtime); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if err := controllerManager(cfg, runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
if !cfg.DisableCCM {
|
|
cloudControllerManager(ctx, cfg, runtime)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) error {
|
|
argsMap := map[string]string{
|
|
"kubeconfig": runtime.KubeConfigController,
|
|
"service-account-private-key-file": runtime.ServiceKey,
|
|
"allocate-node-cidrs": "true",
|
|
"cluster-cidr": cfg.ClusterIPRange.String(),
|
|
"root-ca-file": runtime.ServerCA,
|
|
"port": "10252",
|
|
"profiling": "false",
|
|
"address": localhostIP.String(),
|
|
"bind-address": localhostIP.String(),
|
|
"secure-port": "0",
|
|
"use-service-account-credentials": "true",
|
|
"cluster-signing-cert-file": runtime.ClientCA,
|
|
"cluster-signing-key-file": runtime.ClientCAKey,
|
|
}
|
|
if cfg.NoLeaderElect {
|
|
argsMap["leader-elect"] = "false"
|
|
}
|
|
if !cfg.DisableCCM {
|
|
argsMap["configure-cloud-routes"] = "false"
|
|
argsMap["controllers"] = "*,-service,-route,-cloud-node-lifecycle"
|
|
}
|
|
|
|
args := config.GetArgsList(argsMap, cfg.ExtraControllerArgs)
|
|
logrus.Infof("Running kube-controller-manager %s", config.ArgString(args))
|
|
|
|
return executor.ControllerManager(runtime.APIServerReady, args)
|
|
}
|
|
|
|
func scheduler(cfg *config.Control, runtime *config.ControlRuntime) error {
|
|
argsMap := map[string]string{
|
|
"kubeconfig": runtime.KubeConfigScheduler,
|
|
"port": "10251",
|
|
"address": "127.0.0.1",
|
|
"bind-address": "127.0.0.1",
|
|
"secure-port": "0",
|
|
"profiling": "false",
|
|
}
|
|
if cfg.NoLeaderElect {
|
|
argsMap["leader-elect"] = "false"
|
|
}
|
|
args := config.GetArgsList(argsMap, cfg.ExtraSchedulerAPIArgs)
|
|
|
|
logrus.Infof("Running kube-scheduler %s", config.ArgString(args))
|
|
return executor.Scheduler(runtime.APIServerReady, args)
|
|
}
|
|
|
|
func apiServer(ctx context.Context, cfg *config.Control, runtime *config.ControlRuntime) (authenticator.Request, http.Handler, error) {
|
|
argsMap := make(map[string]string)
|
|
|
|
setupStorageBackend(argsMap, cfg)
|
|
|
|
certDir := filepath.Join(cfg.DataDir, "tls", "temporary-certs")
|
|
os.MkdirAll(certDir, 0700)
|
|
|
|
argsMap["cert-dir"] = certDir
|
|
argsMap["allow-privileged"] = "true"
|
|
argsMap["authorization-mode"] = strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ",")
|
|
argsMap["service-account-signing-key-file"] = runtime.ServiceKey
|
|
argsMap["service-cluster-ip-range"] = cfg.ServiceIPRange.String()
|
|
argsMap["advertise-port"] = strconv.Itoa(cfg.AdvertisePort)
|
|
if cfg.AdvertiseIP != "" {
|
|
argsMap["advertise-address"] = cfg.AdvertiseIP
|
|
}
|
|
argsMap["insecure-port"] = "0"
|
|
argsMap["secure-port"] = strconv.Itoa(cfg.APIServerPort)
|
|
if cfg.APIServerBindAddress == "" {
|
|
argsMap["bind-address"] = localhostIP.String()
|
|
} else {
|
|
argsMap["bind-address"] = cfg.APIServerBindAddress
|
|
}
|
|
argsMap["tls-cert-file"] = runtime.ServingKubeAPICert
|
|
argsMap["tls-private-key-file"] = runtime.ServingKubeAPIKey
|
|
argsMap["service-account-key-file"] = runtime.ServiceKey
|
|
argsMap["service-account-issuer"] = version.Program
|
|
argsMap["api-audiences"] = "unknown"
|
|
argsMap["kubelet-certificate-authority"] = runtime.ServerCA
|
|
argsMap["kubelet-client-certificate"] = runtime.ClientKubeAPICert
|
|
argsMap["kubelet-client-key"] = runtime.ClientKubeAPIKey
|
|
argsMap["requestheader-client-ca-file"] = runtime.RequestHeaderCA
|
|
argsMap["requestheader-allowed-names"] = requestHeaderCN
|
|
argsMap["proxy-client-cert-file"] = runtime.ClientAuthProxyCert
|
|
argsMap["proxy-client-key-file"] = runtime.ClientAuthProxyKey
|
|
argsMap["requestheader-extra-headers-prefix"] = "X-Remote-Extra-"
|
|
argsMap["requestheader-group-headers"] = "X-Remote-Group"
|
|
argsMap["requestheader-username-headers"] = "X-Remote-User"
|
|
argsMap["client-ca-file"] = runtime.ClientCA
|
|
argsMap["enable-admission-plugins"] = "NodeRestriction"
|
|
argsMap["anonymous-auth"] = "false"
|
|
argsMap["profiling"] = "false"
|
|
argsMap["feature-gates=ServiceAccountIssuerDiscovery"] = "false"
|
|
if cfg.EncryptSecrets {
|
|
argsMap["encryption-provider-config"] = runtime.EncryptionConfig
|
|
}
|
|
args := config.GetArgsList(argsMap, cfg.ExtraAPIArgs)
|
|
|
|
logrus.Infof("Running kube-apiserver %s", config.ArgString(args))
|
|
return executor.APIServer(ctx, runtime.ETCDReady, args)
|
|
}
|
|
|
|
func defaults(config *config.Control) {
|
|
if config.ClusterIPRange == nil {
|
|
_, clusterIPNet, _ := net.ParseCIDR("10.42.0.0/16")
|
|
config.ClusterIPRange = clusterIPNet
|
|
}
|
|
|
|
if config.ServiceIPRange == nil {
|
|
_, serviceIPNet, _ := net.ParseCIDR("10.43.0.0/16")
|
|
config.ServiceIPRange = serviceIPNet
|
|
}
|
|
|
|
if len(config.ClusterDNS) == 0 {
|
|
config.ClusterDNS = net.ParseIP("10.43.0.10")
|
|
}
|
|
|
|
if config.AdvertisePort == 0 {
|
|
config.AdvertisePort = config.HTTPSPort
|
|
}
|
|
|
|
if config.APIServerPort == 0 {
|
|
if config.HTTPSPort != 0 {
|
|
config.APIServerPort = config.HTTPSPort + 1
|
|
} else {
|
|
config.APIServerPort = 6444
|
|
}
|
|
}
|
|
|
|
if config.DataDir == "" {
|
|
config.DataDir = "./management-state"
|
|
}
|
|
}
|
|
|
|
func prepare(ctx context.Context, config *config.Control, runtime *config.ControlRuntime) error {
|
|
var err error
|
|
|
|
defaults(config)
|
|
|
|
if err := os.MkdirAll(config.DataDir, 0700); err != nil {
|
|
return err
|
|
}
|
|
|
|
config.DataDir, err = filepath.Abs(config.DataDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
os.MkdirAll(filepath.Join(config.DataDir, "tls"), 0700)
|
|
os.MkdirAll(filepath.Join(config.DataDir, "cred"), 0700)
|
|
|
|
runtime.ClientCA = filepath.Join(config.DataDir, "tls", "client-ca.crt")
|
|
runtime.ClientCAKey = filepath.Join(config.DataDir, "tls", "client-ca.key")
|
|
runtime.ServerCA = filepath.Join(config.DataDir, "tls", "server-ca.crt")
|
|
runtime.ServerCAKey = filepath.Join(config.DataDir, "tls", "server-ca.key")
|
|
runtime.RequestHeaderCA = filepath.Join(config.DataDir, "tls", "request-header-ca.crt")
|
|
runtime.RequestHeaderCAKey = filepath.Join(config.DataDir, "tls", "request-header-ca.key")
|
|
runtime.IPSECKey = filepath.Join(config.DataDir, "cred", "ipsec.psk")
|
|
|
|
runtime.ServiceKey = filepath.Join(config.DataDir, "tls", "service.key")
|
|
runtime.PasswdFile = filepath.Join(config.DataDir, "cred", "passwd")
|
|
runtime.NodePasswdFile = filepath.Join(config.DataDir, "cred", "node-passwd")
|
|
|
|
runtime.KubeConfigAdmin = filepath.Join(config.DataDir, "cred", "admin.kubeconfig")
|
|
runtime.KubeConfigController = filepath.Join(config.DataDir, "cred", "controller.kubeconfig")
|
|
runtime.KubeConfigScheduler = filepath.Join(config.DataDir, "cred", "scheduler.kubeconfig")
|
|
runtime.KubeConfigAPIServer = filepath.Join(config.DataDir, "cred", "api-server.kubeconfig")
|
|
runtime.KubeConfigCloudController = filepath.Join(config.DataDir, "cred", "cloud-controller.kubeconfig")
|
|
|
|
runtime.ClientAdminCert = filepath.Join(config.DataDir, "tls", "client-admin.crt")
|
|
runtime.ClientAdminKey = filepath.Join(config.DataDir, "tls", "client-admin.key")
|
|
runtime.ClientControllerCert = filepath.Join(config.DataDir, "tls", "client-controller.crt")
|
|
runtime.ClientControllerKey = filepath.Join(config.DataDir, "tls", "client-controller.key")
|
|
runtime.ClientCloudControllerCert = filepath.Join(config.DataDir, "tls", "client-cloud-controller.crt")
|
|
runtime.ClientCloudControllerKey = filepath.Join(config.DataDir, "tls", "client-cloud-controller.key")
|
|
runtime.ClientSchedulerCert = filepath.Join(config.DataDir, "tls", "client-scheduler.crt")
|
|
runtime.ClientSchedulerKey = filepath.Join(config.DataDir, "tls", "client-scheduler.key")
|
|
runtime.ClientKubeAPICert = filepath.Join(config.DataDir, "tls", "client-kube-apiserver.crt")
|
|
runtime.ClientKubeAPIKey = filepath.Join(config.DataDir, "tls", "client-kube-apiserver.key")
|
|
runtime.ClientKubeProxyCert = filepath.Join(config.DataDir, "tls", "client-kube-proxy.crt")
|
|
runtime.ClientKubeProxyKey = filepath.Join(config.DataDir, "tls", "client-kube-proxy.key")
|
|
runtime.ClientK3sControllerCert = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-controller.crt")
|
|
runtime.ClientK3sControllerKey = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-controller.key")
|
|
|
|
runtime.ServingKubeAPICert = filepath.Join(config.DataDir, "tls", "serving-kube-apiserver.crt")
|
|
runtime.ServingKubeAPIKey = filepath.Join(config.DataDir, "tls", "serving-kube-apiserver.key")
|
|
|
|
runtime.ClientKubeletKey = filepath.Join(config.DataDir, "tls", "client-kubelet.key")
|
|
runtime.ServingKubeletKey = filepath.Join(config.DataDir, "tls", "serving-kubelet.key")
|
|
|
|
runtime.ClientAuthProxyCert = filepath.Join(config.DataDir, "tls", "client-auth-proxy.crt")
|
|
runtime.ClientAuthProxyKey = filepath.Join(config.DataDir, "tls", "client-auth-proxy.key")
|
|
|
|
runtime.ETCDServerCA = filepath.Join(config.DataDir, "tls", "etcd", "server-ca.crt")
|
|
runtime.ETCDServerCAKey = filepath.Join(config.DataDir, "tls", "etcd", "server-ca.key")
|
|
runtime.ETCDPeerCA = filepath.Join(config.DataDir, "tls", "etcd", "peer-ca.crt")
|
|
runtime.ETCDPeerCAKey = filepath.Join(config.DataDir, "tls", "etcd", "peer-ca.key")
|
|
runtime.ServerETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "server-client.crt")
|
|
runtime.ServerETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "server-client.key")
|
|
runtime.PeerServerClientETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "peer-server-client.crt")
|
|
runtime.PeerServerClientETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "peer-server-client.key")
|
|
runtime.ClientETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "client.crt")
|
|
runtime.ClientETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "client.key")
|
|
|
|
if config.EncryptSecrets {
|
|
runtime.EncryptionConfig = filepath.Join(config.DataDir, "cred", "encryption-config.json")
|
|
}
|
|
|
|
cluster := cluster.New(config)
|
|
|
|
if err := cluster.Bootstrap(ctx); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := genCerts(config, runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := genServiceAccount(runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := genUsers(config, runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := genEncryptedNetworkInfo(config, runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := genEncryptionConfig(config, runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := readTokens(runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
ready, err := cluster.Start(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
runtime.ETCDReady = ready
|
|
return nil
|
|
}
|
|
|
|
func readTokens(runtime *config.ControlRuntime) error {
|
|
tokens, err := passwd.Read(runtime.PasswdFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if nodeToken, ok := tokens.Pass("node"); ok {
|
|
runtime.AgentToken = "node:" + nodeToken
|
|
}
|
|
if serverToken, ok := tokens.Pass("server"); ok {
|
|
runtime.ServerToken = "server:" + serverToken
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.ControlRuntime) error {
|
|
if s, err := os.Stat(runtime.IPSECKey); err == nil && s.Size() > 0 {
|
|
psk, err := ioutil.ReadFile(runtime.IPSECKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
controlConfig.IPSECPSK = strings.TrimSpace(string(psk))
|
|
return nil
|
|
}
|
|
|
|
psk, err := token.Random(ipsecTokenSize)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
controlConfig.IPSECPSK = psk
|
|
if err := ioutil.WriteFile(runtime.IPSECKey, []byte(psk+"\n"), 0600); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func migratePassword(p *passwd.Passwd) error {
|
|
server, _ := p.Pass("server")
|
|
node, _ := p.Pass("node")
|
|
if server == "" && node != "" {
|
|
return p.EnsureUser("server", version.Program+":server", node)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func getServerPass(passwd *passwd.Passwd, config *config.Control) (string, error) {
|
|
var (
|
|
err error
|
|
)
|
|
|
|
serverPass := config.Token
|
|
if serverPass == "" {
|
|
serverPass, _ = passwd.Pass("server")
|
|
}
|
|
if serverPass == "" {
|
|
serverPass, err = token.Random(16)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
|
|
return serverPass, nil
|
|
}
|
|
|
|
func getNodePass(config *config.Control, serverPass string) string {
|
|
if config.AgentToken == "" {
|
|
if _, passwd, ok := clientaccess.ParseUsernamePassword(serverPass); ok {
|
|
return passwd
|
|
}
|
|
return serverPass
|
|
}
|
|
return config.AgentToken
|
|
}
|
|
|
|
func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
|
|
passwd, err := passwd.Read(runtime.PasswdFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := migratePassword(passwd); err != nil {
|
|
return err
|
|
}
|
|
|
|
serverPass, err := getServerPass(passwd, config)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
nodePass := getNodePass(config, serverPass)
|
|
|
|
if err := passwd.EnsureUser("node", version.Program+":agent", nodePass); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := passwd.EnsureUser("server", version.Program+":server", serverPass); err != nil {
|
|
return err
|
|
}
|
|
|
|
return passwd.Write(runtime.PasswdFile)
|
|
}
|
|
|
|
func genCerts(config *config.Control, runtime *config.ControlRuntime) error {
|
|
if err := genClientCerts(config, runtime); err != nil {
|
|
return err
|
|
}
|
|
if err := genServerCerts(config, runtime); err != nil {
|
|
return err
|
|
}
|
|
if err := genRequestHeaderCerts(config, runtime); err != nil {
|
|
return err
|
|
}
|
|
if err := genETCDCerts(config, runtime); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type signedCertFactory = func(commonName string, organization []string, certFile, keyFile string) (bool, error)
|
|
|
|
func getSigningCertFactory(regen bool, altNames *certutil.AltNames, extKeyUsage []x509.ExtKeyUsage, caCertFile, caKeyFile string) signedCertFactory {
|
|
return func(commonName string, organization []string, certFile, keyFile string) (bool, error) {
|
|
return createClientCertKey(regen, commonName, organization, altNames, extKeyUsage, caCertFile, caKeyFile, certFile, keyFile)
|
|
}
|
|
}
|
|
|
|
func genClientCerts(config *config.Control, runtime *config.ControlRuntime) error {
|
|
regen, err := createSigningCertKey(version.Program+"-client", runtime.ClientCA, runtime.ClientCAKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
factory := getSigningCertFactory(regen, nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, runtime.ClientCA, runtime.ClientCAKey)
|
|
|
|
var certGen bool
|
|
apiEndpoint := fmt.Sprintf("https://127.0.0.1:%d", config.APIServerPort)
|
|
|
|
certGen, err = factory("system:admin", []string{"system:masters"}, runtime.ClientAdminCert, runtime.ClientAdminKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if certGen {
|
|
if err := KubeConfig(runtime.KubeConfigAdmin, apiEndpoint, runtime.ServerCA, runtime.ClientAdminCert, runtime.ClientAdminKey); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
certGen, err = factory("system:kube-controller-manager", nil, runtime.ClientControllerCert, runtime.ClientControllerKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if certGen {
|
|
if err := KubeConfig(runtime.KubeConfigController, apiEndpoint, runtime.ServerCA, runtime.ClientControllerCert, runtime.ClientControllerKey); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
certGen, err = factory("system:kube-scheduler", nil, runtime.ClientSchedulerCert, runtime.ClientSchedulerKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if certGen {
|
|
if err := KubeConfig(runtime.KubeConfigScheduler, apiEndpoint, runtime.ServerCA, runtime.ClientSchedulerCert, runtime.ClientSchedulerKey); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
certGen, err = factory("kube-apiserver", nil, runtime.ClientKubeAPICert, runtime.ClientKubeAPIKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if certGen {
|
|
if err := KubeConfig(runtime.KubeConfigAPIServer, apiEndpoint, runtime.ServerCA, runtime.ClientKubeAPICert, runtime.ClientKubeAPIKey); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if _, err = factory("system:kube-proxy", nil, runtime.ClientKubeProxyCert, runtime.ClientKubeProxyKey); err != nil {
|
|
return err
|
|
}
|
|
// This user (system:k3s-controller by default) must be bound to a role in rolebindings.yaml or the downstream equivalent
|
|
if _, err = factory("system:"+version.Program+"-controller", nil, runtime.ClientK3sControllerCert, runtime.ClientK3sControllerKey); err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, _, err := certutil.LoadOrGenerateKeyFile(runtime.ClientKubeletKey, regen); err != nil {
|
|
return err
|
|
}
|
|
|
|
certGen, err = factory("cloud-controller-manager", nil, runtime.ClientCloudControllerCert, runtime.ClientCloudControllerKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if certGen {
|
|
if err := KubeConfig(runtime.KubeConfigCloudController, apiEndpoint, runtime.ServerCA, runtime.ClientCloudControllerCert, runtime.ClientCloudControllerKey); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func createServerSigningCertKey(config *config.Control, runtime *config.ControlRuntime) (bool, error) {
|
|
TokenCA := filepath.Join(config.DataDir, "tls", "token-ca.crt")
|
|
TokenCAKey := filepath.Join(config.DataDir, "tls", "token-ca.key")
|
|
|
|
if exists(TokenCA, TokenCAKey) && !exists(runtime.ServerCA) && !exists(runtime.ServerCAKey) {
|
|
logrus.Infof("Upgrading token-ca files to server-ca")
|
|
if err := os.Link(TokenCA, runtime.ServerCA); err != nil {
|
|
return false, err
|
|
}
|
|
if err := os.Link(TokenCAKey, runtime.ServerCAKey); err != nil {
|
|
return false, err
|
|
}
|
|
return true, nil
|
|
}
|
|
return createSigningCertKey(version.Program+"-server", runtime.ServerCA, runtime.ServerCAKey)
|
|
}
|
|
|
|
func addSANs(altNames *certutil.AltNames, sans []string) {
|
|
for _, san := range sans {
|
|
ip := net.ParseIP(san)
|
|
if ip == nil {
|
|
altNames.DNSNames = append(altNames.DNSNames, san)
|
|
} else {
|
|
altNames.IPs = append(altNames.IPs, ip)
|
|
}
|
|
}
|
|
}
|
|
|
|
func genServerCerts(config *config.Control, runtime *config.ControlRuntime) error {
|
|
regen, err := createServerSigningCertKey(config, runtime)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, apiServerServiceIP, err := controlplane.ServiceIPRange(*config.ServiceIPRange)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
altNames := &certutil.AltNames{
|
|
DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"},
|
|
IPs: []net.IP{apiServerServiceIP},
|
|
}
|
|
|
|
addSANs(altNames, config.SANs)
|
|
|
|
if _, err := createClientCertKey(regen, "kube-apiserver", nil,
|
|
altNames, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
|
runtime.ServerCA, runtime.ServerCAKey,
|
|
runtime.ServingKubeAPICert, runtime.ServingKubeAPIKey); err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, _, err := certutil.LoadOrGenerateKeyFile(runtime.ServingKubeletKey, regen); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func genETCDCerts(config *config.Control, runtime *config.ControlRuntime) error {
|
|
regen, err := createSigningCertKey("etcd-server", runtime.ETCDServerCA, runtime.ETCDServerCAKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
altNames := &certutil.AltNames{
|
|
DNSNames: []string{"localhost"},
|
|
}
|
|
addSANs(altNames, config.SANs)
|
|
|
|
if _, err := createClientCertKey(regen, "etcd-server", nil,
|
|
altNames, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
|
|
runtime.ETCDServerCA, runtime.ETCDServerCAKey,
|
|
runtime.ServerETCDCert, runtime.ServerETCDKey); err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err := createClientCertKey(regen, "etcd-client", nil,
|
|
nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
|
|
runtime.ETCDServerCA, runtime.ETCDServerCAKey,
|
|
runtime.ClientETCDCert, runtime.ClientETCDKey); err != nil {
|
|
return err
|
|
}
|
|
|
|
regen, err = createSigningCertKey("etcd-peer", runtime.ETCDPeerCA, runtime.ETCDPeerCAKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err := createClientCertKey(regen, "etcd-peer", nil,
|
|
altNames, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
|
|
runtime.ETCDPeerCA, runtime.ETCDPeerCAKey,
|
|
runtime.PeerServerClientETCDCert, runtime.PeerServerClientETCDKey); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func genRequestHeaderCerts(config *config.Control, runtime *config.ControlRuntime) error {
|
|
regen, err := createSigningCertKey(version.Program+"-request-header", runtime.RequestHeaderCA, runtime.RequestHeaderCAKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err := createClientCertKey(regen, requestHeaderCN, nil,
|
|
nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
|
|
runtime.RequestHeaderCA, runtime.RequestHeaderCAKey,
|
|
runtime.ClientAuthProxyCert, runtime.ClientAuthProxyKey); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func createClientCertKey(regen bool, commonName string, organization []string, altNames *certutil.AltNames, extKeyUsage []x509.ExtKeyUsage, caCertFile, caKeyFile, certFile, keyFile string) (bool, error) {
|
|
caBytes, err := ioutil.ReadFile(caCertFile)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
pool := x509.NewCertPool()
|
|
pool.AppendCertsFromPEM(caBytes)
|
|
|
|
// check for certificate expiration
|
|
if !regen {
|
|
regen = expired(certFile, pool)
|
|
}
|
|
|
|
if !regen {
|
|
regen = sansChanged(certFile, altNames)
|
|
}
|
|
|
|
if !regen {
|
|
if exists(certFile, keyFile) {
|
|
return false, nil
|
|
}
|
|
}
|
|
|
|
caKeyBytes, err := ioutil.ReadFile(caKeyFile)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
caCert, err := certutil.ParseCertsPEM(caBytes)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
keyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile, regen)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
key, err := certutil.ParsePrivateKeyPEM(keyBytes)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
cfg := certutil.Config{
|
|
CommonName: commonName,
|
|
Organization: organization,
|
|
Usages: extKeyUsage,
|
|
}
|
|
if altNames != nil {
|
|
cfg.AltNames = *altNames
|
|
}
|
|
cert, err := certutil.NewSignedCert(cfg, key.(crypto.Signer), caCert[0], caKey.(crypto.Signer))
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
return true, certutil.WriteCert(certFile, append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...))
|
|
}
|
|
|
|
func exists(files ...string) bool {
|
|
for _, file := range files {
|
|
if _, err := os.Stat(file); err != nil {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func genServiceAccount(runtime *config.ControlRuntime) error {
|
|
_, keyErr := os.Stat(runtime.ServiceKey)
|
|
if keyErr == nil {
|
|
return nil
|
|
}
|
|
|
|
key, err := certutil.NewPrivateKey()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return certutil.WriteKey(runtime.ServiceKey, certutil.EncodePrivateKeyPEM(key))
|
|
}
|
|
|
|
func createSigningCertKey(prefix, certFile, keyFile string) (bool, error) {
|
|
if exists(certFile, keyFile) {
|
|
return false, nil
|
|
}
|
|
|
|
caKeyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile, false)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
cfg := certutil.Config{
|
|
CommonName: fmt.Sprintf("%s-ca@%d", prefix, time.Now().Unix()),
|
|
}
|
|
|
|
cert, err := certutil.NewSelfSignedCACert(cfg, caKey.(crypto.Signer))
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
if err := certutil.WriteCert(certFile, certutil.EncodeCertPEM(cert)); err != nil {
|
|
return false, err
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
func KubeConfig(dest, url, caCert, clientCert, clientKey string) error {
|
|
data := struct {
|
|
URL string
|
|
CACert string
|
|
ClientCert string
|
|
ClientKey string
|
|
}{
|
|
URL: url,
|
|
CACert: caCert,
|
|
ClientCert: clientCert,
|
|
ClientKey: clientKey,
|
|
}
|
|
|
|
output, err := os.Create(dest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer output.Close()
|
|
|
|
return kubeconfigTemplate.Execute(output, &data)
|
|
}
|
|
|
|
func setupStorageBackend(argsMap map[string]string, cfg *config.Control) {
|
|
argsMap["storage-backend"] = "etcd3"
|
|
// specify the endpoints
|
|
if len(cfg.Datastore.Endpoint) > 0 {
|
|
argsMap["etcd-servers"] = cfg.Datastore.Endpoint
|
|
}
|
|
// storage backend tls configuration
|
|
if len(cfg.Datastore.CAFile) > 0 {
|
|
argsMap["etcd-cafile"] = cfg.Datastore.CAFile
|
|
}
|
|
if len(cfg.Datastore.CertFile) > 0 {
|
|
argsMap["etcd-certfile"] = cfg.Datastore.CertFile
|
|
}
|
|
if len(cfg.Datastore.KeyFile) > 0 {
|
|
argsMap["etcd-keyfile"] = cfg.Datastore.KeyFile
|
|
}
|
|
}
|
|
|
|
func sansChanged(certFile string, sans *certutil.AltNames) bool {
|
|
if sans == nil {
|
|
return false
|
|
}
|
|
|
|
certBytes, err := ioutil.ReadFile(certFile)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
|
|
certificates, err := certutil.ParseCertsPEM(certBytes)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
|
|
if len(certificates) == 0 {
|
|
return false
|
|
}
|
|
|
|
if !sets.NewString(certificates[0].DNSNames...).HasAll(sans.DNSNames...) {
|
|
return true
|
|
}
|
|
|
|
ips := sets.NewString()
|
|
for _, ip := range certificates[0].IPAddresses {
|
|
ips.Insert(ip.String())
|
|
}
|
|
|
|
for _, ip := range sans.IPs {
|
|
if !ips.Has(ip.String()) {
|
|
return true
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
func expired(certFile string, pool *x509.CertPool) bool {
|
|
certBytes, err := ioutil.ReadFile(certFile)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
certificates, err := certutil.ParseCertsPEM(certBytes)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
_, err = certificates[0].Verify(x509.VerifyOptions{
|
|
Roots: pool,
|
|
KeyUsages: []x509.ExtKeyUsage{
|
|
x509.ExtKeyUsageAny,
|
|
},
|
|
})
|
|
if err != nil {
|
|
return true
|
|
}
|
|
return certutil.IsCertExpired(certificates[0], config.CertificateRenewDays)
|
|
}
|
|
|
|
func cloudControllerManager(ctx context.Context, cfg *config.Control, runtime *config.ControlRuntime) {
|
|
// Reference: https://github.com/kubernetes/kubernetes/pull/93764
|
|
// The above-linked change made the in-tree cloud-controller-manager command much more flexible
|
|
// but much more complicated to wrap. It now validates some of the configuration very early on, before
|
|
// the CLI args are parsed, so some of the configuration needs to be hardcoded instead of set via flags.
|
|
|
|
argsMap := map[string]string{
|
|
"profiling": "false",
|
|
}
|
|
args := config.GetArgsList(argsMap, cfg.ExtraCloudControllerArgs)
|
|
|
|
s, err := ccmopt.NewCloudControllerManagerOptions()
|
|
if err != nil {
|
|
logrus.Fatalf("Unable to initialize cloudcontroller options: %v", err)
|
|
}
|
|
|
|
s.KubeCloudShared.AllocateNodeCIDRs = true
|
|
s.KubeCloudShared.CloudProvider.Name = version.Program
|
|
s.KubeCloudShared.ClusterCIDR = cfg.ClusterIPRange.String()
|
|
s.KubeCloudShared.ConfigureCloudRoutes = false
|
|
s.Kubeconfig = runtime.KubeConfigCloudController
|
|
s.NodeStatusUpdateFrequency = metav1.Duration{Duration: 1 * time.Minute}
|
|
s.SecureServing.BindAddress = localhostIP
|
|
s.SecureServing.BindPort = 0
|
|
|
|
if cfg.NoLeaderElect {
|
|
s.Generic.LeaderElection.LeaderElect = false
|
|
}
|
|
|
|
c, err := s.Config([]string{}, []string{})
|
|
if err != nil {
|
|
logrus.Fatalf("Unable to create cloudcontroller config from options: %v", err)
|
|
}
|
|
|
|
cloud, err := ccm.InitCloudProvider(version.Program, runtime.KubeConfigCloudController)
|
|
if err != nil {
|
|
logrus.Fatalf("Cloud provider could not be initialized: %v", err)
|
|
}
|
|
if cloud == nil {
|
|
logrus.Fatalf("Cloud provider is nil")
|
|
}
|
|
|
|
cloud.Initialize(c.ClientBuilder, make(chan struct{}))
|
|
if informerUserCloud, ok := cloud.(ccm.InformerUser); ok {
|
|
informerUserCloud.SetInformers(c.SharedInformers)
|
|
}
|
|
|
|
controllerInitializers := ccmapp.DefaultControllerInitializers(c.Complete(), cloud)
|
|
delete(controllerInitializers, "service")
|
|
delete(controllerInitializers, "route")
|
|
|
|
command := ccmapp.NewCloudControllerManagerCommand(s, c, controllerInitializers)
|
|
command.SetArgs(args)
|
|
// register k3s cloud provider
|
|
|
|
go func() {
|
|
for {
|
|
// check for the cloud controller rbac binding
|
|
if err := checkForCloudControllerPrivileges(runtime); err != nil {
|
|
logrus.Infof("Waiting for cloudcontroller rbac role to be created")
|
|
select {
|
|
case <-ctx.Done():
|
|
logrus.Fatalf("cloud-controller-manager context canceled: %v", ctx.Err())
|
|
case <-time.After(time.Second):
|
|
continue
|
|
}
|
|
}
|
|
break
|
|
}
|
|
logrus.Infof("Running cloud-controller-manager for provider %v with args %v", cloud.ProviderName(), config.ArgString(args))
|
|
logrus.Fatalf("cloud-controller-manager exited: %v", command.ExecuteContext(ctx))
|
|
}()
|
|
}
|
|
|
|
func checkForCloudControllerPrivileges(runtime *config.ControlRuntime) error {
|
|
restConfig, err := clientcmd.BuildConfigFromFlags("", runtime.KubeConfigAdmin)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
crb := rbac.NewFactoryFromConfigOrDie(restConfig).Rbac().V1().ClusterRoleBinding()
|
|
_, err = crb.Get("cloud-controller-manager", metav1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func waitForAPIServerInBackground(ctx context.Context, runtime *config.ControlRuntime) error {
|
|
restConfig, err := clientcmd.BuildConfigFromFlags("", runtime.KubeConfigAdmin)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
k8sClient, err := kubernetes.NewForConfig(restConfig)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
done := make(chan struct{})
|
|
runtime.APIServerReady = done
|
|
|
|
go func() {
|
|
defer close(done)
|
|
|
|
etcdLoop:
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-runtime.ETCDReady:
|
|
break etcdLoop
|
|
case <-time.After(30 * time.Second):
|
|
logrus.Infof("Waiting for etcd server to become available")
|
|
}
|
|
}
|
|
|
|
logrus.Infof("Waiting for API server to become available")
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case err := <-promise(func() error { return app2.WaitForAPIServer(k8sClient, 30*time.Second) }):
|
|
if err != nil {
|
|
logrus.Infof("Waiting for API server to become available")
|
|
continue
|
|
}
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
|
|
return nil
|
|
}
|
|
|
|
func promise(f func() error) <-chan error {
|
|
c := make(chan error, 1)
|
|
go func() {
|
|
c <- f()
|
|
close(c)
|
|
}()
|
|
return c
|
|
}
|
|
|
|
func genEncryptionConfig(controlConfig *config.Control, runtime *config.ControlRuntime) error {
|
|
if !controlConfig.EncryptSecrets {
|
|
return nil
|
|
}
|
|
if s, err := os.Stat(runtime.EncryptionConfig); err == nil && s.Size() > 0 {
|
|
return nil
|
|
}
|
|
|
|
aescbcKey := make([]byte, aescbcKeySize, aescbcKeySize)
|
|
_, err := cryptorand.Read(aescbcKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
encodedKey := b64.StdEncoding.EncodeToString(aescbcKey)
|
|
|
|
encConfig := apiserverconfigv1.EncryptionConfiguration{
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "EncryptionConfiguration",
|
|
APIVersion: "apiserver.config.k8s.io/v1",
|
|
},
|
|
Resources: []apiserverconfigv1.ResourceConfiguration{
|
|
{
|
|
Resources: []string{"secrets"},
|
|
Providers: []apiserverconfigv1.ProviderConfiguration{
|
|
{
|
|
AESCBC: &apiserverconfigv1.AESConfiguration{
|
|
Keys: []apiserverconfigv1.Key{
|
|
{
|
|
Name: "aescbckey",
|
|
Secret: encodedKey,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Identity: &apiserverconfigv1.IdentityConfiguration{},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
jsonfile, err := json.Marshal(encConfig)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return ioutil.WriteFile(runtime.EncryptionConfig, jsonfile, 0600)
|
|
}
|