diff --git a/manifests/rolebindings.yaml b/manifests/rolebindings.yaml new file mode 100644 index 0000000000..d048eee998 --- /dev/null +++ b/manifests/rolebindings.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-apiserver-kubelet-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kubelet-api-admin +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: kube-apiserver diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index a32bc5cf69..1559785b90 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -23,6 +23,7 @@ import ( "github.com/rancher/k3s/pkg/cli/cmds" "github.com/rancher/k3s/pkg/clientaccess" "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/k3s/pkg/daemons/control" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/net" @@ -82,6 +83,10 @@ func getNodeNamedCrt(nodeName, nodePasswordFile string) HTTPRequester { } defer resp.Body.Close() + if resp.StatusCode == http.StatusForbidden { + return nil, fmt.Errorf("Node password rejected, contents of '%s' may not match server passwd entry", nodePasswordFile) + } + if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("%s: %s", u, resp.Status) } @@ -101,45 +106,55 @@ func ensureNodePassword(nodePasswordFile string) (string, error) { return "", err } nodePassword := hex.EncodeToString(password) - return nodePassword, ioutil.WriteFile(nodePasswordFile, []byte(nodePassword), 0600) + return nodePassword, ioutil.WriteFile(nodePasswordFile, []byte(nodePassword+"\n"), 0600) } -func getNodeCert(nodeName, nodeCertFile, nodeKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) { - nodeCert, err := Request("/v1-k3s/node.crt", info, getNodeNamedCrt(nodeName, nodePasswordFile)) +func getServingCert(nodeName, servingCertFile, servingKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) { + servingCert, err := Request("/v1-k3s/serving-kubelet.crt", info, getNodeNamedCrt(nodeName, nodePasswordFile)) if err != nil { return nil, err } - if err := ioutil.WriteFile(nodeCertFile, nodeCert, 0600); err != nil { + if err := ioutil.WriteFile(servingCertFile, servingCert, 0600); err != nil { return nil, errors.Wrapf(err, "failed to write node cert") } - nodeKey, err := clientaccess.Get("/v1-k3s/node.key", info) + servingKey, err := clientaccess.Get("/v1-k3s/serving-kubelet.key", info) if err != nil { return nil, err } - if err := ioutil.WriteFile(nodeKeyFile, nodeKey, 0600); err != nil { + if err := ioutil.WriteFile(servingKeyFile, servingKey, 0600); err != nil { return nil, errors.Wrapf(err, "failed to write node key") } - cert, err := tls.X509KeyPair(nodeCert, nodeKey) + cert, err := tls.X509KeyPair(servingCert, servingKey) if err != nil { return nil, err } return &cert, nil } -func writeNodeCA(dataDir string, nodeCert *tls.Certificate) (string, error) { - clientCABytes := pem.EncodeToMemory(&pem.Block{ - Type: "CERTIFICATE", - Bytes: nodeCert.Certificate[1], - }) - - clientCA := filepath.Join(dataDir, "client-ca.pem") - if err := ioutil.WriteFile(clientCA, clientCABytes, 0600); err != nil { - return "", errors.Wrapf(err, "failed to write client CA") +func getHostFile(filename string, info *clientaccess.Info) error { + basename := filepath.Base(filename) + fileBytes, err := clientaccess.Get("/v1-k3s/"+basename, info) + if err != nil { + return err } + if err := ioutil.WriteFile(filename, fileBytes, 0600); err != nil { + return errors.Wrapf(err, "failed to write cert %s", filename) + } + return nil +} - return clientCA, nil +func getNodeNamedHostFile(filename, nodeName, nodePasswordFile string, info *clientaccess.Info) error { + basename := filepath.Base(filename) + fileBytes, err := Request("/v1-k3s/"+basename, info, getNodeNamedCrt(nodeName, nodePasswordFile)) + if err != nil { + return err + } + if err := ioutil.WriteFile(filename, fileBytes, 0600); err != nil { + return errors.Wrapf(err, "failed to write cert %s", filename) + } + return nil } func getHostnameAndIP(info cmds.Agent) (string, string, error) { @@ -169,17 +184,17 @@ func getHostnameAndIP(info cmds.Agent) (string, string, error) { } func localAddress(controlConfig *config.Control) string { - return fmt.Sprintf("127.0.0.1:%d", controlConfig.AdvertisePort) + return fmt.Sprintf("127.0.0.1:%d", controlConfig.ProxyPort) } -func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, controlConfig *config.Control, nodeCert *tls.Certificate) (string, error) { +func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, controlConfig *config.Control, tlsCert *tls.Certificate) (string, error) { os.MkdirAll(envInfo.DataDir, 0700) kubeConfigPath := filepath.Join(envInfo.DataDir, "kubeconfig.yaml") info.URL = "https://" + localAddress(controlConfig) info.CACerts = pem.EncodeToMemory(&pem.Block{ Type: cert.CertificateBlockType, - Bytes: nodeCert.Certificate[1], + Bytes: tlsCert.Certificate[1], }) return kubeConfigPath, info.WriteKubeConfig(kubeConfigPath) @@ -253,25 +268,6 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { return nil, err } - nodeCertFile := filepath.Join(envInfo.DataDir, "token-node.crt") - nodeKeyFile := filepath.Join(envInfo.DataDir, "token-node.key") - nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt") - - nodeCert, err := getNodeCert(nodeName, nodeCertFile, nodeKeyFile, nodePasswordFile, info) - if err != nil { - return nil, err - } - - clientCA, err := writeNodeCA(envInfo.DataDir, nodeCert) - if err != nil { - return nil, err - } - - kubeConfig, err := writeKubeConfig(envInfo, *info, controlConfig, nodeCert) - if err != nil { - return nil, err - } - hostLocal, err := exec.LookPath("host-local") if err != nil { return nil, errors.Wrapf(err, "failed to find host-local") @@ -285,6 +281,61 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { } } + proxyURL := "https://" + localAddress(controlConfig) + + clientCAFile := filepath.Join(envInfo.DataDir, "client-ca.crt") + if err := getHostFile(clientCAFile, info); err != nil { + return nil, err + } + + serverCAFile := filepath.Join(envInfo.DataDir, "server-ca.crt") + if err := getHostFile(serverCAFile, info); err != nil { + return nil, err + } + + servingKubeletCert := filepath.Join(envInfo.DataDir, "serving-kubelet.crt") + servingKubeletKey := filepath.Join(envInfo.DataDir, "serving-kubelet.key") + nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt") + servingCert, err := getServingCert(nodeName, servingKubeletCert, servingKubeletKey, nodePasswordFile, info) + if err != nil { + return nil, err + } + + kubeconfigNode, err := writeKubeConfig(envInfo, *info, controlConfig, servingCert) + if err != nil { + return nil, err + } + + clientKubeletCert := filepath.Join(envInfo.DataDir, "client-kubelet.crt") + if err := getNodeNamedHostFile(clientKubeletCert, nodeName, nodePasswordFile, info); err != nil { + return nil, err + } + + clientKubeletKey := filepath.Join(envInfo.DataDir, "client-kubelet.key") + if err := getHostFile(clientKubeletKey, info); err != nil { + return nil, err + } + + kubeconfigKubelet := filepath.Join(envInfo.DataDir, "kubelet.kubeconfig") + if err := control.KubeConfig(kubeconfigKubelet, proxyURL, serverCAFile, clientKubeletCert, clientKubeletKey); err != nil { + return nil, err + } + + clientKubeProxyCert := filepath.Join(envInfo.DataDir, "client-kube-proxy.crt") + if err := getHostFile(clientKubeProxyCert, info); err != nil { + return nil, err + } + + clientKubeProxyKey := filepath.Join(envInfo.DataDir, "client-kube-proxy.key") + if err := getHostFile(clientKubeProxyKey, info); err != nil { + return nil, err + } + + kubeconfigKubeproxy := filepath.Join(envInfo.DataDir, "kubeproxy.kubeconfig") + if err := control.KubeConfig(kubeconfigKubeproxy, proxyURL, serverCAFile, clientKubeProxyCert, clientKubeProxyKey); err != nil { + return nil, err + } + nodeConfig := &config.Node{ Docker: envInfo.Docker, NoFlannel: envInfo.NoFlannel, @@ -295,14 +346,16 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { nodeConfig.Images = filepath.Join(envInfo.DataDir, "images") nodeConfig.AgentConfig.NodeIP = nodeIP nodeConfig.AgentConfig.NodeName = nodeName - nodeConfig.AgentConfig.NodeCertFile = nodeCertFile - nodeConfig.AgentConfig.NodeKeyFile = nodeKeyFile + nodeConfig.AgentConfig.ServingKubeletCert = servingKubeletCert + nodeConfig.AgentConfig.ServingKubeletKey = servingKubeletKey nodeConfig.AgentConfig.ClusterDNS = controlConfig.ClusterDNS nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo) - nodeConfig.AgentConfig.CACertPath = clientCA + nodeConfig.AgentConfig.ClientCA = clientCAFile nodeConfig.AgentConfig.ListenAddress = "0.0.0.0" - nodeConfig.AgentConfig.KubeConfig = kubeConfig + nodeConfig.AgentConfig.KubeConfigNode = kubeconfigNode + nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet + nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet") nodeConfig.AgentConfig.PauseImage = envInfo.PauseImage nodeConfig.CACerts = info.CACerts @@ -316,7 +369,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { nodeConfig.Containerd.Address = filepath.Join(nodeConfig.Containerd.State, "containerd.sock") nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml.tmpl") nodeConfig.ServerAddress = serverURLParsed.Host - nodeConfig.Certificate = nodeCert + nodeConfig.Certificate = servingCert if !nodeConfig.NoFlannel { nodeConfig.FlannelConf = filepath.Join(envInfo.DataDir, "etc/flannel/net-conf.json") nodeConfig.AgentConfig.CNIBinDir = filepath.Dir(hostLocal) diff --git a/pkg/agent/flannel/setup.go b/pkg/agent/flannel/setup.go index e90158174e..c2da4f347f 100644 --- a/pkg/agent/flannel/setup.go +++ b/pkg/agent/flannel/setup.go @@ -55,7 +55,7 @@ func Prepare(ctx context.Context, config *config.Node) error { func Run(ctx context.Context, config *config.Node) error { nodeName := config.AgentConfig.NodeName - restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig) + restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode) if err != nil { return err } @@ -79,7 +79,7 @@ func Run(ctx context.Context, config *config.Node) error { } go func() { - err := flannel(ctx, config.FlannelIface, config.FlannelConf, config.AgentConfig.KubeConfig) + err := flannel(ctx, config.FlannelIface, config.FlannelConf, config.AgentConfig.KubeConfigNode) logrus.Fatalf("flannel exited: %v", err) }() diff --git a/pkg/agent/proxy/proxy.go b/pkg/agent/proxy/proxy.go index 21d6dc91a9..e93a7be39c 100644 --- a/pkg/agent/proxy/proxy.go +++ b/pkg/agent/proxy/proxy.go @@ -1,35 +1,18 @@ package proxy import ( - "crypto/tls" - "net/http" - - "github.com/pkg/errors" + "github.com/google/tcpproxy" "github.com/rancher/k3s/pkg/daemons/config" - "github.com/rancher/k3s/pkg/proxy" "github.com/sirupsen/logrus" ) func Run(config *config.Node) error { - proxy, err := proxy.NewSimpleProxy(config.ServerAddress, config.CACerts, true) - if err != nil { - return err - } - - listener, err := tls.Listen("tcp", config.LocalAddress, &tls.Config{ - Certificates: []tls.Certificate{ - *config.Certificate, - }, - }) - - if err != nil { - return errors.Wrap(err, "Failed to start tls listener") - } - + logrus.Infof("Starting proxy %s -> %s", config.LocalAddress, config.ServerAddress) + var proxy tcpproxy.Proxy + proxy.AddRoute(config.LocalAddress, tcpproxy.To(config.ServerAddress)) go func() { - err := http.Serve(listener, proxy) + err := proxy.Run() logrus.Fatalf("TLS proxy stopped: %v", err) }() - return nil } diff --git a/pkg/agent/tunnel/tunnel.go b/pkg/agent/tunnel/tunnel.go index 6ce3a9938c..43ec741adc 100644 --- a/pkg/agent/tunnel/tunnel.go +++ b/pkg/agent/tunnel/tunnel.go @@ -26,7 +26,7 @@ var ( ) func Setup(config *config.Node) error { - restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig) + restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode) if err != nil { return err } diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index d4c65e4b31..0a2d2b2b63 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -17,7 +17,7 @@ type Server struct { DisableAgent bool KubeConfigOutput string KubeConfigMode string - KnownIPs cli.StringSlice + TLSSan cli.StringSlice BindAddress string ExtraAPIArgs cli.StringSlice ExtraSchedulerArgs cli.StringSlice @@ -28,6 +28,8 @@ type Server struct { StorageCAFile string StorageCertFile string StorageKeyFile string + AdvertiseIP string + AdvertisePort int } var ServerConfig Server @@ -120,7 +122,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { cli.StringSliceFlag{ Name: "tls-san", Usage: "Add additional hostname or IP as a Subject Alternative Name in the TLS cert", - Value: &ServerConfig.KnownIPs, + Value: &ServerConfig.TLSSan, }, cli.StringSliceFlag{ Name: "kube-apiserver-arg", @@ -172,6 +174,17 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { Destination: &ServerConfig.StorageKeyFile, EnvVar: "K3S_STORAGE_KEYFILE", }, + cli.StringFlag{ + Name: "advertise-address", + Usage: "IP address that apiserver uses to advertise to members of the cluster", + Destination: &ServerConfig.AdvertiseIP, + }, + cli.IntFlag{ + Name: "advertise-port", + Usage: "Port that apiserver uses to advertise to members of the cluster", + Value: 0, + Destination: &ServerConfig.AdvertisePort, + }, NodeIPFlag, NodeNameFlag, DockerFlag, diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 66a8a93938..fa4f8f1141 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -23,6 +23,7 @@ import ( "github.com/sirupsen/logrus" "github.com/urfave/cli" "k8s.io/apimachinery/pkg/util/net" + "k8s.io/kubernetes/pkg/master" "k8s.io/kubernetes/pkg/volume/csi" _ "github.com/go-sql-driver/mysql" // ensure we have mysql @@ -102,8 +103,16 @@ func run(app *cli.Context, cfg *cmds.Server) error { serverConfig.Rootless = cfg.Rootless serverConfig.TLSConfig.HTTPSPort = cfg.HTTPSPort serverConfig.TLSConfig.HTTPPort = cfg.HTTPPort - serverConfig.TLSConfig.KnownIPs = knownIPs(cfg.KnownIPs) + for _, san := range knownIPs(cfg.TLSSan) { + addr := net2.ParseIP(san) + if addr != nil { + serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, san) + } else { + serverConfig.TLSConfig.Domains = append(serverConfig.TLSConfig.Domains, san) + } + } serverConfig.TLSConfig.BindAddress = cfg.BindAddress + serverConfig.ControlConfig.HTTPSPort = cfg.HTTPSPort serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs serverConfig.ControlConfig.ExtraSchedulerAPIArgs = cfg.ExtraSchedulerArgs @@ -113,6 +122,15 @@ func run(app *cli.Context, cfg *cmds.Server) error { serverConfig.ControlConfig.StorageCAFile = cfg.StorageCAFile serverConfig.ControlConfig.StorageCertFile = cfg.StorageCertFile serverConfig.ControlConfig.StorageKeyFile = cfg.StorageKeyFile + serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP + serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort + + if serverConfig.ControlConfig.AdvertiseIP == "" && cmds.AgentConfig.NodeIP != "" { + serverConfig.ControlConfig.AdvertiseIP = cmds.AgentConfig.NodeIP + } + if serverConfig.ControlConfig.AdvertiseIP != "" { + serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, serverConfig.ControlConfig.AdvertiseIP) + } _, serverConfig.ControlConfig.ClusterIPRange, err = net2.ParseCIDR(cfg.ClusterCIDR) if err != nil { @@ -123,6 +141,12 @@ func run(app *cli.Context, cfg *cmds.Server) error { return errors.Wrapf(err, "Invalid CIDR %s: %v", cfg.ServiceCIDR, err) } + _, apiServerServiceIP, err := master.DefaultServiceIPRange(*serverConfig.ControlConfig.ServiceIPRange) + if err != nil { + return err + } + serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, apiServerServiceIP.String()) + // If cluster-dns CLI arg is not set, we set ClusterDNS address to be ServiceCIDR network + 10, // i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10 if cfg.ClusterDNS == "" { diff --git a/pkg/daemons/agent/agent.go b/pkg/daemons/agent/agent.go index 7702e0e56e..1bd83a7e0d 100644 --- a/pkg/daemons/agent/agent.go +++ b/pkg/daemons/agent/agent.go @@ -35,7 +35,7 @@ func kubeProxy(cfg *config.Agent) { argsMap := map[string]string{ "proxy-mode": "iptables", "healthz-bind-address": "127.0.0.1", - "kubeconfig": cfg.KubeConfig, + "kubeconfig": cfg.KubeConfigKubeProxy, "cluster-cidr": cfg.ClusterCIDR.String(), } args := config.GetArgsList(argsMap, cfg.ExtraKubeProxyArgs) @@ -58,7 +58,7 @@ func kubelet(cfg *config.Agent) { "read-only-port": "0", "allow-privileged": "true", "cluster-domain": cfg.ClusterDomain, - "kubeconfig": cfg.KubeConfig, + "kubeconfig": cfg.KubeConfigKubelet, "eviction-hard": "imagefs.available<5%,nodefs.available<5%", "eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%", "fail-swap-on": "false", @@ -95,13 +95,13 @@ func kubelet(cfg *config.Agent) { if cfg.ListenAddress != "" { argsMap["address"] = cfg.ListenAddress } - if cfg.CACertPath != "" { + if cfg.ClientCA != "" { argsMap["anonymous-auth"] = "false" - argsMap["client-ca-file"] = cfg.CACertPath + argsMap["client-ca-file"] = cfg.ClientCA } - if cfg.NodeCertFile != "" && cfg.NodeKeyFile != "" { - argsMap["tls-cert-file"] = cfg.NodeCertFile - argsMap["tls-private-key-file"] = cfg.NodeKeyFile + if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" { + argsMap["tls-cert-file"] = cfg.ServingKubeletCert + argsMap["tls-private-key-file"] = cfg.ServingKubeletKey } if cfg.NodeName != "" { argsMap["hostname-override"] = cfg.NodeName diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 92fc1641b4..489684562e 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -36,32 +36,41 @@ type Containerd struct { } type Agent struct { - NodeName string - NodeCertFile string - NodeKeyFile string - ClusterCIDR net.IPNet - ClusterDNS net.IP - ClusterDomain string - ResolvConf string - RootDir string - KubeConfig string - NodeIP string - RuntimeSocket string - ListenAddress string - CACertPath string - CNIBinDir string - CNIConfDir string - ExtraKubeletArgs []string - ExtraKubeProxyArgs []string - PauseImage string - CNIPlugin bool - NodeTaints []string - NodeLabels []string + NodeName string + ClientKubeletCert string + ClientKubeletKey string + ClientKubeProxyCert string + ClientKubeProxyKey string + ServingKubeletCert string + ServingKubeletKey string + ClusterCIDR net.IPNet + ClusterDNS net.IP + ClusterDomain string + ResolvConf string + RootDir string + KubeConfigNode string + KubeConfigKubelet string + KubeConfigKubeProxy string + NodeIP string + RuntimeSocket string + ListenAddress string + ClientCA string + CNIBinDir string + CNIConfDir string + ExtraKubeletArgs []string + ExtraKubeProxyArgs []string + PauseImage string + CNIPlugin bool + NodeTaints []string + NodeLabels []string } type Control struct { AdvertisePort int + AdvertiseIP string ListenPort int + HTTPSPort int + ProxyPort int ClusterSecret string ClusterIPRange *net.IPNet ServiceIPRange *net.IPNet @@ -87,28 +96,44 @@ type Control struct { } type ControlRuntime struct { - TLSCert string - TLSKey string - TLSCA string - TLSCAKey string - TokenCA string - TokenCAKey string - ServiceKey string - PasswdFile string - KubeConfigSystem string + ClientKubeAPICert string + ClientKubeAPIKey string + ClientCA string + ClientCAKey string + ServerCA string + ServerCAKey string + ServiceKey string + PasswdFile string - NodeCert string - NodeKey string - ClientToken string - NodeToken string - Handler http.Handler - Tunnel http.Handler - Authenticator authenticator.Request + KubeConfigAdmin string + KubeConfigController string + KubeConfigScheduler string + KubeConfigAPIServer string + + ServingKubeAPICert string + ServingKubeAPIKey string + ClientToken string + NodeToken string + Handler http.Handler + Tunnel http.Handler + Authenticator authenticator.Request RequestHeaderCA string RequestHeaderCAKey string ClientAuthProxyCert string ClientAuthProxyKey string + + ClientAdminCert string + ClientAdminKey string + ClientControllerCert string + ClientControllerKey string + ClientSchedulerCert string + ClientSchedulerKey string + ClientKubeProxyCert string + ClientKubeProxyKey string + + ServingKubeletKey string + ClientKubeletKey string } type ArgString []string diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index c9cedc834b..fa9299d71a 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -1,12 +1,10 @@ package control import ( - "bufio" "context" + "crypto" cryptorand "crypto/rand" - "crypto/rsa" "crypto/x509" - "encoding/base64" "encoding/csv" "encoding/hex" "fmt" @@ -41,14 +39,12 @@ import ( var ( localhostIP = net.ParseIP("127.0.0.1") - x509KeyServerOnly = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} - x509KeyClientUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth} - requestHeaderCN = "kubernetes-proxy" + requestHeaderCN = "system:auth-proxy" kubeconfigTemplate = template.Must(template.New("kubeconfig").Parse(`apiVersion: v1 clusters: - cluster: server: {{.URL}} - certificate-authority-data: {{.CACert}} + certificate-authority: {{.CACert}} name: local contexts: - context: @@ -62,8 +58,8 @@ preferences: {} users: - name: user user: - username: {{.User}} - password: {{.Password}} + client-certificate: {{.ClientCert}} + client-key: {{.ClientKey}} `)) ) @@ -99,14 +95,17 @@ func Server(ctx context.Context, cfg *config.Control) error { func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) { argsMap := map[string]string{ - "kubeconfig": runtime.KubeConfigSystem, + "kubeconfig": runtime.KubeConfigController, "service-account-private-key-file": runtime.ServiceKey, "allocate-node-cidrs": "true", "cluster-cidr": cfg.ClusterIPRange.String(), - "root-ca-file": runtime.TokenCA, + "root-ca-file": runtime.ServerCA, "port": "10252", - "bind-address": "127.0.0.1", + "bind-address": localhostIP.String(), "secure-port": "0", + "use-service-account-credentials": "true", + "cluster-signing-cert-file": runtime.ServerCA, + "cluster-signing-key-file": runtime.ServerCAKey, } if cfg.NoLeaderElect { argsMap["leader-elect"] = "false" @@ -125,7 +124,7 @@ func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) { func scheduler(cfg *config.Control, runtime *config.ControlRuntime) { argsMap := map[string]string{ - "kubeconfig": runtime.KubeConfigSystem, + "kubeconfig": runtime.KubeConfigScheduler, "port": "10251", "bind-address": "127.0.0.1", "secure-port": "0", @@ -163,18 +162,20 @@ func apiServer(ctx context.Context, cfg *config.Control, runtime *config.Control argsMap["service-account-signing-key-file"] = runtime.ServiceKey argsMap["service-cluster-ip-range"] = cfg.ServiceIPRange.String() argsMap["advertise-port"] = strconv.Itoa(cfg.AdvertisePort) - argsMap["advertise-address"] = localhostIP.String() + if cfg.AdvertiseIP != "" { + argsMap["advertise-address"] = cfg.AdvertiseIP + } argsMap["insecure-port"] = "0" argsMap["secure-port"] = strconv.Itoa(cfg.ListenPort) argsMap["bind-address"] = localhostIP.String() - argsMap["tls-cert-file"] = runtime.TLSCert - argsMap["tls-private-key-file"] = runtime.TLSKey + argsMap["tls-cert-file"] = runtime.ServingKubeAPICert + argsMap["tls-private-key-file"] = runtime.ServingKubeAPIKey argsMap["service-account-key-file"] = runtime.ServiceKey argsMap["service-account-issuer"] = "k3s" argsMap["api-audiences"] = "unknown" argsMap["basic-auth-file"] = runtime.PasswdFile - argsMap["kubelet-client-certificate"] = runtime.NodeCert - argsMap["kubelet-client-key"] = runtime.NodeKey + argsMap["kubelet-client-certificate"] = runtime.ClientKubeAPICert + argsMap["kubelet-client-key"] = runtime.ClientKubeAPIKey argsMap["requestheader-client-ca-file"] = runtime.RequestHeaderCA argsMap["requestheader-allowed-names"] = requestHeaderCN argsMap["proxy-client-cert-file"] = runtime.ClientAuthProxyCert @@ -182,6 +183,8 @@ func apiServer(ctx context.Context, cfg *config.Control, runtime *config.Control argsMap["requestheader-extra-headers-prefix"] = "X-Remote-Extra-" argsMap["requestheader-group-headers"] = "X-Remote-Group" argsMap["requestheader-username-headers"] = "X-Remote-User" + argsMap["client-ca-file"] = runtime.ClientCA + argsMap["enable-admission-plugins"] = "NodeRestriction" args := config.GetArgsList(argsMap, cfg.ExtraAPIArgs) @@ -214,13 +217,17 @@ func defaults(config *config.Control) { } if config.AdvertisePort == 0 { - config.AdvertisePort = 6445 + config.AdvertisePort = config.HTTPSPort } if config.ListenPort == 0 { config.ListenPort = 6444 } + if config.ProxyPort == 0 { + config.ProxyPort = 6445 + } + if config.DataDir == "" { config.DataDir = "./management-state" } @@ -247,22 +254,40 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error { os.MkdirAll(path.Join(config.DataDir, "tls"), 0700) os.MkdirAll(path.Join(config.DataDir, "cred"), 0700) - name := "localhost" - runtime.TLSCert = path.Join(config.DataDir, "tls", name+".crt") - runtime.TLSKey = path.Join(config.DataDir, "tls", name+".key") - runtime.TLSCA = path.Join(config.DataDir, "tls", "ca.crt") - runtime.TLSCAKey = path.Join(config.DataDir, "tls", "ca.key") - runtime.TokenCA = path.Join(config.DataDir, "tls", "token-ca.crt") - runtime.TokenCAKey = path.Join(config.DataDir, "tls", "token-ca.key") - runtime.ServiceKey = path.Join(config.DataDir, "tls", "service.key") - runtime.PasswdFile = path.Join(config.DataDir, "cred", "passwd") - runtime.KubeConfigSystem = path.Join(config.DataDir, "cred", "kubeconfig-system.yaml") - runtime.NodeKey = path.Join(config.DataDir, "tls", "token-node.key") - runtime.NodeCert = path.Join(config.DataDir, "tls", "token-node-1.crt") + runtime.ClientCA = path.Join(config.DataDir, "tls", "client-ca.crt") + runtime.ClientCAKey = path.Join(config.DataDir, "tls", "client-ca.key") + runtime.ServerCA = path.Join(config.DataDir, "tls", "server-ca.crt") + runtime.ServerCAKey = path.Join(config.DataDir, "tls", "server-ca.key") runtime.RequestHeaderCA = path.Join(config.DataDir, "tls", "request-header-ca.crt") runtime.RequestHeaderCAKey = path.Join(config.DataDir, "tls", "request-header-ca.key") - runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key") + + runtime.ServiceKey = path.Join(config.DataDir, "tls", "service.key") + runtime.PasswdFile = path.Join(config.DataDir, "cred", "passwd") + + runtime.KubeConfigAdmin = path.Join(config.DataDir, "cred", "admin.kubeconfig") + runtime.KubeConfigController = path.Join(config.DataDir, "cred", "controller.kubeconfig") + runtime.KubeConfigScheduler = path.Join(config.DataDir, "cred", "scheduler.kubeconfig") + runtime.KubeConfigAPIServer = path.Join(config.DataDir, "cred", "api-server.kubeconfig") + + runtime.ClientAdminCert = path.Join(config.DataDir, "tls", "client-admin.crt") + runtime.ClientAdminKey = path.Join(config.DataDir, "tls", "client-admin.key") + runtime.ClientControllerCert = path.Join(config.DataDir, "tls", "client-controller.crt") + runtime.ClientControllerKey = path.Join(config.DataDir, "tls", "client-controller.key") + runtime.ClientSchedulerCert = path.Join(config.DataDir, "tls", "client-scheduler.crt") + runtime.ClientSchedulerKey = path.Join(config.DataDir, "tls", "client-scheduler.key") + runtime.ClientKubeAPICert = path.Join(config.DataDir, "tls", "client-kube-apiserver.crt") + runtime.ClientKubeAPIKey = path.Join(config.DataDir, "tls", "client-kube-apiserver.key") + runtime.ClientKubeProxyCert = path.Join(config.DataDir, "tls", "client-kube-proxy.crt") + runtime.ClientKubeProxyKey = path.Join(config.DataDir, "tls", "client-kube-proxy.key") + + runtime.ServingKubeAPICert = path.Join(config.DataDir, "tls", "serving-kube-apiserver.crt") + runtime.ServingKubeAPIKey = path.Join(config.DataDir, "tls", "serving-kube-apiserver.key") + + runtime.ClientKubeletKey = path.Join(config.DataDir, "tls", "client-kubelet.key") + runtime.ServingKubeletKey = path.Join(config.DataDir, "tls", "serving-kubelet.key") + runtime.ClientAuthProxyCert = path.Join(config.DataDir, "tls", "client-auth-proxy.crt") + runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key") if err := genCerts(config, runtime); err != nil { return err @@ -279,32 +304,45 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error { return readTokens(runtime) } -func readTokens(runtime *config.ControlRuntime) error { - f, err := os.Open(runtime.PasswdFile) +func readTokenFile(passwdFile string) (map[string]string, error) { + f, err := os.Open(passwdFile) if err != nil { - return err + return nil, err } + defer f.Close() + reader := csv.NewReader(f) reader.FieldsPerRecord = -1 + tokens := map[string]string{} + for { record, err := reader.Read() if err == io.EOF { break } if err != nil { - return err + return nil, err } if len(record) < 2 { continue } + tokens[record[1]] = record[0] + } + return tokens, nil +} - switch record[1] { - case "node": - runtime.NodeToken = "node:" + record[0] - case "admin": - runtime.ClientToken = "admin:" + record[0] - } +func readTokens(runtime *config.ControlRuntime) error { + tokens, err := readTokenFile(runtime.PasswdFile) + if err != nil { + return err + } + + if nodeToken, ok := tokens["node"]; ok { + runtime.NodeToken = "node:" + nodeToken + } + if clientToken, ok := tokens["admin"]; ok { + runtime.ClientToken = "admin:" + clientToken } return nil @@ -321,31 +359,48 @@ func ensureNodeToken(config *config.Control, runtime *config.ControlRuntime) err } defer f.Close() - buf := &strings.Builder{} - scan := bufio.NewScanner(f) - for scan.Scan() { - line := scan.Text() - parts := strings.Split(line, ",") - if len(parts) < 4 { - continue + records := [][]string{} + reader := csv.NewReader(f) + for { + record, err := reader.Read() + if err == io.EOF { + break } - if parts[1] == "node" { - if parts[0] == config.ClusterSecret { + if err != nil { + return err + } + if len(record) < 3 { + return fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", runtime.PasswdFile, len(record)) + } + if record[1] == "node" { + if record[0] == config.ClusterSecret { return nil } - parts[0] = config.ClusterSecret - line = strings.Join(parts, ",") + record[0] = config.ClusterSecret } - buf.WriteString(line) - buf.WriteString("\n") - } - - if scan.Err() != nil { - return scan.Err() + records = append(records, record) } f.Close() - return ioutil.WriteFile(runtime.PasswdFile, []byte(buf.String()), 0600) + return WritePasswords(runtime.PasswdFile, records) +} + +func WritePasswords(passwdFile string, records [][]string) error { + out, err := os.Create(passwdFile + ".tmp") + if err != nil { + return err + } + defer out.Close() + + if err := out.Chmod(0600); err != nil { + return err + } + + if err := csv.NewWriter(out).WriteAll(records); err != nil { + return err + } + + return os.Rename(passwdFile+".tmp", passwdFile) } func genUsers(config *config.Control, runtime *config.ControlRuntime) error { @@ -370,24 +425,11 @@ func genUsers(config *config.Control, runtime *config.ControlRuntime) error { nodeToken = config.ClusterSecret } - passwd := fmt.Sprintf(`%s,admin,admin,system:masters -%s,system,system,system:masters -%s,node,node,system:masters -`, adminToken, systemToken, nodeToken) - - caCertBytes, err := ioutil.ReadFile(runtime.TLSCA) - if err != nil { - return err - } - - caCert := base64.StdEncoding.EncodeToString(caCertBytes) - - if err := kubeConfig(runtime.KubeConfigSystem, fmt.Sprintf("https://localhost:%d", config.ListenPort), caCert, - "system", systemToken); err != nil { - return err - } - - return ioutil.WriteFile(runtime.PasswdFile, []byte(passwd), 0600) + return WritePasswords(runtime.PasswdFile, [][]string{ + {adminToken, "admin", "admin", "system:masters"}, + {systemToken, "system", "system", "system:masters"}, + {nodeToken, "node", "node", "system:masters"}, + }) } func getToken() (string, error) { @@ -400,10 +442,10 @@ func getToken() (string, error) { } func genCerts(config *config.Control, runtime *config.ControlRuntime) error { - if err := genTLSCerts(config, runtime); err != nil { + if err := genClientCerts(config, runtime); err != nil { return err } - if err := genTokenCerts(config, runtime); err != nil { + if err := genServerCerts(config, runtime); err != nil { return err } if err := genRequestHeaderCerts(config, runtime); err != nil { @@ -412,32 +454,78 @@ func genCerts(config *config.Control, runtime *config.ControlRuntime) error { return nil } -func genTLSCerts(config *config.Control, runtime *config.ControlRuntime) error { - regen, err := createSigningCertKey("k3s-tls", runtime.TLSCA, runtime.TLSCAKey) +type signedCertFactory = func(commonName string, organization []string, certFile, keyFile string) (bool, error) + +func getSigningCertFactory(regen bool, altNames *certutil.AltNames, extKeyUsage []x509.ExtKeyUsage, caCertFile, caKeyFile string) signedCertFactory { + return func(commonName string, organization []string, certFile, keyFile string) (bool, error) { + return createClientCertKey(regen, commonName, organization, altNames, extKeyUsage, caCertFile, caKeyFile, certFile, keyFile) + } +} + +func genClientCerts(config *config.Control, runtime *config.ControlRuntime) error { + regen, err := createSigningCertKey("k3s-client", runtime.ClientCA, runtime.ClientCAKey) if err != nil { return err } - _, apiServerServiceIP, err := master.DefaultServiceIPRange(*config.ServiceIPRange) + factory := getSigningCertFactory(regen, nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, runtime.ClientCA, runtime.ClientCAKey) + + var certGen bool + apiEndpoint := fmt.Sprintf("https://localhost:%d", config.ListenPort) + + certGen, err = factory("system:admin", []string{"system:masters"}, runtime.ClientAdminCert, runtime.ClientAdminKey) if err != nil { return err } + if certGen { + if err := KubeConfig(runtime.KubeConfigAdmin, apiEndpoint, runtime.ServerCA, runtime.ClientAdminCert, runtime.ClientAdminKey); err != nil { + return err + } + } - if err := createClientCertKey(regen, "localhost", - nil, &certutil.AltNames{ - DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"}, - IPs: []net.IP{apiServerServiceIP, localhostIP}, - }, x509KeyServerOnly, - runtime.TLSCA, runtime.TLSCAKey, - runtime.TLSCert, runtime.TLSKey); err != nil { + certGen, err = factory("system:kube-controller-manager", nil, runtime.ClientControllerCert, runtime.ClientControllerKey) + if err != nil { + return err + } + if certGen { + if err := KubeConfig(runtime.KubeConfigController, apiEndpoint, runtime.ServerCA, runtime.ClientControllerCert, runtime.ClientControllerKey); err != nil { + return err + } + } + + certGen, err = factory("system:kube-scheduler", nil, runtime.ClientSchedulerCert, runtime.ClientSchedulerKey) + if err != nil { + return err + } + if certGen { + if err := KubeConfig(runtime.KubeConfigScheduler, apiEndpoint, runtime.ServerCA, runtime.ClientSchedulerCert, runtime.ClientSchedulerKey); err != nil { + return err + } + } + + certGen, err = factory("kube-apiserver", nil, runtime.ClientKubeAPICert, runtime.ClientKubeAPIKey) + if err != nil { + return err + } + if certGen { + if err := KubeConfig(runtime.KubeConfigAPIServer, apiEndpoint, runtime.ServerCA, runtime.ClientKubeAPICert, runtime.ClientKubeAPIKey); err != nil { + return err + } + } + + if _, err = factory("system:kube-proxy", []string{"system:nodes"}, runtime.ClientKubeProxyCert, runtime.ClientKubeProxyKey); err != nil { + return err + } + + if _, _, err := certutil.LoadOrGenerateKeyFile(runtime.ClientKubeletKey); err != nil { return err } return nil } -func genTokenCerts(config *config.Control, runtime *config.ControlRuntime) error { - regen, err := createSigningCertKey("k3s-token", runtime.TokenCA, runtime.TokenCAKey) +func genServerCerts(config *config.Control, runtime *config.ControlRuntime) error { + regen, err := createSigningCertKey("k3s-server", runtime.ServerCA, runtime.ServerCAKey) if err != nil { return err } @@ -447,13 +535,17 @@ func genTokenCerts(config *config.Control, runtime *config.ControlRuntime) error return err } - if err := createClientCertKey(regen, "kubernetes", []string{"system:masters"}, + if _, err := createClientCertKey(regen, "kube-apiserver", nil, &certutil.AltNames{ DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"}, IPs: []net.IP{apiServerServiceIP, localhostIP}, - }, x509KeyClientUsage, - runtime.TokenCA, runtime.TokenCAKey, - runtime.NodeCert, runtime.NodeKey); err != nil { + }, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + runtime.ServerCA, runtime.ServerCAKey, + runtime.ServingKubeAPICert, runtime.ServingKubeAPIKey); err != nil { + return err + } + + if _, _, err := certutil.LoadOrGenerateKeyFile(runtime.ServingKubeletKey); err != nil { return err } @@ -466,8 +558,8 @@ func genRequestHeaderCerts(config *config.Control, runtime *config.ControlRuntim return err } - if err := createClientCertKey(regen, requestHeaderCN, - nil, nil, x509KeyClientUsage, + if _, err := createClientCertKey(regen, requestHeaderCN, nil, + nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, runtime.RequestHeaderCA, runtime.RequestHeaderCAKey, runtime.ClientAuthProxyCert, runtime.ClientAuthProxyKey); err != nil { return err @@ -476,36 +568,41 @@ func genRequestHeaderCerts(config *config.Control, runtime *config.ControlRuntim return nil } -func createClientCertKey(regen bool, commonName string, organization []string, altNames *certutil.AltNames, extKeyUsage []x509.ExtKeyUsage, caCertFile, caKeyFile, certFile, keyFile string) error { +func createClientCertKey(regen bool, commonName string, organization []string, altNames *certutil.AltNames, extKeyUsage []x509.ExtKeyUsage, caCertFile, caKeyFile, certFile, keyFile string) (bool, error) { if !regen { if exists(certFile, keyFile) { - return nil + return false, nil } } caKeyBytes, err := ioutil.ReadFile(caKeyFile) if err != nil { - return err - } - - caBytes, err := ioutil.ReadFile(caCertFile) - if err != nil { - return err + return false, err } caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes) if err != nil { - return err + return false, err + } + + caBytes, err := ioutil.ReadFile(caCertFile) + if err != nil { + return false, err } caCert, err := certutil.ParseCertsPEM(caBytes) if err != nil { - return err + return false, err } - key, err := certutil.NewPrivateKey() + keyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile) if err != nil { - return err + return false, err + } + + key, err := certutil.ParsePrivateKeyPEM(keyBytes) + if err != nil { + return false, err } cfg := certutil.Config{ @@ -516,16 +613,12 @@ func createClientCertKey(regen bool, commonName string, organization []string, a if altNames != nil { cfg.AltNames = *altNames } - cert, err := certutil.NewSignedCert(cfg, key, caCert[0], caKey.(*rsa.PrivateKey)) + cert, err := certutil.NewSignedCert(cfg, key.(crypto.Signer), caCert[0], caKey.(crypto.Signer)) if err != nil { - return err + return false, err } - if err := certutil.WriteKey(keyFile, certutil.EncodePrivateKeyPEM(key)); err != nil { - return err - } - - return certutil.WriteCert(certFile, append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...)) + return true, certutil.WriteCert(certFile, append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...)) } func exists(files ...string) bool { @@ -556,7 +649,12 @@ func createSigningCertKey(prefix, certFile, keyFile string) (bool, error) { return false, nil } - caKey, err := certutil.NewPrivateKey() + caKeyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile) + if err != nil { + return false, err + } + + caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes) if err != nil { return false, err } @@ -565,32 +663,28 @@ func createSigningCertKey(prefix, certFile, keyFile string) (bool, error) { CommonName: fmt.Sprintf("%s-ca@%d", prefix, time.Now().Unix()), } - cert, err := certutil.NewSelfSignedCACert(cfg, caKey) + cert, err := certutil.NewSelfSignedCACert(cfg, caKey.(crypto.Signer)) if err != nil { return false, err } - if err := certutil.WriteKey(keyFile, certutil.EncodePrivateKeyPEM(caKey)); err != nil { - return false, err - } - if err := certutil.WriteCert(certFile, certutil.EncodeCertPEM(cert)); err != nil { return false, err } return true, nil } -func kubeConfig(dest, url, cert, user, password string) error { +func KubeConfig(dest, url, caCert, clientCert, clientKey string) error { data := struct { - URL string - CACert string - User string - Password string + URL string + CACert string + ClientCert string + ClientKey string }{ - URL: url, - CACert: cert, - User: user, - Password: password, + URL: url, + CACert: caCert, + ClientCert: clientCert, + ClientKey: clientKey, } output, err := os.Create(dest) diff --git a/pkg/server/router.go b/pkg/server/router.go index c5a630423b..06adb05b89 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -1,11 +1,12 @@ package server import ( - "bufio" - "crypto/rsa" + "crypto" "crypto/x509" + "encoding/csv" "errors" "fmt" + "io" "io/ioutil" "net" "net/http" @@ -17,10 +18,10 @@ import ( "github.com/gorilla/mux" certutil "github.com/rancher/dynamiclistener/cert" "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/k3s/pkg/daemons/control" "github.com/rancher/k3s/pkg/openapi" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/json" - "k8s.io/kubernetes/pkg/master" ) const ( @@ -38,8 +39,14 @@ func router(serverConfig *config.Control, tunnel http.Handler, cacertsGetter CAC authed.Use(authMiddleware(serverConfig)) authed.NotFoundHandler = serverConfig.Runtime.Handler authed.Path("/v1-k3s/connect").Handler(tunnel) - authed.Path("/v1-k3s/node.crt").Handler(nodeCrt(serverConfig)) - authed.Path("/v1-k3s/node.key").Handler(nodeKey(serverConfig)) + authed.Path("/v1-k3s/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig)) + authed.Path("/v1-k3s/serving-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ServingKubeletKey)) + authed.Path("/v1-k3s/client-kubelet.crt").Handler(clientKubeletCert(serverConfig)) + authed.Path("/v1-k3s/client-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeletKey)) + authed.Path("/v1-k3s/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert)) + authed.Path("/v1-k3s/client-kube-proxy.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyKey)) + authed.Path("/v1-k3s/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA)) + authed.Path("/v1-k3s/server-ca.crt").Handler(fileHandler(serverConfig.Runtime.ServerCA)) authed.Path("/v1-k3s/config").Handler(configHandler(serverConfig)) staticDir := filepath.Join(serverConfig.DataDir, "static") @@ -65,82 +72,85 @@ func cacerts(getter CACertsGetter) http.Handler { }) } -func nodeCrt(server *config.Control) http.Handler { +func getNodeInfo(req *http.Request) (string, string, error) { + nodeNames := req.Header["K3s-Node-Name"] + if len(nodeNames) != 1 || nodeNames[0] == "" { + return "", "", errors.New("node name not set") + } + + nodePasswords := req.Header["K3s-Node-Password"] + if len(nodePasswords) != 1 || nodePasswords[0] == "" { + return "", "", errors.New("node password not set") + } + + return nodeNames[0], nodePasswords[0], nil +} + +func getCACertAndKeys(caCertFile, caKeyFile, signingKeyFile string) ([]*x509.Certificate, crypto.Signer, crypto.Signer, error) { + keyBytes, err := ioutil.ReadFile(signingKeyFile) + if err != nil { + return nil, nil, nil, err + } + + key, err := certutil.ParsePrivateKeyPEM(keyBytes) + if err != nil { + return nil, nil, nil, err + } + + caKeyBytes, err := ioutil.ReadFile(caKeyFile) + if err != nil { + return nil, nil, nil, err + } + + caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes) + if err != nil { + return nil, nil, nil, err + } + + caBytes, err := ioutil.ReadFile(caCertFile) + if err != nil { + return nil, nil, nil, err + } + + caCert, err := certutil.ParseCertsPEM(caBytes) + if err != nil { + return nil, nil, nil, err + } + + return caCert, caKey.(crypto.Signer), key.(crypto.Signer), nil +} + +func servingKubeletCert(server *config.Control) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { if req.TLS == nil { resp.WriteHeader(http.StatusNotFound) return } - nodeNames := req.Header["K3s-Node-Name"] - if len(nodeNames) != 1 || nodeNames[0] == "" { - sendError(errors.New("node name not set"), resp) - return + nodeName, nodePassword, err := getNodeInfo(req) + if err != nil { + sendError(err, resp) } - nodePasswords := req.Header["K3s-Node-Password"] - if len(nodePasswords) != 1 || nodePasswords[0] == "" { - sendError(errors.New("node password not set"), resp) - return - } - - if err := ensureNodePassword(server.Runtime.PasswdFile, nodeNames[0], nodePasswords[0]); err != nil { + if err := ensureNodePassword(server.Runtime.PasswdFile, nodeName, nodePassword); err != nil { sendError(err, resp, http.StatusForbidden) return } - nodeKey, err := ioutil.ReadFile(server.Runtime.NodeKey) + caCert, caKey, key, err := getCACertAndKeys(server.Runtime.ServerCA, server.Runtime.ServerCAKey, server.Runtime.ServingKubeletKey) if err != nil { sendError(err, resp) return } - key, err := certutil.ParsePrivateKeyPEM(nodeKey) - if err != nil { - sendError(err, resp) - return - } - - caKeyBytes, err := ioutil.ReadFile(server.Runtime.TokenCAKey) - if err != nil { - sendError(err, resp) - return - } - - caBytes, err := ioutil.ReadFile(server.Runtime.TokenCA) - if err != nil { - sendError(err, resp) - return - } - - caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes) - if err != nil { - sendError(err, resp) - return - } - - caCert, err := certutil.ParseCertsPEM(caBytes) - if err != nil { - sendError(err, resp) - return - } - - _, apiServerServiceIP, err := master.DefaultServiceIPRange(*server.ServiceIPRange) - if err != nil { - sendError(err, resp) - return - } - - cfg := certutil.Config{ - CommonName: "kubernetes", - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + cert, err := certutil.NewSignedCert(certutil.Config{ + CommonName: nodeName, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, AltNames: certutil.AltNames{ - DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost", nodeNames[0]}, - IPs: []net.IP{apiServerServiceIP, net.ParseIP("127.0.0.1")}, + DNSNames: []string{nodeName, "localhost"}, + IPs: []net.IP{net.ParseIP("127.0.0.1")}, }, - } - - cert, err := certutil.NewSignedCert(cfg, key.(*rsa.PrivateKey), caCert[0], caKey.(*rsa.PrivateKey)) + }, key, caCert[0], caKey) if err != nil { sendError(err, resp) return @@ -150,13 +160,50 @@ func nodeCrt(server *config.Control) http.Handler { }) } -func nodeKey(server *config.Control) http.Handler { +func clientKubeletCert(server *config.Control) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { if req.TLS == nil { resp.WriteHeader(http.StatusNotFound) return } - http.ServeFile(resp, req, server.Runtime.NodeKey) + + nodeName, nodePassword, err := getNodeInfo(req) + if err != nil { + sendError(err, resp) + } + + if err := ensureNodePassword(server.Runtime.PasswdFile, nodeName, nodePassword); err != nil { + sendError(err, resp, http.StatusForbidden) + return + } + + caCert, caKey, key, err := getCACertAndKeys(server.Runtime.ClientCA, server.Runtime.ClientCAKey, server.Runtime.ClientKubeletKey) + if err != nil { + sendError(err, resp) + return + } + + cert, err := certutil.NewSignedCert(certutil.Config{ + CommonName: "system:node:" + nodeName, + Organization: []string{"system:nodes"}, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }, key, caCert[0], caKey) + if err != nil { + sendError(err, resp) + return + } + + resp.Write(append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...)) + }) +} + +func fileHandler(fileName string) http.Handler { + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + if req.TLS == nil { + resp.WriteHeader(http.StatusNotFound) + return + } + http.ServeFile(resp, req, fileName) }) } @@ -225,29 +272,29 @@ func ensureNodePassword(passwdFile, nodeName, passwd string) error { defer f.Close() user := strings.ToLower("node:" + nodeName) - buf := &strings.Builder{} - scan := bufio.NewScanner(f) - for scan.Scan() { - line := scan.Text() - parts := strings.Split(line, ",") - if len(parts) < 4 { - continue + records := [][]string{} + reader := csv.NewReader(f) + for { + record, err := reader.Read() + if err == io.EOF { + break } - if parts[1] == user { - if parts[0] == passwd { + if err != nil { + return err + } + if len(record) < 3 { + return fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", passwdFile, len(record)) + } + if record[1] == user { + if record[0] == passwd { return nil } - return fmt.Errorf("Node password validation failed for [%s]", nodeName) + return fmt.Errorf("Node password validation failed for '%s', using passwd file '%s'", nodeName, passwdFile) } - buf.WriteString(line) - buf.WriteString("\n") - } - buf.WriteString(fmt.Sprintf("%s,%s,%s,system:masters\n", passwd, user, user)) - - if scan.Err() != nil { - return scan.Err() + records = append(records, record) } + records = append(records, []string{passwd, user, user, "system:node:" + nodeName}) f.Close() - return ioutil.WriteFile(passwdFile, []byte(buf.String()), 0600) + return control.WritePasswords(passwdFile, records) } diff --git a/pkg/server/server.go b/pkg/server/server.go index 8ab93a9ff5..7c9be7d208 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -77,6 +77,18 @@ func startWrangler(ctx context.Context, config *Config) (string, error) { controlConfig = &config.ControlConfig ) + caBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCA) + if err != nil { + return "", err + } + caKeyBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCAKey) + if err != nil { + return "", err + } + + tlsConfig.CACerts = string(caBytes) + tlsConfig.CAKey = string(caKeyBytes) + tlsConfig.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, func() (string, error) { if tlsServer == nil { return "", nil @@ -84,7 +96,7 @@ func startWrangler(ctx context.Context, config *Config) (string, error) { return tlsServer.CACert() }) - sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigSystem) + sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigAdmin) if err != nil { return "", err }