github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Refactor certs

pull/359/head
Erik Wilson 2019-05-29 11:53:51 -07:00
parent c745be587e
commit 2c9444399b
12 changed files with 594 additions and 331 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
manifests/rolebindings.yaml Normal file
View File

@ -0,0 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kube-apiserver-kubelet-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:kubelet-api-admin
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: kube-apiserver

141
pkg/agent/config/config.go
View File

@ -23,6 +23,7 @@ import (
"github.com/rancher/k3s/pkg/cli/cmds"
"github.com/rancher/k3s/pkg/clientaccess"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/daemons/control"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/util/json"
"k8s.io/apimachinery/pkg/util/net"
@ -82,6 +83,10 @@ func getNodeNamedCrt(nodeName, nodePasswordFile string) HTTPRequester {
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusForbidden {
return nil, fmt.Errorf("Node password rejected, contents of '%s' may not match server passwd entry", nodePasswordFile)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%s: %s", u, resp.Status)
}
@ -101,45 +106,55 @@ func ensureNodePassword(nodePasswordFile string) (string, error) {
return "", err
}
nodePassword := hex.EncodeToString(password)
return nodePassword, ioutil.WriteFile(nodePasswordFile, []byte(nodePassword), 0600)
return nodePassword, ioutil.WriteFile(nodePasswordFile, []byte(nodePassword+"\n"), 0600)
}
func getNodeCert(nodeName, nodeCertFile, nodeKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) {
nodeCert, err := Request("/v1-k3s/node.crt", info, getNodeNamedCrt(nodeName, nodePasswordFile))
func getServingCert(nodeName, servingCertFile, servingKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) {
servingCert, err := Request("/v1-k3s/serving-kubelet.crt", info, getNodeNamedCrt(nodeName, nodePasswordFile))
if err != nil {
return nil, err
}
if err := ioutil.WriteFile(nodeCertFile, nodeCert, 0600); err != nil {
if err := ioutil.WriteFile(servingCertFile, servingCert, 0600); err != nil {
return nil, errors.Wrapf(err, "failed to write node cert")
}
nodeKey, err := clientaccess.Get("/v1-k3s/node.key", info)
servingKey, err := clientaccess.Get("/v1-k3s/serving-kubelet.key", info)
if err != nil {
return nil, err
}
if err := ioutil.WriteFile(nodeKeyFile, nodeKey, 0600); err != nil {
if err := ioutil.WriteFile(servingKeyFile, servingKey, 0600); err != nil {
return nil, errors.Wrapf(err, "failed to write node key")
}
cert, err := tls.X509KeyPair(nodeCert, nodeKey)
cert, err := tls.X509KeyPair(servingCert, servingKey)
if err != nil {
return nil, err
}
return &cert, nil
}
func writeNodeCA(dataDir string, nodeCert *tls.Certificate) (string, error) {
clientCABytes := pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: nodeCert.Certificate[1],
})
clientCA := filepath.Join(dataDir, "client-ca.pem")
if err := ioutil.WriteFile(clientCA, clientCABytes, 0600); err != nil {
return "", errors.Wrapf(err, "failed to write client CA")
func getHostFile(filename string, info *clientaccess.Info) error {
basename := filepath.Base(filename)
fileBytes, err := clientaccess.Get("/v1-k3s/"+basename, info)
if err != nil {
return err
}
if err := ioutil.WriteFile(filename, fileBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", filename)
}
return nil
}
return clientCA, nil
func getNodeNamedHostFile(filename, nodeName, nodePasswordFile string, info *clientaccess.Info) error {
basename := filepath.Base(filename)
fileBytes, err := Request("/v1-k3s/"+basename, info, getNodeNamedCrt(nodeName, nodePasswordFile))
if err != nil {
return err
}
if err := ioutil.WriteFile(filename, fileBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", filename)
}
return nil
}
func getHostnameAndIP(info cmds.Agent) (string, string, error) {
@ -169,17 +184,17 @@ func getHostnameAndIP(info cmds.Agent) (string, string, error) {
}
func localAddress(controlConfig *config.Control) string {
return fmt.Sprintf("127.0.0.1:%d", controlConfig.AdvertisePort)
return fmt.Sprintf("127.0.0.1:%d", controlConfig.ProxyPort)
}
func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, controlConfig *config.Control, nodeCert *tls.Certificate) (string, error) {
func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, controlConfig *config.Control, tlsCert *tls.Certificate) (string, error) {
os.MkdirAll(envInfo.DataDir, 0700)
kubeConfigPath := filepath.Join(envInfo.DataDir, "kubeconfig.yaml")
info.URL = "https://" + localAddress(controlConfig)
info.CACerts = pem.EncodeToMemory(&pem.Block{
Type: cert.CertificateBlockType,
Bytes: nodeCert.Certificate[1],
Bytes: tlsCert.Certificate[1],
})
return kubeConfigPath, info.WriteKubeConfig(kubeConfigPath)
@ -253,25 +268,6 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
return nil, err
}
nodeCertFile := filepath.Join(envInfo.DataDir, "token-node.crt")
nodeKeyFile := filepath.Join(envInfo.DataDir, "token-node.key")
nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt")
nodeCert, err := getNodeCert(nodeName, nodeCertFile, nodeKeyFile, nodePasswordFile, info)
if err != nil {
return nil, err
}
clientCA, err := writeNodeCA(envInfo.DataDir, nodeCert)
if err != nil {
return nil, err
}
kubeConfig, err := writeKubeConfig(envInfo, *info, controlConfig, nodeCert)
if err != nil {
return nil, err
}
hostLocal, err := exec.LookPath("host-local")
if err != nil {
return nil, errors.Wrapf(err, "failed to find host-local")
@ -285,6 +281,61 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
}
}
proxyURL := "https://" + localAddress(controlConfig)
clientCAFile := filepath.Join(envInfo.DataDir, "client-ca.crt")
if err := getHostFile(clientCAFile, info); err != nil {
return nil, err
}
serverCAFile := filepath.Join(envInfo.DataDir, "server-ca.crt")
if err := getHostFile(serverCAFile, info); err != nil {
return nil, err
}
servingKubeletCert := filepath.Join(envInfo.DataDir, "serving-kubelet.crt")
servingKubeletKey := filepath.Join(envInfo.DataDir, "serving-kubelet.key")
nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt")
servingCert, err := getServingCert(nodeName, servingKubeletCert, servingKubeletKey, nodePasswordFile, info)
if err != nil {
return nil, err
}
kubeconfigNode, err := writeKubeConfig(envInfo, *info, controlConfig, servingCert)
if err != nil {
return nil, err
}
clientKubeletCert := filepath.Join(envInfo.DataDir, "client-kubelet.crt")
if err := getNodeNamedHostFile(clientKubeletCert, nodeName, nodePasswordFile, info); err != nil {
return nil, err
}
clientKubeletKey := filepath.Join(envInfo.DataDir, "client-kubelet.key")
if err := getHostFile(clientKubeletKey, info); err != nil {
return nil, err
}
kubeconfigKubelet := filepath.Join(envInfo.DataDir, "kubelet.kubeconfig")
if err := control.KubeConfig(kubeconfigKubelet, proxyURL, serverCAFile, clientKubeletCert, clientKubeletKey); err != nil {
return nil, err
}
clientKubeProxyCert := filepath.Join(envInfo.DataDir, "client-kube-proxy.crt")
if err := getHostFile(clientKubeProxyCert, info); err != nil {
return nil, err
}
clientKubeProxyKey := filepath.Join(envInfo.DataDir, "client-kube-proxy.key")
if err := getHostFile(clientKubeProxyKey, info); err != nil {
return nil, err
}
kubeconfigKubeproxy := filepath.Join(envInfo.DataDir, "kubeproxy.kubeconfig")
if err := control.KubeConfig(kubeconfigKubeproxy, proxyURL, serverCAFile, clientKubeProxyCert, clientKubeProxyKey); err != nil {
return nil, err
}
nodeConfig := &config.Node{
Docker: envInfo.Docker,
NoFlannel: envInfo.NoFlannel,
@ -295,14 +346,16 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
nodeConfig.Images = filepath.Join(envInfo.DataDir, "images")
nodeConfig.AgentConfig.NodeIP = nodeIP
nodeConfig.AgentConfig.NodeName = nodeName
nodeConfig.AgentConfig.NodeCertFile = nodeCertFile
nodeConfig.AgentConfig.NodeKeyFile = nodeKeyFile
nodeConfig.AgentConfig.ServingKubeletCert = servingKubeletCert
nodeConfig.AgentConfig.ServingKubeletKey = servingKubeletKey
nodeConfig.AgentConfig.ClusterDNS = controlConfig.ClusterDNS
nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain
nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
nodeConfig.AgentConfig.CACertPath = clientCA
nodeConfig.AgentConfig.ClientCA = clientCAFile
nodeConfig.AgentConfig.ListenAddress = "0.0.0.0"
nodeConfig.AgentConfig.KubeConfig = kubeConfig
nodeConfig.AgentConfig.KubeConfigNode = kubeconfigNode
nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet")
nodeConfig.AgentConfig.PauseImage = envInfo.PauseImage
nodeConfig.CACerts = info.CACerts
@ -316,7 +369,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
nodeConfig.Containerd.Address = filepath.Join(nodeConfig.Containerd.State, "containerd.sock")
nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml.tmpl")
nodeConfig.ServerAddress = serverURLParsed.Host
nodeConfig.Certificate = nodeCert
nodeConfig.Certificate = servingCert
if !nodeConfig.NoFlannel {
nodeConfig.FlannelConf = filepath.Join(envInfo.DataDir, "etc/flannel/net-conf.json")
nodeConfig.AgentConfig.CNIBinDir = filepath.Dir(hostLocal)

4
pkg/agent/flannel/setup.go
View File

@ -55,7 +55,7 @@ func Prepare(ctx context.Context, config *config.Node) error {
func Run(ctx context.Context, config *config.Node) error {
nodeName := config.AgentConfig.NodeName
restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig)
restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode)
if err != nil {
return err
}
@ -79,7 +79,7 @@ func Run(ctx context.Context, config *config.Node) error {
}
go func() {
err := flannel(ctx, config.FlannelIface, config.FlannelConf, config.AgentConfig.KubeConfig)
err := flannel(ctx, config.FlannelIface, config.FlannelConf, config.AgentConfig.KubeConfigNode)
logrus.Fatalf("flannel exited: %v", err)
}()

27
pkg/agent/proxy/proxy.go
View File

@ -1,35 +1,18 @@
package proxy
import (
"crypto/tls"
"net/http"
"github.com/pkg/errors"
"github.com/google/tcpproxy"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/proxy"
"github.com/sirupsen/logrus"
)
func Run(config *config.Node) error {
proxy, err := proxy.NewSimpleProxy(config.ServerAddress, config.CACerts, true)
if err != nil {
return err
}
listener, err := tls.Listen("tcp", config.LocalAddress, &tls.Config{
Certificates: []tls.Certificate{
*config.Certificate,
},
})
if err != nil {
return errors.Wrap(err, "Failed to start tls listener")
}
logrus.Infof("Starting proxy %s -> %s", config.LocalAddress, config.ServerAddress)
var proxy tcpproxy.Proxy
proxy.AddRoute(config.LocalAddress, tcpproxy.To(config.ServerAddress))
go func() {
err := http.Serve(listener, proxy)
err := proxy.Run()
logrus.Fatalf("TLS proxy stopped: %v", err)
}()
return nil
}

2
pkg/agent/tunnel/tunnel.go
View File

@ -26,7 +26,7 @@ var (
)
func Setup(config *config.Node) error {
restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig)
restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode)
if err != nil {
return err
}

17
pkg/cli/cmds/server.go
View File

@ -17,7 +17,7 @@ type Server struct {
DisableAgent bool
KubeConfigOutput string
KubeConfigMode string
KnownIPs cli.StringSlice
TLSSan cli.StringSlice
BindAddress string
ExtraAPIArgs cli.StringSlice
ExtraSchedulerArgs cli.StringSlice
@ -28,6 +28,8 @@ type Server struct {
StorageCAFile string
StorageCertFile string
StorageKeyFile string
AdvertiseIP string
AdvertisePort int
}
var ServerConfig Server
@ -120,7 +122,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
cli.StringSliceFlag{
Name: "tls-san",
Usage: "Add additional hostname or IP as a Subject Alternative Name in the TLS cert",
Value: &ServerConfig.KnownIPs,
Value: &ServerConfig.TLSSan,
},
cli.StringSliceFlag{
Name: "kube-apiserver-arg",
@ -172,6 +174,17 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
Destination: &ServerConfig.StorageKeyFile,
EnvVar: "K3S_STORAGE_KEYFILE",
},
cli.StringFlag{
Name: "advertise-address",
Usage: "IP address that apiserver uses to advertise to members of the cluster",
Destination: &ServerConfig.AdvertiseIP,
},
cli.IntFlag{
Name: "advertise-port",
Usage: "Port that apiserver uses to advertise to members of the cluster",
Value: 0,
Destination: &ServerConfig.AdvertisePort,
},
NodeIPFlag,
NodeNameFlag,
DockerFlag,

26
pkg/cli/server/server.go
View File

@ -23,6 +23,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"k8s.io/apimachinery/pkg/util/net"
"k8s.io/kubernetes/pkg/master"
"k8s.io/kubernetes/pkg/volume/csi"
_ "github.com/go-sql-driver/mysql" // ensure we have mysql
@ -102,8 +103,16 @@ func run(app *cli.Context, cfg *cmds.Server) error {
serverConfig.Rootless = cfg.Rootless
serverConfig.TLSConfig.HTTPSPort = cfg.HTTPSPort
serverConfig.TLSConfig.HTTPPort = cfg.HTTPPort
serverConfig.TLSConfig.KnownIPs = knownIPs(cfg.KnownIPs)
for _, san := range knownIPs(cfg.TLSSan) {
addr := net2.ParseIP(san)
if addr != nil {
serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, san)
} else {
serverConfig.TLSConfig.Domains = append(serverConfig.TLSConfig.Domains, san)
}
}
serverConfig.TLSConfig.BindAddress = cfg.BindAddress
serverConfig.ControlConfig.HTTPSPort = cfg.HTTPSPort
serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs
serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs
serverConfig.ControlConfig.ExtraSchedulerAPIArgs = cfg.ExtraSchedulerArgs
@ -113,6 +122,15 @@ func run(app *cli.Context, cfg *cmds.Server) error {
serverConfig.ControlConfig.StorageCAFile = cfg.StorageCAFile
serverConfig.ControlConfig.StorageCertFile = cfg.StorageCertFile
serverConfig.ControlConfig.StorageKeyFile = cfg.StorageKeyFile
serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP
serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
if serverConfig.ControlConfig.AdvertiseIP == "" && cmds.AgentConfig.NodeIP != "" {
serverConfig.ControlConfig.AdvertiseIP = cmds.AgentConfig.NodeIP
}
if serverConfig.ControlConfig.AdvertiseIP != "" {
serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, serverConfig.ControlConfig.AdvertiseIP)
}
_, serverConfig.ControlConfig.ClusterIPRange, err = net2.ParseCIDR(cfg.ClusterCIDR)
if err != nil {
@ -123,6 +141,12 @@ func run(app *cli.Context, cfg *cmds.Server) error {
return errors.Wrapf(err, "Invalid CIDR %s: %v", cfg.ServiceCIDR, err)
}
_, apiServerServiceIP, err := master.DefaultServiceIPRange(*serverConfig.ControlConfig.ServiceIPRange)
if err != nil {
return err
}
serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, apiServerServiceIP.String())
// If cluster-dns CLI arg is not set, we set ClusterDNS address to be ServiceCIDR network + 10,
// i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10
if cfg.ClusterDNS == "" {

14
pkg/daemons/agent/agent.go
View File

@ -35,7 +35,7 @@ func kubeProxy(cfg *config.Agent) {
argsMap := map[string]string{
"proxy-mode": "iptables",
"healthz-bind-address": "127.0.0.1",
"kubeconfig": cfg.KubeConfig,
"kubeconfig": cfg.KubeConfigKubeProxy,
"cluster-cidr": cfg.ClusterCIDR.String(),
}
args := config.GetArgsList(argsMap, cfg.ExtraKubeProxyArgs)
@ -58,7 +58,7 @@ func kubelet(cfg *config.Agent) {
"read-only-port": "0",
"allow-privileged": "true",
"cluster-domain": cfg.ClusterDomain,
"kubeconfig": cfg.KubeConfig,
"kubeconfig": cfg.KubeConfigKubelet,
"eviction-hard": "imagefs.available<5%,nodefs.available<5%",
"eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%",
"fail-swap-on": "false",
@ -95,13 +95,13 @@ func kubelet(cfg *config.Agent) {
if cfg.ListenAddress != "" {
argsMap["address"] = cfg.ListenAddress
}
if cfg.CACertPath != "" {
if cfg.ClientCA != "" {
argsMap["anonymous-auth"] = "false"
argsMap["client-ca-file"] = cfg.CACertPath
argsMap["client-ca-file"] = cfg.ClientCA
}
if cfg.NodeCertFile != "" && cfg.NodeKeyFile != "" {
argsMap["tls-cert-file"] = cfg.NodeCertFile
argsMap["tls-private-key-file"] = cfg.NodeKeyFile
if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" {
argsMap["tls-cert-file"] = cfg.ServingKubeletCert
argsMap["tls-private-key-file"] = cfg.ServingKubeletKey
}
if cfg.NodeName != "" {
argsMap["hostname-override"] = cfg.NodeName

99
pkg/daemons/config/types.go
View File

@ -36,32 +36,41 @@ type Containerd struct {
}
type Agent struct {
NodeName string
NodeCertFile string
NodeKeyFile string
ClusterCIDR net.IPNet
ClusterDNS net.IP
ClusterDomain string
ResolvConf string
RootDir string
KubeConfig string
NodeIP string
RuntimeSocket string
ListenAddress string
CACertPath string
CNIBinDir string
CNIConfDir string
ExtraKubeletArgs []string
ExtraKubeProxyArgs []string
PauseImage string
CNIPlugin bool
NodeTaints []string
NodeLabels []string
NodeName string
ClientKubeletCert string
ClientKubeletKey string
ClientKubeProxyCert string
ClientKubeProxyKey string
ServingKubeletCert string
ServingKubeletKey string
ClusterCIDR net.IPNet
ClusterDNS net.IP
ClusterDomain string
ResolvConf string
RootDir string
KubeConfigNode string
KubeConfigKubelet string
KubeConfigKubeProxy string
NodeIP string
RuntimeSocket string
ListenAddress string
ClientCA string
CNIBinDir string
CNIConfDir string
ExtraKubeletArgs []string
ExtraKubeProxyArgs []string
PauseImage string
CNIPlugin bool
NodeTaints []string
NodeLabels []string
}
type Control struct {
AdvertisePort int
AdvertiseIP string
ListenPort int
HTTPSPort int
ProxyPort int
ClusterSecret string
ClusterIPRange *net.IPNet
ServiceIPRange *net.IPNet
@ -87,28 +96,44 @@ type Control struct {
}
type ControlRuntime struct {
TLSCert string
TLSKey string
TLSCA string
TLSCAKey string
TokenCA string
TokenCAKey string
ServiceKey string
PasswdFile string
KubeConfigSystem string
ClientKubeAPICert string
ClientKubeAPIKey string
ClientCA string
ClientCAKey string
ServerCA string
ServerCAKey string
ServiceKey string
PasswdFile string
NodeCert string
NodeKey string
ClientToken string
NodeToken string
Handler http.Handler
Tunnel http.Handler
Authenticator authenticator.Request
KubeConfigAdmin string
KubeConfigController string
KubeConfigScheduler string
KubeConfigAPIServer string
ServingKubeAPICert string
ServingKubeAPIKey string
ClientToken string
NodeToken string
Handler http.Handler
Tunnel http.Handler
Authenticator authenticator.Request
RequestHeaderCA string
RequestHeaderCAKey string
ClientAuthProxyCert string
ClientAuthProxyKey string
ClientAdminCert string
ClientAdminKey string
ClientControllerCert string
ClientControllerKey string
ClientSchedulerCert string
ClientSchedulerKey string
ClientKubeProxyCert string
ClientKubeProxyKey string
ServingKubeletKey string
ClientKubeletKey string
}
type ArgString []string

358
pkg/daemons/control/server.go
View File

@ -1,12 +1,10 @@
package control
import (
"bufio"
"context"
"crypto"
cryptorand "crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/base64"
"encoding/csv"
"encoding/hex"
"fmt"
@ -41,14 +39,12 @@ import (
var (
localhostIP = net.ParseIP("127.0.0.1")
x509KeyServerOnly = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}
x509KeyClientUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}
requestHeaderCN = "kubernetes-proxy"
requestHeaderCN = "system:auth-proxy"
kubeconfigTemplate = template.Must(template.New("kubeconfig").Parse(`apiVersion: v1
clusters:
- cluster:
server: {{.URL}}
certificate-authority-data: {{.CACert}}
certificate-authority: {{.CACert}}
name: local
contexts:
- context:
@ -62,8 +58,8 @@ preferences: {}
users:
- name: user
user:
username: {{.User}}
password: {{.Password}}
client-certificate: {{.ClientCert}}
client-key: {{.ClientKey}}
`))
)
@ -99,14 +95,17 @@ func Server(ctx context.Context, cfg *config.Control) error {
func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) {
argsMap := map[string]string{
"kubeconfig": runtime.KubeConfigSystem,
"kubeconfig": runtime.KubeConfigController,
"service-account-private-key-file": runtime.ServiceKey,
"allocate-node-cidrs": "true",
"cluster-cidr": cfg.ClusterIPRange.String(),
"root-ca-file": runtime.TokenCA,
"root-ca-file": runtime.ServerCA,
"port": "10252",
"bind-address": "127.0.0.1",
"bind-address": localhostIP.String(),
"secure-port": "0",
"use-service-account-credentials": "true",
"cluster-signing-cert-file": runtime.ServerCA,
"cluster-signing-key-file": runtime.ServerCAKey,
}
if cfg.NoLeaderElect {
argsMap["leader-elect"] = "false"
@ -125,7 +124,7 @@ func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) {
func scheduler(cfg *config.Control, runtime *config.ControlRuntime) {
argsMap := map[string]string{
"kubeconfig": runtime.KubeConfigSystem,
"kubeconfig": runtime.KubeConfigScheduler,
"port": "10251",
"bind-address": "127.0.0.1",
"secure-port": "0",
@ -163,18 +162,20 @@ func apiServer(ctx context.Context, cfg *config.Control, runtime *config.Control
argsMap["service-account-signing-key-file"] = runtime.ServiceKey
argsMap["service-cluster-ip-range"] = cfg.ServiceIPRange.String()
argsMap["advertise-port"] = strconv.Itoa(cfg.AdvertisePort)
argsMap["advertise-address"] = localhostIP.String()
if cfg.AdvertiseIP != "" {
argsMap["advertise-address"] = cfg.AdvertiseIP
}
argsMap["insecure-port"] = "0"
argsMap["secure-port"] = strconv.Itoa(cfg.ListenPort)
argsMap["bind-address"] = localhostIP.String()
argsMap["tls-cert-file"] = runtime.TLSCert
argsMap["tls-private-key-file"] = runtime.TLSKey
argsMap["tls-cert-file"] = runtime.ServingKubeAPICert
argsMap["tls-private-key-file"] = runtime.ServingKubeAPIKey
argsMap["service-account-key-file"] = runtime.ServiceKey
argsMap["service-account-issuer"] = "k3s"
argsMap["api-audiences"] = "unknown"
argsMap["basic-auth-file"] = runtime.PasswdFile
argsMap["kubelet-client-certificate"] = runtime.NodeCert
argsMap["kubelet-client-key"] = runtime.NodeKey
argsMap["kubelet-client-certificate"] = runtime.ClientKubeAPICert
argsMap["kubelet-client-key"] = runtime.ClientKubeAPIKey
argsMap["requestheader-client-ca-file"] = runtime.RequestHeaderCA
argsMap["requestheader-allowed-names"] = requestHeaderCN
argsMap["proxy-client-cert-file"] = runtime.ClientAuthProxyCert
@ -182,6 +183,8 @@ func apiServer(ctx context.Context, cfg *config.Control, runtime *config.Control
argsMap["requestheader-extra-headers-prefix"] = "X-Remote-Extra-"
argsMap["requestheader-group-headers"] = "X-Remote-Group"
argsMap["requestheader-username-headers"] = "X-Remote-User"
argsMap["client-ca-file"] = runtime.ClientCA
argsMap["enable-admission-plugins"] = "NodeRestriction"
args := config.GetArgsList(argsMap, cfg.ExtraAPIArgs)
@ -214,13 +217,17 @@ func defaults(config *config.Control) {
}
if config.AdvertisePort == 0 {
config.AdvertisePort = 6445
config.AdvertisePort = config.HTTPSPort
}
if config.ListenPort == 0 {
config.ListenPort = 6444
}
if config.ProxyPort == 0 {
config.ProxyPort = 6445
}
if config.DataDir == "" {
config.DataDir = "./management-state"
}
@ -247,22 +254,40 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error {
os.MkdirAll(path.Join(config.DataDir, "tls"), 0700)
os.MkdirAll(path.Join(config.DataDir, "cred"), 0700)
name := "localhost"
runtime.TLSCert = path.Join(config.DataDir, "tls", name+".crt")
runtime.TLSKey = path.Join(config.DataDir, "tls", name+".key")
runtime.TLSCA = path.Join(config.DataDir, "tls", "ca.crt")
runtime.TLSCAKey = path.Join(config.DataDir, "tls", "ca.key")
runtime.TokenCA = path.Join(config.DataDir, "tls", "token-ca.crt")
runtime.TokenCAKey = path.Join(config.DataDir, "tls", "token-ca.key")
runtime.ServiceKey = path.Join(config.DataDir, "tls", "service.key")
runtime.PasswdFile = path.Join(config.DataDir, "cred", "passwd")
runtime.KubeConfigSystem = path.Join(config.DataDir, "cred", "kubeconfig-system.yaml")
runtime.NodeKey = path.Join(config.DataDir, "tls", "token-node.key")
runtime.NodeCert = path.Join(config.DataDir, "tls", "token-node-1.crt")
runtime.ClientCA = path.Join(config.DataDir, "tls", "client-ca.crt")
runtime.ClientCAKey = path.Join(config.DataDir, "tls", "client-ca.key")
runtime.ServerCA = path.Join(config.DataDir, "tls", "server-ca.crt")
runtime.ServerCAKey = path.Join(config.DataDir, "tls", "server-ca.key")
runtime.RequestHeaderCA = path.Join(config.DataDir, "tls", "request-header-ca.crt")
runtime.RequestHeaderCAKey = path.Join(config.DataDir, "tls", "request-header-ca.key")
runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key")
runtime.ServiceKey = path.Join(config.DataDir, "tls", "service.key")
runtime.PasswdFile = path.Join(config.DataDir, "cred", "passwd")
runtime.KubeConfigAdmin = path.Join(config.DataDir, "cred", "admin.kubeconfig")
runtime.KubeConfigController = path.Join(config.DataDir, "cred", "controller.kubeconfig")
runtime.KubeConfigScheduler = path.Join(config.DataDir, "cred", "scheduler.kubeconfig")
runtime.KubeConfigAPIServer = path.Join(config.DataDir, "cred", "api-server.kubeconfig")
runtime.ClientAdminCert = path.Join(config.DataDir, "tls", "client-admin.crt")
runtime.ClientAdminKey = path.Join(config.DataDir, "tls", "client-admin.key")
runtime.ClientControllerCert = path.Join(config.DataDir, "tls", "client-controller.crt")
runtime.ClientControllerKey = path.Join(config.DataDir, "tls", "client-controller.key")
runtime.ClientSchedulerCert = path.Join(config.DataDir, "tls", "client-scheduler.crt")
runtime.ClientSchedulerKey = path.Join(config.DataDir, "tls", "client-scheduler.key")
runtime.ClientKubeAPICert = path.Join(config.DataDir, "tls", "client-kube-apiserver.crt")
runtime.ClientKubeAPIKey = path.Join(config.DataDir, "tls", "client-kube-apiserver.key")
runtime.ClientKubeProxyCert = path.Join(config.DataDir, "tls", "client-kube-proxy.crt")
runtime.ClientKubeProxyKey = path.Join(config.DataDir, "tls", "client-kube-proxy.key")
runtime.ServingKubeAPICert = path.Join(config.DataDir, "tls", "serving-kube-apiserver.crt")
runtime.ServingKubeAPIKey = path.Join(config.DataDir, "tls", "serving-kube-apiserver.key")
runtime.ClientKubeletKey = path.Join(config.DataDir, "tls", "client-kubelet.key")
runtime.ServingKubeletKey = path.Join(config.DataDir, "tls", "serving-kubelet.key")
runtime.ClientAuthProxyCert = path.Join(config.DataDir, "tls", "client-auth-proxy.crt")
runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key")
if err := genCerts(config, runtime); err != nil {
return err
@ -279,32 +304,45 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error {
return readTokens(runtime)
}
func readTokens(runtime *config.ControlRuntime) error {
f, err := os.Open(runtime.PasswdFile)
func readTokenFile(passwdFile string) (map[string]string, error) {
f, err := os.Open(passwdFile)
if err != nil {
return err
return nil, err
}
defer f.Close()
reader := csv.NewReader(f)
reader.FieldsPerRecord = -1
tokens := map[string]string{}
for {
record, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
return nil, err
}
if len(record) < 2 {
continue
}
tokens[record[1]] = record[0]
}
return tokens, nil
}
switch record[1] {
case "node":
runtime.NodeToken = "node:" + record[0]
case "admin":
runtime.ClientToken = "admin:" + record[0]
}
func readTokens(runtime *config.ControlRuntime) error {
tokens, err := readTokenFile(runtime.PasswdFile)
if err != nil {
return err
}
if nodeToken, ok := tokens["node"]; ok {
runtime.NodeToken = "node:" + nodeToken
}
if clientToken, ok := tokens["admin"]; ok {
runtime.ClientToken = "admin:" + clientToken
}
return nil
@ -321,31 +359,48 @@ func ensureNodeToken(config *config.Control, runtime *config.ControlRuntime) err
}
defer f.Close()
buf := &strings.Builder{}
scan := bufio.NewScanner(f)
for scan.Scan() {
line := scan.Text()
parts := strings.Split(line, ",")
if len(parts) < 4 {
continue
records := [][]string{}
reader := csv.NewReader(f)
for {
record, err := reader.Read()
if err == io.EOF {
break
}
if parts[1] == "node" {
if parts[0] == config.ClusterSecret {
if err != nil {
return err
}
if len(record) < 3 {
return fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", runtime.PasswdFile, len(record))
}
if record[1] == "node" {
if record[0] == config.ClusterSecret {
return nil
}
parts[0] = config.ClusterSecret
line = strings.Join(parts, ",")
record[0] = config.ClusterSecret
}
buf.WriteString(line)
buf.WriteString("\n")
}
if scan.Err() != nil {
return scan.Err()
records = append(records, record)
}
f.Close()
return ioutil.WriteFile(runtime.PasswdFile, []byte(buf.String()), 0600)
return WritePasswords(runtime.PasswdFile, records)
}
func WritePasswords(passwdFile string, records [][]string) error {
out, err := os.Create(passwdFile + ".tmp")
if err != nil {
return err
}
defer out.Close()
if err := out.Chmod(0600); err != nil {
return err
}
if err := csv.NewWriter(out).WriteAll(records); err != nil {
return err
}
return os.Rename(passwdFile+".tmp", passwdFile)
}
func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
@ -370,24 +425,11 @@ func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
nodeToken = config.ClusterSecret
}
passwd := fmt.Sprintf(`%s,admin,admin,system:masters
%s,system,system,system:masters
%s,node,node,system:masters
`, adminToken, systemToken, nodeToken)
caCertBytes, err := ioutil.ReadFile(runtime.TLSCA)
if err != nil {
return err
}
caCert := base64.StdEncoding.EncodeToString(caCertBytes)
if err := kubeConfig(runtime.KubeConfigSystem, fmt.Sprintf("https://localhost:%d", config.ListenPort), caCert,
"system", systemToken); err != nil {
return err
}
return ioutil.WriteFile(runtime.PasswdFile, []byte(passwd), 0600)
return WritePasswords(runtime.PasswdFile, [][]string{
{adminToken, "admin", "admin", "system:masters"},
{systemToken, "system", "system", "system:masters"},
{nodeToken, "node", "node", "system:masters"},
})
}
func getToken() (string, error) {
@ -400,10 +442,10 @@ func getToken() (string, error) {
}
func genCerts(config *config.Control, runtime *config.ControlRuntime) error {
if err := genTLSCerts(config, runtime); err != nil {
if err := genClientCerts(config, runtime); err != nil {
return err
}
if err := genTokenCerts(config, runtime); err != nil {
if err := genServerCerts(config, runtime); err != nil {
return err
}
if err := genRequestHeaderCerts(config, runtime); err != nil {
@ -412,32 +454,78 @@ func genCerts(config *config.Control, runtime *config.ControlRuntime) error {
return nil
}
func genTLSCerts(config *config.Control, runtime *config.ControlRuntime) error {
regen, err := createSigningCertKey("k3s-tls", runtime.TLSCA, runtime.TLSCAKey)
type signedCertFactory = func(commonName string, organization []string, certFile, keyFile string) (bool, error)
func getSigningCertFactory(regen bool, altNames *certutil.AltNames, extKeyUsage []x509.ExtKeyUsage, caCertFile, caKeyFile string) signedCertFactory {
return func(commonName string, organization []string, certFile, keyFile string) (bool, error) {
return createClientCertKey(regen, commonName, organization, altNames, extKeyUsage, caCertFile, caKeyFile, certFile, keyFile)
}
}
func genClientCerts(config *config.Control, runtime *config.ControlRuntime) error {
regen, err := createSigningCertKey("k3s-client", runtime.ClientCA, runtime.ClientCAKey)
if err != nil {
return err
}
_, apiServerServiceIP, err := master.DefaultServiceIPRange(*config.ServiceIPRange)
factory := getSigningCertFactory(regen, nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, runtime.ClientCA, runtime.ClientCAKey)
var certGen bool
apiEndpoint := fmt.Sprintf("https://localhost:%d", config.ListenPort)
certGen, err = factory("system:admin", []string{"system:masters"}, runtime.ClientAdminCert, runtime.ClientAdminKey)
if err != nil {
return err
}
if certGen {
if err := KubeConfig(runtime.KubeConfigAdmin, apiEndpoint, runtime.ServerCA, runtime.ClientAdminCert, runtime.ClientAdminKey); err != nil {
return err
}
}
if err := createClientCertKey(regen, "localhost",
nil, &certutil.AltNames{
DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"},
IPs: []net.IP{apiServerServiceIP, localhostIP},
}, x509KeyServerOnly,
runtime.TLSCA, runtime.TLSCAKey,
runtime.TLSCert, runtime.TLSKey); err != nil {
certGen, err = factory("system:kube-controller-manager", nil, runtime.ClientControllerCert, runtime.ClientControllerKey)
if err != nil {
return err
}
if certGen {
if err := KubeConfig(runtime.KubeConfigController, apiEndpoint, runtime.ServerCA, runtime.ClientControllerCert, runtime.ClientControllerKey); err != nil {
return err
}
}
certGen, err = factory("system:kube-scheduler", nil, runtime.ClientSchedulerCert, runtime.ClientSchedulerKey)
if err != nil {
return err
}
if certGen {
if err := KubeConfig(runtime.KubeConfigScheduler, apiEndpoint, runtime.ServerCA, runtime.ClientSchedulerCert, runtime.ClientSchedulerKey); err != nil {
return err
}
}
certGen, err = factory("kube-apiserver", nil, runtime.ClientKubeAPICert, runtime.ClientKubeAPIKey)
if err != nil {
return err
}
if certGen {
if err := KubeConfig(runtime.KubeConfigAPIServer, apiEndpoint, runtime.ServerCA, runtime.ClientKubeAPICert, runtime.ClientKubeAPIKey); err != nil {
return err
}
}
if _, err = factory("system:kube-proxy", []string{"system:nodes"}, runtime.ClientKubeProxyCert, runtime.ClientKubeProxyKey); err != nil {
return err
}
if _, _, err := certutil.LoadOrGenerateKeyFile(runtime.ClientKubeletKey); err != nil {
return err
}
return nil
}
func genTokenCerts(config *config.Control, runtime *config.ControlRuntime) error {
regen, err := createSigningCertKey("k3s-token", runtime.TokenCA, runtime.TokenCAKey)
func genServerCerts(config *config.Control, runtime *config.ControlRuntime) error {
regen, err := createSigningCertKey("k3s-server", runtime.ServerCA, runtime.ServerCAKey)
if err != nil {
return err
}
@ -447,13 +535,17 @@ func genTokenCerts(config *config.Control, runtime *config.ControlRuntime) error
return err
}
if err := createClientCertKey(regen, "kubernetes", []string{"system:masters"},
if _, err := createClientCertKey(regen, "kube-apiserver", nil,
&certutil.AltNames{
DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"},
IPs: []net.IP{apiServerServiceIP, localhostIP},
}, x509KeyClientUsage,
runtime.TokenCA, runtime.TokenCAKey,
runtime.NodeCert, runtime.NodeKey); err != nil {
}, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
runtime.ServerCA, runtime.ServerCAKey,
runtime.ServingKubeAPICert, runtime.ServingKubeAPIKey); err != nil {
return err
}
if _, _, err := certutil.LoadOrGenerateKeyFile(runtime.ServingKubeletKey); err != nil {
return err
}
@ -466,8 +558,8 @@ func genRequestHeaderCerts(config *config.Control, runtime *config.ControlRuntim
return err
}
if err := createClientCertKey(regen, requestHeaderCN,
nil, nil, x509KeyClientUsage,
if _, err := createClientCertKey(regen, requestHeaderCN, nil,
nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
runtime.RequestHeaderCA, runtime.RequestHeaderCAKey,
runtime.ClientAuthProxyCert, runtime.ClientAuthProxyKey); err != nil {
return err
@ -476,36 +568,41 @@ func genRequestHeaderCerts(config *config.Control, runtime *config.ControlRuntim
return nil
}
func createClientCertKey(regen bool, commonName string, organization []string, altNames *certutil.AltNames, extKeyUsage []x509.ExtKeyUsage, caCertFile, caKeyFile, certFile, keyFile string) error {
func createClientCertKey(regen bool, commonName string, organization []string, altNames *certutil.AltNames, extKeyUsage []x509.ExtKeyUsage, caCertFile, caKeyFile, certFile, keyFile string) (bool, error) {
if !regen {
if exists(certFile, keyFile) {
return nil
return false, nil
}
}
caKeyBytes, err := ioutil.ReadFile(caKeyFile)
if err != nil {
return err
}
caBytes, err := ioutil.ReadFile(caCertFile)
if err != nil {
return err
return false, err
}
caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
if err != nil {
return err
return false, err
}
caBytes, err := ioutil.ReadFile(caCertFile)
if err != nil {
return false, err
}
caCert, err := certutil.ParseCertsPEM(caBytes)
if err != nil {
return err
return false, err
}
key, err := certutil.NewPrivateKey()
keyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile)
if err != nil {
return err
return false, err
}
key, err := certutil.ParsePrivateKeyPEM(keyBytes)
if err != nil {
return false, err
}
cfg := certutil.Config{
@ -516,16 +613,12 @@ func createClientCertKey(regen bool, commonName string, organization []string, a
if altNames != nil {
cfg.AltNames = *altNames
}
cert, err := certutil.NewSignedCert(cfg, key, caCert[0], caKey.(*rsa.PrivateKey))
cert, err := certutil.NewSignedCert(cfg, key.(crypto.Signer), caCert[0], caKey.(crypto.Signer))
if err != nil {
return err
return false, err
}
if err := certutil.WriteKey(keyFile, certutil.EncodePrivateKeyPEM(key)); err != nil {
return err
}
return certutil.WriteCert(certFile, append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...))
return true, certutil.WriteCert(certFile, append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...))
}
func exists(files ...string) bool {
@ -556,7 +649,12 @@ func createSigningCertKey(prefix, certFile, keyFile string) (bool, error) {
return false, nil
}
caKey, err := certutil.NewPrivateKey()
caKeyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile)
if err != nil {
return false, err
}
caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
if err != nil {
return false, err
}
@ -565,32 +663,28 @@ func createSigningCertKey(prefix, certFile, keyFile string) (bool, error) {
CommonName: fmt.Sprintf("%s-ca@%d", prefix, time.Now().Unix()),
}
cert, err := certutil.NewSelfSignedCACert(cfg, caKey)
cert, err := certutil.NewSelfSignedCACert(cfg, caKey.(crypto.Signer))
if err != nil {
return false, err
}
if err := certutil.WriteKey(keyFile, certutil.EncodePrivateKeyPEM(caKey)); err != nil {
return false, err
}
if err := certutil.WriteCert(certFile, certutil.EncodeCertPEM(cert)); err != nil {
return false, err
}
return true, nil
}
func kubeConfig(dest, url, cert, user, password string) error {
func KubeConfig(dest, url, caCert, clientCert, clientKey string) error {
data := struct {
URL string
CACert string
User string
Password string
URL string
CACert string
ClientCert string
ClientKey string
}{
URL: url,
CACert: cert,
User: user,
Password: password,
URL: url,
CACert: caCert,
ClientCert: clientCert,
ClientKey: clientKey,
}
output, err := os.Create(dest)

211
pkg/server/router.go
View File

@ -1,11 +1,12 @@
package server
import (
"bufio"
"crypto/rsa"
"crypto"
"crypto/x509"
"encoding/csv"
"errors"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
@ -17,10 +18,10 @@ import (
"github.com/gorilla/mux"
certutil "github.com/rancher/dynamiclistener/cert"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/daemons/control"
"github.com/rancher/k3s/pkg/openapi"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/util/json"
"k8s.io/kubernetes/pkg/master"
)
const (
@ -38,8 +39,14 @@ func router(serverConfig *config.Control, tunnel http.Handler, cacertsGetter CAC
authed.Use(authMiddleware(serverConfig))
authed.NotFoundHandler = serverConfig.Runtime.Handler
authed.Path("/v1-k3s/connect").Handler(tunnel)
authed.Path("/v1-k3s/node.crt").Handler(nodeCrt(serverConfig))
authed.Path("/v1-k3s/node.key").Handler(nodeKey(serverConfig))
authed.Path("/v1-k3s/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig))
authed.Path("/v1-k3s/serving-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ServingKubeletKey))
authed.Path("/v1-k3s/client-kubelet.crt").Handler(clientKubeletCert(serverConfig))
authed.Path("/v1-k3s/client-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeletKey))
authed.Path("/v1-k3s/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert))
authed.Path("/v1-k3s/client-kube-proxy.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyKey))
authed.Path("/v1-k3s/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA))
authed.Path("/v1-k3s/server-ca.crt").Handler(fileHandler(serverConfig.Runtime.ServerCA))
authed.Path("/v1-k3s/config").Handler(configHandler(serverConfig))
staticDir := filepath.Join(serverConfig.DataDir, "static")
@ -65,82 +72,85 @@ func cacerts(getter CACertsGetter) http.Handler {
})
}
func nodeCrt(server *config.Control) http.Handler {
func getNodeInfo(req *http.Request) (string, string, error) {
nodeNames := req.Header["K3s-Node-Name"]
if len(nodeNames) != 1 || nodeNames[0] == "" {
return "", "", errors.New("node name not set")
}
nodePasswords := req.Header["K3s-Node-Password"]
if len(nodePasswords) != 1 || nodePasswords[0] == "" {
return "", "", errors.New("node password not set")
}
return nodeNames[0], nodePasswords[0], nil
}
func getCACertAndKeys(caCertFile, caKeyFile, signingKeyFile string) ([]*x509.Certificate, crypto.Signer, crypto.Signer, error) {
keyBytes, err := ioutil.ReadFile(signingKeyFile)
if err != nil {
return nil, nil, nil, err
}
key, err := certutil.ParsePrivateKeyPEM(keyBytes)
if err != nil {
return nil, nil, nil, err
}
caKeyBytes, err := ioutil.ReadFile(caKeyFile)
if err != nil {
return nil, nil, nil, err
}
caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
if err != nil {
return nil, nil, nil, err
}
caBytes, err := ioutil.ReadFile(caCertFile)
if err != nil {
return nil, nil, nil, err
}
caCert, err := certutil.ParseCertsPEM(caBytes)
if err != nil {
return nil, nil, nil, err
}
return caCert, caKey.(crypto.Signer), key.(crypto.Signer), nil
}
func servingKubeletCert(server *config.Control) http.Handler {
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
if req.TLS == nil {
resp.WriteHeader(http.StatusNotFound)
return
}
nodeNames := req.Header["K3s-Node-Name"]
if len(nodeNames) != 1 || nodeNames[0] == "" {
sendError(errors.New("node name not set"), resp)
return
nodeName, nodePassword, err := getNodeInfo(req)
if err != nil {
sendError(err, resp)
}
nodePasswords := req.Header["K3s-Node-Password"]
if len(nodePasswords) != 1 || nodePasswords[0] == "" {
sendError(errors.New("node password not set"), resp)
return
}
if err := ensureNodePassword(server.Runtime.PasswdFile, nodeNames[0], nodePasswords[0]); err != nil {
if err := ensureNodePassword(server.Runtime.PasswdFile, nodeName, nodePassword); err != nil {
sendError(err, resp, http.StatusForbidden)
return
}
nodeKey, err := ioutil.ReadFile(server.Runtime.NodeKey)
caCert, caKey, key, err := getCACertAndKeys(server.Runtime.ServerCA, server.Runtime.ServerCAKey, server.Runtime.ServingKubeletKey)
if err != nil {
sendError(err, resp)
return
}
key, err := certutil.ParsePrivateKeyPEM(nodeKey)
if err != nil {
sendError(err, resp)
return
}
caKeyBytes, err := ioutil.ReadFile(server.Runtime.TokenCAKey)
if err != nil {
sendError(err, resp)
return
}
caBytes, err := ioutil.ReadFile(server.Runtime.TokenCA)
if err != nil {
sendError(err, resp)
return
}
caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
if err != nil {
sendError(err, resp)
return
}
caCert, err := certutil.ParseCertsPEM(caBytes)
if err != nil {
sendError(err, resp)
return
}
_, apiServerServiceIP, err := master.DefaultServiceIPRange(*server.ServiceIPRange)
if err != nil {
sendError(err, resp)
return
}
cfg := certutil.Config{
CommonName: "kubernetes",
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
cert, err := certutil.NewSignedCert(certutil.Config{
CommonName: nodeName,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
AltNames: certutil.AltNames{
DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost", nodeNames[0]},
IPs: []net.IP{apiServerServiceIP, net.ParseIP("127.0.0.1")},
DNSNames: []string{nodeName, "localhost"},
IPs: []net.IP{net.ParseIP("127.0.0.1")},
},
}
cert, err := certutil.NewSignedCert(cfg, key.(*rsa.PrivateKey), caCert[0], caKey.(*rsa.PrivateKey))
}, key, caCert[0], caKey)
if err != nil {
sendError(err, resp)
return
@ -150,13 +160,50 @@ func nodeCrt(server *config.Control) http.Handler {
})
}
func nodeKey(server *config.Control) http.Handler {
func clientKubeletCert(server *config.Control) http.Handler {
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
if req.TLS == nil {
resp.WriteHeader(http.StatusNotFound)
return
}
http.ServeFile(resp, req, server.Runtime.NodeKey)
nodeName, nodePassword, err := getNodeInfo(req)
if err != nil {
sendError(err, resp)
}
if err := ensureNodePassword(server.Runtime.PasswdFile, nodeName, nodePassword); err != nil {
sendError(err, resp, http.StatusForbidden)
return
}
caCert, caKey, key, err := getCACertAndKeys(server.Runtime.ClientCA, server.Runtime.ClientCAKey, server.Runtime.ClientKubeletKey)
if err != nil {
sendError(err, resp)
return
}
cert, err := certutil.NewSignedCert(certutil.Config{
CommonName: "system:node:" + nodeName,
Organization: []string{"system:nodes"},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}, key, caCert[0], caKey)
if err != nil {
sendError(err, resp)
return
}
resp.Write(append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...))
})
}
func fileHandler(fileName string) http.Handler {
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
if req.TLS == nil {
resp.WriteHeader(http.StatusNotFound)
return
}
http.ServeFile(resp, req, fileName)
})
}
@ -225,29 +272,29 @@ func ensureNodePassword(passwdFile, nodeName, passwd string) error {
defer f.Close()
user := strings.ToLower("node:" + nodeName)
buf := &strings.Builder{}
scan := bufio.NewScanner(f)
for scan.Scan() {
line := scan.Text()
parts := strings.Split(line, ",")
if len(parts) < 4 {
continue
records := [][]string{}
reader := csv.NewReader(f)
for {
record, err := reader.Read()
if err == io.EOF {
break
}
if parts[1] == user {
if parts[0] == passwd {
if err != nil {
return err
}
if len(record) < 3 {
return fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", passwdFile, len(record))
}
if record[1] == user {
if record[0] == passwd {
return nil
}
return fmt.Errorf("Node password validation failed for [%s]", nodeName)
return fmt.Errorf("Node password validation failed for '%s', using passwd file '%s'", nodeName, passwdFile)
}
buf.WriteString(line)
buf.WriteString("\n")
}
buf.WriteString(fmt.Sprintf("%s,%s,%s,system:masters\n", passwd, user, user))
if scan.Err() != nil {
return scan.Err()
records = append(records, record)
}
records = append(records, []string{passwd, user, user, "system:node:" + nodeName})
f.Close()
return ioutil.WriteFile(passwdFile, []byte(buf.String()), 0600)
return control.WritePasswords(passwdFile, records)
}

14
pkg/server/server.go
View File

@ -77,6 +77,18 @@ func startWrangler(ctx context.Context, config *Config) (string, error) {
controlConfig = &config.ControlConfig
)
caBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCA)
if err != nil {
return "", err
}
caKeyBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCAKey)
if err != nil {
return "", err
}
tlsConfig.CACerts = string(caBytes)
tlsConfig.CAKey = string(caKeyBytes)
tlsConfig.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, func() (string, error) {
if tlsServer == nil {
return "", nil
@ -84,7 +96,7 @@ func startWrangler(ctx context.Context, config *Config) (string, error) {
return tlsServer.CACert()
})
sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigSystem)
sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigAdmin)
if err != nil {
return "", err
}