k3s/pkg/agent/config/config.go

413 lines
12 KiB
Go
Raw Normal View History

2019-01-01 08:23:01 +00:00
package config
import (
"bufio"
2019-01-09 16:54:15 +00:00
"context"
cryptorand "crypto/rand"
2019-01-01 08:23:01 +00:00
"crypto/tls"
"encoding/hex"
2019-01-01 08:23:01 +00:00
"encoding/pem"
"fmt"
"io/ioutil"
sysnet "net"
2019-04-19 18:20:34 +00:00
"net/http"
2019-01-01 08:23:01 +00:00
"net/url"
"os"
2019-01-09 16:54:15 +00:00
"os/exec"
2019-01-01 08:23:01 +00:00
"path/filepath"
"regexp"
"strings"
2019-01-01 08:23:01 +00:00
"time"
"github.com/pkg/errors"
2019-01-09 16:54:15 +00:00
"github.com/rancher/k3s/pkg/cli/cmds"
2019-05-09 22:05:51 +00:00
"github.com/rancher/k3s/pkg/clientaccess"
2019-01-09 16:54:15 +00:00
"github.com/rancher/k3s/pkg/daemons/config"
2019-05-29 18:53:51 +00:00
"github.com/rancher/k3s/pkg/daemons/control"
2019-01-01 08:23:01 +00:00
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/util/json"
2019-01-09 16:54:15 +00:00
"k8s.io/apimachinery/pkg/util/net"
2019-01-01 08:23:01 +00:00
"k8s.io/client-go/util/cert"
2019-01-22 21:14:58 +00:00
"k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
2019-01-01 08:23:01 +00:00
)
2019-01-09 16:54:15 +00:00
func Get(ctx context.Context, agent cmds.Agent) *config.Node {
2019-01-01 08:23:01 +00:00
for {
2019-01-09 16:54:15 +00:00
agentConfig, err := get(&agent)
2019-01-01 08:23:01 +00:00
if err != nil {
logrus.Error(err)
2019-01-09 16:54:15 +00:00
select {
case <-time.After(5 * time.Second):
continue
case <-ctx.Done():
logrus.Fatalf("Interrupted")
}
2019-01-01 08:23:01 +00:00
}
return agentConfig
}
}
2019-04-19 18:20:34 +00:00
type HTTPRequester func(u string, client *http.Client, username, password string) ([]byte, error)
func Request(path string, info *clientaccess.Info, requester HTTPRequester) ([]byte, error) {
u, err := url.Parse(info.URL)
2019-01-01 08:23:01 +00:00
if err != nil {
return nil, err
}
2019-04-19 18:20:34 +00:00
u.Path = path
username, password, _ := clientaccess.ParseUsernamePassword(info.Token)
return requester(u.String(), clientaccess.GetHTTPClient(info.CACerts), username, password)
}
func getNodeNamedCrt(nodeName, nodePasswordFile string) HTTPRequester {
2019-04-19 18:20:34 +00:00
return func(u string, client *http.Client, username, password string) ([]byte, error) {
req, err := http.NewRequest(http.MethodGet, u, nil)
if err != nil {
return nil, err
}
if username != "" {
req.SetBasicAuth(username, password)
}
req.Header.Set("K3s-Node-Name", nodeName)
nodePassword, err := ensureNodePassword(nodePasswordFile)
if err != nil {
return nil, err
}
2019-04-25 17:53:21 +00:00
req.Header.Set("K3s-Node-Password", nodePassword)
2019-04-19 18:20:34 +00:00
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
2019-05-29 18:53:51 +00:00
if resp.StatusCode == http.StatusForbidden {
return nil, fmt.Errorf("Node password rejected, contents of '%s' may not match server passwd entry", nodePasswordFile)
}
2019-04-19 18:20:34 +00:00
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%s: %s", u, resp.Status)
}
return ioutil.ReadAll(resp.Body)
}
}
2019-04-25 17:53:21 +00:00
func ensureNodePassword(nodePasswordFile string) (string, error) {
if _, err := os.Stat(nodePasswordFile); err == nil {
2019-04-25 17:53:21 +00:00
password, err := ioutil.ReadFile(nodePasswordFile)
return strings.TrimSpace(string(password)), err
}
password := make([]byte, 16, 16)
_, err := cryptorand.Read(password)
if err != nil {
2019-04-25 17:53:21 +00:00
return "", err
}
2019-04-25 17:53:21 +00:00
nodePassword := hex.EncodeToString(password)
2019-05-29 18:53:51 +00:00
return nodePassword, ioutil.WriteFile(nodePasswordFile, []byte(nodePassword+"\n"), 0600)
}
2019-05-29 18:53:51 +00:00
func getServingCert(nodeName, servingCertFile, servingKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) {
servingCert, err := Request("/v1-k3s/serving-kubelet.crt", info, getNodeNamedCrt(nodeName, nodePasswordFile))
2019-01-01 08:23:01 +00:00
if err != nil {
return nil, err
}
2019-05-29 18:53:51 +00:00
if err := ioutil.WriteFile(servingCertFile, servingCert, 0600); err != nil {
2019-04-19 18:20:34 +00:00
return nil, errors.Wrapf(err, "failed to write node cert")
}
2019-01-01 08:23:01 +00:00
2019-05-29 18:53:51 +00:00
servingKey, err := clientaccess.Get("/v1-k3s/serving-kubelet.key", info)
2019-01-01 08:23:01 +00:00
if err != nil {
return nil, err
}
2019-05-29 18:53:51 +00:00
if err := ioutil.WriteFile(servingKeyFile, servingKey, 0600); err != nil {
2019-04-19 18:20:34 +00:00
return nil, errors.Wrapf(err, "failed to write node key")
}
2019-01-01 08:23:01 +00:00
2019-05-29 18:53:51 +00:00
cert, err := tls.X509KeyPair(servingCert, servingKey)
2019-01-01 08:23:01 +00:00
if err != nil {
return nil, err
}
return &cert, nil
}
2019-05-29 18:53:51 +00:00
func getHostFile(filename string, info *clientaccess.Info) error {
basename := filepath.Base(filename)
fileBytes, err := clientaccess.Get("/v1-k3s/"+basename, info)
if err != nil {
return err
}
if err := ioutil.WriteFile(filename, fileBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", filename)
2019-01-01 08:23:01 +00:00
}
2019-05-29 18:53:51 +00:00
return nil
}
2019-01-01 08:23:01 +00:00
2019-05-29 18:53:51 +00:00
func getNodeNamedHostFile(filename, nodeName, nodePasswordFile string, info *clientaccess.Info) error {
basename := filepath.Base(filename)
fileBytes, err := Request("/v1-k3s/"+basename, info, getNodeNamedCrt(nodeName, nodePasswordFile))
if err != nil {
return err
}
if err := ioutil.WriteFile(filename, fileBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", filename)
}
return nil
2019-01-01 08:23:01 +00:00
}
2019-01-09 16:54:15 +00:00
func getHostnameAndIP(info cmds.Agent) (string, string, error) {
2019-01-01 08:23:01 +00:00
ip := info.NodeIP
if ip == "" {
2019-01-09 16:54:15 +00:00
hostIP, err := net.ChooseHostInterface()
2019-01-01 08:23:01 +00:00
if err != nil {
return "", "", err
}
ip = hostIP.String()
}
name := info.NodeName
if name == "" {
hostname, err := os.Hostname()
if err != nil {
return "", "", err
}
2019-01-09 16:54:15 +00:00
name = hostname
2019-01-01 08:23:01 +00:00
}
// Use lower case hostname to comply with kubernetes constraint:
// https://github.com/kubernetes/kubernetes/issues/71140
name = strings.ToLower(name)
2019-01-01 08:23:01 +00:00
return name, ip, nil
}
2019-06-24 17:20:43 +00:00
func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, tlsCert *tls.Certificate) (string, error) {
2019-01-01 08:23:01 +00:00
os.MkdirAll(envInfo.DataDir, 0700)
kubeConfigPath := filepath.Join(envInfo.DataDir, "kubeconfig.yaml")
info.CACerts = pem.EncodeToMemory(&pem.Block{
Type: cert.CertificateBlockType,
2019-05-29 18:53:51 +00:00
Bytes: tlsCert.Certificate[1],
2019-01-01 08:23:01 +00:00
})
return kubeConfigPath, info.WriteKubeConfig(kubeConfigPath)
}
func isValidResolvConf(resolvConfFile string) bool {
file, err := os.Open(resolvConfFile)
if err != nil {
return false
}
defer file.Close()
nameserver := regexp.MustCompile(`^nameserver\s+([^\s]*)`)
scanner := bufio.NewScanner(file)
for scanner.Scan() {
ipMatch := nameserver.FindStringSubmatch(scanner.Text())
if len(ipMatch) == 2 {
ip := sysnet.ParseIP(ipMatch[1])
if ip == nil || !ip.IsGlobalUnicast() {
return false
}
}
}
if err := scanner.Err(); err != nil {
return false
}
return true
}
func locateOrGenerateResolvConf(envInfo *cmds.Agent) string {
if envInfo.ResolvConf != "" {
return envInfo.ResolvConf
}
resolvConfs := []string{"/etc/resolv.conf", "/run/systemd/resolve/resolv.conf"}
for _, conf := range resolvConfs {
if isValidResolvConf(conf) {
return conf
}
}
tmpConf := filepath.Join(os.TempDir(), "k3s-resolv.conf")
if err := ioutil.WriteFile(tmpConf, []byte("nameserver 8.8.8.8\n"), 0444); err != nil {
logrus.Error(err)
return ""
}
return tmpConf
}
2019-01-09 16:54:15 +00:00
func get(envInfo *cmds.Agent) (*config.Node, error) {
if envInfo.Debug {
logrus.SetLevel(logrus.DebugLevel)
2019-01-01 08:23:01 +00:00
}
serverURLParsed, err := url.Parse(envInfo.ServerURL)
if err != nil {
return nil, err
}
info, err := clientaccess.ParseAndValidateToken(envInfo.ServerURL, envInfo.Token)
if err != nil {
return nil, err
}
controlConfig, err := getConfig(info)
if err != nil {
return nil, err
}
2019-04-19 18:20:34 +00:00
nodeName, nodeIP, err := getHostnameAndIP(*envInfo)
2019-01-01 08:23:01 +00:00
if err != nil {
return nil, err
}
2019-05-29 18:53:51 +00:00
hostLocal, err := exec.LookPath("host-local")
2019-01-01 08:23:01 +00:00
if err != nil {
2019-05-29 18:53:51 +00:00
return nil, errors.Wrapf(err, "failed to find host-local")
}
var flannelIface *sysnet.Interface
if !envInfo.NoFlannel && len(envInfo.FlannelIface) > 0 {
flannelIface, err = sysnet.InterfaceByName(envInfo.FlannelIface)
if err != nil {
return nil, errors.Wrapf(err, "unable to find interface")
}
}
clientCAFile := filepath.Join(envInfo.DataDir, "client-ca.crt")
if err := getHostFile(clientCAFile, info); err != nil {
2019-01-01 08:23:01 +00:00
return nil, err
}
2019-05-29 18:53:51 +00:00
serverCAFile := filepath.Join(envInfo.DataDir, "server-ca.crt")
if err := getHostFile(serverCAFile, info); err != nil {
2019-01-01 08:23:01 +00:00
return nil, err
}
2019-05-29 18:53:51 +00:00
servingKubeletCert := filepath.Join(envInfo.DataDir, "serving-kubelet.crt")
servingKubeletKey := filepath.Join(envInfo.DataDir, "serving-kubelet.key")
nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt")
servingCert, err := getServingCert(nodeName, servingKubeletCert, servingKubeletKey, nodePasswordFile, info)
2019-01-01 08:23:01 +00:00
if err != nil {
return nil, err
}
2019-06-24 17:20:43 +00:00
kubeconfigNode, err := writeKubeConfig(envInfo, *info, servingCert)
2019-01-09 16:54:15 +00:00
if err != nil {
2019-05-29 18:53:51 +00:00
return nil, err
2019-01-09 16:54:15 +00:00
}
2019-05-29 18:53:51 +00:00
clientKubeletCert := filepath.Join(envInfo.DataDir, "client-kubelet.crt")
if err := getNodeNamedHostFile(clientKubeletCert, nodeName, nodePasswordFile, info); err != nil {
return nil, err
}
clientKubeletKey := filepath.Join(envInfo.DataDir, "client-kubelet.key")
if err := getHostFile(clientKubeletKey, info); err != nil {
return nil, err
}
kubeconfigKubelet := filepath.Join(envInfo.DataDir, "kubelet.kubeconfig")
2019-06-24 17:20:43 +00:00
if err := control.KubeConfig(kubeconfigKubelet, info.URL, serverCAFile, clientKubeletCert, clientKubeletKey); err != nil {
2019-05-29 18:53:51 +00:00
return nil, err
}
clientKubeProxyCert := filepath.Join(envInfo.DataDir, "client-kube-proxy.crt")
if err := getHostFile(clientKubeProxyCert, info); err != nil {
return nil, err
}
clientKubeProxyKey := filepath.Join(envInfo.DataDir, "client-kube-proxy.key")
if err := getHostFile(clientKubeProxyKey, info); err != nil {
return nil, err
}
kubeconfigKubeproxy := filepath.Join(envInfo.DataDir, "kubeproxy.kubeconfig")
2019-06-24 17:20:43 +00:00
if err := control.KubeConfig(kubeconfigKubeproxy, info.URL, serverCAFile, clientKubeProxyCert, clientKubeProxyKey); err != nil {
2019-05-29 18:53:51 +00:00
return nil, err
}
2019-01-09 16:54:15 +00:00
nodeConfig := &config.Node{
2019-03-04 06:29:06 +00:00
Docker: envInfo.Docker,
ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint,
2019-09-03 23:41:54 +00:00
FlannelBackend: controlConfig.FlannelBackend,
2019-01-09 16:54:15 +00:00
}
nodeConfig.FlannelIface = flannelIface
nodeConfig.Images = filepath.Join(envInfo.DataDir, "images")
2019-01-09 16:54:15 +00:00
nodeConfig.AgentConfig.NodeIP = nodeIP
nodeConfig.AgentConfig.NodeName = nodeName
2019-05-29 18:53:51 +00:00
nodeConfig.AgentConfig.ServingKubeletCert = servingKubeletCert
nodeConfig.AgentConfig.ServingKubeletKey = servingKubeletKey
2019-01-09 16:54:15 +00:00
nodeConfig.AgentConfig.ClusterDNS = controlConfig.ClusterDNS
2019-04-12 06:06:35 +00:00
nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain
nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
2019-05-29 18:53:51 +00:00
nodeConfig.AgentConfig.ClientCA = clientCAFile
nodeConfig.AgentConfig.ListenAddress = "0.0.0.0"
2019-05-29 18:53:51 +00:00
nodeConfig.AgentConfig.KubeConfigNode = kubeconfigNode
nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
2019-01-09 16:54:15 +00:00
nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet")
2019-05-03 17:36:12 +00:00
nodeConfig.AgentConfig.PauseImage = envInfo.PauseImage
2019-09-03 23:41:54 +00:00
nodeConfig.AgentConfig.IPSECPSK = controlConfig.IPSECPSK
2019-09-06 00:39:18 +00:00
nodeConfig.AgentConfig.StrongSwanDir = filepath.Join(envInfo.DataDir, "strongswan")
2019-01-01 08:23:01 +00:00
nodeConfig.CACerts = info.CACerts
2019-01-09 16:54:15 +00:00
nodeConfig.Containerd.Config = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml")
nodeConfig.Containerd.Root = filepath.Join(envInfo.DataDir, "containerd")
nodeConfig.Containerd.Opt = filepath.Join(envInfo.DataDir, "containerd")
if !envInfo.Debug {
nodeConfig.Containerd.Log = filepath.Join(envInfo.DataDir, "containerd/containerd.log")
}
2019-01-09 16:54:15 +00:00
nodeConfig.Containerd.State = "/run/k3s/containerd"
nodeConfig.Containerd.Address = filepath.Join(nodeConfig.Containerd.State, "containerd.sock")
2019-04-19 21:08:05 +00:00
nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml.tmpl")
2019-01-01 08:23:01 +00:00
nodeConfig.ServerAddress = serverURLParsed.Host
2019-05-29 18:53:51 +00:00
nodeConfig.Certificate = servingCert
2019-09-03 23:41:54 +00:00
if nodeConfig.FlannelBackend == config.FlannelBackendNone {
nodeConfig.NoFlannel = true
} else {
nodeConfig.NoFlannel = envInfo.NoFlannel
}
2019-01-09 16:54:15 +00:00
if !nodeConfig.NoFlannel {
2019-08-08 05:56:09 +00:00
if envInfo.FlannelConf == "" {
nodeConfig.FlannelConf = filepath.Join(envInfo.DataDir, "etc/flannel/net-conf.json")
} else {
nodeConfig.FlannelConf = envInfo.FlannelConf
nodeConfig.FlannelConfOverride = true
}
2019-01-09 16:54:15 +00:00
nodeConfig.AgentConfig.CNIBinDir = filepath.Dir(hostLocal)
nodeConfig.AgentConfig.CNIConfDir = filepath.Join(envInfo.DataDir, "etc/cni/net.d")
}
2019-03-04 06:29:06 +00:00
if !nodeConfig.Docker && nodeConfig.ContainerRuntimeEndpoint == "" {
2019-01-09 16:54:15 +00:00
nodeConfig.AgentConfig.RuntimeSocket = "unix://" + nodeConfig.Containerd.Address
2019-03-04 06:29:06 +00:00
} else {
nodeConfig.AgentConfig.RuntimeSocket = "unix://" + nodeConfig.ContainerRuntimeEndpoint
2019-01-09 16:54:15 +00:00
}
if controlConfig.ClusterIPRange != nil {
nodeConfig.AgentConfig.ClusterCIDR = *controlConfig.ClusterIPRange
2019-01-01 08:23:01 +00:00
}
2019-01-09 16:54:15 +00:00
os.Setenv("NODE_NAME", nodeConfig.AgentConfig.NodeName)
v1beta1.KubeletSocket = filepath.Join(envInfo.DataDir, "kubelet/device-plugins/kubelet.sock")
nodeConfig.AgentConfig.ExtraKubeletArgs = envInfo.ExtraKubeletArgs
nodeConfig.AgentConfig.ExtraKubeProxyArgs = envInfo.ExtraKubeProxyArgs
nodeConfig.AgentConfig.NodeTaints = envInfo.Taints
nodeConfig.AgentConfig.NodeLabels = envInfo.Labels
2019-01-01 08:23:01 +00:00
return nodeConfig, nil
}
func getConfig(info *clientaccess.Info) (*config.Control, error) {
data, err := clientaccess.Get("/v1-k3s/config", info)
if err != nil {
return nil, err
}
controlControl := &config.Control{}
return controlControl, json.Unmarshal(data, controlControl)
}