github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Refactor tokens, bootstrap, and cli args

pull/995/head
Darren Shepherd 2019-10-26 22:53:25 -07:00
parent c671b07924
commit ba240d0611
33 changed files with 1292 additions and 856 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
manifests/rolebindings.yaml
View File

@ -10,3 +10,53 @@ subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: kube-apiserver
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:k3s-controller
rules:
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- apiGroups:
- ""
resources:
- namespaces
verbs:
- list
- watch
- apiGroups:
- "networking.k8s.io"
resources:
- networkpolicies
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- endpoints
- pods
verbs:
- list
- get
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:k3s-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:k3s-controller
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: system:k3s-controller

35
pkg/agent/config/config.go
View File

@ -6,7 +6,6 @@ import (
cryptorand "crypto/rand"
"crypto/tls"
"encoding/hex"
"encoding/pem"
"fmt"
"io/ioutil"
sysnet "net"
@ -27,7 +26,6 @@ import (
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/util/json"
"k8s.io/apimachinery/pkg/util/net"
"k8s.io/client-go/util/cert"
"k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
)
@ -183,17 +181,6 @@ func getHostnameAndIP(info cmds.Agent) (string, string, error) {
return name, ip, nil
}
func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, tlsCert *tls.Certificate) (string, error) {
os.MkdirAll(envInfo.DataDir, 0700)
kubeConfigPath := filepath.Join(envInfo.DataDir, "kubeconfig.yaml")
info.CACerts = pem.EncodeToMemory(&pem.Block{
Type: cert.CertificateBlockType,
Bytes: tlsCert.Certificate[1],
})
return kubeConfigPath, info.WriteKubeConfig(kubeConfigPath)
}
func isValidResolvConf(resolvConfFile string) bool {
file, err := os.Open(resolvConfFile)
if err != nil {
@ -293,11 +280,6 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
return nil, err
}
kubeconfigNode, err := writeKubeConfig(envInfo, *info, servingCert)
if err != nil {
return nil, err
}
clientKubeletCert := filepath.Join(envInfo.DataDir, "client-kubelet.crt")
if err := getNodeNamedHostFile(clientKubeletCert, nodeName, nodePasswordFile, info); err != nil {
return nil, err
@ -328,6 +310,21 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
return nil, err
}
clientK3sControllerCert := filepath.Join(envInfo.DataDir, "client-k3s-controller.crt")
if err := getHostFile(clientK3sControllerCert, info); err != nil {
return nil, err
}
clientK3sControllerKey := filepath.Join(envInfo.DataDir, "client-k3s-controller.key")
if err := getHostFile(clientK3sControllerKey, info); err != nil {
return nil, err
}
kubeconfigK3sController := filepath.Join(envInfo.DataDir, "k3scontroller.kubeconfig")
if err := control.KubeConfig(kubeconfigK3sController, info.URL, serverCAFile, clientK3sControllerCert, clientK3sControllerKey); err != nil {
return nil, err
}
nodeConfig := &config.Node{
Docker: envInfo.Docker,
ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint,
@ -345,9 +342,9 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
nodeConfig.AgentConfig.ClientCA = clientCAFile
nodeConfig.AgentConfig.ListenAddress = "0.0.0.0"
nodeConfig.AgentConfig.KubeConfigNode = kubeconfigNode
nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
nodeConfig.AgentConfig.KubeConfigK3sController = kubeconfigK3sController
if envInfo.Rootless {
nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet")
}

21
pkg/agent/flannel/setup.go
View File

@ -13,8 +13,7 @@ import (
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
v1 "k8s.io/client-go/kubernetes/typed/core/v1"
)
const (
@ -74,21 +73,11 @@ func Prepare(ctx context.Context, nodeConfig *config.Node) error {
return createFlannelConf(nodeConfig)
}
func Run(ctx context.Context, nodeConfig *config.Node) error {
func Run(ctx context.Context, nodeConfig *config.Node, nodes v1.NodeInterface) error {
nodeName := nodeConfig.AgentConfig.NodeName
restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigNode)
if err != nil {
return err
}
client, err := kubernetes.NewForConfig(restConfig)
if err != nil {
return err
}
for {
node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
node, err := nodes.Get(nodeName, metav1.GetOptions{})
if err == nil && node.Spec.PodCIDR != "" {
break
}
@ -101,11 +90,11 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
}
go func() {
err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigNode)
err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigKubelet)
logrus.Fatalf("flannel exited: %v", err)
}()
return err
return nil
}
func createCNIConf(dir string) error {

2
pkg/agent/netpol/network_policy.go
View File

@ -10,7 +10,7 @@ import (
)
func Run(ctx context.Context, nodeConfig *config.Node) error {
restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigNode)
restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigK3sController)
if err != nil {
return err
}

120
pkg/agent/run.go
View File

@ -9,6 +9,7 @@ import (
"strings"
"time"
systemd "github.com/coreos/go-systemd/daemon"
"github.com/rancher/k3s/pkg/agent/config"
"github.com/rancher/k3s/pkg/agent/containerd"
"github.com/rancher/k3s/pkg/agent/flannel"
@ -21,10 +22,11 @@ import (
"github.com/rancher/k3s/pkg/daemons/agent"
daemonconfig "github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/rootless"
"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
corev1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
"github.com/rancher/wrangler/pkg/start"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/api/equality"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
v1 "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/tools/clientcmd"
)
@ -64,14 +66,18 @@ func run(ctx context.Context, cfg cmds.Agent, lb *loadbalancer.LoadBalancer) err
return err
}
coreClient, err := coreClient(nodeConfig.AgentConfig.KubeConfigKubelet)
if err != nil {
return err
}
if !nodeConfig.NoFlannel {
if err := flannel.Run(ctx, nodeConfig); err != nil {
if err := flannel.Run(ctx, nodeConfig, coreClient.CoreV1().Nodes()); err != nil {
return err
}
}
if !nodeConfig.AgentConfig.DisableCCM {
if err := syncAddressesLabels(ctx, &nodeConfig.AgentConfig); err != nil {
if err := syncAddressesLabels(ctx, &nodeConfig.AgentConfig, coreClient.CoreV1().Nodes()); err != nil {
return err
}
}
@ -86,6 +92,15 @@ func run(ctx context.Context, cfg cmds.Agent, lb *loadbalancer.LoadBalancer) err
return ctx.Err()
}
func coreClient(cfg string) (kubernetes.Interface, error) {
restConfig, err := clientcmd.BuildConfigFromFlags("", cfg)
if err != nil {
return nil, err
}
return kubernetes.NewForConfig(restConfig)
}
func Run(ctx context.Context, cfg cmds.Agent) error {
if err := validate(); err != nil {
return err
@ -100,10 +115,6 @@ func Run(ctx context.Context, cfg cmds.Agent) error {
cfg.DataDir = filepath.Join(cfg.DataDir, "agent")
os.MkdirAll(cfg.DataDir, 0700)
if cfg.ClusterSecret != "" {
cfg.Token = "K10node:" + cfg.ClusterSecret
}
lb, err := loadbalancer.Setup(ctx, cfg)
if err != nil {
return err
@ -113,7 +124,7 @@ func Run(ctx context.Context, cfg cmds.Agent) error {
}
for {
tmpFile, err := clientaccess.AgentAccessInfoToTempKubeConfig("", cfg.ServerURL, cfg.Token)
newToken, err := clientaccess.NormalizeAndValidateTokenForUser(cfg.ServerURL, cfg.Token, "node")
if err != nil {
logrus.Error(err)
select {
@ -123,10 +134,11 @@ func Run(ctx context.Context, cfg cmds.Agent) error {
}
continue
}
os.Remove(tmpFile)
cfg.Token = newToken
break
}
systemd.SdNotify(true, "READY=1\n")
return run(ctx, cfg, lb)
}
@ -149,73 +161,51 @@ func validate() error {
return nil
}
func syncAddressesLabels(ctx context.Context, agentConfig *daemonconfig.Agent) error {
func syncAddressesLabels(ctx context.Context, agentConfig *daemonconfig.Agent, nodes v1.NodeInterface) error {
for {
nodeController, nodeCache, err := startNodeController(ctx, agentConfig)
node, err := nodes.Get(agentConfig.NodeName, metav1.GetOptions{})
if err != nil {
logrus.Infof("Waiting for kubelet to be ready on node %s: %v", agentConfig.NodeName, err)
time.Sleep(1 * time.Second)
continue
}
nodeCached, err := nodeCache.Get(agentConfig.NodeName)
if err != nil {
logrus.Infof("Waiting for kubelet to be ready on node %s: %v", agentConfig.NodeName, err)
time.Sleep(1 * time.Second)
continue
}
node := nodeCached.DeepCopy()
updated := updateLabelMap(ctx, agentConfig, node.Labels)
if updated {
_, err = nodeController.Update(node)
if err == nil {
logrus.Infof("addresses labels has been set succesfully on node: %s", agentConfig.NodeName)
break
newLabels, update := updateLabelMap(agentConfig, node.Labels)
if update {
node.Labels = newLabels
if _, err := nodes.Update(node); err != nil {
logrus.Infof("Failed to update node %s: %v", agentConfig.NodeName, err)
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Second):
continue
}
}
logrus.Infof("Failed to update node %s: %v", agentConfig.NodeName, err)
time.Sleep(1 * time.Second)
continue
logrus.Infof("addresses labels has been set successfully on node: %s", agentConfig.NodeName)
} else {
logrus.Infof("addresses labels has already been set successfully on node: %s", agentConfig.NodeName)
}
logrus.Infof("addresses labels has already been set succesfully on node: %s", agentConfig.NodeName)
return nil
break
}
return nil
}
func startNodeController(ctx context.Context, agentConfig *daemonconfig.Agent) (corev1.NodeController, corev1.NodeCache, error) {
restConfig, err := clientcmd.BuildConfigFromFlags("", agentConfig.KubeConfigKubelet)
if err != nil {
return nil, nil, err
}
coreFactory := core.NewFactoryFromConfigOrDie(restConfig)
nodeController := coreFactory.Core().V1().Node()
nodeCache := nodeController.Cache()
if err := start.All(ctx, 1, coreFactory); err != nil {
return nil, nil, err
func updateLabelMap(agentConfig *daemonconfig.Agent, nodeLabels map[string]string) (map[string]string, bool) {
result := map[string]string{}
for k, v := range nodeLabels {
result[k] = v
}
return nodeController, nodeCache, nil
}
result[InternalIPLabel] = agentConfig.NodeIP
result[HostnameLabel] = agentConfig.NodeName
if agentConfig.NodeExternalIP == "" {
delete(result, ExternalIPLabel)
} else {
result[ExternalIPLabel] = agentConfig.NodeExternalIP
}
func updateLabelMap(ctx context.Context, agentConfig *daemonconfig.Agent, nodeLabels map[string]string) bool {
if nodeLabels == nil {
nodeLabels = make(map[string]string)
}
updated := false
if internalIPLabel, ok := nodeLabels[InternalIPLabel]; !ok || internalIPLabel != agentConfig.NodeIP {
nodeLabels[InternalIPLabel] = agentConfig.NodeIP
updated = true
}
if hostnameLabel, ok := nodeLabels[HostnameLabel]; !ok || hostnameLabel != agentConfig.NodeName {
nodeLabels[HostnameLabel] = agentConfig.NodeName
updated = true
}
nodeExternalIP := agentConfig.NodeExternalIP
if externalIPLabel := nodeLabels[ExternalIPLabel]; externalIPLabel != nodeExternalIP && nodeExternalIP != "" {
nodeLabels[ExternalIPLabel] = nodeExternalIP
updated = true
} else if nodeExternalIP == "" && externalIPLabel != "" {
delete(nodeLabels, ExternalIPLabel)
updated = true
}
return updated
return result, !equality.Semantic.DeepEqual(nodeLabels, result)
}

49
pkg/agent/tunnel/tunnel.go
View File

@ -3,11 +3,8 @@ package tunnel
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"fmt"
"net"
"net/http"
"reflect"
"sync"
"time"
@ -21,8 +18,8 @@ import (
"k8s.io/apimachinery/pkg/fields"
watchtypes "k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/transport"
)
var (
@ -54,12 +51,7 @@ func getAddresses(endpoint *v1.Endpoints) []string {
}
func Setup(ctx context.Context, config *config.Node, onChange func([]string)) error {
restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode)
if err != nil {
return err
}
transportConfig, err := restConfig.TransportConfig()
restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigK3sController)
if err != nil {
return err
}
@ -69,6 +61,16 @@ func Setup(ctx context.Context, config *config.Node, onChange func([]string)) er
return err
}
nodeRestConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigKubelet)
if err != nil {
return err
}
tlsConfig, err := rest.TLSConfigFor(nodeRestConfig)
if err != nil {
return err
}
addresses := []string{config.ServerAddress}
endpoint, _ := client.CoreV1().Endpoints("default").Get("kubernetes", metav1.GetOptions{})
@ -84,7 +86,7 @@ func Setup(ctx context.Context, config *config.Node, onChange func([]string)) er
wg := &sync.WaitGroup{}
for _, address := range addresses {
if _, ok := disconnect[address]; !ok {
disconnect[address] = connect(ctx, wg, address, config, transportConfig)
disconnect[address] = connect(ctx, wg, address, tlsConfig)
}
}
@ -132,7 +134,7 @@ func Setup(ctx context.Context, config *config.Node, onChange func([]string)) er
for _, address := range addresses {
validEndpoint[address] = true
if _, ok := disconnect[address]; !ok {
disconnect[address] = connect(ctx, nil, address, config, transportConfig)
disconnect[address] = connect(ctx, nil, address, tlsConfig)
}
}
@ -164,25 +166,10 @@ func Setup(ctx context.Context, config *config.Node, onChange func([]string)) er
return nil
}
func connect(rootCtx context.Context, waitGroup *sync.WaitGroup, address string, config *config.Node, transportConfig *transport.Config) context.CancelFunc {
func connect(rootCtx context.Context, waitGroup *sync.WaitGroup, address string, tlsConfig *tls.Config) context.CancelFunc {
wsURL := fmt.Sprintf("wss://%s/v1-k3s/connect", address)
headers := map[string][]string{
"X-K3s-NodeName": {config.AgentConfig.NodeName},
}
ws := &websocket.Dialer{}
if len(config.CACerts) > 0 {
pool := x509.NewCertPool()
pool.AppendCertsFromPEM(config.CACerts)
ws.TLSClientConfig = &tls.Config{
RootCAs: pool,
}
}
if transportConfig.Username != "" {
auth := transportConfig.Username + ":" + transportConfig.Password
auth = base64.StdEncoding.EncodeToString([]byte(auth))
headers["Authorization"] = []string{"Basic " + auth}
ws := &websocket.Dialer{
TLSClientConfig: tlsConfig,
}
once := sync.Once{}
@ -194,7 +181,7 @@ func connect(rootCtx context.Context, waitGroup *sync.WaitGroup, address string,
go func() {
for {
remotedialer.ClientConnect(ctx, wsURL, http.Header(headers), ws, func(proto, address string) bool {
remotedialer.ClientConnect(ctx, wsURL, nil, ws, func(proto, address string) bool {
host, port, err := net.SplitHostPort(address)
return err == nil && proto == "tcp" && ports[port] && host == "127.0.0.1"
}, func(_ context.Context) error {

11
pkg/apis/k3s.cattle.io/v1/types.go
View File

@ -1,7 +1,6 @@
package v1
import (
"github.com/rancher/dynamiclistener"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
)
@ -9,16 +8,6 @@ import (
// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
type ListenerConfig struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Status dynamiclistener.ListenerStatus `json:"status,omitempty"`
}
// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
type Addon struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`

80
pkg/bootstrap/bootstrap.go Normal file
View File

@ -0,0 +1,80 @@
package bootstrap
import (
"encoding/json"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"github.com/pkg/errors"
"github.com/rancher/k3s/pkg/daemons/config"
)
func Handler(bootstrap *config.ControlRuntimeBootstrap) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
rw.Header().Set("Content-Type", "application/json")
Write(rw, bootstrap)
})
}
func Write(w io.Writer, bootstrap *config.ControlRuntimeBootstrap) error {
paths, err := objToMap(bootstrap)
if err != nil {
return nil
}
dataMap := map[string][]byte{}
for pathKey, path := range paths {
if path == "" {
continue
}
data, err := ioutil.ReadFile(path)
if err != nil {
return errors.Wrapf(err, "failed to read %s", path)
}
dataMap[pathKey] = data
}
return json.NewEncoder(w).Encode(dataMap)
}
func Read(r io.Reader, bootstrap *config.ControlRuntimeBootstrap) error {
paths, err := objToMap(bootstrap)
if err != nil {
return err
}
files := map[string][]byte{}
if err := json.NewDecoder(r).Decode(&files); err != nil {
return err
}
for pathKey, data := range files {
path, ok := paths[pathKey]
if !ok {
continue
}
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
return errors.Wrapf(err, "failed to mkdir %s", filepath.Dir(path))
}
if err := ioutil.WriteFile(path, data, 0600); err != nil {
return errors.Wrapf(err, "failed to write to %s", path)
}
}
return nil
}
func objToMap(obj interface{}) (map[string]string, error) {
bytes, err := json.Marshal(obj)
if err != nil {
return nil, err
}
data := map[string]string{}
return data, json.Unmarshal(bytes, &data)
}

28
pkg/cli/agent/agent.go
View File

@ -3,39 +3,18 @@ package agent
import (
"context"
"fmt"
"io/ioutil"
"os"
"strings"
"time"
systemd "github.com/coreos/go-systemd/daemon"
"github.com/rancher/k3s/pkg/agent"
"github.com/rancher/k3s/pkg/cli/cmds"
"github.com/rancher/k3s/pkg/datadir"
"github.com/rancher/k3s/pkg/netutil"
"github.com/rancher/k3s/pkg/token"
"github.com/rancher/wrangler/pkg/signals"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)
func readToken(path string) (string, error) {
if path == "" {
return "", nil
}
for {
tokenBytes, err := ioutil.ReadFile(path)
if err == nil {
return strings.TrimSpace(string(tokenBytes)), nil
} else if os.IsNotExist(err) {
logrus.Infof("Waiting for %s to be available\n", path)
time.Sleep(2 * time.Second)
} else {
return "", err
}
}
}
func Run(ctx *cli.Context) error {
if err := cmds.InitLogging(); err != nil {
return err
@ -45,14 +24,14 @@ func Run(ctx *cli.Context) error {
}
if cmds.AgentConfig.TokenFile != "" {
token, err := readToken(cmds.AgentConfig.TokenFile)
token, err := token.ReadFile(cmds.AgentConfig.TokenFile)
if err != nil {
return err
}
cmds.AgentConfig.Token = token
}
if cmds.AgentConfig.Token == "" && cmds.AgentConfig.ClusterSecret == "" {
if cmds.AgentConfig.Token == "" {
return fmt.Errorf("--token is required")
}
@ -76,7 +55,6 @@ func Run(ctx *cli.Context) error {
cfg.DataDir = dataDir
contextCtx := signals.SetupSignalHandler(context.Background())
systemd.SdNotify(true, "READY=1\n")
return agent.Run(contextCtx, cfg)
}

108
pkg/cli/cmds/agent.go
View File

@ -17,7 +17,6 @@ type Agent struct {
NodeIP string
NodeExternalIP string
NodeName string
ClusterSecret string
PauseImage string
Docker bool
ContainerRuntimeEndpoint string
@ -44,82 +43,82 @@ var (
AgentConfig Agent
NodeIPFlag = cli.StringFlag{
Name: "node-ip,i",
Usage: "(agent) IP address to advertise for node",
Usage: "(agent/networking) IP address to advertise for node",
Destination: &AgentConfig.NodeIP,
}
NodeExternalIPFlag = cli.StringFlag{
Name: "node-external-ip",
Usage: "(agent) External IP address to advertise for node",
Usage: "(agent/networking) External IP address to advertise for node",
Destination: &AgentConfig.NodeExternalIP,
}
NodeNameFlag = cli.StringFlag{
Name: "node-name",
Usage: "(agent) Node name",
Usage: "(agent/node) Node name",
EnvVar: "K3S_NODE_NAME",
Destination: &AgentConfig.NodeName,
}
DockerFlag = cli.BoolFlag{
Name: "docker",
Usage: "(agent) Use docker instead of containerd",
Usage: "(agent/runtime) Use docker instead of containerd",
Destination: &AgentConfig.Docker,
}
CRIEndpointFlag = cli.StringFlag{
Name: "container-runtime-endpoint",
Usage: "(agent/runtime) Disable embedded containerd and use alternative CRI implementation",
Destination: &AgentConfig.ContainerRuntimeEndpoint,
}
PrivateRegistryFlag = cli.StringFlag{
Name: "private-registry",
Usage: "(agent/runtime) Private registry configuration file",
Destination: &AgentConfig.PrivateRegistry,
Value: "/etc/rancher/k3s/registries.yaml",
}
PauseImageFlag = cli.StringFlag{
Name: "pause-image",
Usage: "(agent/runtime) Customized pause image for containerd sandbox",
Destination: &AgentConfig.PauseImage,
}
FlannelFlag = cli.BoolFlag{
Name: "no-flannel",
Usage: "(agent) Disable embedded flannel",
Usage: "(deprecated) use --flannel-backend=none",
Destination: &AgentConfig.NoFlannel,
}
FlannelIfaceFlag = cli.StringFlag{
Name: "flannel-iface",
Usage: "(agent) Override default flannel interface",
Usage: "(agent/networking) Override default flannel interface",
Destination: &AgentConfig.FlannelIface,
}
FlannelConfFlag = cli.StringFlag{
Name: "flannel-conf",
Usage: "(agent) (experimental) Override default flannel config file",
Usage: "(agent/networking) Override default flannel config file",
Destination: &AgentConfig.FlannelConf,
}
CRIEndpointFlag = cli.StringFlag{
Name: "container-runtime-endpoint",
Usage: "(agent) Disable embedded containerd and use alternative CRI implementation",
Destination: &AgentConfig.ContainerRuntimeEndpoint,
}
PauseImageFlag = cli.StringFlag{
Name: "pause-image",
Usage: "(agent) Customized pause image for containerd sandbox",
Destination: &AgentConfig.PauseImage,
}
ResolvConfFlag = cli.StringFlag{
Name: "resolv-conf",
Usage: "(agent) Kubelet resolv.conf file",
Usage: "(agent/networking) Kubelet resolv.conf file",
EnvVar: "K3S_RESOLV_CONF",
Destination: &AgentConfig.ResolvConf,
}
ExtraKubeletArgs = cli.StringSliceFlag{
Name: "kubelet-arg",
Usage: "(agent) Customized flag for kubelet process",
Usage: "(agent/flags) Customized flag for kubelet process",
Value: &AgentConfig.ExtraKubeletArgs,
}
ExtraKubeProxyArgs = cli.StringSliceFlag{
Name: "kube-proxy-arg",
Usage: "(agent) Customized flag for kube-proxy process",
Usage: "(agent/flags) Customized flag for kube-proxy process",
Value: &AgentConfig.ExtraKubeProxyArgs,
}
NodeTaints = cli.StringSliceFlag{
Name: "node-taint",
Usage: "(agent) Registering kubelet with set of taints",
Usage: "(agent/node) Registering kubelet with set of taints",
Value: &AgentConfig.Taints,
}
NodeLabels = cli.StringSliceFlag{
Name: "node-label",
Usage: "(agent) Registering kubelet with set of labels",
Usage: "(agent/node) Registering kubelet with set of labels",
Value: &AgentConfig.Labels,
}
PrivateRegistryFlag = cli.StringFlag{
Name: "private-registry",
Usage: "(agent) Private registry configuration file",
Destination: &AgentConfig.PrivateRegistry,
Value: "/etc/rancher/k3s/registries.yaml",
}
)
func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
@ -135,54 +134,57 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
AlsoLogToStderr,
cli.StringFlag{
Name: "token,t",
Usage: "Token to use for authentication",
Usage: "(cluster) Token to use for authentication",
EnvVar: "K3S_TOKEN",
Destination: &AgentConfig.Token,
},
cli.StringFlag{
Name: "token-file",
Usage: "Token file to use for authentication",
Usage: "(cluster) Token file to use for authentication",
EnvVar: "K3S_TOKEN_FILE",
Destination: &AgentConfig.TokenFile,
},
cli.StringFlag{
Name: "server,s",
Usage: "Server to connect to",
Usage: "(cluster) Server to connect to",
EnvVar: "K3S_URL",
Destination: &AgentConfig.ServerURL,
},
cli.StringFlag{
Name: "data-dir,d",
Usage: "Folder to hold state",
Usage: "(agent/data) Folder to hold state",
Destination: &AgentConfig.DataDir,
Value: "/var/lib/rancher/k3s",
},
cli.StringFlag{
Name: "cluster-secret",
Usage: "Shared secret used to bootstrap a cluster",
Destination: &AgentConfig.ClusterSecret,
EnvVar: "K3S_CLUSTER_SECRET",
},
NodeNameFlag,
NodeLabels,
NodeTaints,
DockerFlag,
CRIEndpointFlag,
PauseImageFlag,
PrivateRegistryFlag,
NodeIPFlag,
NodeExternalIPFlag,
ResolvConfFlag,
FlannelIfaceFlag,
FlannelConfFlag,
ExtraKubeletArgs,
ExtraKubeProxyArgs,
cli.BoolFlag{
Name: "rootless",
Usage: "(experimental) Run rootless",
Destination: &AgentConfig.Rootless,
},
DockerFlag,
// Deprecated/hidden below
FlannelFlag,
FlannelIfaceFlag,
FlannelConfFlag,
NodeNameFlag,
NodeIPFlag,
CRIEndpointFlag,
PauseImageFlag,
ResolvConfFlag,
ExtraKubeletArgs,
ExtraKubeProxyArgs,
NodeLabels,
NodeTaints,
PrivateRegistryFlag,
NodeExternalIPFlag,
cli.StringFlag{
Name: "cluster-secret",
Usage: "(deprecated) use --token",
Destination: &AgentConfig.Token,
EnvVar: "K3S_CLUSTER_SECRET",
},
},
}
}

8
pkg/cli/cmds/log.go
View File

@ -25,22 +25,22 @@ var (
VLevel = cli.IntFlag{
Name: "v",
Usage: "Number for the log level verbosity",
Usage: "(logging) Number for the log level verbosity",
Destination: &LogConfig.VLevel,
}
VModule = cli.StringFlag{
Name: "vmodule",
Usage: "Comma-separated list of pattern=N settings for file-filtered logging",
Usage: "(logging) Comma-separated list of pattern=N settings for file-filtered logging",
Destination: &LogConfig.VModule,
}
LogFile = cli.StringFlag{
Name: "log,l",
Usage: "Log to file",
Usage: "(logging) Log to file",
Destination: &LogConfig.LogFile,
}
AlsoLogToStderr = cli.BoolFlag{
Name: "alsologtostderr",
Usage: "Log to standard error as well as file (if set)",
Usage: "(logging) Log to standard error as well as file (if set)",
Destination: &LogConfig.AlsoLogToStderr,
}
)

7
pkg/cli/cmds/nocluster.go Normal file
View File

@ -0,0 +1,7 @@
// +build !dqlite
package cmds
const (
hideDqlite = true
)

380
pkg/cli/cmds/server.go
View File

@ -9,12 +9,14 @@ import (
type Server struct {
ClusterCIDR string
ClusterSecret string
AgentToken string
AgentTokenFile string
Token string
TokenFile string
ServiceCIDR string
ClusterDNS string
ClusterDomain string
HTTPSPort int
HTTPPort int
DataDir string
DisableAgent bool
KubeConfigOutput string
@ -26,7 +28,6 @@ type Server struct {
ExtraControllerArgs cli.StringSlice
ExtraCloudControllerArgs cli.StringSlice
Rootless bool
StoreBootstrap bool
StorageEndpoint string
StorageCAFile string
StorageCertFile string
@ -34,10 +35,13 @@ type Server struct {
AdvertiseIP string
AdvertisePort int
DisableScheduler bool
ServerURL string
FlannelBackend string
DefaultLocalStoragePath string
DisableCCM bool
DisableNPC bool
ClusterInit bool
ClusterReset bool
}
var ServerConfig Server
@ -55,190 +59,236 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
AlsoLogToStderr,
cli.StringFlag{
Name: "bind-address",
Usage: "k3s bind address (default: localhost)",
Usage: "(listener) k3s bind address (default: 0.0.0.0)",
Destination: &ServerConfig.BindAddress,
},
cli.IntFlag{
Name: "https-listen-port",
Usage: "HTTPS listen port",
Usage: "(listener) HTTPS listen port",
Value: 6443,
Destination: &ServerConfig.HTTPSPort,
},
cli.StringFlag{
Name: "advertise-address",
Usage: "(listener) IP address that apiserver uses to advertise to members of the cluster (default: node-external-ip/node-ip)",
Destination: &ServerConfig.AdvertiseIP,
},
cli.IntFlag{
Name: "http-listen-port",
Usage: "HTTP listen port (for /healthz, HTTPS redirect, and port for TLS terminating LB)",
Value: 0,
Destination: &ServerConfig.HTTPPort,
Name: "advertise-port",
Usage: "(listener) Port that apiserver uses to advertise to members of the cluster (default: listen-port)",
Destination: &ServerConfig.AdvertisePort,
},
cli.StringSliceFlag{
Name: "tls-san",
Usage: "(listener) Add additional hostname or IP as a Subject Alternative Name in the TLS cert",
Value: &ServerConfig.TLSSan,
},
cli.StringFlag{
Name: "data-dir,d",
Usage: "Folder to hold state default /var/lib/rancher/k3s or ${HOME}/.rancher/k3s if not root",
Usage: "(data) Folder to hold state default /var/lib/rancher/k3s or ${HOME}/.rancher/k3s if not root",
Destination: &ServerConfig.DataDir,
},
cli.StringFlag{
Name: "cluster-cidr",
Usage: "(networking) Network CIDR to use for pod IPs",
Destination: &ServerConfig.ClusterCIDR,
Value: "10.42.0.0/16",
},
cli.StringFlag{
Name: "service-cidr",
Usage: "(networking) Network CIDR to use for services IPs",
Destination: &ServerConfig.ServiceCIDR,
Value: "10.43.0.0/16",
},
cli.StringFlag{
Name: "cluster-dns",
Usage: "(networking) Cluster IP for coredns service. Should be in your service-cidr range (default: 10.43.0.10)",
Destination: &ServerConfig.ClusterDNS,
Value: "",
},
cli.StringFlag{
Name: "cluster-domain",
Usage: "(networking) Cluster Domain",
Destination: &ServerConfig.ClusterDomain,
Value: "cluster.local",
},
cli.StringFlag{
Name: "flannel-backend",
Usage: fmt.Sprintf("(networking) One of '%s', '%s', '%s', or '%s'", config.FlannelBackendNone, config.FlannelBackendVXLAN, config.FlannelBackendIPSEC, config.FlannelBackendWireguard),
Destination: &ServerConfig.FlannelBackend,
Value: config.FlannelBackendVXLAN,
},
cli.StringFlag{
Name: "token,t",
Usage: "(cluster) Shared secret used to join a server or agent to a cluster",
Destination: &ServerConfig.Token,
EnvVar: "K3S_CLUSTER_SECRET,K3S_TOKEN",
},
cli.StringFlag{
Name: "token-file",
Usage: "(cluster) File containing the cluster-secret/token",
Destination: &ServerConfig.TokenFile,
EnvVar: "K3S_TOKEN_FILE",
},
cli.StringFlag{
Name: "agent-token",
Usage: "(cluster) Shared secret used to join agents to the cluster, but not agents",
Destination: &ServerConfig.AgentToken,
EnvVar: "K3S_AGENT_TOKEN",
},
cli.StringFlag{
Name: "agent-token-file",
Usage: "(cluster) File containing the agent secret",
Destination: &ServerConfig.AgentTokenFile,
EnvVar: "K3S_AGENT_TOKEN_FILE",
},
cli.StringFlag{
Name: "server,s",
Usage: "(cluster) Server to connect to, used to join a cluster",
EnvVar: "K3S_URL",
Destination: &ServerConfig.ServerURL,
},
cli.BoolFlag{
Name: "new-cluster",
Hidden: hideDqlite,
Usage: "(cluster) Initialize new cluster master",
EnvVar: "K3S_CLUSTER_INIT",
Destination: &ServerConfig.ClusterInit,
},
cli.BoolFlag{
Name: "reset-cluster",
Hidden: hideDqlite,
Usage: "(cluster) Forget all peers and become a single cluster new cluster master",
EnvVar: "K3S_CLUSTER_RESET",
Destination: &ServerConfig.ClusterReset,
},
cli.StringFlag{
Name: "write-kubeconfig,o",
Usage: "(client) Write kubeconfig for admin client to this file",
Destination: &ServerConfig.KubeConfigOutput,
EnvVar: "K3S_KUBECONFIG_OUTPUT",
},
cli.StringFlag{
Name: "write-kubeconfig-mode",
Usage: "(client) Write kubeconfig with this mode",
Destination: &ServerConfig.KubeConfigMode,
EnvVar: "K3S_KUBECONFIG_MODE",
},
cli.StringSliceFlag{
Name: "kube-apiserver-arg",
Usage: "(flags) Customized flag for kube-apiserver process",
Value: &ServerConfig.ExtraAPIArgs,
},
cli.StringSliceFlag{
Name: "kube-scheduler-arg",
Usage: "(flags) Customized flag for kube-scheduler process",
Value: &ServerConfig.ExtraSchedulerArgs,
},
cli.StringSliceFlag{
Name: "kube-controller-manager-arg",
Usage: "(flags) Customized flag for kube-controller-manager process",
Value: &ServerConfig.ExtraControllerArgs,
},
cli.StringSliceFlag{
Name: "kube-cloud-controller-manager-arg",
Usage: "(flags) Customized flag for kube-cloud-controller-manager process",
Value: &ServerConfig.ExtraCloudControllerArgs,
},
cli.StringFlag{
Name: "storage-endpoint",
Usage: "(db) Specify etcd, Mysql, Postgres, or Sqlite (default) data source name",
Destination: &ServerConfig.StorageEndpoint,
EnvVar: "K3S_STORAGE_ENDPOINT",
},
cli.StringFlag{
Name: "storage-cafile",
Usage: "(db) SSL Certificate Authority file used to secure storage backend communication",
Destination: &ServerConfig.StorageCAFile,
EnvVar: "K3S_STORAGE_CAFILE",
},
cli.StringFlag{
Name: "storage-certfile",
Usage: "(db) SSL certification file used to secure storage backend communication",
Destination: &ServerConfig.StorageCertFile,
EnvVar: "K3S_STORAGE_CERTFILE",
},
cli.StringFlag{
Name: "storage-keyfile",
Usage: "(db) SSL key file used to secure storage backend communication",
Destination: &ServerConfig.StorageKeyFile,
EnvVar: "K3S_STORAGE_KEYFILE",
},
cli.StringFlag{
Name: "default-local-storage-path",
Usage: "(storage) Default local storage path for local provisioner storage class",
Destination: &ServerConfig.DefaultLocalStoragePath,
},
cli.StringSliceFlag{
Name: "no-deploy",
Usage: "(components) Do not deploy packaged components (valid items: coredns, servicelb, traefik, local-storage)",
},
cli.BoolFlag{
Name: "disable-scheduler",
Usage: "(components) Disable Kubernetes default scheduler",
Destination: &ServerConfig.DisableScheduler,
},
cli.BoolFlag{
Name: "disable-cloud-controller",
Usage: "(components) Disable k3s default cloud controller manager",
Destination: &ServerConfig.DisableCCM,
},
cli.BoolFlag{
Name: "disable-network-policy",
Usage: "(components) Disable k3s default network policy controller",
Destination: &ServerConfig.DisableNPC,
},
NodeNameFlag,
NodeLabels,
NodeTaints,
DockerFlag,
CRIEndpointFlag,
PauseImageFlag,
PrivateRegistryFlag,
NodeIPFlag,
NodeExternalIPFlag,
ResolvConfFlag,
FlannelIfaceFlag,
FlannelConfFlag,
ExtraKubeletArgs,
ExtraKubeProxyArgs,
cli.BoolFlag{
Name: "rootless",
Usage: "(experimental) Run rootless",
Destination: &ServerConfig.Rootless,
},
// Hidden/Deprecated flags below
FlannelFlag,
cli.StringFlag{
Name: "cluster-secret",
Usage: "(deprecated) use --token",
Destination: &ServerConfig.Token,
EnvVar: "K3S_CLUSTER_SECRET",
},
cli.BoolFlag{
Name: "disable-agent",
Usage: "Do not run a local agent and register a local kubelet",
Hidden: true,
Destination: &ServerConfig.DisableAgent,
},
cli.StringFlag{
Name: "cluster-cidr",
Usage: "Network CIDR to use for pod IPs",
Destination: &ServerConfig.ClusterCIDR,
Value: "10.42.0.0/16",
},
cli.StringFlag{
Name: "cluster-secret",
Usage: "Shared secret used to bootstrap a cluster",
Destination: &ServerConfig.ClusterSecret,
EnvVar: "K3S_CLUSTER_SECRET",
},
cli.StringFlag{
Name: "service-cidr",
Usage: "Network CIDR to use for services IPs",
Destination: &ServerConfig.ServiceCIDR,
Value: "10.43.0.0/16",
},
cli.StringFlag{
Name: "cluster-dns",
Usage: "Cluster IP for coredns service. Should be in your service-cidr range",
Destination: &ServerConfig.ClusterDNS,
Value: "",
},
cli.StringFlag{
Name: "cluster-domain",
Usage: "Cluster Domain",
Destination: &ServerConfig.ClusterDomain,
Value: "cluster.local",
cli.StringSliceFlag{
Hidden: true,
Name: "kube-controller-arg",
Usage: "(flags) Customized flag for kube-controller-manager process",
Value: &ServerConfig.ExtraControllerArgs,
},
cli.StringSliceFlag{
Name: "no-deploy",
Usage: "Do not deploy packaged components (valid items: coredns, servicelb, traefik)",
Hidden: true,
Name: "kube-cloud-controller-arg",
Usage: "(flags) Customized flag for kube-cloud-controller-manager process",
Value: &ServerConfig.ExtraCloudControllerArgs,
},
cli.StringFlag{
Name: "write-kubeconfig,o",
Usage: "Write kubeconfig for admin client to this file",
Destination: &ServerConfig.KubeConfigOutput,
EnvVar: "K3S_KUBECONFIG_OUTPUT",
},
cli.StringFlag{
Name: "write-kubeconfig-mode",
Usage: "Write kubeconfig with this mode",
Destination: &ServerConfig.KubeConfigMode,
EnvVar: "K3S_KUBECONFIG_MODE",
},
cli.StringSliceFlag{
Name: "tls-san",
Usage: "Add additional hostname or IP as a Subject Alternative Name in the TLS cert",
Value: &ServerConfig.TLSSan,
},
cli.StringSliceFlag{
Name: "kube-apiserver-arg",
Usage: "Customized flag for kube-apiserver process",
Value: &ServerConfig.ExtraAPIArgs,
},
cli.StringSliceFlag{
Name: "kube-scheduler-arg",
Usage: "Customized flag for kube-scheduler process",
Value: &ServerConfig.ExtraSchedulerArgs,
},
cli.StringSliceFlag{
Name: "kube-controller-arg",
Usage: "Customized flag for kube-controller-manager process",
Value: &ServerConfig.ExtraControllerArgs,
},
cli.StringSliceFlag{
Name: "kube-cloud-controller-arg",
Usage: "Customized flag for kube-cloud-controller-manager process",
Value: &ServerConfig.ExtraCloudControllerArgs,
},
cli.BoolFlag{
Name: "rootless",
Usage: "(experimental) Run rootless",
Destination: &ServerConfig.Rootless,
},
cli.BoolFlag{
Name: "bootstrap-save",
Usage: "(experimental) Save bootstrap information in the storage endpoint",
Hidden: true,
Destination: &ServerConfig.StoreBootstrap,
},
cli.StringFlag{
Name: "storage-endpoint",
Usage: "Specify etcd, Mysql, Postgres, or Sqlite (default) data source name",
Destination: &ServerConfig.StorageEndpoint,
EnvVar: "K3S_STORAGE_ENDPOINT",
},
cli.StringFlag{
Name: "storage-cafile",
Usage: "SSL Certificate Authority file used to secure storage backend communication",
Destination: &ServerConfig.StorageCAFile,
EnvVar: "K3S_STORAGE_CAFILE",
},
cli.StringFlag{
Name: "storage-certfile",
Usage: "SSL certification file used to secure storage backend communication",
Destination: &ServerConfig.StorageCertFile,
EnvVar: "K3S_STORAGE_CERTFILE",
},
cli.StringFlag{
Name: "storage-keyfile",
Usage: "SSL key file used to secure storage backend communication",
Destination: &ServerConfig.StorageKeyFile,
EnvVar: "K3S_STORAGE_KEYFILE",
},
cli.StringFlag{
Name: "advertise-address",
Usage: "IP address that apiserver uses to advertise to members of the cluster",
Destination: &ServerConfig.AdvertiseIP,
},
cli.IntFlag{
Name: "advertise-port",
Usage: "Port that apiserver uses to advertise to members of the cluster",
Value: 0,
Destination: &ServerConfig.AdvertisePort,
},
cli.BoolFlag{
Name: "disable-scheduler",
Usage: "Disable Kubernetes default scheduler",
Destination: &ServerConfig.DisableScheduler,
},
cli.BoolFlag{
Name: "disable-cloud-controller",
Usage: "Disable k3s default cloud controller manager",
Destination: &ServerConfig.DisableCCM,
},
cli.BoolFlag{
Name: "disable-network-policy",
Usage: "Disable k3s default network policy controller",
Destination: &ServerConfig.DisableNPC,
},
cli.StringFlag{
Name: "flannel-backend",
Usage: fmt.Sprintf("(experimental) One of '%s', '%s', '%s', or '%s'", config.FlannelBackendNone, config.FlannelBackendVXLAN, config.FlannelBackendIPSEC, config.FlannelBackendWireguard),
Destination: &ServerConfig.FlannelBackend,
Value: config.FlannelBackendVXLAN,
},
cli.StringFlag{
Name: "default-local-storage-path",
Usage: "Default local storage path for local provisioner storage class",
Destination: &ServerConfig.DefaultLocalStoragePath,
},
NodeIPFlag,
NodeNameFlag,
DockerFlag,
FlannelFlag,
FlannelIfaceFlag,
FlannelConfFlag,
CRIEndpointFlag,
PauseImageFlag,
ResolvConfFlag,
ExtraKubeletArgs,
ExtraKubeProxyArgs,
NodeLabels,
NodeTaints,
PrivateRegistryFlag,
NodeExternalIPFlag,
},
}
}

54
pkg/cli/server/server.go
View File

@ -16,6 +16,7 @@ import (
"github.com/rancher/k3s/pkg/netutil"
"github.com/rancher/k3s/pkg/rootless"
"github.com/rancher/k3s/pkg/server"
"github.com/rancher/k3s/pkg/token"
"github.com/rancher/wrangler/pkg/signals"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
@ -56,23 +57,28 @@ func run(app *cli.Context, cfg *cmds.Server) error {
serverConfig := server.Config{}
serverConfig.DisableAgent = cfg.DisableAgent
serverConfig.ControlConfig.ClusterSecret = cfg.ClusterSecret
serverConfig.ControlConfig.Token = cfg.Token
serverConfig.ControlConfig.AgentToken = cfg.AgentToken
serverConfig.ControlConfig.JoinURL = cfg.ServerURL
if cfg.AgentTokenFile != "" {
serverConfig.ControlConfig.AgentToken, err = token.ReadFile(cfg.AgentTokenFile)
if err != nil {
return err
}
}
if cfg.TokenFile != "" {
serverConfig.ControlConfig.Token, err = token.ReadFile(cfg.TokenFile)
if err != nil {
return err
}
}
serverConfig.ControlConfig.DataDir = cfg.DataDir
serverConfig.ControlConfig.KubeConfigOutput = cfg.KubeConfigOutput
serverConfig.ControlConfig.KubeConfigMode = cfg.KubeConfigMode
serverConfig.ControlConfig.NoScheduler = cfg.DisableScheduler
serverConfig.Rootless = cfg.Rootless
serverConfig.TLSConfig.HTTPSPort = cfg.HTTPSPort
serverConfig.TLSConfig.HTTPPort = cfg.HTTPPort
for _, san := range knownIPs(cfg.TLSSan) {
addr := net2.ParseIP(san)
if addr != nil {
serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, san)
} else {
serverConfig.TLSConfig.Domains = append(serverConfig.TLSConfig.Domains, san)
}
}
serverConfig.TLSConfig.BindAddress = cfg.BindAddress
serverConfig.ControlConfig.SANs = knownIPs(cfg.TLSSan)
serverConfig.ControlConfig.BindAddress = cfg.BindAddress
serverConfig.ControlConfig.HTTPSPort = cfg.HTTPSPort
serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs
serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs
@ -84,21 +90,25 @@ func run(app *cli.Context, cfg *cmds.Server) error {
serverConfig.ControlConfig.Storage.KeyFile = cfg.StorageKeyFile
serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP
serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
serverConfig.ControlConfig.BootstrapReadOnly = !cfg.StoreBootstrap
serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend
serverConfig.ControlConfig.ExtraCloudControllerArgs = cfg.ExtraCloudControllerArgs
serverConfig.ControlConfig.DisableCCM = cfg.DisableCCM
serverConfig.ControlConfig.DisableNPC = cfg.DisableNPC
serverConfig.ControlConfig.ClusterInit = cfg.ClusterInit
serverConfig.ControlConfig.ClusterReset = cfg.ClusterReset
if cmds.AgentConfig.FlannelIface != "" && cmds.AgentConfig.NodeIP == "" {
cmds.AgentConfig.NodeIP = netutil.GetIPFromInterface(cmds.AgentConfig.FlannelIface)
}
if serverConfig.ControlConfig.AdvertiseIP == "" && cmds.AgentConfig.NodeExternalIP != "" {
serverConfig.ControlConfig.AdvertiseIP = cmds.AgentConfig.NodeExternalIP
}
if serverConfig.ControlConfig.AdvertiseIP == "" && cmds.AgentConfig.NodeIP != "" {
serverConfig.ControlConfig.AdvertiseIP = cmds.AgentConfig.NodeIP
}
if serverConfig.ControlConfig.AdvertiseIP != "" {
serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, serverConfig.ControlConfig.AdvertiseIP)
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, serverConfig.ControlConfig.AdvertiseIP)
}
_, serverConfig.ControlConfig.ClusterIPRange, err = net2.ParseCIDR(cfg.ClusterCIDR)
@ -114,7 +124,7 @@ func run(app *cli.Context, cfg *cmds.Server) error {
if err != nil {
return err
}
serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, apiServerServiceIP.String())
serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, apiServerServiceIP.String())
// If cluster-dns CLI arg is not set, we set ClusterDNS address to be ServiceCIDR network + 10,
// i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10
@ -160,8 +170,7 @@ func run(app *cli.Context, cfg *cmds.Server) error {
os.Unsetenv("NOTIFY_SOCKET")
ctx := signals.SetupSignalHandler(context.Background())
certs, err := server.StartServer(ctx, &serverConfig)
if err != nil {
if err := server.StartServer(ctx, &serverConfig); err != nil {
return err
}
@ -175,12 +184,17 @@ func run(app *cli.Context, cfg *cmds.Server) error {
<-ctx.Done()
return nil
}
ip := serverConfig.TLSConfig.BindAddress
ip := serverConfig.ControlConfig.BindAddress
if ip == "" {
ip = "127.0.0.1"
}
url := fmt.Sprintf("https://%s:%d", ip, serverConfig.TLSConfig.HTTPSPort)
token := server.FormatToken(serverConfig.ControlConfig.Runtime.NodeToken, certs)
url := fmt.Sprintf("https://%s:%d", ip, serverConfig.ControlConfig.HTTPSPort)
token, err := server.FormatToken(serverConfig.ControlConfig.Runtime.AgentToken, serverConfig.ControlConfig.Runtime.ServerCA)
if err != nil {
return err
}
agentConfig := cmds.AgentConfig
agentConfig.Debug = app.GlobalBool("bool")

52
pkg/clientaccess/clientaccess.go
View File

@ -9,7 +9,6 @@ import (
"io/ioutil"
"net/http"
"net/url"
"os"
"strings"
"github.com/pkg/errors"
@ -35,21 +34,6 @@ type clientToken struct {
password string
}
func AgentAccessInfoToTempKubeConfig(tempDir, server, token string) (string, error) {
f, err := ioutil.TempFile(tempDir, "tmp-")
if err != nil {
return "", err
}
if err := f.Close(); err != nil {
return "", err
}
err = accessInfoToKubeConfig(f.Name(), server, token)
if err != nil {
os.Remove(f.Name())
}
return f.Name(), err
}
func AgentAccessInfoToKubeConfig(destFile, server, token string) error {
return accessInfoToKubeConfig(destFile, server, token)
}
@ -62,6 +46,10 @@ type Info struct {
Token string `json:"token,omitempty"`
}
func (i *Info) ToToken() string {
return fmt.Sprintf("K10%s::%s:%s", hashCA(i.CACerts), i.username, i.password)
}
func (i *Info) WriteKubeConfig(destFile string) error {
return clientcmd.WriteToFile(*i.KubeConfig(), destFile)
}
@ -98,6 +86,22 @@ func (i *Info) KubeConfig() *clientcmdapi.Config {
return config
}
func NormalizeAndValidateTokenForUser(server, token, user string) (string, error) {
if !strings.HasPrefix(token, "K10") {
token = "K10::" + user + ":" + token
}
info, err := ParseAndValidateToken(server, token)
if err != nil {
return "", err
}
if info.username != user {
info.username = user
}
return info.ToToken(), nil
}
func ParseAndValidateToken(server, token string) (*Info, error) {
url, err := url.Parse(server)
if err != nil {
@ -132,13 +136,17 @@ func ParseAndValidateToken(server, token string) (*Info, error) {
return nil, err
}
return &Info{
i := &Info{
URL: url.String(),
CACerts: cacerts,
username: parsedToken.username,
password: parsedToken.password,
Token: token,
}, nil
}
// normalize token
i.Token = i.ToToken()
return i, nil
}
func accessInfoToKubeConfig(destFile, server, token string) error {
@ -164,11 +172,15 @@ func validateCACerts(cacerts []byte, hash string) (bool, string, string) {
return true, "", ""
}
digest := sha256.Sum256([]byte(cacerts))
newHash := hex.EncodeToString(digest[:])
newHash := hashCA(cacerts)
return hash == newHash, hash, newHash
}
func hashCA(cacerts []byte) string {
digest := sha256.Sum256(cacerts)
return hex.EncodeToString(digest[:])
}
func ParseUsernamePassword(token string) (string, string, bool) {
parsed, err := parseToken(token)
if err != nil {

48
pkg/cluster/cluster.go Normal file
View File

@ -0,0 +1,48 @@
package cluster
import (
"context"
"github.com/rancher/k3s/pkg/clientaccess"
"github.com/rancher/k3s/pkg/daemons/config"
)
type Cluster struct {
token string
clientAccessInfo *clientaccess.Info
config *config.Control
runtime *config.ControlRuntime
db interface{}
}
func (c *Cluster) Start(ctx context.Context) error {
join, err := c.shouldJoin()
if err != nil {
return err
}
if join {
if err := c.join(); err != nil {
return err
}
}
if err := c.startClusterAndHTTPS(ctx); err != nil {
return err
}
if join {
if err := c.postJoin(ctx); err != nil {
return err
}
}
return c.joined()
}
func New(config *config.Control) *Cluster {
return &Cluster{
config: config,
runtime: config.Runtime,
}
}

81
pkg/cluster/https.go Normal file
View File

@ -0,0 +1,81 @@
package cluster
import (
"context"
"crypto/tls"
"net"
"net/http"
"path/filepath"
"github.com/rancher/dynamiclistener"
"github.com/rancher/dynamiclistener/factory"
"github.com/rancher/dynamiclistener/storage/file"
"github.com/rancher/dynamiclistener/storage/kubernetes"
"github.com/rancher/dynamiclistener/storage/memory"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
"github.com/sirupsen/logrus"
)
func (c *Cluster) newListener(ctx context.Context) (net.Listener, http.Handler, error) {
tcp, err := dynamiclistener.NewTCPListener(c.config.BindAddress, c.config.HTTPSPort)
if err != nil {
return nil, nil, err
}
cert, key, err := factory.LoadCerts(c.runtime.ServerCA, c.runtime.ServerCAKey)
if err != nil {
return nil, nil, err
}
storage := tlsStorage(ctx, c.config.DataDir, c.runtime)
return dynamiclistener.NewListener(tcp, storage, cert, key, dynamiclistener.Config{
CN: "k3s",
Organization: []string{"k3s"},
TLSConfig: tls.Config{
ClientAuth: tls.RequestClientCert,
},
SANs: c.config.SANs,
})
}
func (c *Cluster) startClusterAndHTTPS(ctx context.Context) error {
l, handler, err := c.newListener(ctx)
if err != nil {
return err
}
handler, err = c.getHandler(handler)
if err != nil {
return err
}
l, handler, err = c.initClusterDB(ctx, l, handler)
if err != nil {
return err
}
server := http.Server{
Handler: handler,
}
go func() {
err := server.Serve(l)
logrus.Fatalf("server stopped: %v", err)
}()
go func() {
<-ctx.Done()
server.Shutdown(context.Background())
}()
return nil
}
func tlsStorage(ctx context.Context, dataDir string, runtime *config.ControlRuntime) dynamiclistener.TLSStorage {
fileStorage := file.New(filepath.Join(dataDir, "tls/dynamic-cert.json"))
cache := memory.NewBacked(fileStorage)
return kubernetes.New(ctx, func() *core.Factory {
return runtime.Core
}, "kube-system", "k3s-serving", cache)
}

74
pkg/cluster/join.go Normal file
View File

@ -0,0 +1,74 @@
package cluster
import (
"bytes"
"fmt"
"os"
"path/filepath"
"github.com/rancher/k3s/pkg/bootstrap"
"github.com/rancher/k3s/pkg/clientaccess"
"github.com/sirupsen/logrus"
)
func (c *Cluster) shouldJoin() (bool, error) {
if c.config.JoinURL == "" {
return false, nil
}
stamp := filepath.Join(c.config.DataDir, "db/joined")
if _, err := os.Stat(stamp); err == nil {
logrus.Info("Already joined to cluster, not rejoining")
return false, nil
}
if c.config.Token == "" {
return false, fmt.Errorf("K3S_TOKEN is required to join a cluster")
}
return true, nil
}
func (c *Cluster) joined() error {
if err := os.MkdirAll(filepath.Dir(c.joinStamp()), 0700); err != nil {
return err
}
if _, err := os.Stat(c.joinStamp()); err == nil {
return nil
}
f, err := os.Create(c.joinStamp())
if err != nil {
return err
}
return f.Close()
}
func (c *Cluster) join() error {
c.runtime.Cluster.Join = true
token, err := clientaccess.NormalizeAndValidateTokenForUser(c.config.JoinURL, c.config.Token, "server")
if err != nil {
return err
}
c.token = token
info, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, token)
if err != nil {
return err
}
c.clientAccessInfo = info
content, err := clientaccess.Get("/v1-k3s/server-bootstrap", info)
if err != nil {
return err
}
return bootstrap.Read(bytes.NewBuffer(content), &c.runtime.ControlRuntimeBootstrap)
}
func (c *Cluster) joinStamp() string {
return filepath.Join(c.config.DataDir, "db/joined")
}

17
pkg/cluster/nocluster.go Normal file
View File

@ -0,0 +1,17 @@
// +build !dqlite
package cluster
import (
"context"
"net"
"net/http"
)
func (c *Cluster) initClusterDB(ctx context.Context, l net.Listener, handler http.Handler) (net.Listener, http.Handler, error) {
return l, handler, nil
}
func (c *Cluster) postJoin(ctx context.Context) error {
return nil
}

25
pkg/cluster/router.go Normal file
View File

@ -0,0 +1,25 @@
package cluster
import (
"net/http"
)
func (c *Cluster) getHandler(handler http.Handler) (http.Handler, error) {
next := c.router()
return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
handler.ServeHTTP(rw, req)
next.ServeHTTP(rw, req)
}), nil
}
func (c *Cluster) router() http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
if c.runtime.Handler == nil {
http.Error(rw, "starting", http.StatusServiceUnavailable)
return
}
c.runtime.Handler.ServeHTTP(rw, req)
})
}

1
pkg/codegen/main.go
View File

@ -70,7 +70,6 @@ func main() {
Groups: map[string]args.Group{
"k3s.cattle.io": {
Types: []interface{}{
v1.ListenerConfig{},
v1.Addon{},
},
GenerateTypes: true,

96
pkg/daemons/config/types.go
View File

@ -9,7 +9,7 @@ import (
"strings"
"github.com/rancher/kine/pkg/endpoint"
"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
"k8s.io/apiserver/pkg/authentication/authenticator"
)
@ -47,40 +47,36 @@ type Containerd struct {
}
type Agent struct {
NodeName string
ClientKubeletCert string
ClientKubeletKey string
ClientKubeProxyCert string
ClientKubeProxyKey string
ServingKubeletCert string
ServingKubeletKey string
ClusterCIDR net.IPNet
ClusterDNS net.IP
ClusterDomain string
ResolvConf string
RootDir string
KubeConfigNode string
KubeConfigKubelet string
KubeConfigKubeProxy string
NodeIP string
NodeExternalIP string
RuntimeSocket string
ListenAddress string
ClientCA string
CNIBinDir string
CNIConfDir string
ExtraKubeletArgs []string
ExtraKubeProxyArgs []string
PauseImage string
CNIPlugin bool
NodeTaints []string
NodeLabels []string
IPSECPSK string
StrongSwanDir string
PrivateRegistry string
DisableCCM bool
DisableNPC bool
Rootless bool
NodeName string
ServingKubeletCert string
ServingKubeletKey string
ClusterCIDR net.IPNet
ClusterDNS net.IP
ClusterDomain string
ResolvConf string
RootDir string
KubeConfigKubelet string
KubeConfigKubeProxy string
KubeConfigK3sController string
NodeIP string
NodeExternalIP string
RuntimeSocket string
ListenAddress string
ClientCA string
CNIBinDir string
CNIConfDir string
ExtraKubeletArgs []string
ExtraKubeProxyArgs []string
PauseImage string
CNIPlugin bool
NodeTaints []string
NodeLabels []string
IPSECPSK string
StrongSwanDir string
PrivateRegistry string
DisableCCM bool
DisableNPC bool
Rootless bool
}
type Control struct {
@ -88,7 +84,8 @@ type Control struct {
AdvertiseIP string
ListenPort int
HTTPSPort int
ClusterSecret string
AgentToken string
Token string
ClusterIPRange *net.IPNet
ServiceIPRange *net.IPNet
ClusterDNS net.IP
@ -98,19 +95,24 @@ type Control struct {
KubeConfigMode string
DataDir string
Skips []string
BootstrapReadOnly bool
Storage endpoint.Config
NoScheduler bool
ExtraAPIArgs []string
ExtraControllerArgs []string
ExtraSchedulerAPIArgs []string
ExtraCloudControllerArgs []string
ExtraSchedulerAPIArgs []string
NoLeaderElect bool
JoinURL string
FlannelBackend string
IPSECPSK string
DefaultLocalStoragePath string
DisableCCM bool
DisableNPC bool
ClusterInit bool
ClusterReset bool
BindAddress string
SANs []string
Runtime *ControlRuntime `json:"-"`
}
@ -124,9 +126,6 @@ type ControlRuntimeBootstrap struct {
PasswdFile string
RequestHeaderCA string
RequestHeaderCAKey string
ClientKubeletKey string
ClientKubeProxyKey string
ServingKubeletKey string
IPSECKey string
}
@ -145,8 +144,10 @@ type ControlRuntime struct {
ServingKubeAPICert string
ServingKubeAPIKey string
ServingKubeletKey string
ClientToken string
NodeToken string
ServerToken string
AgentToken string
Handler http.Handler
Tunnel http.Handler
Authenticator authenticator.Request
@ -161,8 +162,19 @@ type ControlRuntime struct {
ClientSchedulerCert string
ClientSchedulerKey string
ClientKubeProxyCert string
ClientKubeProxyKey string
ClientKubeletKey string
ClientCloudControllerCert string
ClientCloudControllerKey string
ClientK3sControllerCert string
ClientK3sControllerKey string
Cluster ClusterConfig
Core *core.Factory
}
type ClusterConfig struct {
Join bool
}
type ArgString []string

100
pkg/daemons/control/bootstrap.go
View File

@ -1,100 +0,0 @@
package control
import (
"context"
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"github.com/pkg/errors"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/kine/pkg/client"
"github.com/sirupsen/logrus"
)
const (
k3sRuntimeEtcdPath = "/k3s/runtime"
)
// fetchBootstrapData copies the bootstrap data (certs, keys, passwords)
// from etcd to individual files specified by cfg.Runtime.
func fetchBootstrapData(ctx context.Context, cfg *config.Control, c client.Client) error {
logrus.Info("Fetching bootstrap data from etcd")
gr, err := c.Get(ctx, k3sRuntimeEtcdPath)
if err != nil {
return err
}
if gr.Modified == 0 {
return nil
}
paths, err := objToMap(&cfg.Runtime.ControlRuntimeBootstrap)
if err != nil {
return err
}
files := map[string][]byte{}
if err := json.Unmarshal(gr.Data, &files); err != nil {
return err
}
for pathKey, data := range files {
path, ok := paths[pathKey]
if !ok {
continue
}
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
return errors.Wrapf(err, "failed to mkdir %s", filepath.Dir(path))
}
if err := ioutil.WriteFile(path, data, 0700); err != nil {
return errors.Wrapf(err, "failed to write to %s", path)
}
}
return nil
}
// storeBootstrapData copies the bootstrap data in the opposite direction to
// fetchBootstrapData.
func storeBootstrapData(ctx context.Context, cfg *config.Control, client client.Client) error {
if cfg.BootstrapReadOnly {
return nil
}
paths, err := objToMap(&cfg.Runtime.ControlRuntimeBootstrap)
if err != nil {
return nil
}
dataMap := map[string][]byte{}
for pathKey, path := range paths {
if path == "" {
continue
}
data, err := ioutil.ReadFile(path)
if err != nil {
return errors.Wrapf(err, "failed to read %s", path)
}
dataMap[pathKey] = data
}
bytes, err := json.Marshal(dataMap)
if err != nil {
return err
}
return client.Put(ctx, k3sRuntimeEtcdPath, bytes)
}
func objToMap(obj interface{}) (map[string]string, error) {
bytes, err := json.Marshal(obj)
if err != nil {
return nil, err
}
data := map[string]string{}
return data, json.Unmarshal(bytes, &data)
}

237
pkg/daemons/control/server.go
View File

@ -3,13 +3,9 @@ package control
import (
"context"
"crypto"
cryptorand "crypto/rand"
"crypto/x509"
"encoding/csv"
"encoding/hex"
"fmt"
"html/template"
"io"
"io/ioutil"
"math/rand"
"net"
@ -21,11 +17,14 @@ import (
"strings"
"time"
certutil "github.com/rancher/dynamiclistener/cert"
// registering k3s cloud provider
_ "github.com/rancher/k3s/pkg/cloudprovider"
certutil "github.com/rancher/dynamiclistener/cert"
"github.com/rancher/k3s/pkg/cluster"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/kine/pkg/client"
"github.com/rancher/k3s/pkg/passwd"
"github.com/rancher/k3s/pkg/token"
"github.com/rancher/kine/pkg/endpoint"
"github.com/rancher/wrangler-api/pkg/generated/controllers/rbac"
"github.com/sirupsen/logrus"
@ -121,6 +120,10 @@ func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) {
"cluster-signing-cert-file": runtime.ServerCA,
"cluster-signing-key-file": runtime.ServerCAKey,
}
offset := cfg.HTTPSPort - 6443
if offset > 0 {
argsMap["port"] = strconv.Itoa(10252 + offset)
}
if cfg.NoLeaderElect {
argsMap["leader-elect"] = "false"
}
@ -143,6 +146,10 @@ func scheduler(cfg *config.Control, runtime *config.ControlRuntime) {
"bind-address": "127.0.0.1",
"secure-port": "0",
}
offset := cfg.HTTPSPort - 6443
if offset > 0 {
argsMap["port"] = strconv.Itoa(10251 + offset)
}
if cfg.NoLeaderElect {
argsMap["leader-elect"] = "false"
}
@ -291,6 +298,8 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
runtime.ClientKubeAPIKey = path.Join(config.DataDir, "tls", "client-kube-apiserver.key")
runtime.ClientKubeProxyCert = path.Join(config.DataDir, "tls", "client-kube-proxy.crt")
runtime.ClientKubeProxyKey = path.Join(config.DataDir, "tls", "client-kube-proxy.key")
runtime.ClientK3sControllerCert = path.Join(config.DataDir, "tls", "client-k3s-controller.crt")
runtime.ClientK3sControllerKey = path.Join(config.DataDir, "tls", "client-k3s-controller.key")
runtime.ServingKubeAPICert = path.Join(config.DataDir, "tls", "serving-kube-apiserver.crt")
runtime.ServingKubeAPIKey = path.Join(config.DataDir, "tls", "serving-kube-apiserver.key")
@ -301,20 +310,14 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
runtime.ClientAuthProxyCert = path.Join(config.DataDir, "tls", "client-auth-proxy.crt")
runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key")
etcdClient, err := prepareStorageBackend(ctx, config)
if err != nil {
return err
}
defer etcdClient.Close()
if err := fetchBootstrapData(ctx, config, etcdClient); err != nil {
return err
}
if err := genCerts(config, runtime); err != nil {
return err
}
if err := cluster.New(config).Start(ctx); err != nil {
return err
}
if err := genServiceAccount(runtime); err != nil {
return err
}
@ -327,124 +330,44 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
return err
}
if err := storeBootstrapData(ctx, config, etcdClient); err != nil {
if err := prepareStorageBackend(ctx, config); err != nil {
return err
}
return readTokens(runtime)
}
func prepareStorageBackend(ctx context.Context, config *config.Control) (client.Client, error) {
func prepareStorageBackend(ctx context.Context, config *config.Control) error {
etcdConfig, err := endpoint.Listen(ctx, config.Storage)
if err != nil {
return nil, err
return err
}
config.Storage.Config = etcdConfig.TLSConfig
config.Storage.Endpoint = strings.Join(etcdConfig.Endpoints, ",")
config.NoLeaderElect = !etcdConfig.LeaderElect
return client.New(etcdConfig)
}
func readTokenFile(passwdFile string) (map[string]string, error) {
f, err := os.Open(passwdFile)
if err != nil {
return nil, err
}
defer f.Close()
reader := csv.NewReader(f)
reader.FieldsPerRecord = -1
tokens := map[string]string{}
for {
record, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
if len(record) < 2 {
continue
}
tokens[record[1]] = record[0]
}
return tokens, nil
return nil
}
func readTokens(runtime *config.ControlRuntime) error {
tokens, err := readTokenFile(runtime.PasswdFile)
tokens, err := passwd.Read(runtime.PasswdFile)
if err != nil {
return err
}
if nodeToken, ok := tokens["node"]; ok {
runtime.NodeToken = "node:" + nodeToken
if nodeToken, ok := tokens.Pass("node"); ok {
runtime.AgentToken = "node:" + nodeToken
}
if clientToken, ok := tokens["admin"]; ok {
if serverToken, ok := tokens.Pass("server"); ok {
runtime.AgentToken = "server:" + serverToken
}
if clientToken, ok := tokens.Pass("admin"); ok {
runtime.ClientToken = "admin:" + clientToken
}
return nil
}
func ensureNodeToken(config *config.Control, runtime *config.ControlRuntime) error {
if config.ClusterSecret == "" {
return nil
}
f, err := os.Open(runtime.PasswdFile)
if err != nil {
return err
}
defer f.Close()
records := [][]string{}
reader := csv.NewReader(f)
for {
record, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}
if len(record) < 3 {
return fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", runtime.PasswdFile, len(record))
}
if record[1] == "node" {
if record[0] == config.ClusterSecret {
return nil
}
record[0] = config.ClusterSecret
}
records = append(records, record)
}
f.Close()
return WritePasswords(runtime.PasswdFile, records)
}
func WritePasswords(passwdFile string, records [][]string) error {
out, err := os.Create(passwdFile + ".tmp")
if err != nil {
return err
}
defer out.Close()
if err := out.Chmod(0600); err != nil {
return err
}
if err := csv.NewWriter(out).WriteAll(records); err != nil {
return err
}
return os.Rename(passwdFile+".tmp", passwdFile)
}
func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.ControlRuntime) error {
if s, err := os.Stat(runtime.IPSECKey); err == nil && s.Size() > 0 {
psk, err := ioutil.ReadFile(runtime.IPSECKey)
@ -455,7 +378,7 @@ func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.Cont
return nil
}
psk, err := getToken(ipsecTokenSize)
psk, err := token.Random(ipsecTokenSize)
if err != nil {
return err
}
@ -468,42 +391,71 @@ func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.Cont
return nil
}
func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
if s, err := os.Stat(runtime.PasswdFile); err == nil && s.Size() > 0 {
return ensureNodeToken(config, runtime)
func migratePassword(p *passwd.Passwd) error {
server, _ := p.Pass("server")
node, _ := p.Pass("node")
if server == "" && node != "" {
return p.EnsureUser("server", "k3s:server", node)
}
adminToken, err := getToken(userTokenSize)
if err != nil {
return err
}
systemToken, err := getToken(userTokenSize)
if err != nil {
return err
}
nodeToken, err := getToken(userTokenSize)
if err != nil {
return err
}
if config.ClusterSecret != "" {
nodeToken = config.ClusterSecret
}
return WritePasswords(runtime.PasswdFile, [][]string{
{adminToken, "admin", "admin", "system:masters"},
{systemToken, "system", "system", "system:masters"},
{nodeToken, "node", "node", "system:masters"},
})
return nil
}
func getToken(size int) (string, error) {
token := make([]byte, size, size)
_, err := cryptorand.Read(token)
if err != nil {
return "", err
func getServerPass(passwd *passwd.Passwd, config *config.Control) (string, error) {
var (
err error
)
serverPass := config.Token
if serverPass == "" {
serverPass, _ = passwd.Pass("server")
}
return hex.EncodeToString(token), err
if serverPass == "" {
serverPass, err = token.Random(16)
if err != nil {
return "", err
}
}
return serverPass, nil
}
func getNodePass(config *config.Control, serverPass string) string {
if config.AgentToken == "" {
return serverPass
}
return config.AgentToken
}
func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
passwd, err := passwd.Read(runtime.PasswdFile)
if err != nil {
return err
}
if err := migratePassword(passwd); err != nil {
return err
}
serverPass, err := getServerPass(passwd, config)
if err != nil {
return err
}
nodePass := getNodePass(config, serverPass)
if err := passwd.EnsureUser("admin", "system:masters", ""); err != nil {
return err
}
if err := passwd.EnsureUser("node", "k3s:agent", nodePass); err != nil {
return err
}
if err := passwd.EnsureUser("server", "k3s:server", serverPass); err != nil {
return err
}
return passwd.Write(runtime.PasswdFile)
}
func genCerts(config *config.Control, runtime *config.ControlRuntime) error {
@ -578,7 +530,10 @@ func genClientCerts(config *config.Control, runtime *config.ControlRuntime) erro
}
}
if _, err = factory("system:kube-proxy", []string{"system:nodes"}, runtime.ClientKubeProxyCert, runtime.ClientKubeProxyKey); err != nil {
if _, err = factory("system:kube-proxy", nil, runtime.ClientKubeProxyCert, runtime.ClientKubeProxyKey); err != nil {
return err
}
if _, err = factory("system:k3s-controller", nil, runtime.ClientK3sControllerCert, runtime.ClientK3sControllerKey); err != nil {
return err
}

12
pkg/daemons/control/tunnel.go
View File

@ -4,6 +4,7 @@ import (
"context"
"net"
"net/http"
"strings"
"time"
"github.com/rancher/remotedialer"
@ -37,14 +38,9 @@ func authorizer(req *http.Request) (clientKey string, authed bool, err error) {
return "", false, nil
}
if user.GetName() != "node" {
return "", false, nil
if strings.HasPrefix(user.GetName(), "system:node:") {
return strings.TrimPrefix(user.GetName(), "system:node:"), true, nil
}
nodeName := req.Header.Get("X-K3s-NodeName")
if nodeName == "" {
return "", false, nil
}
return nodeName, true, nil
return "", false, nil
}

3
pkg/datadir/datadir.go
View File

@ -2,6 +2,7 @@ package datadir
import (
"os"
"path/filepath"
"github.com/pkg/errors"
"github.com/rancher/wrangler/pkg/resolvehome"
@ -32,5 +33,5 @@ func LocalHome(dataDir string, forceLocal bool) (string, error) {
return "", errors.Wrapf(err, "resolving %s", dataDir)
}
return dataDir, nil
return filepath.Abs(dataDir)
}

151
pkg/passwd/passwd.go Normal file
View File

@ -0,0 +1,151 @@
package passwd
import (
"encoding/csv"
"fmt"
"io"
"os"
"strings"
"github.com/rancher/k3s/pkg/token"
)
type entry struct {
pass string
role string
}
type Passwd struct {
changed bool
names map[string]entry
}
func Read(file string) (*Passwd, error) {
result := &Passwd{
names: map[string]entry{},
}
f, err := os.Open(file)
if err != nil {
if os.IsNotExist(err) {
return result, nil
}
return nil, err
}
defer f.Close()
reader := csv.NewReader(f)
reader.FieldsPerRecord = -1
for {
record, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
if len(record) < 3 {
return nil, fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", file, len(record))
}
e := entry{
pass: record[0],
}
if len(record) > 3 {
e.role = record[3]
}
result.names[record[1]] = e
}
return result, nil
}
func (p *Passwd) Check(name, pass string) (matches bool, exists bool) {
e, ok := p.names[name]
if !ok {
return false, false
}
return e.pass == pass, true
}
func (p *Passwd) EnsureUser(name, role, passwd string) error {
tokenPrefix := "::" + name + ":"
idx := strings.Index(passwd, tokenPrefix)
if idx > 0 && strings.HasPrefix(passwd, "K10") {
passwd = passwd[idx+len(tokenPrefix):]
}
if e, ok := p.names[name]; ok {
if passwd != "" && e.pass != passwd {
p.changed = true
e.pass = passwd
}
if e.role != role {
p.changed = true
e.role = role
}
p.names[name] = e
return nil
}
if passwd == "" {
token, err := token.Random(16)
if err != nil {
return err
}
passwd = token
}
p.changed = true
p.names[name] = entry{
pass: passwd,
role: role,
}
return nil
}
func (p *Passwd) Pass(name string) (string, bool) {
e, ok := p.names[name]
if !ok {
return "", false
}
return e.pass, true
}
func (p *Passwd) Write(passwdFile string) error {
if !p.changed {
return nil
}
var records [][]string
for name, e := range p.names {
records = append(records, []string{
e.pass,
name,
name,
e.role,
})
}
return writePasswords(passwdFile, records)
}
func writePasswords(passwdFile string, records [][]string) error {
out, err := os.Create(passwdFile + ".tmp")
if err != nil {
return err
}
defer out.Close()
if err := out.Chmod(0600); err != nil {
return err
}
if err := csv.NewWriter(out).WriteAll(records); err != nil {
return err
}
return os.Rename(passwdFile+".tmp", passwdFile)
}

22
pkg/server/auth.go
View File

@ -9,9 +9,21 @@ import (
"k8s.io/apiserver/pkg/endpoints/request"
)
func doAuth(serverConfig *config.Control, next http.Handler, rw http.ResponseWriter, req *http.Request) {
func hasRole(mustRoles []string, roles []string) bool {
for _, check := range roles {
for _, role := range mustRoles {
if role == check {
return true
}
}
}
return false
}
func doAuth(roles []string, serverConfig *config.Control, next http.Handler, rw http.ResponseWriter, req *http.Request) {
if serverConfig == nil || serverConfig.Runtime.Authenticator == nil {
next.ServeHTTP(rw, req)
logrus.Errorf("authenticate not initialized")
rw.WriteHeader(http.StatusUnauthorized)
return
}
@ -22,7 +34,7 @@ func doAuth(serverConfig *config.Control, next http.Handler, rw http.ResponseWri
return
}
if !ok || resp.User.GetName() != "node" {
if !ok || !hasRole(roles, resp.User.GetGroups()) {
rw.WriteHeader(http.StatusUnauthorized)
return
}
@ -32,10 +44,10 @@ func doAuth(serverConfig *config.Control, next http.Handler, rw http.ResponseWri
next.ServeHTTP(rw, req)
}
func authMiddleware(serverConfig *config.Control) mux.MiddlewareFunc {
func authMiddleware(serverConfig *config.Control, roles ...string) mux.MiddlewareFunc {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
doAuth(serverConfig, next, rw, req)
doAuth(roles, serverConfig, next, rw, req)
})
}
}

1
pkg/server/context.go
View File

@ -62,7 +62,6 @@ func crds(ctx context.Context, config *rest.Config) error {
}
factory.BatchCreateCRDs(ctx, crd.NamespacedTypes(
"ListenerConfig.k3s.cattle.io/v1",
"Addon.k3s.cattle.io/v1",
"HelmChart.helm.cattle.io/v1")...)

91
pkg/server/router.go
View File

@ -3,70 +3,68 @@ package server
import (
"crypto"
"crypto/x509"
"encoding/csv"
"errors"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/gorilla/mux"
certutil "github.com/rancher/dynamiclistener/cert"
"github.com/rancher/k3s/pkg/bootstrap"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/daemons/control"
"github.com/rancher/k3s/pkg/passwd"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/util/json"
)
const (
jsonMediaType = "application/json"
binaryMediaType = "application/octet-stream"
pbMediaType = "application/com.github.proto-openapi.spec.v2@v1.0+protobuf"
openapiPrefix = "openapi."
staticURL = "/static/"
staticURL = "/static/"
)
type CACertsGetter func() (string, error)
func router(serverConfig *config.Control, tunnel http.Handler, cacertsGetter CACertsGetter) http.Handler {
func router(serverConfig *config.Control, tunnel http.Handler, ca []byte) http.Handler {
authed := mux.NewRouter()
authed.Use(authMiddleware(serverConfig))
authed.Use(authMiddleware(serverConfig, "k3s:agent"))
authed.NotFoundHandler = serverConfig.Runtime.Handler
authed.Path("/v1-k3s/connect").Handler(tunnel)
authed.Path("/v1-k3s/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig))
authed.Path("/v1-k3s/serving-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ServingKubeletKey))
authed.Path("/v1-k3s/client-kubelet.crt").Handler(clientKubeletCert(serverConfig))
authed.Path("/v1-k3s/client-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeletKey))
authed.Path("/v1-k3s/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert))
authed.Path("/v1-k3s/client-kube-proxy.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyKey))
authed.Path("/v1-k3s/client-k3s-controller.crt").Handler(fileHandler(serverConfig.Runtime.ClientK3sControllerCert))
authed.Path("/v1-k3s/client-k3s-controller.key").Handler(fileHandler(serverConfig.Runtime.ClientK3sControllerKey))
authed.Path("/v1-k3s/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA))
authed.Path("/v1-k3s/server-ca.crt").Handler(fileHandler(serverConfig.Runtime.ServerCA))
authed.Path("/v1-k3s/config").Handler(configHandler(serverConfig))
nodeAuthed := mux.NewRouter()
nodeAuthed.Use(authMiddleware(serverConfig, "system:nodes"))
nodeAuthed.Path("/v1-k3s/connect").Handler(tunnel)
nodeAuthed.NotFoundHandler = authed
serverAuthed := mux.NewRouter()
serverAuthed.Use(authMiddleware(serverConfig, "k3s:server"))
serverAuthed.NotFoundHandler = nodeAuthed
serverAuthed.Path("/v1-k3s/server-bootstrap").Handler(bootstrap.Handler(&serverConfig.Runtime.ControlRuntimeBootstrap))
staticDir := filepath.Join(serverConfig.DataDir, "static")
router := mux.NewRouter()
router.NotFoundHandler = authed
router.NotFoundHandler = serverAuthed
router.PathPrefix(staticURL).Handler(serveStatic(staticURL, staticDir))
router.Path("/cacerts").Handler(cacerts(cacertsGetter))
router.Path("/cacerts").Handler(cacerts(ca))
router.Path("/ping").Handler(ping())
return router
}
func cacerts(getter CACertsGetter) http.Handler {
func cacerts(ca []byte) http.Handler {
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
content, err := getter()
if err != nil {
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(err.Error()))
}
resp.Header().Set("content-type", "text/plain")
resp.Write([]byte(content))
resp.Write(ca)
})
}
@ -242,38 +240,19 @@ func sendError(err error, resp http.ResponseWriter, status ...int) {
resp.Write([]byte(err.Error()))
}
func ensureNodePassword(passwdFile, nodeName, passwd string) error {
records := [][]string{}
if _, err := os.Stat(passwdFile); !os.IsNotExist(err) {
f, err := os.Open(passwdFile)
if err != nil {
return err
}
defer f.Close()
reader := csv.NewReader(f)
for {
record, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}
if len(record) < 2 {
return fmt.Errorf("password file '%s' must have at least 2 columns (password, nodeName), found %d", passwdFile, len(record))
}
if record[1] == nodeName {
if record[0] == passwd {
return nil
}
return fmt.Errorf("Node password validation failed for '%s', using passwd file '%s'", nodeName, passwdFile)
}
records = append(records, record)
}
f.Close()
func ensureNodePassword(passwdFile, nodeName, pass string) error {
passwd, err := passwd.Read(passwdFile)
if err != nil {
return err
}
records = append(records, []string{passwd, nodeName})
return control.WritePasswords(passwdFile, records)
match, exists := passwd.Check(nodeName, pass)
if exists {
if !match {
return fmt.Errorf("Node password validation failed for '%s', using passwd file '%s'", nodeName, passwdFile)
}
return nil
}
// If user doesn't exist we save this password for future validation
passwd.EnsureUser(nodeName, "", pass)
return passwd.Write(passwdFile)
}

143
pkg/server/server.go
View File

@ -14,7 +14,6 @@ import (
"time"
"github.com/pkg/errors"
"github.com/rancher/dynamiclistener"
"github.com/rancher/helm-controller/pkg/helm"
"github.com/rancher/k3s/pkg/clientaccess"
"github.com/rancher/k3s/pkg/daemons/config"
@ -25,10 +24,11 @@ import (
"github.com/rancher/k3s/pkg/rootlessports"
"github.com/rancher/k3s/pkg/servicelb"
"github.com/rancher/k3s/pkg/static"
"github.com/rancher/k3s/pkg/tls"
v1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
"github.com/rancher/wrangler/pkg/leader"
"github.com/rancher/wrangler/pkg/resolvehome"
"github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/net"
)
@ -39,79 +39,67 @@ func resolveDataDir(dataDir string) (string, error) {
return filepath.Join(dataDir, "server"), err
}
func StartServer(ctx context.Context, config *Config) (string, error) {
func StartServer(ctx context.Context, config *Config) error {
if err := setupDataDirAndChdir(&config.ControlConfig); err != nil {
return "", err
return err
}
if err := setNoProxyEnv(&config.ControlConfig); err != nil {
return "", err
return err
}
if err := control.Server(ctx, &config.ControlConfig); err != nil {
return "", errors.Wrap(err, "starting kubernetes")
return errors.Wrap(err, "starting kubernetes")
}
certs, err := startWrangler(ctx, config)
if err != nil {
return "", errors.Wrap(err, "starting tls server")
if err := startWrangler(ctx, config); err != nil {
return errors.Wrap(err, "starting tls server")
}
ip := net2.ParseIP(config.TLSConfig.BindAddress)
ip := net2.ParseIP(config.ControlConfig.BindAddress)
if ip == nil {
ip, err = net.ChooseHostInterface()
if err != nil {
hostIP, err := net.ChooseHostInterface()
if err == nil {
ip = hostIP
} else {
ip = net2.ParseIP("127.0.0.1")
}
}
printTokens(certs, ip.String(), &config.TLSConfig, &config.ControlConfig)
writeKubeConfig(certs, &config.TLSConfig, config)
if err := printTokens(ip.String(), &config.ControlConfig); err != nil {
return err
}
return certs, nil
return writeKubeConfig(config.ControlConfig.Runtime.ServerCA, config)
}
func startWrangler(ctx context.Context, config *Config) (string, error) {
func startWrangler(ctx context.Context, config *Config) error {
var (
err error
tlsConfig = &config.TLSConfig
controlConfig = &config.ControlConfig
)
caBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCA)
ca, err := ioutil.ReadFile(config.ControlConfig.Runtime.ServerCA)
if err != nil {
return "", err
}
caKeyBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCAKey)
if err != nil {
return "", err
return err
}
certs := string(caBytes)
tlsConfig.CACerts = certs
tlsConfig.CAKey = string(caKeyBytes)
tlsConfig.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, func() (string, error) {
return certs, nil
})
controlConfig.Runtime.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, ca)
sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigAdmin)
if err != nil {
return "", err
return err
}
if err := stageFiles(ctx, sc, controlConfig); err != nil {
return "", err
return err
}
_, err = tls.NewServer(ctx, sc.K3s.K3s().V1().ListenerConfig(), *tlsConfig)
if err != nil {
return "", err
if err := sc.Start(ctx); err != nil {
return err
}
if err := startNodeCache(ctx, sc); err != nil {
return "", err
}
controlConfig.Runtime.Core = sc.Core
start := func(ctx context.Context) {
if err := masterControllers(ctx, sc, config); err != nil {
@ -122,7 +110,7 @@ func startWrangler(ctx context.Context, config *Config) (string, error) {
}
}
if !config.DisableAgent {
go setMasterRoleLabel(ctx, sc)
go setMasterRoleLabel(ctx, sc.Core.Core().V1().Node())
}
if controlConfig.NoLeaderElect {
go func() {
@ -134,7 +122,7 @@ func startWrangler(ctx context.Context, config *Config) (string, error) {
go leader.RunOrDie(ctx, "", "k3s", sc.K8s, start)
}
return certs, nil
return nil
}
func masterControllers(ctx context.Context, sc *Context, config *Config) error {
@ -162,7 +150,7 @@ func masterControllers(ctx context.Context, sc *Context, config *Config) error {
}
if !config.DisableServiceLB && config.Rootless {
return rootlessports.Register(ctx, sc.Core.Core().V1().Service(), config.TLSConfig.HTTPSPort)
return rootlessports.Register(ctx, sc.Core.Core().V1().Service(), config.ControlConfig.HTTPSPort)
}
return nil
@ -203,7 +191,7 @@ func HomeKubeConfig(write, rootless bool) (string, error) {
return resolvehome.Resolve(datadir.HomeConfig)
}
func printTokens(certs, advertiseIP string, tlsConfig *dynamiclistener.UserConfig, config *config.Control) {
func printTokens(advertiseIP string, config *config.Control) error {
var (
nodeFile string
)
@ -212,26 +200,43 @@ func printTokens(certs, advertiseIP string, tlsConfig *dynamiclistener.UserConfi
advertiseIP = "127.0.0.1"
}
if len(config.Runtime.NodeToken) > 0 {
p := filepath.Join(config.DataDir, "node-token")
if err := writeToken(config.Runtime.NodeToken, p, certs); err == nil {
if len(config.Runtime.AgentToken) > 0 {
p := filepath.Join(config.DataDir, "token")
if err := writeToken(config.Runtime.AgentToken, p, config.Runtime.ServerCA); err == nil {
logrus.Infof("Node token is available at %s", p)
nodeFile = p
}
// backwards compatibility
np := filepath.Join(config.DataDir, "node-token")
if !isSymlink(np) {
if err := os.RemoveAll(np); err != nil {
return err
}
if err := os.Symlink(p, np); err != nil {
return err
}
}
}
if len(nodeFile) > 0 {
printToken(tlsConfig.HTTPSPort, advertiseIP, "To join node to cluster:", "agent")
printToken(config.HTTPSPort, advertiseIP, "To join node to cluster:", "agent")
}
return nil
}
func writeKubeConfig(certs string, tlsConfig *dynamiclistener.UserConfig, config *Config) {
clientToken := FormatToken(config.ControlConfig.Runtime.ClientToken, certs)
ip := tlsConfig.BindAddress
func writeKubeConfig(certs string, config *Config) error {
clientToken, err := FormatToken(config.ControlConfig.Runtime.ClientToken, certs)
if err != nil {
return err
}
ip := config.ControlConfig.BindAddress
if ip == "" {
ip = "127.0.0.1"
}
url := fmt.Sprintf("https://%s:%d", ip, tlsConfig.HTTPSPort)
url := fmt.Sprintf("https://%s:%d", ip, config.ControlConfig.HTTPSPort)
kubeConfig, err := HomeKubeConfig(true, config.Rootless)
def := true
if err != nil {
@ -274,6 +279,8 @@ func writeKubeConfig(certs string, tlsConfig *dynamiclistener.UserConfig, config
if def {
logrus.Infof("Run: %s kubectl", filepath.Base(os.Args[0]))
}
return nil
}
func setupDataDirAndChdir(config *config.Control) error {
@ -312,18 +319,22 @@ func printToken(httpsPort int, advertiseIP, prefix, cmd string) {
logrus.Infof("%s k3s %s -s https://%s:%d -t ${NODE_TOKEN}", prefix, cmd, ip, httpsPort)
}
func FormatToken(token string, certs string) string {
func FormatToken(token string, certFile string) (string, error) {
if len(token) == 0 {
return token
return token, nil
}
prefix := "K10"
if len(certs) > 0 {
digest := sha256.Sum256([]byte(certs))
if len(certFile) > 0 {
bytes, err := ioutil.ReadFile(certFile)
if err != nil {
return "", nil
}
digest := sha256.Sum256(bytes)
prefix = "K10" + hex.EncodeToString(digest[:]) + "::"
}
return prefix + token
return prefix + token, nil
}
func writeToken(token, file, certs string) error {
@ -331,7 +342,10 @@ func writeToken(token, file, certs string) error {
return nil
}
token = FormatToken(token, certs)
token, err := FormatToken(token, certs)
if err != nil {
return err
}
return ioutil.WriteFile(file, []byte(token+"\n"), 0600)
}
@ -364,26 +378,23 @@ func isSymlink(config string) bool {
return false
}
func setMasterRoleLabel(ctx context.Context, sc *Context) error {
func setMasterRoleLabel(ctx context.Context, nodes v1.NodeClient) error {
for {
nodeName := os.Getenv("NODE_NAME")
nodeController := sc.Core.Core().V1().Node()
nodeCache := nodeController.Cache()
nodeCached, err := nodeCache.Get(nodeName)
node, err := nodes.Get(nodeName, metav1.GetOptions{})
if err != nil {
logrus.Infof("Waiting for master node %s startup: %v", nodeName, err)
time.Sleep(1 * time.Second)
continue
}
if v, ok := nodeCached.Labels[MasterRoleLabelKey]; ok && v == "true" {
if v, ok := node.Labels[MasterRoleLabelKey]; ok && v == "true" {
break
}
node := nodeCached.DeepCopy()
if node.Labels == nil {
node.Labels = make(map[string]string)
}
node.Labels[MasterRoleLabelKey] = "true"
_, err = nodeController.Update(node)
_, err = nodes.Update(node)
if err == nil {
logrus.Infof("master role label has been set succesfully on node: %s", nodeName)
break
@ -396,9 +407,3 @@ func setMasterRoleLabel(ctx context.Context, sc *Context) error {
}
return nil
}
func startNodeCache(ctx context.Context, sc *Context) error {
sc.Core.Core().V1().Node().Cache()
return sc.Start(ctx)
}

2
pkg/server/types.go
View File

@ -1,14 +1,12 @@
package server
import (
"github.com/rancher/dynamiclistener"
"github.com/rancher/k3s/pkg/daemons/config"
)
type Config struct {
DisableAgent bool
DisableServiceLB bool
TLSConfig dynamiclistener.UserConfig
ControlConfig config.Control
Rootless bool
}

39
pkg/token/read.go Normal file
View File

@ -0,0 +1,39 @@
package token
import (
cryptorand "crypto/rand"
"encoding/hex"
"io/ioutil"
"os"
"strings"
"time"
"github.com/sirupsen/logrus"
)
func Random(size int) (string, error) {
token := make([]byte, size, size)
_, err := cryptorand.Read(token)
if err != nil {
return "", err
}
return hex.EncodeToString(token), err
}
func ReadFile(path string) (string, error) {
if path == "" {
return "", nil
}
for {
tokenBytes, err := ioutil.ReadFile(path)
if err == nil {
return strings.TrimSpace(string(tokenBytes)), nil
} else if os.IsNotExist(err) {
logrus.Infof("Waiting for %s to be available\n", path)
time.Sleep(2 * time.Second)
} else {
return "", err
}
}
}