diff --git a/manifests/rolebindings.yaml b/manifests/rolebindings.yaml index d048eee998..36bc949367 100644 --- a/manifests/rolebindings.yaml +++ b/manifests/rolebindings.yaml @@ -10,3 +10,53 @@ subjects: - apiGroup: rbac.authorization.k8s.io kind: User name: kube-apiserver + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:k3s-controller +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - namespaces + verbs: + - list + - watch +- apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - endpoints + - pods + verbs: + - list + - get + - watch + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:k3s-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:k3s-controller +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: User + name: system:k3s-controller diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 0a001b18c4..1dba940439 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -6,7 +6,6 @@ import ( cryptorand "crypto/rand" "crypto/tls" "encoding/hex" - "encoding/pem" "fmt" "io/ioutil" sysnet "net" @@ -27,7 +26,6 @@ import ( "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/net" - "k8s.io/client-go/util/cert" "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1" ) @@ -183,17 +181,6 @@ func getHostnameAndIP(info cmds.Agent) (string, string, error) { return name, ip, nil } -func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, tlsCert *tls.Certificate) (string, error) { - os.MkdirAll(envInfo.DataDir, 0700) - kubeConfigPath := filepath.Join(envInfo.DataDir, "kubeconfig.yaml") - info.CACerts = pem.EncodeToMemory(&pem.Block{ - Type: cert.CertificateBlockType, - Bytes: tlsCert.Certificate[1], - }) - - return kubeConfigPath, info.WriteKubeConfig(kubeConfigPath) -} - func isValidResolvConf(resolvConfFile string) bool { file, err := os.Open(resolvConfFile) if err != nil { @@ -293,11 +280,6 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { return nil, err } - kubeconfigNode, err := writeKubeConfig(envInfo, *info, servingCert) - if err != nil { - return nil, err - } - clientKubeletCert := filepath.Join(envInfo.DataDir, "client-kubelet.crt") if err := getNodeNamedHostFile(clientKubeletCert, nodeName, nodePasswordFile, info); err != nil { return nil, err @@ -328,6 +310,21 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { return nil, err } + clientK3sControllerCert := filepath.Join(envInfo.DataDir, "client-k3s-controller.crt") + if err := getHostFile(clientK3sControllerCert, info); err != nil { + return nil, err + } + + clientK3sControllerKey := filepath.Join(envInfo.DataDir, "client-k3s-controller.key") + if err := getHostFile(clientK3sControllerKey, info); err != nil { + return nil, err + } + + kubeconfigK3sController := filepath.Join(envInfo.DataDir, "k3scontroller.kubeconfig") + if err := control.KubeConfig(kubeconfigK3sController, info.URL, serverCAFile, clientK3sControllerCert, clientK3sControllerKey); err != nil { + return nil, err + } + nodeConfig := &config.Node{ Docker: envInfo.Docker, ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint, @@ -345,9 +342,9 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo) nodeConfig.AgentConfig.ClientCA = clientCAFile nodeConfig.AgentConfig.ListenAddress = "0.0.0.0" - nodeConfig.AgentConfig.KubeConfigNode = kubeconfigNode nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy + nodeConfig.AgentConfig.KubeConfigK3sController = kubeconfigK3sController if envInfo.Rootless { nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet") } diff --git a/pkg/agent/flannel/setup.go b/pkg/agent/flannel/setup.go index bc354ef92e..d0101da279 100644 --- a/pkg/agent/flannel/setup.go +++ b/pkg/agent/flannel/setup.go @@ -13,8 +13,7 @@ import ( "github.com/rancher/k3s/pkg/daemons/config" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" + v1 "k8s.io/client-go/kubernetes/typed/core/v1" ) const ( @@ -74,21 +73,11 @@ func Prepare(ctx context.Context, nodeConfig *config.Node) error { return createFlannelConf(nodeConfig) } -func Run(ctx context.Context, nodeConfig *config.Node) error { +func Run(ctx context.Context, nodeConfig *config.Node, nodes v1.NodeInterface) error { nodeName := nodeConfig.AgentConfig.NodeName - restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigNode) - if err != nil { - return err - } - - client, err := kubernetes.NewForConfig(restConfig) - if err != nil { - return err - } - for { - node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) + node, err := nodes.Get(nodeName, metav1.GetOptions{}) if err == nil && node.Spec.PodCIDR != "" { break } @@ -101,11 +90,11 @@ func Run(ctx context.Context, nodeConfig *config.Node) error { } go func() { - err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigNode) + err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigKubelet) logrus.Fatalf("flannel exited: %v", err) }() - return err + return nil } func createCNIConf(dir string) error { diff --git a/pkg/agent/netpol/network_policy.go b/pkg/agent/netpol/network_policy.go index 97eac08a90..65e93c6abe 100644 --- a/pkg/agent/netpol/network_policy.go +++ b/pkg/agent/netpol/network_policy.go @@ -10,7 +10,7 @@ import ( ) func Run(ctx context.Context, nodeConfig *config.Node) error { - restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigNode) + restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigK3sController) if err != nil { return err } diff --git a/pkg/agent/run.go b/pkg/agent/run.go index aaf24976c7..7bdb358dbe 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -9,6 +9,7 @@ import ( "strings" "time" + systemd "github.com/coreos/go-systemd/daemon" "github.com/rancher/k3s/pkg/agent/config" "github.com/rancher/k3s/pkg/agent/containerd" "github.com/rancher/k3s/pkg/agent/flannel" @@ -21,10 +22,11 @@ import ( "github.com/rancher/k3s/pkg/daemons/agent" daemonconfig "github.com/rancher/k3s/pkg/daemons/config" "github.com/rancher/k3s/pkg/rootless" - "github.com/rancher/wrangler-api/pkg/generated/controllers/core" - corev1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1" - "github.com/rancher/wrangler/pkg/start" "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + v1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/clientcmd" ) @@ -64,14 +66,18 @@ func run(ctx context.Context, cfg cmds.Agent, lb *loadbalancer.LoadBalancer) err return err } + coreClient, err := coreClient(nodeConfig.AgentConfig.KubeConfigKubelet) + if err != nil { + return err + } if !nodeConfig.NoFlannel { - if err := flannel.Run(ctx, nodeConfig); err != nil { + if err := flannel.Run(ctx, nodeConfig, coreClient.CoreV1().Nodes()); err != nil { return err } } if !nodeConfig.AgentConfig.DisableCCM { - if err := syncAddressesLabels(ctx, &nodeConfig.AgentConfig); err != nil { + if err := syncAddressesLabels(ctx, &nodeConfig.AgentConfig, coreClient.CoreV1().Nodes()); err != nil { return err } } @@ -86,6 +92,15 @@ func run(ctx context.Context, cfg cmds.Agent, lb *loadbalancer.LoadBalancer) err return ctx.Err() } +func coreClient(cfg string) (kubernetes.Interface, error) { + restConfig, err := clientcmd.BuildConfigFromFlags("", cfg) + if err != nil { + return nil, err + } + + return kubernetes.NewForConfig(restConfig) +} + func Run(ctx context.Context, cfg cmds.Agent) error { if err := validate(); err != nil { return err @@ -100,10 +115,6 @@ func Run(ctx context.Context, cfg cmds.Agent) error { cfg.DataDir = filepath.Join(cfg.DataDir, "agent") os.MkdirAll(cfg.DataDir, 0700) - if cfg.ClusterSecret != "" { - cfg.Token = "K10node:" + cfg.ClusterSecret - } - lb, err := loadbalancer.Setup(ctx, cfg) if err != nil { return err @@ -113,7 +124,7 @@ func Run(ctx context.Context, cfg cmds.Agent) error { } for { - tmpFile, err := clientaccess.AgentAccessInfoToTempKubeConfig("", cfg.ServerURL, cfg.Token) + newToken, err := clientaccess.NormalizeAndValidateTokenForUser(cfg.ServerURL, cfg.Token, "node") if err != nil { logrus.Error(err) select { @@ -123,10 +134,11 @@ func Run(ctx context.Context, cfg cmds.Agent) error { } continue } - os.Remove(tmpFile) + cfg.Token = newToken break } + systemd.SdNotify(true, "READY=1\n") return run(ctx, cfg, lb) } @@ -149,73 +161,51 @@ func validate() error { return nil } -func syncAddressesLabels(ctx context.Context, agentConfig *daemonconfig.Agent) error { +func syncAddressesLabels(ctx context.Context, agentConfig *daemonconfig.Agent, nodes v1.NodeInterface) error { for { - nodeController, nodeCache, err := startNodeController(ctx, agentConfig) + node, err := nodes.Get(agentConfig.NodeName, metav1.GetOptions{}) if err != nil { logrus.Infof("Waiting for kubelet to be ready on node %s: %v", agentConfig.NodeName, err) time.Sleep(1 * time.Second) continue } - nodeCached, err := nodeCache.Get(agentConfig.NodeName) - if err != nil { - logrus.Infof("Waiting for kubelet to be ready on node %s: %v", agentConfig.NodeName, err) - time.Sleep(1 * time.Second) - continue - } - node := nodeCached.DeepCopy() - updated := updateLabelMap(ctx, agentConfig, node.Labels) - if updated { - _, err = nodeController.Update(node) - if err == nil { - logrus.Infof("addresses labels has been set succesfully on node: %s", agentConfig.NodeName) - break + + newLabels, update := updateLabelMap(agentConfig, node.Labels) + if update { + node.Labels = newLabels + if _, err := nodes.Update(node); err != nil { + logrus.Infof("Failed to update node %s: %v", agentConfig.NodeName, err) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + continue + } } - logrus.Infof("Failed to update node %s: %v", agentConfig.NodeName, err) - time.Sleep(1 * time.Second) - continue + logrus.Infof("addresses labels has been set successfully on node: %s", agentConfig.NodeName) + } else { + logrus.Infof("addresses labels has already been set successfully on node: %s", agentConfig.NodeName) } - logrus.Infof("addresses labels has already been set succesfully on node: %s", agentConfig.NodeName) - return nil + + break } + return nil } -func startNodeController(ctx context.Context, agentConfig *daemonconfig.Agent) (corev1.NodeController, corev1.NodeCache, error) { - restConfig, err := clientcmd.BuildConfigFromFlags("", agentConfig.KubeConfigKubelet) - if err != nil { - return nil, nil, err - } - coreFactory := core.NewFactoryFromConfigOrDie(restConfig) - nodeController := coreFactory.Core().V1().Node() - nodeCache := nodeController.Cache() - if err := start.All(ctx, 1, coreFactory); err != nil { - return nil, nil, err +func updateLabelMap(agentConfig *daemonconfig.Agent, nodeLabels map[string]string) (map[string]string, bool) { + result := map[string]string{} + for k, v := range nodeLabels { + result[k] = v } - return nodeController, nodeCache, nil -} + result[InternalIPLabel] = agentConfig.NodeIP + result[HostnameLabel] = agentConfig.NodeName + if agentConfig.NodeExternalIP == "" { + delete(result, ExternalIPLabel) + } else { + result[ExternalIPLabel] = agentConfig.NodeExternalIP + } -func updateLabelMap(ctx context.Context, agentConfig *daemonconfig.Agent, nodeLabels map[string]string) bool { - if nodeLabels == nil { - nodeLabels = make(map[string]string) - } - updated := false - if internalIPLabel, ok := nodeLabels[InternalIPLabel]; !ok || internalIPLabel != agentConfig.NodeIP { - nodeLabels[InternalIPLabel] = agentConfig.NodeIP - updated = true - } - if hostnameLabel, ok := nodeLabels[HostnameLabel]; !ok || hostnameLabel != agentConfig.NodeName { - nodeLabels[HostnameLabel] = agentConfig.NodeName - updated = true - } - nodeExternalIP := agentConfig.NodeExternalIP - if externalIPLabel := nodeLabels[ExternalIPLabel]; externalIPLabel != nodeExternalIP && nodeExternalIP != "" { - nodeLabels[ExternalIPLabel] = nodeExternalIP - updated = true - } else if nodeExternalIP == "" && externalIPLabel != "" { - delete(nodeLabels, ExternalIPLabel) - updated = true - } - return updated + return result, !equality.Semantic.DeepEqual(nodeLabels, result) } diff --git a/pkg/agent/tunnel/tunnel.go b/pkg/agent/tunnel/tunnel.go index c139129557..d219236b29 100644 --- a/pkg/agent/tunnel/tunnel.go +++ b/pkg/agent/tunnel/tunnel.go @@ -3,11 +3,8 @@ package tunnel import ( "context" "crypto/tls" - "crypto/x509" - "encoding/base64" "fmt" "net" - "net/http" "reflect" "sync" "time" @@ -21,8 +18,8 @@ import ( "k8s.io/apimachinery/pkg/fields" watchtypes "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/transport" ) var ( @@ -54,12 +51,7 @@ func getAddresses(endpoint *v1.Endpoints) []string { } func Setup(ctx context.Context, config *config.Node, onChange func([]string)) error { - restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode) - if err != nil { - return err - } - - transportConfig, err := restConfig.TransportConfig() + restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigK3sController) if err != nil { return err } @@ -69,6 +61,16 @@ func Setup(ctx context.Context, config *config.Node, onChange func([]string)) er return err } + nodeRestConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigKubelet) + if err != nil { + return err + } + + tlsConfig, err := rest.TLSConfigFor(nodeRestConfig) + if err != nil { + return err + } + addresses := []string{config.ServerAddress} endpoint, _ := client.CoreV1().Endpoints("default").Get("kubernetes", metav1.GetOptions{}) @@ -84,7 +86,7 @@ func Setup(ctx context.Context, config *config.Node, onChange func([]string)) er wg := &sync.WaitGroup{} for _, address := range addresses { if _, ok := disconnect[address]; !ok { - disconnect[address] = connect(ctx, wg, address, config, transportConfig) + disconnect[address] = connect(ctx, wg, address, tlsConfig) } } @@ -132,7 +134,7 @@ func Setup(ctx context.Context, config *config.Node, onChange func([]string)) er for _, address := range addresses { validEndpoint[address] = true if _, ok := disconnect[address]; !ok { - disconnect[address] = connect(ctx, nil, address, config, transportConfig) + disconnect[address] = connect(ctx, nil, address, tlsConfig) } } @@ -164,25 +166,10 @@ func Setup(ctx context.Context, config *config.Node, onChange func([]string)) er return nil } -func connect(rootCtx context.Context, waitGroup *sync.WaitGroup, address string, config *config.Node, transportConfig *transport.Config) context.CancelFunc { +func connect(rootCtx context.Context, waitGroup *sync.WaitGroup, address string, tlsConfig *tls.Config) context.CancelFunc { wsURL := fmt.Sprintf("wss://%s/v1-k3s/connect", address) - headers := map[string][]string{ - "X-K3s-NodeName": {config.AgentConfig.NodeName}, - } - ws := &websocket.Dialer{} - - if len(config.CACerts) > 0 { - pool := x509.NewCertPool() - pool.AppendCertsFromPEM(config.CACerts) - ws.TLSClientConfig = &tls.Config{ - RootCAs: pool, - } - } - - if transportConfig.Username != "" { - auth := transportConfig.Username + ":" + transportConfig.Password - auth = base64.StdEncoding.EncodeToString([]byte(auth)) - headers["Authorization"] = []string{"Basic " + auth} + ws := &websocket.Dialer{ + TLSClientConfig: tlsConfig, } once := sync.Once{} @@ -194,7 +181,7 @@ func connect(rootCtx context.Context, waitGroup *sync.WaitGroup, address string, go func() { for { - remotedialer.ClientConnect(ctx, wsURL, http.Header(headers), ws, func(proto, address string) bool { + remotedialer.ClientConnect(ctx, wsURL, nil, ws, func(proto, address string) bool { host, port, err := net.SplitHostPort(address) return err == nil && proto == "tcp" && ports[port] && host == "127.0.0.1" }, func(_ context.Context) error { diff --git a/pkg/apis/k3s.cattle.io/v1/types.go b/pkg/apis/k3s.cattle.io/v1/types.go index 50353e7470..d1f1fed6cb 100644 --- a/pkg/apis/k3s.cattle.io/v1/types.go +++ b/pkg/apis/k3s.cattle.io/v1/types.go @@ -1,7 +1,6 @@ package v1 import ( - "github.com/rancher/dynamiclistener" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" ) @@ -9,16 +8,6 @@ import ( // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -type ListenerConfig struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Status dynamiclistener.ListenerStatus `json:"status,omitempty"` -} - -// +genclient -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - type Addon struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/pkg/bootstrap/bootstrap.go b/pkg/bootstrap/bootstrap.go new file mode 100644 index 0000000000..832d1a7061 --- /dev/null +++ b/pkg/bootstrap/bootstrap.go @@ -0,0 +1,80 @@ +package bootstrap + +import ( + "encoding/json" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + + "github.com/pkg/errors" + "github.com/rancher/k3s/pkg/daemons/config" +) + +func Handler(bootstrap *config.ControlRuntimeBootstrap) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + rw.Header().Set("Content-Type", "application/json") + Write(rw, bootstrap) + }) +} + +func Write(w io.Writer, bootstrap *config.ControlRuntimeBootstrap) error { + paths, err := objToMap(bootstrap) + if err != nil { + return nil + } + + dataMap := map[string][]byte{} + for pathKey, path := range paths { + if path == "" { + continue + } + data, err := ioutil.ReadFile(path) + if err != nil { + return errors.Wrapf(err, "failed to read %s", path) + } + + dataMap[pathKey] = data + } + + return json.NewEncoder(w).Encode(dataMap) +} + +func Read(r io.Reader, bootstrap *config.ControlRuntimeBootstrap) error { + paths, err := objToMap(bootstrap) + if err != nil { + return err + } + + files := map[string][]byte{} + if err := json.NewDecoder(r).Decode(&files); err != nil { + return err + } + + for pathKey, data := range files { + path, ok := paths[pathKey] + if !ok { + continue + } + + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return errors.Wrapf(err, "failed to mkdir %s", filepath.Dir(path)) + } + + if err := ioutil.WriteFile(path, data, 0600); err != nil { + return errors.Wrapf(err, "failed to write to %s", path) + } + } + + return nil +} + +func objToMap(obj interface{}) (map[string]string, error) { + bytes, err := json.Marshal(obj) + if err != nil { + return nil, err + } + data := map[string]string{} + return data, json.Unmarshal(bytes, &data) +} diff --git a/pkg/cli/agent/agent.go b/pkg/cli/agent/agent.go index bda1df8e35..40c4636a30 100644 --- a/pkg/cli/agent/agent.go +++ b/pkg/cli/agent/agent.go @@ -3,39 +3,18 @@ package agent import ( "context" "fmt" - "io/ioutil" "os" - "strings" - "time" - systemd "github.com/coreos/go-systemd/daemon" "github.com/rancher/k3s/pkg/agent" "github.com/rancher/k3s/pkg/cli/cmds" "github.com/rancher/k3s/pkg/datadir" "github.com/rancher/k3s/pkg/netutil" + "github.com/rancher/k3s/pkg/token" "github.com/rancher/wrangler/pkg/signals" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) -func readToken(path string) (string, error) { - if path == "" { - return "", nil - } - - for { - tokenBytes, err := ioutil.ReadFile(path) - if err == nil { - return strings.TrimSpace(string(tokenBytes)), nil - } else if os.IsNotExist(err) { - logrus.Infof("Waiting for %s to be available\n", path) - time.Sleep(2 * time.Second) - } else { - return "", err - } - } -} - func Run(ctx *cli.Context) error { if err := cmds.InitLogging(); err != nil { return err @@ -45,14 +24,14 @@ func Run(ctx *cli.Context) error { } if cmds.AgentConfig.TokenFile != "" { - token, err := readToken(cmds.AgentConfig.TokenFile) + token, err := token.ReadFile(cmds.AgentConfig.TokenFile) if err != nil { return err } cmds.AgentConfig.Token = token } - if cmds.AgentConfig.Token == "" && cmds.AgentConfig.ClusterSecret == "" { + if cmds.AgentConfig.Token == "" { return fmt.Errorf("--token is required") } @@ -76,7 +55,6 @@ func Run(ctx *cli.Context) error { cfg.DataDir = dataDir contextCtx := signals.SetupSignalHandler(context.Background()) - systemd.SdNotify(true, "READY=1\n") return agent.Run(contextCtx, cfg) } diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go index b376009c2a..8f43666476 100644 --- a/pkg/cli/cmds/agent.go +++ b/pkg/cli/cmds/agent.go @@ -17,7 +17,6 @@ type Agent struct { NodeIP string NodeExternalIP string NodeName string - ClusterSecret string PauseImage string Docker bool ContainerRuntimeEndpoint string @@ -44,82 +43,82 @@ var ( AgentConfig Agent NodeIPFlag = cli.StringFlag{ Name: "node-ip,i", - Usage: "(agent) IP address to advertise for node", + Usage: "(agent/networking) IP address to advertise for node", Destination: &AgentConfig.NodeIP, } NodeExternalIPFlag = cli.StringFlag{ Name: "node-external-ip", - Usage: "(agent) External IP address to advertise for node", + Usage: "(agent/networking) External IP address to advertise for node", Destination: &AgentConfig.NodeExternalIP, } NodeNameFlag = cli.StringFlag{ Name: "node-name", - Usage: "(agent) Node name", + Usage: "(agent/node) Node name", EnvVar: "K3S_NODE_NAME", Destination: &AgentConfig.NodeName, } DockerFlag = cli.BoolFlag{ Name: "docker", - Usage: "(agent) Use docker instead of containerd", + Usage: "(agent/runtime) Use docker instead of containerd", Destination: &AgentConfig.Docker, } + CRIEndpointFlag = cli.StringFlag{ + Name: "container-runtime-endpoint", + Usage: "(agent/runtime) Disable embedded containerd and use alternative CRI implementation", + Destination: &AgentConfig.ContainerRuntimeEndpoint, + } + PrivateRegistryFlag = cli.StringFlag{ + Name: "private-registry", + Usage: "(agent/runtime) Private registry configuration file", + Destination: &AgentConfig.PrivateRegistry, + Value: "/etc/rancher/k3s/registries.yaml", + } + PauseImageFlag = cli.StringFlag{ + Name: "pause-image", + Usage: "(agent/runtime) Customized pause image for containerd sandbox", + Destination: &AgentConfig.PauseImage, + } FlannelFlag = cli.BoolFlag{ Name: "no-flannel", - Usage: "(agent) Disable embedded flannel", + Usage: "(deprecated) use --flannel-backend=none", Destination: &AgentConfig.NoFlannel, } FlannelIfaceFlag = cli.StringFlag{ Name: "flannel-iface", - Usage: "(agent) Override default flannel interface", + Usage: "(agent/networking) Override default flannel interface", Destination: &AgentConfig.FlannelIface, } FlannelConfFlag = cli.StringFlag{ Name: "flannel-conf", - Usage: "(agent) (experimental) Override default flannel config file", + Usage: "(agent/networking) Override default flannel config file", Destination: &AgentConfig.FlannelConf, } - CRIEndpointFlag = cli.StringFlag{ - Name: "container-runtime-endpoint", - Usage: "(agent) Disable embedded containerd and use alternative CRI implementation", - Destination: &AgentConfig.ContainerRuntimeEndpoint, - } - PauseImageFlag = cli.StringFlag{ - Name: "pause-image", - Usage: "(agent) Customized pause image for containerd sandbox", - Destination: &AgentConfig.PauseImage, - } ResolvConfFlag = cli.StringFlag{ Name: "resolv-conf", - Usage: "(agent) Kubelet resolv.conf file", + Usage: "(agent/networking) Kubelet resolv.conf file", EnvVar: "K3S_RESOLV_CONF", Destination: &AgentConfig.ResolvConf, } ExtraKubeletArgs = cli.StringSliceFlag{ Name: "kubelet-arg", - Usage: "(agent) Customized flag for kubelet process", + Usage: "(agent/flags) Customized flag for kubelet process", Value: &AgentConfig.ExtraKubeletArgs, } ExtraKubeProxyArgs = cli.StringSliceFlag{ Name: "kube-proxy-arg", - Usage: "(agent) Customized flag for kube-proxy process", + Usage: "(agent/flags) Customized flag for kube-proxy process", Value: &AgentConfig.ExtraKubeProxyArgs, } NodeTaints = cli.StringSliceFlag{ Name: "node-taint", - Usage: "(agent) Registering kubelet with set of taints", + Usage: "(agent/node) Registering kubelet with set of taints", Value: &AgentConfig.Taints, } NodeLabels = cli.StringSliceFlag{ Name: "node-label", - Usage: "(agent) Registering kubelet with set of labels", + Usage: "(agent/node) Registering kubelet with set of labels", Value: &AgentConfig.Labels, } - PrivateRegistryFlag = cli.StringFlag{ - Name: "private-registry", - Usage: "(agent) Private registry configuration file", - Destination: &AgentConfig.PrivateRegistry, - Value: "/etc/rancher/k3s/registries.yaml", - } ) func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { @@ -135,54 +134,57 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { AlsoLogToStderr, cli.StringFlag{ Name: "token,t", - Usage: "Token to use for authentication", + Usage: "(cluster) Token to use for authentication", EnvVar: "K3S_TOKEN", Destination: &AgentConfig.Token, }, cli.StringFlag{ Name: "token-file", - Usage: "Token file to use for authentication", + Usage: "(cluster) Token file to use for authentication", EnvVar: "K3S_TOKEN_FILE", Destination: &AgentConfig.TokenFile, }, cli.StringFlag{ Name: "server,s", - Usage: "Server to connect to", + Usage: "(cluster) Server to connect to", EnvVar: "K3S_URL", Destination: &AgentConfig.ServerURL, }, cli.StringFlag{ Name: "data-dir,d", - Usage: "Folder to hold state", + Usage: "(agent/data) Folder to hold state", Destination: &AgentConfig.DataDir, Value: "/var/lib/rancher/k3s", }, - cli.StringFlag{ - Name: "cluster-secret", - Usage: "Shared secret used to bootstrap a cluster", - Destination: &AgentConfig.ClusterSecret, - EnvVar: "K3S_CLUSTER_SECRET", - }, + NodeNameFlag, + NodeLabels, + NodeTaints, + DockerFlag, + CRIEndpointFlag, + PauseImageFlag, + PrivateRegistryFlag, + NodeIPFlag, + NodeExternalIPFlag, + ResolvConfFlag, + FlannelIfaceFlag, + FlannelConfFlag, + ExtraKubeletArgs, + ExtraKubeProxyArgs, cli.BoolFlag{ Name: "rootless", Usage: "(experimental) Run rootless", Destination: &AgentConfig.Rootless, }, - DockerFlag, + + // Deprecated/hidden below + FlannelFlag, - FlannelIfaceFlag, - FlannelConfFlag, - NodeNameFlag, - NodeIPFlag, - CRIEndpointFlag, - PauseImageFlag, - ResolvConfFlag, - ExtraKubeletArgs, - ExtraKubeProxyArgs, - NodeLabels, - NodeTaints, - PrivateRegistryFlag, - NodeExternalIPFlag, + cli.StringFlag{ + Name: "cluster-secret", + Usage: "(deprecated) use --token", + Destination: &AgentConfig.Token, + EnvVar: "K3S_CLUSTER_SECRET", + }, }, } } diff --git a/pkg/cli/cmds/log.go b/pkg/cli/cmds/log.go index cf86687dd3..3ae6803db5 100644 --- a/pkg/cli/cmds/log.go +++ b/pkg/cli/cmds/log.go @@ -25,22 +25,22 @@ var ( VLevel = cli.IntFlag{ Name: "v", - Usage: "Number for the log level verbosity", + Usage: "(logging) Number for the log level verbosity", Destination: &LogConfig.VLevel, } VModule = cli.StringFlag{ Name: "vmodule", - Usage: "Comma-separated list of pattern=N settings for file-filtered logging", + Usage: "(logging) Comma-separated list of pattern=N settings for file-filtered logging", Destination: &LogConfig.VModule, } LogFile = cli.StringFlag{ Name: "log,l", - Usage: "Log to file", + Usage: "(logging) Log to file", Destination: &LogConfig.LogFile, } AlsoLogToStderr = cli.BoolFlag{ Name: "alsologtostderr", - Usage: "Log to standard error as well as file (if set)", + Usage: "(logging) Log to standard error as well as file (if set)", Destination: &LogConfig.AlsoLogToStderr, } ) diff --git a/pkg/cli/cmds/nocluster.go b/pkg/cli/cmds/nocluster.go new file mode 100644 index 0000000000..946b60f0c4 --- /dev/null +++ b/pkg/cli/cmds/nocluster.go @@ -0,0 +1,7 @@ +// +build !dqlite + +package cmds + +const ( + hideDqlite = true +) diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index 8d5a85bc84..00481da625 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -9,12 +9,14 @@ import ( type Server struct { ClusterCIDR string - ClusterSecret string + AgentToken string + AgentTokenFile string + Token string + TokenFile string ServiceCIDR string ClusterDNS string ClusterDomain string HTTPSPort int - HTTPPort int DataDir string DisableAgent bool KubeConfigOutput string @@ -26,7 +28,6 @@ type Server struct { ExtraControllerArgs cli.StringSlice ExtraCloudControllerArgs cli.StringSlice Rootless bool - StoreBootstrap bool StorageEndpoint string StorageCAFile string StorageCertFile string @@ -34,10 +35,13 @@ type Server struct { AdvertiseIP string AdvertisePort int DisableScheduler bool + ServerURL string FlannelBackend string DefaultLocalStoragePath string DisableCCM bool DisableNPC bool + ClusterInit bool + ClusterReset bool } var ServerConfig Server @@ -55,190 +59,236 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { AlsoLogToStderr, cli.StringFlag{ Name: "bind-address", - Usage: "k3s bind address (default: localhost)", + Usage: "(listener) k3s bind address (default: 0.0.0.0)", Destination: &ServerConfig.BindAddress, }, cli.IntFlag{ Name: "https-listen-port", - Usage: "HTTPS listen port", + Usage: "(listener) HTTPS listen port", Value: 6443, Destination: &ServerConfig.HTTPSPort, }, + cli.StringFlag{ + Name: "advertise-address", + Usage: "(listener) IP address that apiserver uses to advertise to members of the cluster (default: node-external-ip/node-ip)", + Destination: &ServerConfig.AdvertiseIP, + }, cli.IntFlag{ - Name: "http-listen-port", - Usage: "HTTP listen port (for /healthz, HTTPS redirect, and port for TLS terminating LB)", - Value: 0, - Destination: &ServerConfig.HTTPPort, + Name: "advertise-port", + Usage: "(listener) Port that apiserver uses to advertise to members of the cluster (default: listen-port)", + Destination: &ServerConfig.AdvertisePort, + }, + cli.StringSliceFlag{ + Name: "tls-san", + Usage: "(listener) Add additional hostname or IP as a Subject Alternative Name in the TLS cert", + Value: &ServerConfig.TLSSan, }, cli.StringFlag{ Name: "data-dir,d", - Usage: "Folder to hold state default /var/lib/rancher/k3s or ${HOME}/.rancher/k3s if not root", + Usage: "(data) Folder to hold state default /var/lib/rancher/k3s or ${HOME}/.rancher/k3s if not root", Destination: &ServerConfig.DataDir, }, + cli.StringFlag{ + Name: "cluster-cidr", + Usage: "(networking) Network CIDR to use for pod IPs", + Destination: &ServerConfig.ClusterCIDR, + Value: "10.42.0.0/16", + }, + cli.StringFlag{ + Name: "service-cidr", + Usage: "(networking) Network CIDR to use for services IPs", + Destination: &ServerConfig.ServiceCIDR, + Value: "10.43.0.0/16", + }, + cli.StringFlag{ + Name: "cluster-dns", + Usage: "(networking) Cluster IP for coredns service. Should be in your service-cidr range (default: 10.43.0.10)", + Destination: &ServerConfig.ClusterDNS, + Value: "", + }, + cli.StringFlag{ + Name: "cluster-domain", + Usage: "(networking) Cluster Domain", + Destination: &ServerConfig.ClusterDomain, + Value: "cluster.local", + }, + cli.StringFlag{ + Name: "flannel-backend", + Usage: fmt.Sprintf("(networking) One of '%s', '%s', '%s', or '%s'", config.FlannelBackendNone, config.FlannelBackendVXLAN, config.FlannelBackendIPSEC, config.FlannelBackendWireguard), + Destination: &ServerConfig.FlannelBackend, + Value: config.FlannelBackendVXLAN, + }, + cli.StringFlag{ + Name: "token,t", + Usage: "(cluster) Shared secret used to join a server or agent to a cluster", + Destination: &ServerConfig.Token, + EnvVar: "K3S_CLUSTER_SECRET,K3S_TOKEN", + }, + cli.StringFlag{ + Name: "token-file", + Usage: "(cluster) File containing the cluster-secret/token", + Destination: &ServerConfig.TokenFile, + EnvVar: "K3S_TOKEN_FILE", + }, + cli.StringFlag{ + Name: "agent-token", + Usage: "(cluster) Shared secret used to join agents to the cluster, but not agents", + Destination: &ServerConfig.AgentToken, + EnvVar: "K3S_AGENT_TOKEN", + }, + cli.StringFlag{ + Name: "agent-token-file", + Usage: "(cluster) File containing the agent secret", + Destination: &ServerConfig.AgentTokenFile, + EnvVar: "K3S_AGENT_TOKEN_FILE", + }, + cli.StringFlag{ + Name: "server,s", + Usage: "(cluster) Server to connect to, used to join a cluster", + EnvVar: "K3S_URL", + Destination: &ServerConfig.ServerURL, + }, + cli.BoolFlag{ + Name: "new-cluster", + Hidden: hideDqlite, + Usage: "(cluster) Initialize new cluster master", + EnvVar: "K3S_CLUSTER_INIT", + Destination: &ServerConfig.ClusterInit, + }, + cli.BoolFlag{ + Name: "reset-cluster", + Hidden: hideDqlite, + Usage: "(cluster) Forget all peers and become a single cluster new cluster master", + EnvVar: "K3S_CLUSTER_RESET", + Destination: &ServerConfig.ClusterReset, + }, + cli.StringFlag{ + Name: "write-kubeconfig,o", + Usage: "(client) Write kubeconfig for admin client to this file", + Destination: &ServerConfig.KubeConfigOutput, + EnvVar: "K3S_KUBECONFIG_OUTPUT", + }, + cli.StringFlag{ + Name: "write-kubeconfig-mode", + Usage: "(client) Write kubeconfig with this mode", + Destination: &ServerConfig.KubeConfigMode, + EnvVar: "K3S_KUBECONFIG_MODE", + }, + cli.StringSliceFlag{ + Name: "kube-apiserver-arg", + Usage: "(flags) Customized flag for kube-apiserver process", + Value: &ServerConfig.ExtraAPIArgs, + }, + cli.StringSliceFlag{ + Name: "kube-scheduler-arg", + Usage: "(flags) Customized flag for kube-scheduler process", + Value: &ServerConfig.ExtraSchedulerArgs, + }, + cli.StringSliceFlag{ + Name: "kube-controller-manager-arg", + Usage: "(flags) Customized flag for kube-controller-manager process", + Value: &ServerConfig.ExtraControllerArgs, + }, + cli.StringSliceFlag{ + Name: "kube-cloud-controller-manager-arg", + Usage: "(flags) Customized flag for kube-cloud-controller-manager process", + Value: &ServerConfig.ExtraCloudControllerArgs, + }, + cli.StringFlag{ + Name: "storage-endpoint", + Usage: "(db) Specify etcd, Mysql, Postgres, or Sqlite (default) data source name", + Destination: &ServerConfig.StorageEndpoint, + EnvVar: "K3S_STORAGE_ENDPOINT", + }, + cli.StringFlag{ + Name: "storage-cafile", + Usage: "(db) SSL Certificate Authority file used to secure storage backend communication", + Destination: &ServerConfig.StorageCAFile, + EnvVar: "K3S_STORAGE_CAFILE", + }, + cli.StringFlag{ + Name: "storage-certfile", + Usage: "(db) SSL certification file used to secure storage backend communication", + Destination: &ServerConfig.StorageCertFile, + EnvVar: "K3S_STORAGE_CERTFILE", + }, + cli.StringFlag{ + Name: "storage-keyfile", + Usage: "(db) SSL key file used to secure storage backend communication", + Destination: &ServerConfig.StorageKeyFile, + EnvVar: "K3S_STORAGE_KEYFILE", + }, + cli.StringFlag{ + Name: "default-local-storage-path", + Usage: "(storage) Default local storage path for local provisioner storage class", + Destination: &ServerConfig.DefaultLocalStoragePath, + }, + cli.StringSliceFlag{ + Name: "no-deploy", + Usage: "(components) Do not deploy packaged components (valid items: coredns, servicelb, traefik, local-storage)", + }, + cli.BoolFlag{ + Name: "disable-scheduler", + Usage: "(components) Disable Kubernetes default scheduler", + Destination: &ServerConfig.DisableScheduler, + }, + cli.BoolFlag{ + Name: "disable-cloud-controller", + Usage: "(components) Disable k3s default cloud controller manager", + Destination: &ServerConfig.DisableCCM, + }, + cli.BoolFlag{ + Name: "disable-network-policy", + Usage: "(components) Disable k3s default network policy controller", + Destination: &ServerConfig.DisableNPC, + }, + NodeNameFlag, + NodeLabels, + NodeTaints, + DockerFlag, + CRIEndpointFlag, + PauseImageFlag, + PrivateRegistryFlag, + NodeIPFlag, + NodeExternalIPFlag, + ResolvConfFlag, + FlannelIfaceFlag, + FlannelConfFlag, + ExtraKubeletArgs, + ExtraKubeProxyArgs, + cli.BoolFlag{ + Name: "rootless", + Usage: "(experimental) Run rootless", + Destination: &ServerConfig.Rootless, + }, + + // Hidden/Deprecated flags below + + FlannelFlag, + cli.StringFlag{ + Name: "cluster-secret", + Usage: "(deprecated) use --token", + Destination: &ServerConfig.Token, + EnvVar: "K3S_CLUSTER_SECRET", + }, cli.BoolFlag{ Name: "disable-agent", Usage: "Do not run a local agent and register a local kubelet", Hidden: true, Destination: &ServerConfig.DisableAgent, }, - cli.StringFlag{ - Name: "cluster-cidr", - Usage: "Network CIDR to use for pod IPs", - Destination: &ServerConfig.ClusterCIDR, - Value: "10.42.0.0/16", - }, - cli.StringFlag{ - Name: "cluster-secret", - Usage: "Shared secret used to bootstrap a cluster", - Destination: &ServerConfig.ClusterSecret, - EnvVar: "K3S_CLUSTER_SECRET", - }, - cli.StringFlag{ - Name: "service-cidr", - Usage: "Network CIDR to use for services IPs", - Destination: &ServerConfig.ServiceCIDR, - Value: "10.43.0.0/16", - }, - cli.StringFlag{ - Name: "cluster-dns", - Usage: "Cluster IP for coredns service. Should be in your service-cidr range", - Destination: &ServerConfig.ClusterDNS, - Value: "", - }, - cli.StringFlag{ - Name: "cluster-domain", - Usage: "Cluster Domain", - Destination: &ServerConfig.ClusterDomain, - Value: "cluster.local", + cli.StringSliceFlag{ + Hidden: true, + Name: "kube-controller-arg", + Usage: "(flags) Customized flag for kube-controller-manager process", + Value: &ServerConfig.ExtraControllerArgs, }, cli.StringSliceFlag{ - Name: "no-deploy", - Usage: "Do not deploy packaged components (valid items: coredns, servicelb, traefik)", + Hidden: true, + Name: "kube-cloud-controller-arg", + Usage: "(flags) Customized flag for kube-cloud-controller-manager process", + Value: &ServerConfig.ExtraCloudControllerArgs, }, - cli.StringFlag{ - Name: "write-kubeconfig,o", - Usage: "Write kubeconfig for admin client to this file", - Destination: &ServerConfig.KubeConfigOutput, - EnvVar: "K3S_KUBECONFIG_OUTPUT", - }, - cli.StringFlag{ - Name: "write-kubeconfig-mode", - Usage: "Write kubeconfig with this mode", - Destination: &ServerConfig.KubeConfigMode, - EnvVar: "K3S_KUBECONFIG_MODE", - }, - cli.StringSliceFlag{ - Name: "tls-san", - Usage: "Add additional hostname or IP as a Subject Alternative Name in the TLS cert", - Value: &ServerConfig.TLSSan, - }, - cli.StringSliceFlag{ - Name: "kube-apiserver-arg", - Usage: "Customized flag for kube-apiserver process", - Value: &ServerConfig.ExtraAPIArgs, - }, - cli.StringSliceFlag{ - Name: "kube-scheduler-arg", - Usage: "Customized flag for kube-scheduler process", - Value: &ServerConfig.ExtraSchedulerArgs, - }, - cli.StringSliceFlag{ - Name: "kube-controller-arg", - Usage: "Customized flag for kube-controller-manager process", - Value: &ServerConfig.ExtraControllerArgs, - }, - cli.StringSliceFlag{ - Name: "kube-cloud-controller-arg", - Usage: "Customized flag for kube-cloud-controller-manager process", - Value: &ServerConfig.ExtraCloudControllerArgs, - }, - cli.BoolFlag{ - Name: "rootless", - Usage: "(experimental) Run rootless", - Destination: &ServerConfig.Rootless, - }, - cli.BoolFlag{ - Name: "bootstrap-save", - Usage: "(experimental) Save bootstrap information in the storage endpoint", - Hidden: true, - Destination: &ServerConfig.StoreBootstrap, - }, - cli.StringFlag{ - Name: "storage-endpoint", - Usage: "Specify etcd, Mysql, Postgres, or Sqlite (default) data source name", - Destination: &ServerConfig.StorageEndpoint, - EnvVar: "K3S_STORAGE_ENDPOINT", - }, - cli.StringFlag{ - Name: "storage-cafile", - Usage: "SSL Certificate Authority file used to secure storage backend communication", - Destination: &ServerConfig.StorageCAFile, - EnvVar: "K3S_STORAGE_CAFILE", - }, - cli.StringFlag{ - Name: "storage-certfile", - Usage: "SSL certification file used to secure storage backend communication", - Destination: &ServerConfig.StorageCertFile, - EnvVar: "K3S_STORAGE_CERTFILE", - }, - cli.StringFlag{ - Name: "storage-keyfile", - Usage: "SSL key file used to secure storage backend communication", - Destination: &ServerConfig.StorageKeyFile, - EnvVar: "K3S_STORAGE_KEYFILE", - }, - cli.StringFlag{ - Name: "advertise-address", - Usage: "IP address that apiserver uses to advertise to members of the cluster", - Destination: &ServerConfig.AdvertiseIP, - }, - cli.IntFlag{ - Name: "advertise-port", - Usage: "Port that apiserver uses to advertise to members of the cluster", - Value: 0, - Destination: &ServerConfig.AdvertisePort, - }, - cli.BoolFlag{ - Name: "disable-scheduler", - Usage: "Disable Kubernetes default scheduler", - Destination: &ServerConfig.DisableScheduler, - }, - cli.BoolFlag{ - Name: "disable-cloud-controller", - Usage: "Disable k3s default cloud controller manager", - Destination: &ServerConfig.DisableCCM, - }, - cli.BoolFlag{ - Name: "disable-network-policy", - Usage: "Disable k3s default network policy controller", - Destination: &ServerConfig.DisableNPC, - }, - cli.StringFlag{ - Name: "flannel-backend", - Usage: fmt.Sprintf("(experimental) One of '%s', '%s', '%s', or '%s'", config.FlannelBackendNone, config.FlannelBackendVXLAN, config.FlannelBackendIPSEC, config.FlannelBackendWireguard), - Destination: &ServerConfig.FlannelBackend, - Value: config.FlannelBackendVXLAN, - }, - cli.StringFlag{ - Name: "default-local-storage-path", - Usage: "Default local storage path for local provisioner storage class", - Destination: &ServerConfig.DefaultLocalStoragePath, - }, - NodeIPFlag, - NodeNameFlag, - DockerFlag, - FlannelFlag, - FlannelIfaceFlag, - FlannelConfFlag, - CRIEndpointFlag, - PauseImageFlag, - ResolvConfFlag, - ExtraKubeletArgs, - ExtraKubeProxyArgs, - NodeLabels, - NodeTaints, - PrivateRegistryFlag, - NodeExternalIPFlag, }, } } diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 22dcff2a2c..9699471485 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -16,6 +16,7 @@ import ( "github.com/rancher/k3s/pkg/netutil" "github.com/rancher/k3s/pkg/rootless" "github.com/rancher/k3s/pkg/server" + "github.com/rancher/k3s/pkg/token" "github.com/rancher/wrangler/pkg/signals" "github.com/sirupsen/logrus" "github.com/urfave/cli" @@ -56,23 +57,28 @@ func run(app *cli.Context, cfg *cmds.Server) error { serverConfig := server.Config{} serverConfig.DisableAgent = cfg.DisableAgent - serverConfig.ControlConfig.ClusterSecret = cfg.ClusterSecret + serverConfig.ControlConfig.Token = cfg.Token + serverConfig.ControlConfig.AgentToken = cfg.AgentToken + serverConfig.ControlConfig.JoinURL = cfg.ServerURL + if cfg.AgentTokenFile != "" { + serverConfig.ControlConfig.AgentToken, err = token.ReadFile(cfg.AgentTokenFile) + if err != nil { + return err + } + } + if cfg.TokenFile != "" { + serverConfig.ControlConfig.Token, err = token.ReadFile(cfg.TokenFile) + if err != nil { + return err + } + } serverConfig.ControlConfig.DataDir = cfg.DataDir serverConfig.ControlConfig.KubeConfigOutput = cfg.KubeConfigOutput serverConfig.ControlConfig.KubeConfigMode = cfg.KubeConfigMode serverConfig.ControlConfig.NoScheduler = cfg.DisableScheduler serverConfig.Rootless = cfg.Rootless - serverConfig.TLSConfig.HTTPSPort = cfg.HTTPSPort - serverConfig.TLSConfig.HTTPPort = cfg.HTTPPort - for _, san := range knownIPs(cfg.TLSSan) { - addr := net2.ParseIP(san) - if addr != nil { - serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, san) - } else { - serverConfig.TLSConfig.Domains = append(serverConfig.TLSConfig.Domains, san) - } - } - serverConfig.TLSConfig.BindAddress = cfg.BindAddress + serverConfig.ControlConfig.SANs = knownIPs(cfg.TLSSan) + serverConfig.ControlConfig.BindAddress = cfg.BindAddress serverConfig.ControlConfig.HTTPSPort = cfg.HTTPSPort serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs @@ -84,21 +90,25 @@ func run(app *cli.Context, cfg *cmds.Server) error { serverConfig.ControlConfig.Storage.KeyFile = cfg.StorageKeyFile serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort - serverConfig.ControlConfig.BootstrapReadOnly = !cfg.StoreBootstrap serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend serverConfig.ControlConfig.ExtraCloudControllerArgs = cfg.ExtraCloudControllerArgs serverConfig.ControlConfig.DisableCCM = cfg.DisableCCM serverConfig.ControlConfig.DisableNPC = cfg.DisableNPC + serverConfig.ControlConfig.ClusterInit = cfg.ClusterInit + serverConfig.ControlConfig.ClusterReset = cfg.ClusterReset if cmds.AgentConfig.FlannelIface != "" && cmds.AgentConfig.NodeIP == "" { cmds.AgentConfig.NodeIP = netutil.GetIPFromInterface(cmds.AgentConfig.FlannelIface) } + if serverConfig.ControlConfig.AdvertiseIP == "" && cmds.AgentConfig.NodeExternalIP != "" { + serverConfig.ControlConfig.AdvertiseIP = cmds.AgentConfig.NodeExternalIP + } if serverConfig.ControlConfig.AdvertiseIP == "" && cmds.AgentConfig.NodeIP != "" { serverConfig.ControlConfig.AdvertiseIP = cmds.AgentConfig.NodeIP } if serverConfig.ControlConfig.AdvertiseIP != "" { - serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, serverConfig.ControlConfig.AdvertiseIP) + serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, serverConfig.ControlConfig.AdvertiseIP) } _, serverConfig.ControlConfig.ClusterIPRange, err = net2.ParseCIDR(cfg.ClusterCIDR) @@ -114,7 +124,7 @@ func run(app *cli.Context, cfg *cmds.Server) error { if err != nil { return err } - serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, apiServerServiceIP.String()) + serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, apiServerServiceIP.String()) // If cluster-dns CLI arg is not set, we set ClusterDNS address to be ServiceCIDR network + 10, // i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10 @@ -160,8 +170,7 @@ func run(app *cli.Context, cfg *cmds.Server) error { os.Unsetenv("NOTIFY_SOCKET") ctx := signals.SetupSignalHandler(context.Background()) - certs, err := server.StartServer(ctx, &serverConfig) - if err != nil { + if err := server.StartServer(ctx, &serverConfig); err != nil { return err } @@ -175,12 +184,17 @@ func run(app *cli.Context, cfg *cmds.Server) error { <-ctx.Done() return nil } - ip := serverConfig.TLSConfig.BindAddress + + ip := serverConfig.ControlConfig.BindAddress if ip == "" { ip = "127.0.0.1" } - url := fmt.Sprintf("https://%s:%d", ip, serverConfig.TLSConfig.HTTPSPort) - token := server.FormatToken(serverConfig.ControlConfig.Runtime.NodeToken, certs) + + url := fmt.Sprintf("https://%s:%d", ip, serverConfig.ControlConfig.HTTPSPort) + token, err := server.FormatToken(serverConfig.ControlConfig.Runtime.AgentToken, serverConfig.ControlConfig.Runtime.ServerCA) + if err != nil { + return err + } agentConfig := cmds.AgentConfig agentConfig.Debug = app.GlobalBool("bool") diff --git a/pkg/clientaccess/clientaccess.go b/pkg/clientaccess/clientaccess.go index 8fcaf5ad44..1d403c19f8 100644 --- a/pkg/clientaccess/clientaccess.go +++ b/pkg/clientaccess/clientaccess.go @@ -9,7 +9,6 @@ import ( "io/ioutil" "net/http" "net/url" - "os" "strings" "github.com/pkg/errors" @@ -35,21 +34,6 @@ type clientToken struct { password string } -func AgentAccessInfoToTempKubeConfig(tempDir, server, token string) (string, error) { - f, err := ioutil.TempFile(tempDir, "tmp-") - if err != nil { - return "", err - } - if err := f.Close(); err != nil { - return "", err - } - err = accessInfoToKubeConfig(f.Name(), server, token) - if err != nil { - os.Remove(f.Name()) - } - return f.Name(), err -} - func AgentAccessInfoToKubeConfig(destFile, server, token string) error { return accessInfoToKubeConfig(destFile, server, token) } @@ -62,6 +46,10 @@ type Info struct { Token string `json:"token,omitempty"` } +func (i *Info) ToToken() string { + return fmt.Sprintf("K10%s::%s:%s", hashCA(i.CACerts), i.username, i.password) +} + func (i *Info) WriteKubeConfig(destFile string) error { return clientcmd.WriteToFile(*i.KubeConfig(), destFile) } @@ -98,6 +86,22 @@ func (i *Info) KubeConfig() *clientcmdapi.Config { return config } +func NormalizeAndValidateTokenForUser(server, token, user string) (string, error) { + if !strings.HasPrefix(token, "K10") { + token = "K10::" + user + ":" + token + } + info, err := ParseAndValidateToken(server, token) + if err != nil { + return "", err + } + + if info.username != user { + info.username = user + } + + return info.ToToken(), nil +} + func ParseAndValidateToken(server, token string) (*Info, error) { url, err := url.Parse(server) if err != nil { @@ -132,13 +136,17 @@ func ParseAndValidateToken(server, token string) (*Info, error) { return nil, err } - return &Info{ + i := &Info{ URL: url.String(), CACerts: cacerts, username: parsedToken.username, password: parsedToken.password, Token: token, - }, nil + } + + // normalize token + i.Token = i.ToToken() + return i, nil } func accessInfoToKubeConfig(destFile, server, token string) error { @@ -164,11 +172,15 @@ func validateCACerts(cacerts []byte, hash string) (bool, string, string) { return true, "", "" } - digest := sha256.Sum256([]byte(cacerts)) - newHash := hex.EncodeToString(digest[:]) + newHash := hashCA(cacerts) return hash == newHash, hash, newHash } +func hashCA(cacerts []byte) string { + digest := sha256.Sum256(cacerts) + return hex.EncodeToString(digest[:]) +} + func ParseUsernamePassword(token string) (string, string, bool) { parsed, err := parseToken(token) if err != nil { diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go new file mode 100644 index 0000000000..d5372dae0c --- /dev/null +++ b/pkg/cluster/cluster.go @@ -0,0 +1,48 @@ +package cluster + +import ( + "context" + + "github.com/rancher/k3s/pkg/clientaccess" + "github.com/rancher/k3s/pkg/daemons/config" +) + +type Cluster struct { + token string + clientAccessInfo *clientaccess.Info + config *config.Control + runtime *config.ControlRuntime + db interface{} +} + +func (c *Cluster) Start(ctx context.Context) error { + join, err := c.shouldJoin() + if err != nil { + return err + } + + if join { + if err := c.join(); err != nil { + return err + } + } + + if err := c.startClusterAndHTTPS(ctx); err != nil { + return err + } + + if join { + if err := c.postJoin(ctx); err != nil { + return err + } + } + + return c.joined() +} + +func New(config *config.Control) *Cluster { + return &Cluster{ + config: config, + runtime: config.Runtime, + } +} diff --git a/pkg/cluster/https.go b/pkg/cluster/https.go new file mode 100644 index 0000000000..d57c7f7af5 --- /dev/null +++ b/pkg/cluster/https.go @@ -0,0 +1,81 @@ +package cluster + +import ( + "context" + "crypto/tls" + "net" + "net/http" + "path/filepath" + + "github.com/rancher/dynamiclistener" + "github.com/rancher/dynamiclistener/factory" + "github.com/rancher/dynamiclistener/storage/file" + "github.com/rancher/dynamiclistener/storage/kubernetes" + "github.com/rancher/dynamiclistener/storage/memory" + "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/wrangler-api/pkg/generated/controllers/core" + "github.com/sirupsen/logrus" +) + +func (c *Cluster) newListener(ctx context.Context) (net.Listener, http.Handler, error) { + tcp, err := dynamiclistener.NewTCPListener(c.config.BindAddress, c.config.HTTPSPort) + if err != nil { + return nil, nil, err + } + + cert, key, err := factory.LoadCerts(c.runtime.ServerCA, c.runtime.ServerCAKey) + if err != nil { + return nil, nil, err + } + + storage := tlsStorage(ctx, c.config.DataDir, c.runtime) + return dynamiclistener.NewListener(tcp, storage, cert, key, dynamiclistener.Config{ + CN: "k3s", + Organization: []string{"k3s"}, + TLSConfig: tls.Config{ + ClientAuth: tls.RequestClientCert, + }, + SANs: c.config.SANs, + }) +} + +func (c *Cluster) startClusterAndHTTPS(ctx context.Context) error { + l, handler, err := c.newListener(ctx) + if err != nil { + return err + } + + handler, err = c.getHandler(handler) + if err != nil { + return err + } + + l, handler, err = c.initClusterDB(ctx, l, handler) + if err != nil { + return err + } + + server := http.Server{ + Handler: handler, + } + + go func() { + err := server.Serve(l) + logrus.Fatalf("server stopped: %v", err) + }() + + go func() { + <-ctx.Done() + server.Shutdown(context.Background()) + }() + + return nil +} + +func tlsStorage(ctx context.Context, dataDir string, runtime *config.ControlRuntime) dynamiclistener.TLSStorage { + fileStorage := file.New(filepath.Join(dataDir, "tls/dynamic-cert.json")) + cache := memory.NewBacked(fileStorage) + return kubernetes.New(ctx, func() *core.Factory { + return runtime.Core + }, "kube-system", "k3s-serving", cache) +} diff --git a/pkg/cluster/join.go b/pkg/cluster/join.go new file mode 100644 index 0000000000..99d1bdecdf --- /dev/null +++ b/pkg/cluster/join.go @@ -0,0 +1,74 @@ +package cluster + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + + "github.com/rancher/k3s/pkg/bootstrap" + "github.com/rancher/k3s/pkg/clientaccess" + "github.com/sirupsen/logrus" +) + +func (c *Cluster) shouldJoin() (bool, error) { + if c.config.JoinURL == "" { + return false, nil + } + + stamp := filepath.Join(c.config.DataDir, "db/joined") + if _, err := os.Stat(stamp); err == nil { + logrus.Info("Already joined to cluster, not rejoining") + return false, nil + } + + if c.config.Token == "" { + return false, fmt.Errorf("K3S_TOKEN is required to join a cluster") + } + + return true, nil +} + +func (c *Cluster) joined() error { + if err := os.MkdirAll(filepath.Dir(c.joinStamp()), 0700); err != nil { + return err + } + + if _, err := os.Stat(c.joinStamp()); err == nil { + return nil + } + + f, err := os.Create(c.joinStamp()) + if err != nil { + return err + } + + return f.Close() +} + +func (c *Cluster) join() error { + c.runtime.Cluster.Join = true + + token, err := clientaccess.NormalizeAndValidateTokenForUser(c.config.JoinURL, c.config.Token, "server") + if err != nil { + return err + } + c.token = token + + info, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, token) + if err != nil { + return err + } + c.clientAccessInfo = info + + content, err := clientaccess.Get("/v1-k3s/server-bootstrap", info) + if err != nil { + return err + } + + return bootstrap.Read(bytes.NewBuffer(content), &c.runtime.ControlRuntimeBootstrap) +} + +func (c *Cluster) joinStamp() string { + return filepath.Join(c.config.DataDir, "db/joined") +} diff --git a/pkg/cluster/nocluster.go b/pkg/cluster/nocluster.go new file mode 100644 index 0000000000..ce5029156a --- /dev/null +++ b/pkg/cluster/nocluster.go @@ -0,0 +1,17 @@ +// +build !dqlite + +package cluster + +import ( + "context" + "net" + "net/http" +) + +func (c *Cluster) initClusterDB(ctx context.Context, l net.Listener, handler http.Handler) (net.Listener, http.Handler, error) { + return l, handler, nil +} + +func (c *Cluster) postJoin(ctx context.Context) error { + return nil +} diff --git a/pkg/cluster/router.go b/pkg/cluster/router.go new file mode 100644 index 0000000000..48b14b94ee --- /dev/null +++ b/pkg/cluster/router.go @@ -0,0 +1,25 @@ +package cluster + +import ( + "net/http" +) + +func (c *Cluster) getHandler(handler http.Handler) (http.Handler, error) { + next := c.router() + + return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + handler.ServeHTTP(rw, req) + next.ServeHTTP(rw, req) + }), nil +} + +func (c *Cluster) router() http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + if c.runtime.Handler == nil { + http.Error(rw, "starting", http.StatusServiceUnavailable) + return + } + + c.runtime.Handler.ServeHTTP(rw, req) + }) +} diff --git a/pkg/codegen/main.go b/pkg/codegen/main.go index 45101cb9d9..5d802b1665 100644 --- a/pkg/codegen/main.go +++ b/pkg/codegen/main.go @@ -70,7 +70,6 @@ func main() { Groups: map[string]args.Group{ "k3s.cattle.io": { Types: []interface{}{ - v1.ListenerConfig{}, v1.Addon{}, }, GenerateTypes: true, diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 655dc816cd..8c41740c33 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/rancher/kine/pkg/endpoint" - + "github.com/rancher/wrangler-api/pkg/generated/controllers/core" "k8s.io/apiserver/pkg/authentication/authenticator" ) @@ -47,40 +47,36 @@ type Containerd struct { } type Agent struct { - NodeName string - ClientKubeletCert string - ClientKubeletKey string - ClientKubeProxyCert string - ClientKubeProxyKey string - ServingKubeletCert string - ServingKubeletKey string - ClusterCIDR net.IPNet - ClusterDNS net.IP - ClusterDomain string - ResolvConf string - RootDir string - KubeConfigNode string - KubeConfigKubelet string - KubeConfigKubeProxy string - NodeIP string - NodeExternalIP string - RuntimeSocket string - ListenAddress string - ClientCA string - CNIBinDir string - CNIConfDir string - ExtraKubeletArgs []string - ExtraKubeProxyArgs []string - PauseImage string - CNIPlugin bool - NodeTaints []string - NodeLabels []string - IPSECPSK string - StrongSwanDir string - PrivateRegistry string - DisableCCM bool - DisableNPC bool - Rootless bool + NodeName string + ServingKubeletCert string + ServingKubeletKey string + ClusterCIDR net.IPNet + ClusterDNS net.IP + ClusterDomain string + ResolvConf string + RootDir string + KubeConfigKubelet string + KubeConfigKubeProxy string + KubeConfigK3sController string + NodeIP string + NodeExternalIP string + RuntimeSocket string + ListenAddress string + ClientCA string + CNIBinDir string + CNIConfDir string + ExtraKubeletArgs []string + ExtraKubeProxyArgs []string + PauseImage string + CNIPlugin bool + NodeTaints []string + NodeLabels []string + IPSECPSK string + StrongSwanDir string + PrivateRegistry string + DisableCCM bool + DisableNPC bool + Rootless bool } type Control struct { @@ -88,7 +84,8 @@ type Control struct { AdvertiseIP string ListenPort int HTTPSPort int - ClusterSecret string + AgentToken string + Token string ClusterIPRange *net.IPNet ServiceIPRange *net.IPNet ClusterDNS net.IP @@ -98,19 +95,24 @@ type Control struct { KubeConfigMode string DataDir string Skips []string - BootstrapReadOnly bool Storage endpoint.Config NoScheduler bool ExtraAPIArgs []string ExtraControllerArgs []string - ExtraSchedulerAPIArgs []string ExtraCloudControllerArgs []string + ExtraSchedulerAPIArgs []string NoLeaderElect bool + JoinURL string FlannelBackend string IPSECPSK string DefaultLocalStoragePath string DisableCCM bool DisableNPC bool + ClusterInit bool + ClusterReset bool + + BindAddress string + SANs []string Runtime *ControlRuntime `json:"-"` } @@ -124,9 +126,6 @@ type ControlRuntimeBootstrap struct { PasswdFile string RequestHeaderCA string RequestHeaderCAKey string - ClientKubeletKey string - ClientKubeProxyKey string - ServingKubeletKey string IPSECKey string } @@ -145,8 +144,10 @@ type ControlRuntime struct { ServingKubeAPICert string ServingKubeAPIKey string + ServingKubeletKey string ClientToken string - NodeToken string + ServerToken string + AgentToken string Handler http.Handler Tunnel http.Handler Authenticator authenticator.Request @@ -161,8 +162,19 @@ type ControlRuntime struct { ClientSchedulerCert string ClientSchedulerKey string ClientKubeProxyCert string + ClientKubeProxyKey string + ClientKubeletKey string ClientCloudControllerCert string ClientCloudControllerKey string + ClientK3sControllerCert string + ClientK3sControllerKey string + + Cluster ClusterConfig + Core *core.Factory +} + +type ClusterConfig struct { + Join bool } type ArgString []string diff --git a/pkg/daemons/control/bootstrap.go b/pkg/daemons/control/bootstrap.go deleted file mode 100644 index 8dedee6c2a..0000000000 --- a/pkg/daemons/control/bootstrap.go +++ /dev/null @@ -1,100 +0,0 @@ -package control - -import ( - "context" - "encoding/json" - "io/ioutil" - "os" - "path/filepath" - - "github.com/pkg/errors" - "github.com/rancher/k3s/pkg/daemons/config" - "github.com/rancher/kine/pkg/client" - "github.com/sirupsen/logrus" -) - -const ( - k3sRuntimeEtcdPath = "/k3s/runtime" -) - -// fetchBootstrapData copies the bootstrap data (certs, keys, passwords) -// from etcd to individual files specified by cfg.Runtime. -func fetchBootstrapData(ctx context.Context, cfg *config.Control, c client.Client) error { - logrus.Info("Fetching bootstrap data from etcd") - gr, err := c.Get(ctx, k3sRuntimeEtcdPath) - if err != nil { - return err - } - if gr.Modified == 0 { - return nil - } - - paths, err := objToMap(&cfg.Runtime.ControlRuntimeBootstrap) - if err != nil { - return err - } - - files := map[string][]byte{} - if err := json.Unmarshal(gr.Data, &files); err != nil { - return err - } - - for pathKey, data := range files { - path, ok := paths[pathKey] - if !ok { - continue - } - - if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { - return errors.Wrapf(err, "failed to mkdir %s", filepath.Dir(path)) - } - - if err := ioutil.WriteFile(path, data, 0700); err != nil { - return errors.Wrapf(err, "failed to write to %s", path) - } - } - - return nil -} - -// storeBootstrapData copies the bootstrap data in the opposite direction to -// fetchBootstrapData. -func storeBootstrapData(ctx context.Context, cfg *config.Control, client client.Client) error { - if cfg.BootstrapReadOnly { - return nil - } - - paths, err := objToMap(&cfg.Runtime.ControlRuntimeBootstrap) - if err != nil { - return nil - } - - dataMap := map[string][]byte{} - for pathKey, path := range paths { - if path == "" { - continue - } - data, err := ioutil.ReadFile(path) - if err != nil { - return errors.Wrapf(err, "failed to read %s", path) - } - - dataMap[pathKey] = data - } - - bytes, err := json.Marshal(dataMap) - if err != nil { - return err - } - - return client.Put(ctx, k3sRuntimeEtcdPath, bytes) -} - -func objToMap(obj interface{}) (map[string]string, error) { - bytes, err := json.Marshal(obj) - if err != nil { - return nil, err - } - data := map[string]string{} - return data, json.Unmarshal(bytes, &data) -} diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index d7a7d655ff..40999f6aad 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -3,13 +3,9 @@ package control import ( "context" "crypto" - cryptorand "crypto/rand" "crypto/x509" - "encoding/csv" - "encoding/hex" "fmt" "html/template" - "io" "io/ioutil" "math/rand" "net" @@ -21,11 +17,14 @@ import ( "strings" "time" - certutil "github.com/rancher/dynamiclistener/cert" // registering k3s cloud provider _ "github.com/rancher/k3s/pkg/cloudprovider" + + certutil "github.com/rancher/dynamiclistener/cert" + "github.com/rancher/k3s/pkg/cluster" "github.com/rancher/k3s/pkg/daemons/config" - "github.com/rancher/kine/pkg/client" + "github.com/rancher/k3s/pkg/passwd" + "github.com/rancher/k3s/pkg/token" "github.com/rancher/kine/pkg/endpoint" "github.com/rancher/wrangler-api/pkg/generated/controllers/rbac" "github.com/sirupsen/logrus" @@ -121,6 +120,10 @@ func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) { "cluster-signing-cert-file": runtime.ServerCA, "cluster-signing-key-file": runtime.ServerCAKey, } + offset := cfg.HTTPSPort - 6443 + if offset > 0 { + argsMap["port"] = strconv.Itoa(10252 + offset) + } if cfg.NoLeaderElect { argsMap["leader-elect"] = "false" } @@ -143,6 +146,10 @@ func scheduler(cfg *config.Control, runtime *config.ControlRuntime) { "bind-address": "127.0.0.1", "secure-port": "0", } + offset := cfg.HTTPSPort - 6443 + if offset > 0 { + argsMap["port"] = strconv.Itoa(10251 + offset) + } if cfg.NoLeaderElect { argsMap["leader-elect"] = "false" } @@ -291,6 +298,8 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro runtime.ClientKubeAPIKey = path.Join(config.DataDir, "tls", "client-kube-apiserver.key") runtime.ClientKubeProxyCert = path.Join(config.DataDir, "tls", "client-kube-proxy.crt") runtime.ClientKubeProxyKey = path.Join(config.DataDir, "tls", "client-kube-proxy.key") + runtime.ClientK3sControllerCert = path.Join(config.DataDir, "tls", "client-k3s-controller.crt") + runtime.ClientK3sControllerKey = path.Join(config.DataDir, "tls", "client-k3s-controller.key") runtime.ServingKubeAPICert = path.Join(config.DataDir, "tls", "serving-kube-apiserver.crt") runtime.ServingKubeAPIKey = path.Join(config.DataDir, "tls", "serving-kube-apiserver.key") @@ -301,20 +310,14 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro runtime.ClientAuthProxyCert = path.Join(config.DataDir, "tls", "client-auth-proxy.crt") runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key") - etcdClient, err := prepareStorageBackend(ctx, config) - if err != nil { - return err - } - defer etcdClient.Close() - - if err := fetchBootstrapData(ctx, config, etcdClient); err != nil { - return err - } - if err := genCerts(config, runtime); err != nil { return err } + if err := cluster.New(config).Start(ctx); err != nil { + return err + } + if err := genServiceAccount(runtime); err != nil { return err } @@ -327,124 +330,44 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro return err } - if err := storeBootstrapData(ctx, config, etcdClient); err != nil { + if err := prepareStorageBackend(ctx, config); err != nil { return err } return readTokens(runtime) } -func prepareStorageBackend(ctx context.Context, config *config.Control) (client.Client, error) { +func prepareStorageBackend(ctx context.Context, config *config.Control) error { etcdConfig, err := endpoint.Listen(ctx, config.Storage) if err != nil { - return nil, err + return err } config.Storage.Config = etcdConfig.TLSConfig config.Storage.Endpoint = strings.Join(etcdConfig.Endpoints, ",") config.NoLeaderElect = !etcdConfig.LeaderElect - return client.New(etcdConfig) -} - -func readTokenFile(passwdFile string) (map[string]string, error) { - f, err := os.Open(passwdFile) - if err != nil { - return nil, err - } - defer f.Close() - - reader := csv.NewReader(f) - reader.FieldsPerRecord = -1 - - tokens := map[string]string{} - - for { - record, err := reader.Read() - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - if len(record) < 2 { - continue - } - tokens[record[1]] = record[0] - } - return tokens, nil + return nil } func readTokens(runtime *config.ControlRuntime) error { - tokens, err := readTokenFile(runtime.PasswdFile) + tokens, err := passwd.Read(runtime.PasswdFile) if err != nil { return err } - if nodeToken, ok := tokens["node"]; ok { - runtime.NodeToken = "node:" + nodeToken + if nodeToken, ok := tokens.Pass("node"); ok { + runtime.AgentToken = "node:" + nodeToken } - if clientToken, ok := tokens["admin"]; ok { + if serverToken, ok := tokens.Pass("server"); ok { + runtime.AgentToken = "server:" + serverToken + } + if clientToken, ok := tokens.Pass("admin"); ok { runtime.ClientToken = "admin:" + clientToken } return nil } -func ensureNodeToken(config *config.Control, runtime *config.ControlRuntime) error { - if config.ClusterSecret == "" { - return nil - } - - f, err := os.Open(runtime.PasswdFile) - if err != nil { - return err - } - defer f.Close() - - records := [][]string{} - reader := csv.NewReader(f) - for { - record, err := reader.Read() - if err == io.EOF { - break - } - if err != nil { - return err - } - if len(record) < 3 { - return fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", runtime.PasswdFile, len(record)) - } - if record[1] == "node" { - if record[0] == config.ClusterSecret { - return nil - } - record[0] = config.ClusterSecret - } - records = append(records, record) - } - - f.Close() - return WritePasswords(runtime.PasswdFile, records) -} - -func WritePasswords(passwdFile string, records [][]string) error { - out, err := os.Create(passwdFile + ".tmp") - if err != nil { - return err - } - defer out.Close() - - if err := out.Chmod(0600); err != nil { - return err - } - - if err := csv.NewWriter(out).WriteAll(records); err != nil { - return err - } - - return os.Rename(passwdFile+".tmp", passwdFile) -} - func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.ControlRuntime) error { if s, err := os.Stat(runtime.IPSECKey); err == nil && s.Size() > 0 { psk, err := ioutil.ReadFile(runtime.IPSECKey) @@ -455,7 +378,7 @@ func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.Cont return nil } - psk, err := getToken(ipsecTokenSize) + psk, err := token.Random(ipsecTokenSize) if err != nil { return err } @@ -468,42 +391,71 @@ func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.Cont return nil } -func genUsers(config *config.Control, runtime *config.ControlRuntime) error { - if s, err := os.Stat(runtime.PasswdFile); err == nil && s.Size() > 0 { - return ensureNodeToken(config, runtime) +func migratePassword(p *passwd.Passwd) error { + server, _ := p.Pass("server") + node, _ := p.Pass("node") + if server == "" && node != "" { + return p.EnsureUser("server", "k3s:server", node) } - - adminToken, err := getToken(userTokenSize) - if err != nil { - return err - } - systemToken, err := getToken(userTokenSize) - if err != nil { - return err - } - nodeToken, err := getToken(userTokenSize) - if err != nil { - return err - } - - if config.ClusterSecret != "" { - nodeToken = config.ClusterSecret - } - - return WritePasswords(runtime.PasswdFile, [][]string{ - {adminToken, "admin", "admin", "system:masters"}, - {systemToken, "system", "system", "system:masters"}, - {nodeToken, "node", "node", "system:masters"}, - }) + return nil } -func getToken(size int) (string, error) { - token := make([]byte, size, size) - _, err := cryptorand.Read(token) - if err != nil { - return "", err +func getServerPass(passwd *passwd.Passwd, config *config.Control) (string, error) { + var ( + err error + ) + + serverPass := config.Token + if serverPass == "" { + serverPass, _ = passwd.Pass("server") } - return hex.EncodeToString(token), err + if serverPass == "" { + serverPass, err = token.Random(16) + if err != nil { + return "", err + } + } + + return serverPass, nil +} + +func getNodePass(config *config.Control, serverPass string) string { + if config.AgentToken == "" { + return serverPass + } + return config.AgentToken +} + +func genUsers(config *config.Control, runtime *config.ControlRuntime) error { + passwd, err := passwd.Read(runtime.PasswdFile) + if err != nil { + return err + } + + if err := migratePassword(passwd); err != nil { + return err + } + + serverPass, err := getServerPass(passwd, config) + if err != nil { + return err + } + + nodePass := getNodePass(config, serverPass) + + if err := passwd.EnsureUser("admin", "system:masters", ""); err != nil { + return err + } + + if err := passwd.EnsureUser("node", "k3s:agent", nodePass); err != nil { + return err + } + + if err := passwd.EnsureUser("server", "k3s:server", serverPass); err != nil { + return err + } + + return passwd.Write(runtime.PasswdFile) } func genCerts(config *config.Control, runtime *config.ControlRuntime) error { @@ -578,7 +530,10 @@ func genClientCerts(config *config.Control, runtime *config.ControlRuntime) erro } } - if _, err = factory("system:kube-proxy", []string{"system:nodes"}, runtime.ClientKubeProxyCert, runtime.ClientKubeProxyKey); err != nil { + if _, err = factory("system:kube-proxy", nil, runtime.ClientKubeProxyCert, runtime.ClientKubeProxyKey); err != nil { + return err + } + if _, err = factory("system:k3s-controller", nil, runtime.ClientK3sControllerCert, runtime.ClientK3sControllerKey); err != nil { return err } diff --git a/pkg/daemons/control/tunnel.go b/pkg/daemons/control/tunnel.go index a304c4dd3d..6e1d591ac0 100644 --- a/pkg/daemons/control/tunnel.go +++ b/pkg/daemons/control/tunnel.go @@ -4,6 +4,7 @@ import ( "context" "net" "net/http" + "strings" "time" "github.com/rancher/remotedialer" @@ -37,14 +38,9 @@ func authorizer(req *http.Request) (clientKey string, authed bool, err error) { return "", false, nil } - if user.GetName() != "node" { - return "", false, nil + if strings.HasPrefix(user.GetName(), "system:node:") { + return strings.TrimPrefix(user.GetName(), "system:node:"), true, nil } - nodeName := req.Header.Get("X-K3s-NodeName") - if nodeName == "" { - return "", false, nil - } - - return nodeName, true, nil + return "", false, nil } diff --git a/pkg/datadir/datadir.go b/pkg/datadir/datadir.go index 0e80d3fb9c..298998d872 100644 --- a/pkg/datadir/datadir.go +++ b/pkg/datadir/datadir.go @@ -2,6 +2,7 @@ package datadir import ( "os" + "path/filepath" "github.com/pkg/errors" "github.com/rancher/wrangler/pkg/resolvehome" @@ -32,5 +33,5 @@ func LocalHome(dataDir string, forceLocal bool) (string, error) { return "", errors.Wrapf(err, "resolving %s", dataDir) } - return dataDir, nil + return filepath.Abs(dataDir) } diff --git a/pkg/passwd/passwd.go b/pkg/passwd/passwd.go new file mode 100644 index 0000000000..097a78dac6 --- /dev/null +++ b/pkg/passwd/passwd.go @@ -0,0 +1,151 @@ +package passwd + +import ( + "encoding/csv" + "fmt" + "io" + "os" + "strings" + + "github.com/rancher/k3s/pkg/token" +) + +type entry struct { + pass string + role string +} + +type Passwd struct { + changed bool + names map[string]entry +} + +func Read(file string) (*Passwd, error) { + result := &Passwd{ + names: map[string]entry{}, + } + + f, err := os.Open(file) + if err != nil { + if os.IsNotExist(err) { + return result, nil + } + return nil, err + } + defer f.Close() + + reader := csv.NewReader(f) + reader.FieldsPerRecord = -1 + for { + record, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if len(record) < 3 { + return nil, fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", file, len(record)) + } + e := entry{ + pass: record[0], + } + if len(record) > 3 { + e.role = record[3] + } + result.names[record[1]] = e + } + + return result, nil +} + +func (p *Passwd) Check(name, pass string) (matches bool, exists bool) { + e, ok := p.names[name] + if !ok { + return false, false + } + return e.pass == pass, true +} + +func (p *Passwd) EnsureUser(name, role, passwd string) error { + tokenPrefix := "::" + name + ":" + idx := strings.Index(passwd, tokenPrefix) + if idx > 0 && strings.HasPrefix(passwd, "K10") { + passwd = passwd[idx+len(tokenPrefix):] + } + + if e, ok := p.names[name]; ok { + if passwd != "" && e.pass != passwd { + p.changed = true + e.pass = passwd + } + + if e.role != role { + p.changed = true + e.role = role + } + + p.names[name] = e + return nil + } + + if passwd == "" { + token, err := token.Random(16) + if err != nil { + return err + } + passwd = token + } + + p.changed = true + p.names[name] = entry{ + pass: passwd, + role: role, + } + + return nil +} + +func (p *Passwd) Pass(name string) (string, bool) { + e, ok := p.names[name] + if !ok { + return "", false + } + return e.pass, true +} + +func (p *Passwd) Write(passwdFile string) error { + if !p.changed { + return nil + } + + var records [][]string + for name, e := range p.names { + records = append(records, []string{ + e.pass, + name, + name, + e.role, + }) + } + + return writePasswords(passwdFile, records) +} + +func writePasswords(passwdFile string, records [][]string) error { + out, err := os.Create(passwdFile + ".tmp") + if err != nil { + return err + } + defer out.Close() + + if err := out.Chmod(0600); err != nil { + return err + } + + if err := csv.NewWriter(out).WriteAll(records); err != nil { + return err + } + + return os.Rename(passwdFile+".tmp", passwdFile) +} diff --git a/pkg/server/auth.go b/pkg/server/auth.go index e0cb0a0d6d..34126f438f 100644 --- a/pkg/server/auth.go +++ b/pkg/server/auth.go @@ -9,9 +9,21 @@ import ( "k8s.io/apiserver/pkg/endpoints/request" ) -func doAuth(serverConfig *config.Control, next http.Handler, rw http.ResponseWriter, req *http.Request) { +func hasRole(mustRoles []string, roles []string) bool { + for _, check := range roles { + for _, role := range mustRoles { + if role == check { + return true + } + } + } + return false +} + +func doAuth(roles []string, serverConfig *config.Control, next http.Handler, rw http.ResponseWriter, req *http.Request) { if serverConfig == nil || serverConfig.Runtime.Authenticator == nil { - next.ServeHTTP(rw, req) + logrus.Errorf("authenticate not initialized") + rw.WriteHeader(http.StatusUnauthorized) return } @@ -22,7 +34,7 @@ func doAuth(serverConfig *config.Control, next http.Handler, rw http.ResponseWri return } - if !ok || resp.User.GetName() != "node" { + if !ok || !hasRole(roles, resp.User.GetGroups()) { rw.WriteHeader(http.StatusUnauthorized) return } @@ -32,10 +44,10 @@ func doAuth(serverConfig *config.Control, next http.Handler, rw http.ResponseWri next.ServeHTTP(rw, req) } -func authMiddleware(serverConfig *config.Control) mux.MiddlewareFunc { +func authMiddleware(serverConfig *config.Control, roles ...string) mux.MiddlewareFunc { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - doAuth(serverConfig, next, rw, req) + doAuth(roles, serverConfig, next, rw, req) }) } } diff --git a/pkg/server/context.go b/pkg/server/context.go index 9c65f19e36..eec39b39e3 100644 --- a/pkg/server/context.go +++ b/pkg/server/context.go @@ -62,7 +62,6 @@ func crds(ctx context.Context, config *rest.Config) error { } factory.BatchCreateCRDs(ctx, crd.NamespacedTypes( - "ListenerConfig.k3s.cattle.io/v1", "Addon.k3s.cattle.io/v1", "HelmChart.helm.cattle.io/v1")...) diff --git a/pkg/server/router.go b/pkg/server/router.go index e9cb680596..83ed3e06f0 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -3,70 +3,68 @@ package server import ( "crypto" "crypto/x509" - "encoding/csv" "errors" "fmt" - "io" "io/ioutil" "net" "net/http" - "os" "path/filepath" "strconv" "strings" "github.com/gorilla/mux" certutil "github.com/rancher/dynamiclistener/cert" + "github.com/rancher/k3s/pkg/bootstrap" "github.com/rancher/k3s/pkg/daemons/config" - "github.com/rancher/k3s/pkg/daemons/control" + "github.com/rancher/k3s/pkg/passwd" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/json" ) const ( - jsonMediaType = "application/json" - binaryMediaType = "application/octet-stream" - pbMediaType = "application/com.github.proto-openapi.spec.v2@v1.0+protobuf" - openapiPrefix = "openapi." - staticURL = "/static/" + staticURL = "/static/" ) -type CACertsGetter func() (string, error) - -func router(serverConfig *config.Control, tunnel http.Handler, cacertsGetter CACertsGetter) http.Handler { +func router(serverConfig *config.Control, tunnel http.Handler, ca []byte) http.Handler { authed := mux.NewRouter() - authed.Use(authMiddleware(serverConfig)) + authed.Use(authMiddleware(serverConfig, "k3s:agent")) authed.NotFoundHandler = serverConfig.Runtime.Handler - authed.Path("/v1-k3s/connect").Handler(tunnel) authed.Path("/v1-k3s/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig)) authed.Path("/v1-k3s/serving-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ServingKubeletKey)) authed.Path("/v1-k3s/client-kubelet.crt").Handler(clientKubeletCert(serverConfig)) authed.Path("/v1-k3s/client-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeletKey)) authed.Path("/v1-k3s/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert)) authed.Path("/v1-k3s/client-kube-proxy.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyKey)) + authed.Path("/v1-k3s/client-k3s-controller.crt").Handler(fileHandler(serverConfig.Runtime.ClientK3sControllerCert)) + authed.Path("/v1-k3s/client-k3s-controller.key").Handler(fileHandler(serverConfig.Runtime.ClientK3sControllerKey)) authed.Path("/v1-k3s/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA)) authed.Path("/v1-k3s/server-ca.crt").Handler(fileHandler(serverConfig.Runtime.ServerCA)) authed.Path("/v1-k3s/config").Handler(configHandler(serverConfig)) + nodeAuthed := mux.NewRouter() + nodeAuthed.Use(authMiddleware(serverConfig, "system:nodes")) + nodeAuthed.Path("/v1-k3s/connect").Handler(tunnel) + nodeAuthed.NotFoundHandler = authed + + serverAuthed := mux.NewRouter() + serverAuthed.Use(authMiddleware(serverConfig, "k3s:server")) + serverAuthed.NotFoundHandler = nodeAuthed + serverAuthed.Path("/v1-k3s/server-bootstrap").Handler(bootstrap.Handler(&serverConfig.Runtime.ControlRuntimeBootstrap)) + staticDir := filepath.Join(serverConfig.DataDir, "static") router := mux.NewRouter() - router.NotFoundHandler = authed + router.NotFoundHandler = serverAuthed router.PathPrefix(staticURL).Handler(serveStatic(staticURL, staticDir)) - router.Path("/cacerts").Handler(cacerts(cacertsGetter)) + router.Path("/cacerts").Handler(cacerts(ca)) router.Path("/ping").Handler(ping()) return router } -func cacerts(getter CACertsGetter) http.Handler { +func cacerts(ca []byte) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - content, err := getter() - if err != nil { - resp.WriteHeader(http.StatusInternalServerError) - resp.Write([]byte(err.Error())) - } resp.Header().Set("content-type", "text/plain") - resp.Write([]byte(content)) + resp.Write(ca) }) } @@ -242,38 +240,19 @@ func sendError(err error, resp http.ResponseWriter, status ...int) { resp.Write([]byte(err.Error())) } -func ensureNodePassword(passwdFile, nodeName, passwd string) error { - records := [][]string{} - - if _, err := os.Stat(passwdFile); !os.IsNotExist(err) { - f, err := os.Open(passwdFile) - if err != nil { - return err - } - defer f.Close() - reader := csv.NewReader(f) - for { - record, err := reader.Read() - if err == io.EOF { - break - } - if err != nil { - return err - } - if len(record) < 2 { - return fmt.Errorf("password file '%s' must have at least 2 columns (password, nodeName), found %d", passwdFile, len(record)) - } - if record[1] == nodeName { - if record[0] == passwd { - return nil - } - return fmt.Errorf("Node password validation failed for '%s', using passwd file '%s'", nodeName, passwdFile) - } - records = append(records, record) - } - f.Close() +func ensureNodePassword(passwdFile, nodeName, pass string) error { + passwd, err := passwd.Read(passwdFile) + if err != nil { + return err } - - records = append(records, []string{passwd, nodeName}) - return control.WritePasswords(passwdFile, records) + match, exists := passwd.Check(nodeName, pass) + if exists { + if !match { + return fmt.Errorf("Node password validation failed for '%s', using passwd file '%s'", nodeName, passwdFile) + } + return nil + } + // If user doesn't exist we save this password for future validation + passwd.EnsureUser(nodeName, "", pass) + return passwd.Write(passwdFile) } diff --git a/pkg/server/server.go b/pkg/server/server.go index 0243a26b74..2913c13b5a 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -14,7 +14,6 @@ import ( "time" "github.com/pkg/errors" - "github.com/rancher/dynamiclistener" "github.com/rancher/helm-controller/pkg/helm" "github.com/rancher/k3s/pkg/clientaccess" "github.com/rancher/k3s/pkg/daemons/config" @@ -25,10 +24,11 @@ import ( "github.com/rancher/k3s/pkg/rootlessports" "github.com/rancher/k3s/pkg/servicelb" "github.com/rancher/k3s/pkg/static" - "github.com/rancher/k3s/pkg/tls" + v1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1" "github.com/rancher/wrangler/pkg/leader" "github.com/rancher/wrangler/pkg/resolvehome" "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/net" ) @@ -39,79 +39,67 @@ func resolveDataDir(dataDir string) (string, error) { return filepath.Join(dataDir, "server"), err } -func StartServer(ctx context.Context, config *Config) (string, error) { +func StartServer(ctx context.Context, config *Config) error { if err := setupDataDirAndChdir(&config.ControlConfig); err != nil { - return "", err + return err } if err := setNoProxyEnv(&config.ControlConfig); err != nil { - return "", err + return err } if err := control.Server(ctx, &config.ControlConfig); err != nil { - return "", errors.Wrap(err, "starting kubernetes") + return errors.Wrap(err, "starting kubernetes") } - certs, err := startWrangler(ctx, config) - if err != nil { - return "", errors.Wrap(err, "starting tls server") + if err := startWrangler(ctx, config); err != nil { + return errors.Wrap(err, "starting tls server") } - ip := net2.ParseIP(config.TLSConfig.BindAddress) + ip := net2.ParseIP(config.ControlConfig.BindAddress) if ip == nil { - ip, err = net.ChooseHostInterface() - if err != nil { + hostIP, err := net.ChooseHostInterface() + if err == nil { + ip = hostIP + } else { ip = net2.ParseIP("127.0.0.1") } } - printTokens(certs, ip.String(), &config.TLSConfig, &config.ControlConfig) - writeKubeConfig(certs, &config.TLSConfig, config) + if err := printTokens(ip.String(), &config.ControlConfig); err != nil { + return err + } - return certs, nil + return writeKubeConfig(config.ControlConfig.Runtime.ServerCA, config) } -func startWrangler(ctx context.Context, config *Config) (string, error) { +func startWrangler(ctx context.Context, config *Config) error { var ( err error - tlsConfig = &config.TLSConfig controlConfig = &config.ControlConfig ) - caBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCA) + ca, err := ioutil.ReadFile(config.ControlConfig.Runtime.ServerCA) if err != nil { - return "", err - } - caKeyBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCAKey) - if err != nil { - return "", err + return err } - certs := string(caBytes) - tlsConfig.CACerts = certs - tlsConfig.CAKey = string(caKeyBytes) - - tlsConfig.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, func() (string, error) { - return certs, nil - }) + controlConfig.Runtime.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, ca) sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigAdmin) if err != nil { - return "", err + return err } if err := stageFiles(ctx, sc, controlConfig); err != nil { - return "", err + return err } - _, err = tls.NewServer(ctx, sc.K3s.K3s().V1().ListenerConfig(), *tlsConfig) - if err != nil { - return "", err + if err := sc.Start(ctx); err != nil { + return err } - if err := startNodeCache(ctx, sc); err != nil { - return "", err - } + controlConfig.Runtime.Core = sc.Core start := func(ctx context.Context) { if err := masterControllers(ctx, sc, config); err != nil { @@ -122,7 +110,7 @@ func startWrangler(ctx context.Context, config *Config) (string, error) { } } if !config.DisableAgent { - go setMasterRoleLabel(ctx, sc) + go setMasterRoleLabel(ctx, sc.Core.Core().V1().Node()) } if controlConfig.NoLeaderElect { go func() { @@ -134,7 +122,7 @@ func startWrangler(ctx context.Context, config *Config) (string, error) { go leader.RunOrDie(ctx, "", "k3s", sc.K8s, start) } - return certs, nil + return nil } func masterControllers(ctx context.Context, sc *Context, config *Config) error { @@ -162,7 +150,7 @@ func masterControllers(ctx context.Context, sc *Context, config *Config) error { } if !config.DisableServiceLB && config.Rootless { - return rootlessports.Register(ctx, sc.Core.Core().V1().Service(), config.TLSConfig.HTTPSPort) + return rootlessports.Register(ctx, sc.Core.Core().V1().Service(), config.ControlConfig.HTTPSPort) } return nil @@ -203,7 +191,7 @@ func HomeKubeConfig(write, rootless bool) (string, error) { return resolvehome.Resolve(datadir.HomeConfig) } -func printTokens(certs, advertiseIP string, tlsConfig *dynamiclistener.UserConfig, config *config.Control) { +func printTokens(advertiseIP string, config *config.Control) error { var ( nodeFile string ) @@ -212,26 +200,43 @@ func printTokens(certs, advertiseIP string, tlsConfig *dynamiclistener.UserConfi advertiseIP = "127.0.0.1" } - if len(config.Runtime.NodeToken) > 0 { - p := filepath.Join(config.DataDir, "node-token") - if err := writeToken(config.Runtime.NodeToken, p, certs); err == nil { + if len(config.Runtime.AgentToken) > 0 { + p := filepath.Join(config.DataDir, "token") + if err := writeToken(config.Runtime.AgentToken, p, config.Runtime.ServerCA); err == nil { logrus.Infof("Node token is available at %s", p) nodeFile = p } + + // backwards compatibility + np := filepath.Join(config.DataDir, "node-token") + if !isSymlink(np) { + if err := os.RemoveAll(np); err != nil { + return err + } + if err := os.Symlink(p, np); err != nil { + return err + } + } } if len(nodeFile) > 0 { - printToken(tlsConfig.HTTPSPort, advertiseIP, "To join node to cluster:", "agent") + printToken(config.HTTPSPort, advertiseIP, "To join node to cluster:", "agent") } + + return nil } -func writeKubeConfig(certs string, tlsConfig *dynamiclistener.UserConfig, config *Config) { - clientToken := FormatToken(config.ControlConfig.Runtime.ClientToken, certs) - ip := tlsConfig.BindAddress +func writeKubeConfig(certs string, config *Config) error { + clientToken, err := FormatToken(config.ControlConfig.Runtime.ClientToken, certs) + if err != nil { + return err + } + + ip := config.ControlConfig.BindAddress if ip == "" { ip = "127.0.0.1" } - url := fmt.Sprintf("https://%s:%d", ip, tlsConfig.HTTPSPort) + url := fmt.Sprintf("https://%s:%d", ip, config.ControlConfig.HTTPSPort) kubeConfig, err := HomeKubeConfig(true, config.Rootless) def := true if err != nil { @@ -274,6 +279,8 @@ func writeKubeConfig(certs string, tlsConfig *dynamiclistener.UserConfig, config if def { logrus.Infof("Run: %s kubectl", filepath.Base(os.Args[0])) } + + return nil } func setupDataDirAndChdir(config *config.Control) error { @@ -312,18 +319,22 @@ func printToken(httpsPort int, advertiseIP, prefix, cmd string) { logrus.Infof("%s k3s %s -s https://%s:%d -t ${NODE_TOKEN}", prefix, cmd, ip, httpsPort) } -func FormatToken(token string, certs string) string { +func FormatToken(token string, certFile string) (string, error) { if len(token) == 0 { - return token + return token, nil } prefix := "K10" - if len(certs) > 0 { - digest := sha256.Sum256([]byte(certs)) + if len(certFile) > 0 { + bytes, err := ioutil.ReadFile(certFile) + if err != nil { + return "", nil + } + digest := sha256.Sum256(bytes) prefix = "K10" + hex.EncodeToString(digest[:]) + "::" } - return prefix + token + return prefix + token, nil } func writeToken(token, file, certs string) error { @@ -331,7 +342,10 @@ func writeToken(token, file, certs string) error { return nil } - token = FormatToken(token, certs) + token, err := FormatToken(token, certs) + if err != nil { + return err + } return ioutil.WriteFile(file, []byte(token+"\n"), 0600) } @@ -364,26 +378,23 @@ func isSymlink(config string) bool { return false } -func setMasterRoleLabel(ctx context.Context, sc *Context) error { +func setMasterRoleLabel(ctx context.Context, nodes v1.NodeClient) error { for { nodeName := os.Getenv("NODE_NAME") - nodeController := sc.Core.Core().V1().Node() - nodeCache := nodeController.Cache() - nodeCached, err := nodeCache.Get(nodeName) + node, err := nodes.Get(nodeName, metav1.GetOptions{}) if err != nil { logrus.Infof("Waiting for master node %s startup: %v", nodeName, err) time.Sleep(1 * time.Second) continue } - if v, ok := nodeCached.Labels[MasterRoleLabelKey]; ok && v == "true" { + if v, ok := node.Labels[MasterRoleLabelKey]; ok && v == "true" { break } - node := nodeCached.DeepCopy() if node.Labels == nil { node.Labels = make(map[string]string) } node.Labels[MasterRoleLabelKey] = "true" - _, err = nodeController.Update(node) + _, err = nodes.Update(node) if err == nil { logrus.Infof("master role label has been set succesfully on node: %s", nodeName) break @@ -396,9 +407,3 @@ func setMasterRoleLabel(ctx context.Context, sc *Context) error { } return nil } - -func startNodeCache(ctx context.Context, sc *Context) error { - sc.Core.Core().V1().Node().Cache() - - return sc.Start(ctx) -} diff --git a/pkg/server/types.go b/pkg/server/types.go index 4bccf98fd8..cba0633e56 100644 --- a/pkg/server/types.go +++ b/pkg/server/types.go @@ -1,14 +1,12 @@ package server import ( - "github.com/rancher/dynamiclistener" "github.com/rancher/k3s/pkg/daemons/config" ) type Config struct { DisableAgent bool DisableServiceLB bool - TLSConfig dynamiclistener.UserConfig ControlConfig config.Control Rootless bool } diff --git a/pkg/token/read.go b/pkg/token/read.go new file mode 100644 index 0000000000..1789ef80bc --- /dev/null +++ b/pkg/token/read.go @@ -0,0 +1,39 @@ +package token + +import ( + cryptorand "crypto/rand" + "encoding/hex" + "io/ioutil" + "os" + "strings" + "time" + + "github.com/sirupsen/logrus" +) + +func Random(size int) (string, error) { + token := make([]byte, size, size) + _, err := cryptorand.Read(token) + if err != nil { + return "", err + } + return hex.EncodeToString(token), err +} + +func ReadFile(path string) (string, error) { + if path == "" { + return "", nil + } + + for { + tokenBytes, err := ioutil.ReadFile(path) + if err == nil { + return strings.TrimSpace(string(tokenBytes)), nil + } else if os.IsNotExist(err) { + logrus.Infof("Waiting for %s to be available\n", path) + time.Sleep(2 * time.Second) + } else { + return "", err + } + } +}