Server Token Rotation (#8265)

* Consolidate NewCertCommands
* Add support for user defined new token
* Add E2E testlets

Signed-off-by: Derek Nola <derek.nola@suse.com>

* Ensure agent token also changes

Signed-off-by: Derek Nola <derek.nola@suse.com>

cmd/cert/main.go

@@ -15,11 +15,9 @@ import (
 func main() {
 	app := cmds.NewApp()
 	app.Commands = []cli.Command{
-		cmds.NewCertCommand(
+		cmds.NewCertCommands(
-			cmds.NewCertSubcommands(
+			cert.Rotate,
-				cert.Rotate,
+			cert.RotateCA,
-				cert.RotateCA,
-			),
 		),
 	}
 

cmd/k3s/main.go

@@ -57,6 +57,7 @@ func main() {
 			tokenCommand,
 			tokenCommand,
 			tokenCommand,
+			tokenCommand,
 		),
 		cmds.NewEtcdSnapshotCommands(
 			etcdsnapshotCommand,
@@ -73,11 +74,9 @@ func main() {
 			secretsencryptCommand,
 			secretsencryptCommand,
 		),
-		cmds.NewCertCommand(
+		cmds.NewCertCommands(
-			cmds.NewCertSubcommands(
+			certCommand,
-				certCommand,
+			certCommand,
-				certCommand,
-			),
 		),
 		cmds.NewCompletionCommand(internalCLIAction(version.Program+"-completion", dataDir, os.Args)),
 	}

cmd/server/main.go

@@ -54,6 +54,7 @@ func main() {
 			token.Delete,
 			token.Generate,
 			token.List,
+			token.Rotate,
 		),
 		cmds.NewEtcdSnapshotCommands(
 			etcdsnapshot.Delete,
@@ -70,11 +71,9 @@ func main() {
 			secretsencrypt.Reencrypt,
 			secretsencrypt.RotateKeys,
 		),
-		cmds.NewCertCommand(
+		cmds.NewCertCommands(
-			cmds.NewCertSubcommands(
+			cert.Rotate,
-				cert.Rotate,
+			cert.RotateCA,
-				cert.RotateCA,
-			),
 		),
 		cmds.NewCompletionCommand(completion.Run),
 	}

cmd/token/main.go

@@ -20,6 +20,7 @@ func main() {
 			token.Delete,
 			token.Generate,
 			token.List,
+			token.Rotate,
 		),
 	}
 

main.go

@@ -47,11 +47,9 @@ func main() {
 			secretsencrypt.Reencrypt,
 			secretsencrypt.RotateKeys,
 		),
-		cmds.NewCertCommand(
+		cmds.NewCertCommands(
-			cmds.NewCertSubcommands(
+			cert.Rotate,
-				cert.Rotate,
+			cert.RotateCA,
-				cert.RotateCA,
-			),
 		),
 		cmds.NewCompletionCommand(completion.Run),
 	}

pkg/cli/cmds/certs.go

@@ -54,33 +54,29 @@ var (
 	}
 )
 
-func NewCertCommand(subcommands []cli.Command) cli.Command {
+func NewCertCommands(rotate, rotateCA func(ctx *cli.Context) error) cli.Command {
 	return cli.Command{
 		Name:            CertCommand,
 		Usage:           "Manage K3s certificates",
 		SkipFlagParsing: false,
 		SkipArgReorder:  true,
-		Subcommands:     subcommands,
+		Subcommands: []cli.Command{
-	}
+			{
-}
+				Name:            "rotate",
-
+				Usage:           "Rotate " + version.Program + " component certificates on disk",
-func NewCertSubcommands(rotate, rotateCA func(ctx *cli.Context) error) []cli.Command {
+				SkipFlagParsing: false,
-	return []cli.Command{
+				SkipArgReorder:  true,
-		{
+				Action:          rotate,
-			Name:            "rotate",
+				Flags:           CertRotateCommandFlags,
-			Usage:           "Rotate " + version.Program + " component certificates on disk",
+			},
-			SkipFlagParsing: false,
+			{
-			SkipArgReorder:  true,
+				Name:            "rotate-ca",
-			Action:          rotate,
+				Usage:           "Write updated " + version.Program + " CA certificates to the datastore",
-			Flags:           CertRotateCommandFlags,
+				SkipFlagParsing: false,
-		},
+				SkipArgReorder:  true,
-		{
+				Action:          rotateCA,
-			Name:            "rotate-ca",
+				Flags:           CertRotateCACommandFlags,
-			Usage:           "Write updated " + version.Program + " CA certificates to the datastore",
+			},
-			SkipFlagParsing: false,
-			SkipArgReorder:  true,
-			Action:          rotateCA,
-			Flags:           CertRotateCACommandFlags,
 		},
 	}
 }

pkg/cli/cmds/token.go

@@ -3,6 +3,7 @@ package cmds
 import (
 	"time"
 
+	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/urfave/cli"
 )
 
@@ -12,7 +13,9 @@ const TokenCommand = "token"
 type Token struct {
 	Description string
 	Kubeconfig  string
+	ServerURL   string
 	Token       string
+	NewToken    string
 	Output      string
 	Groups      cli.StringSlice
 	Usages      cli.StringSlice
@@ -32,7 +35,7 @@ var (
 	}
 )
 
-func NewTokenCommands(create, delete, generate, list func(ctx *cli.Context) error) cli.Command {
+func NewTokenCommands(create, delete, generate, list, rotate func(ctx *cli.Context) error) cli.Command {
 	return cli.Command{
 		Name:            TokenCommand,
 		Usage:           "Manage bootstrap tokens",
@@ -92,6 +95,32 @@ func NewTokenCommands(create, delete, generate, list func(ctx *cli.Context) erro
 				SkipArgReorder:  true,
 				Action:          list,
 			},
+			{
+				Name:  "rotate",
+				Usage: "Rotate original server token with a new bootstrap token",
+				Flags: append(TokenFlags,
+					&cli.StringFlag{
+						Name:        "token,t",
+						Usage:       "Existing token used to join a server or agent to a cluster",
+						Destination: &TokenConfig.Token,
+						EnvVar:      version.ProgramUpper + "_TOKEN",
+					},
+					&cli.StringFlag{
+						Name:        "server, s",
+						Usage:       "(cluster) Server to connect to",
+						Destination: &TokenConfig.ServerURL,
+						EnvVar:      version.ProgramUpper + "_URL",
+						Value:       "https://127.0.0.1:6443",
+					},
+					&cli.StringFlag{
+						Name:        "new-token",
+						Usage:       "New token that replaces existing token",
+						Destination: &TokenConfig.NewToken,
+					}),
+				SkipFlagParsing: false,
+				SkipArgReorder:  true,
+				Action:          rotate,
+			},
 		},
 	}
 }

pkg/cli/token/token.go

@@ -1,18 +1,23 @@
 package token
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 	"text/tabwriter"
 	"time"
 
+	"github.com/erikdubbelboer/gspt"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/kubeadm"
+	"github.com/k3s-io/k3s/pkg/server"
 	"github.com/k3s-io/k3s/pkg/util"
+	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
 	"gopkg.in/yaml.v2"
@@ -22,6 +27,7 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
 	bootstraputil "k8s.io/cluster-bootstrap/token/util"
+	"k8s.io/utils/pointer"
 )
 
 func Create(app *cli.Context) error {
@@ -139,6 +145,50 @@ func generate(app *cli.Context, cfg *cmds.Token) error {
 	return nil
 }
 
+func Rotate(app *cli.Context) error {
+	if err := cmds.InitLogging(); err != nil {
+		return err
+	}
+	fmt.Println("\033[33mWARNING\033[0m: Recommended to keep a record of the old token. If restoring from a snapshot, you must use the token associated with that snapshot.")
+	info, err := serverAccess(&cmds.TokenConfig)
+	if err != nil {
+		return err
+	}
+	b, err := json.Marshal(server.TokenRotateRequest{
+		NewToken: pointer.String(cmds.TokenConfig.NewToken),
+	})
+	if err != nil {
+		return err
+	}
+	if err = info.Put("/v1-"+version.Program+"/token", b); err != nil {
+		return err
+	}
+	// wait for etcd db propagation delay
+	time.Sleep(1 * time.Second)
+	fmt.Println("Token rotated, restart k3s with new token")
+	return nil
+}
+
+func serverAccess(cfg *cmds.Token) (*clientaccess.Info, error) {
+	// hide process arguments from ps output, since they likely contain tokens.
+	gspt.SetProcTitle(os.Args[0] + " token")
+
+	dataDir, err := server.ResolveDataDir("")
+	if err != nil {
+		return nil, err
+	}
+
+	if cfg.Token == "" {
+		fp := filepath.Join(dataDir, "token")
+		tokenByte, err := os.ReadFile(fp)
+		if err != nil {
+			return nil, err
+		}
+		cfg.Token = string(bytes.TrimRight(tokenByte, "\n"))
+	}
+	return clientaccess.ParseAndValidateToken(cfg.ServerURL, cfg.Token, clientaccess.WithUser("server"))
+}
+
 func List(app *cli.Context) error {
 	if err := cmds.InitLogging(); err != nil {
 		return err

pkg/cluster/storage.go

@@ -21,6 +21,49 @@ import (
 // After this many attempts, the lock is deleted and the counter reset.
 const maxBootstrapWaitAttempts = 5
 
+func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken string) error {
+
+	token, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
+	if err != nil {
+		return err
+	}
+
+	normalizedToken, err := normalizeToken(token)
+	if err != nil {
+		return err
+	}
+
+	storageClient, err := client.New(config.Runtime.EtcdConfig)
+	if err != nil {
+		return err
+	}
+	defer storageClient.Close()
+
+	tokenKey := storageKey(normalizedToken)
+
+	var bootstrapList []client.Value
+	if err := wait.PollImmediateUntilWithContext(ctx, 5*time.Second, func(ctx context.Context) (bool, error) {
+		bootstrapList, err = storageClient.List(ctx, "/bootstrap", 0)
+		if err != nil {
+			return false, err
+		}
+		return true, nil
+	}); err != nil {
+		return err
+	}
+
+	normalizedOldToken, err := normalizeToken(oldToken)
+	if err != nil {
+		return err
+	}
+	// reuse the existing migration function to reencrypt bootstrap data with new token
+	if err := migrateTokens(ctx, bootstrapList, storageClient, "", tokenKey, normalizedToken, normalizedOldToken); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // Save writes the current ControlRuntimeBootstrap data to the datastore. This contains a complete
 // snapshot of the cluster's CA certs and keys, encryption passphrases, etc - encrypted with the join token.
 // This is used when bootstrapping a cluster from a managed database or external etcd cluster.
@@ -225,7 +268,7 @@ func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client
 		logrus.Warn("found multiple bootstrap keys in storage")
 	}
 	// check for empty string key and for old token format with k10 prefix
-	if err := migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil {
+	if err := migrateTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil {
 		return nil, false, err
 	}
 
@@ -236,6 +279,7 @@ func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client
 	}
 	for _, bootstrapKV := range bootstrapList {
 		// ensure bootstrap is stored in the current token's key
+		logrus.Debugf("checking bootstrap key %s against %s", string(bootstrapKV.Key), tokenKey)
 		if string(bootstrapKV.Key) == tokenKey {
 			return &bootstrapKV, false, nil
 		}
@@ -277,21 +321,24 @@ func normalizeToken(token string) (string, error) {
 	return password, nil
 }
 
-// migrateOldTokens will list all keys that has prefix /bootstrap and will check for key that is
+// migrateTokens will list all keys that has prefix /bootstrap and will check for key that is
 // hashed with empty string and keys that is hashed with old token format before normalizing
 // then migrate those and resave only with the normalized token
-func migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error {
+func migrateTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error {
 	oldTokenKey := storageKey(oldToken)
 
 	for _, bootstrapKV := range bootstrapList {
 		// checking for empty string bootstrap key
+		logrus.Debug("Comparing ", string(bootstrapKV.Key), " to ", oldTokenKey)
 		if string(bootstrapKV.Key) == emptyStringKey {
 			logrus.Warn("Bootstrap data encrypted with empty string, deleting and resaving with token")
 			if err := doMigrateToken(ctx, storageClient, bootstrapKV, "", emptyStringKey, token, tokenKey); err != nil {
 				return err
 			}
 		} else if string(bootstrapKV.Key) == oldTokenKey && oldTokenKey != tokenKey {
-			logrus.Warn("bootstrap data encrypted with old token format string, deleting and resaving with token")
+			if emptyStringKey != "" {
+				logrus.Warn("bootstrap data encrypted with old token format string, deleting and resaving with token")
+			}
 			if err := doMigrateToken(ctx, storageClient, bootstrapKV, oldToken, oldTokenKey, token, tokenKey); err != nil {
 				return err
 			}

pkg/daemons/control/deps/deps.go

@@ -243,6 +243,7 @@ func genUsers(config *config.Control) error {
 		return err
 	}
 
+	// if no token is provided on bootstrap, we generate a random token
 	serverPass, err := getServerPass(passwd, config)
 	if err != nil {
 		return err

pkg/server/router.go

@@ -85,6 +85,7 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler
 	serverAuthed.Path(prefix + "/cert/cacerts").Handler(caCertReplaceHandler(serverConfig))
 	serverAuthed.Path("/db/info").Handler(nodeAuthed)
 	serverAuthed.Path(prefix + "/server-bootstrap").Handler(bootstrapHandler(serverConfig.Runtime))
+	serverAuthed.Path(prefix + "/token").Handler(tokenRequestHandler(ctx, serverConfig))
 
 	systemAuthed := mux.NewRouter().SkipClean(true)
 	systemAuthed.NotFoundHandler = serverAuthed

pkg/server/token.go

@@ -0,0 +1,100 @@
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"path/filepath"
+
+	"github.com/k3s-io/k3s/pkg/cluster"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/passwd"
+	"github.com/k3s-io/k3s/pkg/util"
+	"github.com/k3s-io/k3s/pkg/version"
+	"github.com/sirupsen/logrus"
+)
+
+type TokenRotateRequest struct {
+	NewToken *string `json:"newToken,omitempty"`
+}
+
+func getServerTokenRequest(req *http.Request) (TokenRotateRequest, error) {
+	b, err := io.ReadAll(req.Body)
+	if err != nil {
+		return TokenRotateRequest{}, err
+	}
+	result := TokenRotateRequest{}
+	err = json.Unmarshal(b, &result)
+	return result, err
+}
+
+func tokenRequestHandler(ctx context.Context, server *config.Control) http.Handler {
+	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		if req.TLS == nil || req.Method != http.MethodPut {
+			resp.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		var err error
+		sTokenReq, err := getServerTokenRequest(req)
+		logrus.Debug("Received token request")
+		if err != nil {
+			resp.WriteHeader(http.StatusBadRequest)
+			resp.Write([]byte(err.Error()))
+			return
+		}
+		if err = tokenRotate(ctx, server, *sTokenReq.NewToken); err != nil {
+			genErrorMessage(resp, http.StatusInternalServerError, err, "token")
+			return
+		}
+		resp.WriteHeader(http.StatusOK)
+	})
+}
+
+func tokenRotate(ctx context.Context, server *config.Control, newToken string) error {
+	passwd, err := passwd.Read(server.Runtime.PasswdFile)
+	if err != nil {
+		return err
+	}
+
+	if err != nil {
+		return err
+	}
+	oldToken, found := passwd.Pass("server")
+	if !found {
+		return fmt.Errorf("server token not found")
+	}
+	if newToken == "" {
+		newToken, err = util.Random(16)
+		if err != nil {
+			return err
+		}
+	}
+
+	if err := passwd.EnsureUser("server", version.Program+":server", newToken); err != nil {
+		return err
+	}
+
+	// If the agent token is the same a server, we need to change both
+	if agentToken, found := passwd.Pass("node"); found && agentToken == oldToken && server.AgentToken == "" {
+		if err := passwd.EnsureUser("node", version.Program+":agent", newToken); err != nil {
+			return err
+		}
+	}
+
+	if err := passwd.Write(server.Runtime.PasswdFile); err != nil {
+		return err
+	}
+
+	serverTokenFile := filepath.Join(server.DataDir, "token")
+	if err := writeToken("server:"+newToken, serverTokenFile, server.Runtime.ServerCA); err != nil {
+		return err
+	}
+
+	if err := cluster.RotateBootstrapToken(ctx, server, oldToken); err != nil {
+		return err
+	}
+	server.Token = newToken
+	return cluster.Save(ctx, server, true)
+}

tests/e2e/token/Vagrantfile

@@ -1,6 +1,6 @@
 ENV['VAGRANT_NO_PARALLEL'] = 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
-  ["server-0", "agent-0", "agent-1" ])
+  ["server-0", "server-1", "server-2", "agent-0", "agent-1"])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
   ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
@@ -40,6 +40,18 @@ def provision(vm, roles, role_num, node_num)
       YAML
       k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type]
     end
+  elsif roles.include?("server") && role_num != 0
+    vm.provision 'k3s-secondary-server', type: 'k3s', run: 'once' do |k3s|
+      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
+      k3s.args = "server"
+      k3s.config = <<~YAML
+        server: "https://#{NETWORK_PREFIX}.100:6443"
+        token: vagrant
+        node-external-ip: #{node_ip}
+        flannel-iface: eth1
+      YAML
+      k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type]
+    end
   end
   if roles.include?("agent")
     vm.provision :k3s, run: 'once' do |k3s|

tests/e2e/token/token_test.go

@@ -17,7 +17,7 @@ import (
 // generic/ubuntu2004, generic/centos7, generic/rocky8, opensuse/Leap-15.4.x86_64
 
 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
-var serverCount = flag.Int("serverCount", 1, "number of server nodes")
+var serverCount = flag.Int("serverCount", 3, "number of server nodes")
 var agentCount = flag.Int("agentCount", 2, "number of agent nodes")
 var ci = flag.Bool("ci", false, "running on CI")
 var local = flag.Bool("local", false, "deploy a locally built K3s binary")
@@ -104,7 +104,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
 			Eventually(func(g Gomega) {
 				nodes, err := e2e.ParseNodes(kubeConfigFile, false)
 				g.Expect(err).NotTo(HaveOccurred())
-				g.Expect(len(nodes)).Should(Equal(2))
+				g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 1))
 				for _, node := range nodes {
 					g.Expect(node.Status).Should(Equal("Ready"))
 				}
@@ -122,7 +122,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
 		It("Cleans up 20s token automatically", func() {
 			Eventually(func() (string, error) {
 				return e2e.RunCmdOnNode("k3s token list", serverNodeNames[0])
-			}, "20s", "5s").ShouldNot(ContainSubstring("20sect"))
+			}, "25s", "5s").ShouldNot(ContainSubstring("20sect"))
 		})
 		var tempToken string
 		It("Creates a 10m agent token", func() {
@@ -144,7 +144,49 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
 			Eventually(func(g Gomega) {
 				nodes, err := e2e.ParseNodes(kubeConfigFile, false)
 				g.Expect(err).NotTo(HaveOccurred())
-				g.Expect(len(nodes)).Should(Equal(3))
+				g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 2))
+				for _, node := range nodes {
+					g.Expect(node.Status).Should(Equal("Ready"))
+				}
+			}, "60s", "5s").Should(Succeed())
+		})
+	})
+	Context("Rotate server bootstrap token", func() {
+		serverToken := "1234"
+		It("Creates a new server token", func() {
+			Expect(e2e.RunCmdOnNode("k3s token rotate -t vagrant --new-token="+serverToken, serverNodeNames[0])).
+				To(ContainSubstring("Token rotated, restart k3s with new token"))
+		})
+		It("Restarts servers with the new token", func() {
+			cmd := fmt.Sprintf("sed -i 's/token:.*/token: %s/' /etc/rancher/k3s/config.yaml", serverToken)
+			for _, node := range serverNodeNames {
+				_, err := e2e.RunCmdOnNode(cmd, node)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			for _, node := range serverNodeNames {
+				_, err := e2e.RunCmdOnNode("systemctl restart k3s", node)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			Eventually(func(g Gomega) {
+				nodes, err := e2e.ParseNodes(kubeConfigFile, false)
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 2))
+				for _, node := range nodes {
+					g.Expect(node.Status).Should(Equal("Ready"))
+				}
+			}, "60s", "5s").Should(Succeed())
+		})
+		It("Rejoins an agent with the new server token", func() {
+			cmd := fmt.Sprintf("sed -i 's/token:.*/token: %s/' /etc/rancher/k3s/config.yaml", serverToken)
+			_, err := e2e.RunCmdOnNode(cmd, agentNodeNames[0])
+			Expect(err).NotTo(HaveOccurred())
+			_, err = e2e.RunCmdOnNode("systemctl restart k3s-agent", agentNodeNames[0])
+			Expect(err).NotTo(HaveOccurred())
+
+			Eventually(func(g Gomega) {
+				nodes, err := e2e.ParseNodes(kubeConfigFile, false)
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 2))
 				for _, node := range nodes {
 					g.Expect(node.Status).Should(Equal("Ready"))
 				}