Add server token hash to CR and S3

This required pulling the token hash stuff out of the cluster package, into util.

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

docs/adrs/etcd-snapshot-cr.md

@@ -45,10 +45,13 @@ it into a neutral project for use by both projects.
 3. The new Custom Resource will be cluster-scoped, as etcd and its snapshots are a cluster-level resource.
 4. Snapshot metadata will also be written alongside snapshot files created on disk and/or uploaded to S3. The metadata
    files will have the same basename as their corresponding snapshot file.
-5. Downstream consumers of etcd snapshot lists will migrate to watching Custom Resource types, instead of the ConfigMap.
-6. K3s will observe a three minor version transition period, where both the new Custom Resources, and the existing
+5. A hash of the server token will be stored as an annotation on the Custom Resource, and stored as metadata on snapshots uploaded to S3.
+   This hash should be compared to a current etcd snapshot's token hash to determine if the server token must be rolled back as part of the
+   snapshot restore process.
+6. Downstream consumers of etcd snapshot lists will migrate to watching Custom Resource types, instead of the ConfigMap.
+7. K3s will observe a three minor version transition period, where both the new Custom Resources, and the existing
    ConfigMap, will both be used.
-7. During the transition period, older snapshot metadata may be removed from the ConfigMap while those snapshots still
+8. During the transition period, older snapshot metadata may be removed from the ConfigMap while those snapshots still
    exist and are referenced by new Custom Resources, if the ConfigMap exceeds a preset size or key count limit.
 
 ## Consequences

pkg/cluster/bootstrap.go

@@ -19,6 +19,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/etcd"
+	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/kine/pkg/client"
 	"github.com/k3s-io/kine/pkg/endpoint"
@@ -248,7 +249,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker,
 	if c.managedDB != nil && !isHTTP {
 		token := c.config.Token
 		if token == "" {
-			tokenFromFile, err := readTokenFromFile(c.config.Runtime.ServerToken, c.config.Runtime.ServerCA, c.config.DataDir)
+			tokenFromFile, err := util.ReadTokenFromFile(c.config.Runtime.ServerToken, c.config.Runtime.ServerCA, c.config.DataDir)
 			if err != nil {
 				return err
 			}
@@ -260,7 +261,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker,
 			token = tokenFromFile
 		}
 
-		normalizedToken, err := normalizeToken(token)
+		normalizedToken, err := util.NormalizeToken(token)
 		if err != nil {
 			return err
 		}

pkg/cluster/encrypt.go

@@ -5,9 +5,7 @@ import (
 	"crypto/cipher"
 	"crypto/rand"
 	"crypto/sha1"
-	"crypto/sha256"
 	"encoding/base64"
-	"encoding/hex"
 	"fmt"
 	"io"
 	"strings"
@@ -19,14 +17,7 @@ import (
 // storageKey returns the etcd key for storing bootstrap data for a given passphrase.
 // The key is derived from the sha256 hash of the passphrase.
 func storageKey(passphrase string) string {
-	return "/bootstrap/" + keyHash(passphrase)
-}
-
-// keyHash returns the first 12 characters of the sha256 sum of the passphrase.
-func keyHash(passphrase string) string {
-	d := sha256.New()
-	d.Write([]byte(passphrase))
-	return hex.EncodeToString(d.Sum(nil)[:])[:12]
+	return "/bootstrap/" + util.ShortHash(passphrase, 12)
 }
 
 // encrypt encrypts a byte slice using aes+gcm with a pbkdf2 key derived from the passphrase and a random salt.

pkg/cluster/storage.go

@@ -4,13 +4,11 @@ import (
 	"bytes"
 	"context"
 	"errors"
-	"os"
-	"path/filepath"
 	"time"
 
 	"github.com/k3s-io/k3s/pkg/bootstrap"
-	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/kine/pkg/client"
 	"github.com/sirupsen/logrus"
 	"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
@@ -23,12 +21,12 @@ const maxBootstrapWaitAttempts = 5
 
 func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken string) error {
 
-	token, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
+	token, err := util.ReadTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
 	if err != nil {
 		return err
 	}
 
-	normalizedToken, err := normalizeToken(token)
+	normalizedToken, err := util.NormalizeToken(token)
 	if err != nil {
 		return err
 	}
@@ -52,7 +50,7 @@ func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken
 		return err
 	}
 
-	normalizedOldToken, err := normalizeToken(oldToken)
+	normalizedOldToken, err := util.NormalizeToken(oldToken)
 	if err != nil {
 		return err
 	}
@@ -76,13 +74,13 @@ func Save(ctx context.Context, config *config.Control, override bool) error {
 	}
 	token := config.Token
 	if token == "" {
-		tokenFromFile, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
+		tokenFromFile, err := util.ReadTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
 		if err != nil {
 			return err
 		}
 		token = tokenFromFile
 	}
-	normalizedToken, err := normalizeToken(token)
+	normalizedToken, err := util.NormalizeToken(token)
 	if err != nil {
 		return err
 	}
@@ -165,7 +163,7 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
 
 	token := c.config.Token
 	if token == "" {
-		tokenFromFile, err := readTokenFromFile(c.config.Runtime.ServerToken, c.config.Runtime.ServerCA, c.config.DataDir)
+		tokenFromFile, err := util.ReadTokenFromFile(c.config.Runtime.ServerToken, c.config.Runtime.ServerCA, c.config.DataDir)
 		if err != nil {
 			return err
 		}
@@ -181,7 +179,7 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
 		}
 		token = tokenFromFile
 	}
-	normalizedToken, err := normalizeToken(token)
+	normalizedToken, err := util.NormalizeToken(token)
 	if err != nil {
 		return err
 	}
@@ -288,39 +286,6 @@ func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client
 	return nil, false, errors.New("bootstrap data already found and encrypted with different token")
 }
 
-// readTokenFromFile will attempt to get the token from <data-dir>/token if it the file not found
-// in case of fresh installation it will try to use the runtime serverToken saved in memory
-// after stripping it from any additional information like the username or cahash, if the file
-// found then it will still strip the token from any additional info
-func readTokenFromFile(serverToken, certs, dataDir string) (string, error) {
-	tokenFile := filepath.Join(dataDir, "token")
-
-	b, err := os.ReadFile(tokenFile)
-	if err != nil {
-		if os.IsNotExist(err) {
-			token, err := clientaccess.FormatToken(serverToken, certs)
-			if err != nil {
-				return token, err
-			}
-			return token, nil
-		}
-		return "", err
-	}
-
-	// strip the token from any new line if its read from file
-	return string(bytes.TrimRight(b, "\n")), nil
-}
-
-// normalizeToken will normalize the token read from file or passed as a cli flag
-func normalizeToken(token string) (string, error) {
-	_, password, ok := clientaccess.ParseUsernamePassword(token)
-	if !ok {
-		return password, errors.New("failed to normalize server token; must be in format K10<CA-HASH>::<USERNAME>:<PASSWORD> or <PASSWORD>")
-	}
-
-	return password, nil
-}
-
 // migrateTokens will list all keys that has prefix /bootstrap and will check for key that is
 // hashed with empty string and keys that is hashed with old token format before normalizing
 // then migrate those and resave only with the normalized token

pkg/etcd/s3.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
@@ -31,6 +32,7 @@ import (
 
 var (
 	clusterIDKey = textproto.CanonicalMIMEHeaderKey(version.Program + "-cluster-id")
+	tokenHashKey = textproto.CanonicalMIMEHeaderKey(version.Program + "-token-hash")
 	nodeNameKey  = textproto.CanonicalMIMEHeaderKey(version.Program + "-node-name")
 )
 
@@ -39,6 +41,7 @@ type S3 struct {
 	config    *config.Control
 	client    *minio.Client
 	clusterID string
+	tokenHash string
 	nodeName  string
 }
 
@@ -109,10 +112,16 @@ func NewS3(ctx context.Context, config *config.Control) (*S3, error) {
 		clusterID = string(ns.UID)
 	}
 
+	tokenHash, err := util.GetTokenHash(config)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get server token hash for etcd snapshot")
+	}
+
 	return &S3{
 		config:    config,
 		client:    c,
 		clusterID: clusterID,
+		tokenHash: tokenHash,
 		nodeName:  os.Getenv("NODE_NAME"),
 	}, nil
 }
@@ -154,6 +163,7 @@ func (s *S3) upload(ctx context.Context, snapshot string, extraMetadata *v1.Conf
 	} else {
 		sf.Status = successfulSnapshotStatus
 		sf.Size = uploadInfo.Size
+		sf.tokenHash = s.tokenHash
 	}
 	if _, err := s.uploadSnapshotMetadata(ctx, metadataKey, metadata); err != nil {
 		logrus.Warnf("Failed to upload snapshot metadata to S3: %v", err)
@@ -170,6 +180,7 @@ func (s *S3) uploadSnapshot(ctx context.Context, key, path string) (info minio.U
 		UserMetadata: map[string]string{
 			clusterIDKey: s.clusterID,
 			nodeNameKey:  s.nodeName,
+			tokenHashKey: s.tokenHash,
 		},
 	}
 	if strings.HasSuffix(key, compressedExtension) {
@@ -392,6 +403,7 @@ func (s *S3) listSnapshots(ctx context.Context) (map[string]snapshotFile, error)
 			Status:     successfulSnapshotStatus,
 			Compressed: compressed,
 			nodeSource: obj.UserMetadata[nodeNameKey],
+			tokenHash:  obj.UserMetadata[tokenHashKey],
 		}
 		sfKey := generateSnapshotConfigMapKey(sf)
 		snapshots[sfKey] = sf

pkg/etcd/snapshot.go

@@ -57,6 +57,7 @@ var (
 	labelStorageNode                   = "etcd." + version.Program + ".cattle.io/snapshot-storage-node"
 	annotationLocalReconciled          = "etcd." + version.Program + ".cattle.io/local-snapshots-timestamp"
 	annotationS3Reconciled             = "etcd." + version.Program + ".cattle.io/s3-snapshots-timestamp"
+	annotationTokenHash                = "etcd." + version.Program + ".cattle.io/snapshot-token-hash"
 
 	// snapshotDataBackoff will retry at increasing steps for up to ~30 seconds.
 	// If the ConfigMap update fails, the list won't be reconciled again until next time
@@ -252,6 +253,11 @@ func (e *ETCD) Snapshot(ctx context.Context) error {
 		return errors.Wrap(err, "failed to get config for etcd snapshot")
 	}
 
+	tokenHash, err := util.GetTokenHash(e.config)
+	if err != nil {
+		return errors.Wrap(err, "failed to get server token hash for etcd snapshot")
+	}
+
 	nodeName := os.Getenv("NODE_NAME")
 	now := time.Now().Round(time.Second)
 	snapshotName := fmt.Sprintf("%s-%s-%d", e.config.EtcdSnapshotName, nodeName, now.Unix())
@@ -314,6 +320,7 @@ func (e *ETCD) Snapshot(ctx context.Context) error {
 			Size:           f.Size(),
 			Compressed:     e.config.EtcdSnapshotCompress,
 			metadataSource: extraMetadata,
+			tokenHash:      tokenHash,
 		}
 
 		if err := saveSnapshotMetadata(snapshotPath, extraMetadata); err != nil {
@@ -412,6 +419,7 @@ type snapshotFile struct {
 	// to populate other fields before serialization to the legacy configmap.
 	metadataSource *v1.ConfigMap `json:"-"`
 	nodeSource     string        `json:"-"`
+	tokenHash      string        `json:"-"`
 }
 
 // listLocalSnapshots provides a list of the currently stored
@@ -1016,6 +1024,10 @@ func (sf *snapshotFile) fromETCDSnapshotFile(esf *apisv1.ETCDSnapshotFile) {
 		}
 	}
 
+	if tokenHash := esf.Annotations[annotationTokenHash]; tokenHash != "" {
+		sf.tokenHash = tokenHash
+	}
+
 	if esf.Spec.S3 == nil {
 		sf.NodeName = esf.Spec.NodeName
 	} else {
@@ -1080,6 +1092,14 @@ func (sf *snapshotFile) toETCDSnapshotFile(esf *apisv1.ETCDSnapshotFile) {
 		esf.ObjectMeta.Labels = map[string]string{}
 	}
 
+	if esf.ObjectMeta.Annotations == nil {
+		esf.ObjectMeta.Annotations = map[string]string{}
+	}
+
+	if sf.tokenHash != "" {
+		esf.ObjectMeta.Annotations[annotationTokenHash] = sf.tokenHash
+	}
+
 	if sf.S3 == nil {
 		esf.ObjectMeta.Labels[labelStorageNode] = esf.Spec.NodeName
 	} else {

pkg/util/token.go

@@ -1,8 +1,16 @@
 package util
 
 import (
+	"bytes"
 	cryptorand "crypto/rand"
+	"crypto/sha256"
 	"encoding/hex"
+	"os"
+	"path/filepath"
+
+	"github.com/k3s-io/k3s/pkg/clientaccess"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/pkg/errors"
 )
 
 func Random(size int) (string, error) {
@@ -13,3 +21,57 @@ func Random(size int) (string, error) {
 	}
 	return hex.EncodeToString(token), err
 }
+
+// ReadTokenFromFile will attempt to get the token from <data-dir>/token if it the file not found
+// in case of fresh installation it will try to use the runtime serverToken saved in memory
+// after stripping it from any additional information like the username or cahash, if the file
+// found then it will still strip the token from any additional info
+func ReadTokenFromFile(serverToken, certs, dataDir string) (string, error) {
+	tokenFile := filepath.Join(dataDir, "token")
+
+	b, err := os.ReadFile(tokenFile)
+	if err != nil {
+		if os.IsNotExist(err) {
+			token, err := clientaccess.FormatToken(serverToken, certs)
+			if err != nil {
+				return token, err
+			}
+			return token, nil
+		}
+		return "", err
+	}
+
+	// strip the token from any new line if its read from file
+	return string(bytes.TrimRight(b, "\n")), nil
+}
+
+// NormalizeToken will normalize the token read from file or passed as a cli flag
+func NormalizeToken(token string) (string, error) {
+	_, password, ok := clientaccess.ParseUsernamePassword(token)
+	if !ok {
+		return password, errors.New("failed to normalize server token; must be in format K10<CA-HASH>::<USERNAME>:<PASSWORD> or <PASSWORD>")
+	}
+
+	return password, nil
+}
+
+func GetTokenHash(config *config.Control) (string, error) {
+	token := config.Token
+	if token == "" {
+		tokenFromFile, err := ReadTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
+		if err != nil {
+			return "", err
+		}
+		token = tokenFromFile
+	}
+	normalizedToken, err := NormalizeToken(token)
+	if err != nil {
+		return "", err
+	}
+	return ShortHash(normalizedToken, 12), nil
+}
+
+func ShortHash(s string, i int) string {
+	digest := sha256.Sum256([]byte(s))
+	return hex.EncodeToString(digest[:])[:i]
+}