diff --git a/docs/adrs/etcd-snapshot-cr.md b/docs/adrs/etcd-snapshot-cr.md index 369cbdba64..d4454df7f2 100644 --- a/docs/adrs/etcd-snapshot-cr.md +++ b/docs/adrs/etcd-snapshot-cr.md @@ -45,10 +45,13 @@ it into a neutral project for use by both projects. 3. The new Custom Resource will be cluster-scoped, as etcd and its snapshots are a cluster-level resource. 4. Snapshot metadata will also be written alongside snapshot files created on disk and/or uploaded to S3. The metadata files will have the same basename as their corresponding snapshot file. -5. Downstream consumers of etcd snapshot lists will migrate to watching Custom Resource types, instead of the ConfigMap. -6. K3s will observe a three minor version transition period, where both the new Custom Resources, and the existing +5. A hash of the server token will be stored as an annotation on the Custom Resource, and stored as metadata on snapshots uploaded to S3. + This hash should be compared to a current etcd snapshot's token hash to determine if the server token must be rolled back as part of the + snapshot restore process. +6. Downstream consumers of etcd snapshot lists will migrate to watching Custom Resource types, instead of the ConfigMap. +7. K3s will observe a three minor version transition period, where both the new Custom Resources, and the existing ConfigMap, will both be used. -7. During the transition period, older snapshot metadata may be removed from the ConfigMap while those snapshots still +8. During the transition period, older snapshot metadata may be removed from the ConfigMap while those snapshots still exist and are referenced by new Custom Resources, if the ConfigMap exceeds a preset size or key count limit. ## Consequences diff --git a/pkg/cluster/bootstrap.go b/pkg/cluster/bootstrap.go index 4a5e636a21..a0f8045649 100644 --- a/pkg/cluster/bootstrap.go +++ b/pkg/cluster/bootstrap.go @@ -19,6 +19,7 @@ import ( "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/etcd" + "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" "github.com/k3s-io/kine/pkg/client" "github.com/k3s-io/kine/pkg/endpoint" @@ -248,7 +249,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker, if c.managedDB != nil && !isHTTP { token := c.config.Token if token == "" { - tokenFromFile, err := readTokenFromFile(c.config.Runtime.ServerToken, c.config.Runtime.ServerCA, c.config.DataDir) + tokenFromFile, err := util.ReadTokenFromFile(c.config.Runtime.ServerToken, c.config.Runtime.ServerCA, c.config.DataDir) if err != nil { return err } @@ -260,7 +261,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker, token = tokenFromFile } - normalizedToken, err := normalizeToken(token) + normalizedToken, err := util.NormalizeToken(token) if err != nil { return err } diff --git a/pkg/cluster/encrypt.go b/pkg/cluster/encrypt.go index 1046d61e1a..b39fdc1513 100644 --- a/pkg/cluster/encrypt.go +++ b/pkg/cluster/encrypt.go @@ -5,9 +5,7 @@ import ( "crypto/cipher" "crypto/rand" "crypto/sha1" - "crypto/sha256" "encoding/base64" - "encoding/hex" "fmt" "io" "strings" @@ -19,14 +17,7 @@ import ( // storageKey returns the etcd key for storing bootstrap data for a given passphrase. // The key is derived from the sha256 hash of the passphrase. func storageKey(passphrase string) string { - return "/bootstrap/" + keyHash(passphrase) -} - -// keyHash returns the first 12 characters of the sha256 sum of the passphrase. -func keyHash(passphrase string) string { - d := sha256.New() - d.Write([]byte(passphrase)) - return hex.EncodeToString(d.Sum(nil)[:])[:12] + return "/bootstrap/" + util.ShortHash(passphrase, 12) } // encrypt encrypts a byte slice using aes+gcm with a pbkdf2 key derived from the passphrase and a random salt. diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index 70e3961fdd..5492919612 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -4,13 +4,11 @@ import ( "bytes" "context" "errors" - "os" - "path/filepath" "time" "github.com/k3s-io/k3s/pkg/bootstrap" - "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/kine/pkg/client" "github.com/sirupsen/logrus" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" @@ -23,12 +21,12 @@ const maxBootstrapWaitAttempts = 5 func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken string) error { - token, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) + token, err := util.ReadTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) if err != nil { return err } - normalizedToken, err := normalizeToken(token) + normalizedToken, err := util.NormalizeToken(token) if err != nil { return err } @@ -52,7 +50,7 @@ func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken return err } - normalizedOldToken, err := normalizeToken(oldToken) + normalizedOldToken, err := util.NormalizeToken(oldToken) if err != nil { return err } @@ -76,13 +74,13 @@ func Save(ctx context.Context, config *config.Control, override bool) error { } token := config.Token if token == "" { - tokenFromFile, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) + tokenFromFile, err := util.ReadTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) if err != nil { return err } token = tokenFromFile } - normalizedToken, err := normalizeToken(token) + normalizedToken, err := util.NormalizeToken(token) if err != nil { return err } @@ -165,7 +163,7 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error { token := c.config.Token if token == "" { - tokenFromFile, err := readTokenFromFile(c.config.Runtime.ServerToken, c.config.Runtime.ServerCA, c.config.DataDir) + tokenFromFile, err := util.ReadTokenFromFile(c.config.Runtime.ServerToken, c.config.Runtime.ServerCA, c.config.DataDir) if err != nil { return err } @@ -181,7 +179,7 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error { } token = tokenFromFile } - normalizedToken, err := normalizeToken(token) + normalizedToken, err := util.NormalizeToken(token) if err != nil { return err } @@ -288,39 +286,6 @@ func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client return nil, false, errors.New("bootstrap data already found and encrypted with different token") } -// readTokenFromFile will attempt to get the token from /token if it the file not found -// in case of fresh installation it will try to use the runtime serverToken saved in memory -// after stripping it from any additional information like the username or cahash, if the file -// found then it will still strip the token from any additional info -func readTokenFromFile(serverToken, certs, dataDir string) (string, error) { - tokenFile := filepath.Join(dataDir, "token") - - b, err := os.ReadFile(tokenFile) - if err != nil { - if os.IsNotExist(err) { - token, err := clientaccess.FormatToken(serverToken, certs) - if err != nil { - return token, err - } - return token, nil - } - return "", err - } - - // strip the token from any new line if its read from file - return string(bytes.TrimRight(b, "\n")), nil -} - -// normalizeToken will normalize the token read from file or passed as a cli flag -func normalizeToken(token string) (string, error) { - _, password, ok := clientaccess.ParseUsernamePassword(token) - if !ok { - return password, errors.New("failed to normalize server token; must be in format K10::: or ") - } - - return password, nil -} - // migrateTokens will list all keys that has prefix /bootstrap and will check for key that is // hashed with empty string and keys that is hashed with old token format before normalizing // then migrate those and resave only with the normalized token diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go index d96b536d29..3409337d0b 100644 --- a/pkg/etcd/s3.go +++ b/pkg/etcd/s3.go @@ -20,6 +20,7 @@ import ( "time" "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" @@ -31,6 +32,7 @@ import ( var ( clusterIDKey = textproto.CanonicalMIMEHeaderKey(version.Program + "-cluster-id") + tokenHashKey = textproto.CanonicalMIMEHeaderKey(version.Program + "-token-hash") nodeNameKey = textproto.CanonicalMIMEHeaderKey(version.Program + "-node-name") ) @@ -39,6 +41,7 @@ type S3 struct { config *config.Control client *minio.Client clusterID string + tokenHash string nodeName string } @@ -109,10 +112,16 @@ func NewS3(ctx context.Context, config *config.Control) (*S3, error) { clusterID = string(ns.UID) } + tokenHash, err := util.GetTokenHash(config) + if err != nil { + return nil, errors.Wrap(err, "failed to get server token hash for etcd snapshot") + } + return &S3{ config: config, client: c, clusterID: clusterID, + tokenHash: tokenHash, nodeName: os.Getenv("NODE_NAME"), }, nil } @@ -154,6 +163,7 @@ func (s *S3) upload(ctx context.Context, snapshot string, extraMetadata *v1.Conf } else { sf.Status = successfulSnapshotStatus sf.Size = uploadInfo.Size + sf.tokenHash = s.tokenHash } if _, err := s.uploadSnapshotMetadata(ctx, metadataKey, metadata); err != nil { logrus.Warnf("Failed to upload snapshot metadata to S3: %v", err) @@ -170,6 +180,7 @@ func (s *S3) uploadSnapshot(ctx context.Context, key, path string) (info minio.U UserMetadata: map[string]string{ clusterIDKey: s.clusterID, nodeNameKey: s.nodeName, + tokenHashKey: s.tokenHash, }, } if strings.HasSuffix(key, compressedExtension) { @@ -392,6 +403,7 @@ func (s *S3) listSnapshots(ctx context.Context) (map[string]snapshotFile, error) Status: successfulSnapshotStatus, Compressed: compressed, nodeSource: obj.UserMetadata[nodeNameKey], + tokenHash: obj.UserMetadata[tokenHashKey], } sfKey := generateSnapshotConfigMapKey(sf) snapshots[sfKey] = sf diff --git a/pkg/etcd/snapshot.go b/pkg/etcd/snapshot.go index 4c710d7b51..d11a7fb5b0 100644 --- a/pkg/etcd/snapshot.go +++ b/pkg/etcd/snapshot.go @@ -57,6 +57,7 @@ var ( labelStorageNode = "etcd." + version.Program + ".cattle.io/snapshot-storage-node" annotationLocalReconciled = "etcd." + version.Program + ".cattle.io/local-snapshots-timestamp" annotationS3Reconciled = "etcd." + version.Program + ".cattle.io/s3-snapshots-timestamp" + annotationTokenHash = "etcd." + version.Program + ".cattle.io/snapshot-token-hash" // snapshotDataBackoff will retry at increasing steps for up to ~30 seconds. // If the ConfigMap update fails, the list won't be reconciled again until next time @@ -252,6 +253,11 @@ func (e *ETCD) Snapshot(ctx context.Context) error { return errors.Wrap(err, "failed to get config for etcd snapshot") } + tokenHash, err := util.GetTokenHash(e.config) + if err != nil { + return errors.Wrap(err, "failed to get server token hash for etcd snapshot") + } + nodeName := os.Getenv("NODE_NAME") now := time.Now().Round(time.Second) snapshotName := fmt.Sprintf("%s-%s-%d", e.config.EtcdSnapshotName, nodeName, now.Unix()) @@ -314,6 +320,7 @@ func (e *ETCD) Snapshot(ctx context.Context) error { Size: f.Size(), Compressed: e.config.EtcdSnapshotCompress, metadataSource: extraMetadata, + tokenHash: tokenHash, } if err := saveSnapshotMetadata(snapshotPath, extraMetadata); err != nil { @@ -412,6 +419,7 @@ type snapshotFile struct { // to populate other fields before serialization to the legacy configmap. metadataSource *v1.ConfigMap `json:"-"` nodeSource string `json:"-"` + tokenHash string `json:"-"` } // listLocalSnapshots provides a list of the currently stored @@ -1016,6 +1024,10 @@ func (sf *snapshotFile) fromETCDSnapshotFile(esf *apisv1.ETCDSnapshotFile) { } } + if tokenHash := esf.Annotations[annotationTokenHash]; tokenHash != "" { + sf.tokenHash = tokenHash + } + if esf.Spec.S3 == nil { sf.NodeName = esf.Spec.NodeName } else { @@ -1080,6 +1092,14 @@ func (sf *snapshotFile) toETCDSnapshotFile(esf *apisv1.ETCDSnapshotFile) { esf.ObjectMeta.Labels = map[string]string{} } + if esf.ObjectMeta.Annotations == nil { + esf.ObjectMeta.Annotations = map[string]string{} + } + + if sf.tokenHash != "" { + esf.ObjectMeta.Annotations[annotationTokenHash] = sf.tokenHash + } + if sf.S3 == nil { esf.ObjectMeta.Labels[labelStorageNode] = esf.Spec.NodeName } else { diff --git a/pkg/util/token.go b/pkg/util/token.go index a47a4eefd9..c4d3495af2 100644 --- a/pkg/util/token.go +++ b/pkg/util/token.go @@ -1,8 +1,16 @@ package util import ( + "bytes" cryptorand "crypto/rand" + "crypto/sha256" "encoding/hex" + "os" + "path/filepath" + + "github.com/k3s-io/k3s/pkg/clientaccess" + "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/pkg/errors" ) func Random(size int) (string, error) { @@ -13,3 +21,57 @@ func Random(size int) (string, error) { } return hex.EncodeToString(token), err } + +// ReadTokenFromFile will attempt to get the token from /token if it the file not found +// in case of fresh installation it will try to use the runtime serverToken saved in memory +// after stripping it from any additional information like the username or cahash, if the file +// found then it will still strip the token from any additional info +func ReadTokenFromFile(serverToken, certs, dataDir string) (string, error) { + tokenFile := filepath.Join(dataDir, "token") + + b, err := os.ReadFile(tokenFile) + if err != nil { + if os.IsNotExist(err) { + token, err := clientaccess.FormatToken(serverToken, certs) + if err != nil { + return token, err + } + return token, nil + } + return "", err + } + + // strip the token from any new line if its read from file + return string(bytes.TrimRight(b, "\n")), nil +} + +// NormalizeToken will normalize the token read from file or passed as a cli flag +func NormalizeToken(token string) (string, error) { + _, password, ok := clientaccess.ParseUsernamePassword(token) + if !ok { + return password, errors.New("failed to normalize server token; must be in format K10::: or ") + } + + return password, nil +} + +func GetTokenHash(config *config.Control) (string, error) { + token := config.Token + if token == "" { + tokenFromFile, err := ReadTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) + if err != nil { + return "", err + } + token = tokenFromFile + } + normalizedToken, err := NormalizeToken(token) + if err != nil { + return "", err + } + return ShortHash(normalizedToken, 12), nil +} + +func ShortHash(s string, i int) string { + digest := sha256.Sum256([]byte(s)) + return hex.EncodeToString(digest[:])[:i] +}