diff --git a/cmd/encrypt/main.go b/cmd/encrypt/main.go index c208f4f945..52d02029a9 100644 --- a/cmd/encrypt/main.go +++ b/cmd/encrypt/main.go @@ -22,6 +22,7 @@ func main() { secretsencrypt.Prepare, secretsencrypt.Rotate, secretsencrypt.Reencrypt, + secretsencrypt.RotateKeys, ), } diff --git a/cmd/k3s/main.go b/cmd/k3s/main.go index def64bc9a3..f3b3cfde87 100644 --- a/cmd/k3s/main.go +++ b/cmd/k3s/main.go @@ -71,6 +71,7 @@ func main() { secretsencryptCommand, secretsencryptCommand, secretsencryptCommand, + secretsencryptCommand, ), cmds.NewCertCommand( cmds.NewCertSubcommands( diff --git a/cmd/server/main.go b/cmd/server/main.go index 37bf8d2cd3..5b9cfd6b74 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -68,6 +68,7 @@ func main() { secretsencrypt.Prepare, secretsencrypt.Rotate, secretsencrypt.Reencrypt, + secretsencrypt.RotateKeys, ), cmds.NewCertCommand( cmds.NewCertSubcommands( diff --git a/pkg/cli/cmds/secrets_encrypt.go b/pkg/cli/cmds/secrets_encrypt.go index ba1a74a639..acbb63e3bc 100644 --- a/pkg/cli/cmds/secrets_encrypt.go +++ b/pkg/cli/cmds/secrets_encrypt.go @@ -26,7 +26,7 @@ var ( } ) -func NewSecretsEncryptCommands(status, enable, disable, prepare, rotate, reencrypt func(ctx *cli.Context) error) cli.Command { +func NewSecretsEncryptCommands(status, enable, disable, prepare, rotate, reencrypt, rotateKeys func(ctx *cli.Context) error) cli.Command { return cli.Command{ Name: SecretsEncryptCommand, Usage: "Control secrets encryption and keys rotation", @@ -84,6 +84,13 @@ func NewSecretsEncryptCommands(status, enable, disable, prepare, rotate, reencry Destination: &ServerConfig.EncryptSkip, }), }, + { + Name: "rotate-keys", + Usage: "Add, rotate and rencryption with a new encryption key", + SkipArgReorder: true, + Action: rotateKeys, + Flags: EncryptFlags, + }, }, } } diff --git a/pkg/cli/secretsencrypt/secrets_encrypt.go b/pkg/cli/secretsencrypt/secrets_encrypt.go index d1fa095300..8fdc306adc 100644 --- a/pkg/cli/secretsencrypt/secrets_encrypt.go +++ b/pkg/cli/secretsencrypt/secrets_encrypt.go @@ -156,7 +156,7 @@ func Prepare(app *cli.Context) error { return err } b, err := json.Marshal(server.EncryptionRequest{ - Stage: pointer.StringPtr(secretsencrypt.EncryptionPrepare), + Stage: pointer.String(secretsencrypt.EncryptionPrepare), Force: cmds.ServerConfig.EncryptForce, }) if err != nil { @@ -178,7 +178,7 @@ func Rotate(app *cli.Context) error { return err } b, err := json.Marshal(server.EncryptionRequest{ - Stage: pointer.StringPtr(secretsencrypt.EncryptionRotate), + Stage: pointer.String(secretsencrypt.EncryptionRotate), Force: cmds.ServerConfig.EncryptForce, }) if err != nil { @@ -201,7 +201,7 @@ func Reencrypt(app *cli.Context) error { return err } b, err := json.Marshal(server.EncryptionRequest{ - Stage: pointer.StringPtr(secretsencrypt.EncryptionReencryptActive), + Stage: pointer.String(secretsencrypt.EncryptionReencryptActive), Force: cmds.ServerConfig.EncryptForce, Skip: cmds.ServerConfig.EncryptSkip, }) @@ -214,3 +214,25 @@ func Reencrypt(app *cli.Context) error { fmt.Println("reencryption started") return nil } + +func RotateKeys(app *cli.Context) error { + var err error + if err = cmds.InitLogging(); err != nil { + return err + } + info, err := commandPrep(app, &cmds.ServerConfig) + if err != nil { + return err + } + b, err := json.Marshal(server.EncryptionRequest{ + Stage: pointer.String(secretsencrypt.EncryptionRotateKeys), + }) + if err != nil { + return err + } + if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil { + return wrapServerError(err) + } + fmt.Println("keys rotated, rencryption started") + return nil +} diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 086dcbdf22..ea5ae19bae 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -214,6 +214,7 @@ func apiServer(ctx context.Context, cfg *config.Control) error { } if cfg.EncryptSecrets { argsMap["encryption-provider-config"] = runtime.EncryptionConfig + argsMap["encryption-provider-config-automatic-reload"] = "true" } args := config.GetArgs(argsMap, cfg.ExtraAPIArgs) diff --git a/pkg/secretsencrypt/config.go b/pkg/secretsencrypt/config.go index 56f6904cbd..dd770c7d8b 100644 --- a/pkg/secretsencrypt/config.go +++ b/pkg/secretsencrypt/config.go @@ -21,6 +21,7 @@ const ( EncryptionStart string = "start" EncryptionPrepare string = "prepare" EncryptionRotate string = "rotate" + EncryptionRotateKeys string = "rotate_keys" EncryptionReencryptRequest string = "reencrypt_request" EncryptionReencryptActive string = "reencrypt_active" EncryptionReencryptFinished string = "reencrypt_finished" diff --git a/pkg/server/secrets-encrypt.go b/pkg/server/secrets-encrypt.go index 6075305474..4cb4386147 100644 --- a/pkg/server/secrets-encrypt.go +++ b/pkg/server/secrets-encrypt.go @@ -17,6 +17,7 @@ import ( "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/secretsencrypt" "github.com/k3s-io/k3s/pkg/util" + "github.com/k3s-io/k3s/pkg/version" "github.com/rancher/wrangler/pkg/generated/controllers/core" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -139,6 +140,9 @@ func encryptionEnable(ctx context.Context, server *config.Control, enable bool) logrus.Infoln("Secrets encryption already disabled") return nil } else if providers[0].Identity != nil && providers[1].AESCBC != nil && enable { + if nodeArgs := getNodeArgs(server); !strings.Contains(nodeArgs, "secrets-encryption") { + return fmt.Errorf("secrets encryption cannot be enabled without first starting the server with --secrets-encryption flag") + } logrus.Infoln("Enabling secrets encryption") if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, enable); err != nil { return err @@ -149,7 +153,20 @@ func encryptionEnable(ctx context.Context, server *config.Control, enable bool) } else { return fmt.Errorf("unable to enable/disable secrets encryption, unknown configuration") } - return cluster.Save(ctx, server, true) + if err := cluster.Save(ctx, server, true); err != nil { + return err + } + server.EncryptSkip = true + return setReencryptAnnotation(server) +} + +func getNodeArgs(server *config.Control) string { + nodeName := os.Getenv("NODE_NAME") + node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return "" + } + return node.Annotations[version.Program+".io/node-args"] } func encryptionConfigHandler(ctx context.Context, server *config.Control) http.Handler { @@ -174,6 +191,8 @@ func encryptionConfigHandler(ctx context.Context, server *config.Control) http.H err = encryptionPrepare(ctx, server, encryptReq.Force) case secretsencrypt.EncryptionRotate: err = encryptionRotate(ctx, server, encryptReq.Force) + case secretsencrypt.EncryptionRotateKeys: + err = encryptionRotateKeys(ctx, server) case secretsencrypt.EncryptionReencryptActive: err = encryptionReencrypt(ctx, server, encryptReq.Force, encryptReq.Skip) default: @@ -280,6 +299,63 @@ func encryptionReencrypt(ctx context.Context, server *config.Control, force bool return nil } +func addAndRotateKeys(server *config.Control) error { + + curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime) + if err != nil { + return err + } + + if err := AppendNewEncryptionKey(&curKeys); err != nil { + return err + } + logrus.Infoln("Adding secrets-encryption key: ", curKeys[len(curKeys)-1]) + + if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, true); err != nil { + return err + } + + // Right rotate elements + rotatedKeys := append(curKeys[len(curKeys)-1:], curKeys[:len(curKeys)-1]...) + + return secretsencrypt.WriteEncryptionConfig(server.Runtime, rotatedKeys, true) +} + +// encryptionRotateKeys is both adds and rotates keys, and sets the annotaiton that triggers the +// reencryption process. It is the preferred way to rotate keys, starting with v1.28 +func encryptionRotateKeys(ctx context.Context, server *config.Control) error { + states := secretsencrypt.EncryptionStart + "-" + secretsencrypt.EncryptionReencryptFinished + if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), states); err != nil { + return err + } + + if err := addAndRotateKeys(server); err != nil { + return err + } + + return setReencryptAnnotation(server) +} + +func setReencryptAnnotation(server *config.Control) error { + nodeName := os.Getenv("NODE_NAME") + node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + + reencryptHash, err := secretsencrypt.GenReencryptHash(server.Runtime, secretsencrypt.EncryptionReencryptRequest) + if err != nil { + return err + } + ann := secretsencrypt.EncryptionReencryptRequest + "-" + reencryptHash + node.Annotations[secretsencrypt.EncryptionHashAnnotation] = ann + if _, err = server.Runtime.Core.Core().V1().Node().Update(node); err != nil { + return err + } + logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name) + return nil +} + func AppendNewEncryptionKey(keys *[]apiserverconfigv1.Key) error { aescbcKey := make([]byte, aescbcKeySize) diff --git a/tests/e2e/secretsencryption/secretsencryption_test.go b/tests/e2e/secretsencryption/secretsencryption_test.go index a35545b8f1..907833514c 100644 --- a/tests/e2e/secretsencryption/secretsencryption_test.go +++ b/tests/e2e/secretsencryption/secretsencryption_test.go @@ -3,6 +3,7 @@ package secretsencryption import ( "flag" "fmt" + "os" "strings" "testing" @@ -40,7 +41,7 @@ var _ = ReportAfterEach(e2e.GenReport) var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { Context("Secrets Keys are rotated:", func() { - FIt("Starts up with no issues", func() { + It("Starts up with no issues", func() { var err error if *local { serverNodeNames, _, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, 0) @@ -55,7 +56,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { Expect(err).NotTo(HaveOccurred()) }) - FIt("Checks node and pod status", func() { + It("Checks node and pod status", func() { fmt.Printf("\nFetching node status\n") Eventually(func(g Gomega) { nodes, err := e2e.ParseNodes(kubeConfigFile, false) @@ -81,7 +82,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { _, _ = e2e.ParsePods(kubeConfigFile, true) }) - FIt("Deploys several secrets", func() { + It("Deploys several secrets", func() { _, err := e2e.DeployWorkload("secrets.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "Secrets not deployed") }) @@ -184,6 +185,18 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { Eventually(func() (string, error) { return e2e.RunCmdOnNode(cmd, serverNodeNames[0]) }, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) + + for i, nodeName := range serverNodeNames { + Eventually(func(g Gomega) { + res, err := e2e.RunCmdOnNode(cmd, nodeName) + g.Expect(err).NotTo(HaveOccurred(), res) + if i == 0 { + g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) + } else { + g.Expect(res).Should(ContainSubstring("Encryption Status: Disabled")) + } + }, "420s", "2s").Should(Succeed()) + } }) It("Restarts K3s servers", func() { @@ -197,7 +210,6 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { g.Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Enabled")) }, "420s", "2s").Should(Succeed()) } - }) }) @@ -212,8 +224,8 @@ var _ = AfterSuite(func() { if failed && !*ci { fmt.Println("FAILED!") } else { - // Expect(e2e.GetCoverageReport(serverNodeNames)).To(Succeed()) - // Expect(e2e.DestroyCluster()).To(Succeed()) - // Expect(os.Remove(kubeConfigFile)).To(Succeed()) + Expect(e2e.GetCoverageReport(serverNodeNames)).To(Succeed()) + Expect(e2e.DestroyCluster()).To(Succeed()) + Expect(os.Remove(kubeConfigFile)).To(Succeed()) } })