mirror of https://github.com/k3s-io/k3s
Secrets-encryption rotation (#4372)
* Regular CLI framework for encrypt commands * New secrets-encryption feature * New integration test * fixes for flaky integration test CI * Fix to bootstrap on restart of existing nodes * Consolidate event recorder Signed-off-by: Derek Nola <derek.nola@suse.com>pull/4693/head
Derek Nola
3 years ago
committed by
GitHub
parent
7d3447ceff
commit
bcb662926d
@ -0,0 +1,32 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"github.com/rancher/k3s/pkg/cli/cmds"
|
||||
"github.com/rancher/k3s/pkg/cli/secretsencrypt"
|
||||
"github.com/rancher/k3s/pkg/configfilearg"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
func main() {
|
||||
app := cmds.NewApp()
|
||||
app.Commands = []cli.Command{
|
||||
cmds.NewSecretsEncryptCommand(cli.ShowAppHelp,
|
||||
cmds.NewSecretsEncryptSubcommands(
|
||||
secretsencrypt.Status,
|
||||
secretsencrypt.Enable,
|
||||
secretsencrypt.Disable,
|
||||
secretsencrypt.Prepare,
|
||||
secretsencrypt.Rotate,
|
||||
secretsencrypt.Reencrypt),
|
||||
),
|
||||
}
|
||||
|
||||
if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,94 @@
|
||||
package cmds
|
||||
|
||||
import (
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
const SecretsEncryptCommand = "secrets-encrypt"
|
||||
|
||||
var EncryptFlags = []cli.Flag{
|
||||
DataDirFlag,
|
||||
ServerToken,
|
||||
}
|
||||
|
||||
func NewSecretsEncryptCommand(action func(*cli.Context) error, subcommands []cli.Command) cli.Command {
|
||||
return cli.Command{
|
||||
Name: SecretsEncryptCommand,
|
||||
Usage: "Control secrets encryption and keys rotation",
|
||||
SkipFlagParsing: false,
|
||||
SkipArgReorder: true,
|
||||
Action: action,
|
||||
Subcommands: subcommands,
|
||||
}
|
||||
}
|
||||
|
||||
func NewSecretsEncryptSubcommands(status, enable, disable, prepare, rotate, reencrypt func(ctx *cli.Context) error) []cli.Command {
|
||||
return []cli.Command{
|
||||
{
|
||||
Name: "status",
|
||||
Usage: "Print current status of secrets encryption",
|
||||
SkipFlagParsing: false,
|
||||
SkipArgReorder: true,
|
||||
Action: status,
|
||||
Flags: EncryptFlags,
|
||||
},
|
||||
{
|
||||
Name: "enable",
|
||||
Usage: "Enable secrets encryption",
|
||||
SkipFlagParsing: false,
|
||||
SkipArgReorder: true,
|
||||
Action: enable,
|
||||
Flags: EncryptFlags,
|
||||
},
|
||||
{
|
||||
Name: "disable",
|
||||
Usage: "Disable secrets encryption",
|
||||
SkipFlagParsing: false,
|
||||
SkipArgReorder: true,
|
||||
Action: disable,
|
||||
Flags: EncryptFlags,
|
||||
},
|
||||
{
|
||||
Name: "prepare",
|
||||
Usage: "Prepare for encryption keys rotation",
|
||||
SkipFlagParsing: false,
|
||||
SkipArgReorder: true,
|
||||
Action: prepare,
|
||||
Flags: append(EncryptFlags, &cli.BoolFlag{
|
||||
Name: "f,force",
|
||||
Usage: "Force preparation.",
|
||||
Destination: &ServerConfig.EncryptForce,
|
||||
}),
|
||||
},
|
||||
{
|
||||
Name: "rotate",
|
||||
Usage: "Rotate secrets encryption keys",
|
||||
SkipFlagParsing: false,
|
||||
SkipArgReorder: true,
|
||||
Action: rotate,
|
||||
Flags: append(EncryptFlags, &cli.BoolFlag{
|
||||
Name: "f,force",
|
||||
Usage: "Force key rotation.",
|
||||
Destination: &ServerConfig.EncryptForce,
|
||||
}),
|
||||
},
|
||||
{
|
||||
Name: "reencrypt",
|
||||
Usage: "Reencrypt all data with new encryption key",
|
||||
SkipFlagParsing: false,
|
||||
SkipArgReorder: true,
|
||||
Action: reencrypt,
|
||||
Flags: append(EncryptFlags,
|
||||
&cli.BoolFlag{
|
||||
Name: "f,force",
|
||||
Usage: "Force secrets reencryption.",
|
||||
Destination: &ServerConfig.EncryptForce,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "skip",
|
||||
Usage: "Skip removing old key",
|
||||
Destination: &ServerConfig.EncryptSkip,
|
||||
}),
|
||||
},
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,216 @@
|
||||
package secretsencrypt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"text/tabwriter"
|
||||
|
||||
"github.com/erikdubbelboer/gspt"
|
||||
"github.com/rancher/k3s/pkg/cli/cmds"
|
||||
"github.com/rancher/k3s/pkg/clientaccess"
|
||||
"github.com/rancher/k3s/pkg/daemons/config"
|
||||
"github.com/rancher/k3s/pkg/secretsencrypt"
|
||||
"github.com/rancher/k3s/pkg/server"
|
||||
"github.com/rancher/k3s/pkg/version"
|
||||
"github.com/urfave/cli"
|
||||
"k8s.io/utils/pointer"
|
||||
)
|
||||
|
||||
func commandPrep(app *cli.Context, cfg *cmds.Server) (config.Control, *clientaccess.Info, error) {
|
||||
var controlConfig config.Control
|
||||
var err error
|
||||
// hide process arguments from ps output, since they may contain
|
||||
// database credentials or other secrets.
|
||||
gspt.SetProcTitle(os.Args[0] + " encrypt")
|
||||
|
||||
controlConfig.DataDir, err = server.ResolveDataDir(cfg.DataDir)
|
||||
if err != nil {
|
||||
return controlConfig, nil, err
|
||||
}
|
||||
if cfg.ServerURL == "" {
|
||||
cfg.ServerURL = "https://127.0.0.1:6443"
|
||||
}
|
||||
|
||||
if cfg.Token == "" {
|
||||
fp := filepath.Join(controlConfig.DataDir, "token")
|
||||
tokenByte, err := ioutil.ReadFile(fp)
|
||||
if err != nil {
|
||||
return controlConfig, nil, err
|
||||
}
|
||||
controlConfig.Token = string(bytes.TrimRight(tokenByte, "\n"))
|
||||
} else {
|
||||
controlConfig.Token = cfg.Token
|
||||
}
|
||||
controlConfig.EncryptForce = cfg.EncryptForce
|
||||
controlConfig.EncryptSkip = cfg.EncryptSkip
|
||||
info, err := clientaccess.ParseAndValidateTokenForUser(cmds.ServerConfig.ServerURL, controlConfig.Token, "node")
|
||||
if err != nil {
|
||||
return controlConfig, nil, err
|
||||
}
|
||||
return controlConfig, info, nil
|
||||
}
|
||||
|
||||
func Enable(app *cli.Context) error {
|
||||
var err error
|
||||
if err = cmds.InitLogging(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, info, err := commandPrep(app, &cmds.ServerConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := json.Marshal(server.EncryptionRequest{Enable: pointer.Bool(true)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("secrets-encryption enabled")
|
||||
return nil
|
||||
}
|
||||
|
||||
func Disable(app *cli.Context) error {
|
||||
|
||||
if err := cmds.InitLogging(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, info, err := commandPrep(app, &cmds.ServerConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := json.Marshal(server.EncryptionRequest{Enable: pointer.Bool(false)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("secrets-encryption disabled")
|
||||
return nil
|
||||
}
|
||||
|
||||
func Status(app *cli.Context) error {
|
||||
if err := cmds.InitLogging(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, info, err := commandPrep(app, &cmds.ServerConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := info.Get("/v1-" + version.Program + "/encrypt/status")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status := server.EncryptionState{}
|
||||
if err := json.Unmarshal(data, &status); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if status.Enable == nil {
|
||||
fmt.Println("Encryption Status: Disabled, no configuration file found")
|
||||
return nil
|
||||
}
|
||||
|
||||
var statusOutput string
|
||||
if *status.Enable {
|
||||
statusOutput += "Encryption Status: Enabled\n"
|
||||
} else {
|
||||
statusOutput += "Encryption Status: Disabled\n"
|
||||
}
|
||||
statusOutput += fmt.Sprintln("Current Rotation Stage:", status.Stage)
|
||||
|
||||
if status.HashMatch {
|
||||
statusOutput += fmt.Sprintln("Server Encryption Hashes: All hashes match")
|
||||
} else {
|
||||
statusOutput += fmt.Sprintf("Server Encryption Hashes: %s\n", status.HashError)
|
||||
}
|
||||
|
||||
var tabBuffer bytes.Buffer
|
||||
w := tabwriter.NewWriter(&tabBuffer, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintf(w, "\n")
|
||||
fmt.Fprintf(w, "Active\tKey Type\tName\n")
|
||||
fmt.Fprintf(w, "------\t--------\t----\n")
|
||||
if status.ActiveKey != "" {
|
||||
fmt.Fprintf(w, " *\t%s\t%s\n", "AES-CBC", status.ActiveKey)
|
||||
}
|
||||
for _, k := range status.InactiveKeys {
|
||||
fmt.Fprintf(w, "\t%s\t%s\n", "AES-CBC", k)
|
||||
}
|
||||
w.Flush()
|
||||
fmt.Println(statusOutput + tabBuffer.String())
|
||||
return nil
|
||||
}
|
||||
|
||||
func Prepare(app *cli.Context) error {
|
||||
var err error
|
||||
if err = cmds.InitLogging(); err != nil {
|
||||
return err
|
||||
}
|
||||
controlConfig, info, err := commandPrep(app, &cmds.ServerConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := json.Marshal(server.EncryptionRequest{
|
||||
Stage: pointer.StringPtr(secretsencrypt.EncryptionPrepare),
|
||||
Force: controlConfig.EncryptForce,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("prepare completed successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
func Rotate(app *cli.Context) error {
|
||||
if err := cmds.InitLogging(); err != nil {
|
||||
return err
|
||||
}
|
||||
controlConfig, info, err := commandPrep(app, &cmds.ServerConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := json.Marshal(server.EncryptionRequest{
|
||||
Stage: pointer.StringPtr(secretsencrypt.EncryptionRotate),
|
||||
Force: controlConfig.EncryptForce,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("rotate completed successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
func Reencrypt(app *cli.Context) error {
|
||||
var err error
|
||||
if err = cmds.InitLogging(); err != nil {
|
||||
return err
|
||||
}
|
||||
controlConfig, info, err := commandPrep(app, &cmds.ServerConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := json.Marshal(server.EncryptionRequest{
|
||||
Stage: pointer.StringPtr(secretsencrypt.EncryptionReencryptActive),
|
||||
Force: controlConfig.EncryptForce,
|
||||
Skip: controlConfig.EncryptSkip,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("reencryption started")
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,174 @@
|
||||
package secretsencrypt
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/rancher/k3s/pkg/daemons/config"
|
||||
"github.com/rancher/k3s/pkg/version"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
EncryptionStart string = "start"
|
||||
EncryptionPrepare string = "prepare"
|
||||
EncryptionRotate string = "rotate"
|
||||
EncryptionReencryptRequest string = "reencrypt_request"
|
||||
EncryptionReencryptActive string = "reencrypt_active"
|
||||
EncryptionReencryptFinished string = "reencrypt_finished"
|
||||
)
|
||||
|
||||
var EncryptionHashAnnotation = version.Program + ".io/encryption-config-hash"
|
||||
|
||||
func GetEncryptionProviders(runtime *config.ControlRuntime) ([]apiserverconfigv1.ProviderConfiguration, error) {
|
||||
curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
curEncryption := apiserverconfigv1.EncryptionConfiguration{}
|
||||
if err = json.Unmarshal(curEncryptionByte, &curEncryption); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return curEncryption.Resources[0].Providers, nil
|
||||
}
|
||||
|
||||
func GetEncryptionKeys(runtime *config.ControlRuntime) ([]apiserverconfigv1.Key, error) {
|
||||
|
||||
providers, err := GetEncryptionProviders(runtime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(providers) > 2 {
|
||||
return nil, fmt.Errorf("more than 2 providers (%d) found in secrets encryption", len(providers))
|
||||
}
|
||||
|
||||
var curKeys []apiserverconfigv1.Key
|
||||
for _, p := range providers {
|
||||
if p.AESCBC != nil {
|
||||
curKeys = append(curKeys, p.AESCBC.Keys...)
|
||||
}
|
||||
if p.AESGCM != nil || p.KMS != nil || p.Secretbox != nil {
|
||||
return nil, fmt.Errorf("non-standard encryption keys found")
|
||||
}
|
||||
}
|
||||
return curKeys, nil
|
||||
}
|
||||
|
||||
func WriteEncryptionConfig(runtime *config.ControlRuntime, keys []apiserverconfigv1.Key, enable bool) error {
|
||||
|
||||
// Placing the identity provider first disables encryption
|
||||
var providers []apiserverconfigv1.ProviderConfiguration
|
||||
if enable {
|
||||
providers = []apiserverconfigv1.ProviderConfiguration{
|
||||
{
|
||||
AESCBC: &apiserverconfigv1.AESConfiguration{
|
||||
Keys: keys,
|
||||
},
|
||||
},
|
||||
{
|
||||
Identity: &apiserverconfigv1.IdentityConfiguration{},
|
||||
},
|
||||
}
|
||||
} else {
|
||||
providers = []apiserverconfigv1.ProviderConfiguration{
|
||||
{
|
||||
Identity: &apiserverconfigv1.IdentityConfiguration{},
|
||||
},
|
||||
{
|
||||
AESCBC: &apiserverconfigv1.AESConfiguration{
|
||||
Keys: keys,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
encConfig := apiserverconfigv1.EncryptionConfiguration{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "EncryptionConfiguration",
|
||||
APIVersion: "apiserver.config.k8s.io/v1",
|
||||
},
|
||||
Resources: []apiserverconfigv1.ResourceConfiguration{
|
||||
{
|
||||
Resources: []string{"secrets"},
|
||||
Providers: providers,
|
||||
},
|
||||
},
|
||||
}
|
||||
jsonfile, err := json.Marshal(encConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return ioutil.WriteFile(runtime.EncryptionConfig, jsonfile, 0600)
|
||||
}
|
||||
|
||||
func GenEncryptionConfigHash(runtime *config.ControlRuntime) (string, error) {
|
||||
curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionConfig)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
encryptionConfigHash := sha256.Sum256(curEncryptionByte)
|
||||
return hex.EncodeToString(encryptionConfigHash[:]), nil
|
||||
}
|
||||
|
||||
// GenReencryptHash generates a sha256 hash fom the existing secrets keys and
|
||||
// a new key based on the input arguments.
|
||||
func GenReencryptHash(runtime *config.ControlRuntime, keyName string) (string, error) {
|
||||
|
||||
keys, err := GetEncryptionKeys(runtime)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
newKey := apiserverconfigv1.Key{
|
||||
Name: keyName,
|
||||
Secret: "12345",
|
||||
}
|
||||
keys = append(keys, newKey)
|
||||
b, err := json.Marshal(keys)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
hash := sha256.Sum256(b)
|
||||
return hex.EncodeToString(hash[:]), nil
|
||||
}
|
||||
|
||||
func getEncryptionHashFile(runtime *config.ControlRuntime) (string, error) {
|
||||
curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionHash)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(curEncryptionByte), nil
|
||||
}
|
||||
|
||||
func BootstrapEncryptionHashAnnotation(node *corev1.Node, runtime *config.ControlRuntime) error {
|
||||
existingAnn, err := getEncryptionHashFile(runtime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
node.Annotations[EncryptionHashAnnotation] = existingAnn
|
||||
return nil
|
||||
}
|
||||
|
||||
func WriteEncryptionHashAnnotation(runtime *config.ControlRuntime, node *corev1.Node, stage string) error {
|
||||
encryptionConfigHash, err := GenEncryptionConfigHash(runtime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if node.Annotations == nil {
|
||||
return fmt.Errorf("node annotations do not exist for %s", node.ObjectMeta.Name)
|
||||
}
|
||||
ann := stage + "-" + encryptionConfigHash
|
||||
node.Annotations[EncryptionHashAnnotation] = ann
|
||||
if _, err = runtime.Core.Core().V1().Node().Update(node); err != nil {
|
||||
return err
|
||||
}
|
||||
logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name)
|
||||
return ioutil.WriteFile(runtime.EncryptionHash, []byte(ann), 0600)
|
||||
}
|
||||
@ -0,0 +1,196 @@
|
||||
package secretsencrypt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/rancher/k3s/pkg/cluster"
|
||||
"github.com/rancher/k3s/pkg/daemons/config"
|
||||
"github.com/rancher/k3s/pkg/util"
|
||||
coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
|
||||
"github.com/sirupsen/logrus"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/pager"
|
||||
"k8s.io/client-go/tools/record"
|
||||
)
|
||||
|
||||
const (
|
||||
controllerAgentName string = "reencrypt-controller"
|
||||
secretsUpdateStartEvent string = "SecretsUpdateStart"
|
||||
secretsProgressEvent string = "SecretsProgress"
|
||||
secretsUpdateCompleteEvent string = "SecretsUpdateComplete"
|
||||
secretsUpdateErrorEvent string = "SecretsUpdateError"
|
||||
)
|
||||
|
||||
type handler struct {
|
||||
ctx context.Context
|
||||
controlConfig *config.Control
|
||||
nodes coreclient.NodeController
|
||||
secrets coreclient.SecretController
|
||||
recorder record.EventRecorder
|
||||
}
|
||||
|
||||
func Register(
|
||||
ctx context.Context,
|
||||
k8s kubernetes.Interface,
|
||||
controlConfig *config.Control,
|
||||
nodes coreclient.NodeController,
|
||||
secrets coreclient.SecretController,
|
||||
) error {
|
||||
h := &handler{
|
||||
ctx: ctx,
|
||||
controlConfig: controlConfig,
|
||||
nodes: nodes,
|
||||
secrets: secrets,
|
||||
recorder: util.BuildControllerEventRecorder(k8s, controllerAgentName),
|
||||
}
|
||||
|
||||
nodes.OnChange(ctx, "reencrypt-controller", h.onChangeNode)
|
||||
return nil
|
||||
}
|
||||
|
||||
// onChangeNode handles changes to Nodes. We are looking for a specific annotation change
|
||||
func (h *handler) onChangeNode(key string, node *corev1.Node) (*corev1.Node, error) {
|
||||
if node == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
ann, ok := node.Annotations[EncryptionHashAnnotation]
|
||||
if !ok {
|
||||
return node, nil
|
||||
}
|
||||
|
||||
if valid, err := h.validateReencryptStage(node, ann); err != nil {
|
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
|
||||
return node, err
|
||||
} else if !valid {
|
||||
return node, nil
|
||||
}
|
||||
|
||||
reencryptHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptActive)
|
||||
if err != nil {
|
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
|
||||
return node, err
|
||||
}
|
||||
ann = EncryptionReencryptActive + "-" + reencryptHash
|
||||
node.Annotations[EncryptionHashAnnotation] = ann
|
||||
node, err = h.nodes.Update(node)
|
||||
if err != nil {
|
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
|
||||
return node, err
|
||||
}
|
||||
|
||||
if err := h.updateSecrets(node); err != nil {
|
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
|
||||
return node, err
|
||||
}
|
||||
|
||||
// If skipping, revert back to the previous stage
|
||||
if h.controlConfig.EncryptSkip {
|
||||
BootstrapEncryptionHashAnnotation(node, h.controlConfig.Runtime)
|
||||
if node, err := h.nodes.Update(node); err != nil {
|
||||
return node, err
|
||||
}
|
||||
return node, nil
|
||||
}
|
||||
|
||||
// Remove last key
|
||||
curKeys, err := GetEncryptionKeys(h.controlConfig.Runtime)
|
||||
if err != nil {
|
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
|
||||
return node, err
|
||||
}
|
||||
|
||||
curKeys = curKeys[:len(curKeys)-1]
|
||||
if err = WriteEncryptionConfig(h.controlConfig.Runtime, curKeys, true); err != nil {
|
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
|
||||
return node, err
|
||||
}
|
||||
logrus.Infoln("Removed key: ", curKeys[len(curKeys)-1])
|
||||
if err != nil {
|
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
|
||||
return node, err
|
||||
}
|
||||
if err := WriteEncryptionHashAnnotation(h.controlConfig.Runtime, node, EncryptionReencryptFinished); err != nil {
|
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
|
||||
return node, err
|
||||
}
|
||||
if err := cluster.Save(h.ctx, h.controlConfig, h.controlConfig.Runtime.EtcdConfig, true); err != nil {
|
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
|
||||
return node, err
|
||||
}
|
||||
return node, nil
|
||||
}
|
||||
|
||||
// validateReencryptStage ensures that the request for reencryption is valid and
|
||||
// that there is only one active reencryption at a time
|
||||
func (h *handler) validateReencryptStage(node *corev1.Node, annotation string) (bool, error) {
|
||||
|
||||
split := strings.Split(annotation, "-")
|
||||
if len(split) != 2 {
|
||||
err := fmt.Errorf("invalid annotation %s found on node %s", annotation, node.ObjectMeta.Name)
|
||||
return false, err
|
||||
}
|
||||
stage := split[0]
|
||||
hash := split[1]
|
||||
|
||||
// Validate the specific stage and the request via sha256 hash
|
||||
if stage != EncryptionReencryptRequest {
|
||||
return false, nil
|
||||
}
|
||||
if reencryptRequestHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptRequest); err != nil {
|
||||
return false, err
|
||||
} else if reencryptRequestHash != hash {
|
||||
err = fmt.Errorf("invalid hash: %s found on node %s", hash, node.ObjectMeta.Name)
|
||||
return false, err
|
||||
}
|
||||
|
||||
nodes, err := h.nodes.List(metav1.ListOptions{})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
reencryptActiveHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptActive)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, node := range nodes.Items {
|
||||
if ann, ok := node.Annotations[EncryptionHashAnnotation]; ok {
|
||||
split := strings.Split(ann, "-")
|
||||
if len(split) != 2 {
|
||||
return false, fmt.Errorf("invalid annotation %s found on node %s", ann, node.ObjectMeta.Name)
|
||||
}
|
||||
stage := split[0]
|
||||
hash := split[1]
|
||||
if stage == EncryptionReencryptActive && hash == reencryptActiveHash {
|
||||
return false, fmt.Errorf("another reencrypt is already active")
|
||||
}
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (h *handler) updateSecrets(node *corev1.Node) error {
|
||||
secretPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
|
||||
return h.secrets.List("", opts)
|
||||
}))
|
||||
i := 0
|
||||
secretPager.EachListItem(h.ctx, metav1.ListOptions{}, func(obj runtime.Object) error {
|
||||
if secret, ok := obj.(*corev1.Secret); ok {
|
||||
if _, err := h.secrets.Update(secret); err != nil {
|
||||
return fmt.Errorf("failed to reencrypted secret: %v", err)
|
||||
}
|
||||
if i != 0 && i%10 == 0 {
|
||||
h.recorder.Eventf(node, corev1.EventTypeNormal, secretsProgressEvent, "reencrypted %d secrets", i)
|
||||
}
|
||||
i++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
h.recorder.Eventf(node, corev1.EventTypeNormal, secretsUpdateCompleteEvent, "completed reencrypt of %d secrets", i)
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,379 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rancher/k3s/pkg/cluster"
|
||||
"github.com/rancher/k3s/pkg/daemons/config"
|
||||
"github.com/rancher/k3s/pkg/secretsencrypt"
|
||||
"github.com/rancher/wrangler/pkg/generated/controllers/core"
|
||||
"github.com/sirupsen/logrus"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
|
||||
"k8s.io/utils/pointer"
|
||||
)
|
||||
|
||||
const aescbcKeySize = 32
|
||||
|
||||
type EncryptionState struct {
|
||||
Stage string `json:"stage"`
|
||||
ActiveKey string `json:"activekey"`
|
||||
Enable *bool `json:"enable,omitempty"`
|
||||
HashMatch bool `json:"hashmatch,omitempty"`
|
||||
HashError string `json:"hasherror,omitempty"`
|
||||
InactiveKeys []string `json:"inactivekeys,omitempty"`
|
||||
}
|
||||
|
||||
type EncryptionRequest struct {
|
||||
Stage *string `json:"stage,omitempty"`
|
||||
Enable *bool `json:"enable,omitempty"`
|
||||
Force bool `json:"force"`
|
||||
Skip bool `json:"skip"`
|
||||
}
|
||||
|
||||
func getEncryptionRequest(req *http.Request) (EncryptionRequest, error) {
|
||||
b, err := ioutil.ReadAll(req.Body)
|
||||
if err != nil {
|
||||
return EncryptionRequest{}, err
|
||||
}
|
||||
result := EncryptionRequest{}
|
||||
err = json.Unmarshal(b, &result)
|
||||
return result, err
|
||||
}
|
||||
|
||||
func encryptionStatusHandler(server *config.Control) http.Handler {
|
||||
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
||||
if req.TLS == nil {
|
||||
resp.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
status, err := encryptionStatus(server)
|
||||
if err != nil {
|
||||
genErrorMessage(resp, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
b, err := json.Marshal(status)
|
||||
if err != nil {
|
||||
genErrorMessage(resp, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
resp.Write(b)
|
||||
})
|
||||
}
|
||||
|
||||
func encryptionStatus(server *config.Control) (EncryptionState, error) {
|
||||
state := EncryptionState{}
|
||||
providers, err := secretsencrypt.GetEncryptionProviders(server.Runtime)
|
||||
if os.IsNotExist(err) {
|
||||
return state, nil
|
||||
} else if err != nil {
|
||||
return state, err
|
||||
}
|
||||
if providers[1].Identity != nil && providers[0].AESCBC != nil {
|
||||
state.Enable = pointer.Bool(true)
|
||||
} else if providers[0].Identity != nil && providers[1].AESCBC != nil || !server.EncryptSecrets {
|
||||
state.Enable = pointer.Bool(false)
|
||||
}
|
||||
|
||||
if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), ""); err != nil {
|
||||
state.HashMatch = false
|
||||
state.HashError = err.Error()
|
||||
} else {
|
||||
state.HashMatch = true
|
||||
}
|
||||
stage, _, err := getEncryptionHashAnnotation(server.Runtime.Core.Core())
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
state.Stage = stage
|
||||
active := true
|
||||
for _, p := range providers {
|
||||
if p.AESCBC != nil {
|
||||
for _, aesKey := range p.AESCBC.Keys {
|
||||
if active {
|
||||
active = false
|
||||
state.ActiveKey = aesKey.Name
|
||||
} else {
|
||||
state.InactiveKeys = append(state.InactiveKeys, aesKey.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
if p.Identity != nil {
|
||||
active = false
|
||||
}
|
||||
}
|
||||
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func encryptionEnable(ctx context.Context, server *config.Control, enable bool) error {
|
||||
providers, err := secretsencrypt.GetEncryptionProviders(server.Runtime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(providers) > 2 {
|
||||
return fmt.Errorf("more than 2 providers (%d) found in secrets encryption", len(providers))
|
||||
}
|
||||
curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if providers[1].Identity != nil && providers[0].AESCBC != nil && !enable {
|
||||
logrus.Infoln("Disabling secrets encryption")
|
||||
if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, enable); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if !enable {
|
||||
logrus.Infoln("Secrets encryption already disabled")
|
||||
return nil
|
||||
} else if providers[0].Identity != nil && providers[1].AESCBC != nil && enable {
|
||||
logrus.Infoln("Enabling secrets encryption")
|
||||
if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, enable); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if enable {
|
||||
logrus.Infoln("Secrets encryption already enabled")
|
||||
return nil
|
||||
} else {
|
||||
return fmt.Errorf("unable to enable/disable secrets encryption, unknown configuration")
|
||||
}
|
||||
return cluster.Save(ctx, server, server.Runtime.EtcdConfig, true)
|
||||
}
|
||||
|
||||
func encryptionConfigHandler(ctx context.Context, server *config.Control) http.Handler {
|
||||
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
||||
if req.TLS == nil {
|
||||
resp.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if req.Method != http.MethodPut {
|
||||
resp.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
encryptReq, err := getEncryptionRequest(req)
|
||||
if err != nil {
|
||||
resp.WriteHeader(http.StatusBadRequest)
|
||||
resp.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
if encryptReq.Stage != nil {
|
||||
switch *encryptReq.Stage {
|
||||
case secretsencrypt.EncryptionPrepare:
|
||||
err = encryptionPrepare(ctx, server, encryptReq.Force)
|
||||
case secretsencrypt.EncryptionRotate:
|
||||
err = encryptionRotate(ctx, server, encryptReq.Force)
|
||||
case secretsencrypt.EncryptionReencryptActive:
|
||||
err = encryptionReencrypt(ctx, server, encryptReq.Force, encryptReq.Skip)
|
||||
default:
|
||||
err = fmt.Errorf("unknown stage %s requested", *encryptReq.Stage)
|
||||
}
|
||||
} else if encryptReq.Enable != nil {
|
||||
err = encryptionEnable(ctx, server, *encryptReq.Enable)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
genErrorMessage(resp, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
resp.WriteHeader(http.StatusOK)
|
||||
})
|
||||
}
|
||||
|
||||
func encryptionPrepare(ctx context.Context, server *config.Control, force bool) error {
|
||||
states := secretsencrypt.EncryptionStart + "-" + secretsencrypt.EncryptionReencryptFinished
|
||||
if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), states); err != nil && !force {
|
||||
return err
|
||||
}
|
||||
|
||||
curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := AppendNewEncryptionKey(&curKeys); err != nil {
|
||||
return err
|
||||
}
|
||||
logrus.Infoln("Adding secrets-encryption key: ", curKeys[len(curKeys)-1])
|
||||
|
||||
if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, true); err != nil {
|
||||
return err
|
||||
}
|
||||
nodeName := os.Getenv("NODE_NAME")
|
||||
node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionPrepare); err != nil {
|
||||
return err
|
||||
}
|
||||
return cluster.Save(ctx, server, server.Runtime.EtcdConfig, true)
|
||||
}
|
||||
|
||||
func encryptionRotate(ctx context.Context, server *config.Control, force bool) error {
|
||||
|
||||
if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), secretsencrypt.EncryptionPrepare); err != nil && !force {
|
||||
return err
|
||||
}
|
||||
|
||||
curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Right rotate elements
|
||||
rotatedKeys := append(curKeys[len(curKeys)-1:], curKeys[:len(curKeys)-1]...)
|
||||
|
||||
if err = secretsencrypt.WriteEncryptionConfig(server.Runtime, rotatedKeys, true); err != nil {
|
||||
return err
|
||||
}
|
||||
logrus.Infoln("Encryption keys right rotated")
|
||||
nodeName := os.Getenv("NODE_NAME")
|
||||
node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionRotate); err != nil {
|
||||
return err
|
||||
}
|
||||
return cluster.Save(ctx, server, server.Runtime.EtcdConfig, true)
|
||||
}
|
||||
|
||||
func encryptionReencrypt(ctx context.Context, server *config.Control, force bool, skip bool) error {
|
||||
|
||||
if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), secretsencrypt.EncryptionRotate); err != nil && !force {
|
||||
return err
|
||||
}
|
||||
server.EncryptForce = force
|
||||
server.EncryptSkip = skip
|
||||
nodeName := os.Getenv("NODE_NAME")
|
||||
node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reencryptHash, err := secretsencrypt.GenReencryptHash(server.Runtime, secretsencrypt.EncryptionReencryptRequest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ann := secretsencrypt.EncryptionReencryptRequest + "-" + reencryptHash
|
||||
node.Annotations[secretsencrypt.EncryptionHashAnnotation] = ann
|
||||
if _, err = server.Runtime.Core.Core().V1().Node().Update(node); err != nil {
|
||||
return err
|
||||
}
|
||||
logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func AppendNewEncryptionKey(keys *[]apiserverconfigv1.Key) error {
|
||||
|
||||
aescbcKey := make([]byte, aescbcKeySize)
|
||||
_, err := rand.Read(aescbcKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
encodedKey := base64.StdEncoding.EncodeToString(aescbcKey)
|
||||
|
||||
newKey := []apiserverconfigv1.Key{
|
||||
{
|
||||
Name: "aescbckey-" + time.Now().Format(time.RFC3339),
|
||||
Secret: encodedKey,
|
||||
},
|
||||
}
|
||||
*keys = append(*keys, newKey...)
|
||||
return nil
|
||||
}
|
||||
|
||||
func getEncryptionHashAnnotation(core core.Interface) (string, string, error) {
|
||||
nodeName := os.Getenv("NODE_NAME")
|
||||
node, err := core.V1().Node().Get(nodeName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
ann := node.Annotations[secretsencrypt.EncryptionHashAnnotation]
|
||||
split := strings.Split(ann, "-")
|
||||
if len(split) != 2 {
|
||||
return "", "", fmt.Errorf("invalid annotation %s found on node %s", ann, node.ObjectMeta.Name)
|
||||
}
|
||||
return split[0], split[1], nil
|
||||
}
|
||||
|
||||
func getServerNodes(core core.Interface) ([]corev1.Node, error) {
|
||||
nodes, err := core.V1().Node().List(metav1.ListOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var serverNodes []corev1.Node
|
||||
for _, node := range nodes.Items {
|
||||
if v, ok := node.Labels[ControlPlaneRoleLabelKey]; ok && v == "true" {
|
||||
serverNodes = append(serverNodes, node)
|
||||
}
|
||||
}
|
||||
return serverNodes, nil
|
||||
}
|
||||
|
||||
// verifyEncryptionHashAnnotation checks that all nodes are on the same stage,
|
||||
// and that a request for new stage is valid
|
||||
func verifyEncryptionHashAnnotation(runtime *config.ControlRuntime, core core.Interface, prevStage string) error {
|
||||
|
||||
var firstHash string
|
||||
var firstNodeName string
|
||||
first := true
|
||||
serverNodes, err := getServerNodes(core)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, node := range serverNodes {
|
||||
hash, ok := node.Annotations[secretsencrypt.EncryptionHashAnnotation]
|
||||
if ok && first {
|
||||
firstHash = hash
|
||||
first = false
|
||||
firstNodeName = node.ObjectMeta.Name
|
||||
} else if ok && hash != firstHash {
|
||||
return fmt.Errorf("hash does not match between %s and %s", firstNodeName, node.ObjectMeta.Name)
|
||||
}
|
||||
}
|
||||
|
||||
if prevStage == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
oldStage, oldHash, err := getEncryptionHashAnnotation(core)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
encryptionConfigHash, err := secretsencrypt.GenEncryptionConfigHash(runtime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !strings.Contains(prevStage, oldStage) {
|
||||
return fmt.Errorf("incorrect stage: %s found on node %s", oldStage, serverNodes[0].ObjectMeta.Name)
|
||||
} else if oldHash != encryptionConfigHash {
|
||||
return fmt.Errorf("invalid hash: %s found on node %s", oldHash, serverNodes[0].ObjectMeta.Name)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func genErrorMessage(resp http.ResponseWriter, statusCode int, passedErr error) {
|
||||
errID, err := rand.Int(rand.Reader, big.NewInt(99999))
|
||||
if err != nil {
|
||||
resp.WriteHeader(http.StatusInternalServerError)
|
||||
resp.Write([]byte(err.Error()))
|
||||
return
|
||||
}
|
||||
logrus.Warnf("secrets-encrypt-%s: %s", errID.String(), passedErr.Error())
|
||||
resp.WriteHeader(statusCode)
|
||||
resp.Write([]byte(fmt.Sprintf("error secrets-encrypt-%s: see server logs for more info", errID.String())))
|
||||
}
|
||||
@ -0,0 +1,155 @@
|
||||
package integration
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
"github.com/onsi/ginkgo/reporters"
|
||||
. "github.com/onsi/gomega"
|
||||
testutil "github.com/rancher/k3s/tests/util"
|
||||
)
|
||||
|
||||
var secretsEncryptionServer *testutil.K3sServer
|
||||
var secretsEncryptionDataDir = "/tmp/k3sse"
|
||||
|
||||
var secretsEncryptionServerArgs = []string{"--secrets-encryption", "-d", secretsEncryptionDataDir}
|
||||
var _ = BeforeSuite(func() {
|
||||
if !testutil.IsExistingServer() {
|
||||
var err error
|
||||
Expect(os.MkdirAll(secretsEncryptionDataDir, 0777)).To(Succeed())
|
||||
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
})
|
||||
|
||||
var _ = Describe("secrets encryption rotation", func() {
|
||||
BeforeEach(func() {
|
||||
if testutil.IsExistingServer() {
|
||||
Skip("Test does not support running on existing k3s servers")
|
||||
}
|
||||
})
|
||||
When("A server starts with secrets encryption", func() {
|
||||
It("starts up with no problems", func() {
|
||||
Eventually(func() (string, error) {
|
||||
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
|
||||
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
|
||||
})
|
||||
It("it creates a encryption key", func() {
|
||||
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(result).To(ContainSubstring("Encryption Status: Enabled"))
|
||||
Expect(result).To(ContainSubstring("Current Rotation Stage: start"))
|
||||
})
|
||||
})
|
||||
When("A server rotates encryption keys", func() {
|
||||
It("it prepares to rotate", func() {
|
||||
Expect(testutil.K3sCmd("secrets-encrypt", "prepare", "-d", secretsEncryptionDataDir)).
|
||||
To(ContainSubstring("prepare completed successfully"))
|
||||
|
||||
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(result).To(ContainSubstring("Current Rotation Stage: prepare"))
|
||||
reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
keys := reg.FindAllString(result, -1)
|
||||
Expect(keys).To(HaveLen(2))
|
||||
Expect(keys[0]).To(ContainSubstring("aescbckey"))
|
||||
Expect(keys[1]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year())))
|
||||
})
|
||||
It("restarts the server", func() {
|
||||
var err error
|
||||
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
|
||||
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Eventually(func() (string, error) {
|
||||
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
|
||||
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
|
||||
})
|
||||
It("rotates the keys", func() {
|
||||
Eventually(func() (string, error) {
|
||||
return testutil.K3sCmd("secrets-encrypt", "rotate", "-d", secretsEncryptionDataDir)
|
||||
}, "10s", "2s").Should(ContainSubstring("rotate completed successfully"))
|
||||
|
||||
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(result).To(ContainSubstring("Current Rotation Stage: rotate"))
|
||||
reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
keys := reg.FindAllString(result, -1)
|
||||
Expect(keys).To(HaveLen(2))
|
||||
Expect(keys[0]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year())))
|
||||
Expect(keys[1]).To(ContainSubstring("aescbckey"))
|
||||
})
|
||||
It("restarts the server", func() {
|
||||
var err error
|
||||
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
|
||||
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Eventually(func() (string, error) {
|
||||
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
|
||||
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
|
||||
time.Sleep(10 * time.Second)
|
||||
})
|
||||
It("reencrypts the keys", func() {
|
||||
Expect(testutil.K3sCmd("secrets-encrypt", "reencrypt", "-d", secretsEncryptionDataDir)).
|
||||
To(ContainSubstring("reencryption started"))
|
||||
Eventually(func() (string, error) {
|
||||
return testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
|
||||
}, "30s", "2s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
|
||||
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
keys := reg.FindAllString(result, -1)
|
||||
Expect(keys).To(HaveLen(1))
|
||||
Expect(keys[0]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year())))
|
||||
})
|
||||
})
|
||||
When("A server disables encryption", func() {
|
||||
It("it triggers the disable", func() {
|
||||
Expect(testutil.K3sCmd("secrets-encrypt", "disable", "-d", secretsEncryptionDataDir)).
|
||||
To(ContainSubstring("secrets-encryption disabled"))
|
||||
|
||||
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(result).To(ContainSubstring("Encryption Status: Disabled"))
|
||||
})
|
||||
It("restarts the server", func() {
|
||||
var err error
|
||||
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
|
||||
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Eventually(func() (string, error) {
|
||||
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
|
||||
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
|
||||
time.Sleep(10 * time.Second)
|
||||
})
|
||||
It("reencrypts the keys", func() {
|
||||
result, err := testutil.K3sCmd("secrets-encrypt", "reencrypt", "-f", "--skip", "-d", secretsEncryptionDataDir)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(result).To(ContainSubstring("reencryption started"))
|
||||
|
||||
result, err = testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(result).To(ContainSubstring("Encryption Status: Disabled"))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
var _ = AfterSuite(func() {
|
||||
if !testutil.IsExistingServer() {
|
||||
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
|
||||
Expect(testutil.K3sRemoveDataDir(secretsEncryptionDataDir)).To(Succeed())
|
||||
}
|
||||
})
|
||||
|
||||
func Test_IntegrationSecretsEncryption(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecsWithDefaultAndCustomReporters(t, "Secrets Encryption Suite", []Reporter{
|
||||
reporters.NewJUnitReporter("/tmp/results/junit-se.xml"),
|
||||
})
|
||||
}
|
||||
Loading…
Reference in new issue