mirror of https://github.com/k3s-io/k3s
Browse Source
* Regular CLI framework for encrypt commands * New secrets-encryption feature * New integration test * fixes for flaky integration test CI * Fix to bootstrap on restart of existing nodes * Consolidate event recorder Signed-off-by: Derek Nola <derek.nola@suse.com>pull/4693/head
Derek Nola
3 years ago
committed by
GitHub
32 changed files with 1460 additions and 104 deletions
@ -0,0 +1,32 @@
|
||||
package main |
||||
|
||||
import ( |
||||
"context" |
||||
"errors" |
||||
"os" |
||||
|
||||
"github.com/rancher/k3s/pkg/cli/cmds" |
||||
"github.com/rancher/k3s/pkg/cli/secretsencrypt" |
||||
"github.com/rancher/k3s/pkg/configfilearg" |
||||
"github.com/sirupsen/logrus" |
||||
"github.com/urfave/cli" |
||||
) |
||||
|
||||
func main() { |
||||
app := cmds.NewApp() |
||||
app.Commands = []cli.Command{ |
||||
cmds.NewSecretsEncryptCommand(cli.ShowAppHelp, |
||||
cmds.NewSecretsEncryptSubcommands( |
||||
secretsencrypt.Status, |
||||
secretsencrypt.Enable, |
||||
secretsencrypt.Disable, |
||||
secretsencrypt.Prepare, |
||||
secretsencrypt.Rotate, |
||||
secretsencrypt.Reencrypt), |
||||
), |
||||
} |
||||
|
||||
if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) { |
||||
logrus.Fatal(err) |
||||
} |
||||
} |
||||
@ -0,0 +1,94 @@
|
||||
package cmds |
||||
|
||||
import ( |
||||
"github.com/urfave/cli" |
||||
) |
||||
|
||||
const SecretsEncryptCommand = "secrets-encrypt" |
||||
|
||||
var EncryptFlags = []cli.Flag{ |
||||
DataDirFlag, |
||||
ServerToken, |
||||
} |
||||
|
||||
func NewSecretsEncryptCommand(action func(*cli.Context) error, subcommands []cli.Command) cli.Command { |
||||
return cli.Command{ |
||||
Name: SecretsEncryptCommand, |
||||
Usage: "Control secrets encryption and keys rotation", |
||||
SkipFlagParsing: false, |
||||
SkipArgReorder: true, |
||||
Action: action, |
||||
Subcommands: subcommands, |
||||
} |
||||
} |
||||
|
||||
func NewSecretsEncryptSubcommands(status, enable, disable, prepare, rotate, reencrypt func(ctx *cli.Context) error) []cli.Command { |
||||
return []cli.Command{ |
||||
{ |
||||
Name: "status", |
||||
Usage: "Print current status of secrets encryption", |
||||
SkipFlagParsing: false, |
||||
SkipArgReorder: true, |
||||
Action: status, |
||||
Flags: EncryptFlags, |
||||
}, |
||||
{ |
||||
Name: "enable", |
||||
Usage: "Enable secrets encryption", |
||||
SkipFlagParsing: false, |
||||
SkipArgReorder: true, |
||||
Action: enable, |
||||
Flags: EncryptFlags, |
||||
}, |
||||
{ |
||||
Name: "disable", |
||||
Usage: "Disable secrets encryption", |
||||
SkipFlagParsing: false, |
||||
SkipArgReorder: true, |
||||
Action: disable, |
||||
Flags: EncryptFlags, |
||||
}, |
||||
{ |
||||
Name: "prepare", |
||||
Usage: "Prepare for encryption keys rotation", |
||||
SkipFlagParsing: false, |
||||
SkipArgReorder: true, |
||||
Action: prepare, |
||||
Flags: append(EncryptFlags, &cli.BoolFlag{ |
||||
Name: "f,force", |
||||
Usage: "Force preparation.", |
||||
Destination: &ServerConfig.EncryptForce, |
||||
}), |
||||
}, |
||||
{ |
||||
Name: "rotate", |
||||
Usage: "Rotate secrets encryption keys", |
||||
SkipFlagParsing: false, |
||||
SkipArgReorder: true, |
||||
Action: rotate, |
||||
Flags: append(EncryptFlags, &cli.BoolFlag{ |
||||
Name: "f,force", |
||||
Usage: "Force key rotation.", |
||||
Destination: &ServerConfig.EncryptForce, |
||||
}), |
||||
}, |
||||
{ |
||||
Name: "reencrypt", |
||||
Usage: "Reencrypt all data with new encryption key", |
||||
SkipFlagParsing: false, |
||||
SkipArgReorder: true, |
||||
Action: reencrypt, |
||||
Flags: append(EncryptFlags, |
||||
&cli.BoolFlag{ |
||||
Name: "f,force", |
||||
Usage: "Force secrets reencryption.", |
||||
Destination: &ServerConfig.EncryptForce, |
||||
}, |
||||
&cli.BoolFlag{ |
||||
Name: "skip", |
||||
Usage: "Skip removing old key", |
||||
Destination: &ServerConfig.EncryptSkip, |
||||
}), |
||||
}, |
||||
} |
||||
} |
||||
@ -0,0 +1,216 @@
|
||||
package secretsencrypt |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/json" |
||||
"fmt" |
||||
"io/ioutil" |
||||
"os" |
||||
"path/filepath" |
||||
"text/tabwriter" |
||||
|
||||
"github.com/erikdubbelboer/gspt" |
||||
"github.com/rancher/k3s/pkg/cli/cmds" |
||||
"github.com/rancher/k3s/pkg/clientaccess" |
||||
"github.com/rancher/k3s/pkg/daemons/config" |
||||
"github.com/rancher/k3s/pkg/secretsencrypt" |
||||
"github.com/rancher/k3s/pkg/server" |
||||
"github.com/rancher/k3s/pkg/version" |
||||
"github.com/urfave/cli" |
||||
"k8s.io/utils/pointer" |
||||
) |
||||
|
||||
func commandPrep(app *cli.Context, cfg *cmds.Server) (config.Control, *clientaccess.Info, error) { |
||||
var controlConfig config.Control |
||||
var err error |
||||
// hide process arguments from ps output, since they may contain
|
||||
// database credentials or other secrets.
|
||||
gspt.SetProcTitle(os.Args[0] + " encrypt") |
||||
|
||||
controlConfig.DataDir, err = server.ResolveDataDir(cfg.DataDir) |
||||
if err != nil { |
||||
return controlConfig, nil, err |
||||
} |
||||
if cfg.ServerURL == "" { |
||||
cfg.ServerURL = "https://127.0.0.1:6443" |
||||
} |
||||
|
||||
if cfg.Token == "" { |
||||
fp := filepath.Join(controlConfig.DataDir, "token") |
||||
tokenByte, err := ioutil.ReadFile(fp) |
||||
if err != nil { |
||||
return controlConfig, nil, err |
||||
} |
||||
controlConfig.Token = string(bytes.TrimRight(tokenByte, "\n")) |
||||
} else { |
||||
controlConfig.Token = cfg.Token |
||||
} |
||||
controlConfig.EncryptForce = cfg.EncryptForce |
||||
controlConfig.EncryptSkip = cfg.EncryptSkip |
||||
info, err := clientaccess.ParseAndValidateTokenForUser(cmds.ServerConfig.ServerURL, controlConfig.Token, "node") |
||||
if err != nil { |
||||
return controlConfig, nil, err |
||||
} |
||||
return controlConfig, info, nil |
||||
} |
||||
|
||||
func Enable(app *cli.Context) error { |
||||
var err error |
||||
if err = cmds.InitLogging(); err != nil { |
||||
return err |
||||
} |
||||
_, info, err := commandPrep(app, &cmds.ServerConfig) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
b, err := json.Marshal(server.EncryptionRequest{Enable: pointer.Bool(true)}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil { |
||||
return err |
||||
} |
||||
fmt.Println("secrets-encryption enabled") |
||||
return nil |
||||
} |
||||
|
||||
func Disable(app *cli.Context) error { |
||||
|
||||
if err := cmds.InitLogging(); err != nil { |
||||
return err |
||||
} |
||||
_, info, err := commandPrep(app, &cmds.ServerConfig) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
b, err := json.Marshal(server.EncryptionRequest{Enable: pointer.Bool(false)}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil { |
||||
return err |
||||
} |
||||
fmt.Println("secrets-encryption disabled") |
||||
return nil |
||||
} |
||||
|
||||
func Status(app *cli.Context) error { |
||||
if err := cmds.InitLogging(); err != nil { |
||||
return err |
||||
} |
||||
_, info, err := commandPrep(app, &cmds.ServerConfig) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
data, err := info.Get("/v1-" + version.Program + "/encrypt/status") |
||||
if err != nil { |
||||
return err |
||||
} |
||||
status := server.EncryptionState{} |
||||
if err := json.Unmarshal(data, &status); err != nil { |
||||
return err |
||||
} |
||||
|
||||
if status.Enable == nil { |
||||
fmt.Println("Encryption Status: Disabled, no configuration file found") |
||||
return nil |
||||
} |
||||
|
||||
var statusOutput string |
||||
if *status.Enable { |
||||
statusOutput += "Encryption Status: Enabled\n" |
||||
} else { |
||||
statusOutput += "Encryption Status: Disabled\n" |
||||
} |
||||
statusOutput += fmt.Sprintln("Current Rotation Stage:", status.Stage) |
||||
|
||||
if status.HashMatch { |
||||
statusOutput += fmt.Sprintln("Server Encryption Hashes: All hashes match") |
||||
} else { |
||||
statusOutput += fmt.Sprintf("Server Encryption Hashes: %s\n", status.HashError) |
||||
} |
||||
|
||||
var tabBuffer bytes.Buffer |
||||
w := tabwriter.NewWriter(&tabBuffer, 0, 0, 2, ' ', 0) |
||||
fmt.Fprintf(w, "\n") |
||||
fmt.Fprintf(w, "Active\tKey Type\tName\n") |
||||
fmt.Fprintf(w, "------\t--------\t----\n") |
||||
if status.ActiveKey != "" { |
||||
fmt.Fprintf(w, " *\t%s\t%s\n", "AES-CBC", status.ActiveKey) |
||||
} |
||||
for _, k := range status.InactiveKeys { |
||||
fmt.Fprintf(w, "\t%s\t%s\n", "AES-CBC", k) |
||||
} |
||||
w.Flush() |
||||
fmt.Println(statusOutput + tabBuffer.String()) |
||||
return nil |
||||
} |
||||
|
||||
func Prepare(app *cli.Context) error { |
||||
var err error |
||||
if err = cmds.InitLogging(); err != nil { |
||||
return err |
||||
} |
||||
controlConfig, info, err := commandPrep(app, &cmds.ServerConfig) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
b, err := json.Marshal(server.EncryptionRequest{ |
||||
Stage: pointer.StringPtr(secretsencrypt.EncryptionPrepare), |
||||
Force: controlConfig.EncryptForce, |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil { |
||||
return err |
||||
} |
||||
fmt.Println("prepare completed successfully") |
||||
return nil |
||||
} |
||||
|
||||
func Rotate(app *cli.Context) error { |
||||
if err := cmds.InitLogging(); err != nil { |
||||
return err |
||||
} |
||||
controlConfig, info, err := commandPrep(app, &cmds.ServerConfig) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
b, err := json.Marshal(server.EncryptionRequest{ |
||||
Stage: pointer.StringPtr(secretsencrypt.EncryptionRotate), |
||||
Force: controlConfig.EncryptForce, |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil { |
||||
return err |
||||
} |
||||
fmt.Println("rotate completed successfully") |
||||
return nil |
||||
} |
||||
|
||||
func Reencrypt(app *cli.Context) error { |
||||
var err error |
||||
if err = cmds.InitLogging(); err != nil { |
||||
return err |
||||
} |
||||
controlConfig, info, err := commandPrep(app, &cmds.ServerConfig) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
b, err := json.Marshal(server.EncryptionRequest{ |
||||
Stage: pointer.StringPtr(secretsencrypt.EncryptionReencryptActive), |
||||
Force: controlConfig.EncryptForce, |
||||
Skip: controlConfig.EncryptSkip, |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil { |
||||
return err |
||||
} |
||||
fmt.Println("reencryption started") |
||||
return nil |
||||
} |
||||
@ -0,0 +1,174 @@
|
||||
package secretsencrypt |
||||
|
||||
import ( |
||||
"crypto/sha256" |
||||
"encoding/hex" |
||||
"encoding/json" |
||||
"fmt" |
||||
"io/ioutil" |
||||
|
||||
"github.com/rancher/k3s/pkg/daemons/config" |
||||
"github.com/rancher/k3s/pkg/version" |
||||
corev1 "k8s.io/api/core/v1" |
||||
|
||||
"github.com/sirupsen/logrus" |
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
||||
apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1" |
||||
) |
||||
|
||||
const ( |
||||
EncryptionStart string = "start" |
||||
EncryptionPrepare string = "prepare" |
||||
EncryptionRotate string = "rotate" |
||||
EncryptionReencryptRequest string = "reencrypt_request" |
||||
EncryptionReencryptActive string = "reencrypt_active" |
||||
EncryptionReencryptFinished string = "reencrypt_finished" |
||||
) |
||||
|
||||
var EncryptionHashAnnotation = version.Program + ".io/encryption-config-hash" |
||||
|
||||
func GetEncryptionProviders(runtime *config.ControlRuntime) ([]apiserverconfigv1.ProviderConfiguration, error) { |
||||
curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionConfig) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
curEncryption := apiserverconfigv1.EncryptionConfiguration{} |
||||
if err = json.Unmarshal(curEncryptionByte, &curEncryption); err != nil { |
||||
return nil, err |
||||
} |
||||
return curEncryption.Resources[0].Providers, nil |
||||
} |
||||
|
||||
func GetEncryptionKeys(runtime *config.ControlRuntime) ([]apiserverconfigv1.Key, error) { |
||||
|
||||
providers, err := GetEncryptionProviders(runtime) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if len(providers) > 2 { |
||||
return nil, fmt.Errorf("more than 2 providers (%d) found in secrets encryption", len(providers)) |
||||
} |
||||
|
||||
var curKeys []apiserverconfigv1.Key |
||||
for _, p := range providers { |
||||
if p.AESCBC != nil { |
||||
curKeys = append(curKeys, p.AESCBC.Keys...) |
||||
} |
||||
if p.AESGCM != nil || p.KMS != nil || p.Secretbox != nil { |
||||
return nil, fmt.Errorf("non-standard encryption keys found") |
||||
} |
||||
} |
||||
return curKeys, nil |
||||
} |
||||
|
||||
func WriteEncryptionConfig(runtime *config.ControlRuntime, keys []apiserverconfigv1.Key, enable bool) error { |
||||
|
||||
// Placing the identity provider first disables encryption
|
||||
var providers []apiserverconfigv1.ProviderConfiguration |
||||
if enable { |
||||
providers = []apiserverconfigv1.ProviderConfiguration{ |
||||
{ |
||||
AESCBC: &apiserverconfigv1.AESConfiguration{ |
||||
Keys: keys, |
||||
}, |
||||
}, |
||||
{ |
||||
Identity: &apiserverconfigv1.IdentityConfiguration{}, |
||||
}, |
||||
} |
||||
} else { |
||||
providers = []apiserverconfigv1.ProviderConfiguration{ |
||||
{ |
||||
Identity: &apiserverconfigv1.IdentityConfiguration{}, |
||||
}, |
||||
{ |
||||
AESCBC: &apiserverconfigv1.AESConfiguration{ |
||||
Keys: keys, |
||||
}, |
||||
}, |
||||
} |
||||
} |
||||
|
||||
encConfig := apiserverconfigv1.EncryptionConfiguration{ |
||||
TypeMeta: metav1.TypeMeta{ |
||||
Kind: "EncryptionConfiguration", |
||||
APIVersion: "apiserver.config.k8s.io/v1", |
||||
}, |
||||
Resources: []apiserverconfigv1.ResourceConfiguration{ |
||||
{ |
||||
Resources: []string{"secrets"}, |
||||
Providers: providers, |
||||
}, |
||||
}, |
||||
} |
||||
jsonfile, err := json.Marshal(encConfig) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return ioutil.WriteFile(runtime.EncryptionConfig, jsonfile, 0600) |
||||
} |
||||
|
||||
func GenEncryptionConfigHash(runtime *config.ControlRuntime) (string, error) { |
||||
curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionConfig) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
encryptionConfigHash := sha256.Sum256(curEncryptionByte) |
||||
return hex.EncodeToString(encryptionConfigHash[:]), nil |
||||
} |
||||
|
||||
// GenReencryptHash generates a sha256 hash fom the existing secrets keys and
|
||||
// a new key based on the input arguments.
|
||||
func GenReencryptHash(runtime *config.ControlRuntime, keyName string) (string, error) { |
||||
|
||||
keys, err := GetEncryptionKeys(runtime) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
newKey := apiserverconfigv1.Key{ |
||||
Name: keyName, |
||||
Secret: "12345", |
||||
} |
||||
keys = append(keys, newKey) |
||||
b, err := json.Marshal(keys) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
hash := sha256.Sum256(b) |
||||
return hex.EncodeToString(hash[:]), nil |
||||
} |
||||
|
||||
func getEncryptionHashFile(runtime *config.ControlRuntime) (string, error) { |
||||
curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionHash) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
return string(curEncryptionByte), nil |
||||
} |
||||
|
||||
func BootstrapEncryptionHashAnnotation(node *corev1.Node, runtime *config.ControlRuntime) error { |
||||
existingAnn, err := getEncryptionHashFile(runtime) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
node.Annotations[EncryptionHashAnnotation] = existingAnn |
||||
return nil |
||||
} |
||||
|
||||
func WriteEncryptionHashAnnotation(runtime *config.ControlRuntime, node *corev1.Node, stage string) error { |
||||
encryptionConfigHash, err := GenEncryptionConfigHash(runtime) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if node.Annotations == nil { |
||||
return fmt.Errorf("node annotations do not exist for %s", node.ObjectMeta.Name) |
||||
} |
||||
ann := stage + "-" + encryptionConfigHash |
||||
node.Annotations[EncryptionHashAnnotation] = ann |
||||
if _, err = runtime.Core.Core().V1().Node().Update(node); err != nil { |
||||
return err |
||||
} |
||||
logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name) |
||||
return ioutil.WriteFile(runtime.EncryptionHash, []byte(ann), 0600) |
||||
} |
||||
@ -0,0 +1,196 @@
|
||||
package secretsencrypt |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"strings" |
||||
|
||||
"github.com/rancher/k3s/pkg/cluster" |
||||
"github.com/rancher/k3s/pkg/daemons/config" |
||||
"github.com/rancher/k3s/pkg/util" |
||||
coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1" |
||||
"github.com/sirupsen/logrus" |
||||
corev1 "k8s.io/api/core/v1" |
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
||||
"k8s.io/apimachinery/pkg/runtime" |
||||
|
||||
"k8s.io/client-go/kubernetes" |
||||
"k8s.io/client-go/tools/pager" |
||||
"k8s.io/client-go/tools/record" |
||||
) |
||||
|
||||
const ( |
||||
controllerAgentName string = "reencrypt-controller" |
||||
secretsUpdateStartEvent string = "SecretsUpdateStart" |
||||
secretsProgressEvent string = "SecretsProgress" |
||||
secretsUpdateCompleteEvent string = "SecretsUpdateComplete" |
||||
secretsUpdateErrorEvent string = "SecretsUpdateError" |
||||
) |
||||
|
||||
type handler struct { |
||||
ctx context.Context |
||||
controlConfig *config.Control |
||||
nodes coreclient.NodeController |
||||
secrets coreclient.SecretController |
||||
recorder record.EventRecorder |
||||
} |
||||
|
||||
func Register( |
||||
ctx context.Context, |
||||
k8s kubernetes.Interface, |
||||
controlConfig *config.Control, |
||||
nodes coreclient.NodeController, |
||||
secrets coreclient.SecretController, |
||||
) error { |
||||
h := &handler{ |
||||
ctx: ctx, |
||||
controlConfig: controlConfig, |
||||
nodes: nodes, |
||||
secrets: secrets, |
||||
recorder: util.BuildControllerEventRecorder(k8s, controllerAgentName), |
||||
} |
||||
|
||||
nodes.OnChange(ctx, "reencrypt-controller", h.onChangeNode) |
||||
return nil |
||||
} |
||||
|
||||
// onChangeNode handles changes to Nodes. We are looking for a specific annotation change
|
||||
func (h *handler) onChangeNode(key string, node *corev1.Node) (*corev1.Node, error) { |
||||
if node == nil { |
||||
return nil, nil |
||||
} |
||||
|
||||
ann, ok := node.Annotations[EncryptionHashAnnotation] |
||||
if !ok { |
||||
return node, nil |
||||
} |
||||
|
||||
if valid, err := h.validateReencryptStage(node, ann); err != nil { |
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) |
||||
return node, err |
||||
} else if !valid { |
||||
return node, nil |
||||
} |
||||
|
||||
reencryptHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptActive) |
||||
if err != nil { |
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) |
||||
return node, err |
||||
} |
||||
ann = EncryptionReencryptActive + "-" + reencryptHash |
||||
node.Annotations[EncryptionHashAnnotation] = ann |
||||
node, err = h.nodes.Update(node) |
||||
if err != nil { |
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) |
||||
return node, err |
||||
} |
||||
|
||||
if err := h.updateSecrets(node); err != nil { |
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) |
||||
return node, err |
||||
} |
||||
|
||||
// If skipping, revert back to the previous stage
|
||||
if h.controlConfig.EncryptSkip { |
||||
BootstrapEncryptionHashAnnotation(node, h.controlConfig.Runtime) |
||||
if node, err := h.nodes.Update(node); err != nil { |
||||
return node, err |
||||
} |
||||
return node, nil |
||||
} |
||||
|
||||
// Remove last key
|
||||
curKeys, err := GetEncryptionKeys(h.controlConfig.Runtime) |
||||
if err != nil { |
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) |
||||
return node, err |
||||
} |
||||
|
||||
curKeys = curKeys[:len(curKeys)-1] |
||||
if err = WriteEncryptionConfig(h.controlConfig.Runtime, curKeys, true); err != nil { |
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) |
||||
return node, err |
||||
} |
||||
logrus.Infoln("Removed key: ", curKeys[len(curKeys)-1]) |
||||
if err != nil { |
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) |
||||
return node, err |
||||
} |
||||
if err := WriteEncryptionHashAnnotation(h.controlConfig.Runtime, node, EncryptionReencryptFinished); err != nil { |
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) |
||||
return node, err |
||||
} |
||||
if err := cluster.Save(h.ctx, h.controlConfig, h.controlConfig.Runtime.EtcdConfig, true); err != nil { |
||||
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) |
||||
return node, err |
||||
} |
||||
return node, nil |
||||
} |
||||
|
||||
// validateReencryptStage ensures that the request for reencryption is valid and
|
||||
// that there is only one active reencryption at a time
|
||||
func (h *handler) validateReencryptStage(node *corev1.Node, annotation string) (bool, error) { |
||||
|
||||
split := strings.Split(annotation, "-") |
||||
if len(split) != 2 { |
||||
err := fmt.Errorf("invalid annotation %s found on node %s", annotation, node.ObjectMeta.Name) |
||||
return false, err |
||||
} |
||||
stage := split[0] |
||||
hash := split[1] |
||||
|
||||
// Validate the specific stage and the request via sha256 hash
|
||||
if stage != EncryptionReencryptRequest { |
||||
return false, nil |
||||
} |
||||
if reencryptRequestHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptRequest); err != nil { |
||||
return false, err |
||||
} else if reencryptRequestHash != hash { |
||||
err = fmt.Errorf("invalid hash: %s found on node %s", hash, node.ObjectMeta.Name) |
||||
return false, err |
||||
} |
||||
|
||||
nodes, err := h.nodes.List(metav1.ListOptions{}) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
reencryptActiveHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptActive) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
for _, node := range nodes.Items { |
||||
if ann, ok := node.Annotations[EncryptionHashAnnotation]; ok { |
||||
split := strings.Split(ann, "-") |
||||
if len(split) != 2 { |
||||
return false, fmt.Errorf("invalid annotation %s found on node %s", ann, node.ObjectMeta.Name) |
||||
} |
||||
stage := split[0] |
||||
hash := split[1] |
||||
if stage == EncryptionReencryptActive && hash == reencryptActiveHash { |
||||
return false, fmt.Errorf("another reencrypt is already active") |
||||
} |
||||
} |
||||
} |
||||
return true, nil |
||||
} |
||||
|
||||
func (h *handler) updateSecrets(node *corev1.Node) error { |
||||
secretPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) { |
||||
return h.secrets.List("", opts) |
||||
})) |
||||
i := 0 |
||||
secretPager.EachListItem(h.ctx, metav1.ListOptions{}, func(obj runtime.Object) error { |
||||
if secret, ok := obj.(*corev1.Secret); ok { |
||||
if _, err := h.secrets.Update(secret); err != nil { |
||||
return fmt.Errorf("failed to reencrypted secret: %v", err) |
||||
} |
||||
if i != 0 && i%10 == 0 { |
||||
h.recorder.Eventf(node, corev1.EventTypeNormal, secretsProgressEvent, "reencrypted %d secrets", i) |
||||
} |
||||
i++ |
||||
} |
||||
return nil |
||||
}) |
||||
h.recorder.Eventf(node, corev1.EventTypeNormal, secretsUpdateCompleteEvent, "completed reencrypt of %d secrets", i) |
||||
return nil |
||||
} |
||||
@ -0,0 +1,379 @@
|
||||
package server |
||||
|
||||
import ( |
||||
"context" |
||||
"crypto/rand" |
||||
"encoding/base64" |
||||
"encoding/json" |
||||
"fmt" |
||||
"io/ioutil" |
||||
"math/big" |
||||
"net/http" |
||||
"os" |
||||
"strings" |
||||
"time" |
||||
|
||||
"github.com/rancher/k3s/pkg/cluster" |
||||
"github.com/rancher/k3s/pkg/daemons/config" |
||||
"github.com/rancher/k3s/pkg/secretsencrypt" |
||||
"github.com/rancher/wrangler/pkg/generated/controllers/core" |
||||
"github.com/sirupsen/logrus" |
||||
corev1 "k8s.io/api/core/v1" |
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
||||
apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1" |
||||
"k8s.io/utils/pointer" |
||||
) |
||||
|
||||
const aescbcKeySize = 32 |
||||
|
||||
type EncryptionState struct { |
||||
Stage string `json:"stage"` |
||||
ActiveKey string `json:"activekey"` |
||||
Enable *bool `json:"enable,omitempty"` |
||||
HashMatch bool `json:"hashmatch,omitempty"` |
||||
HashError string `json:"hasherror,omitempty"` |
||||
InactiveKeys []string `json:"inactivekeys,omitempty"` |
||||
} |
||||
|
||||
type EncryptionRequest struct { |
||||
Stage *string `json:"stage,omitempty"` |
||||
Enable *bool `json:"enable,omitempty"` |
||||
Force bool `json:"force"` |
||||
Skip bool `json:"skip"` |
||||
} |
||||
|
||||
func getEncryptionRequest(req *http.Request) (EncryptionRequest, error) { |
||||
b, err := ioutil.ReadAll(req.Body) |
||||
if err != nil { |
||||
return EncryptionRequest{}, err |
||||
} |
||||
result := EncryptionRequest{} |
||||
err = json.Unmarshal(b, &result) |
||||
return result, err |
||||
} |
||||
|
||||
func encryptionStatusHandler(server *config.Control) http.Handler { |
||||
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { |
||||
if req.TLS == nil { |
||||
resp.WriteHeader(http.StatusNotFound) |
||||
return |
||||
} |
||||
status, err := encryptionStatus(server) |
||||
if err != nil { |
||||
genErrorMessage(resp, http.StatusInternalServerError, err) |
||||
return |
||||
} |
||||
b, err := json.Marshal(status) |
||||
if err != nil { |
||||
genErrorMessage(resp, http.StatusInternalServerError, err) |
||||
return |
||||
} |
||||
resp.Write(b) |
||||
}) |
||||
} |
||||
|
||||
func encryptionStatus(server *config.Control) (EncryptionState, error) { |
||||
state := EncryptionState{} |
||||
providers, err := secretsencrypt.GetEncryptionProviders(server.Runtime) |
||||
if os.IsNotExist(err) { |
||||
return state, nil |
||||
} else if err != nil { |
||||
return state, err |
||||
} |
||||
if providers[1].Identity != nil && providers[0].AESCBC != nil { |
||||
state.Enable = pointer.Bool(true) |
||||
} else if providers[0].Identity != nil && providers[1].AESCBC != nil || !server.EncryptSecrets { |
||||
state.Enable = pointer.Bool(false) |
||||
} |
||||
|
||||
if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), ""); err != nil { |
||||
state.HashMatch = false |
||||
state.HashError = err.Error() |
||||
} else { |
||||
state.HashMatch = true |
||||
} |
||||
stage, _, err := getEncryptionHashAnnotation(server.Runtime.Core.Core()) |
||||
if err != nil { |
||||
return state, err |
||||
} |
||||
state.Stage = stage |
||||
active := true |
||||
for _, p := range providers { |
||||
if p.AESCBC != nil { |
||||
for _, aesKey := range p.AESCBC.Keys { |
||||
if active { |
||||
active = false |
||||
state.ActiveKey = aesKey.Name |
||||
} else { |
||||
state.InactiveKeys = append(state.InactiveKeys, aesKey.Name) |
||||
} |
||||
} |
||||
} |
||||
if p.Identity != nil { |
||||
active = false |
||||
} |
||||
} |
||||
|
||||
return state, nil |
||||
} |
||||
|
||||
func encryptionEnable(ctx context.Context, server *config.Control, enable bool) error { |
||||
providers, err := secretsencrypt.GetEncryptionProviders(server.Runtime) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if len(providers) > 2 { |
||||
return fmt.Errorf("more than 2 providers (%d) found in secrets encryption", len(providers)) |
||||
} |
||||
curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if providers[1].Identity != nil && providers[0].AESCBC != nil && !enable { |
||||
logrus.Infoln("Disabling secrets encryption") |
||||
if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, enable); err != nil { |
||||
return err |
||||
} |
||||
} else if !enable { |
||||
logrus.Infoln("Secrets encryption already disabled") |
||||
return nil |
||||
} else if providers[0].Identity != nil && providers[1].AESCBC != nil && enable { |
||||
logrus.Infoln("Enabling secrets encryption") |
||||
if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, enable); err != nil { |
||||
return err |
||||
} |
||||
} else if enable { |
||||
logrus.Infoln("Secrets encryption already enabled") |
||||
return nil |
||||
} else { |
||||
return fmt.Errorf("unable to enable/disable secrets encryption, unknown configuration") |
||||
} |
||||
return cluster.Save(ctx, server, server.Runtime.EtcdConfig, true) |
||||
} |
||||
|
||||
func encryptionConfigHandler(ctx context.Context, server *config.Control) http.Handler { |
||||
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { |
||||
if req.TLS == nil { |
||||
resp.WriteHeader(http.StatusNotFound) |
||||
return |
||||
} |
||||
if req.Method != http.MethodPut { |
||||
resp.WriteHeader(http.StatusBadRequest) |
||||
return |
||||
} |
||||
encryptReq, err := getEncryptionRequest(req) |
||||
if err != nil { |
||||
resp.WriteHeader(http.StatusBadRequest) |
||||
resp.Write([]byte(err.Error())) |
||||
return |
||||
} |
||||
if encryptReq.Stage != nil { |
||||
switch *encryptReq.Stage { |
||||
case secretsencrypt.EncryptionPrepare: |
||||
err = encryptionPrepare(ctx, server, encryptReq.Force) |
||||
case secretsencrypt.EncryptionRotate: |
||||
err = encryptionRotate(ctx, server, encryptReq.Force) |
||||
case secretsencrypt.EncryptionReencryptActive: |
||||
err = encryptionReencrypt(ctx, server, encryptReq.Force, encryptReq.Skip) |
||||
default: |
||||
err = fmt.Errorf("unknown stage %s requested", *encryptReq.Stage) |
||||
} |
||||
} else if encryptReq.Enable != nil { |
||||
err = encryptionEnable(ctx, server, *encryptReq.Enable) |
||||
} |
||||
|
||||
if err != nil { |
||||
genErrorMessage(resp, http.StatusBadRequest, err) |
||||
return |
||||
} |
||||
resp.WriteHeader(http.StatusOK) |
||||
}) |
||||
} |
||||
|
||||
func encryptionPrepare(ctx context.Context, server *config.Control, force bool) error { |
||||
states := secretsencrypt.EncryptionStart + "-" + secretsencrypt.EncryptionReencryptFinished |
||||
if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), states); err != nil && !force { |
||||
return err |
||||
} |
||||
|
||||
curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if err := AppendNewEncryptionKey(&curKeys); err != nil { |
||||
return err |
||||
} |
||||
logrus.Infoln("Adding secrets-encryption key: ", curKeys[len(curKeys)-1]) |
||||
|
||||
if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, true); err != nil { |
||||
return err |
||||
} |
||||
nodeName := os.Getenv("NODE_NAME") |
||||
node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if err = secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionPrepare); err != nil { |
||||
return err |
||||
} |
||||
return cluster.Save(ctx, server, server.Runtime.EtcdConfig, true) |
||||
} |
||||
|
||||
func encryptionRotate(ctx context.Context, server *config.Control, force bool) error { |
||||
|
||||
if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), secretsencrypt.EncryptionPrepare); err != nil && !force { |
||||
return err |
||||
} |
||||
|
||||
curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
// Right rotate elements
|
||||
rotatedKeys := append(curKeys[len(curKeys)-1:], curKeys[:len(curKeys)-1]...) |
||||
|
||||
if err = secretsencrypt.WriteEncryptionConfig(server.Runtime, rotatedKeys, true); err != nil { |
||||
return err |
||||
} |
||||
logrus.Infoln("Encryption keys right rotated") |
||||
nodeName := os.Getenv("NODE_NAME") |
||||
node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if err := secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionRotate); err != nil { |
||||
return err |
||||
} |
||||
return cluster.Save(ctx, server, server.Runtime.EtcdConfig, true) |
||||
} |
||||
|
||||
func encryptionReencrypt(ctx context.Context, server *config.Control, force bool, skip bool) error { |
||||
|
||||
if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), secretsencrypt.EncryptionRotate); err != nil && !force { |
||||
return err |
||||
} |
||||
server.EncryptForce = force |
||||
server.EncryptSkip = skip |
||||
nodeName := os.Getenv("NODE_NAME") |
||||
node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
reencryptHash, err := secretsencrypt.GenReencryptHash(server.Runtime, secretsencrypt.EncryptionReencryptRequest) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
ann := secretsencrypt.EncryptionReencryptRequest + "-" + reencryptHash |
||||
node.Annotations[secretsencrypt.EncryptionHashAnnotation] = ann |
||||
if _, err = server.Runtime.Core.Core().V1().Node().Update(node); err != nil { |
||||
return err |
||||
} |
||||
logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name) |
||||
return nil |
||||
} |
||||
|
||||
func AppendNewEncryptionKey(keys *[]apiserverconfigv1.Key) error { |
||||
|
||||
aescbcKey := make([]byte, aescbcKeySize) |
||||
_, err := rand.Read(aescbcKey) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
encodedKey := base64.StdEncoding.EncodeToString(aescbcKey) |
||||
|
||||
newKey := []apiserverconfigv1.Key{ |
||||
{ |
||||
Name: "aescbckey-" + time.Now().Format(time.RFC3339), |
||||
Secret: encodedKey, |
||||
}, |
||||
} |
||||
*keys = append(*keys, newKey...) |
||||
return nil |
||||
} |
||||
|
||||
func getEncryptionHashAnnotation(core core.Interface) (string, string, error) { |
||||
nodeName := os.Getenv("NODE_NAME") |
||||
node, err := core.V1().Node().Get(nodeName, metav1.GetOptions{}) |
||||
if err != nil { |
||||
return "", "", err |
||||
} |
||||
ann := node.Annotations[secretsencrypt.EncryptionHashAnnotation] |
||||
split := strings.Split(ann, "-") |
||||
if len(split) != 2 { |
||||
return "", "", fmt.Errorf("invalid annotation %s found on node %s", ann, node.ObjectMeta.Name) |
||||
} |
||||
return split[0], split[1], nil |
||||
} |
||||
|
||||
func getServerNodes(core core.Interface) ([]corev1.Node, error) { |
||||
nodes, err := core.V1().Node().List(metav1.ListOptions{}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
var serverNodes []corev1.Node |
||||
for _, node := range nodes.Items { |
||||
if v, ok := node.Labels[ControlPlaneRoleLabelKey]; ok && v == "true" { |
||||
serverNodes = append(serverNodes, node) |
||||
} |
||||
} |
||||
return serverNodes, nil |
||||
} |
||||
|
||||
// verifyEncryptionHashAnnotation checks that all nodes are on the same stage,
|
||||
// and that a request for new stage is valid
|
||||
func verifyEncryptionHashAnnotation(runtime *config.ControlRuntime, core core.Interface, prevStage string) error { |
||||
|
||||
var firstHash string |
||||
var firstNodeName string |
||||
first := true |
||||
serverNodes, err := getServerNodes(core) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
for _, node := range serverNodes { |
||||
hash, ok := node.Annotations[secretsencrypt.EncryptionHashAnnotation] |
||||
if ok && first { |
||||
firstHash = hash |
||||
first = false |
||||
firstNodeName = node.ObjectMeta.Name |
||||
} else if ok && hash != firstHash { |
||||
return fmt.Errorf("hash does not match between %s and %s", firstNodeName, node.ObjectMeta.Name) |
||||
} |
||||
} |
||||
|
||||
if prevStage == "" { |
||||
return nil |
||||
} |
||||
|
||||
oldStage, oldHash, err := getEncryptionHashAnnotation(core) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
encryptionConfigHash, err := secretsencrypt.GenEncryptionConfigHash(runtime) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if !strings.Contains(prevStage, oldStage) { |
||||
return fmt.Errorf("incorrect stage: %s found on node %s", oldStage, serverNodes[0].ObjectMeta.Name) |
||||
} else if oldHash != encryptionConfigHash { |
||||
return fmt.Errorf("invalid hash: %s found on node %s", oldHash, serverNodes[0].ObjectMeta.Name) |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func genErrorMessage(resp http.ResponseWriter, statusCode int, passedErr error) { |
||||
errID, err := rand.Int(rand.Reader, big.NewInt(99999)) |
||||
if err != nil { |
||||
resp.WriteHeader(http.StatusInternalServerError) |
||||
resp.Write([]byte(err.Error())) |
||||
return |
||||
} |
||||
logrus.Warnf("secrets-encrypt-%s: %s", errID.String(), passedErr.Error()) |
||||
resp.WriteHeader(statusCode) |
||||
resp.Write([]byte(fmt.Sprintf("error secrets-encrypt-%s: see server logs for more info", errID.String()))) |
||||
} |
||||
@ -0,0 +1,155 @@
|
||||
package integration |
||||
|
||||
import ( |
||||
"fmt" |
||||
"os" |
||||
"regexp" |
||||
"testing" |
||||
"time" |
||||
|
||||
. "github.com/onsi/ginkgo" |
||||
"github.com/onsi/ginkgo/reporters" |
||||
. "github.com/onsi/gomega" |
||||
testutil "github.com/rancher/k3s/tests/util" |
||||
) |
||||
|
||||
var secretsEncryptionServer *testutil.K3sServer |
||||
var secretsEncryptionDataDir = "/tmp/k3sse" |
||||
|
||||
var secretsEncryptionServerArgs = []string{"--secrets-encryption", "-d", secretsEncryptionDataDir} |
||||
var _ = BeforeSuite(func() { |
||||
if !testutil.IsExistingServer() { |
||||
var err error |
||||
Expect(os.MkdirAll(secretsEncryptionDataDir, 0777)).To(Succeed()) |
||||
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) |
||||
Expect(err).ToNot(HaveOccurred()) |
||||
} |
||||
}) |
||||
|
||||
var _ = Describe("secrets encryption rotation", func() { |
||||
BeforeEach(func() { |
||||
if testutil.IsExistingServer() { |
||||
Skip("Test does not support running on existing k3s servers") |
||||
} |
||||
}) |
||||
When("A server starts with secrets encryption", func() { |
||||
It("starts up with no problems", func() { |
||||
Eventually(func() (string, error) { |
||||
return testutil.K3sCmd("kubectl", "get", "pods", "-A") |
||||
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) |
||||
}) |
||||
It("it creates a encryption key", func() { |
||||
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir) |
||||
Expect(err).NotTo(HaveOccurred()) |
||||
Expect(result).To(ContainSubstring("Encryption Status: Enabled")) |
||||
Expect(result).To(ContainSubstring("Current Rotation Stage: start")) |
||||
}) |
||||
}) |
||||
When("A server rotates encryption keys", func() { |
||||
It("it prepares to rotate", func() { |
||||
Expect(testutil.K3sCmd("secrets-encrypt", "prepare", "-d", secretsEncryptionDataDir)). |
||||
To(ContainSubstring("prepare completed successfully")) |
||||
|
||||
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir) |
||||
Expect(err).NotTo(HaveOccurred()) |
||||
Expect(result).To(ContainSubstring("Current Rotation Stage: prepare")) |
||||
reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`) |
||||
Expect(err).ToNot(HaveOccurred()) |
||||
keys := reg.FindAllString(result, -1) |
||||
Expect(keys).To(HaveLen(2)) |
||||
Expect(keys[0]).To(ContainSubstring("aescbckey")) |
||||
Expect(keys[1]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year()))) |
||||
}) |
||||
It("restarts the server", func() { |
||||
var err error |
||||
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) |
||||
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) |
||||
Expect(err).ToNot(HaveOccurred()) |
||||
Eventually(func() (string, error) { |
||||
return testutil.K3sCmd("kubectl", "get", "pods", "-A") |
||||
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) |
||||
}) |
||||
It("rotates the keys", func() { |
||||
Eventually(func() (string, error) { |
||||
return testutil.K3sCmd("secrets-encrypt", "rotate", "-d", secretsEncryptionDataDir) |
||||
}, "10s", "2s").Should(ContainSubstring("rotate completed successfully")) |
||||
|
||||
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir) |
||||
Expect(err).NotTo(HaveOccurred()) |
||||
Expect(result).To(ContainSubstring("Current Rotation Stage: rotate")) |
||||
reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`) |
||||
Expect(err).ToNot(HaveOccurred()) |
||||
keys := reg.FindAllString(result, -1) |
||||
Expect(keys).To(HaveLen(2)) |
||||
Expect(keys[0]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year()))) |
||||
Expect(keys[1]).To(ContainSubstring("aescbckey")) |
||||
}) |
||||
It("restarts the server", func() { |
||||
var err error |
||||
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) |
||||
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) |
||||
Expect(err).ToNot(HaveOccurred()) |
||||
Eventually(func() (string, error) { |
||||
return testutil.K3sCmd("kubectl", "get", "pods", "-A") |
||||
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) |
||||
time.Sleep(10 * time.Second) |
||||
}) |
||||
It("reencrypts the keys", func() { |
||||
Expect(testutil.K3sCmd("secrets-encrypt", "reencrypt", "-d", secretsEncryptionDataDir)). |
||||
To(ContainSubstring("reencryption started")) |
||||
Eventually(func() (string, error) { |
||||
return testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir) |
||||
}, "30s", "2s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) |
||||
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir) |
||||
Expect(err).NotTo(HaveOccurred()) |
||||
reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`) |
||||
Expect(err).ToNot(HaveOccurred()) |
||||
keys := reg.FindAllString(result, -1) |
||||
Expect(keys).To(HaveLen(1)) |
||||
Expect(keys[0]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year()))) |
||||
}) |
||||
}) |
||||
When("A server disables encryption", func() { |
||||
It("it triggers the disable", func() { |
||||
Expect(testutil.K3sCmd("secrets-encrypt", "disable", "-d", secretsEncryptionDataDir)). |
||||
To(ContainSubstring("secrets-encryption disabled")) |
||||
|
||||
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir) |
||||
Expect(err).NotTo(HaveOccurred()) |
||||
Expect(result).To(ContainSubstring("Encryption Status: Disabled")) |
||||
}) |
||||
It("restarts the server", func() { |
||||
var err error |
||||
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) |
||||
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) |
||||
Expect(err).ToNot(HaveOccurred()) |
||||
Eventually(func() (string, error) { |
||||
return testutil.K3sCmd("kubectl", "get", "pods", "-A") |
||||
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) |
||||
time.Sleep(10 * time.Second) |
||||
}) |
||||
It("reencrypts the keys", func() { |
||||
result, err := testutil.K3sCmd("secrets-encrypt", "reencrypt", "-f", "--skip", "-d", secretsEncryptionDataDir) |
||||
Expect(err).NotTo(HaveOccurred()) |
||||
Expect(result).To(ContainSubstring("reencryption started")) |
||||
|
||||
result, err = testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir) |
||||
Expect(err).NotTo(HaveOccurred()) |
||||
Expect(result).To(ContainSubstring("Encryption Status: Disabled")) |
||||
}) |
||||
}) |
||||
}) |
||||
|
||||
var _ = AfterSuite(func() { |
||||
if !testutil.IsExistingServer() { |
||||
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) |
||||
Expect(testutil.K3sRemoveDataDir(secretsEncryptionDataDir)).To(Succeed()) |
||||
} |
||||
}) |
||||
|
||||
func Test_IntegrationSecretsEncryption(t *testing.T) { |
||||
RegisterFailHandler(Fail) |
||||
RunSpecsWithDefaultAndCustomReporters(t, "Secrets Encryption Suite", []Reporter{ |
||||
reporters.NewJUnitReporter("/tmp/results/junit-se.xml"), |
||||
}) |
||||
} |
||||
Loading…
Reference in new issue