k3s/tests/integration/secretsencryption/secretsencryption_int_test.go

package integration

import (
	"fmt"
	"os"
	"regexp"
	"testing"
	"time"

	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/reporters"
	. "github.com/onsi/gomega"
	testutil "github.com/rancher/k3s/tests/util"
)

var secretsEncryptionServer *testutil.K3sServer
var secretsEncryptionDataDir = "/tmp/k3sse"

var secretsEncryptionServerArgs = []string{"--secrets-encryption", "-d", secretsEncryptionDataDir}
var _ = BeforeSuite(func() {
	if !testutil.IsExistingServer() {
		var err error
		Expect(os.MkdirAll(secretsEncryptionDataDir, 0777)).To(Succeed())
		secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
		Expect(err).ToNot(HaveOccurred())
	}
})

var _ = Describe("secrets encryption rotation", func() {
	BeforeEach(func() {
		if testutil.IsExistingServer() {
			Skip("Test does not support running on existing k3s servers")
		}
	})
	When("A server starts with secrets encryption", func() {
		It("starts up with no problems", func() {
			Eventually(func() (string, error) {
				return testutil.K3sCmd("kubectl get pods -A")
			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
		})
		It("it creates a encryption key", func() {
			result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
			Expect(err).NotTo(HaveOccurred())
			Expect(result).To(ContainSubstring("Encryption Status: Enabled"))
			Expect(result).To(ContainSubstring("Current Rotation Stage: start"))
		})
	})
	When("A server rotates encryption keys", func() {
		It("it prepares to rotate", func() {
			Expect(testutil.K3sCmd("secrets-encrypt prepare -d", secretsEncryptionDataDir)).
				To(ContainSubstring("prepare completed successfully"))

			result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
			Expect(err).NotTo(HaveOccurred())
			Expect(result).To(ContainSubstring("Current Rotation Stage: prepare"))
			reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
			Expect(err).ToNot(HaveOccurred())
			keys := reg.FindAllString(result, -1)
			Expect(keys).To(HaveLen(2))
			Expect(keys[0]).To(ContainSubstring("aescbckey"))
			Expect(keys[1]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year())))
		})
		It("restarts the server", func() {
			var err error
			Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed())
			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
			Expect(err).ToNot(HaveOccurred())
			Eventually(func() (string, error) {
				return testutil.K3sCmd("kubectl get pods -A")
			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
		})
		It("rotates the keys", func() {
			Eventually(func() (string, error) {
				return testutil.K3sCmd("secrets-encrypt rotate -d", secretsEncryptionDataDir)
			}, "10s", "2s").Should(ContainSubstring("rotate completed successfully"))

			result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
			Expect(err).NotTo(HaveOccurred())
			Expect(result).To(ContainSubstring("Current Rotation Stage: rotate"))
			reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
			Expect(err).ToNot(HaveOccurred())
			keys := reg.FindAllString(result, -1)
			Expect(keys).To(HaveLen(2))
			Expect(keys[0]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year())))
			Expect(keys[1]).To(ContainSubstring("aescbckey"))
		})
		It("restarts the server", func() {
			var err error
			Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed())
			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
			Expect(err).ToNot(HaveOccurred())
			Eventually(func() (string, error) {
				return testutil.K3sCmd("kubectl get pods -A")
			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
			time.Sleep(10 * time.Second)
		})
		It("reencrypts the keys", func() {
			Expect(testutil.K3sCmd("secrets-encrypt reencrypt -d", secretsEncryptionDataDir)).
				To(ContainSubstring("reencryption started"))
			Eventually(func() (string, error) {
				return testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
			}, "30s", "2s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
			result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
			Expect(err).NotTo(HaveOccurred())
			reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
			Expect(err).ToNot(HaveOccurred())
			keys := reg.FindAllString(result, -1)
			Expect(keys).To(HaveLen(1))
			Expect(keys[0]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year())))
		})
	})
	When("A server disables encryption", func() {
		It("it triggers the disable", func() {
			Expect(testutil.K3sCmd("secrets-encrypt disable -d", secretsEncryptionDataDir)).
				To(ContainSubstring("secrets-encryption disabled"))

			result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
			Expect(err).NotTo(HaveOccurred())
			Expect(result).To(ContainSubstring("Encryption Status: Disabled"))
		})
		It("restarts the server", func() {
			var err error
			Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed())
			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
			Expect(err).ToNot(HaveOccurred())
			Eventually(func() (string, error) {
				return testutil.K3sCmd("kubectl get pods -A")
			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
			time.Sleep(10 * time.Second)
		})
		It("reencrypts the keys", func() {
			result, err := testutil.K3sCmd("secrets-encrypt reencrypt -f --skip -d", secretsEncryptionDataDir)
			Expect(err).NotTo(HaveOccurred())
			Expect(result).To(ContainSubstring("reencryption started"))

			result, err = testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
			Expect(err).NotTo(HaveOccurred())
			Expect(result).To(ContainSubstring("Encryption Status: Disabled"))
		})
	})
})

var _ = AfterSuite(func() {
	if !testutil.IsExistingServer() {
		Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed())
		Expect(os.RemoveAll(secretsEncryptionDataDir)).To(Succeed())
	}
})

func Test_IntegrationSecretsEncryption(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecsWithDefaultAndCustomReporters(t, "Secrets Encryption Suite", []Reporter{
		reporters.NewJUnitReporter("/tmp/results/junit-se.xml"),
	})
}