Adding benchmarks to envelop encryption integration tests

cmd/kube-apiserver/app/testing/testserver.go

@@ -21,7 +21,6 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
-	"testing"
 	"time"
 
 	pflag "github.com/spf13/pflag"
@@ -46,13 +45,20 @@ type TestServer struct {
 	TmpDir       string                    // Temp Dir used, by the apiserver
 }
 
+// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie
+type Logger interface {
+	Errorf(format string, args ...interface{})
+	Fatalf(format string, args ...interface{})
+	Logf(format string, args ...interface{})
+}
+
 // StartTestServer starts a etcd server and kube-apiserver. A rest client config and a tear-down func,
 // and location of the tmpdir are returned.
 //
-// Note: we return a tear-down func instead of a stop channel because the later will leak temporariy
+// Note: we return a tear-down func instead of a stop channel because the later will leak temporary
-// 		 files that becaues Golang testing's call to os.Exit will not give a stop channel go routine
+// 		 files that because Golang testing's call to os.Exit will not give a stop channel go routine
-// 		 enough time to remove temporariy files.
+// 		 enough time to remove temporary files.
-func StartTestServer(t *testing.T, customFlags []string, storageConfig *storagebackend.Config) (result TestServer, err error) {
+func StartTestServer(t Logger, customFlags []string, storageConfig *storagebackend.Config) (result TestServer, err error) {
 
 	// TODO : Remove TrackStorageCleanup below when PR
 	// https://github.com/kubernetes/kubernetes/pull/50690
@@ -137,7 +143,7 @@ func StartTestServer(t *testing.T, customFlags []string, storageConfig *storageb
 }
 
 // StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed.
-func StartTestServerOrDie(t *testing.T, flags []string, storageConfig *storagebackend.Config) *TestServer {
+func StartTestServerOrDie(t Logger, flags []string, storageConfig *storagebackend.Config) *TestServer {
 
 	result, err := StartTestServer(t, flags, storageConfig)
 	if err == nil {

test/integration/master/secrets_enveloping_test.go

@@ -25,6 +25,8 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -47,6 +49,8 @@ const (
 	testNamespace = "secret-encryption-test"
 	testSecret    = "test-secret"
 
+	encryptionConfigFileName = "encryption.conf"
+
 	aesGCMPrefix = "k8s:enc:aesgcm:v1:key1:"
 	aesCBCPrefix = "k8s:enc:aescbc:v1:key1:"
 
@@ -79,11 +83,129 @@ resources:
         - name: key1
           secret: c2VjcmV0IGlzIHNlY3VyZQ==
 `
+
+	identityConfigYAML = `
+kind: EncryptionConfig
+apiVersion: v1
+resources:
+  - resources:
+    - secrets
+    providers:
+    - identity: {}
+`
 )
 
 type unSealSecret func(cipherText []byte, ctx value.Context, config encryptionconfig.ProviderConfig) ([]byte, error)
 
-// TestSecretsShouldBeEnveloped is an integration test between KubeAPI and ECTD that checks:
+type envelopTest struct {
+	logger            kubeapiservertesting.Logger
+	storageConfig     *storagebackend.Config
+	configDir         string
+	transformerConfig string
+	kubeAPIServer     kubeapiservertesting.TestServer
+	restClient        *kubernetes.Clientset
+	ns                *corev1.Namespace
+	secret            *corev1.Secret
+}
+
+func newEnvelopeTest(l kubeapiservertesting.Logger, transformerConfigYAML string) (*envelopTest, error) {
+	e := envelopTest{
+		logger:            l,
+		transformerConfig: transformerConfigYAML,
+		storageConfig:     framework.SharedEtcd(),
+	}
+
+	var err error
+	if transformerConfigYAML != "" {
+		if e.configDir, err = createKubeAPIServerEncryptionConfig(transformerConfigYAML); err != nil {
+			return nil, fmt.Errorf("error while creating KubeAPIServer encryption config: %v", err)
+		}
+	}
+
+	if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(l, e.getKubeAPIServerEncryptionOptions(), e.storageConfig); err != nil {
+		return nil, fmt.Errorf("failed to start KubeAPI server: %v", err)
+	}
+
+	if e.restClient, err = kubernetes.NewForConfig(e.kubeAPIServer.ClientConfig); err != nil {
+		return nil, fmt.Errorf("error while creating rest client: %v", err)
+	}
+
+	if e.ns, err = createTestNamespace(e.restClient, testNamespace); err != nil {
+		return nil, err
+	}
+
+	if e.secret, err = createTestSecret(e.restClient, testSecret, e.ns.Name); err != nil {
+		return nil, err
+	}
+
+	return &e, nil
+}
+
+func (e *envelopTest) cleanUp() {
+	os.RemoveAll(e.configDir)
+	e.restClient.CoreV1().Namespaces().Delete(e.ns.Name, metav1.NewDeleteOptions(0))
+	e.kubeAPIServer.TearDownFn()
+}
+
+func (e *envelopTest) run(unSealSecretFunc unSealSecret, expectedEnvelopePrefix string) {
+	response, err := readRawRecordFromETCD(&e.kubeAPIServer, e.getETCDPath())
+	if err != nil {
+		e.logger.Errorf("failed to read from etcd: %v", err)
+		return
+	}
+
+	if !bytes.HasPrefix(response.Kvs[0].Value, []byte(expectedEnvelopePrefix)) {
+		e.logger.Errorf("expected secret to be enveloped by %s, but got %s",
+			expectedEnvelopePrefix, response.Kvs[0].Value)
+		return
+	}
+
+	// etcd path of the key is used as the authenticated context - need to pass it to decrypt
+	ctx := value.DefaultContext([]byte(e.getETCDPath()))
+	// Envelope header precedes the payload
+	sealedData := response.Kvs[0].Value[len(expectedEnvelopePrefix):]
+	transformerConfig, err := parseTransformerConfig(e.transformerConfig)
+	if err != nil {
+		e.logger.Errorf("failed to parse transformer config: %v", err)
+	}
+	v, err := unSealSecretFunc(sealedData, ctx, *transformerConfig)
+	if err != nil {
+		e.logger.Errorf("failed to unseal secret: %v", err)
+		return
+	}
+	if !strings.Contains(string(v), secretVal) {
+		e.logger.Errorf("expected %q after decryption, but got %q", secretVal, string(v))
+	}
+
+	// Secrets should be un-enveloped on direct reads from Kube API Server.
+	s, err := e.restClient.CoreV1().Secrets(testNamespace).Get(testSecret, metav1.GetOptions{})
+	if secretVal != string(s.Data[secretKey]) {
+		e.logger.Errorf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
+	}
+}
+
+func (e *envelopTest) benchmark(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_, err := createTestSecret(e.restClient, e.secret.Name+strconv.Itoa(i), e.ns.Name)
+		if err != nil {
+			b.Fatalf("failed to create a secret: %v", err)
+		}
+	}
+}
+
+func (e *envelopTest) getETCDPath() string {
+	return fmt.Sprintf("/%s/secrets/%s/%s", e.storageConfig.Prefix, e.ns.Name, e.secret.Name)
+}
+
+func (e *envelopTest) getKubeAPIServerEncryptionOptions() []string {
+	if e.transformerConfig != "" {
+		return []string{"--experimental-encryption-provider-config", path.Join(e.configDir, encryptionConfigFileName)}
+	}
+
+	return nil
+}
+
+// TestSecretsShouldBeEnveloped is an integration test between KubeAPI and etcd that checks:
 // 1. Secrets are encrypted on write
 // 2. Secrets are decrypted on read
 // when EncryptionConfig is passed to KubeAPI server.
@@ -98,98 +220,62 @@ func TestSecretsShouldBeEnveloped(t *testing.T) {
 		// TODO: add secretbox
 	}
 	for _, tt := range testCases {
-		runEnvelopeTest(t, tt.unSealFunc, tt.transformerConfigContent, tt.transformerPrefix)
+		test, err := newEnvelopeTest(t, tt.transformerConfigContent)
+		if err != nil {
+			test.cleanUp()
+			t.Errorf("failed to setup test for envelop %s, error was %v", tt.transformerPrefix, err)
+			continue
+		}
+		test.run(tt.unSealFunc, tt.transformerPrefix)
+		test.cleanUp()
 	}
 }
 
-func runEnvelopeTest(t *testing.T, unSealSecretFunc unSealSecret, transformerConfigYAML, expectedEnvelopePrefix string) {
+// Baseline (no enveloping) - use to contrast with enveloping benchmarks.
-	transformerConfig := parseTransformerConfigOrDie(t, transformerConfigYAML)
+func BenchmarkBase(b *testing.B) {
-
+	runBenchmark(b, "")
-	storageConfig := framework.SharedEtcd()
-	kubeAPIServer, err := startKubeApiWithEncryption(t, storageConfig, transformerConfigYAML)
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	defer kubeAPIServer.TearDownFn()
-
-	client, err := kubernetes.NewForConfig(kubeAPIServer.ClientConfig)
-	if err != nil {
-		t.Fatalf("error while creating client: %v", err)
-	}
-
-	ns, err := createTestNamespace(client, testNamespace)
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	defer func() {
-		client.CoreV1().Namespaces().Delete(ns.Name, metav1.NewDeleteOptions(0))
-	}()
-
-	_, err = createTestSecret(client, testSecret, ns.Name)
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	etcdPath := getETCDPath(storageConfig.Prefix)
-	response, err := readRawRecordFromETCD(kubeAPIServer, etcdPath)
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	if !bytes.HasPrefix(response.Kvs[0].Value, []byte(expectedEnvelopePrefix)) {
-		t.Errorf("expected secret to be enveloped by %s, but got %s",
-			expectedEnvelopePrefix, response.Kvs[0].Value)
-		return
-	}
-
-	// etcd path of the key is used as authenticated context - need to pass it to decrypt
-	ctx := value.DefaultContext([]byte(etcdPath))
-	// Envelope header precedes the payload
-	sealedData := response.Kvs[0].Value[len(expectedEnvelopePrefix):]
-	v, err := unSealSecretFunc(sealedData, ctx, transformerConfig)
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	if !strings.Contains(string(v), secretVal) {
-		t.Errorf("expected %q after decryption, but got %q", secretVal, string(v))
-	}
-
-	// Secrets should be un-enveloped on direct reads from Kube API Server.
-	s, err := client.CoreV1().Secrets(testNamespace).Get(testSecret, metav1.GetOptions{})
-	if secretVal != string(s.Data[secretKey]) {
-		t.Errorf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey]))
-	}
 }
 
-func startKubeApiWithEncryption(t *testing.T, storageConfig *storagebackend.Config,
+// Identity transformer is a NOOP (crypto-wise) - use to contrast with AESGCM and AESCBC benchmark results.
-	transformerConfig string) (*kubeapiservertesting.TestServer, error) {
+func BenchmarkIdentityWrite(b *testing.B) {
+	runBenchmark(b, identityConfigYAML)
+}
+
+func BenchmarkAESGCMEnvelopeWrite(b *testing.B) {
+	runBenchmark(b, aesGCMConfigYAML)
+}
+
+func BenchmarkAESCBCEnvelopeWrite(b *testing.B) {
+	runBenchmark(b, aesCBCConfigYAML)
+}
+
+func runBenchmark(b *testing.B, transformerConfig string) {
+	b.StopTimer()
+	test, err := newEnvelopeTest(b, transformerConfig)
+	defer test.cleanUp()
+	if err != nil {
+		b.Fatalf("failed to setup benchmark for config %s, error was %v", transformerConfig, err)
+	}
+
+	b.StartTimer()
+	test.benchmark(b)
+	b.StopTimer()
+}
+
+func createKubeAPIServerEncryptionConfig(transformerConfig string) (string, error) {
 	tempDir, err := ioutil.TempDir("", "secrets-encryption-test")
 	if err != nil {
-		return nil, fmt.Errorf("failed to create temp directory: %v", err)
+		return "", fmt.Errorf("failed to create temp directory: %v", err)
-	}
-	defer os.RemoveAll(tempDir)
-
-	encryptionConfig, err := ioutil.TempFile(tempDir, "encryption-config")
-	if err != nil {
-		return nil, fmt.Errorf("error while creating temp file for encryption config %v", err)
 	}
 
-	if _, err := encryptionConfig.Write([]byte(transformerConfig)); err != nil {
+	encryptionConfig := path.Join(tempDir, encryptionConfigFileName)
-		return nil, fmt.Errorf("error while writing encryption config: %v", err)
+
+	if err := ioutil.WriteFile(encryptionConfig, []byte(transformerConfig), 0644); err != nil {
+		os.RemoveAll(tempDir)
+		return "", fmt.Errorf("error while writing encryption config: %v", err)
 	}
 
-	kubeAPIOptions := []string{"--experimental-encryption-provider-config", encryptionConfig.Name()}
+	return tempDir, nil
-	server, err := kubeapiservertesting.StartTestServer(t, kubeAPIOptions, storageConfig)
-	if err != nil {
-		return nil, fmt.Errorf("failed to start KubeAPI Server %v", err)
-	}
-
-	return &server, nil
 }
 
 func createTestNamespace(client *kubernetes.Clientset, name string) (*corev1.Namespace, error) {
@@ -224,7 +310,6 @@ func createTestSecret(client *kubernetes.Clientset, name, namespace string) (*co
 }
 
 func readRawRecordFromETCD(kubeAPIServer *kubeapiservertesting.TestServer, path string) (*clientv3.GetResponse, error) {
-	// Reading secret directly from etcd - expect data to be enveloped and the payload encrypted.
 	etcdClient, err := integration.GetEtcdKVClient(kubeAPIServer.ServerOpts.Etcd.StorageConfig)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create etcd client: %v", err)
@@ -237,18 +322,14 @@ func readRawRecordFromETCD(kubeAPIServer *kubeapiservertesting.TestServer, path
 	return response, nil
 }
 
-func getETCDPath(prefix string) string {
+func parseTransformerConfig(configContent string) (*encryptionconfig.ProviderConfig, error) {
-	return fmt.Sprintf("/%s/secrets/%s/%s", prefix, testNamespace, testSecret)
-}
-
-func parseTransformerConfigOrDie(t *testing.T, configContent string) encryptionconfig.ProviderConfig {
 	var config encryptionconfig.EncryptionConfig
 	err := yaml.Unmarshal([]byte(configContent), &config)
 	if err != nil {
-		t.Errorf("failed to extract transformer key: %v", err)
+		return nil, fmt.Errorf("failed to extract transformer key: %v", err)
 	}
 
-	return config.Resources[0].Providers[0]
+	return &config.Resources[0].Providers[0], nil
 }
 
 func unSealWithGCMTransformer(cipherText []byte, ctx value.Context,