diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go index 469111185e..2d56fbd5d5 100644 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ b/cmd/kube-apiserver/app/testing/testserver.go @@ -21,7 +21,6 @@ import ( "io/ioutil" "net" "os" - "testing" "time" pflag "github.com/spf13/pflag" @@ -46,13 +45,20 @@ type TestServer struct { TmpDir string // Temp Dir used, by the apiserver } +// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie +type Logger interface { + Errorf(format string, args ...interface{}) + Fatalf(format string, args ...interface{}) + Logf(format string, args ...interface{}) +} + // StartTestServer starts a etcd server and kube-apiserver. A rest client config and a tear-down func, // and location of the tmpdir are returned. // -// Note: we return a tear-down func instead of a stop channel because the later will leak temporariy -// files that becaues Golang testing's call to os.Exit will not give a stop channel go routine -// enough time to remove temporariy files. -func StartTestServer(t *testing.T, customFlags []string, storageConfig *storagebackend.Config) (result TestServer, err error) { +// Note: we return a tear-down func instead of a stop channel because the later will leak temporary +// files that because Golang testing's call to os.Exit will not give a stop channel go routine +// enough time to remove temporary files. +func StartTestServer(t Logger, customFlags []string, storageConfig *storagebackend.Config) (result TestServer, err error) { // TODO : Remove TrackStorageCleanup below when PR // https://github.com/kubernetes/kubernetes/pull/50690 @@ -137,7 +143,7 @@ func StartTestServer(t *testing.T, customFlags []string, storageConfig *storageb } // StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed. -func StartTestServerOrDie(t *testing.T, flags []string, storageConfig *storagebackend.Config) *TestServer { +func StartTestServerOrDie(t Logger, flags []string, storageConfig *storagebackend.Config) *TestServer { result, err := StartTestServer(t, flags, storageConfig) if err == nil { diff --git a/test/integration/master/secrets_enveloping_test.go b/test/integration/master/secrets_enveloping_test.go index 4fb6c9daae..94b7b6fb3c 100644 --- a/test/integration/master/secrets_enveloping_test.go +++ b/test/integration/master/secrets_enveloping_test.go @@ -25,6 +25,8 @@ import ( "fmt" "io/ioutil" "os" + "path" + "strconv" "strings" "testing" @@ -47,6 +49,8 @@ const ( testNamespace = "secret-encryption-test" testSecret = "test-secret" + encryptionConfigFileName = "encryption.conf" + aesGCMPrefix = "k8s:enc:aesgcm:v1:key1:" aesCBCPrefix = "k8s:enc:aescbc:v1:key1:" @@ -79,11 +83,129 @@ resources: - name: key1 secret: c2VjcmV0IGlzIHNlY3VyZQ== ` + + identityConfigYAML = ` +kind: EncryptionConfig +apiVersion: v1 +resources: + - resources: + - secrets + providers: + - identity: {} +` ) type unSealSecret func(cipherText []byte, ctx value.Context, config encryptionconfig.ProviderConfig) ([]byte, error) -// TestSecretsShouldBeEnveloped is an integration test between KubeAPI and ECTD that checks: +type envelopTest struct { + logger kubeapiservertesting.Logger + storageConfig *storagebackend.Config + configDir string + transformerConfig string + kubeAPIServer kubeapiservertesting.TestServer + restClient *kubernetes.Clientset + ns *corev1.Namespace + secret *corev1.Secret +} + +func newEnvelopeTest(l kubeapiservertesting.Logger, transformerConfigYAML string) (*envelopTest, error) { + e := envelopTest{ + logger: l, + transformerConfig: transformerConfigYAML, + storageConfig: framework.SharedEtcd(), + } + + var err error + if transformerConfigYAML != "" { + if e.configDir, err = createKubeAPIServerEncryptionConfig(transformerConfigYAML); err != nil { + return nil, fmt.Errorf("error while creating KubeAPIServer encryption config: %v", err) + } + } + + if e.kubeAPIServer, err = kubeapiservertesting.StartTestServer(l, e.getKubeAPIServerEncryptionOptions(), e.storageConfig); err != nil { + return nil, fmt.Errorf("failed to start KubeAPI server: %v", err) + } + + if e.restClient, err = kubernetes.NewForConfig(e.kubeAPIServer.ClientConfig); err != nil { + return nil, fmt.Errorf("error while creating rest client: %v", err) + } + + if e.ns, err = createTestNamespace(e.restClient, testNamespace); err != nil { + return nil, err + } + + if e.secret, err = createTestSecret(e.restClient, testSecret, e.ns.Name); err != nil { + return nil, err + } + + return &e, nil +} + +func (e *envelopTest) cleanUp() { + os.RemoveAll(e.configDir) + e.restClient.CoreV1().Namespaces().Delete(e.ns.Name, metav1.NewDeleteOptions(0)) + e.kubeAPIServer.TearDownFn() +} + +func (e *envelopTest) run(unSealSecretFunc unSealSecret, expectedEnvelopePrefix string) { + response, err := readRawRecordFromETCD(&e.kubeAPIServer, e.getETCDPath()) + if err != nil { + e.logger.Errorf("failed to read from etcd: %v", err) + return + } + + if !bytes.HasPrefix(response.Kvs[0].Value, []byte(expectedEnvelopePrefix)) { + e.logger.Errorf("expected secret to be enveloped by %s, but got %s", + expectedEnvelopePrefix, response.Kvs[0].Value) + return + } + + // etcd path of the key is used as the authenticated context - need to pass it to decrypt + ctx := value.DefaultContext([]byte(e.getETCDPath())) + // Envelope header precedes the payload + sealedData := response.Kvs[0].Value[len(expectedEnvelopePrefix):] + transformerConfig, err := parseTransformerConfig(e.transformerConfig) + if err != nil { + e.logger.Errorf("failed to parse transformer config: %v", err) + } + v, err := unSealSecretFunc(sealedData, ctx, *transformerConfig) + if err != nil { + e.logger.Errorf("failed to unseal secret: %v", err) + return + } + if !strings.Contains(string(v), secretVal) { + e.logger.Errorf("expected %q after decryption, but got %q", secretVal, string(v)) + } + + // Secrets should be un-enveloped on direct reads from Kube API Server. + s, err := e.restClient.CoreV1().Secrets(testNamespace).Get(testSecret, metav1.GetOptions{}) + if secretVal != string(s.Data[secretKey]) { + e.logger.Errorf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey])) + } +} + +func (e *envelopTest) benchmark(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := createTestSecret(e.restClient, e.secret.Name+strconv.Itoa(i), e.ns.Name) + if err != nil { + b.Fatalf("failed to create a secret: %v", err) + } + } +} + +func (e *envelopTest) getETCDPath() string { + return fmt.Sprintf("/%s/secrets/%s/%s", e.storageConfig.Prefix, e.ns.Name, e.secret.Name) +} + +func (e *envelopTest) getKubeAPIServerEncryptionOptions() []string { + if e.transformerConfig != "" { + return []string{"--experimental-encryption-provider-config", path.Join(e.configDir, encryptionConfigFileName)} + } + + return nil +} + +// TestSecretsShouldBeEnveloped is an integration test between KubeAPI and etcd that checks: // 1. Secrets are encrypted on write // 2. Secrets are decrypted on read // when EncryptionConfig is passed to KubeAPI server. @@ -98,98 +220,62 @@ func TestSecretsShouldBeEnveloped(t *testing.T) { // TODO: add secretbox } for _, tt := range testCases { - runEnvelopeTest(t, tt.unSealFunc, tt.transformerConfigContent, tt.transformerPrefix) + test, err := newEnvelopeTest(t, tt.transformerConfigContent) + if err != nil { + test.cleanUp() + t.Errorf("failed to setup test for envelop %s, error was %v", tt.transformerPrefix, err) + continue + } + test.run(tt.unSealFunc, tt.transformerPrefix) + test.cleanUp() } } -func runEnvelopeTest(t *testing.T, unSealSecretFunc unSealSecret, transformerConfigYAML, expectedEnvelopePrefix string) { - transformerConfig := parseTransformerConfigOrDie(t, transformerConfigYAML) - - storageConfig := framework.SharedEtcd() - kubeAPIServer, err := startKubeApiWithEncryption(t, storageConfig, transformerConfigYAML) - if err != nil { - t.Error(err) - return - } - defer kubeAPIServer.TearDownFn() - - client, err := kubernetes.NewForConfig(kubeAPIServer.ClientConfig) - if err != nil { - t.Fatalf("error while creating client: %v", err) - } - - ns, err := createTestNamespace(client, testNamespace) - if err != nil { - t.Error(err) - return - } - defer func() { - client.CoreV1().Namespaces().Delete(ns.Name, metav1.NewDeleteOptions(0)) - }() - - _, err = createTestSecret(client, testSecret, ns.Name) - if err != nil { - t.Error(err) - return - } - - etcdPath := getETCDPath(storageConfig.Prefix) - response, err := readRawRecordFromETCD(kubeAPIServer, etcdPath) - if err != nil { - t.Error(err) - return - } - - if !bytes.HasPrefix(response.Kvs[0].Value, []byte(expectedEnvelopePrefix)) { - t.Errorf("expected secret to be enveloped by %s, but got %s", - expectedEnvelopePrefix, response.Kvs[0].Value) - return - } - - // etcd path of the key is used as authenticated context - need to pass it to decrypt - ctx := value.DefaultContext([]byte(etcdPath)) - // Envelope header precedes the payload - sealedData := response.Kvs[0].Value[len(expectedEnvelopePrefix):] - v, err := unSealSecretFunc(sealedData, ctx, transformerConfig) - if err != nil { - t.Error(err) - return - } - if !strings.Contains(string(v), secretVal) { - t.Errorf("expected %q after decryption, but got %q", secretVal, string(v)) - } - - // Secrets should be un-enveloped on direct reads from Kube API Server. - s, err := client.CoreV1().Secrets(testNamespace).Get(testSecret, metav1.GetOptions{}) - if secretVal != string(s.Data[secretKey]) { - t.Errorf("expected %s from KubeAPI, but got %s", secretVal, string(s.Data[secretKey])) - } +// Baseline (no enveloping) - use to contrast with enveloping benchmarks. +func BenchmarkBase(b *testing.B) { + runBenchmark(b, "") } -func startKubeApiWithEncryption(t *testing.T, storageConfig *storagebackend.Config, - transformerConfig string) (*kubeapiservertesting.TestServer, error) { +// Identity transformer is a NOOP (crypto-wise) - use to contrast with AESGCM and AESCBC benchmark results. +func BenchmarkIdentityWrite(b *testing.B) { + runBenchmark(b, identityConfigYAML) +} + +func BenchmarkAESGCMEnvelopeWrite(b *testing.B) { + runBenchmark(b, aesGCMConfigYAML) +} + +func BenchmarkAESCBCEnvelopeWrite(b *testing.B) { + runBenchmark(b, aesCBCConfigYAML) +} + +func runBenchmark(b *testing.B, transformerConfig string) { + b.StopTimer() + test, err := newEnvelopeTest(b, transformerConfig) + defer test.cleanUp() + if err != nil { + b.Fatalf("failed to setup benchmark for config %s, error was %v", transformerConfig, err) + } + + b.StartTimer() + test.benchmark(b) + b.StopTimer() +} + +func createKubeAPIServerEncryptionConfig(transformerConfig string) (string, error) { tempDir, err := ioutil.TempDir("", "secrets-encryption-test") if err != nil { - return nil, fmt.Errorf("failed to create temp directory: %v", err) - } - defer os.RemoveAll(tempDir) - - encryptionConfig, err := ioutil.TempFile(tempDir, "encryption-config") - if err != nil { - return nil, fmt.Errorf("error while creating temp file for encryption config %v", err) + return "", fmt.Errorf("failed to create temp directory: %v", err) } - if _, err := encryptionConfig.Write([]byte(transformerConfig)); err != nil { - return nil, fmt.Errorf("error while writing encryption config: %v", err) + encryptionConfig := path.Join(tempDir, encryptionConfigFileName) + + if err := ioutil.WriteFile(encryptionConfig, []byte(transformerConfig), 0644); err != nil { + os.RemoveAll(tempDir) + return "", fmt.Errorf("error while writing encryption config: %v", err) } - kubeAPIOptions := []string{"--experimental-encryption-provider-config", encryptionConfig.Name()} - server, err := kubeapiservertesting.StartTestServer(t, kubeAPIOptions, storageConfig) - if err != nil { - return nil, fmt.Errorf("failed to start KubeAPI Server %v", err) - } - - return &server, nil + return tempDir, nil } func createTestNamespace(client *kubernetes.Clientset, name string) (*corev1.Namespace, error) { @@ -224,7 +310,6 @@ func createTestSecret(client *kubernetes.Clientset, name, namespace string) (*co } func readRawRecordFromETCD(kubeAPIServer *kubeapiservertesting.TestServer, path string) (*clientv3.GetResponse, error) { - // Reading secret directly from etcd - expect data to be enveloped and the payload encrypted. etcdClient, err := integration.GetEtcdKVClient(kubeAPIServer.ServerOpts.Etcd.StorageConfig) if err != nil { return nil, fmt.Errorf("failed to create etcd client: %v", err) @@ -237,18 +322,14 @@ func readRawRecordFromETCD(kubeAPIServer *kubeapiservertesting.TestServer, path return response, nil } -func getETCDPath(prefix string) string { - return fmt.Sprintf("/%s/secrets/%s/%s", prefix, testNamespace, testSecret) -} - -func parseTransformerConfigOrDie(t *testing.T, configContent string) encryptionconfig.ProviderConfig { +func parseTransformerConfig(configContent string) (*encryptionconfig.ProviderConfig, error) { var config encryptionconfig.EncryptionConfig err := yaml.Unmarshal([]byte(configContent), &config) if err != nil { - t.Errorf("failed to extract transformer key: %v", err) + return nil, fmt.Errorf("failed to extract transformer key: %v", err) } - return config.Resources[0].Providers[0] + return &config.Resources[0].Providers[0], nil } func unSealWithGCMTransformer(cipherText []byte, ctx value.Context,