From 76906dd73859ea5f9d4b41d12a4ab7a5cd0f2b8f Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Tue, 2 Apr 2019 23:17:59 +0300 Subject: [PATCH] test: partially decouple from cmd/kubeadm The test package imports cmd/kubeadm, which is far from ideal. There are a couple of reasons for the import: 1) Marshaling of Ingress from api/extensions/v1beta1. To fix that include a local function in e2e/manifest/manifest.go that does that same as the kubeadm MarshalToYaml. 2) Using PKI helper function in apimachinery and auth tests. To fix that include a new file under test/utils/pki_helpers.go that only contains the required helpers instead of including the whole kubeadm pkiutil package. There is another related problem: e2e_node/e2e_node_suite_test.go includes: k8s.io/kubernetes/cmd/kubeadm/app/util/system But this has to be done in a follow up. --- test/e2e/apimachinery/BUILD | 1 - test/e2e/apimachinery/certs.go | 16 ++-- test/e2e/auth/BUILD | 1 - test/e2e/auth/certificates.go | 4 +- test/e2e/manifest/BUILD | 2 +- test/e2e/manifest/manifest.go | 18 ++++- test/integration/examples/BUILD | 2 +- test/integration/examples/apiserver_test.go | 16 ++-- test/integration/framework/BUILD | 1 - test/integration/framework/test_server.go | 10 +-- test/utils/BUILD | 3 + test/utils/pki_helpers.go | 87 +++++++++++++++++++++ 12 files changed, 130 insertions(+), 31 deletions(-) create mode 100644 test/utils/pki_helpers.go diff --git a/test/e2e/apimachinery/BUILD b/test/e2e/apimachinery/BUILD index ee718a6d13..49530dada0 100644 --- a/test/e2e/apimachinery/BUILD +++ b/test/e2e/apimachinery/BUILD @@ -28,7 +28,6 @@ go_library( ], importpath = "k8s.io/kubernetes/test/e2e/apimachinery", deps = [ - "//cmd/kubeadm/app/util/pkiutil:go_default_library", "//pkg/api/v1/pod:go_default_library", "//pkg/apis/rbac/v1beta1:go_default_library", "//pkg/printers:go_default_library", diff --git a/test/e2e/apimachinery/certs.go b/test/e2e/apimachinery/certs.go index 3c4f0f8c00..d9926f85e2 100644 --- a/test/e2e/apimachinery/certs.go +++ b/test/e2e/apimachinery/certs.go @@ -23,8 +23,8 @@ import ( "k8s.io/client-go/util/cert" "k8s.io/client-go/util/keyutil" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/utils" ) type certContext struct { @@ -41,7 +41,7 @@ func setupServerCert(namespaceName, serviceName string) *certContext { framework.Failf("Failed to create a temp dir for cert generation %v", err) } defer os.RemoveAll(certDir) - signingKey, err := pkiutil.NewPrivateKey() + signingKey, err := utils.NewPrivateKey() if err != nil { framework.Failf("Failed to create CA private key %v", err) } @@ -53,14 +53,14 @@ func setupServerCert(namespaceName, serviceName string) *certContext { if err != nil { framework.Failf("Failed to create a temp file for ca cert generation %v", err) } - if err := ioutil.WriteFile(caCertFile.Name(), pkiutil.EncodeCertPEM(signingCert), 0644); err != nil { + if err := ioutil.WriteFile(caCertFile.Name(), utils.EncodeCertPEM(signingCert), 0644); err != nil { framework.Failf("Failed to write CA cert %v", err) } - key, err := pkiutil.NewPrivateKey() + key, err := utils.NewPrivateKey() if err != nil { framework.Failf("Failed to create private key for %v", err) } - signedCert, err := pkiutil.NewSignedCert( + signedCert, err := utils.NewSignedCert( &cert.Config{ CommonName: serviceName + "." + namespaceName + ".svc", Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, @@ -78,7 +78,7 @@ func setupServerCert(namespaceName, serviceName string) *certContext { if err != nil { framework.Failf("Failed to create a temp file for key generation %v", err) } - if err = ioutil.WriteFile(certFile.Name(), pkiutil.EncodeCertPEM(signedCert), 0600); err != nil { + if err = ioutil.WriteFile(certFile.Name(), utils.EncodeCertPEM(signedCert), 0600); err != nil { framework.Failf("Failed to write cert file %v", err) } privateKeyPEM, err := keyutil.MarshalPrivateKeyToPEM(key) @@ -89,8 +89,8 @@ func setupServerCert(namespaceName, serviceName string) *certContext { framework.Failf("Failed to write key file %v", err) } return &certContext{ - cert: pkiutil.EncodeCertPEM(signedCert), + cert: utils.EncodeCertPEM(signedCert), key: privateKeyPEM, - signingCert: pkiutil.EncodeCertPEM(signingCert), + signingCert: utils.EncodeCertPEM(signingCert), } } diff --git a/test/e2e/auth/BUILD b/test/e2e/auth/BUILD index e55b179704..31689891c4 100644 --- a/test/e2e/auth/BUILD +++ b/test/e2e/auth/BUILD @@ -20,7 +20,6 @@ go_library( ], importpath = "k8s.io/kubernetes/test/e2e/auth", deps = [ - "//cmd/kubeadm/app/util/pkiutil:go_default_library", "//pkg/master/ports:go_default_library", "//pkg/security/apparmor:go_default_library", "//pkg/security/podsecuritypolicy/seccomp:go_default_library", diff --git a/test/e2e/auth/certificates.go b/test/e2e/auth/certificates.go index 1dd6d74832..3a93fa1f62 100644 --- a/test/e2e/auth/certificates.go +++ b/test/e2e/auth/certificates.go @@ -27,8 +27,8 @@ import ( "k8s.io/apimachinery/pkg/util/wait" v1beta1client "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" "k8s.io/client-go/util/cert" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/utils" . "github.com/onsi/ginkgo" ) @@ -39,7 +39,7 @@ var _ = SIGDescribe("Certificates API", func() { It("should support building a client with a CSR", func() { const commonName = "tester-csr" - pk, err := pkiutil.NewPrivateKey() + pk, err := utils.NewPrivateKey() framework.ExpectNoError(err) pkder := x509.MarshalPKCS1PrivateKey(pk) diff --git a/test/e2e/manifest/BUILD b/test/e2e/manifest/BUILD index 3bf8e3343e..93433648e4 100644 --- a/test/e2e/manifest/BUILD +++ b/test/e2e/manifest/BUILD @@ -11,13 +11,13 @@ go_library( srcs = ["manifest.go"], importpath = "k8s.io/kubernetes/test/e2e/manifest", deps = [ - "//cmd/kubeadm/app/util:go_default_library", "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/extensions/v1beta1:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", "//test/e2e/framework/testfiles:go_default_library", diff --git a/test/e2e/manifest/manifest.go b/test/e2e/manifest/manifest.go index 26c9a4d455..0b0da3930f 100644 --- a/test/e2e/manifest/manifest.go +++ b/test/e2e/manifest/manifest.go @@ -26,9 +26,9 @@ import ( rbac "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" utilyaml "k8s.io/apimachinery/pkg/util/yaml" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/kubernetes/cmd/kubeadm/app/util" + scheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/kubernetes/test/e2e/framework/testfiles" ) @@ -107,7 +107,7 @@ func IngressFromManifest(fileName string) (*extensions.Ingress, error) { // IngressToManifest generates a yaml file in the given path with the given ingress. // Assumes that a directory exists at the given path. func IngressToManifest(ing *extensions.Ingress, path string) error { - serialized, err := util.MarshalToYaml(ing, extensions.SchemeGroupVersion) + serialized, err := marshalToYaml(ing, extensions.SchemeGroupVersion) if err != nil { return fmt.Errorf("failed to marshal ingress %v to YAML: %v", ing, err) } @@ -178,3 +178,15 @@ func RoleFromManifest(fileName, ns string) (*rbac.Role, error) { role.Namespace = ns return &role, nil } + +// marshalToYaml marshals an object into YAML for a given GroupVersion. +// The object must be known in SupportedMediaTypes() for the Codecs under "client-go/kubernetes/scheme". +func marshalToYaml(obj runtime.Object, gv schema.GroupVersion) ([]byte, error) { + mediaType := "application/yaml" + info, ok := runtime.SerializerInfoForMediaType(scheme.Codecs.SupportedMediaTypes(), mediaType) + if !ok { + return []byte{}, fmt.Errorf("unsupported media type %q", mediaType) + } + encoder := scheme.Codecs.EncoderForVersion(info.Serializer, gv) + return runtime.Encode(encoder, obj) +} diff --git a/test/integration/examples/BUILD b/test/integration/examples/BUILD index eb0f8adb2e..7c0f59d8ad 100644 --- a/test/integration/examples/BUILD +++ b/test/integration/examples/BUILD @@ -17,7 +17,6 @@ go_test( deps = [ "//cmd/kube-apiserver/app:go_default_library", "//cmd/kube-apiserver/app/options:go_default_library", - "//cmd/kubeadm/app/util/pkiutil:go_default_library", "//pkg/master:go_default_library", "//pkg/master/reconcilers:go_default_library", "//staging/src/k8s.io/api/admissionregistration/v1beta1:go_default_library", @@ -44,6 +43,7 @@ go_test( "//staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1beta1:go_default_library", "//staging/src/k8s.io/sample-apiserver/pkg/cmd/server:go_default_library", "//test/integration/framework:go_default_library", + "//test/utils:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", ], ) diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index 3359c4c22f..f3ce8862ed 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -49,8 +49,8 @@ import ( kubeaggregatorserver "k8s.io/kube-aggregator/pkg/cmd/server" "k8s.io/kubernetes/cmd/kube-apiserver/app" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" "k8s.io/kubernetes/test/integration/framework" + testutil "k8s.io/kubernetes/test/utils" wardlev1alpha1 "k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1" wardlev1beta1 "k8s.io/sample-apiserver/pkg/apis/wardle/v1beta1" sampleserver "k8s.io/sample-apiserver/pkg/cmd/server" @@ -63,7 +63,7 @@ func TestAggregatedAPIServer(t *testing.T) { certDir, _ := ioutil.TempDir("", "test-integration-apiserver") defer os.RemoveAll(certDir) _, defaultServiceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24") - proxySigningKey, err := pkiutil.NewPrivateKey() + proxySigningKey, err := testutil.NewPrivateKey() if err != nil { t.Fatal(err) } @@ -72,10 +72,10 @@ func TestAggregatedAPIServer(t *testing.T) { t.Fatal(err) } proxyCACertFile, _ := ioutil.TempFile(certDir, "proxy-ca.crt") - if err := ioutil.WriteFile(proxyCACertFile.Name(), pkiutil.EncodeCertPEM(proxySigningCert), 0644); err != nil { + if err := ioutil.WriteFile(proxyCACertFile.Name(), testutil.EncodeCertPEM(proxySigningCert), 0644); err != nil { t.Fatal(err) } - clientSigningKey, err := pkiutil.NewPrivateKey() + clientSigningKey, err := testutil.NewPrivateKey() if err != nil { t.Fatal(err) } @@ -84,7 +84,7 @@ func TestAggregatedAPIServer(t *testing.T) { t.Fatal(err) } clientCACertFile, _ := ioutil.TempFile(certDir, "client-ca.crt") - if err := ioutil.WriteFile(clientCACertFile.Name(), pkiutil.EncodeCertPEM(clientSigningCert), 0644); err != nil { + if err := ioutil.WriteFile(clientCACertFile.Name(), testutil.EncodeCertPEM(clientSigningCert), 0644); err != nil { t.Fatal(err) } @@ -236,11 +236,11 @@ func TestAggregatedAPIServer(t *testing.T) { // start the aggregator aggregatorCertDir, _ := ioutil.TempDir("", "test-integration-aggregator") defer os.RemoveAll(aggregatorCertDir) - proxyClientKey, err := pkiutil.NewPrivateKey() + proxyClientKey, err := testutil.NewPrivateKey() if err != nil { t.Fatal(err) } - proxyClientCert, err := pkiutil.NewSignedCert( + proxyClientCert, err := testutil.NewSignedCert( &cert.Config{ CommonName: "kube-aggregator", Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, @@ -249,7 +249,7 @@ func TestAggregatedAPIServer(t *testing.T) { ) proxyClientCertFile, _ := ioutil.TempFile(aggregatorCertDir, "proxy-client.crt") proxyClientKeyFile, _ := ioutil.TempFile(aggregatorCertDir, "proxy-client.key") - if err := ioutil.WriteFile(proxyClientCertFile.Name(), pkiutil.EncodeCertPEM(proxyClientCert), 0600); err != nil { + if err := ioutil.WriteFile(proxyClientCertFile.Name(), testutil.EncodeCertPEM(proxyClientCert), 0600); err != nil { t.Fatal(err) } proxyClientKeyPEM, err := keyutil.MarshalPrivateKeyToPEM(proxyClientKey) diff --git a/test/integration/framework/BUILD b/test/integration/framework/BUILD index 9d16fe337a..5cc06a3098 100644 --- a/test/integration/framework/BUILD +++ b/test/integration/framework/BUILD @@ -37,7 +37,6 @@ go_library( deps = [ "//cmd/kube-apiserver/app:go_default_library", "//cmd/kube-apiserver/app/options:go_default_library", - "//cmd/kubeadm/app/util/pkiutil:go_default_library", "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/batch:go_default_library", "//pkg/apis/policy/v1beta1:go_default_library", diff --git a/test/integration/framework/test_server.go b/test/integration/framework/test_server.go index 6aecade520..32ba52f409 100644 --- a/test/integration/framework/test_server.go +++ b/test/integration/framework/test_server.go @@ -36,8 +36,8 @@ import ( "k8s.io/client-go/util/cert" "k8s.io/kubernetes/cmd/kube-apiserver/app" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" "k8s.io/kubernetes/pkg/master" + "k8s.io/kubernetes/test/utils" ) // TestServerSetup holds configuration information for a kube-apiserver test server. @@ -55,7 +55,7 @@ func StartTestServer(t *testing.T, stopCh <-chan struct{}, setup TestServerSetup }() _, defaultServiceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24") - proxySigningKey, err := pkiutil.NewPrivateKey() + proxySigningKey, err := utils.NewPrivateKey() if err != nil { t.Fatal(err) } @@ -64,10 +64,10 @@ func StartTestServer(t *testing.T, stopCh <-chan struct{}, setup TestServerSetup t.Fatal(err) } proxyCACertFile, _ := ioutil.TempFile(certDir, "proxy-ca.crt") - if err := ioutil.WriteFile(proxyCACertFile.Name(), pkiutil.EncodeCertPEM(proxySigningCert), 0644); err != nil { + if err := ioutil.WriteFile(proxyCACertFile.Name(), utils.EncodeCertPEM(proxySigningCert), 0644); err != nil { t.Fatal(err) } - clientSigningKey, err := pkiutil.NewPrivateKey() + clientSigningKey, err := utils.NewPrivateKey() if err != nil { t.Fatal(err) } @@ -76,7 +76,7 @@ func StartTestServer(t *testing.T, stopCh <-chan struct{}, setup TestServerSetup t.Fatal(err) } clientCACertFile, _ := ioutil.TempFile(certDir, "client-ca.crt") - if err := ioutil.WriteFile(clientCACertFile.Name(), pkiutil.EncodeCertPEM(clientSigningCert), 0644); err != nil { + if err := ioutil.WriteFile(clientCACertFile.Name(), utils.EncodeCertPEM(clientSigningCert), 0644); err != nil { t.Fatal(err) } diff --git a/test/utils/BUILD b/test/utils/BUILD index b4233c8c83..01c2113ad8 100644 --- a/test/utils/BUILD +++ b/test/utils/BUILD @@ -17,6 +17,7 @@ go_library( "deployment.go", "node.go", "paths.go", + "pki_helpers.go", "pod_store.go", "replicaset.go", "runners.go", @@ -57,8 +58,10 @@ go_library( "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/scale:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", + "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", + "//vendor/github.com/pkg/errors:go_default_library", "//vendor/github.com/stretchr/testify/require:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], diff --git a/test/utils/pki_helpers.go b/test/utils/pki_helpers.go new file mode 100644 index 0000000000..c431aaf1af --- /dev/null +++ b/test/utils/pki_helpers.go @@ -0,0 +1,87 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "crypto" + "crypto/rand" + cryptorand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math" + "math/big" + "time" + + "github.com/pkg/errors" + + certutil "k8s.io/client-go/util/cert" +) + +const ( + certificateBlockType = "CERTIFICATE" + rsaKeySize = 2048 + duration365d = time.Hour * 24 * 365 +) + +// NewPrivateKey creates an RSA private key +func NewPrivateKey() (*rsa.PrivateKey, error) { + return rsa.GenerateKey(cryptorand.Reader, rsaKeySize) +} + +// EncodeCertPEM returns PEM-endcoded certificate data +func EncodeCertPEM(cert *x509.Certificate) []byte { + block := pem.Block{ + Type: certificateBlockType, + Bytes: cert.Raw, + } + return pem.EncodeToMemory(&block) +} + +// NewSignedCert creates a signed certificate using the given CA certificate and key +func NewSignedCert(cfg *certutil.Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) { + serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64)) + if err != nil { + return nil, err + } + if len(cfg.CommonName) == 0 { + return nil, errors.New("must specify a CommonName") + } + if len(cfg.Usages) == 0 { + return nil, errors.New("must specify at least one ExtKeyUsage") + } + + certTmpl := x509.Certificate{ + Subject: pkix.Name{ + CommonName: cfg.CommonName, + Organization: cfg.Organization, + }, + DNSNames: cfg.AltNames.DNSNames, + IPAddresses: cfg.AltNames.IPs, + SerialNumber: serial, + NotBefore: caCert.NotBefore, + NotAfter: time.Now().Add(duration365d).UTC(), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: cfg.Usages, + } + certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey) + if err != nil { + return nil, err + } + return x509.ParseCertificate(certDERBytes) +}