Merge pull request #76052 from neolit123/remove-cmd-kubeadm-from-e2e-fw

test: partially decouple from cmd/kubeadm

test/e2e/apimachinery/BUILD

@@ -28,7 +28,6 @@ go_library(
     ],
     importpath = "k8s.io/kubernetes/test/e2e/apimachinery",
     deps = [
-        "//cmd/kubeadm/app/util/pkiutil:go_default_library",
         "//pkg/api/v1/pod:go_default_library",
         "//pkg/apis/rbac/v1beta1:go_default_library",
         "//pkg/printers:go_default_library",

test/e2e/apimachinery/certs.go

@@ -23,8 +23,8 @@ import (
 
 	"k8s.io/client-go/util/cert"
 	"k8s.io/client-go/util/keyutil"
-	"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/utils"
 )
 
 type certContext struct {
@@ -41,7 +41,7 @@ func setupServerCert(namespaceName, serviceName string) *certContext {
 		framework.Failf("Failed to create a temp dir for cert generation %v", err)
 	}
 	defer os.RemoveAll(certDir)
-	signingKey, err := pkiutil.NewPrivateKey()
+	signingKey, err := utils.NewPrivateKey()
 	if err != nil {
 		framework.Failf("Failed to create CA private key %v", err)
 	}
@@ -53,14 +53,14 @@ func setupServerCert(namespaceName, serviceName string) *certContext {
 	if err != nil {
 		framework.Failf("Failed to create a temp file for ca cert generation %v", err)
 	}
-	if err := ioutil.WriteFile(caCertFile.Name(), pkiutil.EncodeCertPEM(signingCert), 0644); err != nil {
+	if err := ioutil.WriteFile(caCertFile.Name(), utils.EncodeCertPEM(signingCert), 0644); err != nil {
 		framework.Failf("Failed to write CA cert %v", err)
 	}
-	key, err := pkiutil.NewPrivateKey()
+	key, err := utils.NewPrivateKey()
 	if err != nil {
 		framework.Failf("Failed to create private key for %v", err)
 	}
-	signedCert, err := pkiutil.NewSignedCert(
+	signedCert, err := utils.NewSignedCert(
 		&cert.Config{
 			CommonName: serviceName + "." + namespaceName + ".svc",
 			Usages:     []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
@@ -78,7 +78,7 @@ func setupServerCert(namespaceName, serviceName string) *certContext {
 	if err != nil {
 		framework.Failf("Failed to create a temp file for key generation %v", err)
 	}
-	if err = ioutil.WriteFile(certFile.Name(), pkiutil.EncodeCertPEM(signedCert), 0600); err != nil {
+	if err = ioutil.WriteFile(certFile.Name(), utils.EncodeCertPEM(signedCert), 0600); err != nil {
 		framework.Failf("Failed to write cert file %v", err)
 	}
 	privateKeyPEM, err := keyutil.MarshalPrivateKeyToPEM(key)
@@ -89,8 +89,8 @@ func setupServerCert(namespaceName, serviceName string) *certContext {
 		framework.Failf("Failed to write key file %v", err)
 	}
 	return &certContext{
-		cert:        pkiutil.EncodeCertPEM(signedCert),
+		cert:        utils.EncodeCertPEM(signedCert),
 		key:         privateKeyPEM,
-		signingCert: pkiutil.EncodeCertPEM(signingCert),
+		signingCert: utils.EncodeCertPEM(signingCert),
 	}
 }

test/e2e/auth/BUILD

@@ -20,7 +20,6 @@ go_library(
     ],
     importpath = "k8s.io/kubernetes/test/e2e/auth",
     deps = [
-        "//cmd/kubeadm/app/util/pkiutil:go_default_library",
         "//pkg/master/ports:go_default_library",
         "//pkg/security/apparmor:go_default_library",
         "//pkg/security/podsecuritypolicy/seccomp:go_default_library",

test/e2e/auth/certificates.go

@@ -27,8 +27,8 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	v1beta1client "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
 	"k8s.io/client-go/util/cert"
-	"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/utils"
 
 	. "github.com/onsi/ginkgo"
 )
@@ -39,7 +39,7 @@ var _ = SIGDescribe("Certificates API", func() {
 	It("should support building a client with a CSR", func() {
 		const commonName = "tester-csr"
 
-		pk, err := pkiutil.NewPrivateKey()
+		pk, err := utils.NewPrivateKey()
 		framework.ExpectNoError(err)
 
 		pkder := x509.MarshalPKCS1PrivateKey(pk)

test/e2e/manifest/BUILD

@@ -11,13 +11,13 @@ go_library(
     srcs = ["manifest.go"],
     importpath = "k8s.io/kubernetes/test/e2e/manifest",
     deps = [
-        "//cmd/kubeadm/app/util:go_default_library",
         "//staging/src/k8s.io/api/apps/v1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/extensions/v1beta1:go_default_library",
         "//staging/src/k8s.io/api/rbac/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
         "//test/e2e/framework/testfiles:go_default_library",

test/e2e/manifest/manifest.go

@@ -26,9 +26,9 @@ import (
 	rbac "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilyaml "k8s.io/apimachinery/pkg/util/yaml"
-	"k8s.io/client-go/kubernetes/scheme"
+	scheme "k8s.io/client-go/kubernetes/scheme"
-	"k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/test/e2e/framework/testfiles"
 )
 
@@ -107,7 +107,7 @@ func IngressFromManifest(fileName string) (*extensions.Ingress, error) {
 // IngressToManifest generates a yaml file in the given path with the given ingress.
 // Assumes that a directory exists at the given path.
 func IngressToManifest(ing *extensions.Ingress, path string) error {
-	serialized, err := util.MarshalToYaml(ing, extensions.SchemeGroupVersion)
+	serialized, err := marshalToYaml(ing, extensions.SchemeGroupVersion)
 	if err != nil {
 		return fmt.Errorf("failed to marshal ingress %v to YAML: %v", ing, err)
 	}
@@ -178,3 +178,15 @@ func RoleFromManifest(fileName, ns string) (*rbac.Role, error) {
 	role.Namespace = ns
 	return &role, nil
 }
+
+// marshalToYaml marshals an object into YAML for a given GroupVersion.
+// The object must be known in SupportedMediaTypes() for the Codecs under "client-go/kubernetes/scheme".
+func marshalToYaml(obj runtime.Object, gv schema.GroupVersion) ([]byte, error) {
+	mediaType := "application/yaml"
+	info, ok := runtime.SerializerInfoForMediaType(scheme.Codecs.SupportedMediaTypes(), mediaType)
+	if !ok {
+		return []byte{}, fmt.Errorf("unsupported media type %q", mediaType)
+	}
+	encoder := scheme.Codecs.EncoderForVersion(info.Serializer, gv)
+	return runtime.Encode(encoder, obj)
+}

test/integration/examples/BUILD

@@ -17,7 +17,6 @@ go_test(
     deps = [
         "//cmd/kube-apiserver/app:go_default_library",
         "//cmd/kube-apiserver/app/options:go_default_library",
-        "//cmd/kubeadm/app/util/pkiutil:go_default_library",
         "//pkg/master:go_default_library",
         "//pkg/master/reconcilers:go_default_library",
         "//staging/src/k8s.io/api/admissionregistration/v1beta1:go_default_library",
@@ -44,6 +43,7 @@ go_test(
         "//staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/v1beta1:go_default_library",
         "//staging/src/k8s.io/sample-apiserver/pkg/cmd/server:go_default_library",
         "//test/integration/framework:go_default_library",
+        "//test/utils:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
     ],
 )

test/integration/examples/apiserver_test.go

@@ -49,8 +49,8 @@ import (
 	kubeaggregatorserver "k8s.io/kube-aggregator/pkg/cmd/server"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
-	"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
 	"k8s.io/kubernetes/test/integration/framework"
+	testutil "k8s.io/kubernetes/test/utils"
 	wardlev1alpha1 "k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1"
 	wardlev1beta1 "k8s.io/sample-apiserver/pkg/apis/wardle/v1beta1"
 	sampleserver "k8s.io/sample-apiserver/pkg/cmd/server"
@@ -63,7 +63,7 @@ func TestAggregatedAPIServer(t *testing.T) {
 	certDir, _ := ioutil.TempDir("", "test-integration-apiserver")
 	defer os.RemoveAll(certDir)
 	_, defaultServiceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24")
-	proxySigningKey, err := pkiutil.NewPrivateKey()
+	proxySigningKey, err := testutil.NewPrivateKey()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -72,10 +72,10 @@ func TestAggregatedAPIServer(t *testing.T) {
 		t.Fatal(err)
 	}
 	proxyCACertFile, _ := ioutil.TempFile(certDir, "proxy-ca.crt")
-	if err := ioutil.WriteFile(proxyCACertFile.Name(), pkiutil.EncodeCertPEM(proxySigningCert), 0644); err != nil {
+	if err := ioutil.WriteFile(proxyCACertFile.Name(), testutil.EncodeCertPEM(proxySigningCert), 0644); err != nil {
 		t.Fatal(err)
 	}
-	clientSigningKey, err := pkiutil.NewPrivateKey()
+	clientSigningKey, err := testutil.NewPrivateKey()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -84,7 +84,7 @@ func TestAggregatedAPIServer(t *testing.T) {
 		t.Fatal(err)
 	}
 	clientCACertFile, _ := ioutil.TempFile(certDir, "client-ca.crt")
-	if err := ioutil.WriteFile(clientCACertFile.Name(), pkiutil.EncodeCertPEM(clientSigningCert), 0644); err != nil {
+	if err := ioutil.WriteFile(clientCACertFile.Name(), testutil.EncodeCertPEM(clientSigningCert), 0644); err != nil {
 		t.Fatal(err)
 	}
 
@@ -236,11 +236,11 @@ func TestAggregatedAPIServer(t *testing.T) {
 	// start the aggregator
 	aggregatorCertDir, _ := ioutil.TempDir("", "test-integration-aggregator")
 	defer os.RemoveAll(aggregatorCertDir)
-	proxyClientKey, err := pkiutil.NewPrivateKey()
+	proxyClientKey, err := testutil.NewPrivateKey()
 	if err != nil {
 		t.Fatal(err)
 	}
-	proxyClientCert, err := pkiutil.NewSignedCert(
+	proxyClientCert, err := testutil.NewSignedCert(
 		&cert.Config{
 			CommonName: "kube-aggregator",
 			Usages:     []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
@@ -249,7 +249,7 @@ func TestAggregatedAPIServer(t *testing.T) {
 	)
 	proxyClientCertFile, _ := ioutil.TempFile(aggregatorCertDir, "proxy-client.crt")
 	proxyClientKeyFile, _ := ioutil.TempFile(aggregatorCertDir, "proxy-client.key")
-	if err := ioutil.WriteFile(proxyClientCertFile.Name(), pkiutil.EncodeCertPEM(proxyClientCert), 0600); err != nil {
+	if err := ioutil.WriteFile(proxyClientCertFile.Name(), testutil.EncodeCertPEM(proxyClientCert), 0600); err != nil {
 		t.Fatal(err)
 	}
 	proxyClientKeyPEM, err := keyutil.MarshalPrivateKeyToPEM(proxyClientKey)

test/integration/framework/BUILD

@@ -37,7 +37,6 @@ go_library(
     deps = [
         "//cmd/kube-apiserver/app:go_default_library",
         "//cmd/kube-apiserver/app/options:go_default_library",
-        "//cmd/kubeadm/app/util/pkiutil:go_default_library",
         "//pkg/api/legacyscheme:go_default_library",
         "//pkg/apis/batch:go_default_library",
         "//pkg/apis/policy/v1beta1:go_default_library",

test/integration/framework/test_server.go

@@ -36,8 +36,8 @@ import (
 	"k8s.io/client-go/util/cert"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
-	"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
 	"k8s.io/kubernetes/pkg/master"
+	"k8s.io/kubernetes/test/utils"
 )
 
 // TestServerSetup holds configuration information for a kube-apiserver test server.
@@ -55,7 +55,7 @@ func StartTestServer(t *testing.T, stopCh <-chan struct{}, setup TestServerSetup
 	}()
 
 	_, defaultServiceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24")
-	proxySigningKey, err := pkiutil.NewPrivateKey()
+	proxySigningKey, err := utils.NewPrivateKey()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -64,10 +64,10 @@ func StartTestServer(t *testing.T, stopCh <-chan struct{}, setup TestServerSetup
 		t.Fatal(err)
 	}
 	proxyCACertFile, _ := ioutil.TempFile(certDir, "proxy-ca.crt")
-	if err := ioutil.WriteFile(proxyCACertFile.Name(), pkiutil.EncodeCertPEM(proxySigningCert), 0644); err != nil {
+	if err := ioutil.WriteFile(proxyCACertFile.Name(), utils.EncodeCertPEM(proxySigningCert), 0644); err != nil {
 		t.Fatal(err)
 	}
-	clientSigningKey, err := pkiutil.NewPrivateKey()
+	clientSigningKey, err := utils.NewPrivateKey()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -76,7 +76,7 @@ func StartTestServer(t *testing.T, stopCh <-chan struct{}, setup TestServerSetup
 		t.Fatal(err)
 	}
 	clientCACertFile, _ := ioutil.TempFile(certDir, "client-ca.crt")
-	if err := ioutil.WriteFile(clientCACertFile.Name(), pkiutil.EncodeCertPEM(clientSigningCert), 0644); err != nil {
+	if err := ioutil.WriteFile(clientCACertFile.Name(), utils.EncodeCertPEM(clientSigningCert), 0644); err != nil {
 		t.Fatal(err)
 	}
 

test/utils/BUILD

@@ -17,6 +17,7 @@ go_library(
         "deployment.go",
         "node.go",
         "paths.go",
+        "pki_helpers.go",
         "pod_store.go",
         "replicaset.go",
         "runners.go",
@@ -57,8 +58,10 @@ go_library(
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/scale:go_default_library",
         "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
+        "//staging/src/k8s.io/client-go/util/cert:go_default_library",
         "//staging/src/k8s.io/client-go/util/workqueue:go_default_library",
         "//vendor/github.com/davecgh/go-spew/spew:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
         "//vendor/github.com/stretchr/testify/require:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
     ],

test/utils/pki_helpers.go

@@ -0,0 +1,87 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"crypto"
+	"crypto/rand"
+	cryptorand "crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math"
+	"math/big"
+	"time"
+
+	"github.com/pkg/errors"
+
+	certutil "k8s.io/client-go/util/cert"
+)
+
+const (
+	certificateBlockType = "CERTIFICATE"
+	rsaKeySize           = 2048
+	duration365d         = time.Hour * 24 * 365
+)
+
+// NewPrivateKey creates an RSA private key
+func NewPrivateKey() (*rsa.PrivateKey, error) {
+	return rsa.GenerateKey(cryptorand.Reader, rsaKeySize)
+}
+
+// EncodeCertPEM returns PEM-endcoded certificate data
+func EncodeCertPEM(cert *x509.Certificate) []byte {
+	block := pem.Block{
+		Type:  certificateBlockType,
+		Bytes: cert.Raw,
+	}
+	return pem.EncodeToMemory(&block)
+}
+
+// NewSignedCert creates a signed certificate using the given CA certificate and key
+func NewSignedCert(cfg *certutil.Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) {
+	serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64))
+	if err != nil {
+		return nil, err
+	}
+	if len(cfg.CommonName) == 0 {
+		return nil, errors.New("must specify a CommonName")
+	}
+	if len(cfg.Usages) == 0 {
+		return nil, errors.New("must specify at least one ExtKeyUsage")
+	}
+
+	certTmpl := x509.Certificate{
+		Subject: pkix.Name{
+			CommonName:   cfg.CommonName,
+			Organization: cfg.Organization,
+		},
+		DNSNames:     cfg.AltNames.DNSNames,
+		IPAddresses:  cfg.AltNames.IPs,
+		SerialNumber: serial,
+		NotBefore:    caCert.NotBefore,
+		NotAfter:     time.Now().Add(duration365d).UTC(),
+		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:  cfg.Usages,
+	}
+	certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey)
+	if err != nil {
+		return nil, err
+	}
+	return x509.ParseCertificate(certDERBytes)
+}