Merge pull request #73907 from yagonobre/init-upload-certs

Add kubeadm init upload encrypted certs phase

cmd/kubeadm/app/BUILD

@@ -48,6 +48,7 @@ filegroup(
         "//cmd/kubeadm/app/phases/patchnode:all-srcs",
         "//cmd/kubeadm/app/phases/selfhosting:all-srcs",
         "//cmd/kubeadm/app/phases/upgrade:all-srcs",
+        "//cmd/kubeadm/app/phases/uploadcerts:all-srcs",
         "//cmd/kubeadm/app/phases/uploadconfig:all-srcs",
         "//cmd/kubeadm/app/preflight:all-srcs",
         "//cmd/kubeadm/app/util:all-srcs",

cmd/kubeadm/app/apis/kubeadm/validation/validation.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2016 The Kubernetes Authors.
+Copyright 2019 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -423,6 +423,7 @@ func isAllowedFlag(flagName string) bool {
 		kubeadmcmdoptions.NodeName,
 		kubeadmcmdoptions.NodeCRISocket,
 		kubeadmcmdoptions.KubeconfigDir,
+		kubeadmcmdoptions.UploadCerts,
 		"print-join-command", "rootfs", "v")
 	if knownFlags.Has(flagName) {
 		return true

cmd/kubeadm/app/cmd/init.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2018 The Kubernetes Authors.
+Copyright 2019 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -104,6 +104,7 @@ type initOptions struct {
 	ignorePreflightErrors []string
 	bto                   *options.BootstrapTokenOptions
 	externalcfg           *kubeadmapiv1beta1.InitConfiguration
+	uploadCerts           bool
 }
 
 // initData defines all the runtime information used when running the kubeadm init worklow;
@@ -121,6 +122,8 @@ type initData struct {
 	client                clientset.Interface
 	waiter                apiclient.Waiter
 	outputWriter          io.Writer
+	uploadCerts           bool
+	certificateKey        string
 }
 
 // NewCmdInit returns "kubeadm init" command.
@@ -154,7 +157,7 @@ func NewCmdInit(out io.Writer, initOptions *initOptions) *cobra.Command {
 	// adds flags to the init command
 	// init command local flags could be eventually inherited by the sub-commands automatically generated for phases
 	AddInitConfigFlags(cmd.Flags(), initOptions.externalcfg, &initOptions.featureGatesString)
-	AddInitOtherFlags(cmd.Flags(), &initOptions.cfgPath, &initOptions.skipTokenPrint, &initOptions.dryRun, &initOptions.ignorePreflightErrors)
+	AddInitOtherFlags(cmd.Flags(), &initOptions.cfgPath, &initOptions.skipTokenPrint, &initOptions.dryRun, &initOptions.uploadCerts, &initOptions.ignorePreflightErrors)
 	initOptions.bto.AddTokenFlag(cmd.Flags())
 	initOptions.bto.AddTTLFlag(cmd.Flags())
 	options.AddImageMetaFlags(cmd.Flags(), &initOptions.externalcfg.ImageRepository)
@@ -176,6 +179,7 @@ func NewCmdInit(out io.Writer, initOptions *initOptions) *cobra.Command {
 	initRunner.AppendPhase(phases.NewEtcdPhase())
 	initRunner.AppendPhase(phases.NewWaitControlPlanePhase())
 	initRunner.AppendPhase(phases.NewUploadConfigPhase())
+	initRunner.AppendPhase(phases.NewUploadCertsPhase())
 	initRunner.AppendPhase(phases.NewMarkControlPlanePhase())
 	initRunner.AppendPhase(phases.NewBootstrapTokenPhase())
 	initRunner.AppendPhase(phases.NewAddonPhase())
@@ -237,22 +241,30 @@ func AddInitConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta1.InitConfig
 }
 
 // AddInitOtherFlags adds init flags that are not bound to a configuration file to the given flagset
-func AddInitOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipTokenPrint, dryRun *bool, ignorePreflightErrors *[]string) {
+// Note: All flags that are not bound to the cfg object should be allowed in cmd/kubeadm/app/apis/kubeadm/validation/validation.go
+func AddInitOtherFlags(
+	flagSet *flag.FlagSet,
+	cfgPath *string,
+	skipTokenPrint, dryRun, uploadCerts *bool,
+	ignorePreflightErrors *[]string,
+) {
 	options.AddConfigFlag(flagSet, cfgPath)
 	flagSet.StringSliceVar(
 		ignorePreflightErrors, options.IgnorePreflightErrors, *ignorePreflightErrors,
 		"A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.",
 	)
-	// Note: All flags that are not bound to the cfg object should be allowed in cmd/kubeadm/app/apis/kubeadm/validation/validation.go
 	flagSet.BoolVar(
 		skipTokenPrint, options.SkipTokenPrint, *skipTokenPrint,
 		"Skip printing of the default bootstrap token generated by 'kubeadm init'.",
 	)
-	// Note: All flags that are not bound to the cfg object should be allowed in cmd/kubeadm/app/apis/kubeadm/validation/validation.go
 	flagSet.BoolVar(
 		dryRun, options.DryRun, *dryRun,
 		"Don't apply any changes; just output what would be done.",
 	)
+	flagSet.BoolVar(
+		uploadCerts, options.UploadCerts, *uploadCerts,
+		"Upload certfificates to kubeadm-certs secret.",
+	)
 }
 
 // newInitOptions returns a struct ready for being used for creating cmd init flags.
@@ -270,6 +282,7 @@ func newInitOptions() *initOptions {
 		bto:            bto,
 		kubeconfigDir:  kubeadmconstants.KubernetesDir,
 		kubeconfigPath: kubeadmconstants.GetAdminKubeConfigPath(),
+		uploadCerts:    false,
 	}
 }
 
@@ -340,6 +353,9 @@ func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io
 		if err := kubeconfigphase.ValidateKubeconfigsForExternalCA(kubeconfigDir, cfg); err != nil {
 			return nil, err
 		}
+		if options.uploadCerts {
+			return nil, errors.New("can't use externalCA mode and upload-certs")
+		}
 	}
 
 	return &initData{
@@ -353,9 +369,25 @@ func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io
 		ignorePreflightErrors: ignorePreflightErrorsSet,
 		externalCA:            externalCA,
 		outputWriter:          out,
+		uploadCerts:           options.uploadCerts,
 	}, nil
 }
 
+// UploadCerts returns Uploadcerts flag.
+func (d *initData) UploadCerts() bool {
+	return d.uploadCerts
+}
+
+// CertificateKey returns the key used to encrypt the certs.
+func (d *initData) CertificateKey() string {
+	return d.certificateKey
+}
+
+// SetCertificateKey set the key used to encrypt the certs.
+func (d *initData) SetCertificateKey(key string) {
+	d.certificateKey = key
+}
+
 // Cfg returns initConfiguration.
 func (d *initData) Cfg() *kubeadmapi.InitConfiguration {
 	return d.cfg
@@ -461,8 +493,8 @@ func (d *initData) Tokens() []string {
 	return tokens
 }
 
-func printJoinCommand(out io.Writer, adminKubeConfigPath, token string, skipTokenPrint bool) error {
+func printJoinCommand(out io.Writer, adminKubeConfigPath, token, key string, skipTokenPrint, uploadCerts bool) error {
-	joinCommand, err := cmdutil.GetJoinCommand(adminKubeConfigPath, token, skipTokenPrint)
+	joinCommand, err := cmdutil.GetJoinCommand(adminKubeConfigPath, token, key, skipTokenPrint, uploadCerts)
 	if err != nil {
 		return err
 	}
@@ -492,7 +524,7 @@ func showJoinCommand(i *initData, out io.Writer) error {
 
 	// Prints the join command, multiple times in case the user has multiple tokens
 	for _, token := range i.Tokens() {
-		if err := printJoinCommand(out, adminKubeConfigPath, token, i.skipTokenPrint); err != nil {
+		if err := printJoinCommand(out, adminKubeConfigPath, token, i.certificateKey, i.skipTokenPrint, i.uploadCerts); err != nil {
 			return errors.Wrap(err, "failed to print join command")
 		}
 	}

cmd/kubeadm/app/cmd/options/constant.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2018 The Kubernetes Authors.
+Copyright 2019 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -118,4 +118,7 @@ const (
 
 	// ControlPlane flag instruct kubeadm to create a new control plane instance on this node
 	ControlPlane = "experimental-control-plane"
+
+	// UploadCerts flag instruct kubeadm to upload certificates
+	UploadCerts = "experimental-upload-certs"
 )

cmd/kubeadm/app/cmd/phases/init/BUILD

@@ -12,6 +12,7 @@ go_library(
         "kubelet.go",
         "markcontrolplane.go",
         "preflight.go",
+        "uploadcerts.go",
         "uploadconfig.go",
         "waitcontrolplane.go",
     ],
@@ -36,6 +37,7 @@ go_library(
         "//cmd/kubeadm/app/phases/kubelet:go_default_library",
         "//cmd/kubeadm/app/phases/markcontrolplane:go_default_library",
         "//cmd/kubeadm/app/phases/patchnode:go_default_library",
+        "//cmd/kubeadm/app/phases/uploadcerts:go_default_library",
         "//cmd/kubeadm/app/phases/uploadconfig:go_default_library",
         "//cmd/kubeadm/app/preflight:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",

cmd/kubeadm/app/cmd/phases/init/uploadcerts.go

@@ -0,0 +1,83 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package phases
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog"
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadcerts"
+)
+
+type uploadCertsData interface {
+	Client() (clientset.Interface, error)
+	UploadCerts() bool
+	Cfg() *kubeadmapi.InitConfiguration
+	CertificateKey() string
+	SetCertificateKey(key string)
+}
+
+// NewUploadCertsPhase returns the uploadCerts phase
+func NewUploadCertsPhase() workflow.Phase {
+	return workflow.Phase{
+		Name:  "upload-certs",
+		Short: fmt.Sprintf("Upload certificates to %s", kubeadmconstants.KubeadmCertsSecret),
+		Long:  cmdutil.MacroCommandLongDescription,
+		Run:   runUploadCerts,
+		InheritFlags: []string{
+			options.CfgPath,
+			options.UploadCerts,
+		},
+	}
+}
+
+func runUploadCerts(c workflow.RunData) error {
+	data, ok := c.(uploadCertsData)
+	if !ok {
+		return errors.New("upload-certs phase invoked with an invalid data struct")
+	}
+
+	if !data.UploadCerts() {
+		klog.V(1).Infof("[upload-certs] Skipping certs upload")
+		return nil
+	}
+	client, err := data.Client()
+	if err != nil {
+		return err
+	}
+
+	if len(data.CertificateKey()) == 0 {
+		certificateKey, err := uploadcerts.CreateCertificateKey()
+		if err != nil {
+			return err
+		}
+		data.SetCertificateKey(certificateKey)
+	}
+
+	if err := uploadcerts.UploadCerts(client, data.Cfg(), data.CertificateKey()); err != nil {
+		return errors.Wrap(err, "error uploading certs")
+	}
+	return nil
+}

cmd/kubeadm/app/cmd/token.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2016 The Kubernetes Authors.
+Copyright 2019 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -229,7 +229,10 @@ func RunCreateToken(out io.Writer, client clientset.Interface, cfgPath string, c
 	// if --print-join-command was specified, print the full `kubeadm join` command
 	// otherwise, just print the token
 	if printJoinCommand {
-		joinCommand, err := cmdutil.GetJoinCommand(kubeConfigFile, internalcfg.BootstrapTokens[0].Token.String(), false)
+		key := ""
+		skipTokenPrint := false
+		uploadCerts := false
+		joinCommand, err := cmdutil.GetJoinCommand(kubeConfigFile, internalcfg.BootstrapTokens[0].Token.String(), key, skipTokenPrint, uploadCerts)
 		if err != nil {
 			return errors.Wrap(err, "failed to get join command")
 		}

cmd/kubeadm/app/cmd/util/join.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2018 The Kubernetes Authors.
+Copyright 2019 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -30,12 +30,12 @@ import (
 )
 
 var joinCommandTemplate = template.Must(template.New("join").Parse(`` +
-	`kubeadm join {{.MasterHostPort}} --token {{.Token}}{{range $h := .CAPubKeyPins}} --discovery-token-ca-cert-hash {{$h}}{{end}}`,
+	`kubeadm join {{.MasterHostPort}} --token {{.Token}}{{range $h := .CAPubKeyPins}} --discovery-token-ca-cert-hash {{$h}}{{end}}{{if .UploadCerts}} --certificate-key {{.CertificateKey}}{{end}}`,
 ))
 
 // GetJoinCommand returns the kubeadm join command for a given token and
 // and Kubernetes cluster (the current cluster in the kubeconfig file)
-func GetJoinCommand(kubeConfigFile string, token string, skipTokenPrint bool) (string, error) {
+func GetJoinCommand(kubeConfigFile, token, key string, skipTokenPrint, uploadCerts bool) (string, error) {
 	// load the kubeconfig file to get the CA certificate and endpoint
 	config, err := clientcmd.LoadFromFile(kubeConfigFile)
 	if err != nil {
@@ -74,6 +74,8 @@ func GetJoinCommand(kubeConfigFile string, token string, skipTokenPrint bool) (s
 		"Token":          token,
 		"CAPubKeyPins":   publicKeyPins,
 		"MasterHostPort": strings.Replace(clusterConfig.Server, "https://", "", -1),
+		"UploadCerts":    uploadCerts,
+		"CertificateKey": key,
 	}
 
 	if skipTokenPrint {

cmd/kubeadm/app/constants/constants.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2016 The Kubernetes Authors.
+Copyright 2019 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -26,7 +26,7 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/version"
 	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
@@ -191,6 +191,13 @@ const (
 	// Default behaviour is 24 hours
 	DefaultTokenDuration = 24 * time.Hour
 
+	// DefaultCertTokenDuration specifies the default amount of time that the token used by upload certs will be valid
+	// Default behaviour is 2 hours
+	DefaultCertTokenDuration = 2 * time.Hour
+
+	// CertificateKeySize specifies the size of the key used to encrypt certificates on uploadcerts phase
+	CertificateKeySize = 32
+
 	// LabelNodeRoleMaster specifies that a node is a master
 	// This is a duplicate definition of the constant in pkg/controller/service/service_controller.go
 	LabelNodeRoleMaster = "node-role.kubernetes.io/master"
@@ -352,6 +359,9 @@ const (
 
 	// MasterNumCPU is the number of CPUs required on master
 	MasterNumCPU = 2
+
+	// KubeadmCertsSecret specifies in what Secret in the kube-system namespace the certificates should be stored
+	KubeadmCertsSecret = "kubeadm-certs"
 )
 
 var (
@@ -393,6 +403,10 @@ var (
 		13: "3.2.24",
 		14: "3.3.10",
 	}
+
+	// KubeadmCertsClusterRoleName sets the name for the ClusterRole that allows
+	// the bootstrap tokens to access the kubeadm-certs Secret during the join of a new control-plane
+	KubeadmCertsClusterRoleName = fmt.Sprintf("kubeadm:%s", KubeadmCertsSecret)
 )
 
 // EtcdSupportedVersion returns officially supported version of etcd for a specific Kubernetes release

cmd/kubeadm/app/phases/uploadcerts/BUILD

@@ -0,0 +1,50 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["uploadcerts.go"],
+    importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadcerts",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library",
+        "//cmd/kubeadm/app/util/apiclient:go_default_library",
+        "//cmd/kubeadm/app/util/crypto:go_default_library",
+        "//pkg/apis/rbac/v1:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/api/rbac/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/cluster-bootstrap/token/util:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["uploadcerts_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/util/crypto:go_default_library",
+        "//cmd/kubeadm/test:go_default_library",
+        "//vendor/github.com/lithammer/dedent:go_default_library",
+    ],
+)

cmd/kubeadm/app/phases/uploadcerts/uploadcerts.go

@@ -0,0 +1,205 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package uploadcerts
+
+import (
+	"encoding/hex"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	v1 "k8s.io/api/core/v1"
+	rbac "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	clientset "k8s.io/client-go/kubernetes"
+	bootstraputil "k8s.io/cluster-bootstrap/token/util"
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	nodebootstraptokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
+	cryptoutil "k8s.io/kubernetes/cmd/kubeadm/app/util/crypto"
+	rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
+)
+
+const (
+	externalEtcdCA   = "external-etcd-ca.crt"
+	externalEtcdCert = "external-etcd.crt"
+	externalEtcdKey  = "external-etcd.key"
+)
+
+// createShortLivedBootstrapToken creates the token used to manager kubeadm-certs
+// and return the tokenID
+func createShortLivedBootstrapToken(client clientset.Interface) (string, error) {
+	tokenStr, err := bootstraputil.GenerateBootstrapToken()
+	if err != nil {
+		return "", errors.Wrap(err, "error generating token to upload certs")
+	}
+	token, err := kubeadmapi.NewBootstrapTokenString(tokenStr)
+	if err != nil {
+		return "", errors.Wrap(err, "error creating upload certs token")
+	}
+	tokens := []kubeadmapi.BootstrapToken{{
+		Token:       token,
+		Description: "Proxy for managing TTL for the kubeadm-certs secret",
+		TTL: &metav1.Duration{
+			Duration: kubeadmconstants.DefaultCertTokenDuration,
+		},
+	}}
+
+	if err := nodebootstraptokenphase.CreateNewTokens(client, tokens); err != nil {
+		return "", errors.Wrap(err, "error creating token")
+	}
+	return tokens[0].Token.ID, nil
+}
+
+//CreateCertificateKey returns a cryptographically secure random key
+func CreateCertificateKey() (string, error) {
+	randBytes, err := cryptoutil.CreateRandBytes(kubeadmconstants.CertificateKeySize)
+	if err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(randBytes), nil
+}
+
+//UploadCerts save certs needs to join a new control-plane on kubeadm-certs sercret.
+func UploadCerts(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, key string) error {
+	fmt.Printf("[upload-certs] storing the certificates in ConfigMap %q in the %q Namespace\n", kubeadmconstants.KubeadmCertsSecret, metav1.NamespaceSystem)
+	decodedKey, err := hex.DecodeString(key)
+	if err != nil {
+		return err
+	}
+	tokenID, err := createShortLivedBootstrapToken(client)
+	if err != nil {
+		return err
+	}
+
+	secretData, err := getSecretData(cfg, decodedKey)
+	if err != nil {
+		return err
+	}
+	ref, err := getSecretOwnerRef(client, tokenID)
+	if err != nil {
+		return err
+	}
+
+	err = apiclient.CreateOrUpdateSecret(client, &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            kubeadmconstants.KubeadmCertsSecret,
+			Namespace:       metav1.NamespaceSystem,
+			OwnerReferences: ref,
+		},
+		Data: secretData,
+	})
+	if err != nil {
+		return err
+	}
+
+	return createRBAC(client)
+}
+
+func createRBAC(client clientset.Interface) error {
+	err := apiclient.CreateOrUpdateRole(client, &rbac.Role{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      kubeadmconstants.KubeadmCertsClusterRoleName,
+			Namespace: metav1.NamespaceSystem,
+		},
+		Rules: []rbac.PolicyRule{
+			rbachelper.NewRule("get").Groups("").Resources("secrets").Names(kubeadmconstants.KubeadmCertsSecret).RuleOrDie(),
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      kubeadmconstants.KubeadmCertsClusterRoleName,
+			Namespace: metav1.NamespaceSystem,
+		},
+		RoleRef: rbac.RoleRef{
+			APIGroup: rbac.GroupName,
+			Kind:     "Role",
+			Name:     kubeadmconstants.KubeadmCertsClusterRoleName,
+		},
+		Subjects: []rbac.Subject{
+			{
+				Kind: rbac.GroupKind,
+				Name: kubeadmconstants.NodeBootstrapTokenAuthGroup,
+			},
+		},
+	})
+}
+
+func getSecretOwnerRef(client clientset.Interface, tokenID string) ([]metav1.OwnerReference, error) {
+	secretName := bootstraputil.BootstrapTokenSecretName(tokenID)
+	secret, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Get(secretName, metav1.GetOptions{})
+	if err != nil {
+		return nil, errors.Wrap(err, "error to get token reference")
+	}
+
+	gvk := schema.GroupVersionKind{Version: "v1", Kind: "Secret"}
+	ref := metav1.NewControllerRef(secret, gvk)
+	return []metav1.OwnerReference{*ref}, nil
+}
+
+func loadAndEncryptCert(certPath string, key []byte) ([]byte, error) {
+	cert, err := ioutil.ReadFile(certPath)
+	if err != nil {
+		return nil, err
+	}
+	return cryptoutil.EncryptBytes(cert, key)
+}
+
+func certsToUpload(cfg *kubeadmapi.InitConfiguration) map[string]string {
+	certsDir := cfg.CertificatesDir
+	certs := map[string]string{
+		kubeadmconstants.CACertName:                   path.Join(certsDir, kubeadmconstants.CACertName),
+		kubeadmconstants.CAKeyName:                    path.Join(certsDir, kubeadmconstants.CAKeyName),
+		kubeadmconstants.FrontProxyCACertName:         path.Join(certsDir, kubeadmconstants.FrontProxyCACertName),
+		kubeadmconstants.FrontProxyCAKeyName:          path.Join(certsDir, kubeadmconstants.FrontProxyCAKeyName),
+		kubeadmconstants.ServiceAccountPublicKeyName:  path.Join(certsDir, kubeadmconstants.ServiceAccountPublicKeyName),
+		kubeadmconstants.ServiceAccountPrivateKeyName: path.Join(certsDir, kubeadmconstants.ServiceAccountPrivateKeyName),
+	}
+
+	if cfg.Etcd.External == nil {
+		certs[kubeadmconstants.EtcdCACertName] = path.Join(certsDir, kubeadmconstants.EtcdCACertName)
+		certs[kubeadmconstants.EtcdCAKeyName] = path.Join(certsDir, kubeadmconstants.EtcdCAKeyName)
+	} else {
+		certs[externalEtcdCA] = cfg.Etcd.External.CAFile
+		certs[externalEtcdCert] = cfg.Etcd.External.CertFile
+		certs[externalEtcdKey] = cfg.Etcd.External.KeyFile
+	}
+	return certs
+}
+
+func getSecretData(cfg *kubeadmapi.InitConfiguration, key []byte) (map[string][]byte, error) {
+	secretData := map[string][]byte{}
+	for certName, certPath := range certsToUpload(cfg) {
+		cert, err := loadAndEncryptCert(certPath, key)
+		if err == nil || (err != nil && os.IsNotExist(err)) {
+			secretData[strings.Replace(certName, "/", "-", -1)] = cert
+		} else {
+			return nil, err
+		}
+	}
+	return secretData, nil
+}

cmd/kubeadm/app/phases/uploadcerts/uploadcerts_test.go

@@ -0,0 +1,116 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package uploadcerts
+
+import (
+	"encoding/hex"
+	"io/ioutil"
+	"os"
+	"path"
+	"regexp"
+	"testing"
+
+	"github.com/lithammer/dedent"
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	cryptoutil "k8s.io/kubernetes/cmd/kubeadm/app/util/crypto"
+	testutil "k8s.io/kubernetes/cmd/kubeadm/test"
+)
+
+func TestUploadCerts(t *testing.T) {
+	tmpdir := testutil.SetupTempDir(t)
+	defer os.RemoveAll(tmpdir)
+
+}
+
+//teste cert name, teste cert can be decrypted
+func TestGetSecretData(t *testing.T) {
+	certData := []byte("cert-data")
+	tmpdir := testutil.SetupTempDir(t)
+	defer os.RemoveAll(tmpdir)
+	cfg := &kubeadmapi.InitConfiguration{}
+	cfg.CertificatesDir = tmpdir
+
+	key, err := CreateCertificateKey()
+	if err != nil {
+		t.Fatalf(dedent.Dedent("failed to create key.\nfatal error: %v"), err)
+	}
+	decodedKey, err := hex.DecodeString(key)
+	if err != nil {
+		t.Fatalf(dedent.Dedent("failed to decode key.\nfatal error: %v"), err)
+	}
+
+	if err := os.Mkdir(path.Join(tmpdir, "etcd"), 0755); err != nil {
+		t.Fatalf(dedent.Dedent("failed to create etcd cert dir.\nfatal error: %v"), err)
+	}
+
+	certs := certsToUpload(cfg)
+	for name, path := range certs {
+		if err := ioutil.WriteFile(path, certData, 0644); err != nil {
+			t.Fatalf(dedent.Dedent("failed to write cert: %s\nfatal error: %v"), name, err)
+		}
+	}
+
+	secretData, err := getSecretData(cfg, decodedKey)
+	if err != nil {
+		t.Fatalf("failed to get secret data. fatal error: %v", err)
+	}
+
+	re := regexp.MustCompile(`[-._a-zA-Z0-9]+`)
+	for name, data := range secretData {
+		if !re.MatchString(name) {
+			t.Fatalf(dedent.Dedent("failed to validate secretData\n %s isn't a valid secret key"), name)
+		}
+
+		decryptedData, err := cryptoutil.DecryptBytes(data, decodedKey)
+		if string(certData) != string(decryptedData) {
+			t.Fatalf(dedent.Dedent("can't decript cert: %s\nfatal error: %v"), name, err)
+		}
+	}
+}
+
+func TestCertsToUpload(t *testing.T) {
+	localEtcdCfg := &kubeadmapi.InitConfiguration{}
+	externalEtcdCfg := &kubeadmapi.InitConfiguration{}
+	externalEtcdCfg.Etcd = kubeadmapi.Etcd{}
+	externalEtcdCfg.Etcd.External = &kubeadmapi.ExternalEtcd{}
+
+	tests := map[string]struct {
+		config        *kubeadmapi.InitConfiguration
+		expectedCerts []string
+	}{
+		"local etcd": {
+			config:        localEtcdCfg,
+			expectedCerts: []string{kubeadmconstants.EtcdCACertName, kubeadmconstants.EtcdCAKeyName},
+		},
+		"external etcd": {
+			config:        externalEtcdCfg,
+			expectedCerts: []string{externalEtcdCA, externalEtcdCert, externalEtcdKey},
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t2 *testing.T) {
+			certList := certsToUpload(test.config)
+			for _, cert := range test.expectedCerts {
+				if _, found := certList[cert]; !found {
+					t2.Fatalf(dedent.Dedent("failed to get list of certs to upload\ncert %s not found"), cert)
+				}
+			}
+		})
+	}
+}

cmd/kubeadm/app/util/BUILD

@@ -82,6 +82,7 @@ filegroup(
         "//cmd/kubeadm/app/util/audit:all-srcs",
         "//cmd/kubeadm/app/util/certs:all-srcs",
         "//cmd/kubeadm/app/util/config:all-srcs",
+        "//cmd/kubeadm/app/util/crypto:all-srcs",
         "//cmd/kubeadm/app/util/dryrun:all-srcs",
         "//cmd/kubeadm/app/util/etcd:all-srcs",
         "//cmd/kubeadm/app/util/kubeconfig:all-srcs",

cmd/kubeadm/app/util/crypto/BUILD

@@ -0,0 +1,33 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["crypto.go"],
+    importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/crypto",
+    visibility = ["//visibility:public"],
+    deps = ["//vendor/github.com/pkg/errors:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["crypto_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//cmd/kubeadm/app/constants:go_default_library",
+        "//vendor/github.com/lithammer/dedent:go_default_library",
+    ],
+)

cmd/kubeadm/app/util/crypto/crypto.go

@@ -0,0 +1,76 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crypto
+
+import (
+	"github.com/pkg/errors"
+
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+)
+
+// CreateRandBytes returns a cryptographically secure slice of random bytes with a given size
+func CreateRandBytes(size uint32) ([]byte, error) {
+	bytes := make([]byte, size)
+	if _, err := rand.Read(bytes); err != nil {
+		return nil, err
+	}
+	return bytes, nil
+}
+
+// EncryptBytes takes a byte slice of raw data and an encryption key and returns an encrypted byte slice of data.
+// The key must be an AES key, either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256
+func EncryptBytes(data, key []byte) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+	nonce, err := CreateRandBytes(uint32(gcm.NonceSize()))
+	if err != nil {
+		return nil, err
+	}
+	return gcm.Seal(nonce, nonce, data, nil), nil
+}
+
+// DecryptBytes takes a byte slice of encrypted data and an encryption key and returns a decrypted byte slice of data.
+// The key must be an AES key, either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256
+func DecryptBytes(data, key []byte) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+	nonceSize := gcm.NonceSize()
+	if len(data) < nonceSize {
+		return nil, errors.New("size of data is less than the nonce")
+	}
+
+	nonce, out := data[:nonceSize], data[nonceSize:]
+	out, err = gcm.Open(nil, nonce, out, nil)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}

cmd/kubeadm/app/util/crypto/crypto_test.go

@@ -0,0 +1,93 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crypto
+
+import (
+	"testing"
+
+	"github.com/lithammer/dedent"
+
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+)
+
+func TestEncryptAndDecryptData(t *testing.T) {
+	key1, err := CreateRandBytes(kubeadmconstants.CertificateKeySize)
+	if err != nil {
+		t.Fatal(err)
+	}
+	key2, err := CreateRandBytes(kubeadmconstants.CertificateKeySize)
+	if err != nil {
+		t.Fatal(err)
+	}
+	testData := []byte("Lorem ipsum dolor sit amet, consectetur adipiscing elit.")
+
+	tests := map[string]struct {
+		encryptKey       []byte
+		decryptKey       []byte
+		data             []byte
+		expectDecryptErr bool
+	}{
+		"can decrypt using the correct key": {
+			encryptKey:       key1,
+			decryptKey:       key1,
+			data:             testData,
+			expectDecryptErr: false,
+		},
+		"can't decrypt using incorrect key": {
+			encryptKey:       key1,
+			decryptKey:       key2,
+			data:             testData,
+			expectDecryptErr: true,
+		},
+		"can't decrypt without a key": {
+			encryptKey:       key1,
+			decryptKey:       []byte{},
+			data:             testData,
+			expectDecryptErr: true,
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t2 *testing.T) {
+			encryptedData, err := EncryptBytes(test.data, test.encryptKey)
+			if err != nil {
+				t2.Fatalf(dedent.Dedent(
+					"EncryptBytes failed\nerror: %v"),
+					err,
+				)
+			}
+
+			decryptedData, err := DecryptBytes(encryptedData, test.decryptKey)
+			if (err != nil) != test.expectDecryptErr {
+				t2.Fatalf(dedent.Dedent(
+					"DecryptBytes failed\nexpected error: %t\n\tgot: %t\nerror: %v"),
+					test.expectDecryptErr,
+					(err != nil),
+					err,
+				)
+			}
+
+			if (string(decryptedData) != string(test.data)) && !test.expectDecryptErr {
+				t2.Fatalf(dedent.Dedent(
+					"EncryptDecryptBytes failed\nexpected decryptedData equal to data\n\tgot: data=%q decryptedData=%q"),
+					test.data,
+					string(decryptedData),
+				)
+			}
+		})
+	}
+}