diff --git a/cmd/kubeadm/app/BUILD b/cmd/kubeadm/app/BUILD index 26fe4f6063..3f3441f41e 100644 --- a/cmd/kubeadm/app/BUILD +++ b/cmd/kubeadm/app/BUILD @@ -48,6 +48,7 @@ filegroup( "//cmd/kubeadm/app/phases/patchnode:all-srcs", "//cmd/kubeadm/app/phases/selfhosting:all-srcs", "//cmd/kubeadm/app/phases/upgrade:all-srcs", + "//cmd/kubeadm/app/phases/uploadcerts:all-srcs", "//cmd/kubeadm/app/phases/uploadconfig:all-srcs", "//cmd/kubeadm/app/preflight:all-srcs", "//cmd/kubeadm/app/util:all-srcs", diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go index dfc696ddcf..c85d35348d 100644 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go +++ b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -423,6 +423,7 @@ func isAllowedFlag(flagName string) bool { kubeadmcmdoptions.NodeName, kubeadmcmdoptions.NodeCRISocket, kubeadmcmdoptions.KubeconfigDir, + kubeadmcmdoptions.UploadCerts, "print-join-command", "rootfs", "v") if knownFlags.Has(flagName) { return true diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index ac69355eb4..ab81597af2 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -104,6 +104,7 @@ type initOptions struct { ignorePreflightErrors []string bto *options.BootstrapTokenOptions externalcfg *kubeadmapiv1beta1.InitConfiguration + uploadCerts bool } // initData defines all the runtime information used when running the kubeadm init worklow; @@ -121,6 +122,8 @@ type initData struct { client clientset.Interface waiter apiclient.Waiter outputWriter io.Writer + uploadCerts bool + certificateKey string } // NewCmdInit returns "kubeadm init" command. @@ -154,7 +157,7 @@ func NewCmdInit(out io.Writer, initOptions *initOptions) *cobra.Command { // adds flags to the init command // init command local flags could be eventually inherited by the sub-commands automatically generated for phases AddInitConfigFlags(cmd.Flags(), initOptions.externalcfg, &initOptions.featureGatesString) - AddInitOtherFlags(cmd.Flags(), &initOptions.cfgPath, &initOptions.skipTokenPrint, &initOptions.dryRun, &initOptions.ignorePreflightErrors) + AddInitOtherFlags(cmd.Flags(), &initOptions.cfgPath, &initOptions.skipTokenPrint, &initOptions.dryRun, &initOptions.uploadCerts, &initOptions.ignorePreflightErrors) initOptions.bto.AddTokenFlag(cmd.Flags()) initOptions.bto.AddTTLFlag(cmd.Flags()) options.AddImageMetaFlags(cmd.Flags(), &initOptions.externalcfg.ImageRepository) @@ -176,6 +179,7 @@ func NewCmdInit(out io.Writer, initOptions *initOptions) *cobra.Command { initRunner.AppendPhase(phases.NewEtcdPhase()) initRunner.AppendPhase(phases.NewWaitControlPlanePhase()) initRunner.AppendPhase(phases.NewUploadConfigPhase()) + initRunner.AppendPhase(phases.NewUploadCertsPhase()) initRunner.AppendPhase(phases.NewMarkControlPlanePhase()) initRunner.AppendPhase(phases.NewBootstrapTokenPhase()) initRunner.AppendPhase(phases.NewAddonPhase()) @@ -237,22 +241,30 @@ func AddInitConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta1.InitConfig } // AddInitOtherFlags adds init flags that are not bound to a configuration file to the given flagset -func AddInitOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipTokenPrint, dryRun *bool, ignorePreflightErrors *[]string) { +// Note: All flags that are not bound to the cfg object should be allowed in cmd/kubeadm/app/apis/kubeadm/validation/validation.go +func AddInitOtherFlags( + flagSet *flag.FlagSet, + cfgPath *string, + skipTokenPrint, dryRun, uploadCerts *bool, + ignorePreflightErrors *[]string, +) { options.AddConfigFlag(flagSet, cfgPath) flagSet.StringSliceVar( ignorePreflightErrors, options.IgnorePreflightErrors, *ignorePreflightErrors, "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.", ) - // Note: All flags that are not bound to the cfg object should be allowed in cmd/kubeadm/app/apis/kubeadm/validation/validation.go flagSet.BoolVar( skipTokenPrint, options.SkipTokenPrint, *skipTokenPrint, "Skip printing of the default bootstrap token generated by 'kubeadm init'.", ) - // Note: All flags that are not bound to the cfg object should be allowed in cmd/kubeadm/app/apis/kubeadm/validation/validation.go flagSet.BoolVar( dryRun, options.DryRun, *dryRun, "Don't apply any changes; just output what would be done.", ) + flagSet.BoolVar( + uploadCerts, options.UploadCerts, *uploadCerts, + "Upload certfificates to kubeadm-certs secret.", + ) } // newInitOptions returns a struct ready for being used for creating cmd init flags. @@ -270,6 +282,7 @@ func newInitOptions() *initOptions { bto: bto, kubeconfigDir: kubeadmconstants.KubernetesDir, kubeconfigPath: kubeadmconstants.GetAdminKubeConfigPath(), + uploadCerts: false, } } @@ -340,6 +353,9 @@ func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io if err := kubeconfigphase.ValidateKubeconfigsForExternalCA(kubeconfigDir, cfg); err != nil { return nil, err } + if options.uploadCerts { + return nil, errors.New("can't use externalCA mode and upload-certs") + } } return &initData{ @@ -353,9 +369,25 @@ func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io ignorePreflightErrors: ignorePreflightErrorsSet, externalCA: externalCA, outputWriter: out, + uploadCerts: options.uploadCerts, }, nil } +// UploadCerts returns Uploadcerts flag. +func (d *initData) UploadCerts() bool { + return d.uploadCerts +} + +// CertificateKey returns the key used to encrypt the certs. +func (d *initData) CertificateKey() string { + return d.certificateKey +} + +// SetCertificateKey set the key used to encrypt the certs. +func (d *initData) SetCertificateKey(key string) { + d.certificateKey = key +} + // Cfg returns initConfiguration. func (d *initData) Cfg() *kubeadmapi.InitConfiguration { return d.cfg @@ -461,8 +493,8 @@ func (d *initData) Tokens() []string { return tokens } -func printJoinCommand(out io.Writer, adminKubeConfigPath, token string, skipTokenPrint bool) error { - joinCommand, err := cmdutil.GetJoinCommand(adminKubeConfigPath, token, skipTokenPrint) +func printJoinCommand(out io.Writer, adminKubeConfigPath, token, key string, skipTokenPrint, uploadCerts bool) error { + joinCommand, err := cmdutil.GetJoinCommand(adminKubeConfigPath, token, key, skipTokenPrint, uploadCerts) if err != nil { return err } @@ -492,7 +524,7 @@ func showJoinCommand(i *initData, out io.Writer) error { // Prints the join command, multiple times in case the user has multiple tokens for _, token := range i.Tokens() { - if err := printJoinCommand(out, adminKubeConfigPath, token, i.skipTokenPrint); err != nil { + if err := printJoinCommand(out, adminKubeConfigPath, token, i.certificateKey, i.skipTokenPrint, i.uploadCerts); err != nil { return errors.Wrap(err, "failed to print join command") } } diff --git a/cmd/kubeadm/app/cmd/options/constant.go b/cmd/kubeadm/app/cmd/options/constant.go index e5e0d11c94..2386a20edd 100644 --- a/cmd/kubeadm/app/cmd/options/constant.go +++ b/cmd/kubeadm/app/cmd/options/constant.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -118,4 +118,7 @@ const ( // ControlPlane flag instruct kubeadm to create a new control plane instance on this node ControlPlane = "experimental-control-plane" + + // UploadCerts flag instruct kubeadm to upload certificates + UploadCerts = "experimental-upload-certs" ) diff --git a/cmd/kubeadm/app/cmd/phases/init/BUILD b/cmd/kubeadm/app/cmd/phases/init/BUILD index 8840cd1600..df4b683d69 100644 --- a/cmd/kubeadm/app/cmd/phases/init/BUILD +++ b/cmd/kubeadm/app/cmd/phases/init/BUILD @@ -12,6 +12,7 @@ go_library( "kubelet.go", "markcontrolplane.go", "preflight.go", + "uploadcerts.go", "uploadconfig.go", "waitcontrolplane.go", ], @@ -36,6 +37,7 @@ go_library( "//cmd/kubeadm/app/phases/kubelet:go_default_library", "//cmd/kubeadm/app/phases/markcontrolplane:go_default_library", "//cmd/kubeadm/app/phases/patchnode:go_default_library", + "//cmd/kubeadm/app/phases/uploadcerts:go_default_library", "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", diff --git a/cmd/kubeadm/app/cmd/phases/init/uploadcerts.go b/cmd/kubeadm/app/cmd/phases/init/uploadcerts.go new file mode 100644 index 0000000000..c9d388e53c --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/init/uploadcerts.go @@ -0,0 +1,83 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "fmt" + + "github.com/pkg/errors" + + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadcerts" +) + +type uploadCertsData interface { + Client() (clientset.Interface, error) + UploadCerts() bool + Cfg() *kubeadmapi.InitConfiguration + CertificateKey() string + SetCertificateKey(key string) +} + +// NewUploadCertsPhase returns the uploadCerts phase +func NewUploadCertsPhase() workflow.Phase { + return workflow.Phase{ + Name: "upload-certs", + Short: fmt.Sprintf("Upload certificates to %s", kubeadmconstants.KubeadmCertsSecret), + Long: cmdutil.MacroCommandLongDescription, + Run: runUploadCerts, + InheritFlags: []string{ + options.CfgPath, + options.UploadCerts, + }, + } +} + +func runUploadCerts(c workflow.RunData) error { + data, ok := c.(uploadCertsData) + if !ok { + return errors.New("upload-certs phase invoked with an invalid data struct") + } + + if !data.UploadCerts() { + klog.V(1).Infof("[upload-certs] Skipping certs upload") + return nil + } + client, err := data.Client() + if err != nil { + return err + } + + if len(data.CertificateKey()) == 0 { + certificateKey, err := uploadcerts.CreateCertificateKey() + if err != nil { + return err + } + data.SetCertificateKey(certificateKey) + } + + if err := uploadcerts.UploadCerts(client, data.Cfg(), data.CertificateKey()); err != nil { + return errors.Wrap(err, "error uploading certs") + } + return nil +} diff --git a/cmd/kubeadm/app/cmd/token.go b/cmd/kubeadm/app/cmd/token.go index 079e65c37a..3541f8f48e 100644 --- a/cmd/kubeadm/app/cmd/token.go +++ b/cmd/kubeadm/app/cmd/token.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -229,7 +229,10 @@ func RunCreateToken(out io.Writer, client clientset.Interface, cfgPath string, c // if --print-join-command was specified, print the full `kubeadm join` command // otherwise, just print the token if printJoinCommand { - joinCommand, err := cmdutil.GetJoinCommand(kubeConfigFile, internalcfg.BootstrapTokens[0].Token.String(), false) + key := "" + skipTokenPrint := false + uploadCerts := false + joinCommand, err := cmdutil.GetJoinCommand(kubeConfigFile, internalcfg.BootstrapTokens[0].Token.String(), key, skipTokenPrint, uploadCerts) if err != nil { return errors.Wrap(err, "failed to get join command") } diff --git a/cmd/kubeadm/app/cmd/util/join.go b/cmd/kubeadm/app/cmd/util/join.go index 02408dae30..c05bc7e1ac 100644 --- a/cmd/kubeadm/app/cmd/util/join.go +++ b/cmd/kubeadm/app/cmd/util/join.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -30,12 +30,12 @@ import ( ) var joinCommandTemplate = template.Must(template.New("join").Parse(`` + - `kubeadm join {{.MasterHostPort}} --token {{.Token}}{{range $h := .CAPubKeyPins}} --discovery-token-ca-cert-hash {{$h}}{{end}}`, + `kubeadm join {{.MasterHostPort}} --token {{.Token}}{{range $h := .CAPubKeyPins}} --discovery-token-ca-cert-hash {{$h}}{{end}}{{if .UploadCerts}} --certificate-key {{.CertificateKey}}{{end}}`, )) // GetJoinCommand returns the kubeadm join command for a given token and // and Kubernetes cluster (the current cluster in the kubeconfig file) -func GetJoinCommand(kubeConfigFile string, token string, skipTokenPrint bool) (string, error) { +func GetJoinCommand(kubeConfigFile, token, key string, skipTokenPrint, uploadCerts bool) (string, error) { // load the kubeconfig file to get the CA certificate and endpoint config, err := clientcmd.LoadFromFile(kubeConfigFile) if err != nil { @@ -74,6 +74,8 @@ func GetJoinCommand(kubeConfigFile string, token string, skipTokenPrint bool) (s "Token": token, "CAPubKeyPins": publicKeyPins, "MasterHostPort": strings.Replace(clusterConfig.Server, "https://", "", -1), + "UploadCerts": uploadCerts, + "CertificateKey": key, } if skipTokenPrint { diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index 3e8a647bdc..8aef0600a3 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ import ( "time" "github.com/pkg/errors" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/version" bootstrapapi "k8s.io/cluster-bootstrap/token/api" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -191,6 +191,13 @@ const ( // Default behaviour is 24 hours DefaultTokenDuration = 24 * time.Hour + // DefaultCertTokenDuration specifies the default amount of time that the token used by upload certs will be valid + // Default behaviour is 2 hours + DefaultCertTokenDuration = 2 * time.Hour + + // CertificateKeySize specifies the size of the key used to encrypt certificates on uploadcerts phase + CertificateKeySize = 32 + // LabelNodeRoleMaster specifies that a node is a master // This is a duplicate definition of the constant in pkg/controller/service/service_controller.go LabelNodeRoleMaster = "node-role.kubernetes.io/master" @@ -352,6 +359,9 @@ const ( // MasterNumCPU is the number of CPUs required on master MasterNumCPU = 2 + + // KubeadmCertsSecret specifies in what Secret in the kube-system namespace the certificates should be stored + KubeadmCertsSecret = "kubeadm-certs" ) var ( @@ -393,6 +403,10 @@ var ( 13: "3.2.24", 14: "3.3.10", } + + // KubeadmCertsClusterRoleName sets the name for the ClusterRole that allows + // the bootstrap tokens to access the kubeadm-certs Secret during the join of a new control-plane + KubeadmCertsClusterRoleName = fmt.Sprintf("kubeadm:%s", KubeadmCertsSecret) ) // EtcdSupportedVersion returns officially supported version of etcd for a specific Kubernetes release diff --git a/cmd/kubeadm/app/phases/uploadcerts/BUILD b/cmd/kubeadm/app/phases/uploadcerts/BUILD new file mode 100644 index 0000000000..89d6cbc038 --- /dev/null +++ b/cmd/kubeadm/app/phases/uploadcerts/BUILD @@ -0,0 +1,50 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["uploadcerts.go"], + importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadcerts", + visibility = ["//visibility:public"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library", + "//cmd/kubeadm/app/util/apiclient:go_default_library", + "//cmd/kubeadm/app/util/crypto:go_default_library", + "//pkg/apis/rbac/v1:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/api/rbac/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/cluster-bootstrap/token/util:go_default_library", + "//vendor/github.com/pkg/errors:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = ["uploadcerts_test.go"], + embed = [":go_default_library"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/util/crypto:go_default_library", + "//cmd/kubeadm/test:go_default_library", + "//vendor/github.com/lithammer/dedent:go_default_library", + ], +) diff --git a/cmd/kubeadm/app/phases/uploadcerts/uploadcerts.go b/cmd/kubeadm/app/phases/uploadcerts/uploadcerts.go new file mode 100644 index 0000000000..9e5b1f0e1d --- /dev/null +++ b/cmd/kubeadm/app/phases/uploadcerts/uploadcerts.go @@ -0,0 +1,205 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package uploadcerts + +import ( + "encoding/hex" + "fmt" + "io/ioutil" + "os" + "path" + "strings" + + "github.com/pkg/errors" + + v1 "k8s.io/api/core/v1" + rbac "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + clientset "k8s.io/client-go/kubernetes" + bootstraputil "k8s.io/cluster-bootstrap/token/util" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + nodebootstraptokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + cryptoutil "k8s.io/kubernetes/cmd/kubeadm/app/util/crypto" + rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1" +) + +const ( + externalEtcdCA = "external-etcd-ca.crt" + externalEtcdCert = "external-etcd.crt" + externalEtcdKey = "external-etcd.key" +) + +// createShortLivedBootstrapToken creates the token used to manager kubeadm-certs +// and return the tokenID +func createShortLivedBootstrapToken(client clientset.Interface) (string, error) { + tokenStr, err := bootstraputil.GenerateBootstrapToken() + if err != nil { + return "", errors.Wrap(err, "error generating token to upload certs") + } + token, err := kubeadmapi.NewBootstrapTokenString(tokenStr) + if err != nil { + return "", errors.Wrap(err, "error creating upload certs token") + } + tokens := []kubeadmapi.BootstrapToken{{ + Token: token, + Description: "Proxy for managing TTL for the kubeadm-certs secret", + TTL: &metav1.Duration{ + Duration: kubeadmconstants.DefaultCertTokenDuration, + }, + }} + + if err := nodebootstraptokenphase.CreateNewTokens(client, tokens); err != nil { + return "", errors.Wrap(err, "error creating token") + } + return tokens[0].Token.ID, nil +} + +//CreateCertificateKey returns a cryptographically secure random key +func CreateCertificateKey() (string, error) { + randBytes, err := cryptoutil.CreateRandBytes(kubeadmconstants.CertificateKeySize) + if err != nil { + return "", err + } + return hex.EncodeToString(randBytes), nil +} + +//UploadCerts save certs needs to join a new control-plane on kubeadm-certs sercret. +func UploadCerts(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, key string) error { + fmt.Printf("[upload-certs] storing the certificates in ConfigMap %q in the %q Namespace\n", kubeadmconstants.KubeadmCertsSecret, metav1.NamespaceSystem) + decodedKey, err := hex.DecodeString(key) + if err != nil { + return err + } + tokenID, err := createShortLivedBootstrapToken(client) + if err != nil { + return err + } + + secretData, err := getSecretData(cfg, decodedKey) + if err != nil { + return err + } + ref, err := getSecretOwnerRef(client, tokenID) + if err != nil { + return err + } + + err = apiclient.CreateOrUpdateSecret(client, &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: kubeadmconstants.KubeadmCertsSecret, + Namespace: metav1.NamespaceSystem, + OwnerReferences: ref, + }, + Data: secretData, + }) + if err != nil { + return err + } + + return createRBAC(client) +} + +func createRBAC(client clientset.Interface) error { + err := apiclient.CreateOrUpdateRole(client, &rbac.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: kubeadmconstants.KubeadmCertsClusterRoleName, + Namespace: metav1.NamespaceSystem, + }, + Rules: []rbac.PolicyRule{ + rbachelper.NewRule("get").Groups("").Resources("secrets").Names(kubeadmconstants.KubeadmCertsSecret).RuleOrDie(), + }, + }) + if err != nil { + return err + } + + return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: kubeadmconstants.KubeadmCertsClusterRoleName, + Namespace: metav1.NamespaceSystem, + }, + RoleRef: rbac.RoleRef{ + APIGroup: rbac.GroupName, + Kind: "Role", + Name: kubeadmconstants.KubeadmCertsClusterRoleName, + }, + Subjects: []rbac.Subject{ + { + Kind: rbac.GroupKind, + Name: kubeadmconstants.NodeBootstrapTokenAuthGroup, + }, + }, + }) +} + +func getSecretOwnerRef(client clientset.Interface, tokenID string) ([]metav1.OwnerReference, error) { + secretName := bootstraputil.BootstrapTokenSecretName(tokenID) + secret, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Get(secretName, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrap(err, "error to get token reference") + } + + gvk := schema.GroupVersionKind{Version: "v1", Kind: "Secret"} + ref := metav1.NewControllerRef(secret, gvk) + return []metav1.OwnerReference{*ref}, nil +} + +func loadAndEncryptCert(certPath string, key []byte) ([]byte, error) { + cert, err := ioutil.ReadFile(certPath) + if err != nil { + return nil, err + } + return cryptoutil.EncryptBytes(cert, key) +} + +func certsToUpload(cfg *kubeadmapi.InitConfiguration) map[string]string { + certsDir := cfg.CertificatesDir + certs := map[string]string{ + kubeadmconstants.CACertName: path.Join(certsDir, kubeadmconstants.CACertName), + kubeadmconstants.CAKeyName: path.Join(certsDir, kubeadmconstants.CAKeyName), + kubeadmconstants.FrontProxyCACertName: path.Join(certsDir, kubeadmconstants.FrontProxyCACertName), + kubeadmconstants.FrontProxyCAKeyName: path.Join(certsDir, kubeadmconstants.FrontProxyCAKeyName), + kubeadmconstants.ServiceAccountPublicKeyName: path.Join(certsDir, kubeadmconstants.ServiceAccountPublicKeyName), + kubeadmconstants.ServiceAccountPrivateKeyName: path.Join(certsDir, kubeadmconstants.ServiceAccountPrivateKeyName), + } + + if cfg.Etcd.External == nil { + certs[kubeadmconstants.EtcdCACertName] = path.Join(certsDir, kubeadmconstants.EtcdCACertName) + certs[kubeadmconstants.EtcdCAKeyName] = path.Join(certsDir, kubeadmconstants.EtcdCAKeyName) + } else { + certs[externalEtcdCA] = cfg.Etcd.External.CAFile + certs[externalEtcdCert] = cfg.Etcd.External.CertFile + certs[externalEtcdKey] = cfg.Etcd.External.KeyFile + } + return certs +} + +func getSecretData(cfg *kubeadmapi.InitConfiguration, key []byte) (map[string][]byte, error) { + secretData := map[string][]byte{} + for certName, certPath := range certsToUpload(cfg) { + cert, err := loadAndEncryptCert(certPath, key) + if err == nil || (err != nil && os.IsNotExist(err)) { + secretData[strings.Replace(certName, "/", "-", -1)] = cert + } else { + return nil, err + } + } + return secretData, nil +} diff --git a/cmd/kubeadm/app/phases/uploadcerts/uploadcerts_test.go b/cmd/kubeadm/app/phases/uploadcerts/uploadcerts_test.go new file mode 100644 index 0000000000..a8d2faf250 --- /dev/null +++ b/cmd/kubeadm/app/phases/uploadcerts/uploadcerts_test.go @@ -0,0 +1,116 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package uploadcerts + +import ( + "encoding/hex" + "io/ioutil" + "os" + "path" + "regexp" + "testing" + + "github.com/lithammer/dedent" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + cryptoutil "k8s.io/kubernetes/cmd/kubeadm/app/util/crypto" + testutil "k8s.io/kubernetes/cmd/kubeadm/test" +) + +func TestUploadCerts(t *testing.T) { + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + +} + +//teste cert name, teste cert can be decrypted +func TestGetSecretData(t *testing.T) { + certData := []byte("cert-data") + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + cfg := &kubeadmapi.InitConfiguration{} + cfg.CertificatesDir = tmpdir + + key, err := CreateCertificateKey() + if err != nil { + t.Fatalf(dedent.Dedent("failed to create key.\nfatal error: %v"), err) + } + decodedKey, err := hex.DecodeString(key) + if err != nil { + t.Fatalf(dedent.Dedent("failed to decode key.\nfatal error: %v"), err) + } + + if err := os.Mkdir(path.Join(tmpdir, "etcd"), 0755); err != nil { + t.Fatalf(dedent.Dedent("failed to create etcd cert dir.\nfatal error: %v"), err) + } + + certs := certsToUpload(cfg) + for name, path := range certs { + if err := ioutil.WriteFile(path, certData, 0644); err != nil { + t.Fatalf(dedent.Dedent("failed to write cert: %s\nfatal error: %v"), name, err) + } + } + + secretData, err := getSecretData(cfg, decodedKey) + if err != nil { + t.Fatalf("failed to get secret data. fatal error: %v", err) + } + + re := regexp.MustCompile(`[-._a-zA-Z0-9]+`) + for name, data := range secretData { + if !re.MatchString(name) { + t.Fatalf(dedent.Dedent("failed to validate secretData\n %s isn't a valid secret key"), name) + } + + decryptedData, err := cryptoutil.DecryptBytes(data, decodedKey) + if string(certData) != string(decryptedData) { + t.Fatalf(dedent.Dedent("can't decript cert: %s\nfatal error: %v"), name, err) + } + } +} + +func TestCertsToUpload(t *testing.T) { + localEtcdCfg := &kubeadmapi.InitConfiguration{} + externalEtcdCfg := &kubeadmapi.InitConfiguration{} + externalEtcdCfg.Etcd = kubeadmapi.Etcd{} + externalEtcdCfg.Etcd.External = &kubeadmapi.ExternalEtcd{} + + tests := map[string]struct { + config *kubeadmapi.InitConfiguration + expectedCerts []string + }{ + "local etcd": { + config: localEtcdCfg, + expectedCerts: []string{kubeadmconstants.EtcdCACertName, kubeadmconstants.EtcdCAKeyName}, + }, + "external etcd": { + config: externalEtcdCfg, + expectedCerts: []string{externalEtcdCA, externalEtcdCert, externalEtcdKey}, + }, + } + + for name, test := range tests { + t.Run(name, func(t2 *testing.T) { + certList := certsToUpload(test.config) + for _, cert := range test.expectedCerts { + if _, found := certList[cert]; !found { + t2.Fatalf(dedent.Dedent("failed to get list of certs to upload\ncert %s not found"), cert) + } + } + }) + } +} diff --git a/cmd/kubeadm/app/util/BUILD b/cmd/kubeadm/app/util/BUILD index 7356f887c0..259e1d97ca 100644 --- a/cmd/kubeadm/app/util/BUILD +++ b/cmd/kubeadm/app/util/BUILD @@ -82,6 +82,7 @@ filegroup( "//cmd/kubeadm/app/util/audit:all-srcs", "//cmd/kubeadm/app/util/certs:all-srcs", "//cmd/kubeadm/app/util/config:all-srcs", + "//cmd/kubeadm/app/util/crypto:all-srcs", "//cmd/kubeadm/app/util/dryrun:all-srcs", "//cmd/kubeadm/app/util/etcd:all-srcs", "//cmd/kubeadm/app/util/kubeconfig:all-srcs", diff --git a/cmd/kubeadm/app/util/crypto/BUILD b/cmd/kubeadm/app/util/crypto/BUILD new file mode 100644 index 0000000000..8993845f27 --- /dev/null +++ b/cmd/kubeadm/app/util/crypto/BUILD @@ -0,0 +1,33 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["crypto.go"], + importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/crypto", + visibility = ["//visibility:public"], + deps = ["//vendor/github.com/pkg/errors:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = ["crypto_test.go"], + embed = [":go_default_library"], + deps = [ + "//cmd/kubeadm/app/constants:go_default_library", + "//vendor/github.com/lithammer/dedent:go_default_library", + ], +) diff --git a/cmd/kubeadm/app/util/crypto/crypto.go b/cmd/kubeadm/app/util/crypto/crypto.go new file mode 100644 index 0000000000..c3b866bee2 --- /dev/null +++ b/cmd/kubeadm/app/util/crypto/crypto.go @@ -0,0 +1,76 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crypto + +import ( + "github.com/pkg/errors" + + "crypto/aes" + "crypto/cipher" + "crypto/rand" +) + +// CreateRandBytes returns a cryptographically secure slice of random bytes with a given size +func CreateRandBytes(size uint32) ([]byte, error) { + bytes := make([]byte, size) + if _, err := rand.Read(bytes); err != nil { + return nil, err + } + return bytes, nil +} + +// EncryptBytes takes a byte slice of raw data and an encryption key and returns an encrypted byte slice of data. +// The key must be an AES key, either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256 +func EncryptBytes(data, key []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + nonce, err := CreateRandBytes(uint32(gcm.NonceSize())) + if err != nil { + return nil, err + } + return gcm.Seal(nonce, nonce, data, nil), nil +} + +// DecryptBytes takes a byte slice of encrypted data and an encryption key and returns a decrypted byte slice of data. +// The key must be an AES key, either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256 +func DecryptBytes(data, key []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + nonceSize := gcm.NonceSize() + if len(data) < nonceSize { + return nil, errors.New("size of data is less than the nonce") + } + + nonce, out := data[:nonceSize], data[nonceSize:] + out, err = gcm.Open(nil, nonce, out, nil) + if err != nil { + return nil, err + } + return out, nil +} diff --git a/cmd/kubeadm/app/util/crypto/crypto_test.go b/cmd/kubeadm/app/util/crypto/crypto_test.go new file mode 100644 index 0000000000..40fd295954 --- /dev/null +++ b/cmd/kubeadm/app/util/crypto/crypto_test.go @@ -0,0 +1,93 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crypto + +import ( + "testing" + + "github.com/lithammer/dedent" + + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" +) + +func TestEncryptAndDecryptData(t *testing.T) { + key1, err := CreateRandBytes(kubeadmconstants.CertificateKeySize) + if err != nil { + t.Fatal(err) + } + key2, err := CreateRandBytes(kubeadmconstants.CertificateKeySize) + if err != nil { + t.Fatal(err) + } + testData := []byte("Lorem ipsum dolor sit amet, consectetur adipiscing elit.") + + tests := map[string]struct { + encryptKey []byte + decryptKey []byte + data []byte + expectDecryptErr bool + }{ + "can decrypt using the correct key": { + encryptKey: key1, + decryptKey: key1, + data: testData, + expectDecryptErr: false, + }, + "can't decrypt using incorrect key": { + encryptKey: key1, + decryptKey: key2, + data: testData, + expectDecryptErr: true, + }, + "can't decrypt without a key": { + encryptKey: key1, + decryptKey: []byte{}, + data: testData, + expectDecryptErr: true, + }, + } + + for name, test := range tests { + t.Run(name, func(t2 *testing.T) { + encryptedData, err := EncryptBytes(test.data, test.encryptKey) + if err != nil { + t2.Fatalf(dedent.Dedent( + "EncryptBytes failed\nerror: %v"), + err, + ) + } + + decryptedData, err := DecryptBytes(encryptedData, test.decryptKey) + if (err != nil) != test.expectDecryptErr { + t2.Fatalf(dedent.Dedent( + "DecryptBytes failed\nexpected error: %t\n\tgot: %t\nerror: %v"), + test.expectDecryptErr, + (err != nil), + err, + ) + } + + if (string(decryptedData) != string(test.data)) && !test.expectDecryptErr { + t2.Fatalf(dedent.Dedent( + "EncryptDecryptBytes failed\nexpected decryptedData equal to data\n\tgot: data=%q decryptedData=%q"), + test.data, + string(decryptedData), + ) + } + }) + } +}