diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go new file mode 100644 index 0000000000..584909353c --- /dev/null +++ b/cmd/kubeadm/app/kubeadm.go @@ -0,0 +1,65 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "os" + "path" + "strings" + + "github.com/spf13/pflag" + + "k8s.io/kubernetes/pkg/kubeadm/cmd" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/util/logs" +) + +var CommandLine *pflag.FlagSet + +// TODO(phase2) use componentconfig +// we need some params for testing etc, let's keep these hidden for now +func getEnvParams() map[string]string { + globalPrefix := os.Getenv("KUBE_PREFIX_ALL") + if globalPrefix == "" { + globalPrefix = "/etc/kubernetes" + } + + envParams := map[string]string{ + "prefix": globalPrefix, + "host_pki_path": path.Join(globalPrefix, "pki"), + "hyperkube_image": "gcr.io/google_containers/hyperkube:v1.4.0-alpha.3", + "discovery_image": "dgoodwin/kubediscovery:latest", + } + + for k, _ := range envParams { + if v := os.Getenv(fmt.Sprintf("KUBE_%s", strings.ToUpper(k))); v != "" { + envParams[k] = v + } + } + + return envParams +} + +func Run() error { + CommandLine = pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError) + logs.InitLogs() + defer logs.FlushLogs() + + cmd := cmd.NewKubeadmCommand(cmdutil.NewFactory(nil), os.Stdin, os.Stdout, os.Stderr, getEnvParams()) + return cmd.Execute() +} diff --git a/cmd/kubeadm/dummy.go b/cmd/kubeadm/kubeadm.go similarity index 79% rename from cmd/kubeadm/dummy.go rename to cmd/kubeadm/kubeadm.go index 17711df504..6aca5a18f3 100644 --- a/cmd/kubeadm/dummy.go +++ b/cmd/kubeadm/kubeadm.go @@ -1,5 +1,5 @@ /* -Copyright 2014 The Kubernetes Authors. +Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,8 +17,14 @@ limitations under the License. package main import ( - _ "github.com/square/go-jose" + "os" + + "k8s.io/kubernetes/cmd/kubeadm/app" ) func main() { + if err := app.Run(); err != nil { + os.Exit(1) + } + os.Exit(0) } diff --git a/hack/.linted_packages b/hack/.linted_packages index da961ba18c..3fd57c57f7 100644 --- a/hack/.linted_packages +++ b/hack/.linted_packages @@ -243,3 +243,4 @@ test/integration/openshift test/soak/cauldron test/soak/serve_hostnames third_party/forked/golang/expansion +cmd/kubeadm diff --git a/pkg/kubeadm/.gitignore b/pkg/kubeadm/.gitignore new file mode 100644 index 0000000000..a8a20d9f1b --- /dev/null +++ b/pkg/kubeadm/.gitignore @@ -0,0 +1 @@ +kubeadm diff --git a/pkg/kubeadm/api/types.go b/pkg/kubeadm/api/types.go new file mode 100644 index 0000000000..e659162cb0 --- /dev/null +++ b/pkg/kubeadm/api/types.go @@ -0,0 +1,44 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeadmapi + +type BootstrapParams struct { + // A struct with methods that implement Discover() + // kubeadm will do the CSR dance + Discovery *OutOfBandDiscovery + EnvParams map[string]string +} + +type OutOfBandDiscovery struct { + // 'join-node' side + ApiServerURLs string // comma separated + CaCertFile string + GivenToken string // dot-separated `.` set by the user + TokenID string // optional on master side, will be generated if not specified + Token []byte // optional on master side, will be generated if not specified + BearerToken string // set based on Token + // 'init-master' side + ApiServerDNSName string // optional, used in master bootstrap + ListenIP string // optional IP for master to listen on, rather than autodetect +} + +type ClusterInfo struct { + // TODO Kind, apiVersion + // TODO clusterId, fetchedTime, expiredTime + CertificateAuthorities []string `json:"certificateAuthorities"` + Endpoints []string `json:"endpoints"` +} diff --git a/pkg/kubeadm/cmd/cmd.go b/pkg/kubeadm/cmd/cmd.go new file mode 100644 index 0000000000..2fe3ae0f16 --- /dev/null +++ b/pkg/kubeadm/cmd/cmd.go @@ -0,0 +1,97 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "io" + + "github.com/renstrom/dedent" + "github.com/spf13/cobra" + + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/util/flag" + + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" +) + +func NewKubeadmCommand(f *cmdutil.Factory, in io.Reader, out, err io.Writer, envParams map[string]string) *cobra.Command { + cmds := &cobra.Command{ + Use: "kubeadm", + Short: "kubeadm: bootstrap a secure kubernetes cluster easily.", + Long: dedent.Dedent(` + kubeadm: bootstrap a secure kubernetes cluster easily. + + ┌──────────────────────────────────────────────────────────┐ + │ KUBEADM IS ALPHA, DO NOT USE IT FOR PRODUCTION CLUSTERS! │ + │ │ + │ But, please try it out! Give us feedback at: │ + │ https://github.com/kubernetes/kubernetes/issues │ + │ and at-mention @kubernetes/sig-cluster-lifecycle │ + └──────────────────────────────────────────────────────────┘ + + Example usage: + + Create a two-machine cluster with one master (which controls the cluster), + and one node (where workloads, like pods and replica sets run). + + ┌──────────────────────────────────────────────────────────┐ + │ On the first machine │ + ├──────────────────────────────────────────────────────────┤ + │ master# kubeadm init master │ + │ Your token is: │ + └──────────────────────────────────────────────────────────┘ + + ┌──────────────────────────────────────────────────────────┐ + │ On the second machine │ + ├──────────────────────────────────────────────────────────┤ + │ node# kubeadm join node --token= │ + └──────────────────────────────────────────────────────────┘ + + You can then repeat the second step on as many other machines as you like. + + `), + } + // TODO figure out how to avoid running as root + // + // TODO also print the alpha warning when running any commands, as well as + // in the help text. + // + // TODO detect interactive vs non-interactive use and adjust output accordingly + // i.e. make it automation friendly + // + // TODO create an bastraction that defines files and the content that needs to + // be written to disc and write it all in one go at the end as we have a lot of + // crapy little files written from different parts of this code; this could also + // be useful for testing + + bootstrapParams := &kubeadmapi.BootstrapParams{ + Discovery: &kubeadmapi.OutOfBandDiscovery{ + // TODO this type no longer makes sense here + }, + EnvParams: envParams, + } + + cmds.ResetFlags() + cmds.SetGlobalNormalizationFunc(flag.WarnWordSepNormalizeFunc) + + cmds.AddCommand(NewCmdInit(out, bootstrapParams)) + cmds.AddCommand(NewCmdJoin(out, bootstrapParams)) + cmds.AddCommand(NewCmdUser(out, bootstrapParams)) + cmds.AddCommand(NewCmdManual(out, bootstrapParams)) + + return cmds +} diff --git a/pkg/kubeadm/cmd/init.go b/pkg/kubeadm/cmd/init.go new file mode 100644 index 0000000000..5c7a541900 --- /dev/null +++ b/pkg/kubeadm/cmd/init.go @@ -0,0 +1,108 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "io" + + "github.com/renstrom/dedent" + "github.com/spf13/cobra" + + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubemaster "k8s.io/kubernetes/pkg/kubeadm/master" + kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" +) + +var ( + init_done_msgf = dedent.Dedent(` + Kubernetes master initialised successfully! + + You can connect any number of nodes by running: + + kubeadm join --token %s %s + `) +) + +func NewCmdInit(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { + cmd := &cobra.Command{ + Use: "init --token [--listen-ip ]", + Short: "Run this on the first server you deploy onto.", + Run: func(cmd *cobra.Command, args []string) { + err := RunInit(out, cmd, args, params) + cmdutil.CheckErr(err) + }, + } + + cmd.PersistentFlags().StringVarP(¶ms.Discovery.ListenIP, "listen-ip", "", "", + `(optional) IP address to listen on, in case autodetection fails.`) + cmd.PersistentFlags().StringVarP(¶ms.Discovery.GivenToken, "token", "", "", + `(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`) + + return cmd +} + +func RunInit(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error { + if params.Discovery.ListenIP == "" { + ip, err := kubeadmutil.GetDefaultHostIP() + if err != nil { + return err + } + params.Discovery.ListenIP = ip + } + if err := kubemaster.CreateTokenAuthFile(params); err != nil { + return err + } + if err := kubemaster.WriteStaticPodManifests(params); err != nil { + return err + } + caKey, caCert, err := kubemaster.CreatePKIAssets(params) + if err != nil { + return err + } + kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(params, []string{"kubelet", "admin"}, caKey, caCert) + if err != nil { + return err + } + for name, kubeconfig := range kubeconfigs { + if err := kubeadmutil.WriteKubeconfigIfNotExists(params, name, kubeconfig); err != nil { + return err + } + } + + client, err := kubemaster.CreateClientAndWaitForAPI(kubeconfigs["admin"]) + if err != nil { + return err + } + + if err := kubemaster.CreateDiscoveryDeploymentAndSecret(params, client, caCert); err != nil { + return err + } + + if err := kubemaster.CreateEssentialAddons(params, client); err != nil { + return err + } + + // TODO use templates to reference struct fields directly as order of args is fragile + fmt.Fprintf(out, init_done_msgf, + params.Discovery.GivenToken, + params.Discovery.ListenIP, + ) + + return nil +} diff --git a/pkg/kubeadm/cmd/join.go b/pkg/kubeadm/cmd/join.go new file mode 100644 index 0000000000..5c8f7a1ffb --- /dev/null +++ b/pkg/kubeadm/cmd/join.go @@ -0,0 +1,87 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "io" + + "github.com/renstrom/dedent" + "github.com/spf13/cobra" + + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubenode "k8s.io/kubernetes/pkg/kubeadm/node" + kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" +) + +var ( + join_done_msgf = dedent.Dedent(` + Node join complete: + * Certificate signing request sent to master and response + received. + * Kubelet informed of new secure connection details. + + Run 'kubectl get nodes' on the master to see this node join. + `) +) + +func NewCmdJoin(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { + cmd := &cobra.Command{ + Use: "join", + Short: "Run this on other servers to join an existing cluster.", + Run: func(cmd *cobra.Command, args []string) { + err := RunJoin(out, cmd, args, params) + cmdutil.CheckErr(err) + }, + } + + // TODO this should become `kubeadm join --token=<...> ` + cmd.PersistentFlags().StringVarP(¶ms.Discovery.ApiServerURLs, "api-server-urls", "", "", + `Comma separated list of API server URLs. Typically this might be just + https://:8080/`) + cmd.PersistentFlags().StringVarP(¶ms.Discovery.GivenToken, "token", "", "", + `Shared secret used to secure bootstrap. Must match output of 'init-master'.`) + + return cmd +} + +func RunJoin(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error { + ok, err := kubeadmutil.UseGivenTokenIfValid(params) + if !ok { + if err != nil { + return fmt.Errorf("%s (see --help)\n", err) + } + return fmt.Errorf("Must specify --token (see --help)\n") + } + if params.Discovery.ApiServerURLs == "" { + return fmt.Errorf("Must specify --api-server-urls (see --help)\n") + } + + kubeconfig, err := kubenode.RetrieveTrustedClusterInfo(params) + if err != nil { + return err + } + + err = kubeadmutil.WriteKubeconfigIfNotExists(params, "kubelet", kubeconfig) + if err != nil { + return err + } + + fmt.Fprintf(out, join_done_msgf) + return nil +} diff --git a/pkg/kubeadm/cmd/manual.go b/pkg/kubeadm/cmd/manual.go new file mode 100644 index 0000000000..05902986f6 --- /dev/null +++ b/pkg/kubeadm/cmd/manual.go @@ -0,0 +1,195 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "io" + + "github.com/renstrom/dedent" + "github.com/spf13/cobra" + + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubemaster "k8s.io/kubernetes/pkg/kubeadm/master" + kubenode "k8s.io/kubernetes/pkg/kubeadm/node" + kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + netutil "k8s.io/kubernetes/pkg/util/net" + // TODO: cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" +) + +var ( + manual_done_msgf = dedent.Dedent(` + Master initialization complete: + + * Static pods written and kubelet's kubeconfig written. + * Kubelet should start soon. Try 'systemctl restart kubelet' + or equivalent if it doesn't. + + CA cert is written to: + /etc/kubernetes/pki/ca.pem. + + **Please copy this file (scp, rsync or through other means) to + all your nodes and then run on them**: + + kubeadm manual bootstrap join-node --ca-cert-file \ + --token %s --api-server-urls https://%s:443/ + `) +) + +// TODO --token here becomes Discovery.BearerToken and not Discovery.GivenToken +// may be we should make it the same and ask user to pass dot-separated tokens +// in any of the modes; we could also enable discovery API in the manual mode just +// as well, there is no reason we shouldn't let user mix and match modes, unless +// it is too difficult to support + +func NewCmdManual(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { + cmd := &cobra.Command{ + Use: "manual", + Short: "Advanced, less-automated functionality, for power users.", + // TODO put example usage in the Long description here + } + cmd.AddCommand(NewCmdManualBootstrap(out, params)) + return cmd +} + +func NewCmdManualBootstrap(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { + cmd := &cobra.Command{ + Use: "bootstrap", + Short: "Manually bootstrap a cluster 'out-of-band'", + Long: dedent.Dedent(` + Manually bootstrap a cluster 'out-of-band', by generating and distributing a CA + certificate to all your servers and specifying and (list of) API server URLs. + `), + Run: func(cmd *cobra.Command, args []string) { + }, + } + cmd.AddCommand(NewCmdManualBootstrapInitMaster(out, params)) + cmd.AddCommand(NewCmdManualBootstrapJoinNode(out, params)) + + return cmd +} + +func NewCmdManualBootstrapInitMaster(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { + cmd := &cobra.Command{ + Use: "init-master", + Short: "Manually bootstrap a master 'out-of-band'", + Long: dedent.Dedent(` + Manually bootstrap a master 'out-of-band'. + Will create TLS certificates and set up static pods for Kubernetes master + components. + `), + RunE: func(cmd *cobra.Command, args []string) error { + if params.Discovery.ListenIP == "" { + ip, err := netutil.ChooseHostInterface() + if err != nil { + return fmt.Errorf("Unable to autodetect IP address [%s], please specify with --listen-ip", err) + } + params.Discovery.ListenIP = ip + } + if err := kubemaster.CreateTokenAuthFile(params); err != nil { + return err + } + if err := kubemaster.WriteStaticPodManifests(params); err != nil { + return err + } + caKey, caCert, err := kubemaster.CreatePKIAssets(params) + if err != nil { + return err + } + kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(params, []string{"kubelet", "admin"}, caKey, caCert) + if err != nil { + return err + } + for name, kubeconfig := range kubeconfigs { + if err := kubeadmutil.WriteKubeconfigIfNotExists(params, name, kubeconfig); err != nil { + out.Write([]byte(fmt.Sprintf("Unable to write admin for master:\n%s\n", err))) + return nil + } + } + + // TODO use templates to reference struct fields directly as order of args is fragile + fmt.Fprintf(out, manual_done_msgf, + params.Discovery.BearerToken, + params.Discovery.ListenIP, + ) + + return nil + }, + } + + params.Discovery.ApiServerURLs = "http://127.0.0.1:8080/" // On the master, assume you can talk to the API server + cmd.PersistentFlags().StringVarP(¶ms.Discovery.ApiServerDNSName, "api-dns-name", "", "", + `(optional) DNS name for the API server, will be encoded into + subjectAltName in the resulting (generated) TLS certificates`) + cmd.PersistentFlags().StringVarP(¶ms.Discovery.ListenIP, "listen-ip", "", "", + `(optional) IP address to listen on, in case autodetection fails.`) + cmd.PersistentFlags().StringVarP(¶ms.Discovery.BearerToken, "token", "", "", + `(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`) + + return cmd +} + +func NewCmdManualBootstrapJoinNode(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { + cmd := &cobra.Command{ + Use: "join-node", + Short: "Manually bootstrap a node 'out-of-band', joining it into a cluster with extant control plane", + + Run: func(cmd *cobra.Command, args []string) { + if params.Discovery.CaCertFile == "" { + out.Write([]byte(fmt.Sprintf("Must specify --ca-cert-file (see --help)\n"))) + return + } + + if params.Discovery.ApiServerURLs == "" { + out.Write([]byte(fmt.Sprintf("Must specify --api-server-urls (see --help)\n"))) + return + } + + kubeconfig, err := kubenode.PerformTLSBootstrapFromParams(params) + if err != nil { + out.Write([]byte(fmt.Sprintf("Failed to perform TLS bootstrap: %s\n", err))) + return + } + //fmt.Println("recieved signed certificate from the API server, will write `/etc/kubernetes/kubelet.conf`...") + + err = kubeadmutil.WriteKubeconfigIfNotExists(params, "kubelet", kubeconfig) + if err != nil { + out.Write([]byte(fmt.Sprintf("Unable to write config for node:\n%s\n", err))) + return + } + out.Write([]byte(dedent.Dedent(` + Node join complete: + * Certificate signing request sent to master and response + received. + * Kubelet informed of new secure connection details. + + Run 'kubectl get nodes' on the master to see this node join. + + `))) + }, + } + cmd.PersistentFlags().StringVarP(¶ms.Discovery.CaCertFile, "ca-cert-file", "", "", + `Path to a CA cert file in PEM format. The same CA cert must be distributed to + all servers.`) + cmd.PersistentFlags().StringVarP(¶ms.Discovery.ApiServerURLs, "api-server-urls", "", "", + `Comma separated list of API server URLs. Typically this might be just + https://:8080/`) + cmd.PersistentFlags().StringVarP(¶ms.Discovery.BearerToken, "token", "", "", + `Shared secret used to secure bootstrap. Must match output of 'init-master'.`) + + return cmd +} diff --git a/pkg/kubeadm/cmd/user.go b/pkg/kubeadm/cmd/user.go new file mode 100644 index 0000000000..3d30f50d6e --- /dev/null +++ b/pkg/kubeadm/cmd/user.go @@ -0,0 +1,34 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "io" + + "github.com/spf13/cobra" + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" +) + +func NewCmdUser(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { + cmd := &cobra.Command{ + Use: "user", + Short: "Get initial admin credentials for a cluster.", // using TLS bootstrap + Run: func(cmd *cobra.Command, args []string) { + }, + } + return cmd +} diff --git a/pkg/kubeadm/master/addons.go b/pkg/kubeadm/master/addons.go new file mode 100644 index 0000000000..97911ede83 --- /dev/null +++ b/pkg/kubeadm/master/addons.go @@ -0,0 +1,93 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package kubemaster + +import ( + "fmt" + "path" + + "k8s.io/kubernetes/pkg/api" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" +) + +func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { + privilegedTrue := true + return api.PodSpec{ + SecurityContext: &api.PodSecurityContext{HostNetwork: true}, + Containers: []api.Container{{ + Name: "kube-proxy", + Image: params.EnvParams["hyperkube_image"], + Command: []string{ + "/hyperkube", + "proxy", + "--kubeconfig=/run/kubeconfig", + COMPONENT_LOGLEVEL, + }, + SecurityContext: &api.SecurityContext{Privileged: &privilegedTrue}, + VolumeMounts: []api.VolumeMount{ + { + Name: "dbus", + MountPath: "/var/run/dbus", + ReadOnly: false, + }, + { + // TODO there are handful of clever options to get around this, but it's + // easier to just mount kubelet's config here; we should probably just + // make sure that proxy reads the token and CA cert from /run/secrets + // and accepts `--master` at the same time + // + // clever options include: + // - do CSR dance and create kubeconfig and mount it as secrete + // - create a service account with a second secret enconding kubeconfig + // - use init container to convert known information to kubeconfig + // - ...whatever + Name: "kubeconfig", + MountPath: "/run/kubeconfig", + ReadOnly: false, + }, + }, + }}, + Volumes: []api.Volume{ + { + Name: "kubeconfig", + VolumeSource: api.VolumeSource{ + HostPath: &api.HostPathVolumeSource{Path: path.Join(params.EnvParams["prefix"], "kubelet.conf")}, + }, + }, + { + Name: "dbus", + VolumeSource: api.VolumeSource{ + HostPath: &api.HostPathVolumeSource{Path: "/var/run/dbus"}, + }, + }, + }, + } +} + +func CreateEssentialAddons(params *kubeadmapi.BootstrapParams, client *clientset.Clientset) error { + kubeProxyDaemonSet := NewDaemonSet("kube-proxy", createKubeProxyPodSpec(params)) + + if _, err := client.Extensions().DaemonSets(api.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil { + return fmt.Errorf(" failed creating essential kube-proxy addon [%s]", err) + } + + fmt.Println(" created essential addon: kube-proxy") + + // TODO should we wait for it to become ready at least on the master? + + return nil +} diff --git a/pkg/kubeadm/master/apiclient.go b/pkg/kubeadm/master/apiclient.go new file mode 100644 index 0000000000..4a2d404d9b --- /dev/null +++ b/pkg/kubeadm/master/apiclient.go @@ -0,0 +1,111 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubemaster + +import ( + "fmt" + "time" + + "k8s.io/kubernetes/pkg/api" + unversionedapi "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/apis/extensions" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" + clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" + "k8s.io/kubernetes/pkg/util/wait" +) + +func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Clientset, error) { + adminClientConfig, err := clientcmd.NewDefaultClientConfig( + *adminConfig, + &clientcmd.ConfigOverrides{}, + ).ClientConfig() + if err != nil { + return nil, fmt.Errorf(" failed to create API client configuration [%s]", err) + } + + fmt.Println(" created API client configuration") + + client, err := clientset.NewForConfig(adminClientConfig) + if err != nil { + return nil, fmt.Errorf(" failed to create API client [%s]", err) + } + + fmt.Println(" created API client, waiting for the control plane to become ready") + + start := time.Now() + wait.PollInfinite(500*time.Millisecond, func() (bool, error) { + cs, err := client.ComponentStatuses().List(api.ListOptions{}) + if err != nil { + return false, nil + } + if len(cs.Items) < 3 { + fmt.Println(" not all control plane components are ready yet") + return false, nil + } + for _, item := range cs.Items { + for _, condition := range item.Conditions { + if condition.Type != api.ComponentHealthy { + fmt.Printf(" control plane component %q is still unhealthy: %#v\n", item.ObjectMeta.Name, item.Conditions) + return false, nil + } + } + } + + fmt.Printf(" all control plane components are healthy after %s seconds\n", time.Since(start).Seconds()) + return true, nil + }) + + // TODO may be also check node status + return client, nil +} + +func NewDaemonSet(daemonName string, podSpec api.PodSpec) *extensions.DaemonSet { + l := map[string]string{"component": daemonName, "tier": "node"} + return &extensions.DaemonSet{ + ObjectMeta: api.ObjectMeta{Name: daemonName}, + Spec: extensions.DaemonSetSpec{ + Selector: &unversionedapi.LabelSelector{MatchLabels: l}, + Template: api.PodTemplateSpec{ + ObjectMeta: api.ObjectMeta{Labels: l}, + Spec: podSpec, + }, + }, + } +} + +func NewDeployment(deploymentName string, replicas int32, podSpec api.PodSpec) *extensions.Deployment { + l := map[string]string{"name": deploymentName} + return &extensions.Deployment{ + ObjectMeta: api.ObjectMeta{Name: deploymentName}, + Spec: extensions.DeploymentSpec{ + Replicas: replicas, + Selector: &unversionedapi.LabelSelector{MatchLabels: l}, + Template: api.PodTemplateSpec{ + ObjectMeta: api.ObjectMeta{Labels: l}, + Spec: podSpec, + }, + }, + } +} + +func TaintMaster(*clientset.Clientset) error { + // TODO + annotations := make(map[string]string) + annotations[api.TaintsAnnotationKey] = "" + return nil +} diff --git a/pkg/kubeadm/master/discovery.go b/pkg/kubeadm/master/discovery.go new file mode 100644 index 0000000000..76caac9009 --- /dev/null +++ b/pkg/kubeadm/master/discovery.go @@ -0,0 +1,111 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package kubemaster + +import ( + "crypto/x509" + "encoding/hex" + "encoding/json" + "fmt" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/apis/extensions" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + certutil "k8s.io/kubernetes/pkg/util/cert" +) + +type kubeDiscovery struct { + Deployment *extensions.Deployment + Secret *api.Secret +} + +const ( + kubeDiscoverynName = "kube-discovery" + kubeDiscoverySecretName = "clusterinfo" +) + +func encodeKubeDiscoverySecretData(params *kubeadmapi.BootstrapParams, caCert *x509.Certificate) map[string][]byte { + // TODO ListenIP is probably not the right now, although it's best we have right now + // if user provides a DNS name, or anything else, we should use that, may be it's really + // the list of all SANs (minus internal DNS names and service IP)? + + var ( + data = map[string][]byte{} + endpointList = []string{} + tokenMap = map[string]string{} + ) + + endpointList = append(endpointList, fmt.Sprintf("https://%s:443", params.Discovery.ListenIP)) + tokenMap[params.Discovery.TokenID] = hex.EncodeToString(params.Discovery.Token) + + data["endpoint-list.json"], _ = json.Marshal(endpointList) + data["token-map.json"], _ = json.Marshal(tokenMap) + data["ca.pem"] = certutil.EncodeCertPEM(caCert) + + return data +} + +func newKubeDiscoveryPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { + return api.PodSpec{ + SecurityContext: &api.PodSecurityContext{HostNetwork: true}, // TODO we should just use map it to a host port + Containers: []api.Container{{ + Name: kubeDiscoverynName, + Image: params.EnvParams["discovery_image"], + Command: []string{"/usr/bin/kube-discovery"}, + VolumeMounts: []api.VolumeMount{{ + Name: kubeDiscoverySecretName, + MountPath: "/tmp/secret", // TODO use a shared constant + ReadOnly: true, + }}, + }}, + Volumes: []api.Volume{{ + Name: kubeDiscoverySecretName, + VolumeSource: api.VolumeSource{ + Secret: &api.SecretVolumeSource{SecretName: kubeDiscoverySecretName}, + }}, + }, + } +} + +func newKubeDiscovery(params *kubeadmapi.BootstrapParams, caCert *x509.Certificate) kubeDiscovery { + // TODO pin to master + return kubeDiscovery{ + Deployment: NewDeployment(kubeDiscoverynName, 1, newKubeDiscoveryPodSpec(params)), + Secret: &api.Secret{ + ObjectMeta: api.ObjectMeta{Name: kubeDiscoverySecretName}, + Type: api.SecretTypeOpaque, + Data: encodeKubeDiscoverySecretData(params, caCert), + }, + } +} + +func CreateDiscoveryDeploymentAndSecret(params *kubeadmapi.BootstrapParams, client *clientset.Clientset, caCert *x509.Certificate) error { + kd := newKubeDiscovery(params, caCert) + + if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kd.Deployment); err != nil { + return fmt.Errorf(" failed to create %q deployment", kubeDiscoverynName) + } + if _, err := client.Secrets(api.NamespaceSystem).Create(kd.Secret); err != nil { + return fmt.Errorf(" failed to create %q secret", kubeDiscoverySecretName) + } + + fmt.Println(" created essential addon: kube-discovery") + + // TODO we should probably wait for the pod to become ready + + return nil +} diff --git a/pkg/kubeadm/master/kubeconfig.go b/pkg/kubeadm/master/kubeconfig.go new file mode 100644 index 0000000000..9cd9506310 --- /dev/null +++ b/pkg/kubeadm/master/kubeconfig.go @@ -0,0 +1,57 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubemaster + +import ( + "crypto/rsa" + "crypto/x509" + "fmt" + + // TODO: "k8s.io/client-go/client/tools/clientcmd/api" + clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + certutil "k8s.io/kubernetes/pkg/util/cert" +) + +func CreateCertsAndConfigForClients(params *kubeadmapi.BootstrapParams, clientNames []string, caKey *rsa.PrivateKey, caCert *x509.Certificate) (map[string]*clientcmdapi.Config, error) { + + basicClientConfig := kubeadmutil.CreateBasicClientConfig( + "kubernetes", + fmt.Sprintf("https://%s:443", params.Discovery.ListenIP), + certutil.EncodeCertPEM(caCert), + ) + + configs := map[string]*clientcmdapi.Config{} + + for _, client := range clientNames { + key, cert, err := newClientKeyAndCert(caCert, caKey) + if err != nil { + return nil, fmt.Errorf(" failure while creating %s client certificate - %s", client, err) + } + config := kubeadmutil.MakeClientConfigWithCerts( + basicClientConfig, + "kubernetes", + client, + certutil.EncodePrivateKeyPEM(key), + certutil.EncodeCertPEM(cert), + ) + configs[client] = config + } + + return configs, nil +} diff --git a/pkg/kubeadm/master/manifests.go b/pkg/kubeadm/master/manifests.go new file mode 100644 index 0000000000..b8810be2b6 --- /dev/null +++ b/pkg/kubeadm/master/manifests.go @@ -0,0 +1,196 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubemaster + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path" + + "k8s.io/kubernetes/pkg/api/resource" + "k8s.io/kubernetes/pkg/api/unversioned" + api "k8s.io/kubernetes/pkg/api/v1" + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/util/intstr" +) + +// Static pod definitions in golang form are included below so that `kubeadm +// init master` and `kubeadm manual bootstrap master` can get going. + +const ( + COMPONENT_LOGLEVEL = "--v=4" + SERVICE_CLUSTER_IP_RANGE = "--service-cluster-ip-range=10.16.0.0/12" + CLUSTER_NAME = "--cluster-name=kubernetes" + MASTER = "--master=127.0.0.1:8080" +) + +// TODO look into what this really means, scheduler prints it for some reason +// +//E0817 17:53:22.242658 1 event.go:258] Could not construct reference to: '&api.Endpoints{TypeMeta:unversioned.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:api.ObjectMeta{Name:"kube-scheduler", GenerateName:"", Namespace:"kube-system", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:unversioned.Time{Time:time.Time{sec:0, nsec:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*unversioned.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]api.OwnerReference(nil), Finalizers:[]string(nil)}, Subsets:[]api.EndpointSubset(nil)}' due to: 'selfLink was empty, can't make reference'. Will not report event: 'Normal' '%v became leader' 'moby' + +func WriteStaticPodManifests(params *kubeadmapi.BootstrapParams) error { + staticPodSpecs := map[string]api.Pod{ + // TODO this needs a volume + "etcd": componentPod(api.Container{ + Command: []string{ + "/usr/local/bin/etcd", + "--listen-client-urls=http://127.0.0.1:2379,http://127.0.0.1:4001", + "--advertise-client-urls=http://127.0.0.1:2379,http://127.0.0.1:4001", + "--data-dir=/var/etcd/data", + }, + Image: "gcr.io/google_containers/etcd:2.2.1", // TODO parametrise + LivenessProbe: componentProbe(2379, "/health"), + Name: "etcd-server", + Resources: componentResources("200m"), + }), + // TODO bind-mount certs in + "kube-apiserver": componentPod(api.Container{ + Name: "kube-apiserver", + Image: params.EnvParams["hyperkube_image"], + Command: []string{ + "/hyperkube", + "apiserver", + "--address=127.0.0.1", + "--etcd-servers=http://127.0.0.1:2379", + "--cloud-provider=fake", // TODO parametrise + "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota", + SERVICE_CLUSTER_IP_RANGE, + "--service-account-key-file=/etc/kubernetes/pki/apiserver-key.pem", + "--client-ca-file=/etc/kubernetes/pki/ca.pem", + "--tls-cert-file=/etc/kubernetes/pki/apiserver.pem", + "--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", + "--secure-port=443", + "--allow-privileged", + COMPONENT_LOGLEVEL, + "--token-auth-file=/etc/kubernetes/pki/tokens.csv", + }, + VolumeMounts: []api.VolumeMount{pkiVolumeMount()}, + LivenessProbe: componentProbe(8080, "/healthz"), + Resources: componentResources("250m"), + }, pkiVolume(params)), + "kube-controller-manager": componentPod(api.Container{ + Name: "kube-controller-manager", + Image: params.EnvParams["hyperkube_image"], + Command: []string{ + "/hyperkube", + "controller-manager", + "--leader-elect", + MASTER, + CLUSTER_NAME, + "--root-ca-file=/etc/kubernetes/pki/ca.pem", + "--service-account-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", + "--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem", + "--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem", + "--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap", + COMPONENT_LOGLEVEL, + }, + VolumeMounts: []api.VolumeMount{pkiVolumeMount()}, + LivenessProbe: componentProbe(10252, "/healthz"), + Resources: componentResources("200m"), + }, pkiVolume(params)), + "kube-scheduler": componentPod(api.Container{ + Name: "kube-scheduler", + Image: params.EnvParams["hyperkube_image"], + Command: []string{ + "/hyperkube", + "scheduler", + "--leader-elect", + MASTER, + COMPONENT_LOGLEVEL, + }, + LivenessProbe: componentProbe(10251, "/healthz"), + Resources: componentResources("100m"), + }), + } + + manifestsPath := path.Join(params.EnvParams["prefix"], "manifests") + if err := os.MkdirAll(manifestsPath, 0700); err != nil { + return fmt.Errorf(" failed to create directory %q [%s]", manifestsPath, err) + } + for name, spec := range staticPodSpecs { + filename := path.Join(manifestsPath, name+".json") + serialized, err := json.MarshalIndent(spec, "", " ") + if err != nil { + return fmt.Errorf(" failed to marshall manifest for %q to JSON [%s]", name, err) + } + if err := cmdutil.DumpReaderToFile(bytes.NewReader(serialized), filename); err != nil { + return fmt.Errorf(" failed to create static pod manifest file for %q (%q) [%s]", name, filename, err) + } + } + return nil +} + +func pkiVolume(params *kubeadmapi.BootstrapParams) api.Volume { + return api.Volume{ + Name: "pki", + VolumeSource: api.VolumeSource{ + HostPath: &api.HostPathVolumeSource{Path: params.EnvParams["host_pki_path"]}, + }, + } +} + +func pkiVolumeMount() api.VolumeMount { + return api.VolumeMount{ + Name: "pki", + MountPath: "/etc/kubernetes/pki", + ReadOnly: true, + } +} + +func componentResources(cpu string) api.ResourceRequirements { + return api.ResourceRequirements{ + Requests: api.ResourceList{ + api.ResourceName(api.ResourceCPU): resource.MustParse(cpu), + }, + } +} + +func componentProbe(port int, path string) *api.Probe { + return &api.Probe{ + Handler: api.Handler{ + HTTPGet: &api.HTTPGetAction{ + Host: "127.0.0.1", + Path: path, + Port: intstr.FromInt(port), + }, + }, + InitialDelaySeconds: 15, + TimeoutSeconds: 15, + } +} + +func componentPod(container api.Container, volumes ...api.Volume) api.Pod { + return api.Pod{ + TypeMeta: unversioned.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: api.ObjectMeta{ + Name: container.Name, + Namespace: "kube-system", + Labels: map[string]string{"component": container.Name, "tier": "control-plane"}, + }, + Spec: api.PodSpec{ + Containers: []api.Container{container}, + HostNetwork: true, + Volumes: volumes, + }, + } +} diff --git a/pkg/kubeadm/master/pki.go b/pkg/kubeadm/master/pki.go new file mode 100644 index 0000000000..6c23c90a5d --- /dev/null +++ b/pkg/kubeadm/master/pki.go @@ -0,0 +1,180 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubemaster + +import ( + "crypto/rsa" + "crypto/x509" + "fmt" + "net" + "path" + + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + certutil "k8s.io/kubernetes/pkg/util/cert" +) + +/* +func errorf(f string, err error, vargs ...string) error { + return fmt.Errorf(" %s [%s]", fmt.Sprintf(f, v...), err) +} +*/ + +func newCertificateAuthority() (*rsa.PrivateKey, *x509.Certificate, error) { + key, err := certutil.NewPrivateKey() + if err != nil { + return nil, nil, fmt.Errorf("unable to create private key [%s]", err) + } + + config := certutil.CertConfig{ + CommonName: "kubernetes", + } + + cert, err := certutil.NewSelfSignedCACert(config, key) + if err != nil { + return nil, nil, fmt.Errorf("unable to create self-singed certificate [%s]", err) + } + + return key, cert, nil +} + +func newServerKeyAndCert(caCert *x509.Certificate, caKey *rsa.PrivateKey, altNames certutil.AltNames) (*rsa.PrivateKey, *x509.Certificate, error) { + key, err := certutil.NewPrivateKey() + if err != nil { + return nil, nil, fmt.Errorf("unabel to create private key [%s]", err) + } + // TODO these are all hardcoded for now, but we need to figure out what shall we do here exactly + altNames.IPs = append(altNames.IPs, net.ParseIP("10.3.0.1")) + altNames.DNSNames = append(altNames.DNSNames, + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.local", + ) + + config := certutil.CertConfig{ + CommonName: "kube-apiserver", + AltNames: altNames, + } + cert, err := certutil.NewSignedCert(config, key, caCert, caKey) + if err != nil { + return nil, nil, fmt.Errorf("unable to sing certificate [%s]", err) + } + + return key, cert, nil +} + +func newClientKeyAndCert(caCert *x509.Certificate, caKey *rsa.PrivateKey) (*rsa.PrivateKey, *x509.Certificate, error) { + key, err := certutil.NewPrivateKey() + if err != nil { + return nil, nil, fmt.Errorf("unable to create private key [%s]", err) + } + + config := certutil.CertConfig{ + CommonName: "kubernetes-admin", + } + cert, err := certutil.NewSignedCert(config, key, caCert, caKey) + if err != nil { + return nil, nil, fmt.Errorf("unable to sign certificate [%s]", err) + } + + return key, cert, nil +} + +func writeKeysAndCert(pkiPath string, name string, key *rsa.PrivateKey, cert *x509.Certificate) error { + var ( + publicKeyPath = path.Join(pkiPath, fmt.Sprintf("%s-pub.pem", name)) + privateKeyPath = path.Join(pkiPath, fmt.Sprintf("%s-key.pem", name)) + certificatePath = path.Join(pkiPath, fmt.Sprintf("%s.pem", name)) + ) + + if key != nil { + if err := certutil.WriteKey(privateKeyPath, certutil.EncodePrivateKeyPEM(key)); err != nil { + return fmt.Errorf("unable to write private key file (%q) [%s]", privateKeyPath, err) + } + if pubKey, err := certutil.EncodePublicKeyPEM(&key.PublicKey); err == nil { + if err := certutil.WriteKey(publicKeyPath, pubKey); err != nil { + return fmt.Errorf("unable to write public key file (%q) [%s]", publicKeyPath, err) + } + } else { + return fmt.Errorf("unable to encode public key to PEM [%s]", err) + } + } + + if cert != nil { + if err := certutil.WriteCert(certificatePath, certutil.EncodeCertPEM(cert)); err != nil { + return fmt.Errorf("unable to write certificate file (%q) [%s]", err) + } + } + + return nil +} + +func newServiceAccountKey() (*rsa.PrivateKey, error) { + key, err := certutil.NewPrivateKey() + if err != nil { + return nil, err + } + return key, nil +} + +func CreatePKIAssets(params *kubeadmapi.BootstrapParams) (*rsa.PrivateKey, *x509.Certificate, error) { + var ( + err error + altNames certutil.AltNames // TODO actual SANs + ) + + if params.Discovery.ListenIP != "" { + altNames.IPs = append(altNames.IPs, net.ParseIP(params.Discovery.ListenIP)) + } + + if params.Discovery.ApiServerDNSName != "" { + altNames.DNSNames = append(altNames.DNSNames, params.Discovery.ApiServerDNSName) + } + + pkiPath := path.Join(params.EnvParams["host_pki_path"]) + + caKey, caCert, err := newCertificateAuthority() + if err != nil { + return nil, nil, fmt.Errorf(" failure while creating CA keys and certificate - %s", err) + } + + if err := writeKeysAndCert(pkiPath, "ca", caKey, caCert); err != nil { + return nil, nil, fmt.Errorf(" failure while saving CA keys and certificate - %s", err) + } + + apiKey, apiCert, err := newServerKeyAndCert(caCert, caKey, altNames) + if err != nil { + return nil, nil, fmt.Errorf(" failure while creating API server keys and certificate - %s", err) + } + + if err := writeKeysAndCert(pkiPath, "apiserver", apiKey, apiCert); err != nil { + return nil, nil, fmt.Errorf(" failure while saving API server keys and certificate - %s", err) + } + + saKey, err := newServiceAccountKey() + if err != nil { + return nil, nil, fmt.Errorf(" failure while creating service account signing keys [%s]", err) + } + + if err := writeKeysAndCert(pkiPath, "sa", saKey, nil); err != nil { + return nil, nil, fmt.Errorf(" failure while saving service account singing keys - %s", err) + } + + // TODO print a summary of SANs used and checksums (signatures) of each of the certiicates + fmt.Println(" created keys and certificates in %q", params.EnvParams["host_pki_path"]) + return caKey, caCert, nil +} diff --git a/pkg/kubeadm/master/tokens.go b/pkg/kubeadm/master/tokens.go new file mode 100644 index 0000000000..81d899807b --- /dev/null +++ b/pkg/kubeadm/master/tokens.go @@ -0,0 +1,59 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package kubemaster + +import ( + "bytes" + "fmt" + "os" + "path" + + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/util/uuid" +) + +func generateTokenIfNeeded(params *kubeadmapi.BootstrapParams) error { + ok, err := kubeadmutil.UseGivenTokenIfValid(params) + if !ok { + if err != nil { + return err + } + err = kubeadmutil.GenerateToken(params) + if err != nil { + return err + } + fmt.Printf(" generated token: %q\n", params.Discovery.GivenToken) + } + + return nil +} + +func CreateTokenAuthFile(params *kubeadmapi.BootstrapParams) error { + tokenAuthFilePath := path.Join(params.EnvParams["host_pki_path"], "tokens.csv") + if err := generateTokenIfNeeded(params); err != nil { + return fmt.Errorf(" failed to generate token(s) [%s]", err) + } + if err := os.MkdirAll(path.Join(params.EnvParams["host_pki_path"]), 0700); err != nil { + return fmt.Errorf(" failed to create directory %q [%s]", params.EnvParams["host_pki_path"], err) + } + serialized := []byte(fmt.Sprintf("%s,kubeadm-node-csr,%s,system:kubelet-bootstrap\n", params.Discovery.BearerToken, uuid.NewUUID())) + if err := cmdutil.DumpReaderToFile(bytes.NewReader(serialized), tokenAuthFilePath); err != nil { + return fmt.Errorf(" failed to save token auth file (%q) [%s]", tokenAuthFilePath, err) + } + return nil +} diff --git a/pkg/kubeadm/node/csr.go b/pkg/kubeadm/node/csr.go new file mode 100644 index 0000000000..e1da517d59 --- /dev/null +++ b/pkg/kubeadm/node/csr.go @@ -0,0 +1,90 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubenode + +import ( + "fmt" + "io/ioutil" + "strings" + + unversionedcertificates "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/certificates/unversioned" + "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" + clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + kubeadmutil "k8s.io/kubernetes/pkg/kubeadm/util" + "k8s.io/kubernetes/pkg/kubelet/util/csr" + certutil "k8s.io/kubernetes/pkg/util/cert" +) + +func getNodeName() string { + return "TODO" +} + +func PerformTLSBootstrapFromParams(params *kubeadmapi.BootstrapParams) (*clientcmdapi.Config, error) { + caCert, err := ioutil.ReadFile(params.Discovery.CaCertFile) + if err != nil { + return nil, fmt.Errorf(" failed to load CA certificate [%s]", err) + } + + return PerformTLSBootstrap(params, strings.Split(params.Discovery.ApiServerURLs, ",")[0], caCert) +} + +// Create a restful client for doing the certificate signing request. +func PerformTLSBootstrap(params *kubeadmapi.BootstrapParams, apiEndpoint string, caCert []byte) (*clientcmdapi.Config, error) { + // TODO try all the api servers until we find one that works + bareClientConfig := kubeadmutil.CreateBasicClientConfig("kubernetes", apiEndpoint, caCert) + + nodeName := getNodeName() + + bootstrapClientConfig, err := clientcmd.NewDefaultClientConfig( + *kubeadmutil.MakeClientConfigWithToken( + bareClientConfig, "kubernetes", fmt.Sprintf("kubelet-%s", nodeName), params.Discovery.BearerToken, + ), + &clientcmd.ConfigOverrides{}, + ).ClientConfig() + if err != nil { + return nil, fmt.Errorf(" failed to create API client configuration [%s]", err) + } + + client, err := unversionedcertificates.NewForConfig(bootstrapClientConfig) + if err != nil { + return nil, fmt.Errorf(" failed to create API client [%s]", err) + } + csrClient := client.CertificateSigningRequests() + + fmt.Println(" created API client to obtain unique certificate for this node, generating keys and certificate signing request") + + key, err := certutil.MakeEllipticPrivateKeyPEM() + if err != nil { + return nil, fmt.Errorf(" failed to generating private key [%s]", err) + } + + cert, err := csr.RequestNodeCertificate(csrClient, key, nodeName) + if err != nil { + return nil, fmt.Errorf(" failed to request signed certificate from the API server [%s]", err) + } + + // TODO print some basic info about the cert + fmt.Println(" received signed certificate from the API server, generating kubelet configuration") + + finalConfig := kubeadmutil.MakeClientConfigWithCerts( + bareClientConfig, "kubernetes", fmt.Sprintf("kubelet-%s", nodeName), + key, cert, + ) + + return finalConfig, nil +} diff --git a/pkg/kubeadm/node/discovery.go b/pkg/kubeadm/node/discovery.go new file mode 100644 index 0000000000..9609905b65 --- /dev/null +++ b/pkg/kubeadm/node/discovery.go @@ -0,0 +1,87 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package kubenode + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/square/go-jose" + clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" +) + +func RetrieveTrustedClusterInfo(params *kubeadmapi.BootstrapParams) (*clientcmdapi.Config, error) { + firstURL := strings.Split(params.Discovery.ApiServerURLs, ",")[0] // TODO obviously we should do something better.. . + apiServerURL, err := url.Parse(firstURL) + if err != nil { + return nil, fmt.Errorf(" failed to parse given API server URL (%q) [%s]", firstURL, err) + } + + host, port := strings.Split(apiServerURL.Host, ":")[0], 9898 // TODO this is too naive + requestURL := fmt.Sprintf("http://%s:%d/cluster-info/v1/?token-id=%s", host, port, params.Discovery.TokenID) + req, err := http.NewRequest("GET", requestURL, nil) + if err != nil { + return nil, fmt.Errorf(" failed to consturct an HTTP request [%s]", err) + } + + fmt.Println(" created cluster info discovery client, requesting info from %q", requestURL) + + res, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf(" failed to request cluster info [%s]", err) + } + buf := new(bytes.Buffer) + io.Copy(buf, res.Body) + res.Body.Close() + + object, err := jose.ParseSigned(buf.String()) + if err != nil { + return nil, fmt.Errorf(" failed to parse response as JWS object [%s]", err) + } + + fmt.Println(" cluster info object recieved, verifying signature using given token") + + output, err := object.Verify(params.Discovery.Token) + if err != nil { + return nil, fmt.Errorf(" failed to verify JWS signature of recieved cluster info object [%s]", err) + } + + clusterInfo := kubeadmapi.ClusterInfo{} + + if err := json.Unmarshal(output, &clusterInfo); err != nil { + return nil, fmt.Errorf(" failed to unmarshal recieved cluster info object [%s]", err) + } + + if len(clusterInfo.CertificateAuthorities) == 0 || len(clusterInfo.Endpoints) == 0 { + return nil, fmt.Errorf(" cluster info object is invalid - no endpoint(s) and/or root CA certificate(s) found") + } + + // TODO print checksum of the CA certificate + fmt.Printf(" cluser info signature and contents are valid, will use API endpoints %v\n", clusterInfo.Endpoints) + + // TODO we need to configure the client to validate the server + // if it is signed by any of the returned certificates + apiServer := clusterInfo.Endpoints[0] + caCert := []byte(clusterInfo.CertificateAuthorities[0]) + + return PerformTLSBootstrap(params, apiServer, caCert) +} diff --git a/pkg/kubeadm/util/kubeconfig.go b/pkg/kubeadm/util/kubeconfig.go new file mode 100644 index 0000000000..9c595d5758 --- /dev/null +++ b/pkg/kubeadm/util/kubeconfig.go @@ -0,0 +1,103 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeadmutil + +import ( + "fmt" + "os" + "path" + + // TODO: "k8s.io/client-go/client/tools/clientcmd/api" + "k8s.io/kubernetes/pkg/client/unversioned/clientcmd" + clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" +) + +func CreateBasicClientConfig(clusterName string, serverURL string, caCert []byte) *clientcmdapi.Config { + cluster := clientcmdapi.NewCluster() + cluster.Server = serverURL + cluster.CertificateAuthorityData = caCert + + config := clientcmdapi.NewConfig() + config.Clusters[clusterName] = cluster + + return config +} + +func MakeClientConfigWithCerts(config *clientcmdapi.Config, clusterName string, userName string, clientKey []byte, clientCert []byte) *clientcmdapi.Config { + newConfig := config + name := fmt.Sprintf("%s@%s", userName, clusterName) + + authInfo := clientcmdapi.NewAuthInfo() + authInfo.ClientKeyData = clientKey + authInfo.ClientCertificateData = clientCert + + context := clientcmdapi.NewContext() + context.Cluster = clusterName + context.AuthInfo = userName + + newConfig.AuthInfos[userName] = authInfo + newConfig.Contexts[name] = context + newConfig.CurrentContext = name + + return newConfig +} + +func MakeClientConfigWithToken(config *clientcmdapi.Config, clusterName string, userName string, token string) *clientcmdapi.Config { + newConfig := config + name := fmt.Sprintf("%s@%s", userName, clusterName) + + authInfo := clientcmdapi.NewAuthInfo() + authInfo.Token = token + + context := clientcmdapi.NewContext() + context.Cluster = clusterName + context.AuthInfo = userName + + newConfig.AuthInfos[userName] = authInfo + newConfig.Contexts[name] = context + newConfig.CurrentContext = name + + return newConfig +} + +// kubeadm is responsible for writing the following kubeconfig file, which +// kubelet should be waiting for. Help user avoid foot-shooting by refusing to +// write a file that has already been written (the kubelet will be up and +// running in that case - they'd need to stop the kubelet, remove the file, and +// start it again in that case). + +func WriteKubeconfigIfNotExists(params *kubeadmapi.BootstrapParams, name string, kubeconfig *clientcmdapi.Config) error { + filename := path.Join(params.EnvParams["prefix"], fmt.Sprintf("%s.conf", name)) + // Create and open the file, only if it does not already exist. + f, err := os.OpenFile( + filename, + os.O_CREATE|os.O_WRONLY|os.O_EXCL, + 0600, + ) + if err != nil { + return fmt.Errorf(" failed to create %q, it already exists [%s]", filename, err) + } + f.Close() + + if err := clientcmd.WriteToFile(*kubeconfig, filename); err != nil { + return fmt.Errorf(" failed to write to %q [%s]", filename, err) + } + + fmt.Println(" created %q", filename) + return nil +} diff --git a/pkg/kubeadm/util/tokens.go b/pkg/kubeadm/util/tokens.go new file mode 100644 index 0000000000..91f5f78d5f --- /dev/null +++ b/pkg/kubeadm/util/tokens.go @@ -0,0 +1,91 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package kubeadmutil + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "strings" + + kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" +) + +const ( + TokenIDLen = 6 + TokenBytes = 8 +) + +func randBytes(length int) ([]byte, string, error) { + b := make([]byte, length) + _, err := rand.Read(b) + if err != nil { + return nil, "", err + } + // It's only the tokenID that doesn't care about raw byte slice, + // so we just encoded it in place and ignore bytes slice where we + // do not want it + return b, hex.EncodeToString(b), nil +} + +func GenerateToken(params *kubeadmapi.BootstrapParams) error { + _, tokenID, err := randBytes(TokenIDLen / 2) + if err != nil { + return err + } + + tokenBytes, token, err := randBytes(TokenBytes) + if err != nil { + return err + } + + params.Discovery.TokenID = tokenID + params.Discovery.BearerToken = token + params.Discovery.Token = tokenBytes + params.Discovery.GivenToken = fmt.Sprintf("%s.%s", tokenID, token) + return nil +} + +func UseGivenTokenIfValid(params *kubeadmapi.BootstrapParams) (bool, error) { + if params.Discovery.GivenToken == "" { + return false, nil + } + givenToken := strings.Split(strings.ToLower(params.Discovery.GivenToken), ".") + // TODO print desired format + // TODO could also print more specific messages in each case + invalidErr := " provided token is invalid - %s" + if len(givenToken) != 2 { + return false, fmt.Errorf(invalidErr, "not in 2-part dot-separated format") + } + if len(givenToken[0]) != TokenIDLen { + return false, fmt.Errorf(invalidErr, fmt.Sprintf( + "length of first part is incorrect [%d (given) != %d (expected) ]", + len(givenToken[0]), TokenIDLen)) + } + tokenBytes, err := hex.DecodeString(givenToken[1]) + if err != nil { + return false, fmt.Errorf(invalidErr, err) + } + if len(tokenBytes) != TokenBytes { + return false, fmt.Errorf(invalidErr, fmt.Sprintf( + "length of second part is incorrect [%d (given) != %d (expected)]", + len(tokenBytes), TokenBytes)) + } + params.Discovery.TokenID = givenToken[0] + params.Discovery.BearerToken = givenToken[1] + params.Discovery.Token = tokenBytes + return true, nil // given and valid +}