diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go index 749948d5c3..3869a7057c 100644 --- a/cmd/kubeadm/app/kubeadm.go +++ b/cmd/kubeadm/app/kubeadm.go @@ -46,7 +46,6 @@ func getEnvParams() map[string]string { "discovery_image": "dgoodwin/kubediscovery:latest", // TODO(phase1): fmt.Sprintf("gcr.io/google_containers/kube-discovery-%s:%s", runtime.GOARCH, "1.0"), "etcd_image": "", "component_loglevel": "--v=4", - "dns_domain": "cluster.local", } for k := range envParams { diff --git a/pkg/kubeadm/README.md b/pkg/kubeadm/README.md new file mode 100644 index 0000000000..a43cde6a89 --- /dev/null +++ b/pkg/kubeadm/README.md @@ -0,0 +1,76 @@ +# Kubernetes Cluster Boostrap Made Easy + +## Usage + +### `kubeadm init` + +It's usually enough to run `kubeadm init`, but in some case you might like to override the +default behaviour. + +- `--token=` + +By default, a token is generated, but if you are to automate cluster deployment, you want to +set the token ahead of time. Read the docs for more information on the token format. + +- `--api-advertise-addr=` (multiple values allowed) +- `--api-external-dns-name=` (multiple values allowed) + +By default, `kubeadm` will auto detect IP address and use that to generate API server certificates. +If you would like to access the API via any external IPs and/or DNS, which it might not be able +to detect, you can use `--api-advertise-addr` and `--api-external-dns-name` to add multiple +different IP addresses and DNS names. + +- `--service-cidr=` (default: "100.64/12") +- `--service-dns-domain=` (default: "cluster.local") + +- `--use-hyperkube=` (default: "false") + +***TODO(phase1+)*** + +- `--api-bind-addr=` +- `--api-bind-port=` + +***TODO(phase2)*** + +- `--api-bind-loopback-unsecure=` + +***TODO(pahse2)*** + +- `--prefer-private-network=` +- `--prefer-public-network=` + +### `kubeadm join` + +# User Experience Considerations + +> ***TODO*** _Move this into the design document + +a) `kube-apiserver` will listen on `0.0.0.0:443` and `127.0.0.1:8080`, which is not configurable right now and make things a bit easier for the MVP +b) there is `kube-discovery`, which will listen on `0.0.0.0:9898` + + +from the point of view of `kubeadm init`, we need to have +a) a primary IP address as will be seen by the nodes and needs to go into the cert and `kube-discovery` configuration secret +b) some other names and addresses API server may be known by (e.g. external DNS and/or LB and/or NAT) + +from that perspective we don’t can assume default ports for now, but for all the address we really only care about two ports (i.e. 443 and 9898) + +we should make ports configurable and expose some way of making API server bind to a specific address/interface + +but I think for the MVP we need solve the issue with hardcode IPs and DNS names in the certs + +so it sounds rather simple enough to introduce `--api-advertise-addr=` and `--api-external-dns-name=`, and allowing multiple of those sounds also simple enough + +from the `kubeadm join` perspective, it cares about the two ports we mentioned, and we can make those configurable too + +but for now it’s easier to pass just the IP address + +plust it’s also require, so passing it without a named flag sounds convenient, and it’s something users are familiar with + +that’s what Consul does, what what Weave does, and now Docker SwarmKit does the same thing also (edited) + +flags will differ, as there are some Kubernetes-specifics aspects to what join does, but basic join semantics will remain _familiar_ + +if we do marry `kube-discovery` with the API, we might do `kubeadm join host:port`, as we’d end-up with a single port to care about + +but we haven’t yet diff --git a/pkg/kubeadm/api/types.go b/pkg/kubeadm/api/types.go index dab8765e87..b71957172f 100644 --- a/pkg/kubeadm/api/types.go +++ b/pkg/kubeadm/api/types.go @@ -20,29 +20,76 @@ import ( "net" ) -type BootstrapParams struct { - // TODO this is mostly out of date and bloated now, let's revisit this soon - Discovery *OutOfBandDiscovery - EnvParams map[string]string +type KubeadmConfig struct { + InitFlags + JoinFlags + ManualFlags + Secrets struct { + GivenToken string // dot-separated `.` set by the user + TokenID string // optional on master side, will be generated if not specified + Token []byte // optional on master side, will be generated if not specified + BearerToken string // set based on Token + } + EnvParams map[string]string // TODO(phase2) this is likely to be come componentconfig } -type OutOfBandDiscovery struct { - // 'join-node' side +// TODO(phase2) should we add validatin funcs on these structs? + +type InitFlags struct { + API struct { + AdvertiseAddrs []net.IP + ExternalDNSName []string + } + Services struct { + CIDR net.IPNet + DNSDomain string + } + CloudProvider string +} + +const ( + DefaultServiceDNSDomain = "cluster.local" + DefaultServicesCIDRString = "100.64.0.0/12" +) + +var ( + DefaultServicesCIDR *net.IPNet + ListOfCloudProviders = []string{ + "aws", + "azure", + "cloudstack", + "gce", + "mesos", + "openstack", + "ovirt", + "rackspace", + "vsphere", + } + SupportedCloudProviders map[string]bool +) + +func init() { + _, DefaultServicesCIDR, _ = net.ParseCIDR(DefaultServicesCIDRString) + SupportedCloudProviders = make(map[string]bool, len(ListOfCloudProviders)) + for _, v := range ListOfCloudProviders { + SupportedCloudProviders[v] = true + } +} + +type JoinFlags struct { + MasterAddrs []net.IP +} + +// TODO(phase1?) we haven't decided whether manual sub commands should get merged into main commands... +type ManualFlags struct { ApiServerURLs string // comma separated CaCertFile string - GivenToken string // dot-separated `.` set by the user - TokenID string // optional on master side, will be generated if not specified - Token []byte // optional on master side, will be generated if not specified BearerToken string // set based on Token - // 'init-master' side - ApiServerDNSName string // optional, used in master bootstrap - ListenIP net.IP // optional IP for master to listen on, rather than autodetect - UseHyperkubeImage bool + ListenIP net.IP // optional IP for master to listen on, rather than autodetect } type ClusterInfo struct { - // TODO Kind, apiVersion - // TODO clusterId, fetchedTime, expiredTime + // TODO(pahse1?) this may become simply `api.Config` CertificateAuthorities []string `json:"certificateAuthorities"` Endpoints []string `json:"endpoints"` } diff --git a/pkg/kubeadm/cmd/cmd.go b/pkg/kubeadm/cmd/cmd.go index c152f768ec..dda31030e9 100644 --- a/pkg/kubeadm/cmd/cmd.go +++ b/pkg/kubeadm/cmd/cmd.go @@ -73,25 +73,25 @@ func NewKubeadmCommand(f *cmdutil.Factory, in io.Reader, out, err io.Writer, env // TODO(phase2) detect interactive vs non-interactive use and adjust output accordingly // i.e. make it automation friendly // - // TODO(phase2) create an bastraction that defines files and the content that needs to + // TODO(phase2) create an abstraction that defines files and the content that needs to // be written to disc and write it all in one go at the end as we have a lot of // crapy little files written from different parts of this code; this could also // be useful for testing - bootstrapParams := &kubeadmapi.BootstrapParams{ - Discovery: &kubeadmapi.OutOfBandDiscovery{ - // TODO(phase1) this type no longer makes sense here - }, - EnvParams: envParams, - } + s := new(kubeadmapi.KubeadmConfig) + s.EnvParams = envParams + + //s.InitFlags, s.JoinFlags = new(kubeadmapi.InitFlags), new(kubeadmapi.JoinFlags) + + //s.ManualFlags = new(kubeadmapi.ManualFlags) cmds.ResetFlags() cmds.SetGlobalNormalizationFunc(flag.WarnWordSepNormalizeFunc) - cmds.AddCommand(NewCmdInit(out, bootstrapParams)) - cmds.AddCommand(NewCmdJoin(out, bootstrapParams)) - cmds.AddCommand(NewCmdUser(out, bootstrapParams)) - cmds.AddCommand(NewCmdManual(out, bootstrapParams)) + cmds.AddCommand(NewCmdInit(out, s)) + cmds.AddCommand(NewCmdJoin(out, s)) + cmds.AddCommand(NewCmdUser(out, s)) + cmds.AddCommand(NewCmdManual(out, s)) return cmds } diff --git a/pkg/kubeadm/cmd/init.go b/pkg/kubeadm/cmd/init.go index 56422cf643..73baa5cc46 100644 --- a/pkg/kubeadm/cmd/init.go +++ b/pkg/kubeadm/cmd/init.go @@ -19,6 +19,7 @@ package cmd import ( "fmt" "io" + "net" "github.com/renstrom/dedent" "github.com/spf13/cobra" @@ -40,52 +41,91 @@ var ( `) ) -func NewCmdInit(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { +func NewCmdInit(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { + advertiseAddrs := &[]string{} cmd := &cobra.Command{ Use: "init", Short: "Run this on the first server you deploy onto.", Run: func(cmd *cobra.Command, args []string) { - err := RunInit(out, cmd, args, params) - cmdutil.CheckErr(err) + err := RunInit(out, cmd, args, s, advertiseAddrs) + cmdutil.CheckErr(err) // TODO(phase1+) append alpha warning with bugs URL etc }, } - cmd.PersistentFlags().IPVar(¶ms.Discovery.ListenIP, "listen-ip", nil, - `(optional) IP address to listen on, in case autodetection fails.`) - cmd.PersistentFlags().StringVar(¶ms.Discovery.GivenToken, "token", "", - `(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`) - cmd.PersistentFlags().BoolVar(¶ms.Discovery.UseHyperkubeImage, "use-hyperkube", false, - `(optional) Use the hyperkube image for running the apiserver, controller-manager, scheduler and proxy.`) + cmd.PersistentFlags().StringVar( + &s.Secrets.GivenToken, "token", "", + `(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`, + ) + cmd.PersistentFlags().StringSliceVar( + advertiseAddrs, "api-advertise-addr", []string{}, + `(optional) IP address to advertise, in case autodetection fails.`, + ) + cmd.PersistentFlags().StringSliceVar( + &s.InitFlags.API.ExternalDNSName, "api-external-dns-name", []string{}, + `(optional) DNS name to advertise, in case you have configured one yourself.`, + ) + + cmd.PersistentFlags().IPNetVar( + &s.InitFlags.Services.CIDR, "service-cidr", *kubeadmapi.DefaultServicesCIDR, + `(optional) use alterantive range of IP address for service VIPs, e.g. "10.16.0.0/12"`, + ) + cmd.PersistentFlags().StringVar( + &s.InitFlags.Services.DNSDomain, "service-dns-domain", kubeadmapi.DefaultServiceDNSDomain, + `(optional) use alterantive domain name for services, e.g. "myorg.internal"`, + ) + cmd.PersistentFlags().StringVar( + &s.InitFlags.CloudProvider, "cloud-provider", "", + `(optional) enable cloud proiver features (external load-balancers, storage, etc)`, + ) return cmd } -func RunInit(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error { - +func RunInit(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.KubeadmConfig, advertiseAddrs *[]string) error { // Auto-detect the IP - if params.Discovery.ListenIP == nil { + if len(*advertiseAddrs) == 0 { + // TODO(phase1+) perhaps we could actually grab eth0 and eth1 ip, err := netutil.ChooseHostInterface() if err != nil { return err } - params.Discovery.ListenIP = ip + s.InitFlags.API.AdvertiseAddrs = []net.IP{ip} + } else { + for _, i := range *advertiseAddrs { + addr := net.ParseIP(i) + if addr == nil { + return fmt.Errorf(" failed to parse flag (%q) as an IP address", "--api-advertise-addr="+i) + } + s.InitFlags.API.AdvertiseAddrs = append(s.InitFlags.API.AdvertiseAddrs, addr) + } } - if err := kubemaster.CreateTokenAuthFile(params); err != nil { + + if s.InitFlags.CloudProvider != "" { + // TODO(phase2) we should be able to auto-detect it and check whether things like IAM roles are correct + if _, ok := kubeadmapi.SupportedCloudProviders[s.InitFlags.CloudProvider]; !ok { + return fmt.Errorf(" cloud provider %q is not supported, you can use any of %v, or leave it unset", s.InitFlags.CloudProvider, kubeadmapi.ListOfCloudProviders) + } + } + + if err := kubemaster.CreateTokenAuthFile(s); err != nil { return err } - if err := kubemaster.WriteStaticPodManifests(params); err != nil { + + if err := kubemaster.WriteStaticPodManifests(s); err != nil { return err } - caKey, caCert, err := kubemaster.CreatePKIAssets(params) + + caKey, caCert, err := kubemaster.CreatePKIAssets(s) if err != nil { return err } - kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(params, []string{"kubelet", "admin"}, caKey, caCert) + + kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(s, []string{"kubelet", "admin"}, caKey, caCert) if err != nil { return err } for name, kubeconfig := range kubeconfigs { - if err := kubeadmutil.WriteKubeconfigIfNotExists(params, name, kubeconfig); err != nil { + if err := kubeadmutil.WriteKubeconfigIfNotExists(s, name, kubeconfig); err != nil { return err } } @@ -99,18 +139,18 @@ func RunInit(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmap return err } - if err := kubemaster.CreateDiscoveryDeploymentAndSecret(params, client, caCert); err != nil { + if err := kubemaster.CreateDiscoveryDeploymentAndSecret(s, client, caCert); err != nil { return err } - if err := kubemaster.CreateEssentialAddons(params, client); err != nil { + if err := kubemaster.CreateEssentialAddons(s, client); err != nil { return err } - // TODO use templates to reference struct fields directly as order of args is fragile + // TODO(phase1+) use templates to reference struct fields directly as order of args is fragile fmt.Fprintf(out, init_done_msgf, - params.Discovery.GivenToken, - params.Discovery.ListenIP, + s.Secrets.GivenToken, + s.InitFlags.API.AdvertiseAddrs[0].String(), ) return nil diff --git a/pkg/kubeadm/cmd/join.go b/pkg/kubeadm/cmd/join.go index 5c8f7a1ffb..c83f666306 100644 --- a/pkg/kubeadm/cmd/join.go +++ b/pkg/kubeadm/cmd/join.go @@ -19,6 +19,7 @@ package cmd import ( "fmt" "io" + "net" "github.com/renstrom/dedent" "github.com/spf13/cobra" @@ -40,44 +41,52 @@ var ( `) ) -func NewCmdJoin(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { +func NewCmdJoin(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { cmd := &cobra.Command{ Use: "join", Short: "Run this on other servers to join an existing cluster.", Run: func(cmd *cobra.Command, args []string) { - err := RunJoin(out, cmd, args, params) + err := RunJoin(out, cmd, args, s) cmdutil.CheckErr(err) }, } // TODO this should become `kubeadm join --token=<...> ` - cmd.PersistentFlags().StringVarP(¶ms.Discovery.ApiServerURLs, "api-server-urls", "", "", - `Comma separated list of API server URLs. Typically this might be just - https://:8080/`) - cmd.PersistentFlags().StringVarP(¶ms.Discovery.GivenToken, "token", "", "", - `Shared secret used to secure bootstrap. Must match output of 'init-master'.`) + cmd.PersistentFlags().StringVarP( + &s.Secrets.GivenToken, "token", "", "", + `Shared secret used to secure bootstrap. Must match output of 'init-master'.`, + ) return cmd } -func RunJoin(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error { - ok, err := kubeadmutil.UseGivenTokenIfValid(params) +func RunJoin(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.KubeadmConfig) error { + // TODO this we are missing args from the help text, there should be a way to tell cobra about it + if len(args) == 0 { + return fmt.Errorf(" must specify master IP address (see --help)") + } + for _, i := range args { + addr := net.ParseIP(i) // TODO(phase1+) should allow resolvable names too + if addr == nil { + return fmt.Errorf(" failed parse argument (%q) as an IP address", i) + } + s.JoinFlags.MasterAddrs = append(s.JoinFlags.MasterAddrs, addr) + } + + ok, err := kubeadmutil.UseGivenTokenIfValid(s) if !ok { if err != nil { - return fmt.Errorf("%s (see --help)\n", err) + return fmt.Errorf(" %s (see --help)\n", err) } return fmt.Errorf("Must specify --token (see --help)\n") } - if params.Discovery.ApiServerURLs == "" { - return fmt.Errorf("Must specify --api-server-urls (see --help)\n") - } - kubeconfig, err := kubenode.RetrieveTrustedClusterInfo(params) + kubeconfig, err := kubenode.RetrieveTrustedClusterInfo(s) if err != nil { return err } - err = kubeadmutil.WriteKubeconfigIfNotExists(params, "kubelet", kubeconfig) + err = kubeadmutil.WriteKubeconfigIfNotExists(s, "kubelet", kubeconfig) if err != nil { return err } diff --git a/pkg/kubeadm/cmd/manual.go b/pkg/kubeadm/cmd/manual.go index 49a611ced1..a6d0e40d89 100644 --- a/pkg/kubeadm/cmd/manual.go +++ b/pkg/kubeadm/cmd/manual.go @@ -19,6 +19,7 @@ package cmd import ( "fmt" "io" + "net" "github.com/renstrom/dedent" "github.com/spf13/cobra" @@ -58,23 +59,23 @@ var ( `) ) -// TODO --token here becomes Discovery.BearerToken and not Discovery.GivenToken +// TODO --token here becomes `s.Secrets.BearerToken` and not `s.Secrets.GivenToken` // may be we should make it the same and ask user to pass dot-separated tokens // in any of the modes; we could also enable discovery API in the manual mode just // as well, there is no reason we shouldn't let user mix and match modes, unless // it is too difficult to support -func NewCmdManual(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { +func NewCmdManual(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { cmd := &cobra.Command{ Use: "manual", Short: "Advanced, less-automated functionality, for power users.", // TODO put example usage in the Long description here } - cmd.AddCommand(NewCmdManualBootstrap(out, params)) + cmd.AddCommand(NewCmdManualBootstrap(out, s)) return cmd } -func NewCmdManualBootstrap(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { +func NewCmdManualBootstrap(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { cmd := &cobra.Command{ Use: "bootstrap", Short: "Manually bootstrap a cluster 'out-of-band'", @@ -85,13 +86,14 @@ func NewCmdManualBootstrap(out io.Writer, params *kubeadmapi.BootstrapParams) *c Run: func(cmd *cobra.Command, args []string) { }, } - cmd.AddCommand(NewCmdManualBootstrapInitMaster(out, params)) - cmd.AddCommand(NewCmdManualBootstrapJoinNode(out, params)) + cmd.AddCommand(NewCmdManualBootstrapInitMaster(out, s)) + cmd.AddCommand(NewCmdManualBootstrapJoinNode(out, s)) return cmd } -func NewCmdManualBootstrapInitMaster(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { +func NewCmdManualBootstrapInitMaster(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { + advertiseAddrs := &[]string{} cmd := &cobra.Command{ Use: "init-master", Short: "Manually bootstrap a master 'out-of-band'", @@ -101,101 +103,140 @@ func NewCmdManualBootstrapInitMaster(out io.Writer, params *kubeadmapi.Bootstrap components. `), Run: func(cmd *cobra.Command, args []string) { - err := RunManualBootstrapInitMaster(out, cmd, args, params) + err := RunManualBootstrapInitMaster(out, cmd, args, s, advertiseAddrs) cmdutil.CheckErr(err) }, } - params.Discovery.ApiServerURLs = "http://127.0.0.1:8080/" // On the master, assume you can talk to the API server - cmd.PersistentFlags().StringVar(¶ms.Discovery.ApiServerDNSName, "api-dns-name", "", - `(optional) DNS name for the API server, will be encoded into - subjectAltName in the resulting (generated) TLS certificates`) - cmd.PersistentFlags().IPVar(¶ms.Discovery.ListenIP, "listen-ip", nil, - `(optional) IP address to listen on, in case autodetection fails.`) - cmd.PersistentFlags().StringVar(¶ms.Discovery.BearerToken, "token", "", - `(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`) + cmd.PersistentFlags().StringVar( + &s.Secrets.BearerToken, "token", "", + `(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`, + ) + cmd.PersistentFlags().StringSliceVar( + advertiseAddrs, "api-advertise-addr", nil, + `(optional) IP address to advertise, in case autodetection fails.`, + ) + cmd.PersistentFlags().StringSliceVar( + &s.InitFlags.API.ExternalDNSName, "api-external-dns-name", []string{}, + `(optional) DNS name to advertise, in case you have configured one yourself.`, + ) + _, defaultServicesCIDR, _ := net.ParseCIDR("100.64.0.0/12") + cmd.PersistentFlags().IPNetVar( + &s.InitFlags.Services.CIDR, "service-cidr", *defaultServicesCIDR, + `(optional) use alterantive range of IP address for service VIPs, e.g. "10.16.0.0/12"`, + ) + cmd.PersistentFlags().StringVar( + &s.InitFlags.Services.DNSDomain, "service-dns-domain", "cluster.local", + `(optional) use alterantive domain name for services, e.g. "myorg.internal"`, + ) return cmd } -func RunManualBootstrapInitMaster(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error { +func RunManualBootstrapInitMaster(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.KubeadmConfig, advertiseAddrs *[]string) error { // Auto-detect the IP - if params.Discovery.ListenIP == nil { + if len(*advertiseAddrs) == 0 { + // TODO(phase1+) perhaps we could actually grab eth0 and eth1 ip, err := netutil.ChooseHostInterface() if err != nil { return err } - params.Discovery.ListenIP = ip + s.InitFlags.API.AdvertiseAddrs = []net.IP{ip} + } else { + for _, i := range *advertiseAddrs { + addr := net.ParseIP(i) + if addr == nil { + return fmt.Errorf(" failed to parse flag (%q) as an IP address", "--api-advertise-addr="+i) + } + s.InitFlags.API.AdvertiseAddrs = append(s.InitFlags.API.AdvertiseAddrs, addr) + } } - if err := kubemaster.CreateTokenAuthFile(params); err != nil { + if err := kubemaster.CreateTokenAuthFile(s); err != nil { return err } - if err := kubemaster.WriteStaticPodManifests(params); err != nil { + if err := kubemaster.WriteStaticPodManifests(s); err != nil { return err } - caKey, caCert, err := kubemaster.CreatePKIAssets(params) + caKey, caCert, err := kubemaster.CreatePKIAssets(s) if err != nil { return err } - kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(params, []string{"kubelet", "admin"}, caKey, caCert) + kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(s, []string{"kubelet", "admin"}, caKey, caCert) if err != nil { return err } for name, kubeconfig := range kubeconfigs { - if err := kubeadmutil.WriteKubeconfigIfNotExists(params, name, kubeconfig); err != nil { + if err := kubeadmutil.WriteKubeconfigIfNotExists(s, name, kubeconfig); err != nil { return err } } + // TODO we have most of cmd/init functionality here, except for `CreateDiscoveryDeploymentAndSecret()` + // it may be a good idea to just merge the two commands into one, and it's something we have started talking + // about, the only question is where disco service should be an opt-out... + + client, err := kubemaster.CreateClientAndWaitForAPI(kubeconfigs["admin"]) + if err != nil { + return err + } + + if err := kubemaster.UpdateMasterRoleLabelsAndTaints(client); err != nil { + return err + } + + if err := kubemaster.CreateEssentialAddons(s, client); err != nil { + return err + } + // TODO use templates to reference struct fields directly as order of args is fragile fmt.Fprintf(out, manual_init_done_msgf, - params.Discovery.BearerToken, - params.Discovery.ListenIP, + s.Secrets.BearerToken, + s.InitFlags.API.AdvertiseAddrs[0].String(), ) return nil } -func NewCmdManualBootstrapJoinNode(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { +func NewCmdManualBootstrapJoinNode(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { cmd := &cobra.Command{ Use: "join-node", Short: "Manually bootstrap a node 'out-of-band', joining it into a cluster with extant control plane", Run: func(cmd *cobra.Command, args []string) { - err := RunManualBootstrapJoinNode(out, cmd, args, params) + err := RunManualBootstrapJoinNode(out, cmd, args, s) cmdutil.CheckErr(err) }, } - cmd.PersistentFlags().StringVarP(¶ms.Discovery.CaCertFile, "ca-cert-file", "", "", + cmd.PersistentFlags().StringVarP(&s.ManualFlags.CaCertFile, "ca-cert-file", "", "", `Path to a CA cert file in PEM format. The same CA cert must be distributed to all servers.`) - cmd.PersistentFlags().StringVarP(¶ms.Discovery.ApiServerURLs, "api-server-urls", "", "", + cmd.PersistentFlags().StringVarP(&s.ManualFlags.ApiServerURLs, "api-server-urls", "", "", `Comma separated list of API server URLs. Typically this might be just https://:8080/`) - cmd.PersistentFlags().StringVarP(¶ms.Discovery.BearerToken, "token", "", "", + cmd.PersistentFlags().StringVarP(&s.ManualFlags.BearerToken, "token", "", "", `Shared secret used to secure bootstrap. Must match output of 'init-master'.`) return cmd } -func RunManualBootstrapJoinNode(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error { - if params.Discovery.CaCertFile == "" { +func RunManualBootstrapJoinNode(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.KubeadmConfig) error { + if s.ManualFlags.CaCertFile == "" { fmt.Fprintf(out, "Must specify --ca-cert-file (see --help)\n") return nil } - if params.Discovery.ApiServerURLs == "" { + if s.ManualFlags.ApiServerURLs == "" { fmt.Fprintf(out, "Must specify --api-server-urls (see --help)\n") return nil } - kubeconfig, err := kubenode.PerformTLSBootstrapFromParams(params) + kubeconfig, err := kubenode.PerformTLSBootstrapFromConfig(s) if err != nil { fmt.Fprintf(out, "Failed to perform TLS bootstrap: %s\n", err) return err } - err = kubeadmutil.WriteKubeconfigIfNotExists(params, "kubelet", kubeconfig) + err = kubeadmutil.WriteKubeconfigIfNotExists(s, "kubelet", kubeconfig) if err != nil { fmt.Fprintf(out, "Unable to write config for node:\n%s\n", err) return err diff --git a/pkg/kubeadm/cmd/user.go b/pkg/kubeadm/cmd/user.go index 3d30f50d6e..6beda1bf40 100644 --- a/pkg/kubeadm/cmd/user.go +++ b/pkg/kubeadm/cmd/user.go @@ -23,7 +23,7 @@ import ( kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" ) -func NewCmdUser(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command { +func NewCmdUser(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { cmd := &cobra.Command{ Use: "user", Short: "Get initial admin credentials for a cluster.", // using TLS bootstrap diff --git a/pkg/kubeadm/images/images.go b/pkg/kubeadm/images/images.go index 27f3fb03a6..6452456c52 100644 --- a/pkg/kubeadm/images/images.go +++ b/pkg/kubeadm/images/images.go @@ -24,53 +24,42 @@ import ( const ( KubeEtcdImage = "etcd" - KubeApiServerImage = "apiserver" + KubeApiServerImage = "apiserver" KubeControllerManagerImage = "controller-manager" - KubeSchedulerImage = "scheduler" - KubeProxyImage = "proxy" + KubeSchedulerImage = "scheduler" + KubeProxyImage = "proxy" - KubeDnsImage = "kube-dns" - KubeDnsmasqImage = "dnsmasq" + KubeDnsImage = "kube-dns" + KubeDnsmasqImage = "dnsmasq" KubeExechealthzImage = "exechealthz" - - gcrPrefix = "gcr.io/google_containers" + gcrPrefix = "gcr.io/google_containers" etcdVersion = "2.2.5" kubeVersion = "v1.4.0-alpha.3" - kubeDnsVersion = "1.7" - dnsmasqVersion = "1.3" + kubeDnsVersion = "1.7" + dnsmasqVersion = "1.3" exechealthzVersion = "1.1" ) -func GetCoreImage(image string, overrideImage string, useHyperkube bool) string { +func GetCoreImage(image string, overrideImage string) string { if overrideImage != "" { return overrideImage } - if useHyperkube { - return map[string]string{ - KubeEtcdImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "etcd", runtime.GOARCH, etcdVersion), - KubeApiServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "hyperkube", runtime.GOARCH, kubeVersion), - KubeControllerManagerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "hyperkube", runtime.GOARCH, kubeVersion), - KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "hyperkube", runtime.GOARCH, kubeVersion), - KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "hyperkube", runtime.GOARCH, kubeVersion), - }[image] - } - return map[string]string{ - KubeEtcdImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "etcd", runtime.GOARCH, etcdVersion), - KubeApiServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-apiserver", runtime.GOARCH, kubeVersion), + KubeEtcdImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "etcd", runtime.GOARCH, etcdVersion), + KubeApiServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-apiserver", runtime.GOARCH, kubeVersion), KubeControllerManagerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-controller-manager", runtime.GOARCH, kubeVersion), - KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-scheduler", runtime.GOARCH, kubeVersion), - KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-proxy", runtime.GOARCH, kubeVersion), + KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-scheduler", runtime.GOARCH, kubeVersion), + KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-proxy", runtime.GOARCH, kubeVersion), }[image] } func GetAddonImage(image string) string { return map[string]string{ - KubeDnsImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-dns", runtime.GOARCH, kubeDnsVersion), - KubeDnsmasqImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-dnsmasq", runtime.GOARCH, dnsmasqVersion), + KubeDnsImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kubedns", runtime.GOARCH, kubeDnsVersion), + KubeDnsmasqImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-dnsmasq", runtime.GOARCH, dnsmasqVersion), KubeExechealthzImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "exechealthz", runtime.GOARCH, exechealthzVersion), }[image] } diff --git a/pkg/kubeadm/master/addons.go b/pkg/kubeadm/master/addons.go index add2674d3f..362d611241 100644 --- a/pkg/kubeadm/master/addons.go +++ b/pkg/kubeadm/master/addons.go @@ -25,20 +25,18 @@ import ( clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" "k8s.io/kubernetes/pkg/kubeadm/images" + ipallocator "k8s.io/kubernetes/pkg/registry/service/ipallocator" "k8s.io/kubernetes/pkg/util/intstr" ) -func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { +func createKubeProxyPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { privilegedTrue := true return api.PodSpec{ SecurityContext: &api.PodSecurityContext{HostNetwork: true}, Containers: []api.Container{{ - Name: "kube-proxy", - Image: images.GetCoreImage(images.KubeProxyImage, params.EnvParams["hyperkube_image"], params.Discovery.UseHyperkubeImage), - Command: append(getImageEntrypoint("proxy", params.Discovery.UseHyperkubeImage), []string{ - "--kubeconfig=/run/kubeconfig", - params.EnvParams["component_loglevel"], - }...), + Name: kubeProxy, + Image: images.GetCoreImage(images.KubeProxyImage, s.EnvParams["hyperkube_image"]), + Command: append(getComponentCommand("proxy", s), "--kubeconfig=/run/kubeconfig"), SecurityContext: &api.SecurityContext{Privileged: &privilegedTrue}, VolumeMounts: []api.VolumeMount{ { @@ -67,7 +65,7 @@ func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { { Name: "kubeconfig", VolumeSource: api.VolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: path.Join(params.EnvParams["prefix"], "kubelet.conf")}, + HostPath: &api.HostPathVolumeSource{Path: path.Join(s.EnvParams["prefix"], "kubelet.conf")}, }, }, { @@ -80,8 +78,7 @@ func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { } } -// TODO(phase1): Create the DNS service as well -func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { +func createKubeDNSPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { dnsPodResources := api.ResourceList{ api.ResourceName(api.ResourceCPU): resource.MustParse("100m"), @@ -93,6 +90,16 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { api.ResourceName(api.ResourceMemory): resource.MustParse("50Mi"), } + kubeDNSPort := int32(10053) + dnsmasqPort := int32(53) + + nslookup := fmt.Sprintf("nslookup kubernetes.default.svc.%s 127.0.0.1", s.InitFlags.Services.DNSDomain) + + nslookup = fmt.Sprintf("-cmd=%s:%d >/dev/null && %s:%d >/dev/null", + nslookup, dnsmasqPort, + nslookup, kubeDNSPort, + ) + return api.PodSpec{ Containers: []api.Container{ // DNS server @@ -104,8 +111,8 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { Requests: dnsPodResources, }, Args: []string{ - "--domain=" + params.EnvParams["dns_domain"], - "--dns-port=10053", + fmt.Sprintf("--domain=%s", s.InitFlags.Services.DNSDomain), + fmt.Sprintf("--dns-port=%d", kubeDNSPort), // TODO __PILLAR__FEDERATIONS__DOMAIN__MAP__ }, LivenessProbe: &api.Probe{ @@ -136,12 +143,12 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { }, Ports: []api.ContainerPort{ { - ContainerPort: 10053, + ContainerPort: kubeDNSPort, Name: "dns-local", Protocol: api.ProtocolUDP, }, { - ContainerPort: 10053, + ContainerPort: kubeDNSPort, Name: "dns-tcp-local", Protocol: api.ProtocolTCP, }, @@ -158,16 +165,16 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { Args: []string{ "--cache-size=1000", "--no-resolv", - "--server=127.0.0.1#10053", + fmt.Sprintf("--server=127.0.0.1#%d", kubeDNSPort), }, Ports: []api.ContainerPort{ { - ContainerPort: 53, + ContainerPort: dnsmasqPort, Name: "dns", Protocol: api.ProtocolUDP, }, { - ContainerPort: 53, + ContainerPort: dnsmasqPort, Name: "dns-tcp", Protocol: api.ProtocolTCP, }, @@ -182,7 +189,7 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { Requests: healthzPodResources, }, Args: []string{ - "-cmd=nslookup kubernetes.default.svc." + params.EnvParams["dns_domain"] + " 127.0.0.1 >/dev/null && nslookup kubernetes.default.svc." + params.EnvParams["dns_domain"] + " 127.0.0.1:10053 >/dev/null", + nslookup, "-port=8080", "-quiet", }, @@ -196,19 +203,27 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { } } -func createKubeDNSServiceSpec(params *kubeadmapi.BootstrapParams) api.ServiceSpec { - return api.ServiceSpec{ + +func createKubeDNSServiceSpec(s *kubeadmapi.KubeadmConfig) (*api.ServiceSpec, error) { + ip, err := ipallocator.GetIndexedIP(&s.InitFlags.Services.CIDR, 10) + if err != nil { + return nil, fmt.Errorf("unable to allocate IP address for kube-dns addon from the given CIDR (%q) [%s]", s.InitFlags.Services.CIDR, err) + } + + svc := &api.ServiceSpec{ Selector: map[string]string{"name": "kube-dns"}, Ports: []api.ServicePort{ {Name: "dns", Port: 53, Protocol: api.ProtocolUDP}, {Name: "dns-tcp", Port: 53, Protocol: api.ProtocolTCP}, }, - ClusterIP: "100.64.0.2", + ClusterIP: ip.String(), } + + return svc, nil } -func CreateEssentialAddons(params *kubeadmapi.BootstrapParams, client *clientset.Clientset) error { - kubeProxyDaemonSet := NewDaemonSet("kube-proxy", createKubeProxyPodSpec(params)) +func CreateEssentialAddons(s *kubeadmapi.KubeadmConfig, client *clientset.Clientset) error { + kubeProxyDaemonSet := NewDaemonSet(kubeProxy, createKubeProxyPodSpec(s)) SetMasterTaintTolerations(&kubeProxyDaemonSet.Spec.Template.ObjectMeta) if _, err := client.Extensions().DaemonSets(api.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil { @@ -217,15 +232,19 @@ func CreateEssentialAddons(params *kubeadmapi.BootstrapParams, client *clientset fmt.Println(" created essential addon: kube-proxy") - kubeDNSDeployment := NewDeployment("kube-dns", 1, createKubeDNSPodSpec(params)) + kubeDNSDeployment := NewDeployment("kube-dns", 1, createKubeDNSPodSpec(s)) SetMasterTaintTolerations(&kubeDNSDeployment.Spec.Template.ObjectMeta) if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kubeDNSDeployment); err != nil { return fmt.Errorf(" failed creating essential kube-dns addon [%s]", err) } - kubeDNSService := NewService("kube-dns", createKubeDNSServiceSpec(params)) + kubeDNSServiceSpec, err := createKubeDNSServiceSpec(s) + if err != nil { + return fmt.Errorf(" failed creating essential kube-dns addon - %s", err) + } + kubeDNSService := NewService("kube-dns", *kubeDNSServiceSpec) if _, err := client.Services(api.NamespaceSystem).Create(kubeDNSService); err != nil { return fmt.Errorf(" failed creating essential kube-dns addon [%s]", err) } diff --git a/pkg/kubeadm/master/discovery.go b/pkg/kubeadm/master/discovery.go index 15ae71e996..4ae8dcf13e 100644 --- a/pkg/kubeadm/master/discovery.go +++ b/pkg/kubeadm/master/discovery.go @@ -39,7 +39,7 @@ const ( kubeDiscoverySecretName = "clusterinfo" ) -func encodeKubeDiscoverySecretData(params *kubeadmapi.BootstrapParams, caCert *x509.Certificate) map[string][]byte { +func encodeKubeDiscoverySecretData(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate) map[string][]byte { // TODO ListenIP is probably not the right now, although it's best we have right now // if user provides a DNS name, or anything else, we should use that, may be it's really // the list of all SANs (minus internal DNS names and service IP)? @@ -50,8 +50,11 @@ func encodeKubeDiscoverySecretData(params *kubeadmapi.BootstrapParams, caCert *x tokenMap = map[string]string{} ) - endpointList = append(endpointList, fmt.Sprintf("https://%s:443", params.Discovery.ListenIP)) - tokenMap[params.Discovery.TokenID] = hex.EncodeToString(params.Discovery.Token) + for _, addr := range s.InitFlags.API.AdvertiseAddrs { + endpointList = append(endpointList, fmt.Sprintf("https://%s:443", addr.String())) + } + + tokenMap[s.Secrets.TokenID] = hex.EncodeToString(s.Secrets.Token) data["endpoint-list.json"], _ = json.Marshal(endpointList) data["token-map.json"], _ = json.Marshal(tokenMap) @@ -60,7 +63,7 @@ func encodeKubeDiscoverySecretData(params *kubeadmapi.BootstrapParams, caCert *x return data } -func newKubeDiscoveryPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { +func newKubeDiscoveryPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { return api.PodSpec{ // We have to use host network namespace, as `HostPort`/`HostIP` are Docker's // buisness and CNI support isn't quite there yet (except for kubenet) @@ -69,7 +72,7 @@ func newKubeDiscoveryPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { SecurityContext: &api.PodSecurityContext{HostNetwork: true}, Containers: []api.Container{{ Name: kubeDiscoverynName, - Image: params.EnvParams["discovery_image"], + Image: s.EnvParams["discovery_image"], Command: []string{"/usr/bin/kube-discovery"}, VolumeMounts: []api.VolumeMount{{ Name: kubeDiscoverySecretName, @@ -77,7 +80,8 @@ func newKubeDiscoveryPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { ReadOnly: true, }}, Ports: []api.ContainerPort{ - // TODO when CNI issue (#31307) is resolved, we should add `HostIP: params.Discovery.ListenIP` + // TODO when CNI issue (#31307) is resolved, we should consider adding + // `HostIP: s.API.AdvertiseAddrs[0]`, if there is only one address` {Name: "http", ContainerPort: 9898, HostPort: 9898}, }, }}, @@ -90,13 +94,13 @@ func newKubeDiscoveryPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { } } -func newKubeDiscovery(params *kubeadmapi.BootstrapParams, caCert *x509.Certificate) kubeDiscovery { +func newKubeDiscovery(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate) kubeDiscovery { kd := kubeDiscovery{ - Deployment: NewDeployment(kubeDiscoverynName, 1, newKubeDiscoveryPodSpec(params)), + Deployment: NewDeployment(kubeDiscoverynName, 1, newKubeDiscoveryPodSpec(s)), Secret: &api.Secret{ ObjectMeta: api.ObjectMeta{Name: kubeDiscoverySecretName}, Type: api.SecretTypeOpaque, - Data: encodeKubeDiscoverySecretData(params, caCert), + Data: encodeKubeDiscoverySecretData(s, caCert), }, } @@ -106,8 +110,8 @@ func newKubeDiscovery(params *kubeadmapi.BootstrapParams, caCert *x509.Certifica return kd } -func CreateDiscoveryDeploymentAndSecret(params *kubeadmapi.BootstrapParams, client *clientset.Clientset, caCert *x509.Certificate) error { - kd := newKubeDiscovery(params, caCert) +func CreateDiscoveryDeploymentAndSecret(s *kubeadmapi.KubeadmConfig, client *clientset.Clientset, caCert *x509.Certificate) error { + kd := newKubeDiscovery(s, caCert) if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kd.Deployment); err != nil { return fmt.Errorf(" failed to create %q deployment [%s]", kubeDiscoverynName, err) diff --git a/pkg/kubeadm/master/kubeconfig.go b/pkg/kubeadm/master/kubeconfig.go index ea76c3849b..3287ea7a70 100644 --- a/pkg/kubeadm/master/kubeconfig.go +++ b/pkg/kubeadm/master/kubeconfig.go @@ -28,11 +28,14 @@ import ( certutil "k8s.io/kubernetes/pkg/util/cert" ) -func CreateCertsAndConfigForClients(params *kubeadmapi.BootstrapParams, clientNames []string, caKey *rsa.PrivateKey, caCert *x509.Certificate) (map[string]*clientcmdapi.Config, error) { +func CreateCertsAndConfigForClients(s *kubeadmapi.KubeadmConfig, clientNames []string, caKey *rsa.PrivateKey, caCert *x509.Certificate) (map[string]*clientcmdapi.Config, error) { basicClientConfig := kubeadmutil.CreateBasicClientConfig( "kubernetes", - fmt.Sprintf("https://%s:443", params.Discovery.ListenIP), + // TODO this is not great, but there is only one address we can use here + // so we'll pick the first one, there is much of chance to have an empty + // slice by the time this gets called + fmt.Sprintf("https://%s:443", s.InitFlags.API.AdvertiseAddrs[0]), certutil.EncodeCertPEM(caCert), ) diff --git a/pkg/kubeadm/master/manifests.go b/pkg/kubeadm/master/manifests.go index 385813dffe..f2a985b14b 100644 --- a/pkg/kubeadm/master/manifests.go +++ b/pkg/kubeadm/master/manifests.go @@ -36,85 +36,65 @@ import ( // init master` and `kubeadm manual bootstrap master` can get going. const ( - SERVICE_CLUSTER_IP_RANGE = "--service-cluster-ip-range=100.64.0.0/12" - CLUSTER_NAME = "--cluster-name=kubernetes" - MASTER = "--master=127.0.0.1:8080" + DefaultClusterName = "--cluster-name=kubernetes" + + etcd = "etcd" + apiServer = "apiserver" + controllerManager = "controller-manager" + scheduler = "scheduler" + proxy = "proxy" + kubeAPIServer = "kube-apiserver" + kubeControllerManager = "kube-controller-manager" + kubeScheduler = "kube-scheduler" + kubeProxy = "kube-proxy" ) // TODO look into what this really means, scheduler prints it for some reason // //E0817 17:53:22.242658 1 event.go:258] Could not construct reference to: '&api.Endpoints{TypeMeta:unversioned.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:api.ObjectMeta{Name:"kube-scheduler", GenerateName:"", Namespace:"kube-system", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:unversioned.Time{Time:time.Time{sec:0, nsec:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*unversioned.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]api.OwnerReference(nil), Finalizers:[]string(nil)}, Subsets:[]api.EndpointSubset(nil)}' due to: 'selfLink was empty, can't make reference'. Will not report event: 'Normal' '%v became leader' 'moby' -func WriteStaticPodManifests(params *kubeadmapi.BootstrapParams) error { +func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { staticPodSpecs := map[string]api.Pod{ // TODO this needs a volume - "etcd": componentPod(api.Container{ + etcd: componentPod(api.Container{ Command: []string{ "/usr/local/bin/etcd", "--listen-client-urls=http://127.0.0.1:2379", "--advertise-client-urls=http://127.0.0.1:2379", "--data-dir=/var/etcd/data", }, - Image: images.GetCoreImage(images.KubeEtcdImage, params.EnvParams["etcd_image"], params.Discovery.UseHyperkubeImage), + Image: images.GetCoreImage(images.KubeEtcdImage, s.EnvParams["etcd_image"]), LivenessProbe: componentProbe(2379, "/health"), - Name: "etcd-server", + Name: etcd, Resources: componentResources("200m"), }), // TODO bind-mount certs in - "kube-apiserver": componentPod(api.Container{ - Name: "kube-apiserver", - Image: images.GetCoreImage(images.KubeApiServerImage, params.EnvParams["hyperkube_image"], params.Discovery.UseHyperkubeImage), - Command: append(getImageEntrypoint("apiserver", params.Discovery.UseHyperkubeImage), []string{ - "--address=127.0.0.1", - "--etcd-servers=http://127.0.0.1:2379", - "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota", - SERVICE_CLUSTER_IP_RANGE, - "--service-account-key-file=/etc/kubernetes/pki/apiserver-key.pem", - "--client-ca-file=/etc/kubernetes/pki/ca.pem", - "--tls-cert-file=/etc/kubernetes/pki/apiserver.pem", - "--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", - "--secure-port=443", - "--allow-privileged", - params.EnvParams["component_loglevel"], - "--token-auth-file=/etc/kubernetes/pki/tokens.csv", - }...), + kubeAPIServer: componentPod(api.Container{ + Name: kubeAPIServer, + Image: images.GetCoreImage(images.KubeApiServerImage, s.EnvParams["hyperkube_image"]), + Command: getComponentCommand(apiServer, s), VolumeMounts: []api.VolumeMount{pkiVolumeMount()}, LivenessProbe: componentProbe(8080, "/healthz"), Resources: componentResources("250m"), - }, pkiVolume(params)), - "kube-controller-manager": componentPod(api.Container{ - Name: "kube-controller-manager", - Image: images.GetCoreImage(images.KubeControllerManagerImage, params.EnvParams["hyperkube_image"], params.Discovery.UseHyperkubeImage), - Command: append(getImageEntrypoint("controller-manager", params.Discovery.UseHyperkubeImage), []string{ - "--leader-elect", - MASTER, - CLUSTER_NAME, - "--root-ca-file=/etc/kubernetes/pki/ca.pem", - "--service-account-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", - "--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem", - "--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem", - "--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap", - "--cluster-cidr=10.2.0.0/16", - params.EnvParams["component_loglevel"], - }...), + }, pkiVolume(s)), + kubeControllerManager: componentPod(api.Container{ + Name: kubeControllerManager, + Image: images.GetCoreImage(images.KubeControllerManagerImage, s.EnvParams["hyperkube_image"]), + Command: getComponentCommand(controllerManager, s), VolumeMounts: []api.VolumeMount{pkiVolumeMount()}, LivenessProbe: componentProbe(10252, "/healthz"), Resources: componentResources("200m"), - }, pkiVolume(params)), - "kube-scheduler": componentPod(api.Container{ - Name: "kube-scheduler", - Image: images.GetCoreImage(images.KubeSchedulerImage, params.EnvParams["hyperkube_image"], params.Discovery.UseHyperkubeImage), - Command: append(getImageEntrypoint("scheduler", params.Discovery.UseHyperkubeImage), []string{ - "--leader-elect", - MASTER, - params.EnvParams["component_loglevel"], - }...), + }, pkiVolume(s)), + kubeScheduler: componentPod(api.Container{ + Name: kubeScheduler, + Image: images.GetCoreImage(images.KubeSchedulerImage, s.EnvParams["hyperkube_image"]), + Command: getComponentCommand(scheduler, s), LivenessProbe: componentProbe(10251, "/healthz"), Resources: componentResources("100m"), }), } - manifestsPath := path.Join(params.EnvParams["prefix"], "manifests") + manifestsPath := path.Join(s.EnvParams["prefix"], "manifests") if err := os.MkdirAll(manifestsPath, 0700); err != nil { return fmt.Errorf(" failed to create directory %q [%s]", manifestsPath, err) } @@ -131,11 +111,11 @@ func WriteStaticPodManifests(params *kubeadmapi.BootstrapParams) error { return nil } -func pkiVolume(params *kubeadmapi.BootstrapParams) api.Volume { +func pkiVolume(s *kubeadmapi.KubeadmConfig) api.Volume { return api.Volume{ Name: "pki", VolumeSource: api.VolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: params.EnvParams["host_pki_path"]}, + HostPath: &api.HostPathVolumeSource{Path: s.EnvParams["host_pki_path"]}, }, } } @@ -189,10 +169,51 @@ func componentPod(container api.Container, volumes ...api.Volume) api.Pod { } } -func getImageEntrypoint(component string, useHyperkube bool) []string { - if useHyperkube { - return []string{"/hyperkube", component} +func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command []string) { + baseFalgs := map[string][]string{ + apiServer: []string{ + "--address=127.0.0.1", + "--etcd-servers=http://127.0.0.1:2379", + "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota", + "--service-cluster-ip-range=" + s.InitFlags.Services.CIDR.String(), + "--service-account-key-file=/etc/kubernetes/pki/apiserver-key.pem", + "--client-ca-file=/etc/kubernetes/pki/ca.pem", + "--tls-cert-file=/etc/kubernetes/pki/apiserver.pem", + "--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", + "--secure-port=443", + "--allow-privileged", + "--token-auth-file=/etc/kubernetes/pki/tokens.csv", + }, + controllerManager: []string{ + "--leader-elect", + "--master=127.0.0.1:8080", + DefaultClusterName, + "--root-ca-file=/etc/kubernetes/pki/ca.pem", + "--service-account-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", + "--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem", + "--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem", + "--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap", + "--cluster-cidr=" + s.InitFlags.Services.CIDR.String(), + }, + scheduler: []string{ + "--leader-elect", + "--master=127.0.0.1:8080", + }, + proxy: []string{}, } - return []string{"/usr/local/bin/kube-" + component} + if s.EnvParams["hyperkube_image"] != "" { + command = []string{"/hyperkube", component} + } else { + command = []string{"/usr/local/bin/kube-" + component} + } + + command = append(command, s.EnvParams["component_loglevel"]) + command = append(command, baseFalgs[component]...) + + if component == controllerManager && s.InitFlags.CloudProvider != "" { + command = append(command, "--cloud-provider="+s.InitFlags.CloudProvider) + } + + return } diff --git a/pkg/kubeadm/master/pki.go b/pkg/kubeadm/master/pki.go index bbff5ad919..19c8ccac92 100644 --- a/pkg/kubeadm/master/pki.go +++ b/pkg/kubeadm/master/pki.go @@ -20,10 +20,10 @@ import ( "crypto/rsa" "crypto/x509" "fmt" - "net" "path" kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" + ipallocator "k8s.io/kubernetes/pkg/registry/service/ipallocator" certutil "k8s.io/kubernetes/pkg/util/cert" ) @@ -51,19 +51,26 @@ func newCertificateAuthority() (*rsa.PrivateKey, *x509.Certificate, error) { return key, cert, nil } -func newServerKeyAndCert(caCert *x509.Certificate, caKey *rsa.PrivateKey, altNames certutil.AltNames) (*rsa.PrivateKey, *x509.Certificate, error) { +func newServerKeyAndCert(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate, caKey *rsa.PrivateKey, altNames certutil.AltNames) (*rsa.PrivateKey, *x509.Certificate, error) { key, err := certutil.NewPrivateKey() if err != nil { return nil, nil, fmt.Errorf("unabel to create private key [%s]", err) } - // TODO these are all hardcoded for now, but we need to figure out what shall we do here exactly - altNames.IPs = append(altNames.IPs, net.ParseIP("10.3.0.1"), net.ParseIP("10.16.0.1"), net.ParseIP("100.64.0.1")) - altNames.DNSNames = append(altNames.DNSNames, + + internalAPIServerFQDN := []string{ "kubernetes", "kubernetes.default", "kubernetes.default.svc", - "kubernetes.default.svc.cluster.local", - ) + fmt.Sprintf("kubernetes.default.svc.%s", s.InitFlags.Services.DNSDomain), + } + + internalAPIServerVirtualIP, err := ipallocator.GetIndexedIP(&s.InitFlags.Services.CIDR, 1) + if err != nil { + return nil, nil, fmt.Errorf("unable to allocate IP address for the API server from the given CIDR (%q) [%s]") + } + + altNames.IPs = append(altNames.IPs, internalAPIServerVirtualIP) + altNames.DNSNames = append(altNames.DNSNames, internalAPIServerFQDN...) config := certutil.CertConfig{ CommonName: "kube-apiserver", @@ -131,21 +138,21 @@ func newServiceAccountKey() (*rsa.PrivateKey, error) { return key, nil } -func CreatePKIAssets(params *kubeadmapi.BootstrapParams) (*rsa.PrivateKey, *x509.Certificate, error) { +func CreatePKIAssets(s *kubeadmapi.KubeadmConfig) (*rsa.PrivateKey, *x509.Certificate, error) { var ( err error - altNames certutil.AltNames // TODO actual SANs + altNames certutil.AltNames ) - if params.Discovery.ListenIP != nil { - altNames.IPs = append(altNames.IPs, params.Discovery.ListenIP) + if len(s.InitFlags.API.AdvertiseAddrs) > 0 { + altNames.IPs = append(altNames.IPs, s.InitFlags.API.AdvertiseAddrs...) } - if params.Discovery.ApiServerDNSName != "" { - altNames.DNSNames = append(altNames.DNSNames, params.Discovery.ApiServerDNSName) + if len(s.InitFlags.API.ExternalDNSName) > 0 { + altNames.DNSNames = append(altNames.DNSNames, s.InitFlags.API.ExternalDNSName...) } - pkiPath := path.Join(params.EnvParams["host_pki_path"]) + pkiPath := path.Join(s.EnvParams["host_pki_path"]) caKey, caCert, err := newCertificateAuthority() if err != nil { @@ -156,7 +163,7 @@ func CreatePKIAssets(params *kubeadmapi.BootstrapParams) (*rsa.PrivateKey, *x509 return nil, nil, fmt.Errorf(" failure while saving CA keys and certificate - %s", err) } - apiKey, apiCert, err := newServerKeyAndCert(caCert, caKey, altNames) + apiKey, apiCert, err := newServerKeyAndCert(s, caCert, caKey, altNames) if err != nil { return nil, nil, fmt.Errorf(" failure while creating API server keys and certificate - %s", err) } @@ -174,7 +181,7 @@ func CreatePKIAssets(params *kubeadmapi.BootstrapParams) (*rsa.PrivateKey, *x509 return nil, nil, fmt.Errorf(" failure while saving service account singing keys - %s", err) } - // TODO print a summary of SANs used and checksums (signatures) of each of the certiicates - fmt.Printf(" created keys and certificates in %q\n", params.EnvParams["host_pki_path"]) + // TODO(phase1+) print a summary of SANs used and checksums (signatures) of each of the certiicates + fmt.Printf(" created keys and certificates in %q\n", pkiPath) return caKey, caCert, nil } diff --git a/pkg/kubeadm/master/tokens.go b/pkg/kubeadm/master/tokens.go index 844e5ffaaf..f741105d58 100644 --- a/pkg/kubeadm/master/tokens.go +++ b/pkg/kubeadm/master/tokens.go @@ -28,17 +28,17 @@ import ( "k8s.io/kubernetes/pkg/util/uuid" ) -func generateTokenIfNeeded(params *kubeadmapi.BootstrapParams) error { - ok, err := kubeadmutil.UseGivenTokenIfValid(params) +func generateTokenIfNeeded(s *kubeadmapi.KubeadmConfig) error { + ok, err := kubeadmutil.UseGivenTokenIfValid(s) if !ok { if err != nil { return err } - err = kubeadmutil.GenerateToken(params) + err = kubeadmutil.GenerateToken(s) if err != nil { return err } - fmt.Printf(" generated token: %q\n", params.Discovery.GivenToken) + fmt.Printf(" generated token: %q\n", s.Secrets.GivenToken) } else { fmt.Println(" accepted provided token") } @@ -46,15 +46,15 @@ func generateTokenIfNeeded(params *kubeadmapi.BootstrapParams) error { return nil } -func CreateTokenAuthFile(params *kubeadmapi.BootstrapParams) error { - tokenAuthFilePath := path.Join(params.EnvParams["host_pki_path"], "tokens.csv") - if err := generateTokenIfNeeded(params); err != nil { +func CreateTokenAuthFile(s *kubeadmapi.KubeadmConfig) error { + tokenAuthFilePath := path.Join(s.EnvParams["host_pki_path"], "tokens.csv") + if err := generateTokenIfNeeded(s); err != nil { return fmt.Errorf(" failed to generate token(s) [%s]", err) } - if err := os.MkdirAll(params.EnvParams["host_pki_path"], 0700); err != nil { - return fmt.Errorf(" failed to create directory %q [%s]", params.EnvParams["host_pki_path"], err) + if err := os.MkdirAll(s.EnvParams["host_pki_path"], 0700); err != nil { + return fmt.Errorf(" failed to create directory %q [%s]", s.EnvParams["host_pki_path"], err) } - serialized := []byte(fmt.Sprintf("%s,kubeadm-node-csr,%s,system:kubelet-bootstrap\n", params.Discovery.BearerToken, uuid.NewUUID())) + serialized := []byte(fmt.Sprintf("%s,kubeadm-node-csr,%s,system:kubelet-bootstrap\n", s.Secrets.BearerToken, uuid.NewUUID())) if err := cmdutil.DumpReaderToFile(bytes.NewReader(serialized), tokenAuthFilePath); err != nil { return fmt.Errorf(" failed to save token auth file (%q) [%s]", tokenAuthFilePath, err) } diff --git a/pkg/kubeadm/node/csr.go b/pkg/kubeadm/node/csr.go index 0580bc3791..d4f4f0eef5 100644 --- a/pkg/kubeadm/node/csr.go +++ b/pkg/kubeadm/node/csr.go @@ -34,17 +34,17 @@ func getNodeName() string { return "TODO" } -func PerformTLSBootstrapFromParams(params *kubeadmapi.BootstrapParams) (*clientcmdapi.Config, error) { - caCert, err := ioutil.ReadFile(params.Discovery.CaCertFile) +func PerformTLSBootstrapFromConfig(s *kubeadmapi.KubeadmConfig) (*clientcmdapi.Config, error) { + caCert, err := ioutil.ReadFile(s.ManualFlags.CaCertFile) if err != nil { return nil, fmt.Errorf(" failed to load CA certificate [%s]", err) } - return PerformTLSBootstrap(params, strings.Split(params.Discovery.ApiServerURLs, ",")[0], caCert) + return PerformTLSBootstrap(s, strings.Split(s.ManualFlags.ApiServerURLs, ",")[0], caCert) } // Create a restful client for doing the certificate signing request. -func PerformTLSBootstrap(params *kubeadmapi.BootstrapParams, apiEndpoint string, caCert []byte) (*clientcmdapi.Config, error) { +func PerformTLSBootstrap(s *kubeadmapi.KubeadmConfig, apiEndpoint string, caCert []byte) (*clientcmdapi.Config, error) { // TODO try all the api servers until we find one that works bareClientConfig := kubeadmutil.CreateBasicClientConfig("kubernetes", apiEndpoint, caCert) @@ -52,7 +52,7 @@ func PerformTLSBootstrap(params *kubeadmapi.BootstrapParams, apiEndpoint string, bootstrapClientConfig, err := clientcmd.NewDefaultClientConfig( *kubeadmutil.MakeClientConfigWithToken( - bareClientConfig, "kubernetes", fmt.Sprintf("kubelet-%s", nodeName), params.Discovery.BearerToken, + bareClientConfig, "kubernetes", fmt.Sprintf("kubelet-%s", nodeName), s.Secrets.BearerToken, ), &clientcmd.ConfigOverrides{}, ).ClientConfig() diff --git a/pkg/kubeadm/node/discovery.go b/pkg/kubeadm/node/discovery.go index 62e43b8c4d..8ec04b10d1 100644 --- a/pkg/kubeadm/node/discovery.go +++ b/pkg/kubeadm/node/discovery.go @@ -22,23 +22,15 @@ import ( "fmt" "io" "net/http" - "net/url" - "strings" jose "github.com/square/go-jose" clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api" kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" ) -func RetrieveTrustedClusterInfo(params *kubeadmapi.BootstrapParams) (*clientcmdapi.Config, error) { - firstURL := strings.Split(params.Discovery.ApiServerURLs, ",")[0] // TODO obviously we should do something better.. . - apiServerURL, err := url.Parse(firstURL) - if err != nil { - return nil, fmt.Errorf(" failed to parse given API server URL (%q) [%s]", firstURL, err) - } - - host, port := strings.Split(apiServerURL.Host, ":")[0], 9898 // TODO this is too naive - requestURL := fmt.Sprintf("http://%s:%d/cluster-info/v1/?token-id=%s", host, port, params.Discovery.TokenID) +func RetrieveTrustedClusterInfo(s *kubeadmapi.KubeadmConfig) (*clientcmdapi.Config, error) { + host, port := s.JoinFlags.MasterAddrs[0].String(), 9898 + requestURL := fmt.Sprintf("http://%s:%d/cluster-info/v1/?token-id=%s", host, port, s.Secrets.TokenID) req, err := http.NewRequest("GET", requestURL, nil) if err != nil { return nil, fmt.Errorf(" failed to consturct an HTTP request [%s]", err) @@ -61,7 +53,7 @@ func RetrieveTrustedClusterInfo(params *kubeadmapi.BootstrapParams) (*clientcmda fmt.Println(" cluster info object received, verifying signature using given token") - output, err := object.Verify(params.Discovery.Token) + output, err := object.Verify(s.Secrets.Token) if err != nil { return nil, fmt.Errorf(" failed to verify JWS signature of received cluster info object [%s]", err) } @@ -84,5 +76,5 @@ func RetrieveTrustedClusterInfo(params *kubeadmapi.BootstrapParams) (*clientcmda apiServer := clusterInfo.Endpoints[0] caCert := []byte(clusterInfo.CertificateAuthorities[0]) - return PerformTLSBootstrap(params, apiServer, caCert) + return PerformTLSBootstrap(s, apiServer, caCert) } diff --git a/pkg/kubeadm/util/kubeconfig.go b/pkg/kubeadm/util/kubeconfig.go index dd6cd32835..4fe56cf03f 100644 --- a/pkg/kubeadm/util/kubeconfig.go +++ b/pkg/kubeadm/util/kubeconfig.go @@ -81,12 +81,12 @@ func MakeClientConfigWithToken(config *clientcmdapi.Config, clusterName string, // running in that case - they'd need to stop the kubelet, remove the file, and // start it again in that case). -func WriteKubeconfigIfNotExists(params *kubeadmapi.BootstrapParams, name string, kubeconfig *clientcmdapi.Config) error { - if err := os.MkdirAll(params.EnvParams["prefix"], 0700); err != nil { - return fmt.Errorf(" failed to create directory %q [%s]", params.EnvParams["prefix"], err) +func WriteKubeconfigIfNotExists(s *kubeadmapi.KubeadmConfig, name string, kubeconfig *clientcmdapi.Config) error { + if err := os.MkdirAll(s.EnvParams["prefix"], 0700); err != nil { + return fmt.Errorf(" failed to create directory %q [%s]", s.EnvParams["prefix"], err) } - filename := path.Join(params.EnvParams["prefix"], fmt.Sprintf("%s.conf", name)) + filename := path.Join(s.EnvParams["prefix"], fmt.Sprintf("%s.conf", name)) // Create and open the file, only if it does not already exist. f, err := os.OpenFile( filename, diff --git a/pkg/kubeadm/util/tokens.go b/pkg/kubeadm/util/tokens.go index 658044a306..da51ff551a 100644 --- a/pkg/kubeadm/util/tokens.go +++ b/pkg/kubeadm/util/tokens.go @@ -42,7 +42,7 @@ func randBytes(length int) ([]byte, string, error) { return b, hex.EncodeToString(b), nil } -func GenerateToken(params *kubeadmapi.BootstrapParams) error { +func GenerateToken(s *kubeadmapi.KubeadmConfig) error { _, tokenID, err := randBytes(TokenIDLen / 2) if err != nil { return err @@ -53,19 +53,19 @@ func GenerateToken(params *kubeadmapi.BootstrapParams) error { return err } - params.Discovery.TokenID = tokenID - params.Discovery.BearerToken = token - params.Discovery.Token = tokenBytes - params.Discovery.GivenToken = fmt.Sprintf("%s.%s", tokenID, token) + s.Secrets.TokenID = tokenID + s.Secrets.BearerToken = token + s.Secrets.Token = tokenBytes + s.Secrets.GivenToken = fmt.Sprintf("%s.%s", tokenID, token) return nil } -func UseGivenTokenIfValid(params *kubeadmapi.BootstrapParams) (bool, error) { - if params.Discovery.GivenToken == "" { +func UseGivenTokenIfValid(s *kubeadmapi.KubeadmConfig) (bool, error) { + if s.Secrets.GivenToken == "" { return false, nil // not given } fmt.Println(" validating provided token") - givenToken := strings.Split(strings.ToLower(params.Discovery.GivenToken), ".") + givenToken := strings.Split(strings.ToLower(s.Secrets.GivenToken), ".") // TODO print desired format // TODO could also print more specific messages in each case invalidErr := " provided token is invalid - %s" @@ -86,8 +86,8 @@ func UseGivenTokenIfValid(params *kubeadmapi.BootstrapParams) (bool, error) { "length of second part is incorrect [%d (given) != %d (expected)]", len(tokenBytes), TokenBytes)) } - params.Discovery.TokenID = givenToken[0] - params.Discovery.BearerToken = givenToken[1] - params.Discovery.Token = tokenBytes + s.Secrets.TokenID = givenToken[0] + s.Secrets.BearerToken = givenToken[1] + s.Secrets.Token = tokenBytes return true, nil // given and valid }