Refactoring improtant parts and start on docs

pull/6/head
Ilya Dmitrichenko 2016-09-07 17:37:02 +01:00
parent 26c4f593aa
commit b9fd31ff7e
No known key found for this signature in database
GPG Key ID: E7889175A6C0CEB9
19 changed files with 529 additions and 282 deletions

View File

@ -46,7 +46,6 @@ func getEnvParams() map[string]string {
"discovery_image": "dgoodwin/kubediscovery:latest", // TODO(phase1): fmt.Sprintf("gcr.io/google_containers/kube-discovery-%s:%s", runtime.GOARCH, "1.0"),
"etcd_image": "",
"component_loglevel": "--v=4",
"dns_domain": "cluster.local",
}
for k := range envParams {

76
pkg/kubeadm/README.md Normal file
View File

@ -0,0 +1,76 @@
# Kubernetes Cluster Boostrap Made Easy
## Usage
### `kubeadm init`
It's usually enough to run `kubeadm init`, but in some case you might like to override the
default behaviour.
- `--token=<str>`
By default, a token is generated, but if you are to automate cluster deployment, you want to
set the token ahead of time. Read the docs for more information on the token format.
- `--api-advertise-addr=<ip>` (multiple values allowed)
- `--api-external-dns-name=<domain>` (multiple values allowed)
By default, `kubeadm` will auto detect IP address and use that to generate API server certificates.
If you would like to access the API via any external IPs and/or DNS, which it might not be able
to detect, you can use `--api-advertise-addr` and `--api-external-dns-name` to add multiple
different IP addresses and DNS names.
- `--service-cidr=<cidr>` (default: "100.64/12")
- `--service-dns-domain=<domain>` (default: "cluster.local")
- `--use-hyperkube=<bool>` (default: "false")
***TODO(phase1+)***
- `--api-bind-addr=<ip>`
- `--api-bind-port=<port>`
***TODO(phase2)***
- `--api-bind-loopback-unsecure=<bool>`
***TODO(pahse2)***
- `--prefer-private-network=<bool>`
- `--prefer-public-network=<bool>`
### `kubeadm join`
# User Experience Considerations
> ***TODO*** _Move this into the design document
a) `kube-apiserver` will listen on `0.0.0.0:443` and `127.0.0.1:8080`, which is not configurable right now and make things a bit easier for the MVP
b) there is `kube-discovery`, which will listen on `0.0.0.0:9898`
from the point of view of `kubeadm init`, we need to have
a) a primary IP address as will be seen by the nodes and needs to go into the cert and `kube-discovery` configuration secret
b) some other names and addresses API server may be known by (e.g. external DNS and/or LB and/or NAT)
from that perspective we dont can assume default ports for now, but for all the address we really only care about two ports (i.e. 443 and 9898)
we should make ports configurable and expose some way of making API server bind to a specific address/interface
but I think for the MVP we need solve the issue with hardcode IPs and DNS names in the certs
so it sounds rather simple enough to introduce `--api-advertise-addr=<ip>` and `--api-external-dns-name=<domain>`, and allowing multiple of those sounds also simple enough
from the `kubeadm join` perspective, it cares about the two ports we mentioned, and we can make those configurable too
but for now its easier to pass just the IP address
plust its also require, so passing it without a named flag sounds convenient, and its something users are familiar with
thats what Consul does, what what Weave does, and now Docker SwarmKit does the same thing also (edited)
flags will differ, as there are some Kubernetes-specifics aspects to what join does, but basic join semantics will remain _familiar_
if we do marry `kube-discovery` with the API, we might do `kubeadm join host:port`, as wed end-up with a single port to care about
but we havent yet

View File

@ -20,29 +20,76 @@ import (
"net"
)
type BootstrapParams struct {
// TODO this is mostly out of date and bloated now, let's revisit this soon
Discovery *OutOfBandDiscovery
EnvParams map[string]string
type KubeadmConfig struct {
InitFlags
JoinFlags
ManualFlags
Secrets struct {
GivenToken string // dot-separated `<TokenID>.<Token>` set by the user
TokenID string // optional on master side, will be generated if not specified
Token []byte // optional on master side, will be generated if not specified
BearerToken string // set based on Token
}
EnvParams map[string]string // TODO(phase2) this is likely to be come componentconfig
}
type OutOfBandDiscovery struct {
// 'join-node' side
// TODO(phase2) should we add validatin funcs on these structs?
type InitFlags struct {
API struct {
AdvertiseAddrs []net.IP
ExternalDNSName []string
}
Services struct {
CIDR net.IPNet
DNSDomain string
}
CloudProvider string
}
const (
DefaultServiceDNSDomain = "cluster.local"
DefaultServicesCIDRString = "100.64.0.0/12"
)
var (
DefaultServicesCIDR *net.IPNet
ListOfCloudProviders = []string{
"aws",
"azure",
"cloudstack",
"gce",
"mesos",
"openstack",
"ovirt",
"rackspace",
"vsphere",
}
SupportedCloudProviders map[string]bool
)
func init() {
_, DefaultServicesCIDR, _ = net.ParseCIDR(DefaultServicesCIDRString)
SupportedCloudProviders = make(map[string]bool, len(ListOfCloudProviders))
for _, v := range ListOfCloudProviders {
SupportedCloudProviders[v] = true
}
}
type JoinFlags struct {
MasterAddrs []net.IP
}
// TODO(phase1?) we haven't decided whether manual sub commands should get merged into main commands...
type ManualFlags struct {
ApiServerURLs string // comma separated
CaCertFile string
GivenToken string // dot-separated `<TokenID>.<Token>` set by the user
TokenID string // optional on master side, will be generated if not specified
Token []byte // optional on master side, will be generated if not specified
BearerToken string // set based on Token
// 'init-master' side
ApiServerDNSName string // optional, used in master bootstrap
ListenIP net.IP // optional IP for master to listen on, rather than autodetect
UseHyperkubeImage bool
ListenIP net.IP // optional IP for master to listen on, rather than autodetect
}
type ClusterInfo struct {
// TODO Kind, apiVersion
// TODO clusterId, fetchedTime, expiredTime
// TODO(pahse1?) this may become simply `api.Config`
CertificateAuthorities []string `json:"certificateAuthorities"`
Endpoints []string `json:"endpoints"`
}

View File

@ -73,25 +73,25 @@ func NewKubeadmCommand(f *cmdutil.Factory, in io.Reader, out, err io.Writer, env
// TODO(phase2) detect interactive vs non-interactive use and adjust output accordingly
// i.e. make it automation friendly
//
// TODO(phase2) create an bastraction that defines files and the content that needs to
// TODO(phase2) create an abstraction that defines files and the content that needs to
// be written to disc and write it all in one go at the end as we have a lot of
// crapy little files written from different parts of this code; this could also
// be useful for testing
bootstrapParams := &kubeadmapi.BootstrapParams{
Discovery: &kubeadmapi.OutOfBandDiscovery{
// TODO(phase1) this type no longer makes sense here
},
EnvParams: envParams,
}
s := new(kubeadmapi.KubeadmConfig)
s.EnvParams = envParams
//s.InitFlags, s.JoinFlags = new(kubeadmapi.InitFlags), new(kubeadmapi.JoinFlags)
//s.ManualFlags = new(kubeadmapi.ManualFlags)
cmds.ResetFlags()
cmds.SetGlobalNormalizationFunc(flag.WarnWordSepNormalizeFunc)
cmds.AddCommand(NewCmdInit(out, bootstrapParams))
cmds.AddCommand(NewCmdJoin(out, bootstrapParams))
cmds.AddCommand(NewCmdUser(out, bootstrapParams))
cmds.AddCommand(NewCmdManual(out, bootstrapParams))
cmds.AddCommand(NewCmdInit(out, s))
cmds.AddCommand(NewCmdJoin(out, s))
cmds.AddCommand(NewCmdUser(out, s))
cmds.AddCommand(NewCmdManual(out, s))
return cmds
}

View File

@ -19,6 +19,7 @@ package cmd
import (
"fmt"
"io"
"net"
"github.com/renstrom/dedent"
"github.com/spf13/cobra"
@ -40,52 +41,91 @@ var (
`)
)
func NewCmdInit(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command {
func NewCmdInit(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command {
advertiseAddrs := &[]string{}
cmd := &cobra.Command{
Use: "init",
Short: "Run this on the first server you deploy onto.",
Run: func(cmd *cobra.Command, args []string) {
err := RunInit(out, cmd, args, params)
cmdutil.CheckErr(err)
err := RunInit(out, cmd, args, s, advertiseAddrs)
cmdutil.CheckErr(err) // TODO(phase1+) append alpha warning with bugs URL etc
},
}
cmd.PersistentFlags().IPVar(&params.Discovery.ListenIP, "listen-ip", nil,
`(optional) IP address to listen on, in case autodetection fails.`)
cmd.PersistentFlags().StringVar(&params.Discovery.GivenToken, "token", "",
`(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`)
cmd.PersistentFlags().BoolVar(&params.Discovery.UseHyperkubeImage, "use-hyperkube", false,
`(optional) Use the hyperkube image for running the apiserver, controller-manager, scheduler and proxy.`)
cmd.PersistentFlags().StringVar(
&s.Secrets.GivenToken, "token", "",
`(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`,
)
cmd.PersistentFlags().StringSliceVar(
advertiseAddrs, "api-advertise-addr", []string{},
`(optional) IP address to advertise, in case autodetection fails.`,
)
cmd.PersistentFlags().StringSliceVar(
&s.InitFlags.API.ExternalDNSName, "api-external-dns-name", []string{},
`(optional) DNS name to advertise, in case you have configured one yourself.`,
)
cmd.PersistentFlags().IPNetVar(
&s.InitFlags.Services.CIDR, "service-cidr", *kubeadmapi.DefaultServicesCIDR,
`(optional) use alterantive range of IP address for service VIPs, e.g. "10.16.0.0/12"`,
)
cmd.PersistentFlags().StringVar(
&s.InitFlags.Services.DNSDomain, "service-dns-domain", kubeadmapi.DefaultServiceDNSDomain,
`(optional) use alterantive domain name for services, e.g. "myorg.internal"`,
)
cmd.PersistentFlags().StringVar(
&s.InitFlags.CloudProvider, "cloud-provider", "",
`(optional) enable cloud proiver features (external load-balancers, storage, etc)`,
)
return cmd
}
func RunInit(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error {
func RunInit(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.KubeadmConfig, advertiseAddrs *[]string) error {
// Auto-detect the IP
if params.Discovery.ListenIP == nil {
if len(*advertiseAddrs) == 0 {
// TODO(phase1+) perhaps we could actually grab eth0 and eth1
ip, err := netutil.ChooseHostInterface()
if err != nil {
return err
}
params.Discovery.ListenIP = ip
s.InitFlags.API.AdvertiseAddrs = []net.IP{ip}
} else {
for _, i := range *advertiseAddrs {
addr := net.ParseIP(i)
if addr == nil {
return fmt.Errorf("<cmd/init> failed to parse flag (%q) as an IP address", "--api-advertise-addr="+i)
}
s.InitFlags.API.AdvertiseAddrs = append(s.InitFlags.API.AdvertiseAddrs, addr)
}
}
if err := kubemaster.CreateTokenAuthFile(params); err != nil {
if s.InitFlags.CloudProvider != "" {
// TODO(phase2) we should be able to auto-detect it and check whether things like IAM roles are correct
if _, ok := kubeadmapi.SupportedCloudProviders[s.InitFlags.CloudProvider]; !ok {
return fmt.Errorf("<cmd/init> cloud provider %q is not supported, you can use any of %v, or leave it unset", s.InitFlags.CloudProvider, kubeadmapi.ListOfCloudProviders)
}
}
if err := kubemaster.CreateTokenAuthFile(s); err != nil {
return err
}
if err := kubemaster.WriteStaticPodManifests(params); err != nil {
if err := kubemaster.WriteStaticPodManifests(s); err != nil {
return err
}
caKey, caCert, err := kubemaster.CreatePKIAssets(params)
caKey, caCert, err := kubemaster.CreatePKIAssets(s)
if err != nil {
return err
}
kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(params, []string{"kubelet", "admin"}, caKey, caCert)
kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(s, []string{"kubelet", "admin"}, caKey, caCert)
if err != nil {
return err
}
for name, kubeconfig := range kubeconfigs {
if err := kubeadmutil.WriteKubeconfigIfNotExists(params, name, kubeconfig); err != nil {
if err := kubeadmutil.WriteKubeconfigIfNotExists(s, name, kubeconfig); err != nil {
return err
}
}
@ -99,18 +139,18 @@ func RunInit(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmap
return err
}
if err := kubemaster.CreateDiscoveryDeploymentAndSecret(params, client, caCert); err != nil {
if err := kubemaster.CreateDiscoveryDeploymentAndSecret(s, client, caCert); err != nil {
return err
}
if err := kubemaster.CreateEssentialAddons(params, client); err != nil {
if err := kubemaster.CreateEssentialAddons(s, client); err != nil {
return err
}
// TODO use templates to reference struct fields directly as order of args is fragile
// TODO(phase1+) use templates to reference struct fields directly as order of args is fragile
fmt.Fprintf(out, init_done_msgf,
params.Discovery.GivenToken,
params.Discovery.ListenIP,
s.Secrets.GivenToken,
s.InitFlags.API.AdvertiseAddrs[0].String(),
)
return nil

View File

@ -19,6 +19,7 @@ package cmd
import (
"fmt"
"io"
"net"
"github.com/renstrom/dedent"
"github.com/spf13/cobra"
@ -40,44 +41,52 @@ var (
`)
)
func NewCmdJoin(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command {
func NewCmdJoin(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command {
cmd := &cobra.Command{
Use: "join",
Short: "Run this on other servers to join an existing cluster.",
Run: func(cmd *cobra.Command, args []string) {
err := RunJoin(out, cmd, args, params)
err := RunJoin(out, cmd, args, s)
cmdutil.CheckErr(err)
},
}
// TODO this should become `kubeadm join --token=<...> <master-ip-addr>`
cmd.PersistentFlags().StringVarP(&params.Discovery.ApiServerURLs, "api-server-urls", "", "",
`Comma separated list of API server URLs. Typically this might be just
https://<address-of-master>:8080/`)
cmd.PersistentFlags().StringVarP(&params.Discovery.GivenToken, "token", "", "",
`Shared secret used to secure bootstrap. Must match output of 'init-master'.`)
cmd.PersistentFlags().StringVarP(
&s.Secrets.GivenToken, "token", "", "",
`Shared secret used to secure bootstrap. Must match output of 'init-master'.`,
)
return cmd
}
func RunJoin(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error {
ok, err := kubeadmutil.UseGivenTokenIfValid(params)
func RunJoin(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.KubeadmConfig) error {
// TODO this we are missing args from the help text, there should be a way to tell cobra about it
if len(args) == 0 {
return fmt.Errorf("<cmd/join> must specify master IP address (see --help)")
}
for _, i := range args {
addr := net.ParseIP(i) // TODO(phase1+) should allow resolvable names too
if addr == nil {
return fmt.Errorf("<cmd/join> failed parse argument (%q) as an IP address", i)
}
s.JoinFlags.MasterAddrs = append(s.JoinFlags.MasterAddrs, addr)
}
ok, err := kubeadmutil.UseGivenTokenIfValid(s)
if !ok {
if err != nil {
return fmt.Errorf("%s (see --help)\n", err)
return fmt.Errorf("<cmd/join> %s (see --help)\n", err)
}
return fmt.Errorf("Must specify --token (see --help)\n")
}
if params.Discovery.ApiServerURLs == "" {
return fmt.Errorf("Must specify --api-server-urls (see --help)\n")
}
kubeconfig, err := kubenode.RetrieveTrustedClusterInfo(params)
kubeconfig, err := kubenode.RetrieveTrustedClusterInfo(s)
if err != nil {
return err
}
err = kubeadmutil.WriteKubeconfigIfNotExists(params, "kubelet", kubeconfig)
err = kubeadmutil.WriteKubeconfigIfNotExists(s, "kubelet", kubeconfig)
if err != nil {
return err
}

View File

@ -19,6 +19,7 @@ package cmd
import (
"fmt"
"io"
"net"
"github.com/renstrom/dedent"
"github.com/spf13/cobra"
@ -58,23 +59,23 @@ var (
`)
)
// TODO --token here becomes Discovery.BearerToken and not Discovery.GivenToken
// TODO --token here becomes `s.Secrets.BearerToken` and not `s.Secrets.GivenToken`
// may be we should make it the same and ask user to pass dot-separated tokens
// in any of the modes; we could also enable discovery API in the manual mode just
// as well, there is no reason we shouldn't let user mix and match modes, unless
// it is too difficult to support
func NewCmdManual(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command {
func NewCmdManual(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command {
cmd := &cobra.Command{
Use: "manual",
Short: "Advanced, less-automated functionality, for power users.",
// TODO put example usage in the Long description here
}
cmd.AddCommand(NewCmdManualBootstrap(out, params))
cmd.AddCommand(NewCmdManualBootstrap(out, s))
return cmd
}
func NewCmdManualBootstrap(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command {
func NewCmdManualBootstrap(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command {
cmd := &cobra.Command{
Use: "bootstrap",
Short: "Manually bootstrap a cluster 'out-of-band'",
@ -85,13 +86,14 @@ func NewCmdManualBootstrap(out io.Writer, params *kubeadmapi.BootstrapParams) *c
Run: func(cmd *cobra.Command, args []string) {
},
}
cmd.AddCommand(NewCmdManualBootstrapInitMaster(out, params))
cmd.AddCommand(NewCmdManualBootstrapJoinNode(out, params))
cmd.AddCommand(NewCmdManualBootstrapInitMaster(out, s))
cmd.AddCommand(NewCmdManualBootstrapJoinNode(out, s))
return cmd
}
func NewCmdManualBootstrapInitMaster(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command {
func NewCmdManualBootstrapInitMaster(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command {
advertiseAddrs := &[]string{}
cmd := &cobra.Command{
Use: "init-master",
Short: "Manually bootstrap a master 'out-of-band'",
@ -101,101 +103,140 @@ func NewCmdManualBootstrapInitMaster(out io.Writer, params *kubeadmapi.Bootstrap
components.
`),
Run: func(cmd *cobra.Command, args []string) {
err := RunManualBootstrapInitMaster(out, cmd, args, params)
err := RunManualBootstrapInitMaster(out, cmd, args, s, advertiseAddrs)
cmdutil.CheckErr(err)
},
}
params.Discovery.ApiServerURLs = "http://127.0.0.1:8080/" // On the master, assume you can talk to the API server
cmd.PersistentFlags().StringVar(&params.Discovery.ApiServerDNSName, "api-dns-name", "",
`(optional) DNS name for the API server, will be encoded into
subjectAltName in the resulting (generated) TLS certificates`)
cmd.PersistentFlags().IPVar(&params.Discovery.ListenIP, "listen-ip", nil,
`(optional) IP address to listen on, in case autodetection fails.`)
cmd.PersistentFlags().StringVar(&params.Discovery.BearerToken, "token", "",
`(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`)
cmd.PersistentFlags().StringVar(
&s.Secrets.BearerToken, "token", "",
`(optional) Shared secret used to secure bootstrap. Will be generated and displayed if not provided.`,
)
cmd.PersistentFlags().StringSliceVar(
advertiseAddrs, "api-advertise-addr", nil,
`(optional) IP address to advertise, in case autodetection fails.`,
)
cmd.PersistentFlags().StringSliceVar(
&s.InitFlags.API.ExternalDNSName, "api-external-dns-name", []string{},
`(optional) DNS name to advertise, in case you have configured one yourself.`,
)
_, defaultServicesCIDR, _ := net.ParseCIDR("100.64.0.0/12")
cmd.PersistentFlags().IPNetVar(
&s.InitFlags.Services.CIDR, "service-cidr", *defaultServicesCIDR,
`(optional) use alterantive range of IP address for service VIPs, e.g. "10.16.0.0/12"`,
)
cmd.PersistentFlags().StringVar(
&s.InitFlags.Services.DNSDomain, "service-dns-domain", "cluster.local",
`(optional) use alterantive domain name for services, e.g. "myorg.internal"`,
)
return cmd
}
func RunManualBootstrapInitMaster(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error {
func RunManualBootstrapInitMaster(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.KubeadmConfig, advertiseAddrs *[]string) error {
// Auto-detect the IP
if params.Discovery.ListenIP == nil {
if len(*advertiseAddrs) == 0 {
// TODO(phase1+) perhaps we could actually grab eth0 and eth1
ip, err := netutil.ChooseHostInterface()
if err != nil {
return err
}
params.Discovery.ListenIP = ip
s.InitFlags.API.AdvertiseAddrs = []net.IP{ip}
} else {
for _, i := range *advertiseAddrs {
addr := net.ParseIP(i)
if addr == nil {
return fmt.Errorf("<cmd/init> failed to parse flag (%q) as an IP address", "--api-advertise-addr="+i)
}
s.InitFlags.API.AdvertiseAddrs = append(s.InitFlags.API.AdvertiseAddrs, addr)
}
}
if err := kubemaster.CreateTokenAuthFile(params); err != nil {
if err := kubemaster.CreateTokenAuthFile(s); err != nil {
return err
}
if err := kubemaster.WriteStaticPodManifests(params); err != nil {
if err := kubemaster.WriteStaticPodManifests(s); err != nil {
return err
}
caKey, caCert, err := kubemaster.CreatePKIAssets(params)
caKey, caCert, err := kubemaster.CreatePKIAssets(s)
if err != nil {
return err
}
kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(params, []string{"kubelet", "admin"}, caKey, caCert)
kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(s, []string{"kubelet", "admin"}, caKey, caCert)
if err != nil {
return err
}
for name, kubeconfig := range kubeconfigs {
if err := kubeadmutil.WriteKubeconfigIfNotExists(params, name, kubeconfig); err != nil {
if err := kubeadmutil.WriteKubeconfigIfNotExists(s, name, kubeconfig); err != nil {
return err
}
}
// TODO we have most of cmd/init functionality here, except for `CreateDiscoveryDeploymentAndSecret()`
// it may be a good idea to just merge the two commands into one, and it's something we have started talking
// about, the only question is where disco service should be an opt-out...
client, err := kubemaster.CreateClientAndWaitForAPI(kubeconfigs["admin"])
if err != nil {
return err
}
if err := kubemaster.UpdateMasterRoleLabelsAndTaints(client); err != nil {
return err
}
if err := kubemaster.CreateEssentialAddons(s, client); err != nil {
return err
}
// TODO use templates to reference struct fields directly as order of args is fragile
fmt.Fprintf(out, manual_init_done_msgf,
params.Discovery.BearerToken,
params.Discovery.ListenIP,
s.Secrets.BearerToken,
s.InitFlags.API.AdvertiseAddrs[0].String(),
)
return nil
}
func NewCmdManualBootstrapJoinNode(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command {
func NewCmdManualBootstrapJoinNode(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command {
cmd := &cobra.Command{
Use: "join-node",
Short: "Manually bootstrap a node 'out-of-band', joining it into a cluster with extant control plane",
Run: func(cmd *cobra.Command, args []string) {
err := RunManualBootstrapJoinNode(out, cmd, args, params)
err := RunManualBootstrapJoinNode(out, cmd, args, s)
cmdutil.CheckErr(err)
},
}
cmd.PersistentFlags().StringVarP(&params.Discovery.CaCertFile, "ca-cert-file", "", "",
cmd.PersistentFlags().StringVarP(&s.ManualFlags.CaCertFile, "ca-cert-file", "", "",
`Path to a CA cert file in PEM format. The same CA cert must be distributed to
all servers.`)
cmd.PersistentFlags().StringVarP(&params.Discovery.ApiServerURLs, "api-server-urls", "", "",
cmd.PersistentFlags().StringVarP(&s.ManualFlags.ApiServerURLs, "api-server-urls", "", "",
`Comma separated list of API server URLs. Typically this might be just
https://<address-of-master>:8080/`)
cmd.PersistentFlags().StringVarP(&params.Discovery.BearerToken, "token", "", "",
cmd.PersistentFlags().StringVarP(&s.ManualFlags.BearerToken, "token", "", "",
`Shared secret used to secure bootstrap. Must match output of 'init-master'.`)
return cmd
}
func RunManualBootstrapJoinNode(out io.Writer, cmd *cobra.Command, args []string, params *kubeadmapi.BootstrapParams) error {
if params.Discovery.CaCertFile == "" {
func RunManualBootstrapJoinNode(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.KubeadmConfig) error {
if s.ManualFlags.CaCertFile == "" {
fmt.Fprintf(out, "Must specify --ca-cert-file (see --help)\n")
return nil
}
if params.Discovery.ApiServerURLs == "" {
if s.ManualFlags.ApiServerURLs == "" {
fmt.Fprintf(out, "Must specify --api-server-urls (see --help)\n")
return nil
}
kubeconfig, err := kubenode.PerformTLSBootstrapFromParams(params)
kubeconfig, err := kubenode.PerformTLSBootstrapFromConfig(s)
if err != nil {
fmt.Fprintf(out, "Failed to perform TLS bootstrap: %s\n", err)
return err
}
err = kubeadmutil.WriteKubeconfigIfNotExists(params, "kubelet", kubeconfig)
err = kubeadmutil.WriteKubeconfigIfNotExists(s, "kubelet", kubeconfig)
if err != nil {
fmt.Fprintf(out, "Unable to write config for node:\n%s\n", err)
return err

View File

@ -23,7 +23,7 @@ import (
kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api"
)
func NewCmdUser(out io.Writer, params *kubeadmapi.BootstrapParams) *cobra.Command {
func NewCmdUser(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command {
cmd := &cobra.Command{
Use: "user",
Short: "Get initial admin credentials for a cluster.", // using TLS bootstrap

View File

@ -24,53 +24,42 @@ import (
const (
KubeEtcdImage = "etcd"
KubeApiServerImage = "apiserver"
KubeApiServerImage = "apiserver"
KubeControllerManagerImage = "controller-manager"
KubeSchedulerImage = "scheduler"
KubeProxyImage = "proxy"
KubeSchedulerImage = "scheduler"
KubeProxyImage = "proxy"
KubeDnsImage = "kube-dns"
KubeDnsmasqImage = "dnsmasq"
KubeDnsImage = "kube-dns"
KubeDnsmasqImage = "dnsmasq"
KubeExechealthzImage = "exechealthz"
gcrPrefix = "gcr.io/google_containers"
gcrPrefix = "gcr.io/google_containers"
etcdVersion = "2.2.5"
kubeVersion = "v1.4.0-alpha.3"
kubeDnsVersion = "1.7"
dnsmasqVersion = "1.3"
kubeDnsVersion = "1.7"
dnsmasqVersion = "1.3"
exechealthzVersion = "1.1"
)
func GetCoreImage(image string, overrideImage string, useHyperkube bool) string {
func GetCoreImage(image string, overrideImage string) string {
if overrideImage != "" {
return overrideImage
}
if useHyperkube {
return map[string]string{
KubeEtcdImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "etcd", runtime.GOARCH, etcdVersion),
KubeApiServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "hyperkube", runtime.GOARCH, kubeVersion),
KubeControllerManagerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "hyperkube", runtime.GOARCH, kubeVersion),
KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "hyperkube", runtime.GOARCH, kubeVersion),
KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "hyperkube", runtime.GOARCH, kubeVersion),
}[image]
}
return map[string]string{
KubeEtcdImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "etcd", runtime.GOARCH, etcdVersion),
KubeApiServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-apiserver", runtime.GOARCH, kubeVersion),
KubeEtcdImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "etcd", runtime.GOARCH, etcdVersion),
KubeApiServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-apiserver", runtime.GOARCH, kubeVersion),
KubeControllerManagerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-controller-manager", runtime.GOARCH, kubeVersion),
KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-scheduler", runtime.GOARCH, kubeVersion),
KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-proxy", runtime.GOARCH, kubeVersion),
KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-scheduler", runtime.GOARCH, kubeVersion),
KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-proxy", runtime.GOARCH, kubeVersion),
}[image]
}
func GetAddonImage(image string) string {
return map[string]string{
KubeDnsImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-dns", runtime.GOARCH, kubeDnsVersion),
KubeDnsmasqImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-dnsmasq", runtime.GOARCH, dnsmasqVersion),
KubeDnsImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kubedns", runtime.GOARCH, kubeDnsVersion),
KubeDnsmasqImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-dnsmasq", runtime.GOARCH, dnsmasqVersion),
KubeExechealthzImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "exechealthz", runtime.GOARCH, exechealthzVersion),
}[image]
}

View File

@ -25,20 +25,18 @@ import (
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api"
"k8s.io/kubernetes/pkg/kubeadm/images"
ipallocator "k8s.io/kubernetes/pkg/registry/service/ipallocator"
"k8s.io/kubernetes/pkg/util/intstr"
)
func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec {
func createKubeProxyPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec {
privilegedTrue := true
return api.PodSpec{
SecurityContext: &api.PodSecurityContext{HostNetwork: true},
Containers: []api.Container{{
Name: "kube-proxy",
Image: images.GetCoreImage(images.KubeProxyImage, params.EnvParams["hyperkube_image"], params.Discovery.UseHyperkubeImage),
Command: append(getImageEntrypoint("proxy", params.Discovery.UseHyperkubeImage), []string{
"--kubeconfig=/run/kubeconfig",
params.EnvParams["component_loglevel"],
}...),
Name: kubeProxy,
Image: images.GetCoreImage(images.KubeProxyImage, s.EnvParams["hyperkube_image"]),
Command: append(getComponentCommand("proxy", s), "--kubeconfig=/run/kubeconfig"),
SecurityContext: &api.SecurityContext{Privileged: &privilegedTrue},
VolumeMounts: []api.VolumeMount{
{
@ -67,7 +65,7 @@ func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec {
{
Name: "kubeconfig",
VolumeSource: api.VolumeSource{
HostPath: &api.HostPathVolumeSource{Path: path.Join(params.EnvParams["prefix"], "kubelet.conf")},
HostPath: &api.HostPathVolumeSource{Path: path.Join(s.EnvParams["prefix"], "kubelet.conf")},
},
},
{
@ -80,8 +78,7 @@ func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec {
}
}
// TODO(phase1): Create the DNS service as well
func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec {
func createKubeDNSPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec {
dnsPodResources := api.ResourceList{
api.ResourceName(api.ResourceCPU): resource.MustParse("100m"),
@ -93,6 +90,16 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec {
api.ResourceName(api.ResourceMemory): resource.MustParse("50Mi"),
}
kubeDNSPort := int32(10053)
dnsmasqPort := int32(53)
nslookup := fmt.Sprintf("nslookup kubernetes.default.svc.%s 127.0.0.1", s.InitFlags.Services.DNSDomain)
nslookup = fmt.Sprintf("-cmd=%s:%d >/dev/null && %s:%d >/dev/null",
nslookup, dnsmasqPort,
nslookup, kubeDNSPort,
)
return api.PodSpec{
Containers: []api.Container{
// DNS server
@ -104,8 +111,8 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec {
Requests: dnsPodResources,
},
Args: []string{
"--domain=" + params.EnvParams["dns_domain"],
"--dns-port=10053",
fmt.Sprintf("--domain=%s", s.InitFlags.Services.DNSDomain),
fmt.Sprintf("--dns-port=%d", kubeDNSPort),
// TODO __PILLAR__FEDERATIONS__DOMAIN__MAP__
},
LivenessProbe: &api.Probe{
@ -136,12 +143,12 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec {
},
Ports: []api.ContainerPort{
{
ContainerPort: 10053,
ContainerPort: kubeDNSPort,
Name: "dns-local",
Protocol: api.ProtocolUDP,
},
{
ContainerPort: 10053,
ContainerPort: kubeDNSPort,
Name: "dns-tcp-local",
Protocol: api.ProtocolTCP,
},
@ -158,16 +165,16 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec {
Args: []string{
"--cache-size=1000",
"--no-resolv",
"--server=127.0.0.1#10053",
fmt.Sprintf("--server=127.0.0.1#%d", kubeDNSPort),
},
Ports: []api.ContainerPort{
{
ContainerPort: 53,
ContainerPort: dnsmasqPort,
Name: "dns",
Protocol: api.ProtocolUDP,
},
{
ContainerPort: 53,
ContainerPort: dnsmasqPort,
Name: "dns-tcp",
Protocol: api.ProtocolTCP,
},
@ -182,7 +189,7 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec {
Requests: healthzPodResources,
},
Args: []string{
"-cmd=nslookup kubernetes.default.svc." + params.EnvParams["dns_domain"] + " 127.0.0.1 >/dev/null && nslookup kubernetes.default.svc." + params.EnvParams["dns_domain"] + " 127.0.0.1:10053 >/dev/null",
nslookup,
"-port=8080",
"-quiet",
},
@ -196,19 +203,27 @@ func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec {
}
}
func createKubeDNSServiceSpec(params *kubeadmapi.BootstrapParams) api.ServiceSpec {
return api.ServiceSpec{
func createKubeDNSServiceSpec(s *kubeadmapi.KubeadmConfig) (*api.ServiceSpec, error) {
ip, err := ipallocator.GetIndexedIP(&s.InitFlags.Services.CIDR, 10)
if err != nil {
return nil, fmt.Errorf("unable to allocate IP address for kube-dns addon from the given CIDR (%q) [%s]", s.InitFlags.Services.CIDR, err)
}
svc := &api.ServiceSpec{
Selector: map[string]string{"name": "kube-dns"},
Ports: []api.ServicePort{
{Name: "dns", Port: 53, Protocol: api.ProtocolUDP},
{Name: "dns-tcp", Port: 53, Protocol: api.ProtocolTCP},
},
ClusterIP: "100.64.0.2",
ClusterIP: ip.String(),
}
return svc, nil
}
func CreateEssentialAddons(params *kubeadmapi.BootstrapParams, client *clientset.Clientset) error {
kubeProxyDaemonSet := NewDaemonSet("kube-proxy", createKubeProxyPodSpec(params))
func CreateEssentialAddons(s *kubeadmapi.KubeadmConfig, client *clientset.Clientset) error {
kubeProxyDaemonSet := NewDaemonSet(kubeProxy, createKubeProxyPodSpec(s))
SetMasterTaintTolerations(&kubeProxyDaemonSet.Spec.Template.ObjectMeta)
if _, err := client.Extensions().DaemonSets(api.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil {
@ -217,15 +232,19 @@ func CreateEssentialAddons(params *kubeadmapi.BootstrapParams, client *clientset
fmt.Println("<master/addons> created essential addon: kube-proxy")
kubeDNSDeployment := NewDeployment("kube-dns", 1, createKubeDNSPodSpec(params))
kubeDNSDeployment := NewDeployment("kube-dns", 1, createKubeDNSPodSpec(s))
SetMasterTaintTolerations(&kubeDNSDeployment.Spec.Template.ObjectMeta)
if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kubeDNSDeployment); err != nil {
return fmt.Errorf("<master/addons> failed creating essential kube-dns addon [%s]", err)
}
kubeDNSService := NewService("kube-dns", createKubeDNSServiceSpec(params))
kubeDNSServiceSpec, err := createKubeDNSServiceSpec(s)
if err != nil {
return fmt.Errorf("<master/addons> failed creating essential kube-dns addon - %s", err)
}
kubeDNSService := NewService("kube-dns", *kubeDNSServiceSpec)
if _, err := client.Services(api.NamespaceSystem).Create(kubeDNSService); err != nil {
return fmt.Errorf("<master/addons> failed creating essential kube-dns addon [%s]", err)
}

View File

@ -39,7 +39,7 @@ const (
kubeDiscoverySecretName = "clusterinfo"
)
func encodeKubeDiscoverySecretData(params *kubeadmapi.BootstrapParams, caCert *x509.Certificate) map[string][]byte {
func encodeKubeDiscoverySecretData(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate) map[string][]byte {
// TODO ListenIP is probably not the right now, although it's best we have right now
// if user provides a DNS name, or anything else, we should use that, may be it's really
// the list of all SANs (minus internal DNS names and service IP)?
@ -50,8 +50,11 @@ func encodeKubeDiscoverySecretData(params *kubeadmapi.BootstrapParams, caCert *x
tokenMap = map[string]string{}
)
endpointList = append(endpointList, fmt.Sprintf("https://%s:443", params.Discovery.ListenIP))
tokenMap[params.Discovery.TokenID] = hex.EncodeToString(params.Discovery.Token)
for _, addr := range s.InitFlags.API.AdvertiseAddrs {
endpointList = append(endpointList, fmt.Sprintf("https://%s:443", addr.String()))
}
tokenMap[s.Secrets.TokenID] = hex.EncodeToString(s.Secrets.Token)
data["endpoint-list.json"], _ = json.Marshal(endpointList)
data["token-map.json"], _ = json.Marshal(tokenMap)
@ -60,7 +63,7 @@ func encodeKubeDiscoverySecretData(params *kubeadmapi.BootstrapParams, caCert *x
return data
}
func newKubeDiscoveryPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec {
func newKubeDiscoveryPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec {
return api.PodSpec{
// We have to use host network namespace, as `HostPort`/`HostIP` are Docker's
// buisness and CNI support isn't quite there yet (except for kubenet)
@ -69,7 +72,7 @@ func newKubeDiscoveryPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec {
SecurityContext: &api.PodSecurityContext{HostNetwork: true},
Containers: []api.Container{{
Name: kubeDiscoverynName,
Image: params.EnvParams["discovery_image"],
Image: s.EnvParams["discovery_image"],
Command: []string{"/usr/bin/kube-discovery"},
VolumeMounts: []api.VolumeMount{{
Name: kubeDiscoverySecretName,
@ -77,7 +80,8 @@ func newKubeDiscoveryPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec {
ReadOnly: true,
}},
Ports: []api.ContainerPort{
// TODO when CNI issue (#31307) is resolved, we should add `HostIP: params.Discovery.ListenIP`
// TODO when CNI issue (#31307) is resolved, we should consider adding
// `HostIP: s.API.AdvertiseAddrs[0]`, if there is only one address`
{Name: "http", ContainerPort: 9898, HostPort: 9898},
},
}},
@ -90,13 +94,13 @@ func newKubeDiscoveryPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec {
}
}
func newKubeDiscovery(params *kubeadmapi.BootstrapParams, caCert *x509.Certificate) kubeDiscovery {
func newKubeDiscovery(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate) kubeDiscovery {
kd := kubeDiscovery{
Deployment: NewDeployment(kubeDiscoverynName, 1, newKubeDiscoveryPodSpec(params)),
Deployment: NewDeployment(kubeDiscoverynName, 1, newKubeDiscoveryPodSpec(s)),
Secret: &api.Secret{
ObjectMeta: api.ObjectMeta{Name: kubeDiscoverySecretName},
Type: api.SecretTypeOpaque,
Data: encodeKubeDiscoverySecretData(params, caCert),
Data: encodeKubeDiscoverySecretData(s, caCert),
},
}
@ -106,8 +110,8 @@ func newKubeDiscovery(params *kubeadmapi.BootstrapParams, caCert *x509.Certifica
return kd
}
func CreateDiscoveryDeploymentAndSecret(params *kubeadmapi.BootstrapParams, client *clientset.Clientset, caCert *x509.Certificate) error {
kd := newKubeDiscovery(params, caCert)
func CreateDiscoveryDeploymentAndSecret(s *kubeadmapi.KubeadmConfig, client *clientset.Clientset, caCert *x509.Certificate) error {
kd := newKubeDiscovery(s, caCert)
if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kd.Deployment); err != nil {
return fmt.Errorf("<master/discovery> failed to create %q deployment [%s]", kubeDiscoverynName, err)

View File

@ -28,11 +28,14 @@ import (
certutil "k8s.io/kubernetes/pkg/util/cert"
)
func CreateCertsAndConfigForClients(params *kubeadmapi.BootstrapParams, clientNames []string, caKey *rsa.PrivateKey, caCert *x509.Certificate) (map[string]*clientcmdapi.Config, error) {
func CreateCertsAndConfigForClients(s *kubeadmapi.KubeadmConfig, clientNames []string, caKey *rsa.PrivateKey, caCert *x509.Certificate) (map[string]*clientcmdapi.Config, error) {
basicClientConfig := kubeadmutil.CreateBasicClientConfig(
"kubernetes",
fmt.Sprintf("https://%s:443", params.Discovery.ListenIP),
// TODO this is not great, but there is only one address we can use here
// so we'll pick the first one, there is much of chance to have an empty
// slice by the time this gets called
fmt.Sprintf("https://%s:443", s.InitFlags.API.AdvertiseAddrs[0]),
certutil.EncodeCertPEM(caCert),
)

View File

@ -36,85 +36,65 @@ import (
// init master` and `kubeadm manual bootstrap master` can get going.
const (
SERVICE_CLUSTER_IP_RANGE = "--service-cluster-ip-range=100.64.0.0/12"
CLUSTER_NAME = "--cluster-name=kubernetes"
MASTER = "--master=127.0.0.1:8080"
DefaultClusterName = "--cluster-name=kubernetes"
etcd = "etcd"
apiServer = "apiserver"
controllerManager = "controller-manager"
scheduler = "scheduler"
proxy = "proxy"
kubeAPIServer = "kube-apiserver"
kubeControllerManager = "kube-controller-manager"
kubeScheduler = "kube-scheduler"
kubeProxy = "kube-proxy"
)
// TODO look into what this really means, scheduler prints it for some reason
//
//E0817 17:53:22.242658 1 event.go:258] Could not construct reference to: '&api.Endpoints{TypeMeta:unversioned.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:api.ObjectMeta{Name:"kube-scheduler", GenerateName:"", Namespace:"kube-system", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:unversioned.Time{Time:time.Time{sec:0, nsec:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*unversioned.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]api.OwnerReference(nil), Finalizers:[]string(nil)}, Subsets:[]api.EndpointSubset(nil)}' due to: 'selfLink was empty, can't make reference'. Will not report event: 'Normal' '%v became leader' 'moby'
func WriteStaticPodManifests(params *kubeadmapi.BootstrapParams) error {
func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error {
staticPodSpecs := map[string]api.Pod{
// TODO this needs a volume
"etcd": componentPod(api.Container{
etcd: componentPod(api.Container{
Command: []string{
"/usr/local/bin/etcd",
"--listen-client-urls=http://127.0.0.1:2379",
"--advertise-client-urls=http://127.0.0.1:2379",
"--data-dir=/var/etcd/data",
},
Image: images.GetCoreImage(images.KubeEtcdImage, params.EnvParams["etcd_image"], params.Discovery.UseHyperkubeImage),
Image: images.GetCoreImage(images.KubeEtcdImage, s.EnvParams["etcd_image"]),
LivenessProbe: componentProbe(2379, "/health"),
Name: "etcd-server",
Name: etcd,
Resources: componentResources("200m"),
}),
// TODO bind-mount certs in
"kube-apiserver": componentPod(api.Container{
Name: "kube-apiserver",
Image: images.GetCoreImage(images.KubeApiServerImage, params.EnvParams["hyperkube_image"], params.Discovery.UseHyperkubeImage),
Command: append(getImageEntrypoint("apiserver", params.Discovery.UseHyperkubeImage), []string{
"--address=127.0.0.1",
"--etcd-servers=http://127.0.0.1:2379",
"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota",
SERVICE_CLUSTER_IP_RANGE,
"--service-account-key-file=/etc/kubernetes/pki/apiserver-key.pem",
"--client-ca-file=/etc/kubernetes/pki/ca.pem",
"--tls-cert-file=/etc/kubernetes/pki/apiserver.pem",
"--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem",
"--secure-port=443",
"--allow-privileged",
params.EnvParams["component_loglevel"],
"--token-auth-file=/etc/kubernetes/pki/tokens.csv",
}...),
kubeAPIServer: componentPod(api.Container{
Name: kubeAPIServer,
Image: images.GetCoreImage(images.KubeApiServerImage, s.EnvParams["hyperkube_image"]),
Command: getComponentCommand(apiServer, s),
VolumeMounts: []api.VolumeMount{pkiVolumeMount()},
LivenessProbe: componentProbe(8080, "/healthz"),
Resources: componentResources("250m"),
}, pkiVolume(params)),
"kube-controller-manager": componentPod(api.Container{
Name: "kube-controller-manager",
Image: images.GetCoreImage(images.KubeControllerManagerImage, params.EnvParams["hyperkube_image"], params.Discovery.UseHyperkubeImage),
Command: append(getImageEntrypoint("controller-manager", params.Discovery.UseHyperkubeImage), []string{
"--leader-elect",
MASTER,
CLUSTER_NAME,
"--root-ca-file=/etc/kubernetes/pki/ca.pem",
"--service-account-private-key-file=/etc/kubernetes/pki/apiserver-key.pem",
"--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem",
"--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem",
"--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap",
"--cluster-cidr=10.2.0.0/16",
params.EnvParams["component_loglevel"],
}...),
}, pkiVolume(s)),
kubeControllerManager: componentPod(api.Container{
Name: kubeControllerManager,
Image: images.GetCoreImage(images.KubeControllerManagerImage, s.EnvParams["hyperkube_image"]),
Command: getComponentCommand(controllerManager, s),
VolumeMounts: []api.VolumeMount{pkiVolumeMount()},
LivenessProbe: componentProbe(10252, "/healthz"),
Resources: componentResources("200m"),
}, pkiVolume(params)),
"kube-scheduler": componentPod(api.Container{
Name: "kube-scheduler",
Image: images.GetCoreImage(images.KubeSchedulerImage, params.EnvParams["hyperkube_image"], params.Discovery.UseHyperkubeImage),
Command: append(getImageEntrypoint("scheduler", params.Discovery.UseHyperkubeImage), []string{
"--leader-elect",
MASTER,
params.EnvParams["component_loglevel"],
}...),
}, pkiVolume(s)),
kubeScheduler: componentPod(api.Container{
Name: kubeScheduler,
Image: images.GetCoreImage(images.KubeSchedulerImage, s.EnvParams["hyperkube_image"]),
Command: getComponentCommand(scheduler, s),
LivenessProbe: componentProbe(10251, "/healthz"),
Resources: componentResources("100m"),
}),
}
manifestsPath := path.Join(params.EnvParams["prefix"], "manifests")
manifestsPath := path.Join(s.EnvParams["prefix"], "manifests")
if err := os.MkdirAll(manifestsPath, 0700); err != nil {
return fmt.Errorf("<master/manifests> failed to create directory %q [%s]", manifestsPath, err)
}
@ -131,11 +111,11 @@ func WriteStaticPodManifests(params *kubeadmapi.BootstrapParams) error {
return nil
}
func pkiVolume(params *kubeadmapi.BootstrapParams) api.Volume {
func pkiVolume(s *kubeadmapi.KubeadmConfig) api.Volume {
return api.Volume{
Name: "pki",
VolumeSource: api.VolumeSource{
HostPath: &api.HostPathVolumeSource{Path: params.EnvParams["host_pki_path"]},
HostPath: &api.HostPathVolumeSource{Path: s.EnvParams["host_pki_path"]},
},
}
}
@ -189,10 +169,51 @@ func componentPod(container api.Container, volumes ...api.Volume) api.Pod {
}
}
func getImageEntrypoint(component string, useHyperkube bool) []string {
if useHyperkube {
return []string{"/hyperkube", component}
func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command []string) {
baseFalgs := map[string][]string{
apiServer: []string{
"--address=127.0.0.1",
"--etcd-servers=http://127.0.0.1:2379",
"--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota",
"--service-cluster-ip-range=" + s.InitFlags.Services.CIDR.String(),
"--service-account-key-file=/etc/kubernetes/pki/apiserver-key.pem",
"--client-ca-file=/etc/kubernetes/pki/ca.pem",
"--tls-cert-file=/etc/kubernetes/pki/apiserver.pem",
"--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem",
"--secure-port=443",
"--allow-privileged",
"--token-auth-file=/etc/kubernetes/pki/tokens.csv",
},
controllerManager: []string{
"--leader-elect",
"--master=127.0.0.1:8080",
DefaultClusterName,
"--root-ca-file=/etc/kubernetes/pki/ca.pem",
"--service-account-private-key-file=/etc/kubernetes/pki/apiserver-key.pem",
"--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem",
"--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem",
"--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap",
"--cluster-cidr=" + s.InitFlags.Services.CIDR.String(),
},
scheduler: []string{
"--leader-elect",
"--master=127.0.0.1:8080",
},
proxy: []string{},
}
return []string{"/usr/local/bin/kube-" + component}
if s.EnvParams["hyperkube_image"] != "" {
command = []string{"/hyperkube", component}
} else {
command = []string{"/usr/local/bin/kube-" + component}
}
command = append(command, s.EnvParams["component_loglevel"])
command = append(command, baseFalgs[component]...)
if component == controllerManager && s.InitFlags.CloudProvider != "" {
command = append(command, "--cloud-provider="+s.InitFlags.CloudProvider)
}
return
}

View File

@ -20,10 +20,10 @@ import (
"crypto/rsa"
"crypto/x509"
"fmt"
"net"
"path"
kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api"
ipallocator "k8s.io/kubernetes/pkg/registry/service/ipallocator"
certutil "k8s.io/kubernetes/pkg/util/cert"
)
@ -51,19 +51,26 @@ func newCertificateAuthority() (*rsa.PrivateKey, *x509.Certificate, error) {
return key, cert, nil
}
func newServerKeyAndCert(caCert *x509.Certificate, caKey *rsa.PrivateKey, altNames certutil.AltNames) (*rsa.PrivateKey, *x509.Certificate, error) {
func newServerKeyAndCert(s *kubeadmapi.KubeadmConfig, caCert *x509.Certificate, caKey *rsa.PrivateKey, altNames certutil.AltNames) (*rsa.PrivateKey, *x509.Certificate, error) {
key, err := certutil.NewPrivateKey()
if err != nil {
return nil, nil, fmt.Errorf("unabel to create private key [%s]", err)
}
// TODO these are all hardcoded for now, but we need to figure out what shall we do here exactly
altNames.IPs = append(altNames.IPs, net.ParseIP("10.3.0.1"), net.ParseIP("10.16.0.1"), net.ParseIP("100.64.0.1"))
altNames.DNSNames = append(altNames.DNSNames,
internalAPIServerFQDN := []string{
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster.local",
)
fmt.Sprintf("kubernetes.default.svc.%s", s.InitFlags.Services.DNSDomain),
}
internalAPIServerVirtualIP, err := ipallocator.GetIndexedIP(&s.InitFlags.Services.CIDR, 1)
if err != nil {
return nil, nil, fmt.Errorf("unable to allocate IP address for the API server from the given CIDR (%q) [%s]")
}
altNames.IPs = append(altNames.IPs, internalAPIServerVirtualIP)
altNames.DNSNames = append(altNames.DNSNames, internalAPIServerFQDN...)
config := certutil.CertConfig{
CommonName: "kube-apiserver",
@ -131,21 +138,21 @@ func newServiceAccountKey() (*rsa.PrivateKey, error) {
return key, nil
}
func CreatePKIAssets(params *kubeadmapi.BootstrapParams) (*rsa.PrivateKey, *x509.Certificate, error) {
func CreatePKIAssets(s *kubeadmapi.KubeadmConfig) (*rsa.PrivateKey, *x509.Certificate, error) {
var (
err error
altNames certutil.AltNames // TODO actual SANs
altNames certutil.AltNames
)
if params.Discovery.ListenIP != nil {
altNames.IPs = append(altNames.IPs, params.Discovery.ListenIP)
if len(s.InitFlags.API.AdvertiseAddrs) > 0 {
altNames.IPs = append(altNames.IPs, s.InitFlags.API.AdvertiseAddrs...)
}
if params.Discovery.ApiServerDNSName != "" {
altNames.DNSNames = append(altNames.DNSNames, params.Discovery.ApiServerDNSName)
if len(s.InitFlags.API.ExternalDNSName) > 0 {
altNames.DNSNames = append(altNames.DNSNames, s.InitFlags.API.ExternalDNSName...)
}
pkiPath := path.Join(params.EnvParams["host_pki_path"])
pkiPath := path.Join(s.EnvParams["host_pki_path"])
caKey, caCert, err := newCertificateAuthority()
if err != nil {
@ -156,7 +163,7 @@ func CreatePKIAssets(params *kubeadmapi.BootstrapParams) (*rsa.PrivateKey, *x509
return nil, nil, fmt.Errorf("<master/pki> failure while saving CA keys and certificate - %s", err)
}
apiKey, apiCert, err := newServerKeyAndCert(caCert, caKey, altNames)
apiKey, apiCert, err := newServerKeyAndCert(s, caCert, caKey, altNames)
if err != nil {
return nil, nil, fmt.Errorf("<master/pki> failure while creating API server keys and certificate - %s", err)
}
@ -174,7 +181,7 @@ func CreatePKIAssets(params *kubeadmapi.BootstrapParams) (*rsa.PrivateKey, *x509
return nil, nil, fmt.Errorf("<master/pki> failure while saving service account singing keys - %s", err)
}
// TODO print a summary of SANs used and checksums (signatures) of each of the certiicates
fmt.Printf("<master/pki> created keys and certificates in %q\n", params.EnvParams["host_pki_path"])
// TODO(phase1+) print a summary of SANs used and checksums (signatures) of each of the certiicates
fmt.Printf("<master/pki> created keys and certificates in %q\n", pkiPath)
return caKey, caCert, nil
}

View File

@ -28,17 +28,17 @@ import (
"k8s.io/kubernetes/pkg/util/uuid"
)
func generateTokenIfNeeded(params *kubeadmapi.BootstrapParams) error {
ok, err := kubeadmutil.UseGivenTokenIfValid(params)
func generateTokenIfNeeded(s *kubeadmapi.KubeadmConfig) error {
ok, err := kubeadmutil.UseGivenTokenIfValid(s)
if !ok {
if err != nil {
return err
}
err = kubeadmutil.GenerateToken(params)
err = kubeadmutil.GenerateToken(s)
if err != nil {
return err
}
fmt.Printf("<master/tokens> generated token: %q\n", params.Discovery.GivenToken)
fmt.Printf("<master/tokens> generated token: %q\n", s.Secrets.GivenToken)
} else {
fmt.Println("<master/tokens> accepted provided token")
}
@ -46,15 +46,15 @@ func generateTokenIfNeeded(params *kubeadmapi.BootstrapParams) error {
return nil
}
func CreateTokenAuthFile(params *kubeadmapi.BootstrapParams) error {
tokenAuthFilePath := path.Join(params.EnvParams["host_pki_path"], "tokens.csv")
if err := generateTokenIfNeeded(params); err != nil {
func CreateTokenAuthFile(s *kubeadmapi.KubeadmConfig) error {
tokenAuthFilePath := path.Join(s.EnvParams["host_pki_path"], "tokens.csv")
if err := generateTokenIfNeeded(s); err != nil {
return fmt.Errorf("<master/tokens> failed to generate token(s) [%s]", err)
}
if err := os.MkdirAll(params.EnvParams["host_pki_path"], 0700); err != nil {
return fmt.Errorf("<master/tokens> failed to create directory %q [%s]", params.EnvParams["host_pki_path"], err)
if err := os.MkdirAll(s.EnvParams["host_pki_path"], 0700); err != nil {
return fmt.Errorf("<master/tokens> failed to create directory %q [%s]", s.EnvParams["host_pki_path"], err)
}
serialized := []byte(fmt.Sprintf("%s,kubeadm-node-csr,%s,system:kubelet-bootstrap\n", params.Discovery.BearerToken, uuid.NewUUID()))
serialized := []byte(fmt.Sprintf("%s,kubeadm-node-csr,%s,system:kubelet-bootstrap\n", s.Secrets.BearerToken, uuid.NewUUID()))
if err := cmdutil.DumpReaderToFile(bytes.NewReader(serialized), tokenAuthFilePath); err != nil {
return fmt.Errorf("<master/tokens> failed to save token auth file (%q) [%s]", tokenAuthFilePath, err)
}

View File

@ -34,17 +34,17 @@ func getNodeName() string {
return "TODO"
}
func PerformTLSBootstrapFromParams(params *kubeadmapi.BootstrapParams) (*clientcmdapi.Config, error) {
caCert, err := ioutil.ReadFile(params.Discovery.CaCertFile)
func PerformTLSBootstrapFromConfig(s *kubeadmapi.KubeadmConfig) (*clientcmdapi.Config, error) {
caCert, err := ioutil.ReadFile(s.ManualFlags.CaCertFile)
if err != nil {
return nil, fmt.Errorf("<node/csr> failed to load CA certificate [%s]", err)
}
return PerformTLSBootstrap(params, strings.Split(params.Discovery.ApiServerURLs, ",")[0], caCert)
return PerformTLSBootstrap(s, strings.Split(s.ManualFlags.ApiServerURLs, ",")[0], caCert)
}
// Create a restful client for doing the certificate signing request.
func PerformTLSBootstrap(params *kubeadmapi.BootstrapParams, apiEndpoint string, caCert []byte) (*clientcmdapi.Config, error) {
func PerformTLSBootstrap(s *kubeadmapi.KubeadmConfig, apiEndpoint string, caCert []byte) (*clientcmdapi.Config, error) {
// TODO try all the api servers until we find one that works
bareClientConfig := kubeadmutil.CreateBasicClientConfig("kubernetes", apiEndpoint, caCert)
@ -52,7 +52,7 @@ func PerformTLSBootstrap(params *kubeadmapi.BootstrapParams, apiEndpoint string,
bootstrapClientConfig, err := clientcmd.NewDefaultClientConfig(
*kubeadmutil.MakeClientConfigWithToken(
bareClientConfig, "kubernetes", fmt.Sprintf("kubelet-%s", nodeName), params.Discovery.BearerToken,
bareClientConfig, "kubernetes", fmt.Sprintf("kubelet-%s", nodeName), s.Secrets.BearerToken,
),
&clientcmd.ConfigOverrides{},
).ClientConfig()

View File

@ -22,23 +22,15 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"strings"
jose "github.com/square/go-jose"
clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api"
kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api"
)
func RetrieveTrustedClusterInfo(params *kubeadmapi.BootstrapParams) (*clientcmdapi.Config, error) {
firstURL := strings.Split(params.Discovery.ApiServerURLs, ",")[0] // TODO obviously we should do something better.. .
apiServerURL, err := url.Parse(firstURL)
if err != nil {
return nil, fmt.Errorf("<node/discovery> failed to parse given API server URL (%q) [%s]", firstURL, err)
}
host, port := strings.Split(apiServerURL.Host, ":")[0], 9898 // TODO this is too naive
requestURL := fmt.Sprintf("http://%s:%d/cluster-info/v1/?token-id=%s", host, port, params.Discovery.TokenID)
func RetrieveTrustedClusterInfo(s *kubeadmapi.KubeadmConfig) (*clientcmdapi.Config, error) {
host, port := s.JoinFlags.MasterAddrs[0].String(), 9898
requestURL := fmt.Sprintf("http://%s:%d/cluster-info/v1/?token-id=%s", host, port, s.Secrets.TokenID)
req, err := http.NewRequest("GET", requestURL, nil)
if err != nil {
return nil, fmt.Errorf("<node/discovery> failed to consturct an HTTP request [%s]", err)
@ -61,7 +53,7 @@ func RetrieveTrustedClusterInfo(params *kubeadmapi.BootstrapParams) (*clientcmda
fmt.Println("<node/discovery> cluster info object received, verifying signature using given token")
output, err := object.Verify(params.Discovery.Token)
output, err := object.Verify(s.Secrets.Token)
if err != nil {
return nil, fmt.Errorf("<node/discovery> failed to verify JWS signature of received cluster info object [%s]", err)
}
@ -84,5 +76,5 @@ func RetrieveTrustedClusterInfo(params *kubeadmapi.BootstrapParams) (*clientcmda
apiServer := clusterInfo.Endpoints[0]
caCert := []byte(clusterInfo.CertificateAuthorities[0])
return PerformTLSBootstrap(params, apiServer, caCert)
return PerformTLSBootstrap(s, apiServer, caCert)
}

View File

@ -81,12 +81,12 @@ func MakeClientConfigWithToken(config *clientcmdapi.Config, clusterName string,
// running in that case - they'd need to stop the kubelet, remove the file, and
// start it again in that case).
func WriteKubeconfigIfNotExists(params *kubeadmapi.BootstrapParams, name string, kubeconfig *clientcmdapi.Config) error {
if err := os.MkdirAll(params.EnvParams["prefix"], 0700); err != nil {
return fmt.Errorf("<util/kubeconfig> failed to create directory %q [%s]", params.EnvParams["prefix"], err)
func WriteKubeconfigIfNotExists(s *kubeadmapi.KubeadmConfig, name string, kubeconfig *clientcmdapi.Config) error {
if err := os.MkdirAll(s.EnvParams["prefix"], 0700); err != nil {
return fmt.Errorf("<util/kubeconfig> failed to create directory %q [%s]", s.EnvParams["prefix"], err)
}
filename := path.Join(params.EnvParams["prefix"], fmt.Sprintf("%s.conf", name))
filename := path.Join(s.EnvParams["prefix"], fmt.Sprintf("%s.conf", name))
// Create and open the file, only if it does not already exist.
f, err := os.OpenFile(
filename,

View File

@ -42,7 +42,7 @@ func randBytes(length int) ([]byte, string, error) {
return b, hex.EncodeToString(b), nil
}
func GenerateToken(params *kubeadmapi.BootstrapParams) error {
func GenerateToken(s *kubeadmapi.KubeadmConfig) error {
_, tokenID, err := randBytes(TokenIDLen / 2)
if err != nil {
return err
@ -53,19 +53,19 @@ func GenerateToken(params *kubeadmapi.BootstrapParams) error {
return err
}
params.Discovery.TokenID = tokenID
params.Discovery.BearerToken = token
params.Discovery.Token = tokenBytes
params.Discovery.GivenToken = fmt.Sprintf("%s.%s", tokenID, token)
s.Secrets.TokenID = tokenID
s.Secrets.BearerToken = token
s.Secrets.Token = tokenBytes
s.Secrets.GivenToken = fmt.Sprintf("%s.%s", tokenID, token)
return nil
}
func UseGivenTokenIfValid(params *kubeadmapi.BootstrapParams) (bool, error) {
if params.Discovery.GivenToken == "" {
func UseGivenTokenIfValid(s *kubeadmapi.KubeadmConfig) (bool, error) {
if s.Secrets.GivenToken == "" {
return false, nil // not given
}
fmt.Println("<util/tokens> validating provided token")
givenToken := strings.Split(strings.ToLower(params.Discovery.GivenToken), ".")
givenToken := strings.Split(strings.ToLower(s.Secrets.GivenToken), ".")
// TODO print desired format
// TODO could also print more specific messages in each case
invalidErr := "<util/tokens> provided token is invalid - %s"
@ -86,8 +86,8 @@ func UseGivenTokenIfValid(params *kubeadmapi.BootstrapParams) (bool, error) {
"length of second part is incorrect [%d (given) != %d (expected)]",
len(tokenBytes), TokenBytes))
}
params.Discovery.TokenID = givenToken[0]
params.Discovery.BearerToken = givenToken[1]
params.Discovery.Token = tokenBytes
s.Secrets.TokenID = givenToken[0]
s.Secrets.BearerToken = givenToken[1]
s.Secrets.Token = tokenBytes
return true, nil // given and valid
}