From cab23e202e8b34bf1d3547e4652736aa4d02c2bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Thu, 15 Sep 2016 17:40:42 +0300 Subject: [PATCH] Various improvements for kubeadm. Removed the user command, as it's too little time for implementing that. Now it's possible to use multiple arches. --- cmd/kubeadm/app/kubeadm.go | 11 ++---- pkg/kubeadm/cmd/cmd.go | 1 - pkg/kubeadm/cmd/manual.go | 3 +- pkg/kubeadm/cmd/user.go | 34 ----------------- pkg/kubeadm/images/images.go | 12 +++--- pkg/kubeadm/master/addons.go | 25 ++++++++---- pkg/kubeadm/master/manifests.go | 68 +++++++++++++++++++-------------- pkg/kubeadm/master/pki.go | 6 --- pkg/kubeadm/node/csr.go | 11 +++--- pkg/kubeadm/util/kubeconfig.go | 6 +-- pkg/kubeadm/util/tokens.go | 6 +-- 11 files changed, 82 insertions(+), 101 deletions(-) delete mode 100644 pkg/kubeadm/cmd/user.go diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go index 06b0a786b3..6b180b7e08 100644 --- a/cmd/kubeadm/app/kubeadm.go +++ b/cmd/kubeadm/app/kubeadm.go @@ -19,7 +19,6 @@ package app import ( "fmt" "os" - "path" "strings" "github.com/spf13/pflag" @@ -34,14 +33,12 @@ var CommandLine *pflag.FlagSet // TODO(phase2) use componentconfig // we need some params for testing etc, let's keep these hidden for now func getEnvParams() map[string]string { - globalPrefix := os.Getenv("KUBE_PREFIX_ALL") - if globalPrefix == "" { - globalPrefix = "/etc/kubernetes" - } envParams := map[string]string{ - "prefix": globalPrefix, - "host_pki_path": path.Join(globalPrefix, "pki"), + // TODO(phase1): Mode prefix and host_pki_path to another place as constants, and use them everywhere + // Right now they're used here and there, but not consequently + "kubernetes_dir": "/etc/kubernetes", + "host_pki_path": "/etc/kubernetes/pki", "host_etcd_path": "/var/lib/etcd", "hyperkube_image": "", "discovery_image": "dgoodwin/kubediscovery:latest", // TODO(phase1): fmt.Sprintf("gcr.io/google_containers/kube-discovery-%s:%s", runtime.GOARCH, "1.0"), diff --git a/pkg/kubeadm/cmd/cmd.go b/pkg/kubeadm/cmd/cmd.go index dda31030e9..04b51034ed 100644 --- a/pkg/kubeadm/cmd/cmd.go +++ b/pkg/kubeadm/cmd/cmd.go @@ -90,7 +90,6 @@ func NewKubeadmCommand(f *cmdutil.Factory, in io.Reader, out, err io.Writer, env cmds.AddCommand(NewCmdInit(out, s)) cmds.AddCommand(NewCmdJoin(out, s)) - cmds.AddCommand(NewCmdUser(out, s)) cmds.AddCommand(NewCmdManual(out, s)) return cmds diff --git a/pkg/kubeadm/cmd/manual.go b/pkg/kubeadm/cmd/manual.go index a95601931c..2b397a47a8 100644 --- a/pkg/kubeadm/cmd/manual.go +++ b/pkg/kubeadm/cmd/manual.go @@ -120,9 +120,8 @@ func NewCmdManualBootstrapInitMaster(out io.Writer, s *kubeadmapi.KubeadmConfig) &s.InitFlags.API.ExternalDNSName, "api-external-dns-name", []string{}, `(optional) DNS name to advertise, in case you have configured one yourself.`, ) - _, defaultServicesCIDR, _ := net.ParseCIDR("100.64.0.0/12") cmd.PersistentFlags().IPNetVar( - &s.InitFlags.Services.CIDR, "service-cidr", *defaultServicesCIDR, + &s.InitFlags.Services.CIDR, "service-cidr", *kubeadmapi.DefaultServicesCIDR, `(optional) use alterantive range of IP address for service VIPs, e.g. "10.16.0.0/12"`, ) cmd.PersistentFlags().StringVar( diff --git a/pkg/kubeadm/cmd/user.go b/pkg/kubeadm/cmd/user.go deleted file mode 100644 index 6beda1bf40..0000000000 --- a/pkg/kubeadm/cmd/user.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "io" - - "github.com/spf13/cobra" - kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" -) - -func NewCmdUser(out io.Writer, s *kubeadmapi.KubeadmConfig) *cobra.Command { - cmd := &cobra.Command{ - Use: "user", - Short: "Get initial admin credentials for a cluster.", // using TLS bootstrap - Run: func(cmd *cobra.Command, args []string) { - }, - } - return cmd -} diff --git a/pkg/kubeadm/images/images.go b/pkg/kubeadm/images/images.go index e1f5f48c95..f699b382b1 100644 --- a/pkg/kubeadm/images/images.go +++ b/pkg/kubeadm/images/images.go @@ -35,13 +35,15 @@ const ( gcrPrefix = "gcr.io/google_containers" etcdVersion = "2.2.5" - kubeVersion = "v1.4.0-beta.6" kubeDnsVersion = "1.7" dnsmasqVersion = "1.3" exechealthzVersion = "1.1" ) +// TODO(phase1): Make this configurable + default to a v1.4 value fetched from: https://storage.googleapis.com/kubernetes-release/release/stable.txt +var DefaultKubeVersion = "v1.4.0-beta.6" + func GetCoreImage(image string, overrideImage string) string { if overrideImage != "" { return overrideImage @@ -49,10 +51,10 @@ func GetCoreImage(image string, overrideImage string) string { return map[string]string{ KubeEtcdImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "etcd", runtime.GOARCH, etcdVersion), - KubeApiServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-apiserver", runtime.GOARCH, kubeVersion), - KubeControllerManagerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-controller-manager", runtime.GOARCH, kubeVersion), - KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-scheduler", runtime.GOARCH, kubeVersion), - KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-proxy", runtime.GOARCH, kubeVersion), + KubeApiServerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-apiserver", runtime.GOARCH, DefaultKubeVersion), + KubeControllerManagerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-controller-manager", runtime.GOARCH, DefaultKubeVersion), + KubeSchedulerImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-scheduler", runtime.GOARCH, DefaultKubeVersion), + KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-proxy", runtime.GOARCH, DefaultKubeVersion), }[image] } diff --git a/pkg/kubeadm/master/addons.go b/pkg/kubeadm/master/addons.go index 362d611241..44aa05ec5c 100644 --- a/pkg/kubeadm/master/addons.go +++ b/pkg/kubeadm/master/addons.go @@ -19,6 +19,7 @@ package master import ( "fmt" "path" + "runtime" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" @@ -28,11 +29,14 @@ import ( ipallocator "k8s.io/kubernetes/pkg/registry/service/ipallocator" "k8s.io/kubernetes/pkg/util/intstr" ) - -func createKubeProxyPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { +// TODO(phase1+): kube-proxy should be a daemonset, three different daemonsets should not be here +func createKubeProxyPodSpec(s *kubeadmapi.KubeadmConfig, architecture string) api.PodSpec { privilegedTrue := true return api.PodSpec{ SecurityContext: &api.PodSecurityContext{HostNetwork: true}, + NodeSelector: map[string]string{ + "beta.kubernetes.io/arch": architecture, + }, Containers: []api.Container{{ Name: kubeProxy, Image: images.GetCoreImage(images.KubeProxyImage, s.EnvParams["hyperkube_image"]), @@ -65,7 +69,7 @@ func createKubeProxyPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { { Name: "kubeconfig", VolumeSource: api.VolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: path.Join(s.EnvParams["prefix"], "kubelet.conf")}, + HostPath: &api.HostPathVolumeSource{Path: path.Join(s.EnvParams["kubernetes_dir"], "kubelet.conf")}, }, }, { @@ -101,6 +105,9 @@ func createKubeDNSPodSpec(s *kubeadmapi.KubeadmConfig) api.PodSpec { ) return api.PodSpec{ + NodeSelector: map[string]string{ + "beta.kubernetes.io/arch": runtime.GOARCH, + }, Containers: []api.Container{ // DNS server { @@ -223,11 +230,15 @@ func createKubeDNSServiceSpec(s *kubeadmapi.KubeadmConfig) (*api.ServiceSpec, er } func CreateEssentialAddons(s *kubeadmapi.KubeadmConfig, client *clientset.Clientset) error { - kubeProxyDaemonSet := NewDaemonSet(kubeProxy, createKubeProxyPodSpec(s)) - SetMasterTaintTolerations(&kubeProxyDaemonSet.Spec.Template.ObjectMeta) + arches := [3]string{"amd64", "arm", "arm64"} - if _, err := client.Extensions().DaemonSets(api.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil { - return fmt.Errorf(" failed creating essential kube-proxy addon [%s]", err) + for _, arch := range arches { + kubeProxyDaemonSet := NewDaemonSet(kubeProxy + "-" + arch, createKubeProxyPodSpec(s, arch)) + SetMasterTaintTolerations(&kubeProxyDaemonSet.Spec.Template.ObjectMeta) + + if _, err := client.Extensions().DaemonSets(api.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil { + return fmt.Errorf(" failed creating essential kube-proxy addon [%s]", err) + } } fmt.Println(" created essential addon: kube-proxy") diff --git a/pkg/kubeadm/master/manifests.go b/pkg/kubeadm/master/manifests.go index 2766e90d38..d545575d7a 100644 --- a/pkg/kubeadm/master/manifests.go +++ b/pkg/kubeadm/master/manifests.go @@ -36,7 +36,8 @@ import ( // init master` and `kubeadm manual bootstrap master` can get going. const ( - DefaultClusterName = "--cluster-name=kubernetes" + DefaultClusterName = "kubernetes" + DefaultCloudConfigPath = "/etc/kubernetes/cloud-config.json" etcd = "etcd" apiServer = "apiserver" @@ -57,12 +58,7 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { staticPodSpecs := map[string]api.Pod{ // TODO this needs a volume etcd: componentPod(api.Container{ - Command: []string{ - "/usr/local/bin/etcd", - "--listen-client-urls=http://127.0.0.1:2379", - "--advertise-client-urls=http://127.0.0.1:2379", - "--data-dir=/var/etcd/data", - }, + Command: getComponentCommand(etcd, s), VolumeMounts: []api.VolumeMount{etcdVolumeMount()}, Image: images.GetCoreImage(images.KubeEtcdImage, s.EnvParams["etcd_image"]), LivenessProbe: componentProbe(2379, "/health"), @@ -74,18 +70,18 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { Name: kubeAPIServer, Image: images.GetCoreImage(images.KubeApiServerImage, s.EnvParams["hyperkube_image"]), Command: getComponentCommand(apiServer, s), - VolumeMounts: []api.VolumeMount{pkiVolumeMount()}, + VolumeMounts: []api.VolumeMount{k8sVolumeMount()}, LivenessProbe: componentProbe(8080, "/healthz"), Resources: componentResources("250m"), - }, pkiVolume(s)), + }, k8sVolume(s)), kubeControllerManager: componentPod(api.Container{ Name: kubeControllerManager, Image: images.GetCoreImage(images.KubeControllerManagerImage, s.EnvParams["hyperkube_image"]), Command: getComponentCommand(controllerManager, s), - VolumeMounts: []api.VolumeMount{pkiVolumeMount()}, + VolumeMounts: []api.VolumeMount{k8sVolumeMount()}, LivenessProbe: componentProbe(10252, "/healthz"), Resources: componentResources("200m"), - }, pkiVolume(s)), + }, k8sVolume(s)), kubeScheduler: componentPod(api.Container{ Name: kubeScheduler, Image: images.GetCoreImage(images.KubeSchedulerImage, s.EnvParams["hyperkube_image"]), @@ -95,7 +91,7 @@ func WriteStaticPodManifests(s *kubeadmapi.KubeadmConfig) error { }), } - manifestsPath := path.Join(s.EnvParams["prefix"], "manifests") + manifestsPath := path.Join(s.EnvParams["kubernetes_dir"], "manifests") if err := os.MkdirAll(manifestsPath, 0700); err != nil { return fmt.Errorf(" failed to create directory %q [%s]", manifestsPath, err) } @@ -128,19 +124,19 @@ func etcdVolumeMount() api.VolumeMount { } } -func pkiVolume(s *kubeadmapi.KubeadmConfig) api.Volume { +func k8sVolume(s *kubeadmapi.KubeadmConfig) api.Volume { return api.Volume{ Name: "pki", VolumeSource: api.VolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: s.EnvParams["host_pki_path"]}, + HostPath: &api.HostPathVolumeSource{Path: s.EnvParams["kubernetes_dir"]}, }, } } -func pkiVolumeMount() api.VolumeMount { +func k8sVolumeMount() api.VolumeMount { return api.VolumeMount{ Name: "pki", - MountPath: "/etc/kubernetes/pki", + MountPath: "/etc/kubernetes/", ReadOnly: true, } } @@ -187,32 +183,43 @@ func componentPod(container api.Container, volumes ...api.Volume) api.Pod { } func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command []string) { - baseFalgs := map[string][]string{ + // TODO: make a global constant of this + pki_dir := "/etc/kubernetes/pki" + + baseFlags := map[string][]string{ + etcd: []string{ + "/usr/local/bin/etcd", + "--listen-client-urls=http://127.0.0.1:2379", + "--advertise-client-urls=http://127.0.0.1:2379", + "--data-dir=/var/etcd/data", + }, apiServer: []string{ "--address=127.0.0.1", "--etcd-servers=http://127.0.0.1:2379", "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota", "--service-cluster-ip-range=" + s.InitFlags.Services.CIDR.String(), - "--service-account-key-file=/etc/kubernetes/pki/apiserver-key.pem", - "--client-ca-file=/etc/kubernetes/pki/ca.pem", - "--tls-cert-file=/etc/kubernetes/pki/apiserver.pem", - "--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", + "--service-account-key-file=" + pki_dir + "/apiserver-key.pem", + "--client-ca-file=" + pki_dir + "/ca.pem", + "--tls-cert-file=" + pki_dir + "/apiserver.pem", + "--tls-private-key-file=" + pki_dir + "/apiserver-key.pem", + "--token-auth-file=" + pki_dir + "/tokens.csv", "--secure-port=443", "--allow-privileged", - "--token-auth-file=/etc/kubernetes/pki/tokens.csv", }, controllerManager: []string{ + // TODO: consider adding --address=127.0.0.1 in order to not expose the cm port to the rest of the world "--leader-elect", "--master=127.0.0.1:8080", - DefaultClusterName, - "--root-ca-file=/etc/kubernetes/pki/ca.pem", - "--service-account-private-key-file=/etc/kubernetes/pki/apiserver-key.pem", - "--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem", - "--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem", + "--cluster-name=" + DefaultClusterName, + "--root-ca-file=" + pki_dir + "/ca.pem", + "--service-account-private-key-file=" + pki_dir + "/apiserver-key.pem", + "--cluster-signing-cert-file=" + pki_dir + "/ca.pem", + "--cluster-signing-key-file=" + pki_dir + "/ca-key.pem", "--insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap", "--cluster-cidr=" + s.InitFlags.Services.CIDR.String(), }, scheduler: []string{ + // TODO: consider adding --address=127.0.0.1 in order to not expose the scheduler port to the rest of the world "--leader-elect", "--master=127.0.0.1:8080", }, @@ -226,10 +233,15 @@ func getComponentCommand(component string, s *kubeadmapi.KubeadmConfig) (command } command = append(command, s.EnvParams["component_loglevel"]) - command = append(command, baseFalgs[component]...) + command = append(command, baseFlags[component]...) if component == controllerManager && s.InitFlags.CloudProvider != "" { command = append(command, "--cloud-provider="+s.InitFlags.CloudProvider) + + // Only append the --cloud-config option if there's a such file + if _, err := os.Stat(DefaultCloudConfigPath); err == nil { + command = append(command, "--cloud-config=" + DefaultCloudConfigPath) + } } return diff --git a/pkg/kubeadm/master/pki.go b/pkg/kubeadm/master/pki.go index 19c8ccac92..a28189c10a 100644 --- a/pkg/kubeadm/master/pki.go +++ b/pkg/kubeadm/master/pki.go @@ -27,12 +27,6 @@ import ( certutil "k8s.io/kubernetes/pkg/util/cert" ) -/* -func errorf(f string, err error, vargs ...string) error { - return fmt.Errorf(" %s [%s]", fmt.Sprintf(f, v...), err) -} -*/ - func newCertificateAuthority() (*rsa.PrivateKey, *x509.Certificate, error) { key, err := certutil.NewPrivateKey() if err != nil { diff --git a/pkg/kubeadm/node/csr.go b/pkg/kubeadm/node/csr.go index e6d841dd60..28fb584e56 100644 --- a/pkg/kubeadm/node/csr.go +++ b/pkg/kubeadm/node/csr.go @@ -20,6 +20,7 @@ import ( "fmt" "io/ioutil" "strings" + "os" "k8s.io/kubernetes/pkg/apis/certificates" unversionedcertificates "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/certificates/unversioned" @@ -33,10 +34,6 @@ import ( certutil "k8s.io/kubernetes/pkg/util/cert" ) -func getNodeName() string { - return "TODO" -} - func PerformTLSBootstrapFromConfig(s *kubeadmapi.KubeadmConfig) (*clientcmdapi.Config, error) { caCert, err := ioutil.ReadFile(s.ManualFlags.CaCertFile) if err != nil { @@ -51,7 +48,11 @@ func PerformTLSBootstrap(s *kubeadmapi.KubeadmConfig, apiEndpoint string, caCert // TODO try all the api servers until we find one that works bareClientConfig := kubeadmutil.CreateBasicClientConfig("kubernetes", apiEndpoint, caCert) - nodeName := getNodeName() + // Try to fetch the hostname of the node + nodeName, err := os.Hostname() + if err != nil { + return nil, fmt.Errorf(" failed to get node hostname [%v]", err) + } bootstrapClientConfig, err := clientcmd.NewDefaultClientConfig( *kubeadmutil.MakeClientConfigWithToken( diff --git a/pkg/kubeadm/util/kubeconfig.go b/pkg/kubeadm/util/kubeconfig.go index 4fe56cf03f..b7f3919fc0 100644 --- a/pkg/kubeadm/util/kubeconfig.go +++ b/pkg/kubeadm/util/kubeconfig.go @@ -82,11 +82,11 @@ func MakeClientConfigWithToken(config *clientcmdapi.Config, clusterName string, // start it again in that case). func WriteKubeconfigIfNotExists(s *kubeadmapi.KubeadmConfig, name string, kubeconfig *clientcmdapi.Config) error { - if err := os.MkdirAll(s.EnvParams["prefix"], 0700); err != nil { - return fmt.Errorf(" failed to create directory %q [%s]", s.EnvParams["prefix"], err) + if err := os.MkdirAll(s.EnvParams["kubernetes_dir"], 0700); err != nil { + return fmt.Errorf(" failed to create directory %q [%s]", s.EnvParams["kubernetes_dir"], err) } - filename := path.Join(s.EnvParams["prefix"], fmt.Sprintf("%s.conf", name)) + filename := path.Join(s.EnvParams["kubernetes_dir"], fmt.Sprintf("%s.conf", name)) // Create and open the file, only if it does not already exist. f, err := os.OpenFile( filename, diff --git a/pkg/kubeadm/util/tokens.go b/pkg/kubeadm/util/tokens.go index 202a37b245..1e016088b4 100644 --- a/pkg/kubeadm/util/tokens.go +++ b/pkg/kubeadm/util/tokens.go @@ -30,7 +30,7 @@ const ( TokenBytes = 8 ) -func randBytes(length int) ([]byte, string, error) { +func RandBytes(length int) ([]byte, string, error) { b := make([]byte, length) _, err := rand.Read(b) if err != nil { @@ -43,12 +43,12 @@ func randBytes(length int) ([]byte, string, error) { } func GenerateToken(s *kubeadmapi.KubeadmConfig) error { - _, tokenID, err := randBytes(TokenIDLen / 2) + _, tokenID, err := RandBytes(TokenIDLen / 2) if err != nil { return err } - tokenBytes, token, err := randBytes(TokenBytes) + tokenBytes, token, err := RandBytes(TokenBytes) if err != nil { return err }