From b48df06aba04f3e74e7ab38373ef7bd7c469a7ae Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Tue, 6 Sep 2016 16:30:58 +0200 Subject: [PATCH] Refactor kube-dns addon constructors, more labels - also add another IP to SANs - fix mkdir calls - add TODO for ComponentConfig - start tagging TODOs by phases --- cmd/kubeadm/app/kubeadm.go | 5 +- pkg/kubeadm/cmd/cmd.go | 10 ++-- pkg/kubeadm/master/addons.go | 88 ++++++++++++++------------------- pkg/kubeadm/master/apiclient.go | 22 ++++++++- pkg/kubeadm/master/manifests.go | 2 +- pkg/kubeadm/master/pki.go | 2 +- pkg/kubeadm/master/tokens.go | 2 +- pkg/kubeadm/node/discovery.go | 8 +-- pkg/kubeadm/util/kubeconfig.go | 4 ++ 9 files changed, 75 insertions(+), 68 deletions(-) diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go index 8631debd5b..b7249e995b 100644 --- a/cmd/kubeadm/app/kubeadm.go +++ b/cmd/kubeadm/app/kubeadm.go @@ -41,9 +41,8 @@ func getEnvParams() map[string]string { } envParams := map[string]string{ - "prefix": globalPrefix, - "host_pki_path": path.Join(globalPrefix, "pki"), - // TODO find a way to specify image versions for all of these... + "prefix": globalPrefix, + "host_pki_path": path.Join(globalPrefix, "pki"), "hyperkube_image": fmt.Sprintf("gcr.io/google_containers/hyperkube-%s:%s", runtime.GOARCH, "v1.4.0-alpha.3"), "discovery_image": "dgoodwin/kubediscovery:latest", "etcd_image": fmt.Sprintf("gcr.io/google_containers/etcd-%s:%s", runtime.GOARCH, "2.2.5"), diff --git a/pkg/kubeadm/cmd/cmd.go b/pkg/kubeadm/cmd/cmd.go index 2fe3ae0f16..c152f768ec 100644 --- a/pkg/kubeadm/cmd/cmd.go +++ b/pkg/kubeadm/cmd/cmd.go @@ -65,22 +65,22 @@ func NewKubeadmCommand(f *cmdutil.Factory, in io.Reader, out, err io.Writer, env `), } - // TODO figure out how to avoid running as root + // TODO(phase2+) figure out how to avoid running as root // - // TODO also print the alpha warning when running any commands, as well as + // TODO(phase1) also print the alpha warning when running any commands, as well as // in the help text. // - // TODO detect interactive vs non-interactive use and adjust output accordingly + // TODO(phase2) detect interactive vs non-interactive use and adjust output accordingly // i.e. make it automation friendly // - // TODO create an bastraction that defines files and the content that needs to + // TODO(phase2) create an bastraction that defines files and the content that needs to // be written to disc and write it all in one go at the end as we have a lot of // crapy little files written from different parts of this code; this could also // be useful for testing bootstrapParams := &kubeadmapi.BootstrapParams{ Discovery: &kubeadmapi.OutOfBandDiscovery{ - // TODO this type no longer makes sense here + // TODO(phase1) this type no longer makes sense here }, EnvParams: envParams, } diff --git a/pkg/kubeadm/master/addons.go b/pkg/kubeadm/master/addons.go index bae44d7c43..a37ba373d4 100644 --- a/pkg/kubeadm/master/addons.go +++ b/pkg/kubeadm/master/addons.go @@ -19,12 +19,11 @@ package kubemaster import ( "fmt" "path" + "runtime" "strconv" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" - unversionedapi "k8s.io/kubernetes/pkg/api/unversioned" - "k8s.io/kubernetes/pkg/apis/extensions" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" kubeadmapi "k8s.io/kubernetes/pkg/kubeadm/api" "k8s.io/kubernetes/pkg/util/intstr" @@ -84,34 +83,24 @@ func createKubeProxyPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { } } -func createKubeDnsDeployment(params *kubeadmapi.BootstrapParams) *extensions.Deployment { - metaLabels := map[string]string{ - "k8s-app": "kube-dns", - "version": "v19", - "kubernetes.io/cluster-service": "true", - } - - metaAnnotations := map[string]string{ - "scheduler.alpha.kubernetes.io/critical-pod": "''", - "scheduler.alpha.kubernetes.io/tolerations": "'[{\"key\":\"CriticalAddonsOnly\", \"operator\":\"Exists\"}]'", - } +func createKubeDNSPodSpec(params *kubeadmapi.BootstrapParams) api.PodSpec { dnsPodResources := api.ResourceList{ - api.ResourceLimitsCPU: resource.MustParse("100m"), - api.ResourceMemory: resource.MustParse("170Mi"), + api.ResourceName(api.ResourceCPU): resource.MustParse("100m"), + api.ResourceName(api.ResourceMemory): resource.MustParse("170Mi"), } healthzPodResources := api.ResourceList{ - api.ResourceLimitsCPU: resource.MustParse("10m"), - api.ResourceMemory: resource.MustParse("50Mi"), + api.ResourceName(api.ResourceCPU): resource.MustParse("10m"), + api.ResourceName(api.ResourceMemory): resource.MustParse("50Mi"), } - podSpec := api.PodSpec{ + return api.PodSpec{ Containers: []api.Container{ // DNS server { Name: "kube-dns", - Image: "gcr.io/google_containers/kubedns-amd64:1.7", + Image: "gcr.io/google_containers/kubedns-" + runtime.GOARCH + ":1.7", Resources: api.ResourceRequirements{ Limits: dnsPodResources, Requests: dnsPodResources, @@ -163,7 +152,7 @@ func createKubeDnsDeployment(params *kubeadmapi.BootstrapParams) *extensions.Dep // dnsmasq { Name: "dnsmasq", - Image: "gcr.io/google_containers/kube-dnsmasq-amd64:1.3", + Image: "gcr.io/google_containers/kube-dnsmasq-" + runtime.GOARCH + ":1.3", Resources: api.ResourceRequirements{ Limits: dnsPodResources, Requests: dnsPodResources, @@ -189,7 +178,7 @@ func createKubeDnsDeployment(params *kubeadmapi.BootstrapParams) *extensions.Dep // healthz { Name: "healthz", - Image: "gcr.io/google_containers/exechealthz-amd64:1.1", + Image: "gcr.io/google_containers/exechealthz-" + runtime.GOARCH + ":1.1", Resources: api.ResourceRequirements{ Limits: healthzPodResources, Requests: healthzPodResources, @@ -199,41 +188,25 @@ func createKubeDnsDeployment(params *kubeadmapi.BootstrapParams) *extensions.Dep "-port=8080", "-quiet", }, - Ports: []api.ContainerPort{ - { - ContainerPort: 8080, - Protocol: api.ProtocolTCP, - }, - }, + Ports: []api.ContainerPort{{ + ContainerPort: 8080, + Protocol: api.ProtocolTCP, + }}, }, }, DNSPolicy: api.DNSDefault, } - dnsReplicas, err := strconv.Atoi(params.EnvParams["dns_replicas"]) - if err != nil { - dnsReplicas = 1 - } - - return &extensions.Deployment{ - ObjectMeta: api.ObjectMeta{ - Name: "kube-dns-v19", - Namespace: "kube-system", - Labels: metaLabels, - }, - Spec: extensions.DeploymentSpec{ - Replicas: int32(dnsReplicas), - Selector: &unversionedapi.LabelSelector{MatchLabels: metaLabels}, - Template: api.PodTemplateSpec{ - ObjectMeta: api.ObjectMeta{ - Labels: metaLabels, - Annotations: metaAnnotations, - }, - Spec: podSpec, - }, +} +func createKubeDNSServiceSpec(params *kubeadmapi.BootstrapParams) api.ServiceSpec { + return api.ServiceSpec{ + Selector: map[string]string{"name": "kube-dns"}, + Ports: []api.ServicePort{ + {Name: "dns", Port: 53, Protocol: api.ProtocolUDP}, + {Name: "dns-tcp", Port: 53, Protocol: api.ProtocolTCP}, }, + ClusterIP: "100.64.0.2", } - } func CreateEssentialAddons(params *kubeadmapi.BootstrapParams, client *clientset.Clientset) error { @@ -248,8 +221,21 @@ func CreateEssentialAddons(params *kubeadmapi.BootstrapParams, client *clientset // TODO should we wait for it to become ready at least on the master? - kubeDnsDeployment := createKubeDnsDeployment(params) - if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kubeDnsDeployment); err != nil { + dnsReplicas, err := strconv.Atoi(params.EnvParams["dns_replicas"]) + if err != nil { + dnsReplicas = 1 + } + + kubeDNSDeployment := NewDeployment("kube-dns", int32(dnsReplicas), createKubeDNSPodSpec(params)) + SetMasterTaintTolerations(&kubeDNSDeployment.Spec.Template.ObjectMeta) + + if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kubeDNSDeployment); err != nil { + return fmt.Errorf(" failed creating essential kube-dns addon [%s]", err) + } + + kubeDNSService := NewService("kube-dns", createKubeDNSServiceSpec(params)) + + if _, err := client.Services(api.NamespaceSystem).Create(kubeDNSService); err != nil { return fmt.Errorf(" failed creating essential kube-dns addon [%s]", err) } diff --git a/pkg/kubeadm/master/apiclient.go b/pkg/kubeadm/master/apiclient.go index 6c9b19c721..b2e22c895e 100644 --- a/pkg/kubeadm/master/apiclient.go +++ b/pkg/kubeadm/master/apiclient.go @@ -98,8 +98,15 @@ func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Cli return client, nil } +func standardLabels(n string) map[string]string { + return map[string]string{ + "component": n, "name": n, "k8s-app": n, + "kubernetes.io/cluster-service": "true", "tier": "node", + } +} + func NewDaemonSet(daemonName string, podSpec api.PodSpec) *extensions.DaemonSet { - l := map[string]string{"component": daemonName, "tier": "node"} + l := standardLabels(daemonName) return &extensions.DaemonSet{ ObjectMeta: api.ObjectMeta{Name: daemonName}, Spec: extensions.DaemonSetSpec{ @@ -112,8 +119,19 @@ func NewDaemonSet(daemonName string, podSpec api.PodSpec) *extensions.DaemonSet } } +func NewService(serviceName string, spec api.ServiceSpec) *api.Service { + l := standardLabels(serviceName) + return &api.Service{ + ObjectMeta: api.ObjectMeta{ + Name: serviceName, + Labels: l, + }, + Spec: spec, + } +} + func NewDeployment(deploymentName string, replicas int32, podSpec api.PodSpec) *extensions.Deployment { - l := map[string]string{"name": deploymentName} + l := standardLabels(deploymentName) return &extensions.Deployment{ ObjectMeta: api.ObjectMeta{Name: deploymentName}, Spec: extensions.DeploymentSpec{ diff --git a/pkg/kubeadm/master/manifests.go b/pkg/kubeadm/master/manifests.go index 7641ee1066..b104889655 100644 --- a/pkg/kubeadm/master/manifests.go +++ b/pkg/kubeadm/master/manifests.go @@ -35,7 +35,7 @@ import ( // init master` and `kubeadm manual bootstrap master` can get going. const ( - SERVICE_CLUSTER_IP_RANGE = "--service-cluster-ip-range=10.16.0.0/12" + SERVICE_CLUSTER_IP_RANGE = "--service-cluster-ip-range=100.64.0.0/12" CLUSTER_NAME = "--cluster-name=kubernetes" MASTER = "--master=127.0.0.1:8080" ) diff --git a/pkg/kubeadm/master/pki.go b/pkg/kubeadm/master/pki.go index d5b2434985..e908c65f6c 100644 --- a/pkg/kubeadm/master/pki.go +++ b/pkg/kubeadm/master/pki.go @@ -57,7 +57,7 @@ func newServerKeyAndCert(caCert *x509.Certificate, caKey *rsa.PrivateKey, altNam return nil, nil, fmt.Errorf("unabel to create private key [%s]", err) } // TODO these are all hardcoded for now, but we need to figure out what shall we do here exactly - altNames.IPs = append(altNames.IPs, net.ParseIP("10.3.0.1")) + altNames.IPs = append(altNames.IPs, net.ParseIP("10.3.0.1"), net.ParseIP("10.16.0.1"), net.ParseIP("100.64.0.1")) altNames.DNSNames = append(altNames.DNSNames, "kubernetes", "kubernetes.default", diff --git a/pkg/kubeadm/master/tokens.go b/pkg/kubeadm/master/tokens.go index ad184cbbcd..ff052575a7 100644 --- a/pkg/kubeadm/master/tokens.go +++ b/pkg/kubeadm/master/tokens.go @@ -51,7 +51,7 @@ func CreateTokenAuthFile(params *kubeadmapi.BootstrapParams) error { if err := generateTokenIfNeeded(params); err != nil { return fmt.Errorf(" failed to generate token(s) [%s]", err) } - if err := os.MkdirAll(path.Join(params.EnvParams["host_pki_path"]), 0700); err != nil { + if err := os.MkdirAll(params.EnvParams["host_pki_path"], 0700); err != nil { return fmt.Errorf(" failed to create directory %q [%s]", params.EnvParams["host_pki_path"], err) } serialized := []byte(fmt.Sprintf("%s,kubeadm-node-csr,%s,system:kubelet-bootstrap\n", params.Discovery.BearerToken, uuid.NewUUID())) diff --git a/pkg/kubeadm/node/discovery.go b/pkg/kubeadm/node/discovery.go index beb470e98d..3b41cdf0b2 100644 --- a/pkg/kubeadm/node/discovery.go +++ b/pkg/kubeadm/node/discovery.go @@ -59,17 +59,17 @@ func RetrieveTrustedClusterInfo(params *kubeadmapi.BootstrapParams) (*clientcmda return nil, fmt.Errorf(" failed to parse response as JWS object [%s]", err) } - fmt.Println(" cluster info object recieved, verifying signature using given token") + fmt.Println(" cluster info object received, verifying signature using given token") output, err := object.Verify(params.Discovery.Token) if err != nil { - return nil, fmt.Errorf(" failed to verify JWS signature of recieved cluster info object [%s]", err) + return nil, fmt.Errorf(" failed to verify JWS signature of received cluster info object [%s]", err) } clusterInfo := kubeadmapi.ClusterInfo{} if err := json.Unmarshal(output, &clusterInfo); err != nil { - return nil, fmt.Errorf(" failed to unmarshal recieved cluster info object [%s]", err) + return nil, fmt.Errorf(" failed to decode received cluster info object [%s]", err) } if len(clusterInfo.CertificateAuthorities) == 0 || len(clusterInfo.Endpoints) == 0 { @@ -77,7 +77,7 @@ func RetrieveTrustedClusterInfo(params *kubeadmapi.BootstrapParams) (*clientcmda } // TODO print checksum of the CA certificate - fmt.Printf(" cluser info signature and contents are valid, will use API endpoints %v\n", clusterInfo.Endpoints) + fmt.Printf(" cluster info signature and contents are valid, will use API endpoints %v\n", clusterInfo.Endpoints) // TODO we need to configure the client to validate the server // if it is signed by any of the returned certificates diff --git a/pkg/kubeadm/util/kubeconfig.go b/pkg/kubeadm/util/kubeconfig.go index 49e27e3f29..041021bf9c 100644 --- a/pkg/kubeadm/util/kubeconfig.go +++ b/pkg/kubeadm/util/kubeconfig.go @@ -82,6 +82,10 @@ func MakeClientConfigWithToken(config *clientcmdapi.Config, clusterName string, // start it again in that case). func WriteKubeconfigIfNotExists(params *kubeadmapi.BootstrapParams, name string, kubeconfig *clientcmdapi.Config) error { + if err := os.MkdirAll(params.EnvParams["prefix"], 0700); err != nil { + return fmt.Errorf(" failed to create directory %q [%s]", params.EnvParams["prefix"], err) + } + filename := path.Join(params.EnvParams["prefix"], fmt.Sprintf("%s.conf", name)) // Create and open the file, only if it does not already exist. f, err := os.OpenFile(