diff --git a/cmd/kubeadm/app/BUILD b/cmd/kubeadm/app/BUILD index 8f49b8be3a..ae1a1864c4 100644 --- a/cmd/kubeadm/app/BUILD +++ b/cmd/kubeadm/app/BUILD @@ -37,6 +37,7 @@ filegroup( "//cmd/kubeadm/app/images:all-srcs", "//cmd/kubeadm/app/master:all-srcs", "//cmd/kubeadm/app/node:all-srcs", + "//cmd/kubeadm/app/phases/addons:all-srcs", "//cmd/kubeadm/app/phases/apiconfig:all-srcs", "//cmd/kubeadm/app/phases/certs:all-srcs", "//cmd/kubeadm/app/phases/kubeconfig:all-srcs", diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index 90ac20cb7b..73cb8dd678 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -29,6 +29,7 @@ go_library( "//cmd/kubeadm/app/discovery:go_default_library", "//cmd/kubeadm/app/master:go_default_library", "//cmd/kubeadm/app/node:go_default_library", + "//cmd/kubeadm/app/phases/addons:go_default_library", "//cmd/kubeadm/app/phases/apiconfig:go_default_library", "//cmd/kubeadm/app/phases/certs:go_default_library", "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 9dc244f6ec..f87748b709 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -33,6 +33,7 @@ import ( kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/discovery" kubemaster "k8s.io/kubernetes/cmd/kubeadm/app/master" + addonsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons" apiconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/apiconfig" certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" @@ -259,7 +260,7 @@ func (i *Init) Run(out io.Writer) error { } // PHASE 5: Deploy essential addons - if err := kubemaster.CreateEssentialAddons(i.cfg, client); err != nil { + if err := addonsphase.CreateEssentialAddons(i.cfg, client); err != nil { return err } diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index aad72ee827..fb69147aef 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -16,6 +16,8 @@ limitations under the License. package constants +import "time" + const ( CACertAndKeyBaseName = "ca" CACertName = "ca.crt" @@ -43,4 +45,7 @@ const ( // Constants for what we name our ServiceAccounts with limited access to the cluster in case of RBAC KubeDNSServiceAccountName = "kube-dns" KubeProxyServiceAccountName = "kube-proxy" + + // APICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation + APICallRetryInterval = 500 * time.Millisecond ) diff --git a/cmd/kubeadm/app/images/images.go b/cmd/kubeadm/app/images/images.go index 4b78993516..b337d8ab77 100644 --- a/cmd/kubeadm/app/images/images.go +++ b/cmd/kubeadm/app/images/images.go @@ -24,23 +24,13 @@ import ( ) const ( - KubeEtcdImage = "etcd" - + KubeEtcdImage = "etcd" KubeAPIServerImage = "apiserver" KubeControllerManagerImage = "controller-manager" KubeSchedulerImage = "scheduler" KubeProxyImage = "proxy" - KubeDNSImage = "k8s-dns-kube-dns" - KubeDNSmasqImage = "k8s-dns-dnsmasq" - KubeDNSSidecarImage = "k8s-dns-sidecar" - Pause = "pause" - - gcrPrefix = "gcr.io/google_containers" etcdVersion = "3.0.14-kubeadm" - - kubeDNSVersion = "1.11.0" - pauseVersion = "3.0" ) func GetCoreImage(image string, cfg *kubeadmapi.MasterConfiguration, overrideImage string) string { @@ -56,13 +46,3 @@ func GetCoreImage(image string, cfg *kubeadmapi.MasterConfiguration, overrideIma KubeProxyImage: fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "kube-proxy", runtime.GOARCH, cfg.KubernetesVersion), }[image] } - -func GetAddonImage(image string) string { - repoPrefix := kubeadmapi.GlobalEnvParams.RepositoryPrefix - return map[string]string{ - KubeDNSImage: fmt.Sprintf("%s/%s-%s:%s", repoPrefix, KubeDNSImage, runtime.GOARCH, kubeDNSVersion), - KubeDNSmasqImage: fmt.Sprintf("%s/%s-%s:%s", repoPrefix, KubeDNSmasqImage, runtime.GOARCH, kubeDNSVersion), - KubeDNSSidecarImage: fmt.Sprintf("%s/%s-%s:%s", repoPrefix, KubeDNSSidecarImage, runtime.GOARCH, kubeDNSVersion), - Pause: fmt.Sprintf("%s/%s-%s:%s", repoPrefix, Pause, runtime.GOARCH, pauseVersion), - }[image] -} diff --git a/cmd/kubeadm/app/images/images_test.go b/cmd/kubeadm/app/images/images_test.go index 9fa8f8a833..f01b8903c8 100644 --- a/cmd/kubeadm/app/images/images_test.go +++ b/cmd/kubeadm/app/images/images_test.go @@ -30,7 +30,10 @@ type getCoreImageTest struct { o string } -const testversion = "1" +const ( + testversion = "1" + gcrPrefix = "gcr.io/google_containers" +) func TestGetCoreImage(t *testing.T) { var imageTest = []struct { @@ -75,38 +78,3 @@ func TestGetCoreImage(t *testing.T) { } } } - -func TestGetAddonImage(t *testing.T) { - var imageTest = []struct { - t string - expected string - }{ - {"matches nothing", ""}, - { - KubeDNSImage, - fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, KubeDNSImage, runtime.GOARCH, kubeDNSVersion), - }, - { - KubeDNSmasqImage, - fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, KubeDNSmasqImage, runtime.GOARCH, kubeDNSVersion), - }, - { - KubeDNSSidecarImage, - fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, KubeDNSSidecarImage, runtime.GOARCH, kubeDNSVersion), - }, - { - Pause, - fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, Pause, runtime.GOARCH, pauseVersion), - }, - } - for _, it := range imageTest { - actual := GetAddonImage(it.t) - if actual != it.expected { - t.Errorf( - "failed GetAddonImage:\n\texpected: %s\n\t actual: %s", - it.expected, - actual, - ) - } - } -} diff --git a/cmd/kubeadm/app/master/BUILD b/cmd/kubeadm/app/master/BUILD index e010f9da6e..902b610b5d 100644 --- a/cmd/kubeadm/app/master/BUILD +++ b/cmd/kubeadm/app/master/BUILD @@ -11,29 +11,28 @@ load( go_library( name = "go_default_library", srcs = [ - "addons.go", "apiclient.go", "discovery.go", "manifests.go", "selfhosted.go", + "templates.go", "tokens.go", ], tags = ["automanaged"], deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/images:go_default_library", - "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", "//cmd/kubeadm/app/util:go_default_library", + "//pkg/api:go_default_library", "//pkg/api/v1:go_default_library", "//pkg/apis/extensions/v1beta1:go_default_library", "//pkg/client/clientset_generated/clientset:go_default_library", "//pkg/kubectl/cmd/util:go_default_library", - "//pkg/registry/core/service/ipallocator:go_default_library", "//vendor:k8s.io/apimachinery/pkg/api/errors", "//vendor:k8s.io/apimachinery/pkg/api/resource", "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1", + "//vendor:k8s.io/apimachinery/pkg/runtime", "//vendor:k8s.io/apimachinery/pkg/util/intstr", "//vendor:k8s.io/apimachinery/pkg/util/uuid", "//vendor:k8s.io/apimachinery/pkg/util/wait", @@ -44,12 +43,7 @@ go_library( go_test( name = "go_default_test", - srcs = [ - "addons_test.go", - "apiclient_test.go", - "discovery_test.go", - "manifests_test.go", - ], + srcs = ["manifests_test.go"], library = ":go_default_library", tags = ["automanaged"], deps = [ diff --git a/cmd/kubeadm/app/master/addons.go b/cmd/kubeadm/app/master/addons.go deleted file mode 100644 index ecea190918..0000000000 --- a/cmd/kubeadm/app/master/addons.go +++ /dev/null @@ -1,332 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package master - -import ( - "fmt" - "net" - "path" - "runtime" - - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/images" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" - "k8s.io/kubernetes/pkg/api/v1" - "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" - "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" -) - -const KubeDNS = "kube-dns" - -func createKubeProxyPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec { - privilegedTrue := true - return v1.PodSpec{ - HostNetwork: true, - SecurityContext: &v1.PodSecurityContext{}, - Containers: []v1.Container{{ - Name: kubeProxy, - Image: images.GetCoreImage(images.KubeProxyImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), - Command: append(getProxyCommand(cfg), "--kubeconfig=/run/kubeconfig"), - SecurityContext: &v1.SecurityContext{Privileged: &privilegedTrue}, - VolumeMounts: []v1.VolumeMount{ - { - Name: "dbus", - MountPath: "/var/run/dbus", - ReadOnly: false, - }, - { - // TODO there are handful of clever options to get around this, but it's - // easier to just mount kubelet's config here; we should probably just - // make sure that proxy reads the token and CA cert from /run/secrets - // and accepts `--master` at the same time - // - // clever options include: - // - do CSR dance and create kubeconfig and mount it as a secret - // - create a service account with a second secret encoding kubeconfig - // - use init container to convert known information to kubeconfig - // - ...whatever - Name: "kubeconfig", - MountPath: "/run/kubeconfig", - ReadOnly: false, - }, - }, - }}, - Volumes: []v1.Volume{ - { - Name: "kubeconfig", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeconfig.KubeletKubeConfigFileName)}, - }, - }, - { - Name: "dbus", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/var/run/dbus"}, - }, - }, - }, - Affinity: &v1.Affinity{ - NodeAffinity: &v1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "beta.kubernetes.io/arch", - Operator: v1.NodeSelectorOpIn, - Values: []string{runtime.GOARCH}, - }, - }, - }, - }, - }, - }, - }, - } -} - -func createKubeDNSPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec { - kubeDNSPort := int32(10053) - dnsmasqPort := int32(53) - - return v1.PodSpec{ - ServiceAccountName: KubeDNS, - Containers: []v1.Container{ - // DNS server - { - Name: "kubedns", - Image: images.GetAddonImage(images.KubeDNSImage), - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceName(v1.ResourceMemory): resource.MustParse("170Mi"), - }, - Requests: v1.ResourceList{ - v1.ResourceName(v1.ResourceCPU): resource.MustParse("100m"), - v1.ResourceName(v1.ResourceMemory): resource.MustParse("70Mi"), - }, - }, - LivenessProbe: &v1.Probe{ - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Path: "/healthcheck/kubedns", - Port: intstr.FromInt(10054), - Scheme: v1.URISchemeHTTP, - }, - }, - InitialDelaySeconds: 60, - TimeoutSeconds: 5, - SuccessThreshold: 1, - FailureThreshold: 5, - }, - // # we poll on pod startup for the Kubernetes master service and - // # only setup the /readiness HTTP server once that's available. - ReadinessProbe: &v1.Probe{ - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Path: "/readiness", - Port: intstr.FromInt(8081), - Scheme: v1.URISchemeHTTP, - }, - }, - InitialDelaySeconds: 3, - TimeoutSeconds: 5, - }, - Args: []string{ - fmt.Sprintf("--domain=%s", cfg.Networking.DNSDomain), - fmt.Sprintf("--dns-port=%d", kubeDNSPort), - "--config-map=kube-dns", - "--v=2", - }, - Env: []v1.EnvVar{ - { - Name: "PROMETHEUS_PORT", - Value: "10055", - }, - }, - Ports: []v1.ContainerPort{ - { - ContainerPort: kubeDNSPort, - Name: "dns-local", - Protocol: v1.ProtocolUDP, - }, - { - ContainerPort: kubeDNSPort, - Name: "dns-tcp-local", - Protocol: v1.ProtocolTCP, - }, - { - ContainerPort: 10055, - Name: "metrics", - Protocol: v1.ProtocolTCP, - }, - }, - }, - // dnsmasq - { - Name: "dnsmasq", - Image: images.GetAddonImage(images.KubeDNSmasqImage), - LivenessProbe: &v1.Probe{ - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Path: "/healthcheck/dnsmasq", - Port: intstr.FromInt(10054), - Scheme: v1.URISchemeHTTP, - }, - }, - InitialDelaySeconds: 60, - TimeoutSeconds: 5, - SuccessThreshold: 1, - FailureThreshold: 5, - }, - Args: []string{ - "--cache-size=1000", - "--no-resolv", - fmt.Sprintf("--server=127.0.0.1#%d", kubeDNSPort), - "--log-facility=-", - }, - Ports: []v1.ContainerPort{ - { - ContainerPort: dnsmasqPort, - Name: "dns", - Protocol: v1.ProtocolUDP, - }, - { - ContainerPort: dnsmasqPort, - Name: "dns-tcp", - Protocol: v1.ProtocolTCP, - }, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceName(v1.ResourceCPU): resource.MustParse("150m"), - v1.ResourceName(v1.ResourceMemory): resource.MustParse("10Mi"), - }, - }, - }, - { - Name: "sidecar", - Image: images.GetAddonImage(images.KubeDNSSidecarImage), - LivenessProbe: &v1.Probe{ - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Path: "/metrics", - Port: intstr.FromInt(10054), - Scheme: v1.URISchemeHTTP, - }, - }, - InitialDelaySeconds: 60, - TimeoutSeconds: 5, - SuccessThreshold: 1, - FailureThreshold: 5, - }, - Args: []string{ - "--v=2", - "--logtostderr", - fmt.Sprintf("--probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.%s,5,A", cfg.Networking.DNSDomain), - fmt.Sprintf("--probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.%s,5,A", cfg.Networking.DNSDomain), - }, - Ports: []v1.ContainerPort{ - { - ContainerPort: 10054, - Name: "metrics", - Protocol: v1.ProtocolTCP, - }, - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceName(v1.ResourceMemory): resource.MustParse("20Mi"), - v1.ResourceName(v1.ResourceCPU): resource.MustParse("10m"), - }, - }, - }, - }, - DNSPolicy: v1.DNSDefault, - Affinity: &v1.Affinity{ - NodeAffinity: &v1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "beta.kubernetes.io/arch", - Operator: v1.NodeSelectorOpIn, - Values: []string{runtime.GOARCH}, - }, - }, - }, - }, - }, - }, - }, - } -} - -func createKubeDNSServiceSpec(cfg *kubeadmapi.MasterConfiguration) (*v1.ServiceSpec, error) { - _, n, err := net.ParseCIDR(cfg.Networking.ServiceSubnet) - if err != nil { - return nil, fmt.Errorf("could not parse %q: %v", cfg.Networking.ServiceSubnet, err) - } - ip, err := ipallocator.GetIndexedIP(n, 10) - if err != nil { - return nil, fmt.Errorf("unable to allocate IP address for kube-dns addon from the given CIDR %q: [%v]", cfg.Networking.ServiceSubnet, err) - } - - return &v1.ServiceSpec{ - Selector: map[string]string{"name": KubeDNS}, - Ports: []v1.ServicePort{ - {Name: "dns", Port: 53, Protocol: v1.ProtocolUDP}, - {Name: "dns-tcp", Port: 53, Protocol: v1.ProtocolTCP}, - }, - ClusterIP: ip.String(), - }, nil -} - -func CreateEssentialAddons(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error { - kubeProxyDaemonSet := NewDaemonSet(kubeProxy, createKubeProxyPodSpec(cfg)) - SetMasterTaintTolerations(&kubeProxyDaemonSet.Spec.Template.ObjectMeta) - - if _, err := client.Extensions().DaemonSets(metav1.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil { - return fmt.Errorf("failed creating essential kube-proxy addon [%v]", err) - } - - fmt.Println("[addons] Created essential addon: kube-proxy") - - kubeDNSDeployment := NewDeployment(KubeDNS, 1, createKubeDNSPodSpec(cfg)) - SetMasterTaintTolerations(&kubeDNSDeployment.Spec.Template.ObjectMeta) - - if _, err := client.Extensions().Deployments(metav1.NamespaceSystem).Create(kubeDNSDeployment); err != nil { - return fmt.Errorf("failed creating essential kube-dns addon [%v]", err) - } - - kubeDNSServiceSpec, err := createKubeDNSServiceSpec(cfg) - if err != nil { - return fmt.Errorf("failed creating essential kube-dns addon [%v]", err) - } - - kubeDNSService := NewService(KubeDNS, *kubeDNSServiceSpec) - kubeDNSService.ObjectMeta.Labels["kubernetes.io/name"] = "KubeDNS" - if _, err := client.Services(metav1.NamespaceSystem).Create(kubeDNSService); err != nil { - return fmt.Errorf("failed creating essential kube-dns addon [%v]", err) - } - - fmt.Println("[addons] Created essential addon: kube-dns") - - return nil -} diff --git a/cmd/kubeadm/app/master/addons_test.go b/cmd/kubeadm/app/master/addons_test.go deleted file mode 100644 index dd2896e0f9..0000000000 --- a/cmd/kubeadm/app/master/addons_test.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package master - -import ( - "testing" - - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -func TestCreateKubeProxyPodSpec(t *testing.T) { - var tests = []struct { - cfg *kubeadmapi.MasterConfiguration - expected bool - }{ - { - cfg: &kubeadmapi.MasterConfiguration{}, - expected: true, - }, - } - - for _, rt := range tests { - actual := createKubeProxyPodSpec(rt.cfg) - if (actual.Containers[0].Name != "") != rt.expected { - t.Errorf( - "failed createKubeProxyPodSpec:\n\texpected: %t\n\t actual: %t", - rt.expected, - (actual.Containers[0].Name != ""), - ) - } - } -} - -func TestCreateKubeDNSPodSpec(t *testing.T) { - var tests = []struct { - cfg *kubeadmapi.MasterConfiguration - expected string - }{ - { - cfg: &kubeadmapi.MasterConfiguration{ - Networking: kubeadm.Networking{DNSDomain: "localhost"}, - }, - expected: "--domain=localhost", - }, - { - cfg: &kubeadmapi.MasterConfiguration{ - Networking: kubeadm.Networking{DNSDomain: "foo"}, - }, - expected: "--domain=foo", - }, - } - - for _, rt := range tests { - actual := createKubeDNSPodSpec(rt.cfg) - if actual.Containers[0].Args[0] != rt.expected { - t.Errorf( - "failed createKubeDNSPodSpec:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual.Containers[0].Args[0], - ) - } - } -} - -func TestCreateKubeDNSServiceSpec(t *testing.T) { - var tests = []struct { - cfg *kubeadmapi.MasterConfiguration - expected bool - }{ - { - cfg: &kubeadmapi.MasterConfiguration{ - Networking: kubeadm.Networking{ServiceSubnet: "foo"}, - }, - expected: false, - }, - { - cfg: &kubeadmapi.MasterConfiguration{ - Networking: kubeadm.Networking{ServiceSubnet: "10.0.0.1/1"}, - }, - expected: false, - }, - { - cfg: &kubeadmapi.MasterConfiguration{ - Networking: kubeadm.Networking{ServiceSubnet: "10.0.0.1/24"}, - }, - expected: true, - }, - } - - for _, rt := range tests { - _, actual := createKubeDNSServiceSpec(rt.cfg) - if (actual == nil) != rt.expected { - t.Errorf( - "failed createKubeDNSServiceSpec:\n\texpected: %t\n\t actual: %t", - rt.expected, - (actual == nil), - ) - } - } -} diff --git a/cmd/kubeadm/app/master/apiclient.go b/cmd/kubeadm/app/master/apiclient.go index 0b960d88fe..214b71567f 100644 --- a/cmd/kubeadm/app/master/apiclient.go +++ b/cmd/kubeadm/app/master/apiclient.go @@ -17,31 +17,30 @@ limitations under the License. package master import ( - "encoding/json" "fmt" + "runtime" "time" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kuberuntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/clientcmd" - "k8s.io/kubernetes/cmd/kubeadm/app/images" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" extensions "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" ) -const apiCallRetryInterval = 500 * time.Millisecond - func CreateClientFromFile(path string) (*clientset.Clientset, error) { adminKubeconfig, err := clientcmd.LoadFromFile(path) if err != nil { return nil, fmt.Errorf("failed to load admin kubeconfig [%v]", err) } - adminClientConfig, err := clientcmd.NewDefaultClientConfig( - *adminKubeconfig, - &clientcmd.ConfigOverrides{}, - ).ClientConfig() + adminClientConfig, err := clientcmd.NewDefaultClientConfig(*adminKubeconfig, &clientcmd.ConfigOverrides{}).ClientConfig() if err != nil { return nil, fmt.Errorf("failed to create API client configuration [%v]", err) } @@ -64,7 +63,7 @@ func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) { fmt.Println("[apiclient] Waiting for at least one node to register and become ready") start := time.Now() - wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { + wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) { nodeList, err := client.Nodes().List(metav1.ListOptions{}) if err != nil { fmt.Println("[apiclient] Temporarily unable to list nodes (will retry)") @@ -83,21 +82,16 @@ func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) { return true, nil }) - createDummyDeployment(client) + if err := createAndWaitForADummyDeployment(client); err != nil { + return nil, err + } return client, nil } -func standardLabels(n string) map[string]string { - return map[string]string{ - "component": n, "name": n, "k8s-app": n, - "kubernetes.io/cluster-service": "true", "tier": "node", - } -} - func WaitForAPI(client *clientset.Clientset) { start := time.Now() - wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { + wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) { // TODO: use /healthz API instead of this cs, err := client.ComponentStatuses().List(metav1.ListOptions{}) if err != nil { @@ -125,76 +119,31 @@ func WaitForAPI(client *clientset.Clientset) { }) } -func NewDaemonSet(daemonName string, podSpec v1.PodSpec) *extensions.DaemonSet { - l := standardLabels(daemonName) - return &extensions.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{Name: daemonName}, - Spec: extensions.DaemonSetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: l}, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{Labels: l}, - Spec: podSpec, - }, - }, - } -} - -func NewService(serviceName string, spec v1.ServiceSpec) *v1.Service { - l := standardLabels(serviceName) - return &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Labels: l, - }, - Spec: spec, - } -} - -func NewDeployment(deploymentName string, replicas int32, podSpec v1.PodSpec) *extensions.Deployment { - l := standardLabels(deploymentName) - return &extensions.Deployment{ - ObjectMeta: metav1.ObjectMeta{Name: deploymentName}, - Spec: extensions.DeploymentSpec{ - Replicas: &replicas, - Selector: &metav1.LabelSelector{MatchLabels: l}, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{Labels: l}, - Spec: podSpec, - }, - }, - } -} - -func SetMasterTaintTolerations(meta *metav1.ObjectMeta) { - tolerationsAnnotation, _ := json.Marshal([]v1.Toleration{{Key: "dedicated", Value: "master", Effect: "NoSchedule"}}) - if meta.Annotations == nil { - meta.Annotations = map[string]string{} - } - meta.Annotations[v1.TolerationsAnnotationKey] = string(tolerationsAnnotation) -} - -func createDummyDeployment(client *clientset.Clientset) { - fmt.Println("[apiclient] Creating a test deployment") - dummyDeployment := NewDeployment("dummy", 1, v1.PodSpec{ - HostNetwork: true, - SecurityContext: &v1.PodSecurityContext{}, - Containers: []v1.Container{{ - Name: "dummy", - Image: images.GetAddonImage("pause"), - }}, +func createAndWaitForADummyDeployment(client *clientset.Clientset) error { + dummyDeploymentBytes, err := kubeadmutil.ParseTemplate(DummyDeployment, struct{ ImageRepository, Arch string }{ + ImageRepository: kubeadmapi.GlobalEnvParams.RepositoryPrefix, + Arch: runtime.GOARCH, }) + if err != nil { + return fmt.Errorf("error when parsing dummy deployment template: %v", err) + } - wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { + dummyDeployment := &extensions.Deployment{} + if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), dummyDeploymentBytes, dummyDeployment); err != nil { + return fmt.Errorf("unable to decode dummy deployment %v", err) + } + + wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) { // TODO: we should check the error, as some cases may be fatal - if _, err := client.Extensions().Deployments(metav1.NamespaceSystem).Create(dummyDeployment); err != nil { + if _, err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Create(dummyDeployment); err != nil { fmt.Printf("[apiclient] Failed to create test deployment [%v] (will retry)\n", err) return false, nil } return true, nil }) - wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { - d, err := client.Extensions().Deployments(metav1.NamespaceSystem).Get("dummy", metav1.GetOptions{}) + wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) { + d, err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Get("dummy", metav1.GetOptions{}) if err != nil { fmt.Printf("[apiclient] Failed to get test deployment [%v] (will retry)\n", err) return false, nil @@ -208,7 +157,8 @@ func createDummyDeployment(client *clientset.Clientset) { fmt.Println("[apiclient] Test deployment succeeded") // TODO: In the future, make sure the ReplicaSet and Pod are garbage collected - if err := client.Extensions().Deployments(metav1.NamespaceSystem).Delete("dummy", &metav1.DeleteOptions{}); err != nil { + if err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Delete("dummy", &metav1.DeleteOptions{}); err != nil { fmt.Printf("[apiclient] Failed to delete test deployment [%v] (will ignore)\n", err) } + return nil } diff --git a/cmd/kubeadm/app/master/apiclient_test.go b/cmd/kubeadm/app/master/apiclient_test.go deleted file mode 100644 index 9936e96506..0000000000 --- a/cmd/kubeadm/app/master/apiclient_test.go +++ /dev/null @@ -1,190 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package master - -import ( - "testing" - - "k8s.io/kubernetes/pkg/api/v1" -) - -func TestStandardLabels(t *testing.T) { - var tests = []struct { - n string - expected string - }{ - { - n: "foo", - expected: "foo", - }, - { - n: "bar", - expected: "bar", - }, - } - - for _, rt := range tests { - actual := standardLabels(rt.n) - if actual["component"] != rt.expected { - t.Errorf( - "failed standardLabels:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual["component"], - ) - } - if actual["name"] != rt.expected { - t.Errorf( - "failed standardLabels:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual["name"], - ) - } - if actual["k8s-app"] != rt.expected { - t.Errorf( - "failed standardLabels:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual["k8s-app"], - ) - } - } -} - -func TestNewDaemonSet(t *testing.T) { - var tests = []struct { - dn string - expected string - }{ - { - dn: "foo", - expected: "foo", - }, - { - dn: "bar", - expected: "bar", - }, - } - - for _, rt := range tests { - p := v1.PodSpec{} - actual := NewDaemonSet(rt.dn, p) - if actual.Spec.Selector.MatchLabels["k8s-app"] != rt.expected { - t.Errorf( - "failed NewDaemonSet:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual.Spec.Selector.MatchLabels["k8s-app"], - ) - } - if actual.Spec.Selector.MatchLabels["component"] != rt.expected { - t.Errorf( - "failed NewDaemonSet:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual.Spec.Selector.MatchLabels["component"], - ) - } - if actual.Spec.Selector.MatchLabels["name"] != rt.expected { - t.Errorf( - "failed NewDaemonSet:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual.Spec.Selector.MatchLabels["name"], - ) - } - } -} - -func TestNewService(t *testing.T) { - var tests = []struct { - dn string - expected string - }{ - { - dn: "foo", - expected: "foo", - }, - { - dn: "bar", - expected: "bar", - }, - } - - for _, rt := range tests { - p := v1.ServiceSpec{} - actual := NewService(rt.dn, p) - if actual.ObjectMeta.Labels["k8s-app"] != rt.expected { - t.Errorf( - "failed NewService:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual.ObjectMeta.Labels["k8s-app"], - ) - } - if actual.ObjectMeta.Labels["component"] != rt.expected { - t.Errorf( - "failed NewService:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual.ObjectMeta.Labels["component"], - ) - } - if actual.ObjectMeta.Labels["name"] != rt.expected { - t.Errorf( - "failed NewService:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual.ObjectMeta.Labels["name"], - ) - } - } -} - -func TestNewDeployment(t *testing.T) { - var tests = []struct { - dn string - expected string - }{ - { - dn: "foo", - expected: "foo", - }, - { - dn: "bar", - expected: "bar", - }, - } - - for _, rt := range tests { - p := v1.PodSpec{} - actual := NewDeployment(rt.dn, 1, p) - if actual.Spec.Selector.MatchLabels["k8s-app"] != rt.expected { - t.Errorf( - "failed NewDeployment:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual.Spec.Selector.MatchLabels["k8s-app"], - ) - } - if actual.Spec.Selector.MatchLabels["component"] != rt.expected { - t.Errorf( - "failed NewDeployment:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual.Spec.Selector.MatchLabels["component"], - ) - } - if actual.Spec.Selector.MatchLabels["name"] != rt.expected { - t.Errorf( - "failed NewDeployment:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual.Spec.Selector.MatchLabels["name"], - ) - } - } -} diff --git a/cmd/kubeadm/app/master/discovery.go b/cmd/kubeadm/app/master/discovery.go index 205a3d5f65..5e9e75afec 100644 --- a/cmd/kubeadm/app/master/discovery.go +++ b/cmd/kubeadm/app/master/discovery.go @@ -25,27 +25,25 @@ import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kuberuntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" certutil "k8s.io/client-go/util/cert" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" extensions "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" ) -type kubeDiscovery struct { - Deployment *extensions.Deployment - Secret *v1.Secret -} - const ( - kubeDiscoveryName = "kube-discovery" kubeDiscoverySecretName = "clusterinfo" + kubeDiscoveryName = "kube-discovery" ) +// TODO: Remove this file as soon as jbeda's token discovery refactoring PR has merged + func encodeKubeDiscoverySecretData(dcfg *kubeadmapi.TokenDiscovery, apicfg kubeadmapi.API, caCert *x509.Certificate) map[string][]byte { var ( data = map[string][]byte{} @@ -66,79 +64,6 @@ func encodeKubeDiscoverySecretData(dcfg *kubeadmapi.TokenDiscovery, apicfg kubea return data } -func newKubeDiscoveryPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec { - return v1.PodSpec{ - // We have to use host network namespace, as `HostPort`/`HostIP` are Docker's - // business and CNI support isn't quite there yet (except for kubenet) - // (see https://github.com/kubernetes/kubernetes/issues/31307) - // TODO update this when #31307 is resolved - HostNetwork: true, - SecurityContext: &v1.PodSecurityContext{}, - Containers: []v1.Container{{ - Name: kubeDiscoveryName, - Image: kubeadmapi.GlobalEnvParams.DiscoveryImage, - Command: []string{"/usr/local/bin/kube-discovery"}, - VolumeMounts: []v1.VolumeMount{{ - Name: kubeDiscoverySecretName, - MountPath: "/tmp/secret", // TODO use a shared constant - ReadOnly: true, - }}, - Ports: []v1.ContainerPort{ - // TODO when CNI issue (#31307) is resolved, we should consider adding - // `HostIP: s.API.AdvertiseAddrs[0]`, if there is only one address` - {Name: "http", ContainerPort: kubeadmapiext.DefaultDiscoveryBindPort, HostPort: kubeadmutil.DiscoveryPort(cfg.Discovery.Token)}, - }, - SecurityContext: &v1.SecurityContext{ - SELinuxOptions: &v1.SELinuxOptions{ - // TODO: This implies our discovery container is not being restricted by - // SELinux. This is not optimal and would be nice to adjust in future - // so it can read /tmp/secret, but for now this avoids recommending - // setenforce 0 system-wide. - Type: "spc_t", - }, - }, - }}, - Volumes: []v1.Volume{{ - Name: kubeDiscoverySecretName, - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{SecretName: kubeDiscoverySecretName}, - }}, - }, - Affinity: &v1.Affinity{ - NodeAffinity: &v1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "beta.kubernetes.io/arch", - Operator: v1.NodeSelectorOpIn, - Values: []string{runtime.GOARCH}, - }, - }, - }, - }, - }, - }, - }, - } -} - -func newKubeDiscovery(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate) kubeDiscovery { - kd := kubeDiscovery{ - Deployment: NewDeployment(kubeDiscoveryName, 1, newKubeDiscoveryPodSpec(cfg)), - Secret: &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: kubeDiscoverySecretName}, - Type: v1.SecretTypeOpaque, - Data: encodeKubeDiscoverySecretData(cfg.Discovery.Token, cfg.API, caCert), - }, - } - - SetMasterTaintTolerations(&kd.Deployment.Spec.Template.ObjectMeta) - - return kd -} - func CreateDiscoveryDeploymentAndSecret(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error { caCertificatePath := path.Join(kubeadmapi.GlobalEnvParams.HostPKIPath, kubeadmconstants.CACertName) caCerts, err := certutil.CertsFromFile(caCertificatePath) @@ -150,19 +75,23 @@ func CreateDiscoveryDeploymentAndSecret(cfg *kubeadmapi.MasterConfiguration, cli // TODO: Support multiple certs here in order to be able to rotate certs caCert := caCerts[0] - kd := newKubeDiscovery(cfg, caCert) - - if _, err := client.Extensions().Deployments(metav1.NamespaceSystem).Create(kd.Deployment); err != nil { - return fmt.Errorf("failed to create %q deployment [%v]", kubeDiscoveryName, err) + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: kubeDiscoverySecretName}, + Type: v1.SecretTypeOpaque, + Data: encodeKubeDiscoverySecretData(cfg.Discovery.Token, cfg.API, caCert), } - if _, err := client.Secrets(metav1.NamespaceSystem).Create(kd.Secret); err != nil { + if _, err := client.Secrets(metav1.NamespaceSystem).Create(secret); err != nil { return fmt.Errorf("failed to create %q secret [%v]", kubeDiscoverySecretName, err) } + if err := createDiscoveryDeployment(client); err != nil { + return err + } + fmt.Println("[token-discovery] Created the kube-discovery deployment, waiting for it to become ready") start := time.Now() - wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { + wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) { d, err := client.Extensions().Deployments(metav1.NamespaceSystem).Get(kubeDiscoveryName, metav1.GetOptions{}) if err != nil { return false, nil @@ -176,3 +105,22 @@ func CreateDiscoveryDeploymentAndSecret(cfg *kubeadmapi.MasterConfiguration, cli return nil } + +func createDiscoveryDeployment(client *clientset.Clientset) error { + discoveryBytes, err := kubeadmutil.ParseTemplate(KubeDiscoveryDeployment, struct{ ImageRepository, Arch string }{ + ImageRepository: kubeadmapi.GlobalEnvParams.RepositoryPrefix, + Arch: runtime.GOARCH, + }) + if err != nil { + return fmt.Errorf("error when parsing kube-discovery template: %v", err) + } + + discoveryDeployment := &extensions.Deployment{} + if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), discoveryBytes, discoveryDeployment); err != nil { + return fmt.Errorf("unable to decode kube-discovery deployment %v", err) + } + if _, err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Create(discoveryDeployment); err != nil { + return fmt.Errorf("unable to create a new discovery deployment: %v", err) + } + return nil +} diff --git a/cmd/kubeadm/app/master/discovery_test.go b/cmd/kubeadm/app/master/discovery_test.go deleted file mode 100644 index 1f6c16bbc1..0000000000 --- a/cmd/kubeadm/app/master/discovery_test.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package master - -import ( - "crypto/x509" - "testing" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -func TestNewKubeDiscovery(t *testing.T) { - var tests = []struct { - cfg *kubeadmapi.MasterConfiguration - caCert *x509.Certificate - expected bool - }{ - { - cfg: &kubeadmapi.MasterConfiguration{ - API: kubeadmapi.API{Port: 123, AdvertiseAddresses: []string{"10.0.0.1"}}, - Networking: kubeadmapi.Networking{ServiceSubnet: "10.0.0.1/1"}, - Discovery: kubeadmapi.Discovery{Token: &kubeadmapi.TokenDiscovery{}}, - }, - caCert: &x509.Certificate{}, - }, - } - for _, rt := range tests { - actual := newKubeDiscovery(rt.cfg, rt.caCert) - if actual.Deployment == nil || actual.Secret == nil { - t.Errorf( - "failed newKubeDiscovery, kubeDiscovery was nil", - ) - } - } -} diff --git a/cmd/kubeadm/app/master/selfhosted.go b/cmd/kubeadm/app/master/selfhosted.go index df8c8703b0..26c92d2196 100644 --- a/cmd/kubeadm/app/master/selfhosted.go +++ b/cmd/kubeadm/app/master/selfhosted.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/images" "k8s.io/kubernetes/pkg/api/v1" ext "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" @@ -79,7 +80,7 @@ func launchSelfHostedAPIServer(cfg *kubeadmapi.MasterConfiguration, client *clie return fmt.Errorf("failed to create self-hosted %q daemon set [%v]", kubeAPIServer, err) } - wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { + wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) { // TODO: This might be pointless, checking the pods is probably enough. // It does however get us a count of how many there should be which may be useful // with HA. @@ -157,7 +158,7 @@ func launchSelfHostedScheduler(cfg *kubeadmapi.MasterConfiguration, client *clie // waitForPodsWithLabel will lookup pods with the given label and wait until they are all // reporting status as running. func waitForPodsWithLabel(client *clientset.Clientset, appLabel string, mustBeRunning bool) { - wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { + wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) { // TODO: Do we need a stronger label link than this? listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("k8s-app=%s", appLabel)} apiPods, err := client.Pods(metav1.NamespaceSystem).List(listOpts) diff --git a/cmd/kubeadm/app/master/templates.go b/cmd/kubeadm/app/master/templates.go new file mode 100644 index 0000000000..9c51ed88cb --- /dev/null +++ b/cmd/kubeadm/app/master/templates.go @@ -0,0 +1,105 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package master + +const ( + DummyDeployment = ` +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + app: dummy + name: dummy + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: dummy + template: + metadata: + labels: + app: dummy + spec: + containers: + - image: {{ .ImageRepository }}/pause-{{ .Arch }}:3.0 + name: dummy + hostNetwork: true +` + KubeDiscoveryDeployment = ` +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + k8s-app: kube-discovery + kubernetes.io/cluster-service: "true" + name: kube-discovery + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: kube-discovery + kubernetes.io/cluster-service: "true" + template: + metadata: + labels: + k8s-app: kube-discovery + # TODO: I guess we can remove all these cluster-service labels... + kubernetes.io/cluster-service: "true" + annotations: + # TODO: Move this to the beta tolerations field below as soon as the Tolerations field exists in PodSpec + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","value":"master","effect":"NoSchedule"}]' + spec: + containers: + - name: kube-discovery + image: {{ .ImageRepository }}/kube-discovery-{{ .Arch }}:1.0 + imagePullPolicy: IfNotPresent + command: + - /usr/local/bin/kube-discovery + ports: + - containerPort: 9898 + hostPort: 9898 + name: http + volumeMounts: + - mountPath: /tmp/secret + name: clusterinfo + readOnly: true + hostNetwork: true + # tolerations: + # - key: dedicated + # value: master + # effect: NoSchedule + securityContext: + seLinuxOptions: + type: spc_t + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - {{ .Arch }} + volumes: + - name: clusterinfo + secret: + defaultMode: 420 + secretName: clusterinfo +` +) diff --git a/cmd/kubeadm/app/phases/addons/BUILD b/cmd/kubeadm/app/phases/addons/BUILD new file mode 100644 index 0000000000..293c8d873e --- /dev/null +++ b/cmd/kubeadm/app/phases/addons/BUILD @@ -0,0 +1,41 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = [ + "addons.go", + "manifests.go", + ], + tags = ["automanaged"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/util:go_default_library", + "//pkg/api:go_default_library", + "//pkg/api/v1:go_default_library", + "//pkg/apis/extensions/v1beta1:go_default_library", + "//pkg/client/clientset_generated/clientset:go_default_library", + "//pkg/registry/core/service/ipallocator:go_default_library", + "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1", + "//vendor:k8s.io/apimachinery/pkg/runtime", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/cmd/kubeadm/app/phases/addons/addons.go b/cmd/kubeadm/app/phases/addons/addons.go new file mode 100644 index 0000000000..b968106f7c --- /dev/null +++ b/cmd/kubeadm/app/phases/addons/addons.go @@ -0,0 +1,155 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package addons + +import ( + "fmt" + "net" + + "runtime" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kuberuntime "k8s.io/apimachinery/pkg/runtime" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/v1" + extensions "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" + "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" + "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" +) + +// CreateEssentialAddons creates the kube-proxy and kube-dns addons +func CreateEssentialAddons(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error { + + proxyConfigMapBytes, err := kubeadmutil.ParseTemplate(KubeProxyConfigMap, struct{ MasterEndpoint string }{ + // Fetch this value from the kubeconfig file + MasterEndpoint: fmt.Sprintf("https://%s:%d", cfg.API.AdvertiseAddresses[0], cfg.API.Port), + }) + if err != nil { + return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err) + } + + proxyDaemonSetBytes, err := kubeadmutil.ParseTemplate(KubeProxyDaemonSet, struct{ ImageRepository, Arch, Version string }{ + ImageRepository: kubeadmapi.GlobalEnvParams.RepositoryPrefix, + Arch: runtime.GOARCH, + // TODO: Fetch the version from the {API Server IP}/version + Version: cfg.KubernetesVersion, + }) + if err != nil { + return fmt.Errorf("error when parsing kube-proxy daemonset template: %v", err) + } + + dnsDeploymentBytes, err := kubeadmutil.ParseTemplate(KubeDNSDeployment, struct { + ImageRepository, Arch, Version, DNSDomain string + Replicas int + }{ + ImageRepository: kubeadmapi.GlobalEnvParams.RepositoryPrefix, + Arch: runtime.GOARCH, + // TODO: Support larger amount of replicas? + Replicas: 1, + Version: KubeDNSVersion, + DNSDomain: cfg.Networking.DNSDomain, + }) + if err != nil { + return fmt.Errorf("error when parsing kube-dns deployment template: %v", err) + } + + // Get the DNS IP + dnsip, err := getDNSIP(cfg.Networking.ServiceSubnet) + if err != nil { + return err + } + + dnsServiceBytes, err := kubeadmutil.ParseTemplate(KubeDNSService, struct{ DNSIP string }{ + DNSIP: dnsip.String(), + }) + if err != nil { + return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err) + } + + err = CreateKubeProxyAddon(proxyConfigMapBytes, proxyDaemonSetBytes, client) + if err != nil { + return err + } + fmt.Println("[addons] Created essential addon: kube-proxy") + + err = CreateKubeDNSAddon(dnsDeploymentBytes, dnsServiceBytes, client) + if err != nil { + return err + } + fmt.Println("[addons] Created essential addon: kube-dns") + return nil +} + +func CreateKubeProxyAddon(configMapBytes, daemonSetbytes []byte, client *clientset.Clientset) error { + kubeproxyConfigMap := &v1.ConfigMap{} + if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), configMapBytes, kubeproxyConfigMap); err != nil { + return fmt.Errorf("unable to decode kube-proxy configmap %v", err) + } + + if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Create(kubeproxyConfigMap); err != nil { + return fmt.Errorf("unable to create a new kube-proxy configmap: %v", err) + } + + kubeproxyDaemonSet := &extensions.DaemonSet{} + if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), daemonSetbytes, kubeproxyDaemonSet); err != nil { + return fmt.Errorf("unable to decode kube-proxy daemonset %v", err) + } + + if _, err := client.ExtensionsV1beta1().DaemonSets(metav1.NamespaceSystem).Create(kubeproxyDaemonSet); err != nil { + return fmt.Errorf("unable to create a new kube-proxy daemonset: %v", err) + } + return nil +} + +func CreateKubeDNSAddon(deploymentBytes, serviceBytes []byte, client *clientset.Clientset) error { + kubednsDeployment := &extensions.Deployment{} + if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), deploymentBytes, kubednsDeployment); err != nil { + return fmt.Errorf("unable to decode kube-dns deployment %v", err) + } + + // TODO: All these .Create(foo) calls should instead be more like "kubectl apply -f" commands; they should not fail if there are existing objects with the same name + if _, err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Create(kubednsDeployment); err != nil { + return fmt.Errorf("unable to create a new kube-dns deployment: %v", err) + } + + kubednsService := &v1.Service{} + if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), serviceBytes, kubednsService); err != nil { + return fmt.Errorf("unable to decode kube-dns service %v", err) + } + + if _, err := client.CoreV1().Services(metav1.NamespaceSystem).Create(kubednsService); err != nil { + return fmt.Errorf("unable to create a new kube-dns service: %v", err) + } + return nil +} + +// TODO: Instead of looking at the subnet given to kubeadm, it should be possible to only use /28 or larger subnets and then +// kubeadm should look at the kubernetes service (e.g. 10.96.0.1 or 10.0.0.1) and just append a "0" at the end. +// This way, we don't need the information about the subnet in this phase => good +func getDNSIP(subnet string) (net.IP, error) { + _, n, err := net.ParseCIDR(subnet) + if err != nil { + return nil, fmt.Errorf("could not parse %q: %v", subnet, err) + } + ip, err := ipallocator.GetIndexedIP(n, 10) + if err != nil { + return nil, fmt.Errorf("unable to allocate IP address for kube-dns addon from the given CIDR %q: [%v]", subnet, err) + } + return ip, nil +} diff --git a/cmd/kubeadm/app/phases/addons/manifests.go b/cmd/kubeadm/app/phases/addons/manifests.go new file mode 100644 index 0000000000..f79b7e111a --- /dev/null +++ b/cmd/kubeadm/app/phases/addons/manifests.go @@ -0,0 +1,273 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package addons + +const ( + KubeProxyConfigMap = ` +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-proxy + namespace: kube-system + labels: + app: kube-proxy +data: + kubeconfig.conf: | + apiVersion: v1 + kind: Config + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: {{ .MasterEndpoint }} + name: default + contexts: + - context: + cluster: default + namespace: default + user: default + name: default + current-context: default + users: + - name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +` + + KubeProxyDaemonSet = ` +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-proxy + kubernetes.io/cluster-service: "true" + name: kube-proxy + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-proxy + template: + metadata: + labels: + k8s-app: kube-proxy + kubernetes.io/cluster-service: "true" + annotations: + # TODO: Move this to the beta tolerations field below as soon as the Tolerations field exists in PodSpec + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","value":"master","effect":"NoSchedule"}]' + spec: + containers: + - name: kube-proxy + image: {{ .ImageRepository }}/kube-proxy-{{ .Arch }}:{{ .Version }} + imagePullPolicy: IfNotPresent + command: + - kube-proxy + - --kubeconfig=/var/lib/kube-proxy/kubeconfig.conf + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + hostNetwork: true + serviceAccountName: kube-proxy + # Tolerate running on the master + # tolerations: + # - key: dedicated + # value: master + # effect: NoSchedule + volumes: + - name: kube-proxy + configMap: + name: kube-proxy +` + + KubeDNSVersion = "1.11.0" + + KubeDNSDeployment = ` +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + name: kube-dns + namespace: kube-system +spec: + replicas: {{ .Replicas }} + selector: + matchLabels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + annotations: + # TODO: Move this to the beta tolerations field below as soon as the Tolerations field exists in PodSpec + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","value":"master","effect":"NoSchedule"}]' + spec: + containers: + - name: kubedns + image: {{ .ImageRepository }}/k8s-dns-kube-dns-{{ .Arch }}:{{ .Version }} + imagePullPolicy: IfNotPresent + args: + - --domain={{ .DNSDomain }} + - --dns-port=10053 + - --config-map=kube-dns + - --v=2 + env: + - name: PROMETHEUS_PORT + value: "10055" + ports: + - containerPort: 10053 + name: dns-local + protocol: UDP + - containerPort: 10053 + name: dns-tcp-local + protocol: TCP + - containerPort: 10055 + name: metrics + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthcheck/kubedns + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readiness + port: 8081 + scheme: HTTP + initialDelaySeconds: 3 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + - name: dnsmasq + image: {{ .ImageRepository }}/k8s-dns-dnsmasq-{{ .Arch }}:{{ .Version }} + imagePullPolicy: IfNotPresent + args: + - --cache-size=1000 + - --no-resolv + - --server=127.0.0.1#10053 + - --log-facility=- + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthcheck/dnsmasq + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + resources: + requests: + cpu: 150m + memory: 10Mi + - name: sidecar + image: {{ .ImageRepository }}/k8s-dns-sidecar-{{ .Arch }}:{{ .Version }} + imagePullPolicy: IfNotPresent + args: + - --v=2 + - --logtostderr + - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.{{ .DNSDomain }},5,A + - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.{{ .DNSDomain }},5,A + ports: + - containerPort: 10054 + name: metrics + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /metrics + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + resources: + requests: + cpu: 10m + memory: 20Mi + dnsPolicy: Default + serviceAccountName: kube-dns + # tolerations: + # - key: dedicated + # value: master + # effect: NoSchedule + # TODO: Remove this affinity field as soon as we are using manifest lists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - {{ .Arch }} +` + + KubeDNSService = ` +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "KubeDNS" + name: kube-dns + namespace: kube-system +spec: + clusterIP: {{ .DNSIP }} + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 + selector: + k8s-app: kube-dns +` +) diff --git a/cmd/kubeadm/app/util/BUILD b/cmd/kubeadm/app/util/BUILD index 062590cdc1..8e36965304 100644 --- a/cmd/kubeadm/app/util/BUILD +++ b/cmd/kubeadm/app/util/BUILD @@ -12,6 +12,7 @@ go_library( name = "go_default_library", srcs = [ "error.go", + "template.go", "tokens.go", "version.go", ], @@ -32,6 +33,7 @@ go_test( name = "go_default_test", srcs = [ "error_test.go", + "template_test.go", "tokens_test.go", "version_test.go", ], diff --git a/cmd/kubeadm/app/util/template.go b/cmd/kubeadm/app/util/template.go new file mode 100644 index 0000000000..7e1e8702c5 --- /dev/null +++ b/cmd/kubeadm/app/util/template.go @@ -0,0 +1,37 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "bytes" + "fmt" + "text/template" +) + +// TODO: Should be unit-tested +func ParseTemplate(strtmpl string, obj interface{}) ([]byte, error) { + var buf bytes.Buffer + tmpl, err := template.New("template").Parse(strtmpl) + if err != nil { + return nil, fmt.Errorf("error when parsing template: %v", err) + } + err = tmpl.Execute(&buf, obj) + if err != nil { + return nil, fmt.Errorf("error when executing template: %v", err) + } + return buf.Bytes(), nil +} diff --git a/cmd/kubeadm/app/util/template_test.go b/cmd/kubeadm/app/util/template_test.go new file mode 100644 index 0000000000..3a00e05e60 --- /dev/null +++ b/cmd/kubeadm/app/util/template_test.go @@ -0,0 +1,90 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" +) + +const ( + validTmpl = "image: {{ .ImageRepository }}/pause-{{ .Arch }}:3.0" + validTmplOut = "image: gcr.io/google_containers/pause-amd64:3.0" + doNothing = "image: gcr.io/google_containers/pause-amd64:3.0" + invalidTmpl1 = "{{ .baz }/d}" + invalidTmpl2 = "{{ !foobar }}" +) + +func TestParseTemplate(t *testing.T) { + var tmplTests = []struct { + template string + data interface{} + output string + errExpected bool + }{ + // should parse a valid template and set the right values + { + template: validTmpl, + data: struct{ ImageRepository, Arch string }{ + ImageRepository: "gcr.io/google_containers", + Arch: "amd64", + }, + output: validTmplOut, + errExpected: false, + }, + // should noop if there aren't any {{ .foo }} present + { + template: doNothing, + data: struct{ ImageRepository, Arch string }{ + ImageRepository: "gcr.io/google_containers", + Arch: "amd64", + }, + output: doNothing, + errExpected: false, + }, + // invalid syntax, passing nil + { + template: invalidTmpl1, + data: nil, + output: "", + errExpected: true, + }, + // invalid syntax + { + template: invalidTmpl2, + data: struct{}{}, + output: "", + errExpected: true, + }, + } + for _, tt := range tmplTests { + outbytes, err := ParseTemplate(tt.template, tt.data) + if tt.errExpected != (err != nil) { + t.Errorf( + "failed TestParseTemplate:\n\texpected err: %t\n\t actual: %s", + tt.errExpected, + err, + ) + } + if tt.output != string(outbytes) { + t.Errorf( + "failed TestParseTemplate:\n\texpected bytes: %s\n\t actual: %s", + tt.output, + outbytes, + ) + } + } +}