Merge pull request #47435 from luxas/kubeadm_new_selfhosting

Automatic merge from submit-queue (batch tested with PRs 47435, 46044)

kubeadm: Make self-hosting work and split it out to a phase

**What this PR does / why we need it**:

 - Removes the old self-hosting code
 - Puts the new self-hosting code in `phases/selfhosting`
   - The new code reads manifests from disk (static pods)...
   - ...mutates the PodSpec as necessary...
   - ...and posts the DaemonSet to the API Server...
   - ...and waits for it to come up
 - Uses DaemonSets for all control plane components
 - Creates a `kubeadm alpha phase selfhosting` command that can be invoked against any kubeadm-cluster after install.

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #

fixes: https://github.com/kubernetes/kubeadm/issues/127
(large part of at least)

**Special notes for your reviewer**:

Please only review the fourth commit, based on https://github.com/kubernetes/kubernetes/pull/47345

**Release note**:

```release-note
kubeadm: Make self-hosting work by using DaemonSets and split it out to a phase that can be invoked via the CLI
```
@kubernetes/sig-cluster-lifecycle-pr-reviews @jbeda
pull/6/head
Kubernetes Submit Queue 2017-07-06 12:43:39 -07:00 committed by GitHub
commit b00df7eb89
18 changed files with 1116 additions and 387 deletions

View File

@ -41,6 +41,7 @@ filegroup(
"//cmd/kubeadm/app/phases/certs:all-srcs",
"//cmd/kubeadm/app/phases/controlplane:all-srcs",
"//cmd/kubeadm/app/phases/kubeconfig:all-srcs",
"//cmd/kubeadm/app/phases/selfhosting:all-srcs",
"//cmd/kubeadm/app/phases/token:all-srcs",
"//cmd/kubeadm/app/preflight:all-srcs",
"//cmd/kubeadm/app/util:all-srcs",

View File

@ -34,6 +34,7 @@ go_library(
"//cmd/kubeadm/app/phases/certs:go_default_library",
"//cmd/kubeadm/app/phases/controlplane:go_default_library",
"//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
"//cmd/kubeadm/app/phases/selfhosting:go_default_library",
"//cmd/kubeadm/app/phases/token:go_default_library",
"//cmd/kubeadm/app/preflight:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",

View File

@ -37,6 +37,7 @@ import (
certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
selfhostingphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting"
tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/token"
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
@ -132,6 +133,10 @@ func NewCmdInit(out io.Writer) *cobra.Command {
&skipTokenPrint, "skip-token-print", skipTokenPrint,
"Skip printing of the default bootstrap token generated by 'kubeadm init'",
)
cmd.PersistentFlags().BoolVar(
&cfg.SelfHosted, "self-hosted", cfg.SelfHosted,
"[experimental] If kubeadm should make this control plane self-hosted",
)
cmd.PersistentFlags().StringVar(
&cfg.Token, "token", cfg.Token,
@ -266,7 +271,7 @@ func (i *Init) Run(out io.Writer) error {
// Temporary control plane is up, now we create our self hosted control
// plane components and remove the static manifests:
fmt.Println("[self-hosted] Creating self-hosted control plane...")
if err := controlplanephase.CreateSelfHostedControlPlane(i.cfg, client); err != nil {
if err := selfhostingphase.CreateSelfHostedControlPlane(client); err != nil {
return err
}
}

View File

@ -14,6 +14,7 @@ go_library(
"kubeconfig.go",
"phase.go",
"preflight.go",
"selfhosting.go",
],
tags = ["automanaged"],
deps = [
@ -22,8 +23,10 @@ go_library(
"//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library",
"//cmd/kubeadm/app/phases/certs:go_default_library",
"//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
"//cmd/kubeadm/app/phases/selfhosting:go_default_library",
"//cmd/kubeadm/app/preflight:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/kubeconfig:go_default_library",
"//pkg/api:go_default_library",
"//vendor/github.com/spf13/cobra:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",

View File

@ -33,6 +33,7 @@ func NewCmdPhase(out io.Writer) *cobra.Command {
cmd.AddCommand(NewCmdKubeConfig(out))
cmd.AddCommand(NewCmdCerts())
cmd.AddCommand(NewCmdPreFlight())
cmd.AddCommand(NewCmdSelfhosting())
return cmd
}

View File

@ -0,0 +1,45 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package phases
import (
"github.com/spf13/cobra"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
)
// NewCmdSelfhosting returns the self-hosting Cobra command
func NewCmdSelfhosting() *cobra.Command {
var kubeConfigFile string
cmd := &cobra.Command{
Use: "selfhosting",
Aliases: []string{"selfhosted"},
Short: "Make a kubeadm cluster self-hosted.",
Run: func(cmd *cobra.Command, args []string) {
client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile)
kubeadmutil.CheckErr(err)
err = selfhosting.CreateSelfHostedControlPlane(client)
kubeadmutil.CheckErr(err)
},
}
cmd.Flags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use for talking to the cluster")
return cmd
}

View File

@ -109,3 +109,8 @@ var (
// MinimumControlPlaneVersion specifies the minimum control plane version kubeadm can deploy
MinimumControlPlaneVersion = version.MustParseSemantic("v1.7.0")
)
// BuildStaticManifestFilepath returns the location on the disk where the Static Pod should be present
func BuildStaticManifestFilepath(componentName string) string {
return filepath.Join(KubernetesDir, ManifestsSubDirName, componentName+".yaml")
}

View File

@ -25,30 +25,23 @@ go_test(
go_library(
name = "go_default_library",
srcs = [
"manifests.go",
"selfhosted.go",
],
srcs = ["manifests.go"],
tags = ["automanaged"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/images:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//pkg/kubeapiserver/authorizer/modes:go_default_library",
"//pkg/kubectl/cmd/util:go_default_library",
"//pkg/kubelet/types:go_default_library",
"//pkg/util/version:go_default_library",
"//vendor/github.com/ghodss/yaml:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
],
)

View File

@ -229,23 +229,6 @@ func pkiVolumeMount() v1.VolumeMount {
}
}
func flockVolume() v1.Volume {
return v1.Volume{
Name: "var-lock",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{Path: "/var/lock"},
},
}
}
func flockVolumeMount() v1.VolumeMount {
return v1.VolumeMount{
Name: "var-lock",
MountPath: "/var/lock",
ReadOnly: false,
}
}
func k8sVolume() v1.Volume {
return v1.Volume{
Name: "k8s",
@ -447,19 +430,6 @@ func getProxyEnvVars() []v1.EnvVar {
return envs
}
func getSelfHostedAPIServerEnv() []v1.EnvVar {
podIPEnvVar := v1.EnvVar{
Name: "POD_IP",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "status.podIP",
},
},
}
return append(getProxyEnvVars(), podIPEnvVar)
}
// getAuthzParameters gets the authorization-related parameters to the api server
// At this point, we can assume the list of authorization modes is valid (due to that it has been validated in the API machinery code already)
// If the list is empty; it's defaulted (mostly for unit testing)

View File

@ -1,348 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controlplane
import (
"fmt"
"os"
"path/filepath"
"time"
"k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/images"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/pkg/util/version"
)
var (
// maximum unavailable and surge instances per self-hosted component deployment
maxUnavailable = intstr.FromInt(0)
maxSurge = intstr.FromInt(1)
)
func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error {
volumes := []v1.Volume{k8sVolume()}
volumeMounts := []v1.VolumeMount{k8sVolumeMount()}
if isCertsVolumeMountNeeded() {
volumes = append(volumes, certsVolume(cfg))
volumeMounts = append(volumeMounts, certsVolumeMount())
}
if isPkiVolumeMountNeeded() {
volumes = append(volumes, pkiVolume())
volumeMounts = append(volumeMounts, pkiVolumeMount())
}
// Need lock for self-hosted
volumes = append(volumes, flockVolume())
volumeMounts = append(volumeMounts, flockVolumeMount())
if err := launchSelfHostedAPIServer(cfg, client, volumes, volumeMounts); err != nil {
return err
}
if err := launchSelfHostedScheduler(cfg, client, volumes, volumeMounts); err != nil {
return err
}
if err := launchSelfHostedControllerManager(cfg, client, volumes, volumeMounts); err != nil {
return err
}
return nil
}
func launchSelfHostedAPIServer(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error {
start := time.Now()
kubeVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
if err != nil {
return err
}
apiServer := getAPIServerDS(cfg, volumes, volumeMounts, kubeVersion)
if _, err := client.Extensions().DaemonSets(metav1.NamespaceSystem).Create(&apiServer); err != nil {
return fmt.Errorf("failed to create self-hosted %q daemon set [%v]", kubeAPIServer, err)
}
wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
// TODO: This might be pointless, checking the pods is probably enough.
// It does however get us a count of how many there should be which may be useful
// with HA.
apiDS, err := client.DaemonSets(metav1.NamespaceSystem).Get("self-hosted-"+kubeAPIServer,
metav1.GetOptions{})
if err != nil {
fmt.Println("[self-hosted] error getting apiserver DaemonSet:", err)
return false, nil
}
fmt.Printf("[self-hosted] %s DaemonSet current=%d, desired=%d\n",
kubeAPIServer,
apiDS.Status.CurrentNumberScheduled,
apiDS.Status.DesiredNumberScheduled)
if apiDS.Status.CurrentNumberScheduled != apiDS.Status.DesiredNumberScheduled {
return false, nil
}
return true, nil
})
// Wait for self-hosted API server to take ownership
waitForPodsWithLabel(client, "self-hosted-"+kubeAPIServer, true)
// Remove temporary API server
apiServerStaticManifestPath := buildStaticManifestFilepath(kubeAPIServer)
if err := os.RemoveAll(apiServerStaticManifestPath); err != nil {
return fmt.Errorf("unable to delete temporary API server manifest [%v]", err)
}
kubeadmutil.WaitForAPI(client)
fmt.Printf("[self-hosted] self-hosted kube-apiserver ready after %f seconds\n", time.Since(start).Seconds())
return nil
}
func launchSelfHostedControllerManager(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error {
start := time.Now()
kubeVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
if err != nil {
return err
}
ctrlMgr := getControllerManagerDeployment(cfg, volumes, volumeMounts, kubeVersion)
if _, err := client.Extensions().Deployments(metav1.NamespaceSystem).Create(&ctrlMgr); err != nil {
return fmt.Errorf("failed to create self-hosted %q deployment [%v]", kubeControllerManager, err)
}
waitForPodsWithLabel(client, "self-hosted-"+kubeControllerManager, true)
ctrlMgrStaticManifestPath := buildStaticManifestFilepath(kubeControllerManager)
if err := os.RemoveAll(ctrlMgrStaticManifestPath); err != nil {
return fmt.Errorf("unable to delete temporary controller manager manifest [%v]", err)
}
fmt.Printf("[self-hosted] self-hosted kube-controller-manager ready after %f seconds\n", time.Since(start).Seconds())
return nil
}
func launchSelfHostedScheduler(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error {
start := time.Now()
scheduler := getSchedulerDeployment(cfg, volumes, volumeMounts)
if _, err := client.Extensions().Deployments(metav1.NamespaceSystem).Create(&scheduler); err != nil {
return fmt.Errorf("failed to create self-hosted %q deployment [%v]", kubeScheduler, err)
}
waitForPodsWithLabel(client, "self-hosted-"+kubeScheduler, true)
schedulerStaticManifestPath := buildStaticManifestFilepath(kubeScheduler)
if err := os.RemoveAll(schedulerStaticManifestPath); err != nil {
return fmt.Errorf("unable to delete temporary scheduler manifest [%v]", err)
}
fmt.Printf("[self-hosted] self-hosted kube-scheduler ready after %f seconds\n", time.Since(start).Seconds())
return nil
}
// waitForPodsWithLabel will lookup pods with the given label and wait until they are all
// reporting status as running.
func waitForPodsWithLabel(client *clientset.Clientset, appLabel string, mustBeRunning bool) {
wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
// TODO: Do we need a stronger label link than this?
listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("k8s-app=%s", appLabel)}
apiPods, err := client.Pods(metav1.NamespaceSystem).List(listOpts)
if err != nil {
fmt.Printf("[self-hosted] error getting %s pods [%v]\n", appLabel, err)
return false, nil
}
fmt.Printf("[self-hosted] Found %d %s pods\n", len(apiPods.Items), appLabel)
// TODO: HA
if int32(len(apiPods.Items)) != 1 {
return false, nil
}
for _, pod := range apiPods.Items {
fmt.Printf("[self-hosted] Pod %s status: %s\n", pod.Name, pod.Status.Phase)
if mustBeRunning && pod.Status.Phase != "Running" {
return false, nil
}
}
return true, nil
})
}
// Sources from bootkube templates.go
func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount, kubeVersion *version.Version) extensions.DaemonSet {
ds := extensions.DaemonSet{
TypeMeta: metav1.TypeMeta{
APIVersion: "extensions/v1beta1",
Kind: "DaemonSet",
},
ObjectMeta: metav1.ObjectMeta{
Name: "self-hosted-" + kubeAPIServer,
Namespace: "kube-system",
Labels: map[string]string{"k8s-app": "self-hosted-" + kubeAPIServer},
},
Spec: extensions.DaemonSetSpec{
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"k8s-app": "self-hosted-" + kubeAPIServer,
"component": kubeAPIServer,
"tier": "control-plane",
},
},
Spec: v1.PodSpec{
NodeSelector: map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""},
HostNetwork: true,
Volumes: volumes,
Containers: []v1.Container{
{
Name: "self-hosted-" + kubeAPIServer,
Image: images.GetCoreImage(images.KubeAPIServerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
Command: getAPIServerCommand(cfg, true, kubeVersion),
Env: getSelfHostedAPIServerEnv(),
VolumeMounts: volumeMounts,
LivenessProbe: componentProbe(6443, "/healthz", v1.URISchemeHTTPS),
Resources: componentResources("250m"),
},
},
Tolerations: []v1.Toleration{kubeadmconstants.MasterToleration},
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
},
}
return ds
}
func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount, kubeVersion *version.Version) extensions.Deployment {
d := extensions.Deployment{
TypeMeta: metav1.TypeMeta{
APIVersion: "extensions/v1beta1",
Kind: "Deployment",
},
ObjectMeta: metav1.ObjectMeta{
Name: "self-hosted-" + kubeControllerManager,
Namespace: "kube-system",
Labels: map[string]string{"k8s-app": "self-hosted-" + kubeControllerManager},
},
Spec: extensions.DeploymentSpec{
// TODO bootkube uses 2 replicas
Strategy: extensions.DeploymentStrategy{
Type: extensions.RollingUpdateDeploymentStrategyType,
RollingUpdate: &extensions.RollingUpdateDeployment{
MaxUnavailable: &maxUnavailable,
MaxSurge: &maxSurge,
},
},
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"k8s-app": "self-hosted-" + kubeControllerManager,
"component": kubeControllerManager,
"tier": "control-plane",
},
},
Spec: v1.PodSpec{
NodeSelector: map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""},
HostNetwork: true,
Volumes: volumes,
Containers: []v1.Container{
{
Name: "self-hosted-" + kubeControllerManager,
Image: images.GetCoreImage(images.KubeControllerManagerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
Command: getControllerManagerCommand(cfg, true, kubeVersion),
VolumeMounts: volumeMounts,
LivenessProbe: componentProbe(10252, "/healthz", v1.URISchemeHTTP),
Resources: componentResources("200m"),
Env: getProxyEnvVars(),
},
},
Tolerations: []v1.Toleration{kubeadmconstants.MasterToleration},
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
},
}
return d
}
func getSchedulerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount) extensions.Deployment {
d := extensions.Deployment{
TypeMeta: metav1.TypeMeta{
APIVersion: "extensions/v1beta1",
Kind: "Deployment",
},
ObjectMeta: metav1.ObjectMeta{
Name: "self-hosted-" + kubeScheduler,
Namespace: "kube-system",
Labels: map[string]string{"k8s-app": "self-hosted-" + kubeScheduler},
},
Spec: extensions.DeploymentSpec{
// TODO bootkube uses 2 replicas
Strategy: extensions.DeploymentStrategy{
Type: extensions.RollingUpdateDeploymentStrategyType,
RollingUpdate: &extensions.RollingUpdateDeployment{
MaxUnavailable: &maxUnavailable,
MaxSurge: &maxSurge,
},
},
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"k8s-app": "self-hosted-" + kubeScheduler,
"component": kubeScheduler,
"tier": "control-plane",
},
},
Spec: v1.PodSpec{
NodeSelector: map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""},
HostNetwork: true,
Volumes: volumes,
Containers: []v1.Container{
{
Name: "self-hosted-" + kubeScheduler,
Image: images.GetCoreImage(images.KubeSchedulerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
Command: getSchedulerCommand(cfg, true),
VolumeMounts: volumeMounts,
LivenessProbe: componentProbe(10251, "/healthz", v1.URISchemeHTTP),
Resources: componentResources("100m"),
Env: getProxyEnvVars(),
},
},
Tolerations: []v1.Toleration{kubeadmconstants.MasterToleration},
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
},
}
return d
}
func buildStaticManifestFilepath(name string) string {
return filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.ManifestsSubDirName, name+".yaml")
}

View File

@ -0,0 +1,57 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = [
"podspec_mutation_test.go",
"selfhosting_test.go",
],
library = ":go_default_library",
tags = ["automanaged"],
deps = [
"//cmd/kubeadm/app/constants:go_default_library",
"//vendor/github.com/ghodss/yaml:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"podspec_mutation.go",
"selfhosting.go",
],
tags = ["automanaged"],
deps = [
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//pkg/api:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View File

@ -0,0 +1,74 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package selfhosting
import (
"k8s.io/api/core/v1"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
)
// mutatePodSpec makes a Static Pod-hosted PodSpec suitable for self-hosting
func mutatePodSpec(name string, podSpec *v1.PodSpec) {
mutators := map[string][]func(*v1.PodSpec){
kubeAPIServer: {
addNodeSelectorToPodSpec,
setMasterTolerationOnPodSpec,
setRightDNSPolicyOnPodSpec,
},
kubeControllerManager: {
addNodeSelectorToPodSpec,
setMasterTolerationOnPodSpec,
setRightDNSPolicyOnPodSpec,
},
kubeScheduler: {
addNodeSelectorToPodSpec,
setMasterTolerationOnPodSpec,
setRightDNSPolicyOnPodSpec,
},
}
// Get the mutator functions for the component in question, then loop through and execute them
mutatorsForComponent := mutators[name]
for _, mutateFunc := range mutatorsForComponent {
mutateFunc(podSpec)
}
}
// addNodeSelectorToPodSpec makes Pod require to be scheduled on a node marked with the master label
func addNodeSelectorToPodSpec(podSpec *v1.PodSpec) {
if podSpec.NodeSelector == nil {
podSpec.NodeSelector = map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""}
return
}
podSpec.NodeSelector[kubeadmconstants.LabelNodeRoleMaster] = ""
}
// setMasterTolerationOnPodSpec makes the Pod tolerate the master taint
func setMasterTolerationOnPodSpec(podSpec *v1.PodSpec) {
if podSpec.Tolerations == nil {
podSpec.Tolerations = []v1.Toleration{kubeadmconstants.MasterToleration}
return
}
podSpec.Tolerations = append(podSpec.Tolerations, kubeadmconstants.MasterToleration)
}
// setRightDNSPolicyOnPodSpec makes sure the self-hosted components can look up things via kube-dns if necessary
func setRightDNSPolicyOnPodSpec(podSpec *v1.PodSpec) {
podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
}

View File

@ -0,0 +1,185 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package selfhosting
import (
"reflect"
"testing"
"k8s.io/api/core/v1"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
)
func TestMutatePodSpec(t *testing.T) {
var tests = []struct {
component string
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
component: kubeAPIServer,
podSpec: &v1.PodSpec{},
expected: v1.PodSpec{
NodeSelector: map[string]string{
kubeadmconstants.LabelNodeRoleMaster: "",
},
Tolerations: []v1.Toleration{
kubeadmconstants.MasterToleration,
},
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
{
component: kubeControllerManager,
podSpec: &v1.PodSpec{},
expected: v1.PodSpec{
NodeSelector: map[string]string{
kubeadmconstants.LabelNodeRoleMaster: "",
},
Tolerations: []v1.Toleration{
kubeadmconstants.MasterToleration,
},
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
{
component: kubeScheduler,
podSpec: &v1.PodSpec{},
expected: v1.PodSpec{
NodeSelector: map[string]string{
kubeadmconstants.LabelNodeRoleMaster: "",
},
Tolerations: []v1.Toleration{
kubeadmconstants.MasterToleration,
},
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
}
for _, rt := range tests {
mutatePodSpec(rt.component, rt.podSpec)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed mutatePodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
}
}
func TestAddNodeSelectorToPodSpec(t *testing.T) {
var tests = []struct {
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
podSpec: &v1.PodSpec{},
expected: v1.PodSpec{
NodeSelector: map[string]string{
kubeadmconstants.LabelNodeRoleMaster: "",
},
},
},
{
podSpec: &v1.PodSpec{
NodeSelector: map[string]string{
"foo": "bar",
},
},
expected: v1.PodSpec{
NodeSelector: map[string]string{
"foo": "bar",
kubeadmconstants.LabelNodeRoleMaster: "",
},
},
},
}
for _, rt := range tests {
addNodeSelectorToPodSpec(rt.podSpec)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed addNodeSelectorToPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
}
}
func TestSetMasterTolerationOnPodSpec(t *testing.T) {
var tests = []struct {
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
podSpec: &v1.PodSpec{},
expected: v1.PodSpec{
Tolerations: []v1.Toleration{
kubeadmconstants.MasterToleration,
},
},
},
{
podSpec: &v1.PodSpec{
Tolerations: []v1.Toleration{
{Key: "foo", Value: "bar"},
},
},
expected: v1.PodSpec{
Tolerations: []v1.Toleration{
{Key: "foo", Value: "bar"},
kubeadmconstants.MasterToleration,
},
},
},
}
for _, rt := range tests {
setMasterTolerationOnPodSpec(rt.podSpec)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed setMasterTolerationOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
}
}
func TestSetRightDNSPolicyOnPodSpec(t *testing.T) {
var tests = []struct {
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
podSpec: &v1.PodSpec{},
expected: v1.PodSpec{
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
{
podSpec: &v1.PodSpec{
DNSPolicy: v1.DNSClusterFirst,
},
expected: v1.PodSpec{
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
}
for _, rt := range tests {
setRightDNSPolicyOnPodSpec(rt.podSpec)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed setRightDNSPolicyOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
}
}

View File

@ -0,0 +1,158 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package selfhosting
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"time"
"k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kuberuntime "k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/pkg/api"
)
const (
kubeAPIServer = "kube-apiserver"
kubeControllerManager = "kube-controller-manager"
kubeScheduler = "kube-scheduler"
selfHostingPrefix = "self-hosted-"
)
// CreateSelfHostedControlPlane is responsible for turning a Static Pod-hosted control plane to a self-hosted one
// It achieves that task this way:
// 1. Load the Static Pod specification from disk (from /etc/kubernetes/manifests)
// 2. Extract the PodSpec from that Static Pod specification
// 3. Mutate the PodSpec to be compatible with self-hosting (add the right labels, taints, etc. so it can schedule correctly)
// 4. Build a new DaemonSet object for the self-hosted component in question. Use the above mentioned PodSpec
// 5. Create the DaemonSet resource. Wait until the Pods are running.
// 6. Remove the Static Pod manifest file. The kubelet will stop the original Static Pod-hosted component that was running.
// 7. The self-hosted containers should now step up and take over.
// 8. In order to avoid race conditions, we're still making sure the API /healthz endpoint is healthy
// 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop
func CreateSelfHostedControlPlane(client *clientset.Clientset) error {
// The sequence here isn't set in stone, but seems to work well to start with the API server
components := []string{kubeAPIServer, kubeControllerManager, kubeScheduler}
for _, componentName := range components {
start := time.Now()
manifestPath := buildStaticManifestFilepath(componentName)
// Load the Static Pod file in order to be able to create a self-hosted variant of that file
podSpec, err := loadPodSpecFromFile(manifestPath)
if err != nil {
return err
}
// Build a DaemonSet object from the loaded PodSpec
ds := buildDaemonSet(componentName, podSpec)
// Create the DaemonSet in the API Server
if _, err := client.ExtensionsV1beta1().DaemonSets(metav1.NamespaceSystem).Create(ds); err != nil {
if !apierrors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create self-hosted %q daemonset [%v]", componentName, err)
}
if _, err := client.ExtensionsV1beta1().DaemonSets(metav1.NamespaceSystem).Update(ds); err != nil {
// TODO: We should retry on 409 responses
return fmt.Errorf("failed to update self-hosted %q daemonset [%v]", componentName, err)
}
}
// Wait for the self-hosted component to come up
kubeadmutil.WaitForPodsWithLabel(client, buildSelfHostedWorkloadLabelQuery(componentName))
// Remove the old Static Pod manifest
if err := os.RemoveAll(manifestPath); err != nil {
return fmt.Errorf("unable to delete static pod manifest for %s [%v]", componentName, err)
}
// Make sure the API is responsive at /healthz
kubeadmutil.WaitForAPI(client)
fmt.Printf("[self-hosted] self-hosted %s ready after %f seconds\n", componentName, time.Since(start).Seconds())
}
return nil
}
// buildDaemonSet is responsible for mutating the PodSpec and return a DaemonSet which is suitable for the self-hosting purporse
func buildDaemonSet(name string, podSpec *v1.PodSpec) *extensions.DaemonSet {
// Mutate the PodSpec so it's suitable for self-hosting
mutatePodSpec(name, podSpec)
// Return a DaemonSet based on that Spec
return &extensions.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Name: addSelfHostedPrefix(name),
Namespace: metav1.NamespaceSystem,
Labels: map[string]string{
"k8s-app": addSelfHostedPrefix(name),
},
},
Spec: extensions.DaemonSetSpec{
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"k8s-app": addSelfHostedPrefix(name),
},
},
Spec: *podSpec,
},
},
}
}
// loadPodSpecFromFile reads and decodes a file containing a specification of a Pod
// TODO: Consider using "k8s.io/kubernetes/pkg/volume/util".LoadPodFromFile(filename string) in the future instead.
func loadPodSpecFromFile(manifestPath string) (*v1.PodSpec, error) {
podBytes, err := ioutil.ReadFile(manifestPath)
if err != nil {
return nil, err
}
staticPod := &v1.Pod{}
if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), podBytes, staticPod); err != nil {
return nil, fmt.Errorf("unable to decode static pod %v", err)
}
return &staticPod.Spec, nil
}
// buildStaticManifestFilepath returns the location on the disk where the Static Pod should be present
func buildStaticManifestFilepath(componentName string) string {
return filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, componentName+".yaml")
}
// buildSelfHostedWorkloadLabelQuery creates the right query for matching a self-hosted Pod
func buildSelfHostedWorkloadLabelQuery(componentName string) string {
return fmt.Sprintf("k8s-app=%s", addSelfHostedPrefix(componentName))
}
// addSelfHostedPrefix adds the self-hosted- prefix to the component name
func addSelfHostedPrefix(componentName string) string {
return fmt.Sprintf("%s%s", selfHostingPrefix, componentName)
}

View File

@ -0,0 +1,544 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package selfhosting
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"testing"
"github.com/ghodss/yaml"
)
const (
testAPIServerPod = `
apiVersion: v1
kind: Pod
metadata:
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ""
creationTimestamp: null
labels:
component: kube-apiserver
tier: control-plane
name: kube-apiserver
namespace: kube-system
spec:
containers:
- command:
- kube-apiserver
- --client-ca-file=/etc/kubernetes/pki/ca.crt
- --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
- --allow-privileged=true
- --service-cluster-ip-range=10.96.0.0/12
- --service-account-key-file=/etc/kubernetes/pki/sa.pub
- --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
- --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
- --secure-port=6443
- --insecure-port=0
- --admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota
- --requestheader-extra-headers-prefix=X-Remote-Extra-
- --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
- --experimental-bootstrap-token-auth=true
- --requestheader-group-headers=X-Remote-Group
- --requestheader-allowed-names=front-proxy-client
- --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
- --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
- --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --requestheader-username-headers=X-Remote-User
- --authorization-mode=Node,RBAC
- --advertise-address=192.168.200.101
- --etcd-servers=http://127.0.0.1:2379
image: gcr.io/google_containers/kube-apiserver-amd64:v1.7.0
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 6443
scheme: HTTPS
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-apiserver
resources:
requests:
cpu: 250m
volumeMounts:
- mountPath: /etc/kubernetes
name: k8s
readOnly: true
- mountPath: /etc/ssl/certs
name: certs
- mountPath: /etc/pki
name: pki
hostNetwork: true
volumes:
- hostPath:
path: /etc/kubernetes
name: k8s
- hostPath:
path: /etc/ssl/certs
name: certs
- hostPath:
path: /etc/pki
name: pki
status: {}
`
testAPIServerDaemonSet = `metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-apiserver
name: self-hosted-kube-apiserver
namespace: kube-system
spec:
template:
metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-apiserver
spec:
containers:
- command:
- kube-apiserver
- --client-ca-file=/etc/kubernetes/pki/ca.crt
- --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
- --allow-privileged=true
- --service-cluster-ip-range=10.96.0.0/12
- --service-account-key-file=/etc/kubernetes/pki/sa.pub
- --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
- --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
- --secure-port=6443
- --insecure-port=0
- --admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota
- --requestheader-extra-headers-prefix=X-Remote-Extra-
- --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
- --experimental-bootstrap-token-auth=true
- --requestheader-group-headers=X-Remote-Group
- --requestheader-allowed-names=front-proxy-client
- --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
- --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
- --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --requestheader-username-headers=X-Remote-User
- --authorization-mode=Node,RBAC
- --advertise-address=192.168.200.101
- --etcd-servers=http://127.0.0.1:2379
image: gcr.io/google_containers/kube-apiserver-amd64:v1.7.0
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 6443
scheme: HTTPS
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-apiserver
resources:
requests:
cpu: 250m
volumeMounts:
- mountPath: /etc/kubernetes
name: k8s
readOnly: true
- mountPath: /etc/ssl/certs
name: certs
- mountPath: /etc/pki
name: pki
dnsPolicy: ClusterFirstWithHostNet
hostNetwork: true
nodeSelector:
node-role.kubernetes.io/master: ""
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
volumes:
- hostPath:
path: /etc/kubernetes
name: k8s
- hostPath:
path: /etc/ssl/certs
name: certs
- hostPath:
path: /etc/pki
name: pki
updateStrategy: {}
status:
currentNumberScheduled: 0
desiredNumberScheduled: 0
numberMisscheduled: 0
numberReady: 0
`
testControllerManagerPod = `
apiVersion: v1
kind: Pod
metadata:
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ""
creationTimestamp: null
labels:
component: kube-controller-manager
tier: control-plane
name: kube-controller-manager
namespace: kube-system
spec:
containers:
- command:
- kube-controller-manager
- --service-account-private-key-file=/etc/kubernetes/pki/sa.key
- --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
- --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
- --leader-elect=true
- --kubeconfig=/etc/kubernetes/controller-manager.conf
- --controllers=*,bootstrapsigner,tokencleaner
- --root-ca-file=/etc/kubernetes/pki/ca.crt
- --address=127.0.0.1
- --use-service-account-credentials=true
image: gcr.io/google_containers/kube-controller-manager-amd64:v1.7.0
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 10252
scheme: HTTP
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-controller-manager
resources:
requests:
cpu: 200m
volumeMounts:
- mountPath: /etc/kubernetes
name: k8s
readOnly: true
- mountPath: /etc/ssl/certs
name: certs
- mountPath: /etc/pki
name: pki
hostNetwork: true
volumes:
- hostPath:
path: /etc/kubernetes
name: k8s
- hostPath:
path: /etc/ssl/certs
name: certs
- hostPath:
path: /etc/pki
name: pki
status: {}
`
testControllerManagerDaemonSet = `metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-controller-manager
name: self-hosted-kube-controller-manager
namespace: kube-system
spec:
template:
metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-controller-manager
spec:
containers:
- command:
- kube-controller-manager
- --service-account-private-key-file=/etc/kubernetes/pki/sa.key
- --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
- --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
- --leader-elect=true
- --kubeconfig=/etc/kubernetes/controller-manager.conf
- --controllers=*,bootstrapsigner,tokencleaner
- --root-ca-file=/etc/kubernetes/pki/ca.crt
- --address=127.0.0.1
- --use-service-account-credentials=true
image: gcr.io/google_containers/kube-controller-manager-amd64:v1.7.0
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 10252
scheme: HTTP
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-controller-manager
resources:
requests:
cpu: 200m
volumeMounts:
- mountPath: /etc/kubernetes
name: k8s
readOnly: true
- mountPath: /etc/ssl/certs
name: certs
- mountPath: /etc/pki
name: pki
dnsPolicy: ClusterFirstWithHostNet
hostNetwork: true
nodeSelector:
node-role.kubernetes.io/master: ""
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
volumes:
- hostPath:
path: /etc/kubernetes
name: k8s
- hostPath:
path: /etc/ssl/certs
name: certs
- hostPath:
path: /etc/pki
name: pki
updateStrategy: {}
status:
currentNumberScheduled: 0
desiredNumberScheduled: 0
numberMisscheduled: 0
numberReady: 0
`
testSchedulerPod = `
apiVersion: v1
kind: Pod
metadata:
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ""
creationTimestamp: null
labels:
component: kube-scheduler
tier: control-plane
name: kube-scheduler
namespace: kube-system
spec:
containers:
- command:
- kube-scheduler
- --address=127.0.0.1
- --leader-elect=true
- --kubeconfig=/etc/kubernetes/scheduler.conf
image: gcr.io/google_containers/kube-scheduler-amd64:v1.7.0
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 10251
scheme: HTTP
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-scheduler
resources:
requests:
cpu: 100m
volumeMounts:
- mountPath: /etc/kubernetes
name: k8s
readOnly: true
hostNetwork: true
volumes:
- hostPath:
path: /etc/kubernetes
name: k8s
status: {}
`
testSchedulerDaemonSet = `metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-scheduler
name: self-hosted-kube-scheduler
namespace: kube-system
spec:
template:
metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-scheduler
spec:
containers:
- command:
- kube-scheduler
- --address=127.0.0.1
- --leader-elect=true
- --kubeconfig=/etc/kubernetes/scheduler.conf
image: gcr.io/google_containers/kube-scheduler-amd64:v1.7.0
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 10251
scheme: HTTP
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-scheduler
resources:
requests:
cpu: 100m
volumeMounts:
- mountPath: /etc/kubernetes
name: k8s
readOnly: true
dnsPolicy: ClusterFirstWithHostNet
hostNetwork: true
nodeSelector:
node-role.kubernetes.io/master: ""
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
volumes:
- hostPath:
path: /etc/kubernetes
name: k8s
updateStrategy: {}
status:
currentNumberScheduled: 0
desiredNumberScheduled: 0
numberMisscheduled: 0
numberReady: 0
`
)
func TestBuildDaemonSet(t *testing.T) {
var tests = []struct {
component string
podBytes []byte
dsBytes []byte
}{
{
component: kubeAPIServer,
podBytes: []byte(testAPIServerPod),
dsBytes: []byte(testAPIServerDaemonSet),
},
{
component: kubeControllerManager,
podBytes: []byte(testControllerManagerPod),
dsBytes: []byte(testControllerManagerDaemonSet),
},
{
component: kubeScheduler,
podBytes: []byte(testSchedulerPod),
dsBytes: []byte(testSchedulerDaemonSet),
},
}
for _, rt := range tests {
tempFile, err := createTempFileWithContent(rt.podBytes)
defer os.Remove(tempFile)
podSpec, err := loadPodSpecFromFile(tempFile)
if err != nil {
t.Fatalf("couldn't load the specified Pod")
}
ds := buildDaemonSet(rt.component, podSpec)
dsBytes, err := yaml.Marshal(ds)
if err != nil {
t.Fatalf("failed to marshal daemonset to YAML: %v", err)
}
if !bytes.Equal(dsBytes, rt.dsBytes) {
t.Errorf("failed TestBuildDaemonSet:\nexpected:\n%s\nsaw:\n%s", rt.dsBytes, dsBytes)
}
}
}
func TestLoadPodSpecFromFile(t *testing.T) {
tests := []struct {
content string
expectError bool
}{
{
// Good YAML
content: `
apiVersion: v1
kind: Pod
metadata:
name: testpod
spec:
containers:
- image: gcr.io/google_containers/busybox
`,
expectError: false,
},
{
// Good JSON
content: `
{
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": "testpod"
},
"spec": {
"containers": [
{
"image": "gcr.io/google_containers/busybox"
}
]
}
}`,
expectError: false,
},
{
// Bad PodSpec
content: `
apiVersion: v1
kind: Pod
metadata:
name: testpod
spec:
- image: gcr.io/google_containers/busybox
`,
expectError: true,
},
}
for _, rt := range tests {
tempFile, err := createTempFileWithContent([]byte(rt.content))
defer os.Remove(tempFile)
_, err = loadPodSpecFromFile(tempFile)
if (err != nil) != rt.expectError {
t.Errorf("failed TestLoadPodSpecFromFile:\nexpected error:\n%t\nsaw:\n%v", rt.expectError, err)
}
}
}
func createTempFileWithContent(content []byte) (string, error) {
tempFile, err := ioutil.TempFile("", "")
if err != nil {
return "", fmt.Errorf("cannot create temporary file: %v", err)
}
if _, err = tempFile.Write([]byte(content)); err != nil {
return "", fmt.Errorf("cannot save temporary file: %v", err)
}
if err = tempFile.Close(); err != nil {
return "", fmt.Errorf("cannot close temporary file: %v", err)
}
return tempFile.Name(), nil
}

View File

@ -21,6 +21,8 @@ go_library(
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/preflight:go_default_library",
"//cmd/kubeadm/app/util/kubeconfig:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",

View File

@ -21,12 +21,15 @@ import (
"net/http"
"time"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
)
// CreateClientAndWaitForAPI takes a path to a kubeconfig file, makes a client of it and waits for the API to be healthy
func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) {
client, err := kubeconfigutil.ClientSetFromFile(file)
if err != nil {
@ -39,6 +42,7 @@ func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) {
return client, nil
}
// WaitForAPI waits for the API Server's /healthz endpoint to report "ok"
func WaitForAPI(client *clientset.Clientset) {
start := time.Now()
wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
@ -52,3 +56,30 @@ func WaitForAPI(client *clientset.Clientset) {
return true, nil
})
}
// WaitForPodsWithLabel will lookup pods with the given label and wait until they are all
// reporting status as running.
func WaitForPodsWithLabel(client *clientset.Clientset, labelKeyValPair string) {
// TODO: Implement a timeout
// TODO: Implement a verbosity switch
wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
listOpts := metav1.ListOptions{LabelSelector: labelKeyValPair}
apiPods, err := client.CoreV1().Pods(metav1.NamespaceSystem).List(listOpts)
if err != nil {
fmt.Printf("[apiclient] Error getting Pods with label selector %q [%v]\n", labelKeyValPair, err)
return false, nil
}
if len(apiPods.Items) == 0 {
return false, nil
}
for _, pod := range apiPods.Items {
fmt.Printf("[apiclient] Pod %s status: %s\n", pod.Name, pod.Status.Phase)
if pod.Status.Phase != v1.PodRunning {
return false, nil
}
}
return true, nil
})
}

View File

@ -22,7 +22,9 @@ cmd/kubeadm/app/apis/kubeadm/install
cmd/kubeadm/app/discovery/https
cmd/kubeadm/app/phases/apiconfig
cmd/kubeadm/app/phases/certs
cmd/kubeadm/app/phases/controlplane
cmd/kubeadm/app/phases/kubeconfig
cmd/kubeadm/app/phases/selfhosting
cmd/kubectl
cmd/kubelet
cmd/libs/go2idl/client-gen