mirror of https://github.com/k3s-io/k3s
Merge pull request #47435 from luxas/kubeadm_new_selfhosting
Automatic merge from submit-queue (batch tested with PRs 47435, 46044) kubeadm: Make self-hosting work and split it out to a phase **What this PR does / why we need it**: - Removes the old self-hosting code - Puts the new self-hosting code in `phases/selfhosting` - The new code reads manifests from disk (static pods)... - ...mutates the PodSpec as necessary... - ...and posts the DaemonSet to the API Server... - ...and waits for it to come up - Uses DaemonSets for all control plane components - Creates a `kubeadm alpha phase selfhosting` command that can be invoked against any kubeadm-cluster after install. **Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes # fixes: https://github.com/kubernetes/kubeadm/issues/127 (large part of at least) **Special notes for your reviewer**: Please only review the fourth commit, based on https://github.com/kubernetes/kubernetes/pull/47345 **Release note**: ```release-note kubeadm: Make self-hosting work by using DaemonSets and split it out to a phase that can be invoked via the CLI ``` @kubernetes/sig-cluster-lifecycle-pr-reviews @jbedapull/6/head
commit
b00df7eb89
|
@ -41,6 +41,7 @@ filegroup(
|
|||
"//cmd/kubeadm/app/phases/certs:all-srcs",
|
||||
"//cmd/kubeadm/app/phases/controlplane:all-srcs",
|
||||
"//cmd/kubeadm/app/phases/kubeconfig:all-srcs",
|
||||
"//cmd/kubeadm/app/phases/selfhosting:all-srcs",
|
||||
"//cmd/kubeadm/app/phases/token:all-srcs",
|
||||
"//cmd/kubeadm/app/preflight:all-srcs",
|
||||
"//cmd/kubeadm/app/util:all-srcs",
|
||||
|
|
|
@ -34,6 +34,7 @@ go_library(
|
|||
"//cmd/kubeadm/app/phases/certs:go_default_library",
|
||||
"//cmd/kubeadm/app/phases/controlplane:go_default_library",
|
||||
"//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
|
||||
"//cmd/kubeadm/app/phases/selfhosting:go_default_library",
|
||||
"//cmd/kubeadm/app/phases/token:go_default_library",
|
||||
"//cmd/kubeadm/app/preflight:go_default_library",
|
||||
"//cmd/kubeadm/app/util:go_default_library",
|
||||
|
|
|
@ -37,6 +37,7 @@ import (
|
|||
certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
|
||||
controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
|
||||
kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
|
||||
selfhostingphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting"
|
||||
tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/token"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
|
||||
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
|
||||
|
@ -132,6 +133,10 @@ func NewCmdInit(out io.Writer) *cobra.Command {
|
|||
&skipTokenPrint, "skip-token-print", skipTokenPrint,
|
||||
"Skip printing of the default bootstrap token generated by 'kubeadm init'",
|
||||
)
|
||||
cmd.PersistentFlags().BoolVar(
|
||||
&cfg.SelfHosted, "self-hosted", cfg.SelfHosted,
|
||||
"[experimental] If kubeadm should make this control plane self-hosted",
|
||||
)
|
||||
|
||||
cmd.PersistentFlags().StringVar(
|
||||
&cfg.Token, "token", cfg.Token,
|
||||
|
@ -266,7 +271,7 @@ func (i *Init) Run(out io.Writer) error {
|
|||
// Temporary control plane is up, now we create our self hosted control
|
||||
// plane components and remove the static manifests:
|
||||
fmt.Println("[self-hosted] Creating self-hosted control plane...")
|
||||
if err := controlplanephase.CreateSelfHostedControlPlane(i.cfg, client); err != nil {
|
||||
if err := selfhostingphase.CreateSelfHostedControlPlane(client); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
@ -14,6 +14,7 @@ go_library(
|
|||
"kubeconfig.go",
|
||||
"phase.go",
|
||||
"preflight.go",
|
||||
"selfhosting.go",
|
||||
],
|
||||
tags = ["automanaged"],
|
||||
deps = [
|
||||
|
@ -22,8 +23,10 @@ go_library(
|
|||
"//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library",
|
||||
"//cmd/kubeadm/app/phases/certs:go_default_library",
|
||||
"//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
|
||||
"//cmd/kubeadm/app/phases/selfhosting:go_default_library",
|
||||
"//cmd/kubeadm/app/preflight:go_default_library",
|
||||
"//cmd/kubeadm/app/util:go_default_library",
|
||||
"//cmd/kubeadm/app/util/kubeconfig:go_default_library",
|
||||
"//pkg/api:go_default_library",
|
||||
"//vendor/github.com/spf13/cobra:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
|
||||
|
|
|
@ -33,6 +33,7 @@ func NewCmdPhase(out io.Writer) *cobra.Command {
|
|||
cmd.AddCommand(NewCmdKubeConfig(out))
|
||||
cmd.AddCommand(NewCmdCerts())
|
||||
cmd.AddCommand(NewCmdPreFlight())
|
||||
cmd.AddCommand(NewCmdSelfhosting())
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
|
|
@ -0,0 +1,45 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package phases
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting"
|
||||
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
|
||||
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
|
||||
)
|
||||
|
||||
// NewCmdSelfhosting returns the self-hosting Cobra command
|
||||
func NewCmdSelfhosting() *cobra.Command {
|
||||
var kubeConfigFile string
|
||||
cmd := &cobra.Command{
|
||||
Use: "selfhosting",
|
||||
Aliases: []string{"selfhosted"},
|
||||
Short: "Make a kubeadm cluster self-hosted.",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile)
|
||||
kubeadmutil.CheckErr(err)
|
||||
|
||||
err = selfhosting.CreateSelfHostedControlPlane(client)
|
||||
kubeadmutil.CheckErr(err)
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use for talking to the cluster")
|
||||
return cmd
|
||||
}
|
|
@ -109,3 +109,8 @@ var (
|
|||
// MinimumControlPlaneVersion specifies the minimum control plane version kubeadm can deploy
|
||||
MinimumControlPlaneVersion = version.MustParseSemantic("v1.7.0")
|
||||
)
|
||||
|
||||
// BuildStaticManifestFilepath returns the location on the disk where the Static Pod should be present
|
||||
func BuildStaticManifestFilepath(componentName string) string {
|
||||
return filepath.Join(KubernetesDir, ManifestsSubDirName, componentName+".yaml")
|
||||
}
|
||||
|
|
|
@ -25,30 +25,23 @@ go_test(
|
|||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"manifests.go",
|
||||
"selfhosted.go",
|
||||
],
|
||||
srcs = ["manifests.go"],
|
||||
tags = ["automanaged"],
|
||||
deps = [
|
||||
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
|
||||
"//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library",
|
||||
"//cmd/kubeadm/app/constants:go_default_library",
|
||||
"//cmd/kubeadm/app/images:go_default_library",
|
||||
"//cmd/kubeadm/app/util:go_default_library",
|
||||
"//pkg/kubeapiserver/authorizer/modes:go_default_library",
|
||||
"//pkg/kubectl/cmd/util:go_default_library",
|
||||
"//pkg/kubelet/types:go_default_library",
|
||||
"//pkg/util/version:go_default_library",
|
||||
"//vendor/github.com/ghodss/yaml:go_default_library",
|
||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||
"//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
|
||||
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
|
|
|
@ -229,23 +229,6 @@ func pkiVolumeMount() v1.VolumeMount {
|
|||
}
|
||||
}
|
||||
|
||||
func flockVolume() v1.Volume {
|
||||
return v1.Volume{
|
||||
Name: "var-lock",
|
||||
VolumeSource: v1.VolumeSource{
|
||||
HostPath: &v1.HostPathVolumeSource{Path: "/var/lock"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func flockVolumeMount() v1.VolumeMount {
|
||||
return v1.VolumeMount{
|
||||
Name: "var-lock",
|
||||
MountPath: "/var/lock",
|
||||
ReadOnly: false,
|
||||
}
|
||||
}
|
||||
|
||||
func k8sVolume() v1.Volume {
|
||||
return v1.Volume{
|
||||
Name: "k8s",
|
||||
|
@ -447,19 +430,6 @@ func getProxyEnvVars() []v1.EnvVar {
|
|||
return envs
|
||||
}
|
||||
|
||||
func getSelfHostedAPIServerEnv() []v1.EnvVar {
|
||||
podIPEnvVar := v1.EnvVar{
|
||||
Name: "POD_IP",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
FieldRef: &v1.ObjectFieldSelector{
|
||||
FieldPath: "status.podIP",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return append(getProxyEnvVars(), podIPEnvVar)
|
||||
}
|
||||
|
||||
// getAuthzParameters gets the authorization-related parameters to the api server
|
||||
// At this point, we can assume the list of authorization modes is valid (due to that it has been validated in the API machinery code already)
|
||||
// If the list is empty; it's defaulted (mostly for unit testing)
|
||||
|
|
|
@ -1,348 +0,0 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package controlplane
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
extensions "k8s.io/api/extensions/v1beta1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/images"
|
||||
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
|
||||
"k8s.io/kubernetes/pkg/util/version"
|
||||
)
|
||||
|
||||
var (
|
||||
// maximum unavailable and surge instances per self-hosted component deployment
|
||||
maxUnavailable = intstr.FromInt(0)
|
||||
maxSurge = intstr.FromInt(1)
|
||||
)
|
||||
|
||||
func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error {
|
||||
volumes := []v1.Volume{k8sVolume()}
|
||||
volumeMounts := []v1.VolumeMount{k8sVolumeMount()}
|
||||
if isCertsVolumeMountNeeded() {
|
||||
volumes = append(volumes, certsVolume(cfg))
|
||||
volumeMounts = append(volumeMounts, certsVolumeMount())
|
||||
}
|
||||
|
||||
if isPkiVolumeMountNeeded() {
|
||||
volumes = append(volumes, pkiVolume())
|
||||
volumeMounts = append(volumeMounts, pkiVolumeMount())
|
||||
}
|
||||
|
||||
// Need lock for self-hosted
|
||||
volumes = append(volumes, flockVolume())
|
||||
volumeMounts = append(volumeMounts, flockVolumeMount())
|
||||
|
||||
if err := launchSelfHostedAPIServer(cfg, client, volumes, volumeMounts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := launchSelfHostedScheduler(cfg, client, volumes, volumeMounts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := launchSelfHostedControllerManager(cfg, client, volumes, volumeMounts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func launchSelfHostedAPIServer(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error {
|
||||
start := time.Now()
|
||||
|
||||
kubeVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
apiServer := getAPIServerDS(cfg, volumes, volumeMounts, kubeVersion)
|
||||
if _, err := client.Extensions().DaemonSets(metav1.NamespaceSystem).Create(&apiServer); err != nil {
|
||||
return fmt.Errorf("failed to create self-hosted %q daemon set [%v]", kubeAPIServer, err)
|
||||
}
|
||||
|
||||
wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
|
||||
// TODO: This might be pointless, checking the pods is probably enough.
|
||||
// It does however get us a count of how many there should be which may be useful
|
||||
// with HA.
|
||||
apiDS, err := client.DaemonSets(metav1.NamespaceSystem).Get("self-hosted-"+kubeAPIServer,
|
||||
metav1.GetOptions{})
|
||||
if err != nil {
|
||||
fmt.Println("[self-hosted] error getting apiserver DaemonSet:", err)
|
||||
return false, nil
|
||||
}
|
||||
fmt.Printf("[self-hosted] %s DaemonSet current=%d, desired=%d\n",
|
||||
kubeAPIServer,
|
||||
apiDS.Status.CurrentNumberScheduled,
|
||||
apiDS.Status.DesiredNumberScheduled)
|
||||
|
||||
if apiDS.Status.CurrentNumberScheduled != apiDS.Status.DesiredNumberScheduled {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
|
||||
// Wait for self-hosted API server to take ownership
|
||||
waitForPodsWithLabel(client, "self-hosted-"+kubeAPIServer, true)
|
||||
|
||||
// Remove temporary API server
|
||||
apiServerStaticManifestPath := buildStaticManifestFilepath(kubeAPIServer)
|
||||
if err := os.RemoveAll(apiServerStaticManifestPath); err != nil {
|
||||
return fmt.Errorf("unable to delete temporary API server manifest [%v]", err)
|
||||
}
|
||||
|
||||
kubeadmutil.WaitForAPI(client)
|
||||
|
||||
fmt.Printf("[self-hosted] self-hosted kube-apiserver ready after %f seconds\n", time.Since(start).Seconds())
|
||||
return nil
|
||||
}
|
||||
|
||||
func launchSelfHostedControllerManager(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error {
|
||||
start := time.Now()
|
||||
|
||||
kubeVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctrlMgr := getControllerManagerDeployment(cfg, volumes, volumeMounts, kubeVersion)
|
||||
if _, err := client.Extensions().Deployments(metav1.NamespaceSystem).Create(&ctrlMgr); err != nil {
|
||||
return fmt.Errorf("failed to create self-hosted %q deployment [%v]", kubeControllerManager, err)
|
||||
}
|
||||
|
||||
waitForPodsWithLabel(client, "self-hosted-"+kubeControllerManager, true)
|
||||
|
||||
ctrlMgrStaticManifestPath := buildStaticManifestFilepath(kubeControllerManager)
|
||||
if err := os.RemoveAll(ctrlMgrStaticManifestPath); err != nil {
|
||||
return fmt.Errorf("unable to delete temporary controller manager manifest [%v]", err)
|
||||
}
|
||||
|
||||
fmt.Printf("[self-hosted] self-hosted kube-controller-manager ready after %f seconds\n", time.Since(start).Seconds())
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func launchSelfHostedScheduler(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error {
|
||||
start := time.Now()
|
||||
scheduler := getSchedulerDeployment(cfg, volumes, volumeMounts)
|
||||
if _, err := client.Extensions().Deployments(metav1.NamespaceSystem).Create(&scheduler); err != nil {
|
||||
return fmt.Errorf("failed to create self-hosted %q deployment [%v]", kubeScheduler, err)
|
||||
}
|
||||
|
||||
waitForPodsWithLabel(client, "self-hosted-"+kubeScheduler, true)
|
||||
|
||||
schedulerStaticManifestPath := buildStaticManifestFilepath(kubeScheduler)
|
||||
if err := os.RemoveAll(schedulerStaticManifestPath); err != nil {
|
||||
return fmt.Errorf("unable to delete temporary scheduler manifest [%v]", err)
|
||||
}
|
||||
|
||||
fmt.Printf("[self-hosted] self-hosted kube-scheduler ready after %f seconds\n", time.Since(start).Seconds())
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForPodsWithLabel will lookup pods with the given label and wait until they are all
|
||||
// reporting status as running.
|
||||
func waitForPodsWithLabel(client *clientset.Clientset, appLabel string, mustBeRunning bool) {
|
||||
wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
|
||||
// TODO: Do we need a stronger label link than this?
|
||||
listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("k8s-app=%s", appLabel)}
|
||||
apiPods, err := client.Pods(metav1.NamespaceSystem).List(listOpts)
|
||||
if err != nil {
|
||||
fmt.Printf("[self-hosted] error getting %s pods [%v]\n", appLabel, err)
|
||||
return false, nil
|
||||
}
|
||||
fmt.Printf("[self-hosted] Found %d %s pods\n", len(apiPods.Items), appLabel)
|
||||
|
||||
// TODO: HA
|
||||
if int32(len(apiPods.Items)) != 1 {
|
||||
return false, nil
|
||||
}
|
||||
for _, pod := range apiPods.Items {
|
||||
fmt.Printf("[self-hosted] Pod %s status: %s\n", pod.Name, pod.Status.Phase)
|
||||
if mustBeRunning && pod.Status.Phase != "Running" {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
}
|
||||
|
||||
// Sources from bootkube templates.go
|
||||
func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount, kubeVersion *version.Version) extensions.DaemonSet {
|
||||
ds := extensions.DaemonSet{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "extensions/v1beta1",
|
||||
Kind: "DaemonSet",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "self-hosted-" + kubeAPIServer,
|
||||
Namespace: "kube-system",
|
||||
Labels: map[string]string{"k8s-app": "self-hosted-" + kubeAPIServer},
|
||||
},
|
||||
Spec: extensions.DaemonSetSpec{
|
||||
Template: v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"k8s-app": "self-hosted-" + kubeAPIServer,
|
||||
"component": kubeAPIServer,
|
||||
"tier": "control-plane",
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
NodeSelector: map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""},
|
||||
HostNetwork: true,
|
||||
Volumes: volumes,
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: "self-hosted-" + kubeAPIServer,
|
||||
Image: images.GetCoreImage(images.KubeAPIServerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
|
||||
Command: getAPIServerCommand(cfg, true, kubeVersion),
|
||||
Env: getSelfHostedAPIServerEnv(),
|
||||
VolumeMounts: volumeMounts,
|
||||
LivenessProbe: componentProbe(6443, "/healthz", v1.URISchemeHTTPS),
|
||||
Resources: componentResources("250m"),
|
||||
},
|
||||
},
|
||||
Tolerations: []v1.Toleration{kubeadmconstants.MasterToleration},
|
||||
DNSPolicy: v1.DNSClusterFirstWithHostNet,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
return ds
|
||||
}
|
||||
|
||||
func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount, kubeVersion *version.Version) extensions.Deployment {
|
||||
d := extensions.Deployment{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "extensions/v1beta1",
|
||||
Kind: "Deployment",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "self-hosted-" + kubeControllerManager,
|
||||
Namespace: "kube-system",
|
||||
Labels: map[string]string{"k8s-app": "self-hosted-" + kubeControllerManager},
|
||||
},
|
||||
Spec: extensions.DeploymentSpec{
|
||||
// TODO bootkube uses 2 replicas
|
||||
Strategy: extensions.DeploymentStrategy{
|
||||
Type: extensions.RollingUpdateDeploymentStrategyType,
|
||||
RollingUpdate: &extensions.RollingUpdateDeployment{
|
||||
MaxUnavailable: &maxUnavailable,
|
||||
MaxSurge: &maxSurge,
|
||||
},
|
||||
},
|
||||
Template: v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"k8s-app": "self-hosted-" + kubeControllerManager,
|
||||
"component": kubeControllerManager,
|
||||
"tier": "control-plane",
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
NodeSelector: map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""},
|
||||
HostNetwork: true,
|
||||
Volumes: volumes,
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: "self-hosted-" + kubeControllerManager,
|
||||
Image: images.GetCoreImage(images.KubeControllerManagerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
|
||||
Command: getControllerManagerCommand(cfg, true, kubeVersion),
|
||||
VolumeMounts: volumeMounts,
|
||||
LivenessProbe: componentProbe(10252, "/healthz", v1.URISchemeHTTP),
|
||||
Resources: componentResources("200m"),
|
||||
Env: getProxyEnvVars(),
|
||||
},
|
||||
},
|
||||
Tolerations: []v1.Toleration{kubeadmconstants.MasterToleration},
|
||||
DNSPolicy: v1.DNSClusterFirstWithHostNet,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func getSchedulerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount) extensions.Deployment {
|
||||
d := extensions.Deployment{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "extensions/v1beta1",
|
||||
Kind: "Deployment",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "self-hosted-" + kubeScheduler,
|
||||
Namespace: "kube-system",
|
||||
Labels: map[string]string{"k8s-app": "self-hosted-" + kubeScheduler},
|
||||
},
|
||||
Spec: extensions.DeploymentSpec{
|
||||
// TODO bootkube uses 2 replicas
|
||||
Strategy: extensions.DeploymentStrategy{
|
||||
Type: extensions.RollingUpdateDeploymentStrategyType,
|
||||
RollingUpdate: &extensions.RollingUpdateDeployment{
|
||||
MaxUnavailable: &maxUnavailable,
|
||||
MaxSurge: &maxSurge,
|
||||
},
|
||||
},
|
||||
Template: v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"k8s-app": "self-hosted-" + kubeScheduler,
|
||||
"component": kubeScheduler,
|
||||
"tier": "control-plane",
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
NodeSelector: map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""},
|
||||
HostNetwork: true,
|
||||
Volumes: volumes,
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: "self-hosted-" + kubeScheduler,
|
||||
Image: images.GetCoreImage(images.KubeSchedulerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
|
||||
Command: getSchedulerCommand(cfg, true),
|
||||
VolumeMounts: volumeMounts,
|
||||
LivenessProbe: componentProbe(10251, "/healthz", v1.URISchemeHTTP),
|
||||
Resources: componentResources("100m"),
|
||||
Env: getProxyEnvVars(),
|
||||
},
|
||||
},
|
||||
Tolerations: []v1.Toleration{kubeadmconstants.MasterToleration},
|
||||
DNSPolicy: v1.DNSClusterFirstWithHostNet,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return d
|
||||
}
|
||||
|
||||
func buildStaticManifestFilepath(name string) string {
|
||||
return filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.ManifestsSubDirName, name+".yaml")
|
||||
}
|
|
@ -0,0 +1,57 @@
|
|||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
load(
|
||||
"@io_bazel_rules_go//go:def.bzl",
|
||||
"go_library",
|
||||
"go_test",
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = [
|
||||
"podspec_mutation_test.go",
|
||||
"selfhosting_test.go",
|
||||
],
|
||||
library = ":go_default_library",
|
||||
tags = ["automanaged"],
|
||||
deps = [
|
||||
"//cmd/kubeadm/app/constants:go_default_library",
|
||||
"//vendor/github.com/ghodss/yaml:go_default_library",
|
||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"podspec_mutation.go",
|
||||
"selfhosting.go",
|
||||
],
|
||||
tags = ["automanaged"],
|
||||
deps = [
|
||||
"//cmd/kubeadm/app/constants:go_default_library",
|
||||
"//cmd/kubeadm/app/util:go_default_library",
|
||||
"//pkg/api:go_default_library",
|
||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||
"//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
|
||||
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [":package-srcs"],
|
||||
tags = ["automanaged"],
|
||||
)
|
|
@ -0,0 +1,74 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package selfhosting
|
||||
|
||||
import (
|
||||
"k8s.io/api/core/v1"
|
||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
)
|
||||
|
||||
// mutatePodSpec makes a Static Pod-hosted PodSpec suitable for self-hosting
|
||||
func mutatePodSpec(name string, podSpec *v1.PodSpec) {
|
||||
mutators := map[string][]func(*v1.PodSpec){
|
||||
kubeAPIServer: {
|
||||
addNodeSelectorToPodSpec,
|
||||
setMasterTolerationOnPodSpec,
|
||||
setRightDNSPolicyOnPodSpec,
|
||||
},
|
||||
kubeControllerManager: {
|
||||
addNodeSelectorToPodSpec,
|
||||
setMasterTolerationOnPodSpec,
|
||||
setRightDNSPolicyOnPodSpec,
|
||||
},
|
||||
kubeScheduler: {
|
||||
addNodeSelectorToPodSpec,
|
||||
setMasterTolerationOnPodSpec,
|
||||
setRightDNSPolicyOnPodSpec,
|
||||
},
|
||||
}
|
||||
|
||||
// Get the mutator functions for the component in question, then loop through and execute them
|
||||
mutatorsForComponent := mutators[name]
|
||||
for _, mutateFunc := range mutatorsForComponent {
|
||||
mutateFunc(podSpec)
|
||||
}
|
||||
}
|
||||
|
||||
// addNodeSelectorToPodSpec makes Pod require to be scheduled on a node marked with the master label
|
||||
func addNodeSelectorToPodSpec(podSpec *v1.PodSpec) {
|
||||
if podSpec.NodeSelector == nil {
|
||||
podSpec.NodeSelector = map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""}
|
||||
return
|
||||
}
|
||||
|
||||
podSpec.NodeSelector[kubeadmconstants.LabelNodeRoleMaster] = ""
|
||||
}
|
||||
|
||||
// setMasterTolerationOnPodSpec makes the Pod tolerate the master taint
|
||||
func setMasterTolerationOnPodSpec(podSpec *v1.PodSpec) {
|
||||
if podSpec.Tolerations == nil {
|
||||
podSpec.Tolerations = []v1.Toleration{kubeadmconstants.MasterToleration}
|
||||
return
|
||||
}
|
||||
|
||||
podSpec.Tolerations = append(podSpec.Tolerations, kubeadmconstants.MasterToleration)
|
||||
}
|
||||
|
||||
// setRightDNSPolicyOnPodSpec makes sure the self-hosted components can look up things via kube-dns if necessary
|
||||
func setRightDNSPolicyOnPodSpec(podSpec *v1.PodSpec) {
|
||||
podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
}
|
|
@ -0,0 +1,185 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package selfhosting
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
)
|
||||
|
||||
func TestMutatePodSpec(t *testing.T) {
|
||||
var tests = []struct {
|
||||
component string
|
||||
podSpec *v1.PodSpec
|
||||
expected v1.PodSpec
|
||||
}{
|
||||
{
|
||||
component: kubeAPIServer,
|
||||
podSpec: &v1.PodSpec{},
|
||||
expected: v1.PodSpec{
|
||||
NodeSelector: map[string]string{
|
||||
kubeadmconstants.LabelNodeRoleMaster: "",
|
||||
},
|
||||
Tolerations: []v1.Toleration{
|
||||
kubeadmconstants.MasterToleration,
|
||||
},
|
||||
DNSPolicy: v1.DNSClusterFirstWithHostNet,
|
||||
},
|
||||
},
|
||||
{
|
||||
component: kubeControllerManager,
|
||||
podSpec: &v1.PodSpec{},
|
||||
expected: v1.PodSpec{
|
||||
NodeSelector: map[string]string{
|
||||
kubeadmconstants.LabelNodeRoleMaster: "",
|
||||
},
|
||||
Tolerations: []v1.Toleration{
|
||||
kubeadmconstants.MasterToleration,
|
||||
},
|
||||
DNSPolicy: v1.DNSClusterFirstWithHostNet,
|
||||
},
|
||||
},
|
||||
{
|
||||
component: kubeScheduler,
|
||||
podSpec: &v1.PodSpec{},
|
||||
expected: v1.PodSpec{
|
||||
NodeSelector: map[string]string{
|
||||
kubeadmconstants.LabelNodeRoleMaster: "",
|
||||
},
|
||||
Tolerations: []v1.Toleration{
|
||||
kubeadmconstants.MasterToleration,
|
||||
},
|
||||
DNSPolicy: v1.DNSClusterFirstWithHostNet,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, rt := range tests {
|
||||
mutatePodSpec(rt.component, rt.podSpec)
|
||||
|
||||
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
|
||||
t.Errorf("failed mutatePodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddNodeSelectorToPodSpec(t *testing.T) {
|
||||
var tests = []struct {
|
||||
podSpec *v1.PodSpec
|
||||
expected v1.PodSpec
|
||||
}{
|
||||
{
|
||||
podSpec: &v1.PodSpec{},
|
||||
expected: v1.PodSpec{
|
||||
NodeSelector: map[string]string{
|
||||
kubeadmconstants.LabelNodeRoleMaster: "",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
podSpec: &v1.PodSpec{
|
||||
NodeSelector: map[string]string{
|
||||
"foo": "bar",
|
||||
},
|
||||
},
|
||||
expected: v1.PodSpec{
|
||||
NodeSelector: map[string]string{
|
||||
"foo": "bar",
|
||||
kubeadmconstants.LabelNodeRoleMaster: "",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, rt := range tests {
|
||||
addNodeSelectorToPodSpec(rt.podSpec)
|
||||
|
||||
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
|
||||
t.Errorf("failed addNodeSelectorToPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetMasterTolerationOnPodSpec(t *testing.T) {
|
||||
var tests = []struct {
|
||||
podSpec *v1.PodSpec
|
||||
expected v1.PodSpec
|
||||
}{
|
||||
{
|
||||
podSpec: &v1.PodSpec{},
|
||||
expected: v1.PodSpec{
|
||||
Tolerations: []v1.Toleration{
|
||||
kubeadmconstants.MasterToleration,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
podSpec: &v1.PodSpec{
|
||||
Tolerations: []v1.Toleration{
|
||||
{Key: "foo", Value: "bar"},
|
||||
},
|
||||
},
|
||||
expected: v1.PodSpec{
|
||||
Tolerations: []v1.Toleration{
|
||||
{Key: "foo", Value: "bar"},
|
||||
kubeadmconstants.MasterToleration,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, rt := range tests {
|
||||
setMasterTolerationOnPodSpec(rt.podSpec)
|
||||
|
||||
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
|
||||
t.Errorf("failed setMasterTolerationOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetRightDNSPolicyOnPodSpec(t *testing.T) {
|
||||
var tests = []struct {
|
||||
podSpec *v1.PodSpec
|
||||
expected v1.PodSpec
|
||||
}{
|
||||
{
|
||||
podSpec: &v1.PodSpec{},
|
||||
expected: v1.PodSpec{
|
||||
DNSPolicy: v1.DNSClusterFirstWithHostNet,
|
||||
},
|
||||
},
|
||||
{
|
||||
podSpec: &v1.PodSpec{
|
||||
DNSPolicy: v1.DNSClusterFirst,
|
||||
},
|
||||
expected: v1.PodSpec{
|
||||
DNSPolicy: v1.DNSClusterFirstWithHostNet,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, rt := range tests {
|
||||
setRightDNSPolicyOnPodSpec(rt.podSpec)
|
||||
|
||||
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
|
||||
t.Errorf("failed setRightDNSPolicyOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,158 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package selfhosting
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
extensions "k8s.io/api/extensions/v1beta1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
kuberuntime "k8s.io/apimachinery/pkg/runtime"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
|
||||
"k8s.io/kubernetes/pkg/api"
|
||||
)
|
||||
|
||||
const (
|
||||
kubeAPIServer = "kube-apiserver"
|
||||
kubeControllerManager = "kube-controller-manager"
|
||||
kubeScheduler = "kube-scheduler"
|
||||
|
||||
selfHostingPrefix = "self-hosted-"
|
||||
)
|
||||
|
||||
// CreateSelfHostedControlPlane is responsible for turning a Static Pod-hosted control plane to a self-hosted one
|
||||
// It achieves that task this way:
|
||||
// 1. Load the Static Pod specification from disk (from /etc/kubernetes/manifests)
|
||||
// 2. Extract the PodSpec from that Static Pod specification
|
||||
// 3. Mutate the PodSpec to be compatible with self-hosting (add the right labels, taints, etc. so it can schedule correctly)
|
||||
// 4. Build a new DaemonSet object for the self-hosted component in question. Use the above mentioned PodSpec
|
||||
// 5. Create the DaemonSet resource. Wait until the Pods are running.
|
||||
// 6. Remove the Static Pod manifest file. The kubelet will stop the original Static Pod-hosted component that was running.
|
||||
// 7. The self-hosted containers should now step up and take over.
|
||||
// 8. In order to avoid race conditions, we're still making sure the API /healthz endpoint is healthy
|
||||
// 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop
|
||||
func CreateSelfHostedControlPlane(client *clientset.Clientset) error {
|
||||
|
||||
// The sequence here isn't set in stone, but seems to work well to start with the API server
|
||||
components := []string{kubeAPIServer, kubeControllerManager, kubeScheduler}
|
||||
|
||||
for _, componentName := range components {
|
||||
start := time.Now()
|
||||
manifestPath := buildStaticManifestFilepath(componentName)
|
||||
|
||||
// Load the Static Pod file in order to be able to create a self-hosted variant of that file
|
||||
podSpec, err := loadPodSpecFromFile(manifestPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Build a DaemonSet object from the loaded PodSpec
|
||||
ds := buildDaemonSet(componentName, podSpec)
|
||||
|
||||
// Create the DaemonSet in the API Server
|
||||
if _, err := client.ExtensionsV1beta1().DaemonSets(metav1.NamespaceSystem).Create(ds); err != nil {
|
||||
if !apierrors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create self-hosted %q daemonset [%v]", componentName, err)
|
||||
}
|
||||
|
||||
if _, err := client.ExtensionsV1beta1().DaemonSets(metav1.NamespaceSystem).Update(ds); err != nil {
|
||||
// TODO: We should retry on 409 responses
|
||||
return fmt.Errorf("failed to update self-hosted %q daemonset [%v]", componentName, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for the self-hosted component to come up
|
||||
kubeadmutil.WaitForPodsWithLabel(client, buildSelfHostedWorkloadLabelQuery(componentName))
|
||||
|
||||
// Remove the old Static Pod manifest
|
||||
if err := os.RemoveAll(manifestPath); err != nil {
|
||||
return fmt.Errorf("unable to delete static pod manifest for %s [%v]", componentName, err)
|
||||
}
|
||||
|
||||
// Make sure the API is responsive at /healthz
|
||||
kubeadmutil.WaitForAPI(client)
|
||||
|
||||
fmt.Printf("[self-hosted] self-hosted %s ready after %f seconds\n", componentName, time.Since(start).Seconds())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildDaemonSet is responsible for mutating the PodSpec and return a DaemonSet which is suitable for the self-hosting purporse
|
||||
func buildDaemonSet(name string, podSpec *v1.PodSpec) *extensions.DaemonSet {
|
||||
// Mutate the PodSpec so it's suitable for self-hosting
|
||||
mutatePodSpec(name, podSpec)
|
||||
|
||||
// Return a DaemonSet based on that Spec
|
||||
return &extensions.DaemonSet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: addSelfHostedPrefix(name),
|
||||
Namespace: metav1.NamespaceSystem,
|
||||
Labels: map[string]string{
|
||||
"k8s-app": addSelfHostedPrefix(name),
|
||||
},
|
||||
},
|
||||
Spec: extensions.DaemonSetSpec{
|
||||
Template: v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"k8s-app": addSelfHostedPrefix(name),
|
||||
},
|
||||
},
|
||||
Spec: *podSpec,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// loadPodSpecFromFile reads and decodes a file containing a specification of a Pod
|
||||
// TODO: Consider using "k8s.io/kubernetes/pkg/volume/util".LoadPodFromFile(filename string) in the future instead.
|
||||
func loadPodSpecFromFile(manifestPath string) (*v1.PodSpec, error) {
|
||||
podBytes, err := ioutil.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
staticPod := &v1.Pod{}
|
||||
if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), podBytes, staticPod); err != nil {
|
||||
return nil, fmt.Errorf("unable to decode static pod %v", err)
|
||||
}
|
||||
|
||||
return &staticPod.Spec, nil
|
||||
}
|
||||
|
||||
// buildStaticManifestFilepath returns the location on the disk where the Static Pod should be present
|
||||
func buildStaticManifestFilepath(componentName string) string {
|
||||
return filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, componentName+".yaml")
|
||||
}
|
||||
|
||||
// buildSelfHostedWorkloadLabelQuery creates the right query for matching a self-hosted Pod
|
||||
func buildSelfHostedWorkloadLabelQuery(componentName string) string {
|
||||
return fmt.Sprintf("k8s-app=%s", addSelfHostedPrefix(componentName))
|
||||
}
|
||||
|
||||
// addSelfHostedPrefix adds the self-hosted- prefix to the component name
|
||||
func addSelfHostedPrefix(componentName string) string {
|
||||
return fmt.Sprintf("%s%s", selfHostingPrefix, componentName)
|
||||
}
|
|
@ -0,0 +1,544 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package selfhosting
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/ghodss/yaml"
|
||||
)
|
||||
|
||||
const (
|
||||
testAPIServerPod = `
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
annotations:
|
||||
scheduler.alpha.kubernetes.io/critical-pod: ""
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
component: kube-apiserver
|
||||
tier: control-plane
|
||||
name: kube-apiserver
|
||||
namespace: kube-system
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- kube-apiserver
|
||||
- --client-ca-file=/etc/kubernetes/pki/ca.crt
|
||||
- --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
|
||||
- --allow-privileged=true
|
||||
- --service-cluster-ip-range=10.96.0.0/12
|
||||
- --service-account-key-file=/etc/kubernetes/pki/sa.pub
|
||||
- --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
|
||||
- --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
|
||||
- --secure-port=6443
|
||||
- --insecure-port=0
|
||||
- --admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota
|
||||
- --requestheader-extra-headers-prefix=X-Remote-Extra-
|
||||
- --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
|
||||
- --experimental-bootstrap-token-auth=true
|
||||
- --requestheader-group-headers=X-Remote-Group
|
||||
- --requestheader-allowed-names=front-proxy-client
|
||||
- --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
|
||||
- --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
|
||||
- --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
|
||||
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
|
||||
- --requestheader-username-headers=X-Remote-User
|
||||
- --authorization-mode=Node,RBAC
|
||||
- --advertise-address=192.168.200.101
|
||||
- --etcd-servers=http://127.0.0.1:2379
|
||||
image: gcr.io/google_containers/kube-apiserver-amd64:v1.7.0
|
||||
livenessProbe:
|
||||
failureThreshold: 8
|
||||
httpGet:
|
||||
host: 127.0.0.1
|
||||
path: /healthz
|
||||
port: 6443
|
||||
scheme: HTTPS
|
||||
initialDelaySeconds: 15
|
||||
timeoutSeconds: 15
|
||||
name: kube-apiserver
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
volumeMounts:
|
||||
- mountPath: /etc/kubernetes
|
||||
name: k8s
|
||||
readOnly: true
|
||||
- mountPath: /etc/ssl/certs
|
||||
name: certs
|
||||
- mountPath: /etc/pki
|
||||
name: pki
|
||||
hostNetwork: true
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /etc/kubernetes
|
||||
name: k8s
|
||||
- hostPath:
|
||||
path: /etc/ssl/certs
|
||||
name: certs
|
||||
- hostPath:
|
||||
path: /etc/pki
|
||||
name: pki
|
||||
status: {}
|
||||
`
|
||||
|
||||
testAPIServerDaemonSet = `metadata:
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
k8s-app: self-hosted-kube-apiserver
|
||||
name: self-hosted-kube-apiserver
|
||||
namespace: kube-system
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
k8s-app: self-hosted-kube-apiserver
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- kube-apiserver
|
||||
- --client-ca-file=/etc/kubernetes/pki/ca.crt
|
||||
- --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
|
||||
- --allow-privileged=true
|
||||
- --service-cluster-ip-range=10.96.0.0/12
|
||||
- --service-account-key-file=/etc/kubernetes/pki/sa.pub
|
||||
- --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
|
||||
- --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
|
||||
- --secure-port=6443
|
||||
- --insecure-port=0
|
||||
- --admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota
|
||||
- --requestheader-extra-headers-prefix=X-Remote-Extra-
|
||||
- --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
|
||||
- --experimental-bootstrap-token-auth=true
|
||||
- --requestheader-group-headers=X-Remote-Group
|
||||
- --requestheader-allowed-names=front-proxy-client
|
||||
- --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
|
||||
- --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
|
||||
- --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
|
||||
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
|
||||
- --requestheader-username-headers=X-Remote-User
|
||||
- --authorization-mode=Node,RBAC
|
||||
- --advertise-address=192.168.200.101
|
||||
- --etcd-servers=http://127.0.0.1:2379
|
||||
image: gcr.io/google_containers/kube-apiserver-amd64:v1.7.0
|
||||
livenessProbe:
|
||||
failureThreshold: 8
|
||||
httpGet:
|
||||
host: 127.0.0.1
|
||||
path: /healthz
|
||||
port: 6443
|
||||
scheme: HTTPS
|
||||
initialDelaySeconds: 15
|
||||
timeoutSeconds: 15
|
||||
name: kube-apiserver
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
volumeMounts:
|
||||
- mountPath: /etc/kubernetes
|
||||
name: k8s
|
||||
readOnly: true
|
||||
- mountPath: /etc/ssl/certs
|
||||
name: certs
|
||||
- mountPath: /etc/pki
|
||||
name: pki
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
hostNetwork: true
|
||||
nodeSelector:
|
||||
node-role.kubernetes.io/master: ""
|
||||
tolerations:
|
||||
- effect: NoSchedule
|
||||
key: node-role.kubernetes.io/master
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /etc/kubernetes
|
||||
name: k8s
|
||||
- hostPath:
|
||||
path: /etc/ssl/certs
|
||||
name: certs
|
||||
- hostPath:
|
||||
path: /etc/pki
|
||||
name: pki
|
||||
updateStrategy: {}
|
||||
status:
|
||||
currentNumberScheduled: 0
|
||||
desiredNumberScheduled: 0
|
||||
numberMisscheduled: 0
|
||||
numberReady: 0
|
||||
`
|
||||
|
||||
testControllerManagerPod = `
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
annotations:
|
||||
scheduler.alpha.kubernetes.io/critical-pod: ""
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
component: kube-controller-manager
|
||||
tier: control-plane
|
||||
name: kube-controller-manager
|
||||
namespace: kube-system
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- kube-controller-manager
|
||||
- --service-account-private-key-file=/etc/kubernetes/pki/sa.key
|
||||
- --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
|
||||
- --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
|
||||
- --leader-elect=true
|
||||
- --kubeconfig=/etc/kubernetes/controller-manager.conf
|
||||
- --controllers=*,bootstrapsigner,tokencleaner
|
||||
- --root-ca-file=/etc/kubernetes/pki/ca.crt
|
||||
- --address=127.0.0.1
|
||||
- --use-service-account-credentials=true
|
||||
image: gcr.io/google_containers/kube-controller-manager-amd64:v1.7.0
|
||||
livenessProbe:
|
||||
failureThreshold: 8
|
||||
httpGet:
|
||||
host: 127.0.0.1
|
||||
path: /healthz
|
||||
port: 10252
|
||||
scheme: HTTP
|
||||
initialDelaySeconds: 15
|
||||
timeoutSeconds: 15
|
||||
name: kube-controller-manager
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
volumeMounts:
|
||||
- mountPath: /etc/kubernetes
|
||||
name: k8s
|
||||
readOnly: true
|
||||
- mountPath: /etc/ssl/certs
|
||||
name: certs
|
||||
- mountPath: /etc/pki
|
||||
name: pki
|
||||
hostNetwork: true
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /etc/kubernetes
|
||||
name: k8s
|
||||
- hostPath:
|
||||
path: /etc/ssl/certs
|
||||
name: certs
|
||||
- hostPath:
|
||||
path: /etc/pki
|
||||
name: pki
|
||||
status: {}
|
||||
`
|
||||
|
||||
testControllerManagerDaemonSet = `metadata:
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
k8s-app: self-hosted-kube-controller-manager
|
||||
name: self-hosted-kube-controller-manager
|
||||
namespace: kube-system
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
k8s-app: self-hosted-kube-controller-manager
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- kube-controller-manager
|
||||
- --service-account-private-key-file=/etc/kubernetes/pki/sa.key
|
||||
- --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
|
||||
- --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
|
||||
- --leader-elect=true
|
||||
- --kubeconfig=/etc/kubernetes/controller-manager.conf
|
||||
- --controllers=*,bootstrapsigner,tokencleaner
|
||||
- --root-ca-file=/etc/kubernetes/pki/ca.crt
|
||||
- --address=127.0.0.1
|
||||
- --use-service-account-credentials=true
|
||||
image: gcr.io/google_containers/kube-controller-manager-amd64:v1.7.0
|
||||
livenessProbe:
|
||||
failureThreshold: 8
|
||||
httpGet:
|
||||
host: 127.0.0.1
|
||||
path: /healthz
|
||||
port: 10252
|
||||
scheme: HTTP
|
||||
initialDelaySeconds: 15
|
||||
timeoutSeconds: 15
|
||||
name: kube-controller-manager
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
volumeMounts:
|
||||
- mountPath: /etc/kubernetes
|
||||
name: k8s
|
||||
readOnly: true
|
||||
- mountPath: /etc/ssl/certs
|
||||
name: certs
|
||||
- mountPath: /etc/pki
|
||||
name: pki
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
hostNetwork: true
|
||||
nodeSelector:
|
||||
node-role.kubernetes.io/master: ""
|
||||
tolerations:
|
||||
- effect: NoSchedule
|
||||
key: node-role.kubernetes.io/master
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /etc/kubernetes
|
||||
name: k8s
|
||||
- hostPath:
|
||||
path: /etc/ssl/certs
|
||||
name: certs
|
||||
- hostPath:
|
||||
path: /etc/pki
|
||||
name: pki
|
||||
updateStrategy: {}
|
||||
status:
|
||||
currentNumberScheduled: 0
|
||||
desiredNumberScheduled: 0
|
||||
numberMisscheduled: 0
|
||||
numberReady: 0
|
||||
`
|
||||
|
||||
testSchedulerPod = `
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
annotations:
|
||||
scheduler.alpha.kubernetes.io/critical-pod: ""
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
component: kube-scheduler
|
||||
tier: control-plane
|
||||
name: kube-scheduler
|
||||
namespace: kube-system
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- kube-scheduler
|
||||
- --address=127.0.0.1
|
||||
- --leader-elect=true
|
||||
- --kubeconfig=/etc/kubernetes/scheduler.conf
|
||||
image: gcr.io/google_containers/kube-scheduler-amd64:v1.7.0
|
||||
livenessProbe:
|
||||
failureThreshold: 8
|
||||
httpGet:
|
||||
host: 127.0.0.1
|
||||
path: /healthz
|
||||
port: 10251
|
||||
scheme: HTTP
|
||||
initialDelaySeconds: 15
|
||||
timeoutSeconds: 15
|
||||
name: kube-scheduler
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
volumeMounts:
|
||||
- mountPath: /etc/kubernetes
|
||||
name: k8s
|
||||
readOnly: true
|
||||
hostNetwork: true
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /etc/kubernetes
|
||||
name: k8s
|
||||
status: {}
|
||||
`
|
||||
|
||||
testSchedulerDaemonSet = `metadata:
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
k8s-app: self-hosted-kube-scheduler
|
||||
name: self-hosted-kube-scheduler
|
||||
namespace: kube-system
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
k8s-app: self-hosted-kube-scheduler
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- kube-scheduler
|
||||
- --address=127.0.0.1
|
||||
- --leader-elect=true
|
||||
- --kubeconfig=/etc/kubernetes/scheduler.conf
|
||||
image: gcr.io/google_containers/kube-scheduler-amd64:v1.7.0
|
||||
livenessProbe:
|
||||
failureThreshold: 8
|
||||
httpGet:
|
||||
host: 127.0.0.1
|
||||
path: /healthz
|
||||
port: 10251
|
||||
scheme: HTTP
|
||||
initialDelaySeconds: 15
|
||||
timeoutSeconds: 15
|
||||
name: kube-scheduler
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
volumeMounts:
|
||||
- mountPath: /etc/kubernetes
|
||||
name: k8s
|
||||
readOnly: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
hostNetwork: true
|
||||
nodeSelector:
|
||||
node-role.kubernetes.io/master: ""
|
||||
tolerations:
|
||||
- effect: NoSchedule
|
||||
key: node-role.kubernetes.io/master
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /etc/kubernetes
|
||||
name: k8s
|
||||
updateStrategy: {}
|
||||
status:
|
||||
currentNumberScheduled: 0
|
||||
desiredNumberScheduled: 0
|
||||
numberMisscheduled: 0
|
||||
numberReady: 0
|
||||
`
|
||||
)
|
||||
|
||||
func TestBuildDaemonSet(t *testing.T) {
|
||||
var tests = []struct {
|
||||
component string
|
||||
podBytes []byte
|
||||
dsBytes []byte
|
||||
}{
|
||||
{
|
||||
component: kubeAPIServer,
|
||||
podBytes: []byte(testAPIServerPod),
|
||||
dsBytes: []byte(testAPIServerDaemonSet),
|
||||
},
|
||||
{
|
||||
component: kubeControllerManager,
|
||||
podBytes: []byte(testControllerManagerPod),
|
||||
dsBytes: []byte(testControllerManagerDaemonSet),
|
||||
},
|
||||
{
|
||||
component: kubeScheduler,
|
||||
podBytes: []byte(testSchedulerPod),
|
||||
dsBytes: []byte(testSchedulerDaemonSet),
|
||||
},
|
||||
}
|
||||
|
||||
for _, rt := range tests {
|
||||
tempFile, err := createTempFileWithContent(rt.podBytes)
|
||||
defer os.Remove(tempFile)
|
||||
|
||||
podSpec, err := loadPodSpecFromFile(tempFile)
|
||||
if err != nil {
|
||||
t.Fatalf("couldn't load the specified Pod")
|
||||
}
|
||||
|
||||
ds := buildDaemonSet(rt.component, podSpec)
|
||||
dsBytes, err := yaml.Marshal(ds)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to marshal daemonset to YAML: %v", err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(dsBytes, rt.dsBytes) {
|
||||
t.Errorf("failed TestBuildDaemonSet:\nexpected:\n%s\nsaw:\n%s", rt.dsBytes, dsBytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPodSpecFromFile(t *testing.T) {
|
||||
tests := []struct {
|
||||
content string
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
// Good YAML
|
||||
content: `
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: testpod
|
||||
spec:
|
||||
containers:
|
||||
- image: gcr.io/google_containers/busybox
|
||||
`,
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
// Good JSON
|
||||
content: `
|
||||
{
|
||||
"apiVersion": "v1",
|
||||
"kind": "Pod",
|
||||
"metadata": {
|
||||
"name": "testpod"
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "gcr.io/google_containers/busybox"
|
||||
}
|
||||
]
|
||||
}
|
||||
}`,
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
// Bad PodSpec
|
||||
content: `
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: testpod
|
||||
spec:
|
||||
- image: gcr.io/google_containers/busybox
|
||||
`,
|
||||
expectError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, rt := range tests {
|
||||
tempFile, err := createTempFileWithContent([]byte(rt.content))
|
||||
defer os.Remove(tempFile)
|
||||
|
||||
_, err = loadPodSpecFromFile(tempFile)
|
||||
if (err != nil) != rt.expectError {
|
||||
t.Errorf("failed TestLoadPodSpecFromFile:\nexpected error:\n%t\nsaw:\n%v", rt.expectError, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func createTempFileWithContent(content []byte) (string, error) {
|
||||
tempFile, err := ioutil.TempFile("", "")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot create temporary file: %v", err)
|
||||
}
|
||||
if _, err = tempFile.Write([]byte(content)); err != nil {
|
||||
return "", fmt.Errorf("cannot save temporary file: %v", err)
|
||||
}
|
||||
if err = tempFile.Close(); err != nil {
|
||||
return "", fmt.Errorf("cannot close temporary file: %v", err)
|
||||
}
|
||||
return tempFile.Name(), nil
|
||||
}
|
|
@ -21,6 +21,8 @@ go_library(
|
|||
"//cmd/kubeadm/app/constants:go_default_library",
|
||||
"//cmd/kubeadm/app/preflight:go_default_library",
|
||||
"//cmd/kubeadm/app/util/kubeconfig:go_default_library",
|
||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
|
||||
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
|
||||
|
|
|
@ -21,12 +21,15 @@ import (
|
|||
"net/http"
|
||||
"time"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
|
||||
)
|
||||
|
||||
// CreateClientAndWaitForAPI takes a path to a kubeconfig file, makes a client of it and waits for the API to be healthy
|
||||
func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) {
|
||||
client, err := kubeconfigutil.ClientSetFromFile(file)
|
||||
if err != nil {
|
||||
|
@ -39,6 +42,7 @@ func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) {
|
|||
return client, nil
|
||||
}
|
||||
|
||||
// WaitForAPI waits for the API Server's /healthz endpoint to report "ok"
|
||||
func WaitForAPI(client *clientset.Clientset) {
|
||||
start := time.Now()
|
||||
wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
|
||||
|
@ -52,3 +56,30 @@ func WaitForAPI(client *clientset.Clientset) {
|
|||
return true, nil
|
||||
})
|
||||
}
|
||||
|
||||
// WaitForPodsWithLabel will lookup pods with the given label and wait until they are all
|
||||
// reporting status as running.
|
||||
func WaitForPodsWithLabel(client *clientset.Clientset, labelKeyValPair string) {
|
||||
// TODO: Implement a timeout
|
||||
// TODO: Implement a verbosity switch
|
||||
wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
|
||||
listOpts := metav1.ListOptions{LabelSelector: labelKeyValPair}
|
||||
apiPods, err := client.CoreV1().Pods(metav1.NamespaceSystem).List(listOpts)
|
||||
if err != nil {
|
||||
fmt.Printf("[apiclient] Error getting Pods with label selector %q [%v]\n", labelKeyValPair, err)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if len(apiPods.Items) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
for _, pod := range apiPods.Items {
|
||||
fmt.Printf("[apiclient] Pod %s status: %s\n", pod.Name, pod.Status.Phase)
|
||||
if pod.Status.Phase != v1.PodRunning {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
}
|
||||
|
|
|
@ -22,7 +22,9 @@ cmd/kubeadm/app/apis/kubeadm/install
|
|||
cmd/kubeadm/app/discovery/https
|
||||
cmd/kubeadm/app/phases/apiconfig
|
||||
cmd/kubeadm/app/phases/certs
|
||||
cmd/kubeadm/app/phases/controlplane
|
||||
cmd/kubeadm/app/phases/kubeconfig
|
||||
cmd/kubeadm/app/phases/selfhosting
|
||||
cmd/kubectl
|
||||
cmd/kubelet
|
||||
cmd/libs/go2idl/client-gen
|
||||
|
|
Loading…
Reference in New Issue