mirror of https://github.com/k3s-io/k3s
kubeadm: Make self-hosting work and split out to a phase
Lucas Käldström
parent
9385d50100
commit
d14478f27a
|
|
@ -37,6 +37,7 @@ import (
|
|||
certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
|
||||
controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
|
||||
kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
|
||||
selfhostingphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting"
|
||||
tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/token"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
|
||||
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
|
||||
|
|
@ -132,6 +133,10 @@ func NewCmdInit(out io.Writer) *cobra.Command {
|
|||
&skipTokenPrint, "skip-token-print", skipTokenPrint,
|
||||
"Skip printing of the default bootstrap token generated by 'kubeadm init'",
|
||||
)
|
||||
cmd.PersistentFlags().BoolVar(
|
||||
&cfg.SelfHosted, "self-hosted", cfg.SelfHosted,
|
||||
"[experimental] If kubeadm should make this control plane self-hosted",
|
||||
)
|
||||
|
||||
cmd.PersistentFlags().StringVar(
|
||||
&cfg.Token, "token", cfg.Token,
|
||||
|
|
@ -266,7 +271,7 @@ func (i *Init) Run(out io.Writer) error {
|
|||
// Temporary control plane is up, now we create our self hosted control
|
||||
// plane components and remove the static manifests:
|
||||
fmt.Println("[self-hosted] Creating self-hosted control plane...")
|
||||
if err := controlplanephase.CreateSelfHostedControlPlane(i.cfg, client); err != nil {
|
||||
if err := selfhostingphase.CreateSelfHostedControlPlane(client); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ func NewCmdPhase(out io.Writer) *cobra.Command {
|
|||
cmd.AddCommand(NewCmdKubeConfig(out))
|
||||
cmd.AddCommand(NewCmdCerts())
|
||||
cmd.AddCommand(NewCmdPreFlight())
|
||||
cmd.AddCommand(NewCmdSelfhosting())
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package phases
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting"
|
||||
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
|
||||
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
|
||||
)
|
||||
|
||||
// NewCmdSelfhosting returns the self-hosting Cobra command
|
||||
func NewCmdSelfhosting() *cobra.Command {
|
||||
var kubeConfigFile string
|
||||
cmd := &cobra.Command{
|
||||
Use: "selfhosting",
|
||||
Aliases: []string{"selfhosted"},
|
||||
Short: "Make a kubeadm cluster self-hosted.",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile)
|
||||
kubeadmutil.CheckErr(err)
|
||||
|
||||
err = selfhosting.CreateSelfHostedControlPlane(client)
|
||||
kubeadmutil.CheckErr(err)
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use for talking to the cluster")
|
||||
return cmd
|
||||
}
|
||||
|
|
@ -109,3 +109,8 @@ var (
|
|||
// MinimumControlPlaneVersion specifies the minimum control plane version kubeadm can deploy
|
||||
MinimumControlPlaneVersion = version.MustParseSemantic("v1.7.0")
|
||||
)
|
||||
|
||||
// BuildStaticManifestFilepath returns the location on the disk where the Static Pod should be present
|
||||
func BuildStaticManifestFilepath(componentName string) string {
|
||||
return filepath.Join(KubernetesDir, ManifestsSubDirName, componentName+".yaml")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -229,23 +229,6 @@ func pkiVolumeMount() v1.VolumeMount {
|
|||
}
|
||||
}
|
||||
|
||||
func flockVolume() v1.Volume {
|
||||
return v1.Volume{
|
||||
Name: "var-lock",
|
||||
VolumeSource: v1.VolumeSource{
|
||||
HostPath: &v1.HostPathVolumeSource{Path: "/var/lock"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func flockVolumeMount() v1.VolumeMount {
|
||||
return v1.VolumeMount{
|
||||
Name: "var-lock",
|
||||
MountPath: "/var/lock",
|
||||
ReadOnly: false,
|
||||
}
|
||||
}
|
||||
|
||||
func k8sVolume() v1.Volume {
|
||||
return v1.Volume{
|
||||
Name: "k8s",
|
||||
|
|
@ -447,19 +430,6 @@ func getProxyEnvVars() []v1.EnvVar {
|
|||
return envs
|
||||
}
|
||||
|
||||
func getSelfHostedAPIServerEnv() []v1.EnvVar {
|
||||
podIPEnvVar := v1.EnvVar{
|
||||
Name: "POD_IP",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
FieldRef: &v1.ObjectFieldSelector{
|
||||
FieldPath: "status.podIP",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return append(getProxyEnvVars(), podIPEnvVar)
|
||||
}
|
||||
|
||||
// getAuthzParameters gets the authorization-related parameters to the api server
|
||||
// At this point, we can assume the list of authorization modes is valid (due to that it has been validated in the API machinery code already)
|
||||
// If the list is empty; it's defaulted (mostly for unit testing)
|
||||
|
|
|
|||
|
|
@ -1,348 +0,0 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package controlplane
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
extensions "k8s.io/api/extensions/v1beta1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
|
||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/images"
|
||||
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
|
||||
"k8s.io/kubernetes/pkg/util/version"
|
||||
)
|
||||
|
||||
var (
|
||||
// maximum unavailable and surge instances per self-hosted component deployment
|
||||
maxUnavailable = intstr.FromInt(0)
|
||||
maxSurge = intstr.FromInt(1)
|
||||
)
|
||||
|
||||
func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error {
|
||||
volumes := []v1.Volume{k8sVolume()}
|
||||
volumeMounts := []v1.VolumeMount{k8sVolumeMount()}
|
||||
if isCertsVolumeMountNeeded() {
|
||||
volumes = append(volumes, certsVolume(cfg))
|
||||
volumeMounts = append(volumeMounts, certsVolumeMount())
|
||||
}
|
||||
|
||||
if isPkiVolumeMountNeeded() {
|
||||
volumes = append(volumes, pkiVolume())
|
||||
volumeMounts = append(volumeMounts, pkiVolumeMount())
|
||||
}
|
||||
|
||||
// Need lock for self-hosted
|
||||
volumes = append(volumes, flockVolume())
|
||||
volumeMounts = append(volumeMounts, flockVolumeMount())
|
||||
|
||||
if err := launchSelfHostedAPIServer(cfg, client, volumes, volumeMounts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := launchSelfHostedScheduler(cfg, client, volumes, volumeMounts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := launchSelfHostedControllerManager(cfg, client, volumes, volumeMounts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func launchSelfHostedAPIServer(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error {
|
||||
start := time.Now()
|
||||
|
||||
kubeVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
apiServer := getAPIServerDS(cfg, volumes, volumeMounts, kubeVersion)
|
||||
if _, err := client.Extensions().DaemonSets(metav1.NamespaceSystem).Create(&apiServer); err != nil {
|
||||
return fmt.Errorf("failed to create self-hosted %q daemon set [%v]", kubeAPIServer, err)
|
||||
}
|
||||
|
||||
wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
|
||||
// TODO: This might be pointless, checking the pods is probably enough.
|
||||
// It does however get us a count of how many there should be which may be useful
|
||||
// with HA.
|
||||
apiDS, err := client.DaemonSets(metav1.NamespaceSystem).Get("self-hosted-"+kubeAPIServer,
|
||||
metav1.GetOptions{})
|
||||
if err != nil {
|
||||
fmt.Println("[self-hosted] error getting apiserver DaemonSet:", err)
|
||||
return false, nil
|
||||
}
|
||||
fmt.Printf("[self-hosted] %s DaemonSet current=%d, desired=%d\n",
|
||||
kubeAPIServer,
|
||||
apiDS.Status.CurrentNumberScheduled,
|
||||
apiDS.Status.DesiredNumberScheduled)
|
||||
|
||||
if apiDS.Status.CurrentNumberScheduled != apiDS.Status.DesiredNumberScheduled {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
|
||||
// Wait for self-hosted API server to take ownership
|
||||
waitForPodsWithLabel(client, "self-hosted-"+kubeAPIServer, true)
|
||||
|
||||
// Remove temporary API server
|
||||
apiServerStaticManifestPath := buildStaticManifestFilepath(kubeAPIServer)
|
||||
if err := os.RemoveAll(apiServerStaticManifestPath); err != nil {
|
||||
return fmt.Errorf("unable to delete temporary API server manifest [%v]", err)
|
||||
}
|
||||
|
||||
kubeadmutil.WaitForAPI(client)
|
||||
|
||||
fmt.Printf("[self-hosted] self-hosted kube-apiserver ready after %f seconds\n", time.Since(start).Seconds())
|
||||
return nil
|
||||
}
|
||||
|
||||
func launchSelfHostedControllerManager(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error {
|
||||
start := time.Now()
|
||||
|
||||
kubeVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctrlMgr := getControllerManagerDeployment(cfg, volumes, volumeMounts, kubeVersion)
|
||||
if _, err := client.Extensions().Deployments(metav1.NamespaceSystem).Create(&ctrlMgr); err != nil {
|
||||
return fmt.Errorf("failed to create self-hosted %q deployment [%v]", kubeControllerManager, err)
|
||||
}
|
||||
|
||||
waitForPodsWithLabel(client, "self-hosted-"+kubeControllerManager, true)
|
||||
|
||||
ctrlMgrStaticManifestPath := buildStaticManifestFilepath(kubeControllerManager)
|
||||
if err := os.RemoveAll(ctrlMgrStaticManifestPath); err != nil {
|
||||
return fmt.Errorf("unable to delete temporary controller manager manifest [%v]", err)
|
||||
}
|
||||
|
||||
fmt.Printf("[self-hosted] self-hosted kube-controller-manager ready after %f seconds\n", time.Since(start).Seconds())
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func launchSelfHostedScheduler(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error {
|
||||
start := time.Now()
|
||||
scheduler := getSchedulerDeployment(cfg, volumes, volumeMounts)
|
||||
if _, err := client.Extensions().Deployments(metav1.NamespaceSystem).Create(&scheduler); err != nil {
|
||||
return fmt.Errorf("failed to create self-hosted %q deployment [%v]", kubeScheduler, err)
|
||||
}
|
||||
|
||||
waitForPodsWithLabel(client, "self-hosted-"+kubeScheduler, true)
|
||||
|
||||
schedulerStaticManifestPath := buildStaticManifestFilepath(kubeScheduler)
|
||||
if err := os.RemoveAll(schedulerStaticManifestPath); err != nil {
|
||||
return fmt.Errorf("unable to delete temporary scheduler manifest [%v]", err)
|
||||
}
|
||||
|
||||
fmt.Printf("[self-hosted] self-hosted kube-scheduler ready after %f seconds\n", time.Since(start).Seconds())
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForPodsWithLabel will lookup pods with the given label and wait until they are all
|
||||
// reporting status as running.
|
||||
func waitForPodsWithLabel(client *clientset.Clientset, appLabel string, mustBeRunning bool) {
|
||||
wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
|
||||
// TODO: Do we need a stronger label link than this?
|
||||
listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("k8s-app=%s", appLabel)}
|
||||
apiPods, err := client.Pods(metav1.NamespaceSystem).List(listOpts)
|
||||
if err != nil {
|
||||
fmt.Printf("[self-hosted] error getting %s pods [%v]\n", appLabel, err)
|
||||
return false, nil
|
||||
}
|
||||
fmt.Printf("[self-hosted] Found %d %s pods\n", len(apiPods.Items), appLabel)
|
||||
|
||||
// TODO: HA
|
||||
if int32(len(apiPods.Items)) != 1 {
|
||||
return false, nil
|
||||
}
|
||||
for _, pod := range apiPods.Items {
|
||||
fmt.Printf("[self-hosted] Pod %s status: %s\n", pod.Name, pod.Status.Phase)
|
||||
if mustBeRunning && pod.Status.Phase != "Running" {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
}
|
||||
|
||||
// Sources from bootkube templates.go
|
||||
func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount, kubeVersion *version.Version) extensions.DaemonSet {
|
||||
ds := extensions.DaemonSet{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "extensions/v1beta1",
|
||||
Kind: "DaemonSet",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "self-hosted-" + kubeAPIServer,
|
||||
Namespace: "kube-system",
|
||||
Labels: map[string]string{"k8s-app": "self-hosted-" + kubeAPIServer},
|
||||
},
|
||||
Spec: extensions.DaemonSetSpec{
|
||||
Template: v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"k8s-app": "self-hosted-" + kubeAPIServer,
|
||||
"component": kubeAPIServer,
|
||||
"tier": "control-plane",
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
NodeSelector: map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""},
|
||||
HostNetwork: true,
|
||||
Volumes: volumes,
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: "self-hosted-" + kubeAPIServer,
|
||||
Image: images.GetCoreImage(images.KubeAPIServerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
|
||||
Command: getAPIServerCommand(cfg, true, kubeVersion),
|
||||
Env: getSelfHostedAPIServerEnv(),
|
||||
VolumeMounts: volumeMounts,
|
||||
LivenessProbe: componentProbe(6443, "/healthz", v1.URISchemeHTTPS),
|
||||
Resources: componentResources("250m"),
|
||||
},
|
||||
},
|
||||
Tolerations: []v1.Toleration{kubeadmconstants.MasterToleration},
|
||||
DNSPolicy: v1.DNSClusterFirstWithHostNet,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
return ds
|
||||
}
|
||||
|
||||
func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount, kubeVersion *version.Version) extensions.Deployment {
|
||||
d := extensions.Deployment{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "extensions/v1beta1",
|
||||
Kind: "Deployment",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "self-hosted-" + kubeControllerManager,
|
||||
Namespace: "kube-system",
|
||||
Labels: map[string]string{"k8s-app": "self-hosted-" + kubeControllerManager},
|
||||
},
|
||||
Spec: extensions.DeploymentSpec{
|
||||
// TODO bootkube uses 2 replicas
|
||||
Strategy: extensions.DeploymentStrategy{
|
||||
Type: extensions.RollingUpdateDeploymentStrategyType,
|
||||
RollingUpdate: &extensions.RollingUpdateDeployment{
|
||||
MaxUnavailable: &maxUnavailable,
|
||||
MaxSurge: &maxSurge,
|
||||
},
|
||||
},
|
||||
Template: v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"k8s-app": "self-hosted-" + kubeControllerManager,
|
||||
"component": kubeControllerManager,
|
||||
"tier": "control-plane",
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
NodeSelector: map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""},
|
||||
HostNetwork: true,
|
||||
Volumes: volumes,
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: "self-hosted-" + kubeControllerManager,
|
||||
Image: images.GetCoreImage(images.KubeControllerManagerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
|
||||
Command: getControllerManagerCommand(cfg, true, kubeVersion),
|
||||
VolumeMounts: volumeMounts,
|
||||
LivenessProbe: componentProbe(10252, "/healthz", v1.URISchemeHTTP),
|
||||
Resources: componentResources("200m"),
|
||||
Env: getProxyEnvVars(),
|
||||
},
|
||||
},
|
||||
Tolerations: []v1.Toleration{kubeadmconstants.MasterToleration},
|
||||
DNSPolicy: v1.DNSClusterFirstWithHostNet,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func getSchedulerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount) extensions.Deployment {
|
||||
d := extensions.Deployment{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "extensions/v1beta1",
|
||||
Kind: "Deployment",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "self-hosted-" + kubeScheduler,
|
||||
Namespace: "kube-system",
|
||||
Labels: map[string]string{"k8s-app": "self-hosted-" + kubeScheduler},
|
||||
},
|
||||
Spec: extensions.DeploymentSpec{
|
||||
// TODO bootkube uses 2 replicas
|
||||
Strategy: extensions.DeploymentStrategy{
|
||||
Type: extensions.RollingUpdateDeploymentStrategyType,
|
||||
RollingUpdate: &extensions.RollingUpdateDeployment{
|
||||
MaxUnavailable: &maxUnavailable,
|
||||
MaxSurge: &maxSurge,
|
||||
},
|
||||
},
|
||||
Template: v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"k8s-app": "self-hosted-" + kubeScheduler,
|
||||
"component": kubeScheduler,
|
||||
"tier": "control-plane",
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
NodeSelector: map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""},
|
||||
HostNetwork: true,
|
||||
Volumes: volumes,
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: "self-hosted-" + kubeScheduler,
|
||||
Image: images.GetCoreImage(images.KubeSchedulerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
|
||||
Command: getSchedulerCommand(cfg, true),
|
||||
VolumeMounts: volumeMounts,
|
||||
LivenessProbe: componentProbe(10251, "/healthz", v1.URISchemeHTTP),
|
||||
Resources: componentResources("100m"),
|
||||
Env: getProxyEnvVars(),
|
||||
},
|
||||
},
|
||||
Tolerations: []v1.Toleration{kubeadmconstants.MasterToleration},
|
||||
DNSPolicy: v1.DNSClusterFirstWithHostNet,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return d
|
||||
}
|
||||
|
||||
func buildStaticManifestFilepath(name string) string {
|
||||
return filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.ManifestsSubDirName, name+".yaml")
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package selfhosting
|
||||
|
||||
import (
|
||||
"k8s.io/api/core/v1"
|
||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
)
|
||||
|
||||
// mutatePodSpec makes a Static Pod-hosted PodSpec suitable for self-hosting
|
||||
func mutatePodSpec(name string, podSpec *v1.PodSpec) {
|
||||
mutators := map[string][]func(*v1.PodSpec){
|
||||
kubeAPIServer: {
|
||||
addNodeSelectorToPodSpec,
|
||||
setMasterTolerationOnPodSpec,
|
||||
setRightDNSPolicyOnPodSpec,
|
||||
},
|
||||
kubeControllerManager: {
|
||||
addNodeSelectorToPodSpec,
|
||||
setMasterTolerationOnPodSpec,
|
||||
setRightDNSPolicyOnPodSpec,
|
||||
},
|
||||
kubeScheduler: {
|
||||
addNodeSelectorToPodSpec,
|
||||
setMasterTolerationOnPodSpec,
|
||||
setRightDNSPolicyOnPodSpec,
|
||||
},
|
||||
}
|
||||
|
||||
// Get the mutator functions for the component in question, then loop through and execute them
|
||||
mutatorsForComponent := mutators[name]
|
||||
for _, mutateFunc := range mutatorsForComponent {
|
||||
mutateFunc(podSpec)
|
||||
}
|
||||
}
|
||||
|
||||
// addNodeSelectorToPodSpec makes Pod require to be scheduled on a node marked with the master label
|
||||
func addNodeSelectorToPodSpec(podSpec *v1.PodSpec) {
|
||||
if podSpec.NodeSelector == nil {
|
||||
podSpec.NodeSelector = map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""}
|
||||
return
|
||||
}
|
||||
|
||||
podSpec.NodeSelector[kubeadmconstants.LabelNodeRoleMaster] = ""
|
||||
}
|
||||
|
||||
// setMasterTolerationOnPodSpec makes the Pod tolerate the master taint
|
||||
func setMasterTolerationOnPodSpec(podSpec *v1.PodSpec) {
|
||||
if podSpec.Tolerations == nil {
|
||||
podSpec.Tolerations = []v1.Toleration{kubeadmconstants.MasterToleration}
|
||||
return
|
||||
}
|
||||
|
||||
podSpec.Tolerations = append(podSpec.Tolerations, kubeadmconstants.MasterToleration)
|
||||
}
|
||||
|
||||
// setRightDNSPolicyOnPodSpec makes sure the self-hosted components can look up things via kube-dns if necessary
|
||||
func setRightDNSPolicyOnPodSpec(podSpec *v1.PodSpec) {
|
||||
podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
}
|
||||
|
|
@ -0,0 +1,158 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package selfhosting
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
extensions "k8s.io/api/extensions/v1beta1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
kuberuntime "k8s.io/apimachinery/pkg/runtime"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
|
||||
"k8s.io/kubernetes/pkg/api"
|
||||
)
|
||||
|
||||
const (
|
||||
kubeAPIServer = "kube-apiserver"
|
||||
kubeControllerManager = "kube-controller-manager"
|
||||
kubeScheduler = "kube-scheduler"
|
||||
|
||||
selfHostingPrefix = "self-hosted-"
|
||||
)
|
||||
|
||||
// CreateSelfHostedControlPlane is responsible for turning a Static Pod-hosted control plane to a self-hosted one
|
||||
// It achieves that task this way:
|
||||
// 1. Load the Static Pod specification from disk (from /etc/kubernetes/manifests)
|
||||
// 2. Extract the PodSpec from that Static Pod specification
|
||||
// 3. Mutate the PodSpec to be compatible with self-hosting (add the right labels, taints, etc. so it can schedule correctly)
|
||||
// 4. Build a new DaemonSet object for the self-hosted component in question. Use the above mentioned PodSpec
|
||||
// 5. Create the DaemonSet resource. Wait until the Pods are running.
|
||||
// 6. Remove the Static Pod manifest file. The kubelet will stop the original Static Pod-hosted component that was running.
|
||||
// 7. The self-hosted containers should now step up and take over.
|
||||
// 8. In order to avoid race conditions, we're still making sure the API /healthz endpoint is healthy
|
||||
// 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop
|
||||
func CreateSelfHostedControlPlane(client *clientset.Clientset) error {
|
||||
|
||||
// The sequence here isn't set in stone, but seems to work well to start with the API server
|
||||
components := []string{kubeAPIServer, kubeControllerManager, kubeScheduler}
|
||||
|
||||
for _, componentName := range components {
|
||||
start := time.Now()
|
||||
manifestPath := buildStaticManifestFilepath(componentName)
|
||||
|
||||
// Load the Static Pod file in order to be able to create a self-hosted variant of that file
|
||||
podSpec, err := loadPodSpecFromFile(manifestPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Build a DaemonSet object from the loaded PodSpec
|
||||
ds := buildDaemonSet(componentName, podSpec)
|
||||
|
||||
// Create the DaemonSet in the API Server
|
||||
if _, err := client.ExtensionsV1beta1().DaemonSets(metav1.NamespaceSystem).Create(ds); err != nil {
|
||||
if !apierrors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create self-hosted %q daemonset [%v]", componentName, err)
|
||||
}
|
||||
|
||||
if _, err := client.ExtensionsV1beta1().DaemonSets(metav1.NamespaceSystem).Update(ds); err != nil {
|
||||
// TODO: We should retry on 409 responses
|
||||
return fmt.Errorf("failed to update self-hosted %q daemonset [%v]", componentName, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for the self-hosted component to come up
|
||||
kubeadmutil.WaitForPodsWithLabel(client, buildSelfHostedWorkloadLabelQuery(componentName))
|
||||
|
||||
// Remove the old Static Pod manifest
|
||||
if err := os.RemoveAll(manifestPath); err != nil {
|
||||
return fmt.Errorf("unable to delete static pod manifest for %s [%v]", componentName, err)
|
||||
}
|
||||
|
||||
// Make sure the API is responsive at /healthz
|
||||
kubeadmutil.WaitForAPI(client)
|
||||
|
||||
fmt.Printf("[self-hosted] self-hosted %s ready after %f seconds\n", componentName, time.Since(start).Seconds())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildDaemonSet is responsible for mutating the PodSpec and return a DaemonSet which is suitable for the self-hosting purporse
|
||||
func buildDaemonSet(name string, podSpec *v1.PodSpec) *extensions.DaemonSet {
|
||||
// Mutate the PodSpec so it's suitable for self-hosting
|
||||
mutatePodSpec(name, podSpec)
|
||||
|
||||
// Return a DaemonSet based on that Spec
|
||||
return &extensions.DaemonSet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: addSelfHostedPrefix(name),
|
||||
Namespace: metav1.NamespaceSystem,
|
||||
Labels: map[string]string{
|
||||
"k8s-app": addSelfHostedPrefix(name),
|
||||
},
|
||||
},
|
||||
Spec: extensions.DaemonSetSpec{
|
||||
Template: v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"k8s-app": addSelfHostedPrefix(name),
|
||||
},
|
||||
},
|
||||
Spec: *podSpec,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// loadPodSpecFromFile reads and decodes a file containing a specification of a Pod
|
||||
// TODO: Consider using "k8s.io/kubernetes/pkg/volume/util".LoadPodFromFile(filename string) in the future instead.
|
||||
func loadPodSpecFromFile(manifestPath string) (*v1.PodSpec, error) {
|
||||
podBytes, err := ioutil.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
staticPod := &v1.Pod{}
|
||||
if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), podBytes, staticPod); err != nil {
|
||||
return nil, fmt.Errorf("unable to decode static pod %v", err)
|
||||
}
|
||||
|
||||
return &staticPod.Spec, nil
|
||||
}
|
||||
|
||||
// buildStaticManifestFilepath returns the location on the disk where the Static Pod should be present
|
||||
func buildStaticManifestFilepath(componentName string) string {
|
||||
return filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, componentName+".yaml")
|
||||
}
|
||||
|
||||
// buildSelfHostedWorkloadLabelQuery creates the right query for matching a self-hosted Pod
|
||||
func buildSelfHostedWorkloadLabelQuery(componentName string) string {
|
||||
return fmt.Sprintf("k8s-app=%s", addSelfHostedPrefix(componentName))
|
||||
}
|
||||
|
||||
// addSelfHostedPrefix adds the self-hosted- prefix to the component name
|
||||
func addSelfHostedPrefix(componentName string) string {
|
||||
return fmt.Sprintf("%s%s", selfHostingPrefix, componentName)
|
||||
}
|
||||
|
|
@ -21,12 +21,15 @@ import (
|
|||
"net/http"
|
||||
"time"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
|
||||
)
|
||||
|
||||
// CreateClientAndWaitForAPI takes a path to a kubeconfig file, makes a client of it and waits for the API to be healthy
|
||||
func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) {
|
||||
client, err := kubeconfigutil.ClientSetFromFile(file)
|
||||
if err != nil {
|
||||
|
|
@ -39,6 +42,7 @@ func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) {
|
|||
return client, nil
|
||||
}
|
||||
|
||||
// WaitForAPI waits for the API Server's /healthz endpoint to report "ok"
|
||||
func WaitForAPI(client *clientset.Clientset) {
|
||||
start := time.Now()
|
||||
wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
|
||||
|
|
@ -52,3 +56,30 @@ func WaitForAPI(client *clientset.Clientset) {
|
|||
return true, nil
|
||||
})
|
||||
}
|
||||
|
||||
// WaitForPodsWithLabel will lookup pods with the given label and wait until they are all
|
||||
// reporting status as running.
|
||||
func WaitForPodsWithLabel(client *clientset.Clientset, labelKeyValPair string) {
|
||||
// TODO: Implement a timeout
|
||||
// TODO: Implement a verbosity switch
|
||||
wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
|
||||
listOpts := metav1.ListOptions{LabelSelector: labelKeyValPair}
|
||||
apiPods, err := client.CoreV1().Pods(metav1.NamespaceSystem).List(listOpts)
|
||||
if err != nil {
|
||||
fmt.Printf("[apiclient] Error getting Pods with label selector %q [%v]\n", labelKeyValPair, err)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if len(apiPods.Items) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
for _, pod := range apiPods.Items {
|
||||
fmt.Printf("[apiclient] Pod %s status: %s\n", pod.Name, pod.Status.Phase)
|
||||
if pod.Status.Phase != v1.PodRunning {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue