github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

upload and fetch of kubeam config v1alpha3 from cluster

pull/8/head
fabriziopandini 2018-09-04 09:09:55 +02:00
parent 62315e88c0
commit 3f70af3685
17 changed files with 1319 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
cmd/kubeadm/app/componentconfigs/config.go Normal file
View File

@ -0,0 +1,80 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package componentconfigs
import (
"fmt"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config"
"k8s.io/kubernetes/pkg/util/version"
)
// GetFromKubeletConfigMap returns the pointer to the ComponentConfig API object read from the kubelet-config-version
// ConfigMap map stored in the cluster
func GetFromKubeletConfigMap(client clientset.Interface, version *version.Version) (runtime.Object, error) {
// Read the ConfigMap from the cluster based on what version the kubelet is
configMapName := kubeadmconstants.GetKubeletConfigMapName(version)
kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{})
if err != nil {
return nil, err
}
kubeletConfigData, ok := kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey]
if !ok {
return nil, fmt.Errorf("unexpected error when reading %s ConfigMap: %s key value pair missing", configMapName, kubeadmconstants.KubeletBaseConfigurationConfigMapKey)
}
// Decodes the kubeletConfigData into the internal component config
obj := &kubeletconfig.KubeletConfiguration{}
err = unmarshalObject(obj, []byte(kubeletConfigData))
if err != nil {
return nil, err
}
return obj, nil
}
// GetFromKubeProxyConfigMap returns the pointer to the ComponentConfig API object read from the kube-proxy
// ConfigMap map stored in the cluster
func GetFromKubeProxyConfigMap(client clientset.Interface, version *version.Version) (runtime.Object, error) {
// Read the ConfigMap from the cluster
kubeproxyCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeProxyConfigMap, metav1.GetOptions{})
if err != nil {
return nil, err
}
kubeproxyConfigData, ok := kubeproxyCfg.Data[kubeadmconstants.KubeProxyConfigMapKey]
if !ok {
return nil, fmt.Errorf("unexpected error when reading %s ConfigMap: %s key value pair missing", kubeadmconstants.KubeProxyConfigMap, kubeadmconstants.KubeProxyConfigMapKey)
}
// Decodes the Config map dat into the internal component config
obj := &kubeproxyconfig.KubeProxyConfiguration{}
err = unmarshalObject(obj, []byte(kubeproxyConfigData))
if err != nil {
return nil, err
}
return obj, nil
}

146
cmd/kubeadm/app/componentconfigs/config_test.go Normal file
View File

@ -0,0 +1,146 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package componentconfigs
import (
"testing"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
clientsetfake "k8s.io/client-go/kubernetes/fake"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
"k8s.io/kubernetes/pkg/util/version"
)
var cfgFiles = map[string][]byte{
"Kube-proxy_componentconfig": []byte(`
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
`),
"Kubelet_componentconfig": []byte(`
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
`),
}
func TestGetFromConfigMap(t *testing.T) {
k8sVersion := version.MustParseGeneric("v1.12.0")
var tests = []struct {
name string
component RegistrationKind
configMap *fakeConfigMap
expectedError bool
}{
{
name: "valid kube-proxy",
component: KubeProxyConfigurationKind,
configMap: &fakeConfigMap{
name: kubeadmconstants.KubeProxyConfigMap,
data: map[string]string{
kubeadmconstants.KubeProxyConfigMapKey: string(cfgFiles["Kube-proxy_componentconfig"]),
},
},
},
{
name: "invalid kube-proxy - missing ConfigMap",
component: KubeProxyConfigurationKind,
configMap: nil,
expectedError: true,
},
{
name: "invalid kube-proxy - missing key",
component: KubeProxyConfigurationKind,
configMap: &fakeConfigMap{
name: kubeadmconstants.KubeProxyConfigMap,
data: map[string]string{},
},
expectedError: true,
},
{
name: "valid kubelet",
component: KubeletConfigurationKind,
configMap: &fakeConfigMap{
name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion),
data: map[string]string{
kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]),
},
},
},
{
name: "invalid kubelet - missing ConfigMap",
component: KubeletConfigurationKind,
configMap: nil,
expectedError: true,
},
{
name: "invalid kubelet - missing key",
component: KubeletConfigurationKind,
configMap: &fakeConfigMap{
name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion),
data: map[string]string{},
},
expectedError: true,
},
}
for _, rt := range tests {
t.Run(rt.name, func(t2 *testing.T) {
client := clientsetfake.NewSimpleClientset()
if rt.configMap != nil {
err := rt.configMap.create(client)
if err != nil {
t.Errorf("unexpected create ConfigMap %s", rt.configMap.name)
return
}
}
registration := Known[rt.component]
obj, err := registration.GetFromConfigMap(client, k8sVersion)
if rt.expectedError != (err != nil) {
t.Errorf("unexpected return err from GetFromConfigMap: %v", err)
return
}
if rt.expectedError {
return
}
if obj == nil {
t.Error("unexpected nil return value")
}
})
}
}
type fakeConfigMap struct {
name string
data map[string]string
}
func (c *fakeConfigMap) create(client clientset.Interface) error {
return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: c.name,
Namespace: metav1.NamespaceSystem,
},
Data: c.data,
})
}

17
cmd/kubeadm/app/componentconfigs/registrations.go
View File

@ -20,12 +20,14 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/validation/field"
clientset "k8s.io/client-go/kubernetes"
kubeproxyconfigv1alpha1 "k8s.io/kube-proxy/config/v1alpha1"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1"
kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config"
"k8s.io/kubernetes/pkg/util/version"
)
// AddToSchemeFunc is a function that adds known types and API GroupVersions to a scheme
@ -47,6 +49,8 @@ type Registration struct {
GetFromInternalConfig func(*kubeadmapi.ClusterConfiguration) (runtime.Object, bool)
// SetToInternalConfig sets the pointer to a ComponentConfig API object embedded in the internal kubeadm config struct
SetToInternalConfig func(runtime.Object, *kubeadmapi.ClusterConfiguration) bool
// GetFromConfigMap returns the pointer to the ComponentConfig API object read from the config map stored in the cluster
GetFromConfigMap func(clientset.Interface, *version.Version) (runtime.Object, error)
}
// Marshal marshals obj to bytes for the current Registration
@ -60,13 +64,20 @@ func (r Registration) Unmarshal(fileContent []byte) (runtime.Object, error) {
obj := r.EmptyValue.DeepCopyObject()
// Decode the file content into obj which is a pointer to an empty struct of the internal ComponentConfig
// object, using the componentconfig Codecs that knows about all APIs
if err := runtime.DecodeInto(Codecs.UniversalDecoder(), fileContent, obj); err != nil {
if err := unmarshalObject(obj, fileContent); err != nil {
return nil, err
}
return obj, nil
}
func unmarshalObject(obj runtime.Object, fileContent []byte) error {
// Decode the file content using the componentconfig Codecs that knows about all APIs
if err := runtime.DecodeInto(Codecs.UniversalDecoder(), fileContent, obj); err != nil {
return err
}
return nil
}
const (
// KubeletConfigurationKind is the kind for the kubelet ComponentConfig
KubeletConfigurationKind RegistrationKind = "KubeletConfiguration"
@ -99,6 +110,7 @@ var Known Registrations = map[RegistrationKind]Registration{
}
return ok
},
GetFromConfigMap: GetFromKubeProxyConfigMap,
},
KubeletConfigurationKind: {
MarshalGroupVersion: kubeletconfigv1beta1.SchemeGroupVersion,
@ -116,6 +128,7 @@ var Known Registrations = map[RegistrationKind]Registration{
}
return ok
},
GetFromConfigMap: GetFromKubeletConfigMap,
},
}

26
cmd/kubeadm/app/constants/constants.go
View File

@ -154,6 +154,8 @@ const (
MastersGroup = "system:masters"
// NodesGroup defines the well-known group for all nodes.
NodesGroup = "system:nodes"
// NodesUserPrefix defines the user name prefix as requested by the Node authorizer.
NodesUserPrefix = "system:node:"
// NodesClusterRoleBinding defines the well-known ClusterRoleBinding which binds the too permissive system:node
// ClusterRole to the system:nodes group. Since kubeadm is using the Node Authorizer, this ClusterRoleBinding's
// system:nodes group subject is removed if present.
@ -187,13 +189,24 @@ const (
AnnotationKubeadmCRISocket = "kubeadm.alpha.kubernetes.io/cri-socket"
// InitConfigurationConfigMap specifies in what ConfigMap in the kube-system namespace the `kubeadm init` configuration should be stored
// TODO: Rename this to ClusterConfigurationConfigMap
// TODO: Rename this to KubeadmConfigConfigMap
InitConfigurationConfigMap = "kubeadm-config"
// InitConfigurationConfigMapKey specifies in what ConfigMap key the master configuration should be stored
// TODO: Rename this to ClusterConfigurationConfigMapKey, and migrate the value to ClusterConfiguration,
// but still support the old MasterConfiguration naming in earlier versions
InitConfigurationConfigMapKey = "InitConfiguration"
// TODO: This was used in v1.11 with vi1alpha2 config and older. Remove in v1.13
InitConfigurationConfigMapKey = "MasterConfiguration"
// ClusterConfigurationConfigMapKey specifies in what ConfigMap key the cluster configuration should be stored
ClusterConfigurationConfigMapKey = "ClusterConfiguration"
// ClusterStatusConfigMapKey specifies in what ConfigMap key the cluster status should be stored
ClusterStatusConfigMapKey = "ClusterStatus"
// KubeProxyConfigMap specifies in what ConfigMap in the kube-system namespace the kube-proxy configuration should be stored
KubeProxyConfigMap = "kube-proxy"
// KubeProxyConfigMapKey specifies in what ConfigMap key the component config of kube-proxy should be stored
KubeProxyConfigMapKey = "config.conf"
// KubeletBaseConfigurationConfigMapPrefix specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored
KubeletBaseConfigurationConfigMapPrefix = "kubelet-config-"
@ -460,3 +473,8 @@ func GetDNSVersion(dnsType string) string {
return KubeDNSVersion
}
}
// GetKubeletConfigMapName returns the right ConfigMap name for the right branch of k8s
func GetKubeletConfigMapName(k8sVersion *version.Version) string {
return fmt.Sprintf("%s%d.%d", KubeletBaseConfigurationConfigMapPrefix, k8sVersion.Major(), k8sVersion.Minor())
}

8
cmd/kubeadm/app/phases/addons/proxy/manifests.go
View File

@ -22,7 +22,7 @@ const (
kind: ConfigMap
apiVersion: v1
metadata:
name: kube-proxy
name: {{ .ProxyConfigMap }}
namespace: kube-system
labels:
app: kube-proxy
@ -46,7 +46,7 @@ data:
- name: default
user:
tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
config.conf: |-
{{ .ProxyConfigMapKey }}: |-
{{ .ProxyConfig}}
`
@ -79,7 +79,7 @@ spec:
imagePullPolicy: IfNotPresent
command:
- /usr/local/bin/kube-proxy
- --config=/var/lib/kube-proxy/config.conf
- --config=/var/lib/kube-proxy/{{ .ProxyConfigMapKey }}
securityContext:
privileged: true
volumeMounts:
@ -96,7 +96,7 @@ spec:
volumes:
- name: kube-proxy
configMap:
name: kube-proxy
name: {{ .ProxyConfigMap }}
- name: xtables-lock
hostPath:
path: /run/xtables.lock

55
cmd/kubeadm/app/phases/addons/proxy/proxy.go
View File

@ -33,6 +33,7 @@ import (
"k8s.io/kubernetes/cmd/kubeadm/app/images"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
)
const (
@ -65,17 +66,23 @@ func EnsureProxyAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interf
var proxyConfigMapBytes, proxyDaemonSetBytes []byte
proxyConfigMapBytes, err = kubeadmutil.ParseTemplate(KubeProxyConfigMap19,
struct {
MasterEndpoint string
ProxyConfig string
MasterEndpoint string
ProxyConfig string
ProxyConfigMap string
ProxyConfigMapKey string
}{
MasterEndpoint: masterEndpoint,
ProxyConfig: prefixBytes.String(),
MasterEndpoint: masterEndpoint,
ProxyConfig: prefixBytes.String(),
ProxyConfigMap: constants.KubeProxyConfigMap,
ProxyConfigMapKey: constants.KubeProxyConfigMapKey,
})
if err != nil {
return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err)
}
proxyDaemonSetBytes, err = kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ Image string }{
Image: images.GetKubeControlPlaneImage(constants.KubeProxy, &cfg.ClusterConfiguration),
proxyDaemonSetBytes, err = kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ Image, ProxyConfigMap, ProxyConfigMapKey string }{
Image: images.GetKubeControlPlaneImage(constants.KubeProxy, &cfg.ClusterConfiguration),
ProxyConfigMap: constants.KubeProxyConfigMap,
ProxyConfigMapKey: constants.KubeProxyConfigMapKey,
})
if err != nil {
return fmt.Errorf("error when parsing kube-proxy daemonset template: %v", err)
@ -128,7 +135,7 @@ func createKubeProxyAddon(configMapBytes, daemonSetbytes []byte, client clientse
}
func createClusterRoleBindings(client clientset.Interface) error {
return apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{
if err := apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "kubeadm:node-proxier",
},
@ -144,5 +151,39 @@ func createClusterRoleBindings(client clientset.Interface) error {
Namespace: metav1.NamespaceSystem,
},
},
}); err != nil {
return err
}
// Create a role for granting read only access to the kube-proxy component config ConfigMap
if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{
ObjectMeta: metav1.ObjectMeta{
Name: constants.KubeProxyConfigMap,
Namespace: metav1.NamespaceSystem,
},
Rules: []rbac.PolicyRule{
rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(constants.KubeProxyConfigMap).RuleOrDie(),
},
}); err != nil {
return err
}
// Bind the role to bootstrap tokens for allowing fetchConfiguration during join
return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: constants.KubeProxyConfigMap,
Namespace: metav1.NamespaceSystem,
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "Role",
Name: constants.KubeProxyConfigMap,
},
Subjects: []rbac.Subject{
{
Kind: rbac.GroupKind,
Name: constants.NodeBootstrapTokenAuthGroup,
},
},
})
}

14
cmd/kubeadm/app/phases/addons/proxy/proxy_test.go
View File

@ -99,17 +99,21 @@ func TestCompileManifests(t *testing.T) {
{
manifest: KubeProxyConfigMap19,
data: struct {
MasterEndpoint, ProxyConfig string
MasterEndpoint, ProxyConfig, ProxyConfigMap, ProxyConfigMapKey string
}{
MasterEndpoint: "foo",
ProxyConfig: " bindAddress: 0.0.0.0\n clusterCIDR: 192.168.1.1\n enableProfiling: false",
MasterEndpoint: "foo",
ProxyConfig: " bindAddress: 0.0.0.0\n clusterCIDR: 192.168.1.1\n enableProfiling: false",
ProxyConfigMap: "bar",
ProxyConfigMapKey: "baz",
},
expected: true,
},
{
manifest: KubeProxyDaemonSet19,
data: struct{ Image string }{
Image: "foo",
data: struct{ Image, ProxyConfigMap, ProxyConfigMapKey string }{
Image: "foo",
ProxyConfigMap: "bar",
ProxyConfigMapKey: "baz",
},
expected: true,
},

2
cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go
View File

@ -174,7 +174,7 @@ func getKubeConfigSpecs(cfg *kubeadmapi.InitConfiguration) (map[string]*kubeConf
kubeadmconstants.KubeletKubeConfigFileName: {
CACert: caCert,
APIServer: masterEndpoint,
ClientName: fmt.Sprintf("system:node:%s", cfg.NodeRegistration.Name),
ClientName: fmt.Sprintf("%s%s", kubeadmconstants.NodesUserPrefix, cfg.NodeRegistration.Name),
ClientCertAuth: &clientCertAuth{
CAKey: caKey,
Organizations: []string{kubeadmconstants.NodesGroup},

2
cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go
View File

@ -119,7 +119,7 @@ func TestGetKubeConfigSpecs(t *testing.T) {
},
{
kubeConfigFile: kubeadmconstants.KubeletKubeConfigFileName,
clientName: fmt.Sprintf("system:node:%s", cfg.NodeRegistration.Name),
clientName: fmt.Sprintf("%s%s", kubeadmconstants.NodesUserPrefix, cfg.NodeRegistration.Name),
organizations: []string{kubeadmconstants.NodesGroup},
},
{

11
cmd/kubeadm/app/phases/kubelet/config.go
View File

@ -56,7 +56,7 @@ func CreateConfigMap(cfg *kubeadmapi.InitConfiguration, client clientset.Interfa
return err
}
configMapName := configMapName(k8sVersion)
configMapName := kubeadmconstants.GetKubeletConfigMapName(k8sVersion)
fmt.Printf("[kubelet] Creating a ConfigMap %q in namespace %s with the configuration for the kubelets in the cluster\n", configMapName, metav1.NamespaceSystem)
kubeletBytes, err := getConfigBytes(cfg.ComponentConfigs.Kubelet)
@ -90,7 +90,7 @@ func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Ve
Namespace: metav1.NamespaceSystem,
},
Rules: []rbac.PolicyRule{
rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(configMapName(k8sVersion)).RuleOrDie(),
rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(kubeadmconstants.GetKubeletConfigMapName(k8sVersion)).RuleOrDie(),
},
}); err != nil {
return err
@ -124,7 +124,7 @@ func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Ve
func DownloadConfig(client clientset.Interface, kubeletVersion *version.Version, kubeletDir string) error {
// Download the ConfigMap from the cluster based on what version the kubelet is
configMapName := configMapName(kubeletVersion)
configMapName := kubeadmconstants.GetKubeletConfigMapName(kubeletVersion)
fmt.Printf("[kubelet] Downloading configuration for the kubelet from the %q ConfigMap in the %s namespace\n",
configMapName, metav1.NamespaceSystem)
@ -142,11 +142,6 @@ func DownloadConfig(client clientset.Interface, kubeletVersion *version.Version,
return writeConfigBytesToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey]), kubeletDir)
}
// configMapName returns the right ConfigMap name for the right branch of k8s
func configMapName(k8sVersion *version.Version) string {
return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigurationConfigMapPrefix, k8sVersion.Major(), k8sVersion.Minor())
}
// configMapRBACName returns the name for the Role/RoleBinding for the kubelet config configmap for the right branch of k8s
func configMapRBACName(k8sVersion *version.Version) string {
return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigMapRolePrefix, k8sVersion.Major(), k8sVersion.Minor())

2
cmd/kubeadm/app/phases/kubelet/dynamic.go
View File

@ -32,7 +32,7 @@ import (
// This func is ONLY run if the user enables the `DynamicKubeletConfig` feature gate, which is by default off
func EnableDynamicConfigForNode(client clientset.Interface, nodeName string, kubeletVersion *version.Version) error {
configMapName := configMapName(kubeletVersion)
configMapName := kubeadmconstants.GetKubeletConfigMapName(kubeletVersion)
fmt.Printf("[kubelet] Enabling Dynamic Kubelet Config for Node %q; config sourced from ConfigMap %q in namespace %s\n",
nodeName, configMapName, metav1.NamespaceSystem)
fmt.Println("[kubelet] WARNING: The Dynamic Kubelet Config feature is beta, but off by default. It hasn't been well-tested yet at this stage, use with caution.")

61
cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go
View File

@ -21,9 +21,12 @@ import (
"k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
@ -39,20 +42,36 @@ const (
// UploadConfiguration saves the InitConfiguration used for later reference (when upgrading for instance)
func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error {
fmt.Printf("[uploadconfig] storing the configuration used in ConfigMap %q in the %q Namespace\n", kubeadmconstants.InitConfigurationConfigMap, metav1.NamespaceSystem)
// Prepare the ClusterConfiguration for upload
// The components store their config in their own ConfigMaps, then reset the .ComponentConfig struct;
// We don't want to mutate the cfg itself, so create a copy of it using .DeepCopy of it first
clusterConfigToUpload := cfg.ClusterConfiguration.DeepCopy()
// TODO: Reset the .ComponentConfig struct like this:
// cfgToUpload.ComponentConfigs = kubeadmapi.ComponentConfigs{}
// in order to not upload any other components' config to the kubeadm-config
// ConfigMap. The components store their config in their own ConfigMaps.
// Before this line can be uncommented util/config.loadConfigurationBytes()
// needs to support reading the different components' ConfigMaps first.
clusterConfigurationToUpload := cfg.ClusterConfiguration.DeepCopy()
clusterConfigurationToUpload.ComponentConfigs = kubeadmapi.ComponentConfigs{}
// Marshal the object into YAML
cfgYaml, err := configutil.MarshalKubeadmConfigObject(clusterConfigToUpload)
// Marshal the ClusterConfiguration into YAML
clusterConfigurationYaml, err := configutil.MarshalKubeadmConfigObject(clusterConfigurationToUpload)
if err != nil {
return err
}
// Prepare the ClusterStatus for upload
// Gets the current cluster status
// TODO: use configmap locks on this object on the get before the update.
clusterStatus, err := getClusterStatus(client)
if err != nil {
return err
}
// Updates the ClusterStatus with the current control plane instance
if clusterStatus.APIEndpoints == nil {
clusterStatus.APIEndpoints = map[string]kubeadmapi.APIEndpoint{}
}
clusterStatus.APIEndpoints[cfg.NodeRegistration.Name] = cfg.APIEndpoint
// Marshal the ClusterStatus back into into YAML
clusterStatusYaml, err := configutil.MarshalKubeadmConfigObject(clusterStatus)
if err != nil {
return err
}
@ -63,7 +82,8 @@ func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Int
Namespace: metav1.NamespaceSystem,
},
Data: map[string]string{
kubeadmconstants.InitConfigurationConfigMapKey: string(cfgYaml),
kubeadmconstants.ClusterConfigurationConfigMapKey: string(clusterConfigurationYaml),
kubeadmconstants.ClusterStatusConfigMapKey: string(clusterStatusYaml),
},
})
if err != nil {
@ -109,3 +129,22 @@ func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Int
},
})
}
func getClusterStatus(client clientset.Interface) (*kubeadmapi.ClusterStatus, error) {
obj := &kubeadmapi.ClusterStatus{}
// Read the ConfigMap from the cluster
configMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.InitConfigurationConfigMap, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
return obj, nil
}
if err != nil {
return nil, err
}
// Decode the file content using the componentconfig Codecs that knows about all APIs
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(configMap.Data[kubeadmconstants.ClusterStatusConfigMapKey]), obj); err != nil {
return nil, err
}
return obj, nil
}

86
cmd/kubeadm/app/phases/uploadconfig/uploadconfig_test.go
View File

@ -20,15 +20,18 @@ import (
"reflect"
"testing"
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
clientsetfake "k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
)
@ -89,6 +92,12 @@ func TestUploadConfiguration(t *testing.T) {
t2.Fatalf("UploadConfiguration() error = %v", err)
}
status := &kubeadmapi.ClusterStatus{
APIEndpoints: map[string]kubeadmapi.APIEndpoint{
"node-foo": cfg.APIEndpoint,
},
}
client := clientsetfake.NewSimpleClientset()
if tt.errOnCreate != nil {
client.PrependReactor("create", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
@ -114,9 +123,9 @@ func TestUploadConfiguration(t *testing.T) {
if err != nil {
t2.Fatalf("Fail to query ConfigMap error = %v", err)
}
configData := masterCfg.Data[kubeadmconstants.InitConfigurationConfigMapKey]
configData := masterCfg.Data[kubeadmconstants.ClusterConfigurationConfigMapKey]
if configData == "" {
t2.Fatalf("Fail to find ConfigMap key")
t2.Fatal("Fail to find ClusterConfigurationConfigMapKey key")
}
decodedCfg := &kubeadmapi.ClusterConfiguration{}
@ -127,7 +136,80 @@ func TestUploadConfiguration(t *testing.T) {
if !reflect.DeepEqual(decodedCfg, &cfg.ClusterConfiguration) {
t2.Errorf("the initial and decoded ClusterConfiguration didn't match")
}
statusData := masterCfg.Data[kubeadmconstants.ClusterStatusConfigMapKey]
if statusData == "" {
t2.Fatal("failed to find ClusterStatusConfigMapKey key")
}
decodedStatus := &kubeadmapi.ClusterStatus{}
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(statusData), decodedStatus); err != nil {
t2.Fatalf("unable to decode status from bytes: %v", err)
}
if !reflect.DeepEqual(decodedStatus, status) {
t2.Error("the initial and decoded ClusterStatus didn't match")
}
}
})
}
}
func TestGetClusterStatus(t *testing.T) {
var tests = []struct {
name string
clusterStatus *kubeadmapi.ClusterStatus
expectedClusterEndpoints int
}{
{
name: "return empty ClusterStatus if cluster kubeadm-config doesn't exist (e.g init)",
expectedClusterEndpoints: 0,
},
{
name: "return ClusterStatus if cluster kubeadm-config exist (e.g upgrade)",
clusterStatus: &kubeadmapi.ClusterStatus{
APIEndpoints: map[string]kubeadmapi.APIEndpoint{
"dummy": {AdvertiseAddress: "1.2.3.4", BindPort: 1234},
},
},
expectedClusterEndpoints: 1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := clientsetfake.NewSimpleClientset()
if tt.clusterStatus != nil {
createConfigMapWithStatus(tt.clusterStatus, client)
}
actual, err := getClusterStatus(client)
if err != nil {
t.Error("GetClusterStatus returned unexpected error")
return
}
if tt.expectedClusterEndpoints != len(actual.APIEndpoints) {
t.Error("actual ClusterStatus doesn't return expected endpoints")
}
})
}
}
// createConfigMapWithStatus create a ConfigMap with ClusterStatus for TestGetClusterStatus
func createConfigMapWithStatus(statusToCreate *kubeadmapi.ClusterStatus, client clientset.Interface) error {
statusYaml, err := configutil.MarshalKubeadmConfigObject(statusToCreate)
if err != nil {
return err
}
return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: kubeadmconstants.InitConfigurationConfigMap,
Namespace: metav1.NamespaceSystem,
},
Data: map[string]string{
kubeadmconstants.ClusterStatusConfigMapKey: string(statusYaml),
},
})
}

238
cmd/kubeadm/app/util/config/cluster.go
View File

@ -17,60 +17,244 @@ limitations under the License.
package config
import (
"crypto/x509"
"errors"
"fmt"
"io"
"io/ioutil"
"path/filepath"
"strings"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
certutil "k8s.io/client-go/util/cert"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/pkg/util/version"
)
// FetchConfigFromFileOrCluster fetches configuration required for upgrading your cluster from a file (which has precedence) or a ConfigMap in the cluster
// TODO: This func should be renamed FetchClusterConfigFromFileOrCluster, and return a ClusterConfiguration instead of an InitConfiguration
func FetchConfigFromFileOrCluster(client clientset.Interface, w io.Writer, logPrefix, cfgPath string) (*kubeadmapi.InitConfiguration, error) {
func FetchConfigFromFileOrCluster(client clientset.Interface, w io.Writer, logPrefix, cfgPath string, newControlPlane bool) (*kubeadmapi.InitConfiguration, error) {
// Load the configuration from a file or the cluster
configBytes, err := loadConfigurationBytes(client, w, logPrefix, cfgPath)
initcfg, err := loadConfiguration(client, w, logPrefix, cfgPath, newControlPlane)
if err != nil {
return nil, err
}
// Unmarshal the versioned configuration populated from the file or ConfigMap, convert it to the internal API types, then default and validate
initcfg, err := BytesToInternalConfig(configBytes)
if err != nil {
return nil, err
}
//TODO: this will be reviewed in the following PR for reading/storing the kubeadm-config ConfigMap
// Apply dynamic defaults
if err := SetInitDynamicDefaults(initcfg); err != nil {
return nil, err
}
return initcfg, err
}
// loadConfigurationBytes loads the configuration byte slice from either a file or the cluster ConfigMap
func loadConfigurationBytes(client clientset.Interface, w io.Writer, logPrefix, cfgPath string) ([]byte, error) {
// loadConfiguration loads the configuration byte slice from either a file or the cluster ConfigMap
func loadConfiguration(client clientset.Interface, w io.Writer, logPrefix, cfgPath string, newControlPlane bool) (*kubeadmapi.InitConfiguration, error) {
// The config file has the highest priority
if cfgPath != "" {
fmt.Fprintf(w, "[%s] Reading configuration options from a file: %s\n", logPrefix, cfgPath)
return ioutil.ReadFile(cfgPath)
return loadInitConfigurationFromFile(cfgPath)
}
fmt.Fprintf(w, "[%s] Reading configuration from the cluster...\n", logPrefix)
// TODO: This code should support reading the MasterConfiguration key as well for backwards-compat
// Also, the key really should be ClusterConfiguration...
configMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(constants.InitConfigurationConfigMap, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
// Return the apierror directly so the caller of this function can know what type of error occurred and act based on that
return []byte{}, err
} else if err != nil {
return []byte{}, fmt.Errorf("an unexpected error happened when trying to get the ConfigMap %q in the %s namespace: %v", constants.InitConfigurationConfigMap, metav1.NamespaceSystem, err)
}
// TODO: Load the kube-proxy and kubelet ComponentConfig ConfigMaps here as different YAML documents and append to the byte slice
fmt.Fprintf(w, "[%s] FYI: You can look at this config file with 'kubectl -n %s get cm %s -oyaml'\n", logPrefix, metav1.NamespaceSystem, constants.InitConfigurationConfigMap)
return []byte(configMap.Data[constants.InitConfigurationConfigMapKey]), nil
return getInitConfigurationFromCluster(constants.KubernetesDir, client, newControlPlane)
}
func loadInitConfigurationFromFile(cfgPath string) (*kubeadmapi.InitConfiguration, error) {
configBytes, err := ioutil.ReadFile(cfgPath)
if err != nil {
return nil, err
}
// Unmarshal the versioned configuration populated from the file,
// convert it to the internal API types, then default and validate
// NB the file can be one of
// - a single YAML, with a v1alpha2.MasterConfiguration object (with embedded component configs)
// - multiple YAML, with a combination of
// - a YAML with a v1alpha3.InitConfiguration object
// - a YAML with a v1alpha3.ClusterConfiguration object (without embedded component configs)
// - separated YAML for components configs
initcfg, err := BytesToInternalConfig(configBytes)
if err != nil {
return nil, err
}
return initcfg, nil
}
func getInitConfigurationFromCluster(kubeconfigDir string, client clientset.Interface, newControlPlane bool) (*kubeadmapi.InitConfiguration, error) {
// TODO: This code should support reading the MasterConfiguration key as well for backwards-compat
// Also, the config map really should be KubeadmConfigConfigMap...
configMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(constants.InitConfigurationConfigMap, metav1.GetOptions{})
if err != nil {
return nil, err
}
// TODO: remove in V1.13
// If InitConfigurationConfigMapKey exist, the kubeadm-config was created with v1.11
if _, ok := configMap.Data[constants.InitConfigurationConfigMapKey]; ok {
return getInitConfigurationFromConfigMapV11(configMap.Data)
}
return getInitConfigurationFromConfigMaps(kubeconfigDir, client, configMap.Data, newControlPlane)
}
func getInitConfigurationFromConfigMapV11(data map[string]string) (*kubeadmapi.InitConfiguration, error) {
configBytes := []byte(data[constants.InitConfigurationConfigMapKey])
// Unmarshal the versioned configuration populated from the file,
// convert it to the internal API types, then default and validate
// NB the config map created with v11 is a single YAML, with a v1alpha2.MasterConfiguration object (with embedded component configs)
initcfg, err := BytesToInternalConfig(configBytes)
if err != nil {
return nil, err
}
return initcfg, nil
}
func getInitConfigurationFromConfigMaps(kubeconfigDir string, client clientset.Interface, data map[string]string, newControlPlane bool) (*kubeadmapi.InitConfiguration, error) {
// In case of cluster crated with v1.12 InitConfiguration is composed with data from different places
initcfg := &kubeadmapi.InitConfiguration{}
// gets ClusterConfiguration from kubeadm-config
clusterConfigurationData, ok := data[constants.ClusterConfigurationConfigMapKey]
if !ok {
return nil, fmt.Errorf("unexpected error when reading kubeadm-config ConfigMap: %s key value pair missing", constants.ClusterConfigurationConfigMapKey)
}
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigurationData), &initcfg.ClusterConfiguration); err != nil {
return nil, err
}
// gets the component configs from the corresponding config maps
if err := getComponentConfigs(client, &initcfg.ClusterConfiguration); err != nil {
return nil, err
}
// if this isn't a new controlplane instance (e.g. in case of kubeadm upgrades)
// get nodes specific information as well
if !newControlPlane {
// gets the nodeRegistration for the current from the node object
if err := getNodeRegistration(kubeconfigDir, client, &initcfg.NodeRegistration); err != nil {
return nil, err
}
// gets the APIEndpoint for the current node from then ClusterStatus in the kubeadm-config ConfigMap
if err := getAPIEndpoint(data, initcfg.NodeRegistration.Name, &initcfg.APIEndpoint); err != nil {
return nil, err
}
}
return initcfg, nil
}
// getNodeRegistration returns the nodeRegistration for the current node
func getNodeRegistration(kubeconfigDir string, client clientset.Interface, nodeRegistration *kubeadmapi.NodeRegistrationOptions) error {
// gets the name of the current node
nodeName, err := getNodeNameFromKubeletConfig(kubeconfigDir)
if err != nil {
return err
}
// gets the corresponding node and retrives attributes stored there.
node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
if err != nil {
return err
}
criSocket, ok := node.ObjectMeta.Annotations[constants.AnnotationKubeadmCRISocket]
if !ok {
return fmt.Errorf("Node %s doesn't have %s annotation", nodeName, constants.AnnotationKubeadmCRISocket)
}
// returns the nodeRegistration attributes
nodeRegistration.Name = nodeName
nodeRegistration.CRISocket = criSocket
nodeRegistration.Taints = node.Spec.Taints
// NB. currently nodeRegistration.KubeletExtraArgs isn't stored at node level but only in the kubeadm-flags.env
// that isn't modified during upgrades
// in future we might reconsider this thus enabling changes to the kubeadm-flags.env during upgrades as well
return nil
}
// getNodeNameFromConfig gets the node name from a kubelet config file
// TODO: in future we want to switch to a more canonical way for doing this e.g. by having this
// information in the local kubelet config.yaml
func getNodeNameFromKubeletConfig(kubeconfigDir string) (string, error) {
// loads the kubelet.conf file
fileName := filepath.Join(kubeconfigDir, constants.KubeletKubeConfigFileName)
config, err := clientcmd.LoadFromFile(fileName)
if err != nil {
return "", err
}
// gets the info about the current user
authInfo := config.AuthInfos[config.Contexts[config.CurrentContext].AuthInfo]
// gets the X509 certificate with current user credentials
var certs []*x509.Certificate
if len(authInfo.ClientCertificateData) > 0 {
// if the config file uses an embedded x509 certificate (e.g. kubelet.conf created by kubeadm), parse it
if certs, err = certutil.ParseCertsPEM(authInfo.ClientCertificateData); err != nil {
return "", err
}
} else if len(authInfo.ClientCertificate) > 0 {
// if the config file links an external x509 certificate (e.g. kubelet.conf created by TLS bootstrap), load it
if certs, err = certutil.CertsFromFile(authInfo.ClientCertificate); err != nil {
return "", err
}
} else {
return "", errors.New("Invalid kubelet.conf. X509 certificate expected")
}
// We are only putting one certificate in the certificate pem file, so it's safe to just pick the first one
// TODO: Support multiple certs here in order to be able to rotate certs
cert := certs[0]
// gets the node name from the certificate common name
return strings.TrimPrefix(cert.Subject.CommonName, constants.NodesUserPrefix), nil
}
// getAPIEndpoint returns the APIEndpoint for the current node
func getAPIEndpoint(data map[string]string, nodeName string, apiEndpoint *kubeadmapi.APIEndpoint) error {
// gets the ClusterStatus from kubeadm-config
clusterStatusData, ok := data[constants.ClusterStatusConfigMapKey]
if !ok {
return fmt.Errorf("unexpected error when reading kubeadm-config ConfigMap: %s key value pair missing", constants.ClusterStatusConfigMapKey)
}
clusterStatus := &kubeadmapi.ClusterStatus{}
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterStatusData), clusterStatus); err != nil {
return err
}
// gets the APIEndpoint for the current machine from the ClusterStatus
e, ok := clusterStatus.APIEndpoints[nodeName]
if !ok {
return errors.New("failed to get APIEndpoint information for this node")
}
apiEndpoint.AdvertiseAddress = e.AdvertiseAddress
apiEndpoint.BindPort = e.BindPort
return nil
}
// getComponentConfigs gets the component configs from the corresponding config maps
func getComponentConfigs(client clientset.Interface, clusterConfiguration *kubeadmapi.ClusterConfiguration) error {
// some config maps is versioned, so we need the KubernetesVersion for getting the right config map
k8sVersion := version.MustParseGeneric(clusterConfiguration.KubernetesVersion)
for kind, registration := range componentconfigs.Known {
obj, err := registration.GetFromConfigMap(client, k8sVersion)
if err != nil {
return err
}
if ok := registration.SetToInternalConfig(obj, clusterConfiguration); !ok {
return fmt.Errorf("couldn't save componentconfig value for kind %q", string(kind))
}
}
return nil
}

766
cmd/kubeadm/app/util/config/cluster_test.go
View File

@ -17,7 +17,10 @@ limitations under the License.
package config
import (
"bytes"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
@ -25,176 +28,657 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
clientsetfake "k8s.io/client-go/kubernetes/fake"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
"k8s.io/kubernetes/pkg/util/version"
)
func TestFetchConfigFromFileOrCluster(t *testing.T) {
var k8sVersionString = "v1.12.0"
var k8sVersion = version.MustParseGeneric(k8sVersionString)
var nodeName = "mynode"
var cfgFiles = map[string][]byte{
"MasterConfiguration_v1alpha2": []byte(`
apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
kubernetesVersion: ` + k8sVersionString + `
api:
advertiseAddress: 1.2.3.4
bindPort: 1234
`),
"InitConfiguration_v1alpha3": []byte(`
apiVersion: kubeadm.k8s.io/v1alpha3
kind: InitConfiguration
`),
"ClusterConfiguration_v1alpha3": []byte(`
apiVersion: kubeadm.k8s.io/v1alpha3
kind: ClusterConfiguration
kubernetesVersion: ` + k8sVersionString + `
`),
"ClusterStatus_v1alpha3": []byte(`
apiVersion: kubeadm.k8s.io/v1alpha3
kind: ClusterStatus
apiEndpoints:
` + nodeName + `:
advertiseAddress: 1.2.3.4
bindPort: 1234
`),
"ClusterStatus_v1alpha3_Without_APIEndpoints": []byte(`
apiVersion: kubeadm.k8s.io/v1alpha3
kind: ClusterStatus
`),
"Kube-proxy_componentconfig": []byte(`
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
`),
"Kubelet_componentconfig": []byte(`
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
`),
}
var kubeletConfFiles = map[string][]byte{
"withoutX509Cert": []byte(`
apiVersion: v1
clusters:
- cluster:
server: https://10.0.2.15:6443
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: system:node:mynode
name: system:node:mynode@kubernetes
current-context: system:node:mynode@kubernetes
kind: Config
preferences: {}
users:
- name: system:node:mynode
user:
`),
"configWithEmbeddedCert": []byte(`
apiVersion: v1
clusters:
- cluster:
server: https://10.0.2.15:6443
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: system:node:mynode
name: system:node:mynode@kubernetes
current-context: system:node:mynode@kubernetes
kind: Config
preferences: {}
users:
- name: system:node:mynode
user:
client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM4akNDQWRxZ0F3SUJBZ0lJQWl3VURhYk5vZ1F3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4T0RBNU1ERXhOVE14TWpaYUZ3MHhPVEE1TURFeE5qQXhOVGxhTURReApGVEFUQmdOVkJBb1RESE41YzNSbGJUcHViMlJsY3pFYk1Ca0dBMVVFQXhNU2MzbHpkR1Z0T201dlpHVTZiWGx1CmIyUmxNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQWs2UXUzeStyNEZYUzZ4VkoKWU1vNE9kSkt3R1d1MHY4TEJIUnhhOUhvVHo1RXZLQnB1OVJoemt5dStUaFczb0xta2ZTRmNJcitHa0M5MW0zOApFelRmVE5JY2dsL0V5YkpmR1QvdGdUazZYd1kxY1UrUUdmSEFNNTBCVzFXTFVHc25CSllJZjA5eENnZTVoTkxLCnREeUJOWWNQZzg1bUJpOU9CNFJ2UlgyQVFRMjJwZ0xrQUpJWklOU0FEdUFrODN2b051SXM2YVY2bHBkR2Vva3YKdDlpTFdNR3p3a3ZTZUZQTlNGeWZ3Q055eENjb1FDQUNtSnJRS3NoeUE2bWNzdVhORWVXdlRQdVVFSWZPVFl4dwpxdkszRVBOK0xUYlA2anhUMWtTcFJUOSt4Z29uSlFhU0RsbUNBd20zRGJkSVppWUt3R2ppMkxKL0kvYWc0cTlzCjNLb0J2UUlEQVFBQm95Y3dKVEFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFLcVVrU21jdW85OG5EK015b005VFdEV0pyTndySXpQTUNqRQpCSkdyREhVaHIwcEZlRjc0RHViODNzRXlaNjFxNUVQd2Y0enNLSzdzdDRUTzZhcE9pZWJYVmN3dnZoa09HQ2dFCmFVdGNOMjFhUGxtU0tOd0c4ai8yK3ZhbU80bGplK1NnZzRUUVB0eDZWejh5VXN2RFhxSUZycjNNd1gzSDA1RW4KWXAzN05JYkhKbGxHUW5LVHA5aTg5aXF4WXVhSERqZldiVHlEY3B5NldNVjdVaFYvY1plc3lGL0NBamNHd1V6YgowRlo5bW5tMnFONlBGWHZ4RmdMSGFWZzN2SVVCbkNmVVVyY1BDNE94VFNPK21aUmUxazh3eUFpVWovSk0rZllvCkcrMi9sbThUYVZqb1U3Rmk1S2E1RzVIWTJHTGFSN1ArSXhZY3JNSENsNjJZN1JxY3JuYz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
`),
"configWithLinkedCert": []byte(`
apiVersion: v1
clusters:
- cluster:
server: https://10.0.2.15:6443
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: system:node:mynode
name: system:node:mynode@kubernetes
current-context: system:node:mynode@kubernetes
kind: Config
preferences: {}
users:
- name: system:node:mynode
user:
client-certificate: kubelet.pem
`),
}
var pemFiles = map[string][]byte{
"mynode.pem": []byte(`
-----BEGIN CERTIFICATE-----
MIIC8jCCAdqgAwIBAgIIAiwUDabNogQwDQYJKoZIhvcNAQELBQAwFTETMBEGA1UE
AxMKa3ViZXJuZXRlczAeFw0xODA5MDExNTMxMjZaFw0xOTA5MDExNjAxNTlaMDQx
FTATBgNVBAoTDHN5c3RlbTpub2RlczEbMBkGA1UEAxMSc3lzdGVtOm5vZGU6bXlu
b2RlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk6Qu3y+r4FXS6xVJ
YMo4OdJKwGWu0v8LBHRxa9HoTz5EvKBpu9Rhzkyu+ThW3oLmkfSFcIr+GkC91m38
EzTfTNIcgl/EybJfGT/tgTk6XwY1cU+QGfHAM50BW1WLUGsnBJYIf09xCge5hNLK
tDyBNYcPg85mBi9OB4RvRX2AQQ22pgLkAJIZINSADuAk83voNuIs6aV6lpdGeokv
t9iLWMGzwkvSeFPNSFyfwCNyxCcoQCACmJrQKshyA6mcsuXNEeWvTPuUEIfOTYxw
qvK3EPN+LTbP6jxT1kSpRT9+xgonJQaSDlmCAwm3DbdIZiYKwGji2LJ/I/ag4q9s
3KoBvQIDAQABoycwJTAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUH
AwIwDQYJKoZIhvcNAQELBQADggEBAKqUkSmcuo98nD+MyoM9TWDWJrNwrIzPMCjE
BJGrDHUhr0pFeF74Dub83sEyZ61q5EPwf4zsKK7st4TO6apOiebXVcwvvhkOGCgE
aUtcN21aPlmSKNwG8j/2+vamO4lje+Sgg4TQPtx6Vz8yUsvDXqIFrr3MwX3H05En
Yp37NIbHJllGQnKTp9i89iqxYuaHDjfWbTyDcpy6WMV7UhV/cZesyF/CAjcGwUzb
0FZ9mnm2qN6PFXvxFgLHaVg3vIUBnCfUUrcPC4OxTSO+mZRe1k8wyAiUj/JM+fYo
G+2/lm8TaVjoU7Fi5Ka5G5HY2GLaR7P+IxYcrMHCl62Y7Rqcrnc=
-----END CERTIFICATE-----
`),
}
func TestLoadInitConfigurationFromFile(t *testing.T) {
// Create temp folder for the test case
tmpdir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Couldn't create tmpdir")
}
defer os.RemoveAll(tmpdir)
var tests = []struct {
name string
cfgPath string
testCfg *kubeadmapi.InitConfiguration
expectErr string
name string
fileContents []byte
}{
{
name: "fetch valid config from configMap",
testCfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{
AdvertiseAddress: "1.2.3.4",
BindPort: 6443,
},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
KubernetesVersion: "v1.10.3",
Etcd: kubeadm.Etcd{
Local: &kubeadm.LocalEtcd{
DataDir: "/some/path",
},
},
Networking: kubeadm.Networking{
ServiceSubnet: "10.96.0.1/12",
DNSDomain: "cluster.local",
PodSubnet: "10.0.1.15/16",
},
CertificatesDir: "/some/other/cert/dir",
},
BootstrapTokens: []kubeadm.BootstrapToken{
{
Token: &kubeadm.BootstrapTokenString{
ID: "abcdef",
Secret: "abcdef0123456789",
},
},
},
NodeRegistration: kubeadm.NodeRegistrationOptions{
Name: "node-foo",
CRISocket: "/var/run/custom-cri.sock",
},
},
name: "v1alpha2.MasterConfiguration",
fileContents: cfgFiles["MasterConfiguration_v1alpha2"],
},
{
name: "fetch invalid config from configMap",
testCfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{
AdvertiseAddress: "1.2.3.4",
BindPort: 6443,
},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
KubernetesVersion: "v1.10.3",
Etcd: kubeadm.Etcd{
Local: &kubeadm.LocalEtcd{
DataDir: "/some/path",
},
},
Networking: kubeadm.Networking{
ServiceSubnet: "10.96.0.1/12",
DNSDomain: "cluster.local",
PodSubnet: "10.0.1.15", // wrong
},
CertificatesDir: "/some/other/cert/dir",
},
BootstrapTokens: []kubeadm.BootstrapToken{
{
Token: &kubeadm.BootstrapTokenString{
ID: "abcdef",
Secret: "abcdef0123456789",
},
},
},
NodeRegistration: kubeadm.NodeRegistrationOptions{
Name: "node-foo",
CRISocket: "/var/run/custom-cri.sock",
},
},
expectErr: "couldn't parse subnet",
name: "v1alpha3.partial1",
fileContents: cfgFiles["InitConfiguration_v1alpha3"],
},
{
name: "fetch valid config from cfgPath",
cfgPath: "testdata/conversion/master/v1alpha3.yaml",
testCfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{
AdvertiseAddress: "1.2.3.4",
BindPort: 6443,
},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
KubernetesVersion: "v1.10.3",
Etcd: kubeadm.Etcd{
Local: &kubeadm.LocalEtcd{
DataDir: "/some/path",
},
},
Networking: kubeadm.Networking{
ServiceSubnet: "10.96.0.1/12",
DNSDomain: "cluster.local",
PodSubnet: "10.0.1.15",
},
CertificatesDir: "/some/other/cert/dir",
},
BootstrapTokens: []kubeadm.BootstrapToken{
{
Token: &kubeadm.BootstrapTokenString{
ID: "abcdef",
Secret: "abcdef0123456789",
},
},
},
NodeRegistration: kubeadm.NodeRegistrationOptions{
Name: "node-foo",
CRISocket: "/var/run/custom-cri.sock",
},
},
name: "v1alpha3.partial2",
fileContents: cfgFiles["ClusterConfiguration_v1alpha3"],
},
{
name: "fetch invalid config from cfgPath",
cfgPath: "testdata/validation/invalid_mastercfg.yaml",
expectErr: "was not of the form",
},
{
name: "fetch config from not exist cfgPath",
cfgPath: "doesnotexist.yaml",
expectErr: "no such file or directory",
},
{
name: "fetch config when no configMap and no cfgPath",
expectErr: "not found",
name: "v1alpha3.full",
fileContents: bytes.Join([][]byte{
cfgFiles["InitConfiguration_v1alpha3"],
cfgFiles["ClusterConfiguration_v1alpha3"],
cfgFiles["Kube-proxy_componentconfig"],
cfgFiles["Kubelet_componentconfig"],
}, []byte(kubeadmconstants.YAMLDocumentSeparator)),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := clientsetfake.NewSimpleClientset()
if tt.testCfg != nil {
if err := createConfigMapWithCfg(tt.testCfg, client); err != nil {
t.Errorf("UploadConfiguration failed err: %v", err)
}
}
_, err := FetchConfigFromFileOrCluster(client, os.Stdout, "upgrade/config", tt.cfgPath)
for _, rt := range tests {
t.Run(rt.name, func(t2 *testing.T) {
cfgPath := filepath.Join(tmpdir, rt.name)
err := ioutil.WriteFile(cfgPath, rt.fileContents, 0644)
if err != nil {
if len(tt.expectErr) == 0 {
t.Fatalf("expected no err, but got err: %v", err)
} else if !strings.Contains(err.Error(), tt.expectErr) {
t.Errorf("expected contain err: %v, but got err: %v", tt.expectErr, err)
}
t.Errorf("Couldn't create file")
return
}
obj, err := loadInitConfigurationFromFile(cfgPath)
if err != nil {
t.Errorf("Error reading file: %v", err)
return
}
if obj == nil {
t.Errorf("Unexpected nil return value")
}
})
}
}
// createConfigMapWithCfg create a ConfigMap with InitConfiguration for TestFetchConfigFromFileOrCluster
func createConfigMapWithCfg(cfgToCreate *kubeadmapi.InitConfiguration, client clientset.Interface) error {
cfgYaml, err := MarshalKubeadmConfigObject(cfgToCreate)
func TestGetNodeNameFromKubeletConfig(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "")
if err != nil {
return err
t.Fatalf("Couldn't create tmpdir")
}
defer os.RemoveAll(tmpdir)
var tests = []struct {
name string
kubeconfigContent []byte
pemContent []byte
expectedError bool
}{
{
name: "valid - with embedded cert",
kubeconfigContent: kubeletConfFiles["configWithEmbeddedCert"],
},
{
name: "invalid - linked cert missing",
kubeconfigContent: kubeletConfFiles["configWithLinkedCert"],
expectedError: true,
},
{
name: "valid - with linked cert",
kubeconfigContent: kubeletConfFiles["configWithLinkedCert"],
pemContent: pemFiles["mynode.pem"],
},
{
name: "invalid - without embedded or linked X509Cert",
kubeconfigContent: kubeletConfFiles["withoutX509Cert"],
expectedError: true,
},
}
for _, rt := range tests {
t.Run(rt.name, func(t2 *testing.T) {
if len(rt.pemContent) > 0 {
pemPath := filepath.Join(tmpdir, "kubelet.pem")
err := ioutil.WriteFile(pemPath, rt.pemContent, 0644)
if err != nil {
t.Errorf("Couldn't create pem file: %v", err)
return
}
rt.kubeconfigContent = []byte(strings.Replace(string(rt.kubeconfigContent), "kubelet.pem", pemPath, -1))
}
kubeconfigPath := filepath.Join(tmpdir, kubeadmconstants.KubeletKubeConfigFileName)
err := ioutil.WriteFile(kubeconfigPath, rt.kubeconfigContent, 0644)
if err != nil {
t.Errorf("Couldn't create kubeconfig: %v", err)
return
}
name, err := getNodeNameFromKubeletConfig(tmpdir)
if rt.expectedError != (err != nil) {
t.Errorf("unexpected return err from getNodeRegistration: %v", err)
return
}
if rt.expectedError {
return
}
if name != nodeName {
t.Errorf("invalid name")
}
})
}
}
func TestGetNodeRegistration(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Couldn't create tmpdir")
}
defer os.RemoveAll(tmpdir)
var tests = []struct {
name string
fileContents []byte
node *v1.Node
expectedError bool
}{
{
name: "invalid - no kubelet.conf",
expectedError: true,
},
{
name: "valid",
fileContents: kubeletConfFiles["configWithEmbeddedCert"],
node: &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: nodeName,
Annotations: map[string]string{
kubeadmconstants.AnnotationKubeadmCRISocket: "myCRIsocket",
},
},
Spec: v1.NodeSpec{
Taints: []v1.Taint{kubeadmconstants.MasterTaint},
},
},
},
{
name: "invalid - no node",
fileContents: kubeletConfFiles["configWithEmbeddedCert"],
expectedError: true,
},
}
for _, rt := range tests {
t.Run(rt.name, func(t2 *testing.T) {
cfgPath := filepath.Join(tmpdir, kubeadmconstants.KubeletKubeConfigFileName)
if len(rt.fileContents) > 0 {
err := ioutil.WriteFile(cfgPath, rt.fileContents, 0644)
if err != nil {
t.Errorf("Couldn't create file")
return
}
}
client := clientsetfake.NewSimpleClientset()
if rt.node != nil {
_, err := client.CoreV1().Nodes().Create(rt.node)
if err != nil {
t.Errorf("couldn't create Node")
return
}
}
cfg := &kubeadmapi.InitConfiguration{}
err = getNodeRegistration(tmpdir, client, &cfg.NodeRegistration)
if rt.expectedError != (err != nil) {
t.Errorf("unexpected return err from getNodeRegistration: %v", err)
return
}
if rt.expectedError {
return
}
if cfg.NodeRegistration.Name != nodeName {
t.Errorf("invalid cfg.NodeRegistration.Name")
}
if cfg.NodeRegistration.CRISocket != "myCRIsocket" {
t.Errorf("invalid cfg.NodeRegistration.CRISocket")
}
if len(cfg.NodeRegistration.Taints) != 1 {
t.Errorf("invalid cfg.NodeRegistration.Taints")
}
})
}
}
func TestGetAPIEndpoint(t *testing.T) {
var tests = []struct {
name string
configMap fakeConfigMap
expectedError bool
}{
{
name: "valid",
configMap: fakeConfigMap{
name: kubeadmconstants.InitConfigurationConfigMap, // ClusterConfiguration from kubeadm-config.
data: map[string]string{
kubeadmconstants.ClusterStatusConfigMapKey: string(cfgFiles["ClusterStatus_v1alpha3"]),
},
},
},
{
name: "invalid - No CLusterStatus in kubeadm-config ConfigMap",
configMap: fakeConfigMap{
name: kubeadmconstants.InitConfigurationConfigMap, // ClusterConfiguration from kubeadm-config.
data: map[string]string{},
},
expectedError: true,
},
{
name: "invalid - CLusterStatus without APIEndopoints",
configMap: fakeConfigMap{
name: kubeadmconstants.InitConfigurationConfigMap, // ClusterConfiguration from kubeadm-config.
data: map[string]string{
kubeadmconstants.ClusterStatusConfigMapKey: string(cfgFiles["ClusterStatus_v1alpha3_Without_APIEndpoints"]),
},
},
expectedError: true,
},
}
for _, rt := range tests {
t.Run(rt.name, func(t *testing.T) {
cfg := &kubeadmapi.InitConfiguration{}
err := getAPIEndpoint(rt.configMap.data, nodeName, &cfg.APIEndpoint)
if rt.expectedError != (err != nil) {
t.Errorf("unexpected return err from getInitConfigurationFromCluster: %v", err)
return
}
if rt.expectedError {
return
}
if cfg.APIEndpoint.AdvertiseAddress != "1.2.3.4" || cfg.APIEndpoint.BindPort != 1234 {
t.Errorf("invalid cfg.APIEndpoint")
}
})
}
}
func TestGetComponentConfigs(t *testing.T) {
var tests = []struct {
name string
configMaps []fakeConfigMap
expectedError bool
}{
{
name: "valid",
configMaps: []fakeConfigMap{
{
name: kubeadmconstants.KubeProxyConfigMap, // Kube-proxy component config from corresponding ConfigMap.
data: map[string]string{
kubeadmconstants.KubeProxyConfigMapKey: string(cfgFiles["Kube-proxy_componentconfig"]),
},
},
{
name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion), // Kubelet component config from corresponding ConfigMap.
data: map[string]string{
kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]),
},
},
},
},
{
name: "invalid - No kubelet component config ConfigMap",
configMaps: []fakeConfigMap{
{
name: kubeadmconstants.KubeProxyConfigMap,
data: map[string]string{
kubeadmconstants.KubeProxyConfigMapKey: string(cfgFiles["Kube-proxy_componentconfig"]),
},
},
},
expectedError: true,
},
{
name: "invalid - No kube-proxy component config ConfigMap",
configMaps: []fakeConfigMap{
{
name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion),
data: map[string]string{
kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]),
},
},
},
expectedError: true,
},
}
for _, rt := range tests {
t.Run(rt.name, func(t *testing.T) {
client := clientsetfake.NewSimpleClientset()
for _, c := range rt.configMaps {
err := c.create(client)
if err != nil {
t.Errorf("couldn't create ConfigMap %s", c.name)
return
}
}
cfg := &kubeadmapi.InitConfiguration{
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
KubernetesVersion: k8sVersionString,
},
}
err := getComponentConfigs(client, &cfg.ClusterConfiguration)
if rt.expectedError != (err != nil) {
t.Errorf("unexpected return err from getInitConfigurationFromCluster: %v", err)
return
}
if rt.expectedError {
return
}
// Test expected values in InitConfiguration
if cfg.ComponentConfigs.Kubelet == nil {
t.Errorf("invalid cfg.ComponentConfigs.Kubelet")
}
if cfg.ComponentConfigs.KubeProxy == nil {
t.Errorf("invalid cfg.ComponentConfigs.KubeProxy")
}
})
}
}
func TestGetInitConfigurationFromCluster(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Couldn't create tmpdir")
}
defer os.RemoveAll(tmpdir)
var tests = []struct {
name string
fileContents []byte
node *v1.Node
configMaps []fakeConfigMap
newControlPlane bool
expectedError bool
}{
{ //TODO: remove in V1.13
name: "before v1.11", // single YAML, with a v1alpha2.MasterConfiguration object (with embedded component configs)
configMaps: []fakeConfigMap{
{
name: kubeadmconstants.InitConfigurationConfigMap,
data: map[string]string{
kubeadmconstants.InitConfigurationConfigMapKey: string(cfgFiles["MasterConfiguration_v1alpha2"]),
},
},
},
},
{
name: "invalid - No kubeadm-config ConfigMap",
expectedError: true,
},
{
name: "invalid - No CLusterConfiguration in kubeadm-config ConfigMap",
configMaps: []fakeConfigMap{
{
name: kubeadmconstants.InitConfigurationConfigMap, // ClusterConfiguration from kubeadm-config.
data: map[string]string{},
},
},
expectedError: true,
},
{
name: "valid - new control plane == false", // InitConfiguration composed with data from different places, with also node specific information from ClusterStatus and node
configMaps: []fakeConfigMap{
{
name: kubeadmconstants.InitConfigurationConfigMap, // ClusterConfiguration from kubeadm-config.
data: map[string]string{
kubeadmconstants.ClusterConfigurationConfigMapKey: string(cfgFiles["ClusterConfiguration_v1alpha3"]),
kubeadmconstants.ClusterStatusConfigMapKey: string(cfgFiles["ClusterStatus_v1alpha3"]),
},
},
{
name: kubeadmconstants.KubeProxyConfigMap, // Kube-proxy component config from corresponding ConfigMap.
data: map[string]string{
kubeadmconstants.KubeProxyConfigMapKey: string(cfgFiles["Kube-proxy_componentconfig"]),
},
},
{
name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion), // Kubelet component config from corresponding ConfigMap.
data: map[string]string{
kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]),
},
},
},
fileContents: kubeletConfFiles["configWithEmbeddedCert"],
node: &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: nodeName,
Annotations: map[string]string{
kubeadmconstants.AnnotationKubeadmCRISocket: "myCRIsocket",
},
},
Spec: v1.NodeSpec{
Taints: []v1.Taint{kubeadmconstants.MasterTaint},
},
},
},
{
name: "valid - new control plane == true", // InitConfiguration composed with data from different places, without node specific information
configMaps: []fakeConfigMap{
{
name: kubeadmconstants.InitConfigurationConfigMap, // ClusterConfiguration from kubeadm-config.
data: map[string]string{
kubeadmconstants.ClusterConfigurationConfigMapKey: string(cfgFiles["ClusterConfiguration_v1alpha3"]),
},
},
{
name: kubeadmconstants.KubeProxyConfigMap, // Kube-proxy component config from corresponding ConfigMap.
data: map[string]string{
kubeadmconstants.KubeProxyConfigMapKey: string(cfgFiles["Kube-proxy_componentconfig"]),
},
},
{
name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion), // Kubelet component config from corresponding ConfigMap.
data: map[string]string{
kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]),
},
},
},
newControlPlane: true,
},
}
for _, rt := range tests {
t.Run(rt.name, func(t *testing.T) {
cfgPath := filepath.Join(tmpdir, kubeadmconstants.KubeletKubeConfigFileName)
if len(rt.fileContents) > 0 {
err := ioutil.WriteFile(cfgPath, rt.fileContents, 0644)
if err != nil {
t.Errorf("Couldn't create file")
return
}
}
client := clientsetfake.NewSimpleClientset()
if rt.node != nil {
_, err := client.CoreV1().Nodes().Create(rt.node)
if err != nil {
t.Errorf("couldn't create Node")
return
}
}
for _, c := range rt.configMaps {
err := c.create(client)
if err != nil {
t.Errorf("couldn't create ConfigMap %s", c.name)
return
}
}
cfg, err := getInitConfigurationFromCluster(tmpdir, client, rt.newControlPlane)
if rt.expectedError != (err != nil) {
t.Errorf("unexpected return err from getInitConfigurationFromCluster: %v", err)
return
}
if rt.expectedError {
return
}
// Test expected values in InitConfiguration
if cfg == nil {
t.Errorf("unexpected nil return value")
}
if cfg.ClusterConfiguration.KubernetesVersion != k8sVersionString {
t.Errorf("invalid ClusterConfiguration.KubernetesVersion")
}
if !rt.newControlPlane && (cfg.APIEndpoint.AdvertiseAddress != "1.2.3.4" || cfg.APIEndpoint.BindPort != 1234) {
t.Errorf("invalid cfg.APIEndpoint")
}
if cfg.ComponentConfigs.Kubelet == nil {
t.Errorf("invalid cfg.ComponentConfigs.Kubelet")
}
if cfg.ComponentConfigs.KubeProxy == nil {
t.Errorf("invalid cfg.ComponentConfigs.KubeProxy")
}
})
}
}
type fakeConfigMap struct {
name string
data map[string]string
}
func (c *fakeConfigMap) create(client clientset.Interface) error {
return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: kubeadmconstants.InitConfigurationConfigMap,
Name: c.name,
Namespace: metav1.NamespaceSystem,
},
Data: map[string]string{
kubeadmconstants.InitConfigurationConfigMapKey: string(cfgYaml),
},
Data: c.data,
})
}

18
cmd/kubeadm/app/util/config/masterconfig.go
View File

@ -23,6 +23,7 @@ import (
"net"
"reflect"
"sort"
"strconv"
"github.com/golang/glog"
@ -52,7 +53,7 @@ func SetInitDynamicDefaults(cfg *kubeadmapi.InitConfiguration) error {
if err := SetAPIEndpointDynamicDefaults(&cfg.APIEndpoint); err != nil {
return err
}
if err := SetClusterDynamicDefaults(&cfg.ClusterConfiguration, cfg.APIEndpoint.AdvertiseAddress); err != nil {
if err := SetClusterDynamicDefaults(&cfg.ClusterConfiguration, cfg.APIEndpoint.AdvertiseAddress, cfg.APIEndpoint.BindPort); err != nil {
return err
}
return nil
@ -119,7 +120,7 @@ func SetAPIEndpointDynamicDefaults(cfg *kubeadmapi.APIEndpoint) error {
}
// SetClusterDynamicDefaults checks and sets configuration values for the InitConfiguration object
func SetClusterDynamicDefaults(cfg *kubeadmapi.ClusterConfiguration, advertiseAddress string) error {
func SetClusterDynamicDefaults(cfg *kubeadmapi.ClusterConfiguration, advertiseAddress string, bindPort int32) error {
// Default all the embedded ComponentConfig structs
componentconfigs.Known.Default(cfg)
@ -135,6 +136,19 @@ func SetClusterDynamicDefaults(cfg *kubeadmapi.ClusterConfiguration, advertiseAd
return err
}
// If ControlPlaneEndpoint is specified without a port number defaults it to
// the bindPort number of the APIEndpoint.
// This will allow join of additional control plane instances with different bindPort number
if cfg.ControlPlaneEndpoint != "" {
host, port, err := kubeadmutil.ParseHostPort(cfg.ControlPlaneEndpoint)
if err != nil {
return err
}
if port == "" {
cfg.ControlPlaneEndpoint = net.JoinHostPort(host, strconv.FormatInt(int64(bindPort), 10))
}
}
// Downcase SANs. Some domain names (like ELBs) have capitals in them.
LowercaseSANs(cfg.APIServerCertSANs)
return nil

4
cmd/kubeadm/app/util/endpoint.go
View File

@ -60,7 +60,9 @@ func GetMasterEndpoint(cfg *kubeadmapi.InitConfiguration) (string, error) {
// if a port is provided within the controlPlaneAddress warn the users we are using it, else use the bindport
if port != "" {
fmt.Println("[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address")
if port != bindPortString {
fmt.Println("[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address")
}
} else {
port = bindPortString
}