mirror of https://github.com/k3s-io/k3s
1198 lines
47 KiB
Go
1198 lines
47 KiB
Go
/*
|
|
Copyright 2016 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package e2e_node
|
|
|
|
import (
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/davecgh/go-spew/spew"
|
|
|
|
apiv1 "k8s.io/api/core/v1"
|
|
apiequality "k8s.io/apimachinery/pkg/api/equality"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/util/sets"
|
|
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
|
|
controller "k8s.io/kubernetes/pkg/kubelet/kubeletconfig"
|
|
"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/status"
|
|
"k8s.io/kubernetes/pkg/kubelet/metrics"
|
|
frameworkmetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
|
|
|
|
"k8s.io/kubernetes/test/e2e/framework"
|
|
|
|
"github.com/prometheus/common/model"
|
|
|
|
. "github.com/onsi/ginkgo"
|
|
. "github.com/onsi/gomega"
|
|
)
|
|
|
|
const itDescription = "status and events should match expectations"
|
|
|
|
type expectNodeConfigStatus struct {
|
|
lastKnownGood *apiv1.NodeConfigSource
|
|
err string
|
|
// If true, expect Status.Config.Active == Status.Config.LastKnownGood,
|
|
// otherwise expect Status.Config.Active == Status.Config.Assigned.
|
|
lkgActive bool
|
|
}
|
|
|
|
type nodeConfigTestCase struct {
|
|
desc string
|
|
configSource *apiv1.NodeConfigSource
|
|
configMap *apiv1.ConfigMap
|
|
expectConfigStatus expectNodeConfigStatus
|
|
expectConfig *kubeletconfig.KubeletConfiguration
|
|
// whether to expect this substring in an error returned from the API server when updating the config source
|
|
apierr string
|
|
// whether the state would cause a config change event as a result of the update to Node.Spec.ConfigSource,
|
|
// assuming that the current source would have also caused a config change event.
|
|
// for example, some malformed references may result in a download failure, in which case the Kubelet
|
|
// does not restart to change config, while an invalid payload will be detected upon restart
|
|
event bool
|
|
}
|
|
|
|
// This test is marked [Disruptive] because the Kubelet restarts several times during this test.
|
|
var _ = framework.KubeDescribe("[Feature:DynamicKubeletConfig][NodeFeature:DynamicKubeletConfig][Serial][Disruptive]", func() {
|
|
f := framework.NewDefaultFramework("dynamic-kubelet-configuration-test")
|
|
var beforeNode *apiv1.Node
|
|
var beforeConfigMap *apiv1.ConfigMap
|
|
var beforeKC *kubeletconfig.KubeletConfiguration
|
|
var localKC *kubeletconfig.KubeletConfiguration
|
|
|
|
// Dummy context to prevent framework's AfterEach from cleaning up before this test's AfterEach can run
|
|
Context("", func() {
|
|
BeforeEach(func() {
|
|
// make sure Dynamic Kubelet Configuration feature is enabled on the Kubelet we are about to test
|
|
enabled, err := isKubeletConfigEnabled(f)
|
|
framework.ExpectNoError(err)
|
|
if !enabled {
|
|
framework.ExpectNoError(fmt.Errorf("The Dynamic Kubelet Configuration feature is not enabled.\n" +
|
|
"Pass --feature-gates=DynamicKubeletConfig=true to the Kubelet and API server to enable this feature.\n" +
|
|
"For `make test-e2e-node`, you can set `TEST_ARGS='--feature-gates=DynamicKubeletConfig=true'`."))
|
|
}
|
|
// record before state so we can restore it after the test
|
|
if beforeNode == nil {
|
|
node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
|
|
framework.ExpectNoError(err)
|
|
beforeNode = node
|
|
}
|
|
if source := beforeNode.Spec.ConfigSource; source != nil {
|
|
if source.ConfigMap != nil {
|
|
cm, err := f.ClientSet.CoreV1().ConfigMaps(source.ConfigMap.Namespace).Get(source.ConfigMap.Name, metav1.GetOptions{})
|
|
framework.ExpectNoError(err)
|
|
beforeConfigMap = cm
|
|
}
|
|
}
|
|
if beforeKC == nil {
|
|
kc, err := getCurrentKubeletConfig()
|
|
framework.ExpectNoError(err)
|
|
beforeKC = kc
|
|
}
|
|
// reset the node's assigned/active/last-known-good config by setting the source to nil,
|
|
// so each test starts from a clean-slate
|
|
(&nodeConfigTestCase{
|
|
desc: "reset via nil config source",
|
|
configSource: nil,
|
|
}).run(f, setConfigSourceFunc, false, 0)
|
|
// record local KC so we can check it during tests that roll back to nil last-known-good
|
|
if localKC == nil {
|
|
kc, err := getCurrentKubeletConfig()
|
|
framework.ExpectNoError(err)
|
|
localKC = kc
|
|
}
|
|
})
|
|
|
|
AfterEach(func() {
|
|
// clean-slate the Node again (prevents last-known-good from any tests from leaking through)
|
|
(&nodeConfigTestCase{
|
|
desc: "reset via nil config source",
|
|
configSource: nil,
|
|
}).run(f, setConfigSourceFunc, false, 0)
|
|
// restore the values from before the test before moving on
|
|
restore := &nodeConfigTestCase{
|
|
desc: "restore values from before test",
|
|
configSource: beforeNode.Spec.ConfigSource,
|
|
configMap: beforeConfigMap,
|
|
expectConfig: beforeKC,
|
|
}
|
|
restore.run(f, setConfigSourceFunc, false, 0)
|
|
})
|
|
|
|
Context("update Node.Spec.ConfigSource: state transitions:", func() {
|
|
It(itDescription, func() {
|
|
var err error
|
|
// we base the "correct" configmap off of the configuration from before the test
|
|
correctKC := beforeKC.DeepCopy()
|
|
correctConfigMap := newKubeletConfigMap("dynamic-kubelet-config-test-correct", correctKC)
|
|
correctConfigMap, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(correctConfigMap)
|
|
framework.ExpectNoError(err)
|
|
|
|
// fail to parse, we insert some bogus stuff into the configMap
|
|
failParseConfigMap := &apiv1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "dynamic-kubelet-config-test-fail-parse"},
|
|
Data: map[string]string{
|
|
"kubelet": "{0xdeadbeef}",
|
|
},
|
|
}
|
|
failParseConfigMap, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(failParseConfigMap)
|
|
framework.ExpectNoError(err)
|
|
|
|
// fail to validate, we make a copy of correct and set an invalid KubeAPIQPS on kc before serializing
|
|
invalidKC := correctKC.DeepCopy()
|
|
invalidKC.KubeAPIQPS = -1
|
|
failValidateConfigMap := newKubeletConfigMap("dynamic-kubelet-config-test-fail-validate", invalidKC)
|
|
failValidateConfigMap, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(failValidateConfigMap)
|
|
framework.ExpectNoError(err)
|
|
|
|
correctSource := &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Namespace: correctConfigMap.Namespace,
|
|
Name: correctConfigMap.Name,
|
|
KubeletConfigKey: "kubelet",
|
|
}}
|
|
failParseSource := &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Namespace: failParseConfigMap.Namespace,
|
|
Name: failParseConfigMap.Name,
|
|
KubeletConfigKey: "kubelet",
|
|
}}
|
|
failValidateSource := &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Namespace: failValidateConfigMap.Namespace,
|
|
Name: failValidateConfigMap.Name,
|
|
KubeletConfigKey: "kubelet",
|
|
}}
|
|
|
|
cases := []nodeConfigTestCase{
|
|
{
|
|
desc: "Node.Spec.ConfigSource is nil",
|
|
configSource: nil,
|
|
expectConfigStatus: expectNodeConfigStatus{},
|
|
expectConfig: nil,
|
|
event: true,
|
|
},
|
|
{
|
|
desc: "Node.Spec.ConfigSource has all nil subfields",
|
|
configSource: &apiv1.NodeConfigSource{},
|
|
apierr: "exactly one reference subfield must be non-nil",
|
|
},
|
|
{
|
|
desc: "Node.Spec.ConfigSource.ConfigMap is missing namespace",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Name: "bar",
|
|
KubeletConfigKey: "kubelet",
|
|
}}, // missing Namespace
|
|
apierr: "spec.configSource.configMap.namespace: Required value: namespace must be set",
|
|
},
|
|
{
|
|
desc: "Node.Spec.ConfigSource.ConfigMap is missing name",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Namespace: "foo",
|
|
KubeletConfigKey: "kubelet",
|
|
}}, // missing Name
|
|
apierr: "spec.configSource.configMap.name: Required value: name must be set",
|
|
},
|
|
{
|
|
desc: "Node.Spec.ConfigSource.ConfigMap is missing kubeletConfigKey",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Namespace: "foo",
|
|
Name: "bar",
|
|
}}, // missing KubeletConfigKey
|
|
apierr: "spec.configSource.configMap.kubeletConfigKey: Required value: kubeletConfigKey must be set",
|
|
},
|
|
{
|
|
desc: "Node.Spec.ConfigSource.ConfigMap.UID is illegally specified",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
UID: "foo",
|
|
Name: "bar",
|
|
Namespace: "baz",
|
|
KubeletConfigKey: "kubelet",
|
|
}},
|
|
apierr: "spec.configSource.configMap.uid: Forbidden: uid must not be set in spec",
|
|
},
|
|
{
|
|
desc: "Node.Spec.ConfigSource.ConfigMap.ResourceVersion is illegally specified",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Name: "bar",
|
|
Namespace: "baz",
|
|
ResourceVersion: "1",
|
|
KubeletConfigKey: "kubelet",
|
|
}},
|
|
apierr: "spec.configSource.configMap.resourceVersion: Forbidden: resourceVersion must not be set in spec",
|
|
},
|
|
{
|
|
desc: "Node.Spec.ConfigSource.ConfigMap has invalid namespace",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Name: "bar",
|
|
Namespace: "../baz",
|
|
KubeletConfigKey: "kubelet",
|
|
}},
|
|
apierr: "spec.configSource.configMap.namespace: Invalid value",
|
|
},
|
|
{
|
|
desc: "Node.Spec.ConfigSource.ConfigMap has invalid name",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Name: "../bar",
|
|
Namespace: "baz",
|
|
KubeletConfigKey: "kubelet",
|
|
}},
|
|
apierr: "spec.configSource.configMap.name: Invalid value",
|
|
},
|
|
{
|
|
desc: "Node.Spec.ConfigSource.ConfigMap has invalid kubeletConfigKey",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Name: "bar",
|
|
Namespace: "baz",
|
|
KubeletConfigKey: "../qux",
|
|
}},
|
|
apierr: "spec.configSource.configMap.kubeletConfigKey: Invalid value",
|
|
},
|
|
{
|
|
desc: "correct",
|
|
configSource: correctSource,
|
|
configMap: correctConfigMap,
|
|
expectConfig: correctKC,
|
|
event: true,
|
|
},
|
|
{
|
|
desc: "fail-parse",
|
|
configSource: failParseSource,
|
|
configMap: failParseConfigMap,
|
|
expectConfigStatus: expectNodeConfigStatus{
|
|
err: status.LoadError,
|
|
lkgActive: true,
|
|
},
|
|
expectConfig: localKC,
|
|
event: true,
|
|
},
|
|
{
|
|
desc: "fail-validate",
|
|
configSource: failValidateSource,
|
|
configMap: failValidateConfigMap,
|
|
expectConfigStatus: expectNodeConfigStatus{
|
|
err: status.ValidateError,
|
|
lkgActive: true,
|
|
},
|
|
expectConfig: localKC,
|
|
event: true,
|
|
},
|
|
}
|
|
|
|
L := len(cases)
|
|
for i := 1; i <= L; i++ { // need one less iteration than the number of cases
|
|
testBothDirections(f, setConfigSourceFunc, &cases[i-1 : i][0], cases[i:L], 0)
|
|
}
|
|
|
|
})
|
|
})
|
|
|
|
Context("update Node.Spec.ConfigSource: recover to last-known-good ConfigMap:", func() {
|
|
It(itDescription, func() {
|
|
var err error
|
|
// we base the "lkg" configmap off of the configuration from before the test
|
|
lkgKC := beforeKC.DeepCopy()
|
|
lkgConfigMap := newKubeletConfigMap("dynamic-kubelet-config-test-intended-lkg", lkgKC)
|
|
lkgConfigMap, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(lkgConfigMap)
|
|
framework.ExpectNoError(err)
|
|
|
|
// bad config map, we insert some bogus stuff into the configMap
|
|
badConfigMap := &apiv1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "dynamic-kubelet-config-test-bad"},
|
|
Data: map[string]string{
|
|
"kubelet": "{0xdeadbeef}",
|
|
},
|
|
}
|
|
badConfigMap, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(badConfigMap)
|
|
framework.ExpectNoError(err)
|
|
|
|
lkgSource := &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Namespace: lkgConfigMap.Namespace,
|
|
Name: lkgConfigMap.Name,
|
|
KubeletConfigKey: "kubelet",
|
|
}}
|
|
lkgStatus := lkgSource.DeepCopy()
|
|
lkgStatus.ConfigMap.UID = lkgConfigMap.UID
|
|
lkgStatus.ConfigMap.ResourceVersion = lkgConfigMap.ResourceVersion
|
|
|
|
badSource := &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Namespace: badConfigMap.Namespace,
|
|
Name: badConfigMap.Name,
|
|
KubeletConfigKey: "kubelet",
|
|
}}
|
|
|
|
cases := []nodeConfigTestCase{
|
|
{
|
|
desc: "intended last-known-good",
|
|
configSource: lkgSource,
|
|
configMap: lkgConfigMap,
|
|
expectConfigStatus: expectNodeConfigStatus{
|
|
lastKnownGood: lkgStatus,
|
|
},
|
|
expectConfig: lkgKC,
|
|
event: true,
|
|
},
|
|
{
|
|
desc: "bad config",
|
|
configSource: badSource,
|
|
configMap: badConfigMap,
|
|
expectConfigStatus: expectNodeConfigStatus{
|
|
lastKnownGood: lkgStatus,
|
|
err: status.LoadError,
|
|
lkgActive: true,
|
|
},
|
|
expectConfig: lkgKC,
|
|
event: true,
|
|
},
|
|
}
|
|
|
|
// wait 12 minutes after setting the first config to ensure it has time to pass the trial duration
|
|
testBothDirections(f, setConfigSourceFunc, &cases[0], cases[1:], 12*time.Minute)
|
|
})
|
|
})
|
|
|
|
Context("update Node.Spec.ConfigSource: recover to last-known-good ConfigMap.KubeletConfigKey:", func() {
|
|
It(itDescription, func() {
|
|
const badConfigKey = "bad"
|
|
var err error
|
|
// we base the "lkg" configmap off of the configuration from before the test
|
|
lkgKC := beforeKC.DeepCopy()
|
|
combinedConfigMap := newKubeletConfigMap("dynamic-kubelet-config-test-combined", lkgKC)
|
|
combinedConfigMap.Data[badConfigKey] = "{0xdeadbeef}"
|
|
combinedConfigMap, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(combinedConfigMap)
|
|
framework.ExpectNoError(err)
|
|
|
|
lkgSource := &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Namespace: combinedConfigMap.Namespace,
|
|
Name: combinedConfigMap.Name,
|
|
KubeletConfigKey: "kubelet",
|
|
}}
|
|
|
|
lkgStatus := lkgSource.DeepCopy()
|
|
lkgStatus.ConfigMap.UID = combinedConfigMap.UID
|
|
lkgStatus.ConfigMap.ResourceVersion = combinedConfigMap.ResourceVersion
|
|
|
|
badSource := lkgSource.DeepCopy()
|
|
badSource.ConfigMap.KubeletConfigKey = badConfigKey
|
|
|
|
cases := []nodeConfigTestCase{
|
|
{
|
|
desc: "intended last-known-good",
|
|
configSource: lkgSource,
|
|
configMap: combinedConfigMap,
|
|
expectConfigStatus: expectNodeConfigStatus{
|
|
lastKnownGood: lkgStatus,
|
|
},
|
|
expectConfig: lkgKC,
|
|
event: true,
|
|
},
|
|
{
|
|
desc: "bad config",
|
|
configSource: badSource,
|
|
configMap: combinedConfigMap,
|
|
expectConfigStatus: expectNodeConfigStatus{
|
|
lastKnownGood: lkgStatus,
|
|
err: status.LoadError,
|
|
lkgActive: true,
|
|
},
|
|
expectConfig: lkgKC,
|
|
event: true,
|
|
},
|
|
}
|
|
|
|
// wait 12 minutes after setting the first config to ensure it has time to pass the trial duration
|
|
testBothDirections(f, setConfigSourceFunc, &cases[0], cases[1:], 12*time.Minute)
|
|
})
|
|
})
|
|
|
|
// previously, we missed a panic because we were not exercising this path
|
|
Context("update Node.Spec.ConfigSource: non-nil last-known-good to a new non-nil last-known-good", func() {
|
|
It(itDescription, func() {
|
|
var err error
|
|
// we base the "lkg" configmap off of the configuration from before the test
|
|
lkgKC := beforeKC.DeepCopy()
|
|
lkgConfigMap1 := newKubeletConfigMap("dynamic-kubelet-config-test-lkg-1", lkgKC)
|
|
lkgConfigMap1, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(lkgConfigMap1)
|
|
framework.ExpectNoError(err)
|
|
|
|
lkgSource1 := &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Namespace: lkgConfigMap1.Namespace,
|
|
Name: lkgConfigMap1.Name,
|
|
KubeletConfigKey: "kubelet",
|
|
}}
|
|
lkgStatus1 := lkgSource1.DeepCopy()
|
|
lkgStatus1.ConfigMap.UID = lkgConfigMap1.UID
|
|
lkgStatus1.ConfigMap.ResourceVersion = lkgConfigMap1.ResourceVersion
|
|
|
|
lkgConfigMap2 := newKubeletConfigMap("dynamic-kubelet-config-test-lkg-2", lkgKC)
|
|
lkgConfigMap2, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(lkgConfigMap2)
|
|
framework.ExpectNoError(err)
|
|
|
|
lkgSource2 := &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Namespace: lkgConfigMap2.Namespace,
|
|
Name: lkgConfigMap2.Name,
|
|
KubeletConfigKey: "kubelet",
|
|
}}
|
|
lkgStatus2 := lkgSource2.DeepCopy()
|
|
lkgStatus2.ConfigMap.UID = lkgConfigMap2.UID
|
|
lkgStatus2.ConfigMap.ResourceVersion = lkgConfigMap2.ResourceVersion
|
|
|
|
// cases
|
|
first := nodeConfigTestCase{
|
|
desc: "last-known-good-1",
|
|
configSource: lkgSource1,
|
|
configMap: lkgConfigMap1,
|
|
expectConfigStatus: expectNodeConfigStatus{
|
|
lastKnownGood: lkgStatus1,
|
|
},
|
|
expectConfig: lkgKC,
|
|
event: true,
|
|
}
|
|
|
|
second := nodeConfigTestCase{
|
|
desc: "last-known-good-2",
|
|
configSource: lkgSource2,
|
|
configMap: lkgConfigMap2,
|
|
expectConfigStatus: expectNodeConfigStatus{
|
|
lastKnownGood: lkgStatus2,
|
|
},
|
|
expectConfig: lkgKC,
|
|
event: true,
|
|
}
|
|
|
|
// Manually actuate this to ensure we wait for each case to become the last-known-good
|
|
const lkgDuration = 12 * time.Minute
|
|
By(fmt.Sprintf("setting initial state %q", first.desc))
|
|
first.run(f, setConfigSourceFunc, true, lkgDuration)
|
|
By(fmt.Sprintf("from %q to %q", first.desc, second.desc))
|
|
second.run(f, setConfigSourceFunc, true, lkgDuration)
|
|
})
|
|
})
|
|
|
|
// exposes resource leaks across config changes
|
|
Context("update Node.Spec.ConfigSource: 100 update stress test:", func() {
|
|
It(itDescription, func() {
|
|
var err error
|
|
|
|
// we just create two configmaps with the same config but different names and toggle between them
|
|
kc1 := beforeKC.DeepCopy()
|
|
cm1 := newKubeletConfigMap("dynamic-kubelet-config-test-cm1", kc1)
|
|
cm1, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(cm1)
|
|
framework.ExpectNoError(err)
|
|
|
|
// slightly change the config
|
|
kc2 := kc1.DeepCopy()
|
|
kc2.EventRecordQPS = kc1.EventRecordQPS + 1
|
|
cm2 := newKubeletConfigMap("dynamic-kubelet-config-test-cm2", kc2)
|
|
cm2, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(cm2)
|
|
framework.ExpectNoError(err)
|
|
|
|
cm1Source := &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Namespace: cm1.Namespace,
|
|
Name: cm1.Name,
|
|
KubeletConfigKey: "kubelet",
|
|
}}
|
|
|
|
cm2Source := &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Namespace: cm2.Namespace,
|
|
Name: cm2.Name,
|
|
KubeletConfigKey: "kubelet",
|
|
}}
|
|
|
|
cases := []nodeConfigTestCase{
|
|
{
|
|
desc: "cm1",
|
|
configSource: cm1Source,
|
|
configMap: cm1,
|
|
expectConfig: kc1,
|
|
event: true,
|
|
},
|
|
{
|
|
desc: "cm2",
|
|
configSource: cm2Source,
|
|
configMap: cm2,
|
|
expectConfig: kc2,
|
|
event: true,
|
|
},
|
|
}
|
|
|
|
for i := 0; i < 50; i++ { // change the config 101 times (changes 3 times in the first iteration, 2 times in each subsequent iteration)
|
|
testBothDirections(f, setConfigSourceFunc, &cases[0], cases[1:], 0)
|
|
}
|
|
})
|
|
})
|
|
|
|
// Please note: This behavior is tested to ensure implementation correctness. We do not, however, recommend ConfigMap mutations
|
|
// as a usage pattern for dynamic Kubelet config in large clusters. It is much safer to create a new ConfigMap, and incrementally
|
|
// roll out a new Node.Spec.ConfigSource that references the new ConfigMap. In-place ConfigMap updates, including deletion
|
|
// followed by re-creation, will cause all observing Kubelets to immediately restart for new config, because these operations
|
|
// change the ResourceVersion of the ConfigMap.
|
|
Context("update ConfigMap in-place: state transitions:", func() {
|
|
It(itDescription, func() {
|
|
var err error
|
|
// we base the "correct" configmap off of the configuration from before the test
|
|
correctKC := beforeKC.DeepCopy()
|
|
correctConfigMap := newKubeletConfigMap("dynamic-kubelet-config-test-in-place", correctKC)
|
|
correctConfigMap, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(correctConfigMap)
|
|
framework.ExpectNoError(err)
|
|
|
|
// we reuse the same name, namespace
|
|
failParseConfigMap := correctConfigMap.DeepCopy()
|
|
failParseConfigMap.Data = map[string]string{
|
|
"kubelet": "{0xdeadbeef}",
|
|
}
|
|
|
|
// fail to validate, we make a copy and set an invalid KubeAPIQPS on kc before serializing
|
|
invalidKC := correctKC.DeepCopy()
|
|
invalidKC.KubeAPIQPS = -1
|
|
failValidateConfigMap := correctConfigMap.DeepCopy()
|
|
failValidateConfigMap.Data = newKubeletConfigMap("", invalidKC).Data
|
|
|
|
// ensure node config source is set to the config map we will mutate in-place,
|
|
// since updateConfigMapFunc doesn't mutate Node.Spec.ConfigSource
|
|
source := &apiv1.NodeConfigSource{
|
|
ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Namespace: correctConfigMap.Namespace,
|
|
Name: correctConfigMap.Name,
|
|
KubeletConfigKey: "kubelet",
|
|
},
|
|
}
|
|
(&nodeConfigTestCase{
|
|
desc: "initial state (correct)",
|
|
configSource: source,
|
|
configMap: correctConfigMap,
|
|
expectConfig: correctKC,
|
|
}).run(f, setConfigSourceFunc, false, 0)
|
|
|
|
cases := []nodeConfigTestCase{
|
|
{
|
|
desc: "correct",
|
|
configSource: source,
|
|
configMap: correctConfigMap,
|
|
expectConfig: correctKC,
|
|
event: true,
|
|
},
|
|
{
|
|
desc: "fail-parse",
|
|
configSource: source,
|
|
configMap: failParseConfigMap,
|
|
expectConfigStatus: expectNodeConfigStatus{
|
|
err: status.LoadError,
|
|
lkgActive: true,
|
|
},
|
|
expectConfig: localKC,
|
|
event: true,
|
|
},
|
|
{
|
|
desc: "fail-validate",
|
|
configSource: source,
|
|
configMap: failValidateConfigMap,
|
|
expectConfigStatus: expectNodeConfigStatus{
|
|
err: status.ValidateError,
|
|
lkgActive: true,
|
|
},
|
|
expectConfig: localKC,
|
|
event: true,
|
|
},
|
|
}
|
|
L := len(cases)
|
|
for i := 1; i <= L; i++ { // need one less iteration than the number of cases
|
|
testBothDirections(f, updateConfigMapFunc, &cases[i-1 : i][0], cases[i:L], 0)
|
|
}
|
|
})
|
|
})
|
|
|
|
// Please note: This behavior is tested to ensure implementation correctness. We do not, however, recommend ConfigMap mutations
|
|
// as a usage pattern for dynamic Kubelet config in large clusters. It is much safer to create a new ConfigMap, and incrementally
|
|
// roll out a new Node.Spec.ConfigSource that references the new ConfigMap. In-place ConfigMap updates, including deletion
|
|
// followed by re-creation, will cause all observing Kubelets to immediately restart for new config, because these operations
|
|
// change the ResourceVersion of the ConfigMap.
|
|
Context("update ConfigMap in-place: recover to last-known-good version:", func() {
|
|
It(itDescription, func() {
|
|
var err error
|
|
// we base the "lkg" configmap off of the configuration from before the test
|
|
lkgKC := beforeKC.DeepCopy()
|
|
lkgConfigMap := newKubeletConfigMap("dynamic-kubelet-config-test-in-place-lkg", lkgKC)
|
|
lkgConfigMap, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(lkgConfigMap)
|
|
framework.ExpectNoError(err)
|
|
|
|
// bad config map, we insert some bogus stuff into the configMap
|
|
badConfigMap := lkgConfigMap.DeepCopy()
|
|
badConfigMap.Data = map[string]string{
|
|
"kubelet": "{0xdeadbeef}",
|
|
}
|
|
// ensure node config source is set to the config map we will mutate in-place
|
|
source := &apiv1.NodeConfigSource{
|
|
ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Namespace: lkgConfigMap.Namespace,
|
|
Name: lkgConfigMap.Name,
|
|
KubeletConfigKey: "kubelet",
|
|
},
|
|
}
|
|
|
|
// Even though the first test case will PUT the lkgConfigMap again, no-op writes don't increment
|
|
// ResourceVersion, so the expected status we record here will still be correct.
|
|
lkgStatus := source.DeepCopy()
|
|
lkgStatus.ConfigMap.UID = lkgConfigMap.UID
|
|
lkgStatus.ConfigMap.ResourceVersion = lkgConfigMap.ResourceVersion
|
|
|
|
(&nodeConfigTestCase{
|
|
desc: "initial state (correct)",
|
|
configSource: source,
|
|
configMap: lkgConfigMap,
|
|
expectConfig: lkgKC,
|
|
}).run(f, setConfigSourceFunc, false, 0) // wait 0 here, and we should not expect LastKnownGood to have changed yet (hence nil)
|
|
|
|
cases := []nodeConfigTestCase{
|
|
{
|
|
desc: "intended last-known-good",
|
|
configSource: source,
|
|
configMap: lkgConfigMap,
|
|
expectConfigStatus: expectNodeConfigStatus{
|
|
lastKnownGood: lkgStatus,
|
|
},
|
|
expectConfig: lkgKC,
|
|
event: true,
|
|
},
|
|
{
|
|
// NOTE(mtaufen): If you see a strange "expected assigned x but got assigned y" error on this case,
|
|
// it is possible that the Kubelet didn't start the informer that watches the currently assigned
|
|
// ConfigMap, or didn't get updates from that informer. Other tests don't always catch this because
|
|
// they quickly change config. The sync loop will always happen once, a bit after the Kubelet starts
|
|
// up, because other informers' initial "add" events can queue a sync. If you wait long enough before
|
|
// changing config (waiting for the config to become last-known-good, for example), the syncs queued by
|
|
// add events will have already been processed, and the lack of a running ConfigMap informer will result
|
|
// in a missed update, no config change, and the above error when we check the status.
|
|
desc: "bad config",
|
|
configSource: source,
|
|
configMap: badConfigMap,
|
|
expectConfigStatus: expectNodeConfigStatus{
|
|
lastKnownGood: lkgStatus,
|
|
err: status.LoadError,
|
|
lkgActive: true,
|
|
},
|
|
expectConfig: lkgKC,
|
|
event: true,
|
|
},
|
|
}
|
|
|
|
// wait 12 minutes after setting the first config to ensure it has time to pass the trial duration
|
|
testBothDirections(f, updateConfigMapFunc, &cases[0], cases[1:], 12*time.Minute)
|
|
})
|
|
})
|
|
|
|
// Please note: This behavior is tested to ensure implementation correctness. We do not, however, recommend ConfigMap mutations
|
|
// as a usage pattern for dynamic Kubelet config in large clusters. It is much safer to create a new ConfigMap, and incrementally
|
|
// roll out a new Node.Spec.ConfigSource that references the new ConfigMap. In-place ConfigMap updates, including deletion
|
|
// followed by re-creation, will cause all observing Kubelets to immediately restart for new config, because these operations
|
|
// change the ResourceVersion of the ConfigMap.
|
|
Context("delete and recreate ConfigMap: state transitions:", func() {
|
|
It(itDescription, func() {
|
|
var err error
|
|
// we base the "correct" configmap off of the configuration from before the test
|
|
correctKC := beforeKC.DeepCopy()
|
|
correctConfigMap := newKubeletConfigMap("dynamic-kubelet-config-test-delete-createe", correctKC)
|
|
correctConfigMap, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(correctConfigMap)
|
|
framework.ExpectNoError(err)
|
|
|
|
// we reuse the same name, namespace
|
|
failParseConfigMap := correctConfigMap.DeepCopy()
|
|
failParseConfigMap.Data = map[string]string{
|
|
"kubelet": "{0xdeadbeef}",
|
|
}
|
|
|
|
// fail to validate, we make a copy and set an invalid KubeAPIQPS on kc before serializing
|
|
invalidKC := correctKC.DeepCopy()
|
|
invalidKC.KubeAPIQPS = -1
|
|
failValidateConfigMap := correctConfigMap.DeepCopy()
|
|
failValidateConfigMap.Data = newKubeletConfigMap("", invalidKC).Data
|
|
|
|
// ensure node config source is set to the config map we will mutate in-place,
|
|
// since recreateConfigMapFunc doesn't mutate Node.Spec.ConfigSource
|
|
source := &apiv1.NodeConfigSource{
|
|
ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Namespace: correctConfigMap.Namespace,
|
|
Name: correctConfigMap.Name,
|
|
KubeletConfigKey: "kubelet",
|
|
},
|
|
}
|
|
(&nodeConfigTestCase{
|
|
desc: "initial state (correct)",
|
|
configSource: source,
|
|
configMap: correctConfigMap,
|
|
expectConfig: correctKC,
|
|
}).run(f, setConfigSourceFunc, false, 0)
|
|
|
|
cases := []nodeConfigTestCase{
|
|
{
|
|
desc: "correct",
|
|
configSource: source,
|
|
configMap: correctConfigMap,
|
|
expectConfig: correctKC,
|
|
event: true,
|
|
},
|
|
{
|
|
desc: "fail-parse",
|
|
configSource: source,
|
|
configMap: failParseConfigMap,
|
|
expectConfigStatus: expectNodeConfigStatus{
|
|
err: status.LoadError,
|
|
lkgActive: true,
|
|
},
|
|
expectConfig: localKC,
|
|
event: true,
|
|
},
|
|
{
|
|
desc: "fail-validate",
|
|
configSource: source,
|
|
configMap: failValidateConfigMap,
|
|
expectConfigStatus: expectNodeConfigStatus{
|
|
err: status.ValidateError,
|
|
lkgActive: true,
|
|
},
|
|
expectConfig: localKC,
|
|
event: true,
|
|
},
|
|
}
|
|
L := len(cases)
|
|
for i := 1; i <= L; i++ { // need one less iteration than the number of cases
|
|
testBothDirections(f, recreateConfigMapFunc, &cases[i-1 : i][0], cases[i:L], 0)
|
|
}
|
|
})
|
|
})
|
|
|
|
// Please note: This behavior is tested to ensure implementation correctness. We do not, however, recommend ConfigMap mutations
|
|
// as a usage pattern for dynamic Kubelet config in large clusters. It is much safer to create a new ConfigMap, and incrementally
|
|
// roll out a new Node.Spec.ConfigSource that references the new ConfigMap. In-place ConfigMap updates, including deletion
|
|
// followed by re-creation, will cause all observing Kubelets to immediately restart for new config, because these operations
|
|
// change the ResourceVersion of the ConfigMap.
|
|
Context("delete and recreate ConfigMap: error while ConfigMap is absent:", func() {
|
|
It(itDescription, func() {
|
|
var err error
|
|
// we base the "correct" configmap off of the configuration from before the test
|
|
correctKC := beforeKC.DeepCopy()
|
|
correctConfigMap := newKubeletConfigMap("dynamic-kubelet-config-test-delete-createe", correctKC)
|
|
correctConfigMap, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(correctConfigMap)
|
|
framework.ExpectNoError(err)
|
|
|
|
// ensure node config source is set to the config map we will mutate in-place,
|
|
// since our mutation functions don't mutate Node.Spec.ConfigSource
|
|
source := &apiv1.NodeConfigSource{
|
|
ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
|
Namespace: correctConfigMap.Namespace,
|
|
Name: correctConfigMap.Name,
|
|
KubeletConfigKey: "kubelet",
|
|
},
|
|
}
|
|
(&nodeConfigTestCase{
|
|
desc: "correct",
|
|
configSource: source,
|
|
configMap: correctConfigMap,
|
|
expectConfig: correctKC,
|
|
}).run(f, setConfigSourceFunc, false, 0)
|
|
|
|
// delete the ConfigMap, and ensure an error is reported by the Kubelet while the ConfigMap is absent
|
|
(&nodeConfigTestCase{
|
|
desc: "correct",
|
|
configSource: source,
|
|
configMap: correctConfigMap,
|
|
expectConfigStatus: expectNodeConfigStatus{
|
|
err: fmt.Sprintf(status.SyncErrorFmt, status.DownloadError),
|
|
},
|
|
expectConfig: correctKC,
|
|
}).run(f, deleteConfigMapFunc, false, 0)
|
|
|
|
// re-create the ConfigMap, and ensure the error disappears
|
|
(&nodeConfigTestCase{
|
|
desc: "correct",
|
|
configSource: source,
|
|
configMap: correctConfigMap,
|
|
expectConfig: correctKC,
|
|
}).run(f, createConfigMapFunc, false, 0)
|
|
})
|
|
})
|
|
})
|
|
})
|
|
|
|
// testBothDirections tests the state change represented by each edge, where each case is a vertex,
|
|
// and there are edges in each direction between first and each of the cases.
|
|
func testBothDirections(f *framework.Framework, fn func(f *framework.Framework, tc *nodeConfigTestCase) error,
|
|
first *nodeConfigTestCase, cases []nodeConfigTestCase, waitAfterFirst time.Duration) {
|
|
// set to first and check that everything got set up properly
|
|
By(fmt.Sprintf("setting initial state %q", first.desc))
|
|
// we don't always expect an event here, because setting "first" might not represent
|
|
// a change from the current configuration
|
|
first.run(f, fn, false, waitAfterFirst)
|
|
|
|
// for each case, set up, check expectations, then reset to first and check again
|
|
for i := range cases {
|
|
tc := &cases[i]
|
|
By(fmt.Sprintf("from %q to %q", first.desc, tc.desc))
|
|
// from first -> tc, tc.event fully describes whether we should get a config change event
|
|
tc.run(f, fn, tc.event, 0)
|
|
|
|
By(fmt.Sprintf("back to %q from %q", first.desc, tc.desc))
|
|
// whether first -> tc should have produced a config change event partially determines whether tc -> first should produce an event
|
|
first.run(f, fn, first.event && tc.event, 0)
|
|
}
|
|
}
|
|
|
|
// run tests that, after performing fn, the node spec, status, configz, and latest event match
|
|
// the expectations described by state.
|
|
func (tc *nodeConfigTestCase) run(f *framework.Framework, fn func(f *framework.Framework, tc *nodeConfigTestCase) error,
|
|
expectEvent bool, wait time.Duration) {
|
|
// set the desired state, retry a few times in case we are competing with other editors
|
|
Eventually(func() error {
|
|
if err := fn(f, tc); err != nil {
|
|
if len(tc.apierr) == 0 {
|
|
return fmt.Errorf("case %s: expect nil error but got %q", tc.desc, err.Error())
|
|
} else if !strings.Contains(err.Error(), tc.apierr) {
|
|
return fmt.Errorf("case %s: expect error to contain %q but got %q", tc.desc, tc.apierr, err.Error())
|
|
}
|
|
} else if len(tc.apierr) > 0 {
|
|
return fmt.Errorf("case %s: expect error to contain %q but got nil error", tc.desc, tc.apierr)
|
|
}
|
|
return nil
|
|
}, time.Minute, time.Second).Should(BeNil())
|
|
// skip further checks if we expected an API error
|
|
if len(tc.apierr) > 0 {
|
|
return
|
|
}
|
|
// wait for the designated duration before checking the reconciliation
|
|
time.Sleep(wait)
|
|
// check config source
|
|
tc.checkNodeConfigSource(f)
|
|
// check status
|
|
tc.checkConfigStatus(f)
|
|
// check that the Kubelet's config-related metrics are correct
|
|
tc.checkConfigMetrics(f)
|
|
// check expectConfig
|
|
if tc.expectConfig != nil {
|
|
tc.checkConfig(f)
|
|
}
|
|
// check that an event was sent for the config change
|
|
if expectEvent {
|
|
tc.checkEvent(f)
|
|
}
|
|
}
|
|
|
|
// setConfigSourceFunc sets Node.Spec.ConfigSource to tc.configSource
|
|
func setConfigSourceFunc(f *framework.Framework, tc *nodeConfigTestCase) error {
|
|
return setNodeConfigSource(f, tc.configSource)
|
|
}
|
|
|
|
// updateConfigMapFunc updates the ConfigMap described by tc.configMap to contain matching data.
|
|
// It also updates the resourceVersion in any non-nil NodeConfigSource.ConfigMap in the expected
|
|
// status to match the resourceVersion of the updated ConfigMap.
|
|
func updateConfigMapFunc(f *framework.Framework, tc *nodeConfigTestCase) error {
|
|
// Clear ResourceVersion from the ConfigMap objects we use to initiate mutations
|
|
// so that we don't get 409 (conflict) responses. ConfigMaps always allow updates
|
|
// (with respect to concurrency control) when you omit ResourceVersion.
|
|
// We know that we won't perform concurrent updates during this test.
|
|
tc.configMap.ResourceVersion = ""
|
|
cm, err := f.ClientSet.CoreV1().ConfigMaps(tc.configMap.Namespace).Update(tc.configMap)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// update tc.configMap's ResourceVersion to match the updated ConfigMap, this makes
|
|
// sure our derived status checks have up-to-date information
|
|
tc.configMap.ResourceVersion = cm.ResourceVersion
|
|
return nil
|
|
}
|
|
|
|
// recreateConfigMapFunc deletes and recreates the ConfigMap described by tc.configMap.
|
|
// The new ConfigMap will match tc.configMap.
|
|
func recreateConfigMapFunc(f *framework.Framework, tc *nodeConfigTestCase) error {
|
|
// need to ignore NotFound error, since there could be cases where delete
|
|
// fails during a retry because the delete in a previous attempt succeeded,
|
|
// before some other error occurred.
|
|
err := deleteConfigMapFunc(f, tc)
|
|
if err != nil && !apierrors.IsNotFound(err) {
|
|
return err
|
|
}
|
|
return createConfigMapFunc(f, tc)
|
|
}
|
|
|
|
// deleteConfigMapFunc simply deletes tc.configMap
|
|
func deleteConfigMapFunc(f *framework.Framework, tc *nodeConfigTestCase) error {
|
|
return f.ClientSet.CoreV1().ConfigMaps(tc.configMap.Namespace).Delete(tc.configMap.Name, &metav1.DeleteOptions{})
|
|
}
|
|
|
|
// createConfigMapFunc creates tc.configMap and updates the UID and ResourceVersion on tc.configMap
|
|
// to match the created configMap
|
|
func createConfigMapFunc(f *framework.Framework, tc *nodeConfigTestCase) error {
|
|
tc.configMap.ResourceVersion = ""
|
|
cm, err := f.ClientSet.CoreV1().ConfigMaps(tc.configMap.Namespace).Create(tc.configMap)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// update tc.configMap's UID and ResourceVersion to match the new ConfigMap, this makes
|
|
// sure our derived status checks have up-to-date information
|
|
tc.configMap.UID = cm.UID
|
|
tc.configMap.ResourceVersion = cm.ResourceVersion
|
|
return nil
|
|
}
|
|
|
|
// make sure the node's config source matches what we expect, after setting it
|
|
func (tc *nodeConfigTestCase) checkNodeConfigSource(f *framework.Framework) {
|
|
const (
|
|
timeout = time.Minute
|
|
interval = time.Second
|
|
)
|
|
Eventually(func() error {
|
|
node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("checkNodeConfigSource: case %s: %v", tc.desc, err)
|
|
}
|
|
actual := node.Spec.ConfigSource
|
|
if !apiequality.Semantic.DeepEqual(tc.configSource, actual) {
|
|
return fmt.Errorf(spew.Sprintf("checkNodeConfigSource: case %s: expected %#v but got %#v", tc.desc, tc.configSource, actual))
|
|
}
|
|
return nil
|
|
}, timeout, interval).Should(BeNil())
|
|
}
|
|
|
|
// make sure the node status eventually matches what we expect
|
|
func (tc *nodeConfigTestCase) checkConfigStatus(f *framework.Framework) {
|
|
const (
|
|
timeout = time.Minute
|
|
interval = time.Second
|
|
)
|
|
errFmt := fmt.Sprintf("checkConfigStatus: case %s:", tc.desc) + " %v"
|
|
Eventually(func() error {
|
|
node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf(errFmt, err)
|
|
}
|
|
if err := expectConfigStatus(tc, node.Status.Config); err != nil {
|
|
return fmt.Errorf(errFmt, err)
|
|
}
|
|
return nil
|
|
}, timeout, interval).Should(BeNil())
|
|
}
|
|
|
|
func expectConfigStatus(tc *nodeConfigTestCase, actual *apiv1.NodeConfigStatus) error {
|
|
var errs []string
|
|
if actual == nil {
|
|
return fmt.Errorf("expectConfigStatus requires actual to be non-nil (possible Kubelet failed to update status)")
|
|
}
|
|
// check Assigned matches tc.configSource, with UID and ResourceVersion from tc.configMap
|
|
expectAssigned := tc.configSource.DeepCopy()
|
|
if expectAssigned != nil && expectAssigned.ConfigMap != nil {
|
|
expectAssigned.ConfigMap.UID = tc.configMap.UID
|
|
expectAssigned.ConfigMap.ResourceVersion = tc.configMap.ResourceVersion
|
|
}
|
|
if !apiequality.Semantic.DeepEqual(expectAssigned, actual.Assigned) {
|
|
errs = append(errs, spew.Sprintf("expected Assigned %#v but got %#v", expectAssigned, actual.Assigned))
|
|
}
|
|
// check LastKnownGood matches tc.expectConfigStatus.lastKnownGood
|
|
if !apiequality.Semantic.DeepEqual(tc.expectConfigStatus.lastKnownGood, actual.LastKnownGood) {
|
|
errs = append(errs, spew.Sprintf("expected LastKnownGood %#v but got %#v", tc.expectConfigStatus.lastKnownGood, actual.LastKnownGood))
|
|
}
|
|
// check Active matches Assigned or LastKnownGood, depending on tc.expectConfigStatus.lkgActive
|
|
expectActive := expectAssigned
|
|
if tc.expectConfigStatus.lkgActive {
|
|
expectActive = tc.expectConfigStatus.lastKnownGood
|
|
}
|
|
if !apiequality.Semantic.DeepEqual(expectActive, actual.Active) {
|
|
errs = append(errs, spew.Sprintf("expected Active %#v but got %#v", expectActive, actual.Active))
|
|
}
|
|
// check Error
|
|
if tc.expectConfigStatus.err != actual.Error {
|
|
errs = append(errs, fmt.Sprintf("expected Error %q but got %q", tc.expectConfigStatus.err, actual.Error))
|
|
}
|
|
// format error list
|
|
if len(errs) > 0 {
|
|
return fmt.Errorf("%s", strings.Join(errs, ", "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// make sure config exposed on configz matches what we expect
|
|
func (tc *nodeConfigTestCase) checkConfig(f *framework.Framework) {
|
|
const (
|
|
timeout = time.Minute
|
|
interval = time.Second
|
|
)
|
|
Eventually(func() error {
|
|
actual, err := getCurrentKubeletConfig()
|
|
if err != nil {
|
|
return fmt.Errorf("checkConfig: case %s: %v", tc.desc, err)
|
|
}
|
|
if !apiequality.Semantic.DeepEqual(tc.expectConfig, actual) {
|
|
return fmt.Errorf(spew.Sprintf("checkConfig: case %s: expected %#v but got %#v", tc.desc, tc.expectConfig, actual))
|
|
}
|
|
return nil
|
|
}, timeout, interval).Should(BeNil())
|
|
}
|
|
|
|
// checkEvent makes sure an event was sent marking the Kubelet's restart to use new config,
|
|
// and that it mentions the config we expect.
|
|
func (tc *nodeConfigTestCase) checkEvent(f *framework.Framework) {
|
|
const (
|
|
timeout = time.Minute
|
|
interval = time.Second
|
|
)
|
|
Eventually(func() error {
|
|
events, err := f.ClientSet.CoreV1().Events("").List(metav1.ListOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("checkEvent: case %s: %v", tc.desc, err)
|
|
}
|
|
// find config changed event with most recent timestamp
|
|
var recent *apiv1.Event
|
|
for i := range events.Items {
|
|
if events.Items[i].Reason == controller.KubeletConfigChangedEventReason {
|
|
if recent == nil {
|
|
recent = &events.Items[i]
|
|
continue
|
|
}
|
|
// for these events, first and last timestamp are always the same
|
|
if events.Items[i].FirstTimestamp.Time.After(recent.FirstTimestamp.Time) {
|
|
recent = &events.Items[i]
|
|
}
|
|
}
|
|
}
|
|
// we expect at least one config change event
|
|
if recent == nil {
|
|
return fmt.Errorf("checkEvent: case %s: no events found with reason %s", tc.desc, controller.KubeletConfigChangedEventReason)
|
|
}
|
|
// construct expected message, based on the test case
|
|
expectMessage := controller.LocalEventMessage
|
|
if tc.configSource != nil {
|
|
if tc.configSource.ConfigMap != nil {
|
|
expectMessage = fmt.Sprintf(controller.RemoteEventMessageFmt,
|
|
fmt.Sprintf("/api/v1/namespaces/%s/configmaps/%s", tc.configSource.ConfigMap.Namespace, tc.configSource.ConfigMap.Name),
|
|
tc.configMap.UID, tc.configMap.ResourceVersion, tc.configSource.ConfigMap.KubeletConfigKey)
|
|
}
|
|
}
|
|
// compare messages
|
|
if expectMessage != recent.Message {
|
|
return fmt.Errorf("checkEvent: case %s: expected event message %q but got %q", tc.desc, expectMessage, recent.Message)
|
|
}
|
|
return nil
|
|
}, timeout, interval).Should(BeNil())
|
|
}
|
|
|
|
// checkConfigMetrics makes sure the Kubelet's config related metrics are as we expect, given the test case
|
|
func (tc *nodeConfigTestCase) checkConfigMetrics(f *framework.Framework) {
|
|
const (
|
|
timeout = time.Minute
|
|
interval = time.Second
|
|
assignedConfigKey = metrics.KubeletSubsystem + "_" + metrics.AssignedConfigKey
|
|
activeConfigKey = metrics.KubeletSubsystem + "_" + metrics.ActiveConfigKey
|
|
lastKnownGoodConfigKey = metrics.KubeletSubsystem + "_" + metrics.LastKnownGoodConfigKey
|
|
configErrorKey = metrics.KubeletSubsystem + "_" + metrics.ConfigErrorKey
|
|
)
|
|
// local config helper
|
|
mkLocalSample := func(name model.LabelValue) *model.Sample {
|
|
return &model.Sample{
|
|
Metric: model.Metric(map[model.LabelName]model.LabelValue{
|
|
model.MetricNameLabel: name,
|
|
metrics.ConfigSourceLabelKey: metrics.ConfigSourceLabelValueLocal,
|
|
metrics.ConfigUIDLabelKey: "",
|
|
metrics.ConfigResourceVersionLabelKey: "",
|
|
metrics.KubeletConfigKeyLabelKey: "",
|
|
}),
|
|
Value: 1,
|
|
}
|
|
}
|
|
// remote config helper
|
|
mkRemoteSample := func(name model.LabelValue, source *apiv1.NodeConfigSource) *model.Sample {
|
|
return &model.Sample{
|
|
Metric: model.Metric(map[model.LabelName]model.LabelValue{
|
|
model.MetricNameLabel: name,
|
|
metrics.ConfigSourceLabelKey: model.LabelValue(fmt.Sprintf("/api/v1/namespaces/%s/configmaps/%s", source.ConfigMap.Namespace, source.ConfigMap.Name)),
|
|
metrics.ConfigUIDLabelKey: model.LabelValue(source.ConfigMap.UID),
|
|
metrics.ConfigResourceVersionLabelKey: model.LabelValue(source.ConfigMap.ResourceVersion),
|
|
metrics.KubeletConfigKeyLabelKey: model.LabelValue(source.ConfigMap.KubeletConfigKey),
|
|
}),
|
|
Value: 1,
|
|
}
|
|
}
|
|
// error helper
|
|
mkErrorSample := func(expectError bool) *model.Sample {
|
|
v := model.SampleValue(0)
|
|
if expectError {
|
|
v = model.SampleValue(1)
|
|
}
|
|
return &model.Sample{
|
|
Metric: model.Metric(map[model.LabelName]model.LabelValue{model.MetricNameLabel: configErrorKey}),
|
|
Value: v,
|
|
}
|
|
}
|
|
// construct expected metrics
|
|
// assigned
|
|
assignedSamples := model.Samples{mkLocalSample(assignedConfigKey)}
|
|
assignedSource := tc.configSource.DeepCopy()
|
|
if assignedSource != nil && assignedSource.ConfigMap != nil {
|
|
assignedSource.ConfigMap.UID = tc.configMap.UID
|
|
assignedSource.ConfigMap.ResourceVersion = tc.configMap.ResourceVersion
|
|
assignedSamples = model.Samples{mkRemoteSample(assignedConfigKey, assignedSource)}
|
|
}
|
|
// last-known-good
|
|
lastKnownGoodSamples := model.Samples{mkLocalSample(lastKnownGoodConfigKey)}
|
|
lastKnownGoodSource := tc.expectConfigStatus.lastKnownGood
|
|
if lastKnownGoodSource != nil && lastKnownGoodSource.ConfigMap != nil {
|
|
lastKnownGoodSamples = model.Samples{mkRemoteSample(lastKnownGoodConfigKey, lastKnownGoodSource)}
|
|
}
|
|
// active
|
|
activeSamples := model.Samples{mkLocalSample(activeConfigKey)}
|
|
activeSource := assignedSource
|
|
if tc.expectConfigStatus.lkgActive {
|
|
activeSource = lastKnownGoodSource
|
|
}
|
|
if activeSource != nil && activeSource.ConfigMap != nil {
|
|
activeSamples = model.Samples{mkRemoteSample(activeConfigKey, activeSource)}
|
|
}
|
|
// error
|
|
errorSamples := model.Samples{mkErrorSample(len(tc.expectConfigStatus.err) > 0)}
|
|
// expected metrics
|
|
expect := frameworkmetrics.KubeletMetrics(map[string]model.Samples{
|
|
assignedConfigKey: assignedSamples,
|
|
activeConfigKey: activeSamples,
|
|
lastKnownGoodConfigKey: lastKnownGoodSamples,
|
|
configErrorKey: errorSamples,
|
|
})
|
|
// wait for expected metrics to appear
|
|
Eventually(func() error {
|
|
actual, err := getKubeletMetrics(sets.NewString(
|
|
assignedConfigKey,
|
|
activeConfigKey,
|
|
lastKnownGoodConfigKey,
|
|
configErrorKey,
|
|
))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// clear timestamps from actual, so DeepEqual is time-invariant
|
|
for _, samples := range actual {
|
|
for _, sample := range samples {
|
|
sample.Timestamp = 0
|
|
}
|
|
}
|
|
// compare to expected
|
|
if !reflect.DeepEqual(expect, actual) {
|
|
return fmt.Errorf("checkConfigMetrics: case: %s: expect metrics %s but got %s", tc.desc, spew.Sprintf("%#v", expect), spew.Sprintf("%#v", actual))
|
|
}
|
|
return nil
|
|
}, timeout, interval).Should(BeNil())
|
|
}
|
|
|
|
// constructs the expected SelfLink for a config map
|
|
func configMapAPIPath(cm *apiv1.ConfigMap) string {
|
|
return fmt.Sprintf("/api/v1/namespaces/%s/configmaps/%s", cm.Namespace, cm.Name)
|
|
}
|