mirror of https://github.com/k3s-io/k3s
407 lines
17 KiB
Go
407 lines
17 KiB
Go
/*
|
|
Copyright 2016 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package e2e_node
|
|
|
|
import (
|
|
"fmt"
|
|
"reflect"
|
|
"time"
|
|
|
|
"github.com/davecgh/go-spew/spew"
|
|
|
|
apiv1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
|
|
"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/status"
|
|
|
|
"k8s.io/kubernetes/test/e2e/framework"
|
|
|
|
. "github.com/onsi/ginkgo"
|
|
. "github.com/onsi/gomega"
|
|
)
|
|
|
|
type configState struct {
|
|
desc string
|
|
configSource *apiv1.NodeConfigSource
|
|
expectConfigOK *apiv1.NodeCondition
|
|
expectConfig *kubeletconfig.KubeletConfiguration
|
|
}
|
|
|
|
// This test is marked [Disruptive] because the Kubelet restarts several times during this test.
|
|
var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKubeletConfig] [Serial] [Disruptive]", func() {
|
|
f := framework.NewDefaultFramework("dynamic-kubelet-configuration-test")
|
|
var originalKC *kubeletconfig.KubeletConfiguration
|
|
var originalConfigMap *apiv1.ConfigMap
|
|
|
|
// Dummy context to prevent framework's AfterEach from cleaning up before this test's AfterEach can run
|
|
Context("", func() {
|
|
BeforeEach(func() {
|
|
var err error
|
|
if originalConfigMap == nil {
|
|
originalKC, err = getCurrentKubeletConfig()
|
|
framework.ExpectNoError(err)
|
|
originalConfigMap = newKubeletConfigMap("original-values", originalKC)
|
|
originalConfigMap, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(originalConfigMap)
|
|
framework.ExpectNoError(err)
|
|
}
|
|
// make sure Dynamic Kubelet Configuration feature is enabled on the Kubelet we are about to test
|
|
enabled, err := isKubeletConfigEnabled(f)
|
|
framework.ExpectNoError(err)
|
|
if !enabled {
|
|
framework.ExpectNoError(fmt.Errorf("The Dynamic Kubelet Configuration feature is not enabled.\n" +
|
|
"Pass --feature-gates=DynamicKubeletConfig=true to the Kubelet to enable this feature.\n" +
|
|
"For `make test-e2e-node`, you can set `TEST_ARGS='--feature-gates=DynamicKubeletConfig=true'`."))
|
|
}
|
|
})
|
|
|
|
AfterEach(func() {
|
|
// Set the config back to the original values before moving on.
|
|
// We care that the values are the same, not where they come from, so it
|
|
// should be fine to reset the values using a remote config, even if they
|
|
// were initially set via the locally provisioned configuration.
|
|
// This is the same strategy several other e2e node tests use.
|
|
setAndTestKubeletConfigState(f, &configState{desc: "reset to original values",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{
|
|
UID: originalConfigMap.UID,
|
|
Namespace: originalConfigMap.Namespace,
|
|
Name: originalConfigMap.Name}},
|
|
expectConfigOK: &apiv1.NodeCondition{Type: apiv1.NodeConfigOK, Status: apiv1.ConditionTrue,
|
|
Message: fmt.Sprintf(status.CurRemoteMessageFmt, originalConfigMap.UID),
|
|
Reason: status.CurRemoteOKReason},
|
|
expectConfig: originalKC})
|
|
})
|
|
|
|
Context("When setting new NodeConfigSources that cause transitions between ConfigOK conditions", func() {
|
|
It("the Kubelet should report the appropriate status and configz", func() {
|
|
var err error
|
|
// we base the "correct" configmap off of the current configuration,
|
|
// but we also set the trial duration very high to prevent changing the last-known-good
|
|
correctKC := originalKC.DeepCopy()
|
|
correctKC.ConfigTrialDuration = &metav1.Duration{Duration: time.Hour}
|
|
correctConfigMap := newKubeletConfigMap("dynamic-kubelet-config-test-correct", correctKC)
|
|
correctConfigMap, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(correctConfigMap)
|
|
framework.ExpectNoError(err)
|
|
|
|
// fail to parse, we insert some bogus stuff into the configMap
|
|
failParseConfigMap := &apiv1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "dynamic-kubelet-config-test-fail-parse"},
|
|
Data: map[string]string{
|
|
"kubelet": "{0xdeadbeef}",
|
|
},
|
|
}
|
|
failParseConfigMap, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(failParseConfigMap)
|
|
framework.ExpectNoError(err)
|
|
|
|
// fail to validate, we make a copy and set an invalid KubeAPIQPS on kc before serializing
|
|
invalidKC := correctKC.DeepCopy()
|
|
|
|
invalidKC.KubeAPIQPS = -1
|
|
failValidateConfigMap := newKubeletConfigMap("dynamic-kubelet-config-test-fail-validate", invalidKC)
|
|
failValidateConfigMap, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(failValidateConfigMap)
|
|
framework.ExpectNoError(err)
|
|
|
|
states := []configState{
|
|
// Node.Spec.ConfigSource is nil
|
|
{desc: "Node.Spec.ConfigSource is nil",
|
|
configSource: nil,
|
|
expectConfigOK: &apiv1.NodeCondition{Type: apiv1.NodeConfigOK, Status: apiv1.ConditionTrue,
|
|
Message: status.CurDefaultMessage,
|
|
Reason: status.CurDefaultOKReason},
|
|
expectConfig: nil},
|
|
|
|
// Node.Spec.ConfigSource has all nil subfields
|
|
{desc: "Node.Spec.ConfigSource has all nil subfields",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMapRef: nil},
|
|
expectConfigOK: &apiv1.NodeCondition{Type: apiv1.NodeConfigOK, Status: apiv1.ConditionFalse,
|
|
Message: "",
|
|
Reason: fmt.Sprintf(status.FailSyncReasonFmt, status.FailSyncReasonAllNilSubfields)},
|
|
expectConfig: nil},
|
|
|
|
// Node.Spec.ConfigSource.ConfigMapRef is partial
|
|
{desc: "Node.Spec.ConfigSource.ConfigMapRef is partial",
|
|
// TODO(mtaufen): check the other 7 partials in a unit test
|
|
configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{
|
|
UID: "foo",
|
|
Name: "bar"}}, // missing Namespace
|
|
expectConfigOK: &apiv1.NodeCondition{Type: apiv1.NodeConfigOK, Status: apiv1.ConditionFalse,
|
|
Message: "",
|
|
Reason: fmt.Sprintf(status.FailSyncReasonFmt, status.FailSyncReasonPartialObjectReference)},
|
|
expectConfig: nil},
|
|
|
|
// Node.Spec.ConfigSource's UID does not align with namespace/name
|
|
{desc: "Node.Spec.ConfigSource's UID does not align with namespace/name",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{UID: "foo",
|
|
Namespace: correctConfigMap.Namespace,
|
|
Name: correctConfigMap.Name}},
|
|
expectConfigOK: &apiv1.NodeCondition{Type: apiv1.NodeConfigOK, Status: apiv1.ConditionFalse,
|
|
Message: "",
|
|
Reason: fmt.Sprintf(status.FailSyncReasonFmt, fmt.Sprintf(status.FailSyncReasonUIDMismatchFmt, "foo", correctConfigMap.UID))},
|
|
expectConfig: nil},
|
|
|
|
// correct
|
|
{desc: "correct",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{
|
|
UID: correctConfigMap.UID,
|
|
Namespace: correctConfigMap.Namespace,
|
|
Name: correctConfigMap.Name}},
|
|
expectConfigOK: &apiv1.NodeCondition{Type: apiv1.NodeConfigOK, Status: apiv1.ConditionTrue,
|
|
Message: fmt.Sprintf(status.CurRemoteMessageFmt, correctConfigMap.UID),
|
|
Reason: status.CurRemoteOKReason},
|
|
expectConfig: correctKC},
|
|
|
|
// fail-parse
|
|
{desc: "fail-parse",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{
|
|
UID: failParseConfigMap.UID,
|
|
Namespace: failParseConfigMap.Namespace,
|
|
Name: failParseConfigMap.Name}},
|
|
expectConfigOK: &apiv1.NodeCondition{Type: apiv1.NodeConfigOK, Status: apiv1.ConditionFalse,
|
|
Message: status.LkgDefaultMessage,
|
|
Reason: fmt.Sprintf(status.CurFailParseReasonFmt, failParseConfigMap.UID)},
|
|
expectConfig: nil},
|
|
|
|
// fail-validate
|
|
{desc: "fail-validate",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{
|
|
UID: failValidateConfigMap.UID,
|
|
Namespace: failValidateConfigMap.Namespace,
|
|
Name: failValidateConfigMap.Name}},
|
|
expectConfigOK: &apiv1.NodeCondition{Type: apiv1.NodeConfigOK, Status: apiv1.ConditionFalse,
|
|
Message: status.LkgDefaultMessage,
|
|
Reason: fmt.Sprintf(status.CurFailValidateReasonFmt, failValidateConfigMap.UID)},
|
|
expectConfig: nil},
|
|
}
|
|
|
|
L := len(states)
|
|
for i := 1; i <= L; i++ { // need one less iteration than the number of states
|
|
testBothDirections(f, &states[i-1 : i][0], states[i:L])
|
|
}
|
|
|
|
})
|
|
})
|
|
|
|
Context("When a remote config becomes the new last-known-good before the Kubelet is updated to use a new, bad config", func() {
|
|
It("it should report a status and configz indicating that it rolled back to the new last-known-good", func() {
|
|
var err error
|
|
// we base the "lkg" configmap off of the current configuration, but set the trial
|
|
// duration very low so that it quickly becomes the last-known-good
|
|
lkgKC := originalKC.DeepCopy()
|
|
lkgKC.ConfigTrialDuration = &metav1.Duration{Duration: time.Nanosecond}
|
|
lkgConfigMap := newKubeletConfigMap("dynamic-kubelet-config-test-intended-lkg", lkgKC)
|
|
lkgConfigMap, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(lkgConfigMap)
|
|
framework.ExpectNoError(err)
|
|
|
|
// bad config map, we insert some bogus stuff into the configMap
|
|
badConfigMap := &apiv1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "dynamic-kubelet-config-test-bad"},
|
|
Data: map[string]string{
|
|
"kubelet": "{0xdeadbeef}",
|
|
},
|
|
}
|
|
badConfigMap, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(badConfigMap)
|
|
framework.ExpectNoError(err)
|
|
|
|
states := []configState{
|
|
// intended lkg
|
|
{desc: "intended last-known-good",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{
|
|
UID: lkgConfigMap.UID,
|
|
Namespace: lkgConfigMap.Namespace,
|
|
Name: lkgConfigMap.Name}},
|
|
expectConfigOK: &apiv1.NodeCondition{Type: apiv1.NodeConfigOK, Status: apiv1.ConditionTrue,
|
|
Message: fmt.Sprintf(status.CurRemoteMessageFmt, lkgConfigMap.UID),
|
|
Reason: status.CurRemoteOKReason},
|
|
expectConfig: lkgKC},
|
|
|
|
// bad config
|
|
{desc: "bad config",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{
|
|
UID: badConfigMap.UID,
|
|
Namespace: badConfigMap.Namespace,
|
|
Name: badConfigMap.Name}},
|
|
expectConfigOK: &apiv1.NodeCondition{Type: apiv1.NodeConfigOK, Status: apiv1.ConditionFalse,
|
|
Message: fmt.Sprintf(status.LkgRemoteMessageFmt, lkgConfigMap.UID),
|
|
Reason: fmt.Sprintf(status.CurFailParseReasonFmt, badConfigMap.UID)},
|
|
expectConfig: lkgKC},
|
|
}
|
|
|
|
testBothDirections(f, &states[0], states[1:])
|
|
})
|
|
})
|
|
|
|
// This stress test will help turn up resource leaks across kubelet restarts that can, over time,
|
|
// break our ability to dynamically update kubelet config
|
|
Context("When changing the configuration 100 times", func() {
|
|
It("the Kubelet should report the appropriate status and configz", func() {
|
|
var err error
|
|
|
|
// we just create two configmaps with the same config but different names and toggle between them
|
|
kc1 := originalKC.DeepCopy()
|
|
cm1 := newKubeletConfigMap("dynamic-kubelet-config-test-cm1", kc1)
|
|
cm1, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(cm1)
|
|
framework.ExpectNoError(err)
|
|
|
|
// slightly change the config
|
|
kc2 := kc1.DeepCopy()
|
|
kc2.EventRecordQPS = kc1.EventRecordQPS + 1
|
|
cm2 := newKubeletConfigMap("dynamic-kubelet-config-test-cm2", kc2)
|
|
cm2, err = f.ClientSet.CoreV1().ConfigMaps("kube-system").Create(cm2)
|
|
framework.ExpectNoError(err)
|
|
|
|
states := []configState{
|
|
{desc: "cm1",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{
|
|
UID: cm1.UID,
|
|
Namespace: cm1.Namespace,
|
|
Name: cm1.Name}},
|
|
expectConfigOK: &apiv1.NodeCondition{Type: apiv1.NodeConfigOK, Status: apiv1.ConditionTrue,
|
|
Message: fmt.Sprintf(status.CurRemoteMessageFmt, cm1.UID),
|
|
Reason: status.CurRemoteOKReason},
|
|
expectConfig: kc1},
|
|
{desc: "cm2",
|
|
configSource: &apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{
|
|
UID: cm2.UID,
|
|
Namespace: cm2.Namespace,
|
|
Name: cm2.Name}},
|
|
expectConfigOK: &apiv1.NodeCondition{Type: apiv1.NodeConfigOK, Status: apiv1.ConditionTrue,
|
|
Message: fmt.Sprintf(status.CurRemoteMessageFmt, cm2.UID),
|
|
Reason: status.CurRemoteOKReason},
|
|
expectConfig: kc2},
|
|
}
|
|
|
|
for i := 0; i < 50; i++ { // change the config 101 times (changes 3 times in the first iteration, 2 times in each subsequent iteration)
|
|
testBothDirections(f, &states[0], states[1:])
|
|
}
|
|
})
|
|
})
|
|
})
|
|
})
|
|
|
|
// testBothDirections tests the state change represented by each edge, where each state is a vertex,
|
|
// and there are edges in each direction between first and each of the states.
|
|
func testBothDirections(f *framework.Framework, first *configState, states []configState) {
|
|
// set to first and check that everything got set up properly
|
|
By(fmt.Sprintf("setting configSource to state %q", first.desc))
|
|
setAndTestKubeletConfigState(f, first)
|
|
|
|
// for each state, set to that state, check condition and configz, then reset to first and check again
|
|
for i := range states {
|
|
By(fmt.Sprintf("from %q to %q", first.desc, states[i].desc))
|
|
setAndTestKubeletConfigState(f, &states[i])
|
|
|
|
By(fmt.Sprintf("back to %q from %q", first.desc, states[i].desc))
|
|
setAndTestKubeletConfigState(f, first)
|
|
}
|
|
}
|
|
|
|
// setAndTestKubeletConfigState tests that after setting the config source, the ConfigOK condition
|
|
// and (if appropriate) configuration exposed via conifgz are as expected.
|
|
// The configuration will be converted to the internal type prior to comparison.
|
|
func setAndTestKubeletConfigState(f *framework.Framework, state *configState) {
|
|
// set the desired state, retry a few times in case we are competing with other editors
|
|
Eventually(func() error {
|
|
if err := setNodeConfigSource(f, state.configSource); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}, time.Minute, time.Second).Should(BeNil())
|
|
// check that config source actually got set to what we expect
|
|
checkNodeConfigSource(f, state.configSource)
|
|
// check condition
|
|
checkConfigOKCondition(f, state.expectConfigOK)
|
|
// check expectConfig
|
|
if state.expectConfig != nil {
|
|
checkConfig(f, state.expectConfig)
|
|
}
|
|
}
|
|
|
|
// make sure the node's config source matches what we expect, after setting it
|
|
func checkNodeConfigSource(f *framework.Framework, expect *apiv1.NodeConfigSource) {
|
|
const (
|
|
timeout = time.Minute
|
|
interval = time.Second
|
|
)
|
|
|
|
Eventually(func() error {
|
|
node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
actual := node.Spec.ConfigSource
|
|
if !reflect.DeepEqual(expect, actual) {
|
|
return fmt.Errorf(spew.Sprintf("expected %#v but got %#v", expect, actual))
|
|
}
|
|
return nil
|
|
}, timeout, interval).Should(BeNil())
|
|
}
|
|
|
|
// make sure the ConfigOK node condition eventually matches what we expect
|
|
func checkConfigOKCondition(f *framework.Framework, expect *apiv1.NodeCondition) {
|
|
const (
|
|
timeout = time.Minute
|
|
interval = time.Second
|
|
)
|
|
|
|
Eventually(func() error {
|
|
node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
actual := getConfigOKCondition(node.Status.Conditions)
|
|
if actual == nil {
|
|
return fmt.Errorf("ConfigOK condition not found on node %q", framework.TestContext.NodeName)
|
|
}
|
|
if err := expectConfigOK(expect, actual); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}, timeout, interval).Should(BeNil())
|
|
}
|
|
|
|
// if the actual matches the expect, return nil, else error explaining the mismatch
|
|
// if a subfield of the expect is the empty string, that check is skipped
|
|
func expectConfigOK(expect, actual *apiv1.NodeCondition) error {
|
|
if expect.Status != actual.Status {
|
|
return fmt.Errorf("expected condition Status %q but got %q", expect.Status, actual.Status)
|
|
}
|
|
if len(expect.Message) > 0 && expect.Message != actual.Message {
|
|
return fmt.Errorf("expected condition Message %q but got %q", expect.Message, actual.Message)
|
|
}
|
|
if len(expect.Reason) > 0 && expect.Reason != actual.Reason {
|
|
return fmt.Errorf("expected condition Reason %q but got %q", expect.Reason, actual.Reason)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// make sure config exposed on configz matches what we expect
|
|
func checkConfig(f *framework.Framework, expect *kubeletconfig.KubeletConfiguration) {
|
|
const (
|
|
timeout = time.Minute
|
|
interval = time.Second
|
|
)
|
|
Eventually(func() error {
|
|
actual, err := getCurrentKubeletConfig()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !reflect.DeepEqual(expect, actual) {
|
|
return fmt.Errorf(spew.Sprintf("expected %#v but got %#v", expect, actual))
|
|
}
|
|
return nil
|
|
}, timeout, interval).Should(BeNil())
|
|
}
|