/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubelet

import (
	"encoding/json"
	"fmt"
	"net"
	goruntime "runtime"
	"sort"
	"strconv"
	"sync/atomic"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	cadvisorapi "github.com/google/cadvisor/info/v1"
	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
	"k8s.io/api/core/v1"
	apiequality "k8s.io/apimachinery/pkg/api/equality"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/diff"
	"k8s.io/apimachinery/pkg/util/rand"
	"k8s.io/apimachinery/pkg/util/strategicpatch"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes/fake"
	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/rest"
	core "k8s.io/client-go/testing"
	fakecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/fake"
	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
	"k8s.io/kubernetes/pkg/kubelet/cm"
	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
	"k8s.io/kubernetes/pkg/kubelet/util/sliceutils"
	"k8s.io/kubernetes/pkg/version"
	"k8s.io/kubernetes/pkg/volume/util/volumehelper"
)

const (
	maxImageTagsForTest = 20
)

// generateTestingImageList generate randomly generated image list and corresponding expectedImageList.
func generateTestingImageList(count int) ([]kubecontainer.Image, []v1.ContainerImage) {
	// imageList is randomly generated image list
	var imageList []kubecontainer.Image
	for ; count > 0; count-- {
		imageItem := kubecontainer.Image{
			ID:       string(uuid.NewUUID()),
			RepoTags: generateImageTags(),
			Size:     rand.Int63nRange(minImgSize, maxImgSize+1),
		}
		imageList = append(imageList, imageItem)
	}

	// expectedImageList is generated by imageList according to size and maxImagesInNodeStatus
	// 1. sort the imageList by size
	sort.Sort(sliceutils.ByImageSize(imageList))
	// 2. convert sorted imageList to v1.ContainerImage list
	var expectedImageList []v1.ContainerImage
	for _, kubeImage := range imageList {
		apiImage := v1.ContainerImage{
			Names:     kubeImage.RepoTags[0:maxNamesPerImageInNodeStatus],
			SizeBytes: kubeImage.Size,
		}

		expectedImageList = append(expectedImageList, apiImage)
	}
	// 3. only returns the top maxImagesInNodeStatus images in expectedImageList
	return imageList, expectedImageList[0:maxImagesInNodeStatus]
}

func generateImageTags() []string {
	var tagList []string
	// Generate > maxNamesPerImageInNodeStatus tags so that the test can verify
	// that kubelet report up to maxNamesPerImageInNodeStatus tags.
	count := rand.IntnRange(maxNamesPerImageInNodeStatus+1, maxImageTagsForTest+1)
	for ; count > 0; count-- {
		tagList = append(tagList, "gcr.io/google_containers:v"+strconv.Itoa(count))
	}
	return tagList
}

func applyNodeStatusPatch(originalNode *v1.Node, patch []byte) (*v1.Node, error) {
	original, err := json.Marshal(originalNode)
	if err != nil {
		return nil, fmt.Errorf("failed to marshal original node %#v: %v", originalNode, err)
	}
	updated, err := strategicpatch.StrategicMergePatch(original, patch, v1.Node{})
	if err != nil {
		return nil, fmt.Errorf("failed to apply strategic merge patch %q on node %#v: %v",
			patch, originalNode, err)
	}
	updatedNode := &v1.Node{}
	if err := json.Unmarshal(updated, updatedNode); err != nil {
		return nil, fmt.Errorf("failed to unmarshal updated node %q: %v", updated, err)
	}
	return updatedNode, nil
}

type localCM struct {
	cm.ContainerManager
	allocatable v1.ResourceList
	capacity    v1.ResourceList
}

func (lcm *localCM) GetNodeAllocatableReservation() v1.ResourceList {
	return lcm.allocatable
}

func (lcm *localCM) GetCapacity() v1.ResourceList {
	return lcm.capacity
}

func TestNodeStatusWithCloudProviderNodeIP(t *testing.T) {
	testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
	defer testKubelet.Cleanup()
	kubelet := testKubelet.kubelet
	kubelet.kubeClient = nil // ensure only the heartbeat client is used
	kubelet.hostname = testKubeletHostname

	existingNode := v1.Node{
		ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname, Annotations: make(map[string]string)},
		Spec:       v1.NodeSpec{},
	}

	// TODO : is it possible to mock validateNodeIP() to avoid relying on the host interface addresses ?
	addrs, err := net.InterfaceAddrs()
	assert.NoError(t, err)
	for _, addr := range addrs {
		var ip net.IP
		switch v := addr.(type) {
		case *net.IPNet:
			ip = v.IP
		case *net.IPAddr:
			ip = v.IP
		}
		if ip != nil && !ip.IsLoopback() && ip.To4() != nil {
			kubelet.nodeIP = ip
			break
		}
	}
	assert.NotNil(t, kubelet.nodeIP)

	fakeCloud := &fakecloud.FakeCloud{
		Addresses: []v1.NodeAddress{
			{
				Type:    v1.NodeExternalIP,
				Address: "132.143.154.163",
			},
			{
				Type:    v1.NodeExternalIP,
				Address: kubelet.nodeIP.String(),
			},
			{
				Type:    v1.NodeInternalIP,
				Address: "132.143.154.164",
			},
			{
				Type:    v1.NodeInternalIP,
				Address: kubelet.nodeIP.String(),
			},
			{
				Type:    v1.NodeInternalIP,
				Address: "132.143.154.165",
			},
			{
				Type:    v1.NodeHostName,
				Address: testKubeletHostname,
			},
		},
		Err: nil,
	}
	kubelet.cloud = fakeCloud

	kubelet.setNodeAddress(&existingNode)

	expectedAddresses := []v1.NodeAddress{
		{
			Type:    v1.NodeExternalIP,
			Address: kubelet.nodeIP.String(),
		},
		{
			Type:    v1.NodeInternalIP,
			Address: kubelet.nodeIP.String(),
		},
		{
			Type:    v1.NodeHostName,
			Address: testKubeletHostname,
		},
	}
	assert.True(t, apiequality.Semantic.DeepEqual(expectedAddresses, existingNode.Status.Addresses), "%s", diff.ObjectDiff(expectedAddresses, existingNode.Status.Addresses))
}

func TestUpdateNewNodeStatus(t *testing.T) {
	// generate one more than maxImagesInNodeStatus in inputImageList
	inputImageList, expectedImageList := generateTestingImageList(maxImagesInNodeStatus + 1)
	testKubelet := newTestKubeletWithImageList(
		t, inputImageList, false /* controllerAttachDetachEnabled */)
	defer testKubelet.Cleanup()
	kubelet := testKubelet.kubelet
	kubelet.kubeClient = nil // ensure only the heartbeat client is used
	kubelet.containerManager = &localCM{
		ContainerManager: cm.NewStubContainerManager(),
		allocatable: v1.ResourceList{
			v1.ResourceCPU:    *resource.NewMilliQuantity(200, resource.DecimalSI),
			v1.ResourceMemory: *resource.NewQuantity(100E6, resource.BinarySI),
		},
		capacity: v1.ResourceList{
			v1.ResourceCPU:    *resource.NewMilliQuantity(2000, resource.DecimalSI),
			v1.ResourceMemory: *resource.NewQuantity(10E9, resource.BinarySI),
		},
	}
	kubeClient := testKubelet.fakeKubeClient
	existingNode := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}}
	kubeClient.ReactionChain = fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{existingNode}}).ReactionChain
	machineInfo := &cadvisorapi.MachineInfo{
		MachineID:      "123",
		SystemUUID:     "abc",
		BootID:         "1b3",
		NumCores:       2,
		MemoryCapacity: 10E9, // 10G
	}
	mockCadvisor := testKubelet.fakeCadvisor
	mockCadvisor.On("Start").Return(nil)
	mockCadvisor.On("MachineInfo").Return(machineInfo, nil)
	versionInfo := &cadvisorapi.VersionInfo{
		KernelVersion:      "3.16.0-0.bpo.4-amd64",
		ContainerOsVersion: "Debian GNU/Linux 7 (wheezy)",
	}
	mockCadvisor.On("VersionInfo").Return(versionInfo, nil)

	expectedNode := &v1.Node{
		ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
		Spec:       v1.NodeSpec{},
		Status: v1.NodeStatus{
			Conditions: []v1.NodeCondition{
				{
					Type:               v1.NodeOutOfDisk,
					Status:             v1.ConditionFalse,
					Reason:             "KubeletHasSufficientDisk",
					Message:            fmt.Sprintf("kubelet has sufficient disk space available"),
					LastHeartbeatTime:  metav1.Time{},
					LastTransitionTime: metav1.Time{},
				},
				{
					Type:               v1.NodeMemoryPressure,
					Status:             v1.ConditionFalse,
					Reason:             "KubeletHasSufficientMemory",
					Message:            fmt.Sprintf("kubelet has sufficient memory available"),
					LastHeartbeatTime:  metav1.Time{},
					LastTransitionTime: metav1.Time{},
				},
				{
					Type:               v1.NodeDiskPressure,
					Status:             v1.ConditionFalse,
					Reason:             "KubeletHasNoDiskPressure",
					Message:            fmt.Sprintf("kubelet has no disk pressure"),
					LastHeartbeatTime:  metav1.Time{},
					LastTransitionTime: metav1.Time{},
				},
				{
					Type:               v1.NodeReady,
					Status:             v1.ConditionTrue,
					Reason:             "KubeletReady",
					Message:            fmt.Sprintf("kubelet is posting ready status"),
					LastHeartbeatTime:  metav1.Time{},
					LastTransitionTime: metav1.Time{},
				},
			},
			NodeInfo: v1.NodeSystemInfo{
				MachineID:               "123",
				SystemUUID:              "abc",
				BootID:                  "1b3",
				KernelVersion:           "3.16.0-0.bpo.4-amd64",
				OSImage:                 "Debian GNU/Linux 7 (wheezy)",
				OperatingSystem:         goruntime.GOOS,
				Architecture:            goruntime.GOARCH,
				ContainerRuntimeVersion: "test://1.5.0",
				KubeletVersion:          version.Get().String(),
				KubeProxyVersion:        version.Get().String(),
			},
			Capacity: v1.ResourceList{
				v1.ResourceCPU:    *resource.NewMilliQuantity(2000, resource.DecimalSI),
				v1.ResourceMemory: *resource.NewQuantity(10E9, resource.BinarySI),
				v1.ResourcePods:   *resource.NewQuantity(0, resource.DecimalSI),
			},
			Allocatable: v1.ResourceList{
				v1.ResourceCPU:    *resource.NewMilliQuantity(1800, resource.DecimalSI),
				v1.ResourceMemory: *resource.NewQuantity(9900E6, resource.BinarySI),
				v1.ResourcePods:   *resource.NewQuantity(0, resource.DecimalSI),
			},
			Addresses: []v1.NodeAddress{
				{Type: v1.NodeInternalIP, Address: "127.0.0.1"},
				{Type: v1.NodeHostName, Address: testKubeletHostname},
			},
			Images: expectedImageList,
		},
	}

	kubelet.updateRuntimeUp()
	assert.NoError(t, kubelet.updateNodeStatus())
	actions := kubeClient.Actions()
	require.Len(t, actions, 2)
	require.True(t, actions[1].Matches("patch", "nodes"))
	require.Equal(t, actions[1].GetSubresource(), "status")

	updatedNode, err := applyNodeStatusPatch(&existingNode, actions[1].(core.PatchActionImpl).GetPatch())
	assert.NoError(t, err)
	for i, cond := range updatedNode.Status.Conditions {
		assert.False(t, cond.LastHeartbeatTime.IsZero(), "LastHeartbeatTime for %v condition is zero", cond.Type)
		assert.False(t, cond.LastTransitionTime.IsZero(), "LastTransitionTime for %v condition  is zero", cond.Type)
		updatedNode.Status.Conditions[i].LastHeartbeatTime = metav1.Time{}
		updatedNode.Status.Conditions[i].LastTransitionTime = metav1.Time{}
	}

	// Version skew workaround. See: https://github.com/kubernetes/kubernetes/issues/16961
	assert.Equal(t, v1.NodeReady, updatedNode.Status.Conditions[len(updatedNode.Status.Conditions)-1].Type, "NotReady should be last")
	assert.Len(t, updatedNode.Status.Images, maxImagesInNodeStatus)
	assert.True(t, apiequality.Semantic.DeepEqual(expectedNode, updatedNode), "%s", diff.ObjectDiff(expectedNode, updatedNode))
}

func TestUpdateExistingNodeStatus(t *testing.T) {
	testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
	defer testKubelet.Cleanup()
	kubelet := testKubelet.kubelet
	kubelet.kubeClient = nil // ensure only the heartbeat client is used
	kubelet.containerManager = &localCM{
		ContainerManager: cm.NewStubContainerManager(),
		allocatable: v1.ResourceList{
			v1.ResourceCPU:    *resource.NewMilliQuantity(200, resource.DecimalSI),
			v1.ResourceMemory: *resource.NewQuantity(100E6, resource.BinarySI),
		},
		capacity: v1.ResourceList{
			v1.ResourceCPU:    *resource.NewMilliQuantity(2000, resource.DecimalSI),
			v1.ResourceMemory: *resource.NewQuantity(20E9, resource.BinarySI),
		},
	}

	kubeClient := testKubelet.fakeKubeClient
	existingNode := v1.Node{
		ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
		Spec:       v1.NodeSpec{},
		Status: v1.NodeStatus{
			Conditions: []v1.NodeCondition{
				{
					Type:               v1.NodeOutOfDisk,
					Status:             v1.ConditionFalse,
					Reason:             "KubeletHasSufficientDisk",
					Message:            fmt.Sprintf("kubelet has sufficient disk space available"),
					LastHeartbeatTime:  metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
					LastTransitionTime: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
				},
				{
					Type:               v1.NodeMemoryPressure,
					Status:             v1.ConditionFalse,
					Reason:             "KubeletHasSufficientMemory",
					Message:            fmt.Sprintf("kubelet has sufficient memory available"),
					LastHeartbeatTime:  metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
					LastTransitionTime: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
				},
				{
					Type:               v1.NodeDiskPressure,
					Status:             v1.ConditionFalse,
					Reason:             "KubeletHasSufficientDisk",
					Message:            fmt.Sprintf("kubelet has sufficient disk space available"),
					LastHeartbeatTime:  metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
					LastTransitionTime: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
				},
				{
					Type:               v1.NodeReady,
					Status:             v1.ConditionTrue,
					Reason:             "KubeletReady",
					Message:            fmt.Sprintf("kubelet is posting ready status"),
					LastHeartbeatTime:  metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
					LastTransitionTime: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
				},
			},
			Capacity: v1.ResourceList{
				v1.ResourceCPU:    *resource.NewMilliQuantity(3000, resource.DecimalSI),
				v1.ResourceMemory: *resource.NewQuantity(20E9, resource.BinarySI),
				v1.ResourcePods:   *resource.NewQuantity(0, resource.DecimalSI),
			},
			Allocatable: v1.ResourceList{
				v1.ResourceCPU:    *resource.NewMilliQuantity(2800, resource.DecimalSI),
				v1.ResourceMemory: *resource.NewQuantity(19900E6, resource.BinarySI),
				v1.ResourcePods:   *resource.NewQuantity(0, resource.DecimalSI),
			},
		},
	}
	kubeClient.ReactionChain = fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{existingNode}}).ReactionChain
	mockCadvisor := testKubelet.fakeCadvisor
	mockCadvisor.On("Start").Return(nil)
	machineInfo := &cadvisorapi.MachineInfo{
		MachineID:      "123",
		SystemUUID:     "abc",
		BootID:         "1b3",
		NumCores:       2,
		MemoryCapacity: 20E9,
	}
	mockCadvisor.On("MachineInfo").Return(machineInfo, nil)
	versionInfo := &cadvisorapi.VersionInfo{
		KernelVersion:      "3.16.0-0.bpo.4-amd64",
		ContainerOsVersion: "Debian GNU/Linux 7 (wheezy)",
	}
	mockCadvisor.On("VersionInfo").Return(versionInfo, nil)

	expectedNode := &v1.Node{
		ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
		Spec:       v1.NodeSpec{},
		Status: v1.NodeStatus{
			Conditions: []v1.NodeCondition{
				{
					Type:               v1.NodeOutOfDisk,
					Status:             v1.ConditionFalse,
					Reason:             "KubeletHasSufficientDisk",
					Message:            fmt.Sprintf("kubelet has sufficient disk space available"),
					LastHeartbeatTime:  metav1.Time{},
					LastTransitionTime: metav1.Time{},
				},
				{
					Type:               v1.NodeMemoryPressure,
					Status:             v1.ConditionFalse,
					Reason:             "KubeletHasSufficientMemory",
					Message:            fmt.Sprintf("kubelet has sufficient memory available"),
					LastHeartbeatTime:  metav1.Time{},
					LastTransitionTime: metav1.Time{},
				},
				{
					Type:               v1.NodeDiskPressure,
					Status:             v1.ConditionFalse,
					Reason:             "KubeletHasSufficientDisk",
					Message:            fmt.Sprintf("kubelet has sufficient disk space available"),
					LastHeartbeatTime:  metav1.Time{},
					LastTransitionTime: metav1.Time{},
				},
				{
					Type:               v1.NodeReady,
					Status:             v1.ConditionTrue,
					Reason:             "KubeletReady",
					Message:            fmt.Sprintf("kubelet is posting ready status"),
					LastHeartbeatTime:  metav1.Time{}, // placeholder
					LastTransitionTime: metav1.Time{}, // placeholder
				},
			},
			NodeInfo: v1.NodeSystemInfo{
				MachineID:               "123",
				SystemUUID:              "abc",
				BootID:                  "1b3",
				KernelVersion:           "3.16.0-0.bpo.4-amd64",
				OSImage:                 "Debian GNU/Linux 7 (wheezy)",
				OperatingSystem:         goruntime.GOOS,
				Architecture:            goruntime.GOARCH,
				ContainerRuntimeVersion: "test://1.5.0",
				KubeletVersion:          version.Get().String(),
				KubeProxyVersion:        version.Get().String(),
			},
			Capacity: v1.ResourceList{
				v1.ResourceCPU:    *resource.NewMilliQuantity(2000, resource.DecimalSI),
				v1.ResourceMemory: *resource.NewQuantity(20E9, resource.BinarySI),
				v1.ResourcePods:   *resource.NewQuantity(0, resource.DecimalSI),
			},
			Allocatable: v1.ResourceList{
				v1.ResourceCPU:    *resource.NewMilliQuantity(1800, resource.DecimalSI),
				v1.ResourceMemory: *resource.NewQuantity(19900E6, resource.BinarySI),
				v1.ResourcePods:   *resource.NewQuantity(0, resource.DecimalSI),
			},
			Addresses: []v1.NodeAddress{
				{Type: v1.NodeInternalIP, Address: "127.0.0.1"},
				{Type: v1.NodeHostName, Address: testKubeletHostname},
			},
			// images will be sorted from max to min in node status.
			Images: []v1.ContainerImage{
				{
					Names:     []string{"gcr.io/google_containers:v3", "gcr.io/google_containers:v4"},
					SizeBytes: 456,
				},
				{
					Names:     []string{"gcr.io/google_containers:v1", "gcr.io/google_containers:v2"},
					SizeBytes: 123,
				},
			},
		},
	}

	kubelet.updateRuntimeUp()
	assert.NoError(t, kubelet.updateNodeStatus())

	actions := kubeClient.Actions()
	assert.Len(t, actions, 2)

	assert.IsType(t, core.PatchActionImpl{}, actions[1])
	patchAction := actions[1].(core.PatchActionImpl)

	updatedNode, err := applyNodeStatusPatch(&existingNode, patchAction.GetPatch())
	require.NoError(t, err)

	for i, cond := range updatedNode.Status.Conditions {
		old := metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC).Time
		// Expect LastHearbeat to be updated to Now, while LastTransitionTime to be the same.
		assert.NotEqual(t, old, cond.LastHeartbeatTime.Rfc3339Copy().UTC(), "LastHeartbeatTime for condition %v", cond.Type)
		assert.EqualValues(t, old, cond.LastTransitionTime.Rfc3339Copy().UTC(), "LastTransitionTime for condition %v", cond.Type)

		updatedNode.Status.Conditions[i].LastHeartbeatTime = metav1.Time{}
		updatedNode.Status.Conditions[i].LastTransitionTime = metav1.Time{}
	}

	// Version skew workaround. See: https://github.com/kubernetes/kubernetes/issues/16961
	assert.Equal(t, v1.NodeReady, updatedNode.Status.Conditions[len(updatedNode.Status.Conditions)-1].Type,
		"NodeReady should be the last condition")
	assert.True(t, apiequality.Semantic.DeepEqual(expectedNode, updatedNode), "%s", diff.ObjectDiff(expectedNode, updatedNode))
}

func TestUpdateExistingNodeStatusTimeout(t *testing.T) {
	attempts := int64(0)

	// set up a listener that hangs connections
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	assert.NoError(t, err)
	defer ln.Close()
	go func() {
		// accept connections and just let them hang
		for {
			_, err := ln.Accept()
			if err != nil {
				t.Log(err)
				return
			}
			t.Log("accepted connection")
			atomic.AddInt64(&attempts, 1)
		}
	}()

	config := &rest.Config{
		Host:    "http://" + ln.Addr().String(),
		QPS:     -1,
		Timeout: time.Second,
	}
	assert.NoError(t, err)

	testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
	defer testKubelet.Cleanup()
	kubelet := testKubelet.kubelet
	kubelet.kubeClient = nil // ensure only the heartbeat client is used
	kubelet.heartbeatClient, err = v1core.NewForConfig(config)
	kubelet.containerManager = &localCM{
		ContainerManager: cm.NewStubContainerManager(),
		allocatable: v1.ResourceList{
			v1.ResourceCPU:    *resource.NewMilliQuantity(200, resource.DecimalSI),
			v1.ResourceMemory: *resource.NewQuantity(100E6, resource.BinarySI),
		},
		capacity: v1.ResourceList{
			v1.ResourceCPU:    *resource.NewMilliQuantity(2000, resource.DecimalSI),
			v1.ResourceMemory: *resource.NewQuantity(20E9, resource.BinarySI),
		},
	}

	// should return an error, but not hang
	assert.Error(t, kubelet.updateNodeStatus())

	// should have attempted multiple times
	if actualAttempts := atomic.LoadInt64(&attempts); actualAttempts != nodeStatusUpdateRetry {
		t.Errorf("Expected %d attempts, got %d", nodeStatusUpdateRetry, actualAttempts)
	}
}

func TestUpdateNodeStatusWithRuntimeStateError(t *testing.T) {
	testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
	defer testKubelet.Cleanup()
	kubelet := testKubelet.kubelet
	kubelet.kubeClient = nil // ensure only the heartbeat client is used
	kubelet.containerManager = &localCM{
		ContainerManager: cm.NewStubContainerManager(),
		allocatable: v1.ResourceList{
			v1.ResourceCPU:    *resource.NewMilliQuantity(200, resource.DecimalSI),
			v1.ResourceMemory: *resource.NewQuantity(100E6, resource.BinarySI),
		},
		capacity: v1.ResourceList{
			v1.ResourceCPU:    *resource.NewMilliQuantity(2000, resource.DecimalSI),
			v1.ResourceMemory: *resource.NewQuantity(10E9, resource.BinarySI),
		},
	}

	clock := testKubelet.fakeClock
	kubeClient := testKubelet.fakeKubeClient
	existingNode := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}}
	kubeClient.ReactionChain = fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{existingNode}}).ReactionChain
	mockCadvisor := testKubelet.fakeCadvisor
	mockCadvisor.On("Start").Return(nil)
	machineInfo := &cadvisorapi.MachineInfo{
		MachineID:      "123",
		SystemUUID:     "abc",
		BootID:         "1b3",
		NumCores:       2,
		MemoryCapacity: 10E9,
	}
	mockCadvisor.On("MachineInfo").Return(machineInfo, nil)
	versionInfo := &cadvisorapi.VersionInfo{
		KernelVersion:      "3.16.0-0.bpo.4-amd64",
		ContainerOsVersion: "Debian GNU/Linux 7 (wheezy)",
	}
	mockCadvisor.On("VersionInfo").Return(versionInfo, nil)

	expectedNode := &v1.Node{
		ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
		Spec:       v1.NodeSpec{},
		Status: v1.NodeStatus{
			Conditions: []v1.NodeCondition{
				{
					Type:               v1.NodeOutOfDisk,
					Status:             v1.ConditionFalse,
					Reason:             "KubeletHasSufficientDisk",
					Message:            fmt.Sprintf("kubelet has sufficient disk space available"),
					LastHeartbeatTime:  metav1.Time{},
					LastTransitionTime: metav1.Time{},
				},
				{
					Type:               v1.NodeMemoryPressure,
					Status:             v1.ConditionFalse,
					Reason:             "KubeletHasSufficientMemory",
					Message:            fmt.Sprintf("kubelet has sufficient memory available"),
					LastHeartbeatTime:  metav1.Time{},
					LastTransitionTime: metav1.Time{},
				},
				{
					Type:               v1.NodeDiskPressure,
					Status:             v1.ConditionFalse,
					Reason:             "KubeletHasNoDiskPressure",
					Message:            fmt.Sprintf("kubelet has no disk pressure"),
					LastHeartbeatTime:  metav1.Time{},
					LastTransitionTime: metav1.Time{},
				},
				{}, //placeholder
			},
			NodeInfo: v1.NodeSystemInfo{
				MachineID:               "123",
				SystemUUID:              "abc",
				BootID:                  "1b3",
				KernelVersion:           "3.16.0-0.bpo.4-amd64",
				OSImage:                 "Debian GNU/Linux 7 (wheezy)",
				OperatingSystem:         goruntime.GOOS,
				Architecture:            goruntime.GOARCH,
				ContainerRuntimeVersion: "test://1.5.0",
				KubeletVersion:          version.Get().String(),
				KubeProxyVersion:        version.Get().String(),
			},
			Capacity: v1.ResourceList{
				v1.ResourceCPU:    *resource.NewMilliQuantity(2000, resource.DecimalSI),
				v1.ResourceMemory: *resource.NewQuantity(10E9, resource.BinarySI),
				v1.ResourcePods:   *resource.NewQuantity(0, resource.DecimalSI),
			},
			Allocatable: v1.ResourceList{
				v1.ResourceCPU:    *resource.NewMilliQuantity(1800, resource.DecimalSI),
				v1.ResourceMemory: *resource.NewQuantity(9900E6, resource.BinarySI),
				v1.ResourcePods:   *resource.NewQuantity(0, resource.DecimalSI),
			},
			Addresses: []v1.NodeAddress{
				{Type: v1.NodeInternalIP, Address: "127.0.0.1"},
				{Type: v1.NodeHostName, Address: testKubeletHostname},
			},
			Images: []v1.ContainerImage{
				{
					Names:     []string{"gcr.io/google_containers:v3", "gcr.io/google_containers:v4"},
					SizeBytes: 456,
				},
				{
					Names:     []string{"gcr.io/google_containers:v1", "gcr.io/google_containers:v2"},
					SizeBytes: 123,
				},
			},
		},
	}

	checkNodeStatus := func(status v1.ConditionStatus, reason string) {
		kubeClient.ClearActions()
		assert.NoError(t, kubelet.updateNodeStatus())
		actions := kubeClient.Actions()
		require.Len(t, actions, 2)
		require.True(t, actions[1].Matches("patch", "nodes"))
		require.Equal(t, actions[1].GetSubresource(), "status")

		updatedNode, err := applyNodeStatusPatch(&existingNode, actions[1].(core.PatchActionImpl).GetPatch())
		require.NoError(t, err, "can't apply node status patch")

		for i, cond := range updatedNode.Status.Conditions {
			assert.False(t, cond.LastHeartbeatTime.IsZero(), "LastHeartbeatTime for %v condition is zero", cond.Type)
			assert.False(t, cond.LastTransitionTime.IsZero(), "LastTransitionTime for %v condition  is zero", cond.Type)
			updatedNode.Status.Conditions[i].LastHeartbeatTime = metav1.Time{}
			updatedNode.Status.Conditions[i].LastTransitionTime = metav1.Time{}
		}

		// Version skew workaround. See: https://github.com/kubernetes/kubernetes/issues/16961
		lastIndex := len(updatedNode.Status.Conditions) - 1

		assert.Equal(t, v1.NodeReady, updatedNode.Status.Conditions[lastIndex].Type, "NodeReady should be the last condition")
		assert.NotEmpty(t, updatedNode.Status.Conditions[lastIndex].Message)

		updatedNode.Status.Conditions[lastIndex].Message = ""
		expectedNode.Status.Conditions[lastIndex] = v1.NodeCondition{
			Type:               v1.NodeReady,
			Status:             status,
			Reason:             reason,
			LastHeartbeatTime:  metav1.Time{},
			LastTransitionTime: metav1.Time{},
		}
		assert.True(t, apiequality.Semantic.DeepEqual(expectedNode, updatedNode), "%s", diff.ObjectDiff(expectedNode, updatedNode))
	}

	// TODO(random-liu): Refactor the unit test to be table driven test.
	// Should report kubelet not ready if the runtime check is out of date
	clock.SetTime(time.Now().Add(-maxWaitForContainerRuntime))
	kubelet.updateRuntimeUp()
	checkNodeStatus(v1.ConditionFalse, "KubeletNotReady")

	// Should report kubelet ready if the runtime check is updated
	clock.SetTime(time.Now())
	kubelet.updateRuntimeUp()
	checkNodeStatus(v1.ConditionTrue, "KubeletReady")

	// Should report kubelet not ready if the runtime check is out of date
	clock.SetTime(time.Now().Add(-maxWaitForContainerRuntime))
	kubelet.updateRuntimeUp()
	checkNodeStatus(v1.ConditionFalse, "KubeletNotReady")

	// Should report kubelet not ready if the runtime check failed
	fakeRuntime := testKubelet.fakeRuntime
	// Inject error into fake runtime status check, node should be NotReady
	fakeRuntime.StatusErr = fmt.Errorf("injected runtime status error")
	clock.SetTime(time.Now())
	kubelet.updateRuntimeUp()
	checkNodeStatus(v1.ConditionFalse, "KubeletNotReady")

	fakeRuntime.StatusErr = nil

	// Should report node not ready if runtime status is nil.
	fakeRuntime.RuntimeStatus = nil
	kubelet.updateRuntimeUp()
	checkNodeStatus(v1.ConditionFalse, "KubeletNotReady")

	// Should report node not ready if runtime status is empty.
	fakeRuntime.RuntimeStatus = &kubecontainer.RuntimeStatus{}
	kubelet.updateRuntimeUp()
	checkNodeStatus(v1.ConditionFalse, "KubeletNotReady")

	// Should report node not ready if RuntimeReady is false.
	fakeRuntime.RuntimeStatus = &kubecontainer.RuntimeStatus{
		Conditions: []kubecontainer.RuntimeCondition{
			{Type: kubecontainer.RuntimeReady, Status: false},
			{Type: kubecontainer.NetworkReady, Status: true},
		},
	}
	kubelet.updateRuntimeUp()
	checkNodeStatus(v1.ConditionFalse, "KubeletNotReady")

	// Should report node ready if RuntimeReady is true.
	fakeRuntime.RuntimeStatus = &kubecontainer.RuntimeStatus{
		Conditions: []kubecontainer.RuntimeCondition{
			{Type: kubecontainer.RuntimeReady, Status: true},
			{Type: kubecontainer.NetworkReady, Status: true},
		},
	}
	kubelet.updateRuntimeUp()
	checkNodeStatus(v1.ConditionTrue, "KubeletReady")

	// Should report node not ready if NetworkReady is false.
	fakeRuntime.RuntimeStatus = &kubecontainer.RuntimeStatus{
		Conditions: []kubecontainer.RuntimeCondition{
			{Type: kubecontainer.RuntimeReady, Status: true},
			{Type: kubecontainer.NetworkReady, Status: false},
		},
	}
	kubelet.updateRuntimeUp()
	checkNodeStatus(v1.ConditionFalse, "KubeletNotReady")
}

func TestUpdateNodeStatusError(t *testing.T) {
	testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
	defer testKubelet.Cleanup()
	kubelet := testKubelet.kubelet
	kubelet.kubeClient = nil // ensure only the heartbeat client is used
	// No matching node for the kubelet
	testKubelet.fakeKubeClient.ReactionChain = fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{}}).ReactionChain
	assert.Error(t, kubelet.updateNodeStatus())
	assert.Len(t, testKubelet.fakeKubeClient.Actions(), nodeStatusUpdateRetry)
}

func TestRegisterWithApiServer(t *testing.T) {
	testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
	defer testKubelet.Cleanup()
	kubelet := testKubelet.kubelet
	kubeClient := testKubelet.fakeKubeClient
	kubeClient.AddReactor("create", "nodes", func(action core.Action) (bool, runtime.Object, error) {
		// Return an error on create.
		return true, &v1.Node{}, &apierrors.StatusError{
			ErrStatus: metav1.Status{Reason: metav1.StatusReasonAlreadyExists},
		}
	})
	kubeClient.AddReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) {
		// Return an existing (matching) node on get.
		return true, &v1.Node{
			ObjectMeta: metav1.ObjectMeta{
				Name: testKubeletHostname,
				Labels: map[string]string{
					kubeletapis.LabelHostname: testKubeletHostname,
					kubeletapis.LabelOS:       goruntime.GOOS,
					kubeletapis.LabelArch:     goruntime.GOARCH,
				},
			},
			Spec: v1.NodeSpec{ExternalID: testKubeletHostname},
		}, nil
	})
	kubeClient.AddReactor("*", "*", func(action core.Action) (bool, runtime.Object, error) {
		return true, nil, fmt.Errorf("no reaction implemented for %s", action)
	})
	machineInfo := &cadvisorapi.MachineInfo{
		MachineID:      "123",
		SystemUUID:     "abc",
		BootID:         "1b3",
		NumCores:       2,
		MemoryCapacity: 1024,
	}
	mockCadvisor := testKubelet.fakeCadvisor
	mockCadvisor.On("MachineInfo").Return(machineInfo, nil)
	versionInfo := &cadvisorapi.VersionInfo{
		KernelVersion:      "3.16.0-0.bpo.4-amd64",
		ContainerOsVersion: "Debian GNU/Linux 7 (wheezy)",
		DockerVersion:      "1.5.0",
	}
	mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
	mockCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{
		Usage:     400,
		Capacity:  1000,
		Available: 600,
	}, nil)
	mockCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{
		Usage:    9,
		Capacity: 10,
	}, nil)

	done := make(chan struct{})
	go func() {
		kubelet.registerWithAPIServer()
		done <- struct{}{}
	}()
	select {
	case <-time.After(wait.ForeverTestTimeout):
		assert.Fail(t, "timed out waiting for registration")
	case <-done:
		return
	}
}

func TestTryRegisterWithApiServer(t *testing.T) {
	alreadyExists := &apierrors.StatusError{
		ErrStatus: metav1.Status{Reason: metav1.StatusReasonAlreadyExists},
	}

	conflict := &apierrors.StatusError{
		ErrStatus: metav1.Status{Reason: metav1.StatusReasonConflict},
	}

	newNode := func(cmad bool, externalID string) *v1.Node {
		node := &v1.Node{
			ObjectMeta: metav1.ObjectMeta{
				Labels: map[string]string{
					kubeletapis.LabelHostname: testKubeletHostname,
					kubeletapis.LabelOS:       goruntime.GOOS,
					kubeletapis.LabelArch:     goruntime.GOARCH,
				},
			},
			Spec: v1.NodeSpec{
				ExternalID: externalID,
			},
		}

		if cmad {
			node.Annotations = make(map[string]string)
			node.Annotations[volumehelper.ControllerManagedAttachAnnotation] = "true"
		}

		return node
	}

	cases := []struct {
		name            string
		newNode         *v1.Node
		existingNode    *v1.Node
		createError     error
		getError        error
		patchError      error
		deleteError     error
		expectedResult  bool
		expectedActions int
		testSavedNode   bool
		savedNodeIndex  int
		savedNodeCMAD   bool
	}{
		{
			name:            "success case - new node",
			newNode:         &v1.Node{},
			expectedResult:  true,
			expectedActions: 1,
		},
		{
			name:            "success case - existing node - no change in CMAD",
			newNode:         newNode(true, "a"),
			createError:     alreadyExists,
			existingNode:    newNode(true, "a"),
			expectedResult:  true,
			expectedActions: 2,
		},
		{
			name:            "success case - existing node - CMAD disabled",
			newNode:         newNode(false, "a"),
			createError:     alreadyExists,
			existingNode:    newNode(true, "a"),
			expectedResult:  true,
			expectedActions: 3,
			testSavedNode:   true,
			savedNodeIndex:  2,
			savedNodeCMAD:   false,
		},
		{
			name:            "success case - existing node - CMAD enabled",
			newNode:         newNode(true, "a"),
			createError:     alreadyExists,
			existingNode:    newNode(false, "a"),
			expectedResult:  true,
			expectedActions: 3,
			testSavedNode:   true,
			savedNodeIndex:  2,
			savedNodeCMAD:   true,
		},
		{
			name:            "success case - external ID changed",
			newNode:         newNode(false, "b"),
			createError:     alreadyExists,
			existingNode:    newNode(false, "a"),
			expectedResult:  false,
			expectedActions: 3,
		},
		{
			name:            "create failed",
			newNode:         newNode(false, "b"),
			createError:     conflict,
			expectedResult:  false,
			expectedActions: 1,
		},
		{
			name:            "get existing node failed",
			newNode:         newNode(false, "a"),
			createError:     alreadyExists,
			getError:        conflict,
			expectedResult:  false,
			expectedActions: 2,
		},
		{
			name:            "update existing node failed",
			newNode:         newNode(false, "a"),
			createError:     alreadyExists,
			existingNode:    newNode(true, "a"),
			patchError:      conflict,
			expectedResult:  false,
			expectedActions: 3,
		},
		{
			name:            "delete existing node failed",
			newNode:         newNode(false, "b"),
			createError:     alreadyExists,
			existingNode:    newNode(false, "a"),
			deleteError:     conflict,
			expectedResult:  false,
			expectedActions: 3,
		},
	}

	notImplemented := func(action core.Action) (bool, runtime.Object, error) {
		return true, nil, fmt.Errorf("no reaction implemented for %s", action)
	}

	for _, tc := range cases {
		testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled is a don't-care for this test */)
		defer testKubelet.Cleanup()
		kubelet := testKubelet.kubelet
		kubeClient := testKubelet.fakeKubeClient

		kubeClient.AddReactor("create", "nodes", func(action core.Action) (bool, runtime.Object, error) {
			return true, nil, tc.createError
		})
		kubeClient.AddReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) {
			// Return an existing (matching) node on get.
			return true, tc.existingNode, tc.getError
		})
		kubeClient.AddReactor("patch", "nodes", func(action core.Action) (bool, runtime.Object, error) {
			if action.GetSubresource() == "status" {
				return true, nil, tc.patchError
			}
			return notImplemented(action)
		})
		kubeClient.AddReactor("delete", "nodes", func(action core.Action) (bool, runtime.Object, error) {
			return true, nil, tc.deleteError
		})
		kubeClient.AddReactor("*", "*", func(action core.Action) (bool, runtime.Object, error) {
			return notImplemented(action)
		})

		result := kubelet.tryRegisterWithAPIServer(tc.newNode)
		require.Equal(t, tc.expectedResult, result, "test [%s]", tc.name)

		actions := kubeClient.Actions()
		assert.Len(t, actions, tc.expectedActions, "test [%s]", tc.name)

		if tc.testSavedNode {
			var savedNode *v1.Node

			t.Logf("actions: %v: %+v", len(actions), actions)
			action := actions[tc.savedNodeIndex]
			if action.GetVerb() == "create" {
				createAction := action.(core.CreateAction)
				obj := createAction.GetObject()
				require.IsType(t, &v1.Node{}, obj)
				savedNode = obj.(*v1.Node)
			} else if action.GetVerb() == "patch" {
				patchAction := action.(core.PatchActionImpl)
				var err error
				savedNode, err = applyNodeStatusPatch(tc.existingNode, patchAction.GetPatch())
				require.NoError(t, err)
			}

			actualCMAD, _ := strconv.ParseBool(savedNode.Annotations[volumehelper.ControllerManagedAttachAnnotation])
			assert.Equal(t, tc.savedNodeCMAD, actualCMAD, "test [%s]", tc.name)
		}
	}
}

func TestUpdateNewNodeStatusTooLargeReservation(t *testing.T) {
	// generate one more than maxImagesInNodeStatus in inputImageList
	inputImageList, _ := generateTestingImageList(maxImagesInNodeStatus + 1)
	testKubelet := newTestKubeletWithImageList(
		t, inputImageList, false /* controllerAttachDetachEnabled */)
	defer testKubelet.Cleanup()
	kubelet := testKubelet.kubelet
	kubelet.kubeClient = nil // ensure only the heartbeat client is used
	kubelet.containerManager = &localCM{
		ContainerManager: cm.NewStubContainerManager(),
		allocatable: v1.ResourceList{
			v1.ResourceCPU: *resource.NewMilliQuantity(40000, resource.DecimalSI),
		},
		capacity: v1.ResourceList{
			v1.ResourceCPU:    *resource.NewMilliQuantity(2000, resource.DecimalSI),
			v1.ResourceMemory: *resource.NewQuantity(10E9, resource.BinarySI),
		},
	}
	kubeClient := testKubelet.fakeKubeClient
	existingNode := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}}
	kubeClient.ReactionChain = fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{existingNode}}).ReactionChain
	machineInfo := &cadvisorapi.MachineInfo{
		MachineID:      "123",
		SystemUUID:     "abc",
		BootID:         "1b3",
		NumCores:       2,
		MemoryCapacity: 10E9, // 10G
	}
	mockCadvisor := testKubelet.fakeCadvisor
	mockCadvisor.On("Start").Return(nil)
	mockCadvisor.On("MachineInfo").Return(machineInfo, nil)
	versionInfo := &cadvisorapi.VersionInfo{
		KernelVersion:      "3.16.0-0.bpo.4-amd64",
		ContainerOsVersion: "Debian GNU/Linux 7 (wheezy)",
	}
	mockCadvisor.On("VersionInfo").Return(versionInfo, nil)

	expectedNode := &v1.Node{
		ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
		Spec:       v1.NodeSpec{},
		Status: v1.NodeStatus{
			Capacity: v1.ResourceList{
				v1.ResourceCPU:    *resource.NewMilliQuantity(2000, resource.DecimalSI),
				v1.ResourceMemory: *resource.NewQuantity(10E9, resource.BinarySI),
				v1.ResourcePods:   *resource.NewQuantity(0, resource.DecimalSI),
			},
			Allocatable: v1.ResourceList{
				v1.ResourceCPU:    *resource.NewMilliQuantity(0, resource.DecimalSI),
				v1.ResourceMemory: *resource.NewQuantity(10E9, resource.BinarySI),
				v1.ResourcePods:   *resource.NewQuantity(0, resource.DecimalSI),
			},
		},
	}

	kubelet.updateRuntimeUp()
	assert.NoError(t, kubelet.updateNodeStatus())
	actions := kubeClient.Actions()
	require.Len(t, actions, 2)
	require.True(t, actions[1].Matches("patch", "nodes"))
	require.Equal(t, actions[1].GetSubresource(), "status")

	updatedNode, err := applyNodeStatusPatch(&existingNode, actions[1].(core.PatchActionImpl).GetPatch())
	assert.NoError(t, err)
	assert.True(t, apiequality.Semantic.DeepEqual(expectedNode.Status.Allocatable, updatedNode.Status.Allocatable), "%s", diff.ObjectDiff(expectedNode.Status.Allocatable, updatedNode.Status.Allocatable))
}

func TestUpdateDefaultLabels(t *testing.T) {
	testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
	testKubelet.kubelet.kubeClient = nil // ensure only the heartbeat client is used

	cases := []struct {
		name         string
		initialNode  *v1.Node
		existingNode *v1.Node
		needsUpdate  bool
		finalLabels  map[string]string
	}{
		{
			name: "make sure default labels exist",
			initialNode: &v1.Node{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						kubeletapis.LabelHostname:          "new-hostname",
						kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain",
						kubeletapis.LabelZoneRegion:        "new-zone-region",
						kubeletapis.LabelInstanceType:      "new-instance-type",
						kubeletapis.LabelOS:                "new-os",
						kubeletapis.LabelArch:              "new-arch",
					},
				},
			},
			existingNode: &v1.Node{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{},
				},
			},
			needsUpdate: true,
			finalLabels: map[string]string{
				kubeletapis.LabelHostname:          "new-hostname",
				kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain",
				kubeletapis.LabelZoneRegion:        "new-zone-region",
				kubeletapis.LabelInstanceType:      "new-instance-type",
				kubeletapis.LabelOS:                "new-os",
				kubeletapis.LabelArch:              "new-arch",
			},
		},
		{
			name: "make sure default labels are up to date",
			initialNode: &v1.Node{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						kubeletapis.LabelHostname:          "new-hostname",
						kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain",
						kubeletapis.LabelZoneRegion:        "new-zone-region",
						kubeletapis.LabelInstanceType:      "new-instance-type",
						kubeletapis.LabelOS:                "new-os",
						kubeletapis.LabelArch:              "new-arch",
					},
				},
			},
			existingNode: &v1.Node{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						kubeletapis.LabelHostname:          "old-hostname",
						kubeletapis.LabelZoneFailureDomain: "old-zone-failure-domain",
						kubeletapis.LabelZoneRegion:        "old-zone-region",
						kubeletapis.LabelInstanceType:      "old-instance-type",
						kubeletapis.LabelOS:                "old-os",
						kubeletapis.LabelArch:              "old-arch",
					},
				},
			},
			needsUpdate: true,
			finalLabels: map[string]string{
				kubeletapis.LabelHostname:          "new-hostname",
				kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain",
				kubeletapis.LabelZoneRegion:        "new-zone-region",
				kubeletapis.LabelInstanceType:      "new-instance-type",
				kubeletapis.LabelOS:                "new-os",
				kubeletapis.LabelArch:              "new-arch",
			},
		},
		{
			name: "make sure existing labels do not get deleted",
			initialNode: &v1.Node{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						kubeletapis.LabelHostname:          "new-hostname",
						kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain",
						kubeletapis.LabelZoneRegion:        "new-zone-region",
						kubeletapis.LabelInstanceType:      "new-instance-type",
						kubeletapis.LabelOS:                "new-os",
						kubeletapis.LabelArch:              "new-arch",
					},
				},
			},
			existingNode: &v1.Node{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						kubeletapis.LabelHostname:          "new-hostname",
						kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain",
						kubeletapis.LabelZoneRegion:        "new-zone-region",
						kubeletapis.LabelInstanceType:      "new-instance-type",
						kubeletapis.LabelOS:                "new-os",
						kubeletapis.LabelArch:              "new-arch",
						"please-persist":                   "foo",
					},
				},
			},
			needsUpdate: false,
			finalLabels: map[string]string{
				kubeletapis.LabelHostname:          "new-hostname",
				kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain",
				kubeletapis.LabelZoneRegion:        "new-zone-region",
				kubeletapis.LabelInstanceType:      "new-instance-type",
				kubeletapis.LabelOS:                "new-os",
				kubeletapis.LabelArch:              "new-arch",
				"please-persist":                   "foo",
			},
		},
		{
			name: "make sure existing labels do not get deleted when initial node has no opinion",
			initialNode: &v1.Node{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{},
				},
			},
			existingNode: &v1.Node{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						kubeletapis.LabelHostname:          "new-hostname",
						kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain",
						kubeletapis.LabelZoneRegion:        "new-zone-region",
						kubeletapis.LabelInstanceType:      "new-instance-type",
						kubeletapis.LabelOS:                "new-os",
						kubeletapis.LabelArch:              "new-arch",
						"please-persist":                   "foo",
					},
				},
			},
			needsUpdate: false,
			finalLabels: map[string]string{
				kubeletapis.LabelHostname:          "new-hostname",
				kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain",
				kubeletapis.LabelZoneRegion:        "new-zone-region",
				kubeletapis.LabelInstanceType:      "new-instance-type",
				kubeletapis.LabelOS:                "new-os",
				kubeletapis.LabelArch:              "new-arch",
				"please-persist":                   "foo",
			},
		},
		{
			name: "no update needed",
			initialNode: &v1.Node{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						kubeletapis.LabelHostname:          "new-hostname",
						kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain",
						kubeletapis.LabelZoneRegion:        "new-zone-region",
						kubeletapis.LabelInstanceType:      "new-instance-type",
						kubeletapis.LabelOS:                "new-os",
						kubeletapis.LabelArch:              "new-arch",
					},
				},
			},
			existingNode: &v1.Node{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						kubeletapis.LabelHostname:          "new-hostname",
						kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain",
						kubeletapis.LabelZoneRegion:        "new-zone-region",
						kubeletapis.LabelInstanceType:      "new-instance-type",
						kubeletapis.LabelOS:                "new-os",
						kubeletapis.LabelArch:              "new-arch",
					},
				},
			},
			needsUpdate: false,
			finalLabels: map[string]string{
				kubeletapis.LabelHostname:          "new-hostname",
				kubeletapis.LabelZoneFailureDomain: "new-zone-failure-domain",
				kubeletapis.LabelZoneRegion:        "new-zone-region",
				kubeletapis.LabelInstanceType:      "new-instance-type",
				kubeletapis.LabelOS:                "new-os",
				kubeletapis.LabelArch:              "new-arch",
			},
		},
	}

	for _, tc := range cases {
		defer testKubelet.Cleanup()
		kubelet := testKubelet.kubelet

		needsUpdate := kubelet.updateDefaultLabels(tc.initialNode, tc.existingNode)
		assert.Equal(t, tc.needsUpdate, needsUpdate, tc.name)
		assert.Equal(t, tc.finalLabels, tc.existingNode.Labels, tc.name)
	}
}