mirror of https://github.com/k3s-io/k3s
Refactor TestPodContainerDeviceAllocation to make it readable and extensible
vikaschoudhary16
parent
4fb55d9136
commit
8c51d235d6
|
|
@ -366,37 +366,25 @@ func (m *MockEndpoint) allocate(devs []string) (*pluginapi.AllocateResponse, err
|
|||
return nil, nil
|
||||
}
|
||||
|
||||
func TestPodContainerDeviceAllocation(t *testing.T) {
|
||||
flag.Set("alsologtostderr", fmt.Sprintf("%t", true))
|
||||
var logLevel string
|
||||
flag.StringVar(&logLevel, "logLevel", "4", "test")
|
||||
flag.Lookup("v").Value.Set(logLevel)
|
||||
|
||||
resourceName1 := "domain1.com/resource1"
|
||||
resourceQuantity1 := *resource.NewQuantity(int64(2), resource.DecimalSI)
|
||||
devID1 := "dev1"
|
||||
devID2 := "dev2"
|
||||
resourceName2 := "domain2.com/resource2"
|
||||
resourceQuantity2 := *resource.NewQuantity(int64(1), resource.DecimalSI)
|
||||
devID3 := "dev3"
|
||||
devID4 := "dev4"
|
||||
|
||||
as := require.New(t)
|
||||
monitorCallback := func(resourceName string, added, updated, deleted []pluginapi.Device) {}
|
||||
podsStub := activePodsStub{
|
||||
activePods: []*v1.Pod{},
|
||||
}
|
||||
cachedNode := &v1.Node{
|
||||
Status: v1.NodeStatus{
|
||||
Allocatable: v1.ResourceList{},
|
||||
func makePod(limits v1.ResourceList) *v1.Pod {
|
||||
return &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
UID: uuid.NewUUID(),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Resources: v1.ResourceRequirements{
|
||||
Limits: limits,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
nodeInfo := &schedulercache.NodeInfo{}
|
||||
nodeInfo.SetNode(cachedNode)
|
||||
}
|
||||
|
||||
tmpDir, err := ioutil.TempDir("", "checkpoint")
|
||||
as.Nil(err)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
func getTestManager(tmpDir string, activePods ActivePodsFunc, testRes []TestResource) *ManagerImpl {
|
||||
monitorCallback := func(resourceName string, added, updated, deleted []pluginapi.Device) {}
|
||||
testManager := &ManagerImpl{
|
||||
socketdir: tmpDir,
|
||||
callback: monitorCallback,
|
||||
|
|
@ -404,158 +392,221 @@ func TestPodContainerDeviceAllocation(t *testing.T) {
|
|||
allocatedDevices: make(map[string]sets.String),
|
||||
endpoints: make(map[string]endpoint),
|
||||
podDevices: make(podDevices),
|
||||
activePods: podsStub.getActivePods,
|
||||
activePods: activePods,
|
||||
sourcesReady: &sourcesReadyStub{},
|
||||
}
|
||||
testManager.store, _ = utilstore.NewFileStore("/tmp/", utilfs.DefaultFs{})
|
||||
for _, res := range testRes {
|
||||
testManager.allDevices[res.resourceName] = sets.NewString()
|
||||
for _, dev := range res.devs {
|
||||
testManager.allDevices[res.resourceName].Insert(dev)
|
||||
}
|
||||
if res.resourceName == "domain1.com/resource1" {
|
||||
testManager.endpoints[res.resourceName] = &MockEndpoint{
|
||||
allocateFunc: func(devs []string) (*pluginapi.AllocateResponse, error) {
|
||||
resp := new(pluginapi.AllocateResponse)
|
||||
resp.Envs = make(map[string]string)
|
||||
for _, dev := range devs {
|
||||
switch dev {
|
||||
case "dev1":
|
||||
resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
|
||||
ContainerPath: "/dev/aaa",
|
||||
HostPath: "/dev/aaa",
|
||||
Permissions: "mrw",
|
||||
})
|
||||
|
||||
testManager.allDevices[resourceName1] = sets.NewString()
|
||||
testManager.allDevices[resourceName1].Insert(devID1)
|
||||
testManager.allDevices[resourceName1].Insert(devID2)
|
||||
testManager.allDevices[resourceName2] = sets.NewString()
|
||||
testManager.allDevices[resourceName2].Insert(devID3)
|
||||
testManager.allDevices[resourceName2].Insert(devID4)
|
||||
resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
|
||||
ContainerPath: "/dev/bbb",
|
||||
HostPath: "/dev/bbb",
|
||||
Permissions: "mrw",
|
||||
})
|
||||
|
||||
testManager.endpoints[resourceName1] = &MockEndpoint{
|
||||
allocateFunc: func(devs []string) (*pluginapi.AllocateResponse, error) {
|
||||
resp := new(pluginapi.AllocateResponse)
|
||||
resp.Envs = make(map[string]string)
|
||||
for _, dev := range devs {
|
||||
switch dev {
|
||||
case "dev1":
|
||||
resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
|
||||
ContainerPath: "/dev/aaa",
|
||||
HostPath: "/dev/aaa",
|
||||
Permissions: "mrw",
|
||||
})
|
||||
resp.Mounts = append(resp.Mounts, &pluginapi.Mount{
|
||||
ContainerPath: "/container_dir1/file1",
|
||||
HostPath: "host_dir1/file1",
|
||||
ReadOnly: true,
|
||||
})
|
||||
|
||||
resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
|
||||
ContainerPath: "/dev/bbb",
|
||||
HostPath: "/dev/bbb",
|
||||
Permissions: "mrw",
|
||||
})
|
||||
case "dev2":
|
||||
resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
|
||||
ContainerPath: "/dev/ccc",
|
||||
HostPath: "/dev/ccc",
|
||||
Permissions: "mrw",
|
||||
})
|
||||
|
||||
resp.Mounts = append(resp.Mounts, &pluginapi.Mount{
|
||||
ContainerPath: "/container_dir1/file1",
|
||||
HostPath: "host_dir1/file1",
|
||||
ReadOnly: true,
|
||||
})
|
||||
resp.Mounts = append(resp.Mounts, &pluginapi.Mount{
|
||||
ContainerPath: "/container_dir1/file2",
|
||||
HostPath: "host_dir1/file2",
|
||||
ReadOnly: true,
|
||||
})
|
||||
|
||||
case "dev2":
|
||||
resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
|
||||
ContainerPath: "/dev/ccc",
|
||||
HostPath: "/dev/ccc",
|
||||
Permissions: "mrw",
|
||||
})
|
||||
|
||||
resp.Mounts = append(resp.Mounts, &pluginapi.Mount{
|
||||
ContainerPath: "/container_dir1/file2",
|
||||
HostPath: "host_dir1/file2",
|
||||
ReadOnly: true,
|
||||
})
|
||||
|
||||
resp.Envs["key1"] = "val1"
|
||||
}
|
||||
resp.Envs["key1"] = "val1"
|
||||
}
|
||||
}
|
||||
return resp, nil
|
||||
},
|
||||
}
|
||||
return resp, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
if res.resourceName == "domain2.com/resource2" {
|
||||
testManager.endpoints[res.resourceName] = &MockEndpoint{
|
||||
allocateFunc: func(devs []string) (*pluginapi.AllocateResponse, error) {
|
||||
resp := new(pluginapi.AllocateResponse)
|
||||
resp.Envs = make(map[string]string)
|
||||
for _, dev := range devs {
|
||||
switch dev {
|
||||
case "dev3":
|
||||
resp.Envs["key2"] = "val2"
|
||||
|
||||
testManager.endpoints[resourceName2] = &MockEndpoint{
|
||||
allocateFunc: func(devs []string) (*pluginapi.AllocateResponse, error) {
|
||||
resp := new(pluginapi.AllocateResponse)
|
||||
resp.Envs = make(map[string]string)
|
||||
for _, dev := range devs {
|
||||
switch dev {
|
||||
case "dev3":
|
||||
resp.Envs["key2"] = "val2"
|
||||
|
||||
case "dev4":
|
||||
resp.Envs["key2"] = "val3"
|
||||
}
|
||||
case "dev4":
|
||||
resp.Envs["key2"] = "val3"
|
||||
}
|
||||
}
|
||||
return resp, nil
|
||||
},
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
}
|
||||
return testManager
|
||||
}
|
||||
|
||||
func getTestNodeInfo(allocatable v1.ResourceList) *schedulercache.NodeInfo {
|
||||
cachedNode := &v1.Node{
|
||||
Status: v1.NodeStatus{
|
||||
Allocatable: allocatable,
|
||||
},
|
||||
}
|
||||
nodeInfo := &schedulercache.NodeInfo{}
|
||||
nodeInfo.SetNode(cachedNode)
|
||||
return nodeInfo
|
||||
}
|
||||
|
||||
pod := &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
UID: uuid.NewUUID(),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: string(uuid.NewUUID()),
|
||||
Resources: v1.ResourceRequirements{
|
||||
Limits: v1.ResourceList{
|
||||
v1.ResourceName(resourceName1): resourceQuantity1,
|
||||
v1.ResourceName("cpu"): resourceQuantity1,
|
||||
v1.ResourceName(resourceName2): resourceQuantity2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
type TestResource struct {
|
||||
resourceName string
|
||||
resourceQuantity resource.Quantity
|
||||
devs []string
|
||||
}
|
||||
|
||||
func TestPodContainerDeviceAllocation(t *testing.T) {
|
||||
flag.Set("alsologtostderr", fmt.Sprintf("%t", true))
|
||||
var logLevel string
|
||||
flag.StringVar(&logLevel, "logLevel", "4", "test")
|
||||
flag.Lookup("v").Value.Set(logLevel)
|
||||
res1 := TestResource{
|
||||
resourceName: "domain1.com/resource1",
|
||||
resourceQuantity: *resource.NewQuantity(int64(2), resource.DecimalSI),
|
||||
devs: []string{"dev1", "dev2"},
|
||||
}
|
||||
|
||||
podsStub.updateActivePods([]*v1.Pod{pod})
|
||||
err = testManager.Allocate(nodeInfo, &lifecycle.PodAdmitAttributes{Pod: pod})
|
||||
res2 := TestResource{
|
||||
resourceName: "domain2.com/resource2",
|
||||
resourceQuantity: *resource.NewQuantity(int64(1), resource.DecimalSI),
|
||||
devs: []string{"dev3", "dev4"},
|
||||
}
|
||||
testResources := make([]TestResource, 2)
|
||||
testResources = append(testResources, res1)
|
||||
testResources = append(testResources, res2)
|
||||
as := require.New(t)
|
||||
podsStub := activePodsStub{
|
||||
activePods: []*v1.Pod{},
|
||||
}
|
||||
tmpDir, err := ioutil.TempDir("", "checkpoint")
|
||||
as.Nil(err)
|
||||
runContainerOpts := testManager.GetDeviceRunContainerOptions(pod, &pod.Spec.Containers[0])
|
||||
as.NotNil(runContainerOpts)
|
||||
as.Equal(len(runContainerOpts.Devices), 3)
|
||||
as.Equal(len(runContainerOpts.Mounts), 2)
|
||||
as.Equal(len(runContainerOpts.Envs), 2)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
nodeInfo := getTestNodeInfo(v1.ResourceList{})
|
||||
testManager := getTestManager(tmpDir, podsStub.getActivePods, testResources)
|
||||
|
||||
// Requesting to create a pod without enough resources should fail.
|
||||
as.Equal(2, testManager.allocatedDevices[resourceName1].Len())
|
||||
failPod := &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
UID: uuid.NewUUID(),
|
||||
testPods := []*v1.Pod{
|
||||
makePod(v1.ResourceList{
|
||||
v1.ResourceName(res1.resourceName): res1.resourceQuantity,
|
||||
v1.ResourceName("cpu"): res1.resourceQuantity,
|
||||
v1.ResourceName(res2.resourceName): res2.resourceQuantity}),
|
||||
makePod(v1.ResourceList{
|
||||
v1.ResourceName(res1.resourceName): res2.resourceQuantity}),
|
||||
makePod(v1.ResourceList{
|
||||
v1.ResourceName(res2.resourceName): res2.resourceQuantity}),
|
||||
}
|
||||
testCases := []struct {
|
||||
description string
|
||||
testPod *v1.Pod
|
||||
expectedContainerOptsLen []int
|
||||
expectedAllocatedResName1 int
|
||||
expectedAllocatedResName2 int
|
||||
expErr error
|
||||
}{
|
||||
{
|
||||
description: "Successfull allocation of two Res1 resources and one Res2 resource",
|
||||
testPod: testPods[0],
|
||||
expectedContainerOptsLen: []int{3, 2, 2},
|
||||
expectedAllocatedResName1: 2,
|
||||
expectedAllocatedResName2: 1,
|
||||
expErr: nil,
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: string(uuid.NewUUID()),
|
||||
Resources: v1.ResourceRequirements{
|
||||
Limits: v1.ResourceList{
|
||||
v1.ResourceName(resourceName1): resourceQuantity2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Requesting to create a pod without enough resources should fail",
|
||||
testPod: testPods[1],
|
||||
expectedContainerOptsLen: nil,
|
||||
expectedAllocatedResName1: 2,
|
||||
expectedAllocatedResName2: 1,
|
||||
expErr: fmt.Errorf("requested number of devices unavailable for domain1.com/resource1. Requested: 1, Available: 0"),
|
||||
},
|
||||
{
|
||||
description: "Successfull allocation of all available Res1 resources and Res2 resources",
|
||||
testPod: testPods[2],
|
||||
expectedContainerOptsLen: []int{0, 0, 1},
|
||||
expectedAllocatedResName1: 2,
|
||||
expectedAllocatedResName2: 2,
|
||||
expErr: nil,
|
||||
},
|
||||
}
|
||||
err = testManager.Allocate(nodeInfo, &lifecycle.PodAdmitAttributes{Pod: failPod})
|
||||
as.NotNil(err)
|
||||
runContainerOpts2 := testManager.GetDeviceRunContainerOptions(failPod, &failPod.Spec.Containers[0])
|
||||
as.Nil(runContainerOpts2)
|
||||
|
||||
// Requesting to create a new pod with a single resourceName2 should succeed.
|
||||
newPod := &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
UID: uuid.NewUUID(),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: string(uuid.NewUUID()),
|
||||
Resources: v1.ResourceRequirements{
|
||||
Limits: v1.ResourceList{
|
||||
v1.ResourceName(resourceName2): resourceQuantity2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
activePods := []*v1.Pod{}
|
||||
for _, testCase := range testCases {
|
||||
pod := testCase.testPod
|
||||
activePods = append(activePods, pod)
|
||||
podsStub.updateActivePods(activePods)
|
||||
err := testManager.Allocate(nodeInfo, &lifecycle.PodAdmitAttributes{Pod: pod})
|
||||
if !reflect.DeepEqual(err, testCase.expErr) {
|
||||
t.Errorf("DevicePluginManager error (%v). expected error: %v but got: %v",
|
||||
testCase.description, testCase.expErr, err)
|
||||
}
|
||||
runContainerOpts := testManager.GetDeviceRunContainerOptions(pod, &pod.Spec.Containers[0])
|
||||
if testCase.expectedContainerOptsLen == nil {
|
||||
as.Nil(runContainerOpts)
|
||||
} else {
|
||||
as.Equal(len(runContainerOpts.Devices), testCase.expectedContainerOptsLen[0])
|
||||
as.Equal(len(runContainerOpts.Mounts), testCase.expectedContainerOptsLen[1])
|
||||
as.Equal(len(runContainerOpts.Envs), testCase.expectedContainerOptsLen[2])
|
||||
}
|
||||
as.Equal(testCase.expectedAllocatedResName1, testManager.allocatedDevices[res1.resourceName].Len())
|
||||
as.Equal(testCase.expectedAllocatedResName2, testManager.allocatedDevices[res2.resourceName].Len())
|
||||
}
|
||||
err = testManager.Allocate(nodeInfo, &lifecycle.PodAdmitAttributes{Pod: newPod})
|
||||
as.Nil(err)
|
||||
runContainerOpts3 := testManager.GetDeviceRunContainerOptions(newPod, &newPod.Spec.Containers[0])
|
||||
as.Equal(1, len(runContainerOpts3.Envs))
|
||||
|
||||
}
|
||||
|
||||
func TestInitContainerDeviceAllocation(t *testing.T) {
|
||||
// Requesting to create a pod that requests resourceName1 in init containers and normal containers
|
||||
// should succeed with devices allocated to init containers reallocated to normal containers.
|
||||
res1 := TestResource{
|
||||
resourceName: "domain1.com/resource1",
|
||||
resourceQuantity: *resource.NewQuantity(int64(2), resource.DecimalSI),
|
||||
devs: []string{"dev1", "dev2"},
|
||||
}
|
||||
res2 := TestResource{
|
||||
resourceName: "domain2.com/resource2",
|
||||
resourceQuantity: *resource.NewQuantity(int64(1), resource.DecimalSI),
|
||||
devs: []string{"dev3", "dev4"},
|
||||
}
|
||||
testResources := make([]TestResource, 2)
|
||||
testResources = append(testResources, res1)
|
||||
testResources = append(testResources, res2)
|
||||
as := require.New(t)
|
||||
podsStub := activePodsStub{
|
||||
activePods: []*v1.Pod{},
|
||||
}
|
||||
nodeInfo := getTestNodeInfo(v1.ResourceList{})
|
||||
tmpDir, err := ioutil.TempDir("", "checkpoint")
|
||||
as.Nil(err)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
testManager := getTestManager(tmpDir, podsStub.getActivePods, testResources)
|
||||
|
||||
podWithPluginResourcesInInitContainers := &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
UID: uuid.NewUUID(),
|
||||
|
|
@ -566,7 +617,7 @@ func TestPodContainerDeviceAllocation(t *testing.T) {
|
|||
Name: string(uuid.NewUUID()),
|
||||
Resources: v1.ResourceRequirements{
|
||||
Limits: v1.ResourceList{
|
||||
v1.ResourceName(resourceName1): resourceQuantity2,
|
||||
v1.ResourceName(res1.resourceName): res2.resourceQuantity,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -574,7 +625,7 @@ func TestPodContainerDeviceAllocation(t *testing.T) {
|
|||
Name: string(uuid.NewUUID()),
|
||||
Resources: v1.ResourceRequirements{
|
||||
Limits: v1.ResourceList{
|
||||
v1.ResourceName(resourceName1): resourceQuantity1,
|
||||
v1.ResourceName(res1.resourceName): res1.resourceQuantity,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -584,8 +635,8 @@ func TestPodContainerDeviceAllocation(t *testing.T) {
|
|||
Name: string(uuid.NewUUID()),
|
||||
Resources: v1.ResourceRequirements{
|
||||
Limits: v1.ResourceList{
|
||||
v1.ResourceName(resourceName1): resourceQuantity2,
|
||||
v1.ResourceName(resourceName2): resourceQuantity2,
|
||||
v1.ResourceName(res1.resourceName): res2.resourceQuantity,
|
||||
v1.ResourceName(res2.resourceName): res2.resourceQuantity,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -593,8 +644,8 @@ func TestPodContainerDeviceAllocation(t *testing.T) {
|
|||
Name: string(uuid.NewUUID()),
|
||||
Resources: v1.ResourceRequirements{
|
||||
Limits: v1.ResourceList{
|
||||
v1.ResourceName(resourceName1): resourceQuantity2,
|
||||
v1.ResourceName(resourceName2): resourceQuantity2,
|
||||
v1.ResourceName(res1.resourceName): res2.resourceQuantity,
|
||||
v1.ResourceName(res2.resourceName): res2.resourceQuantity,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -609,10 +660,10 @@ func TestPodContainerDeviceAllocation(t *testing.T) {
|
|||
initCont2 := podWithPluginResourcesInInitContainers.Spec.InitContainers[1].Name
|
||||
normalCont1 := podWithPluginResourcesInInitContainers.Spec.Containers[0].Name
|
||||
normalCont2 := podWithPluginResourcesInInitContainers.Spec.Containers[1].Name
|
||||
initCont1Devices := testManager.podDevices.containerDevices(podUID, initCont1, resourceName1)
|
||||
initCont2Devices := testManager.podDevices.containerDevices(podUID, initCont2, resourceName1)
|
||||
normalCont1Devices := testManager.podDevices.containerDevices(podUID, normalCont1, resourceName1)
|
||||
normalCont2Devices := testManager.podDevices.containerDevices(podUID, normalCont2, resourceName1)
|
||||
initCont1Devices := testManager.podDevices.containerDevices(podUID, initCont1, res1.resourceName)
|
||||
initCont2Devices := testManager.podDevices.containerDevices(podUID, initCont2, res1.resourceName)
|
||||
normalCont1Devices := testManager.podDevices.containerDevices(podUID, normalCont1, res1.resourceName)
|
||||
normalCont2Devices := testManager.podDevices.containerDevices(podUID, normalCont2, res1.resourceName)
|
||||
as.True(initCont2Devices.IsSuperset(initCont1Devices))
|
||||
as.True(initCont2Devices.IsSuperset(normalCont1Devices))
|
||||
as.True(initCont2Devices.IsSuperset(normalCont2Devices))
|
||||
|
|
|
|||
Loading…
Reference in New Issue