Merge pull request #42264 from yujuhong/kubemark_cri

Automatic merge from submit-queue

kubemark: enable CRI for the hollow nodes

This fixes #41488
pull/6/head
Kubernetes Submit Queue 2017-03-07 13:04:39 -08:00 committed by GitHub
commit 1ed3aa6750
11 changed files with 320 additions and 290 deletions

View File

@ -51,7 +51,7 @@ func imageToRuntimeAPIImage(image *dockertypes.Image) (*runtimeapi.Image, error)
}
func imageInspectToRuntimeAPIImage(image *dockertypes.ImageInspect) (*runtimeapi.Image, error) {
if image == nil {
if image == nil || image.Config == nil {
return nil, fmt.Errorf("unable to convert a nil pointer to a runtime API image")
}

View File

@ -217,7 +217,7 @@ func (ds *dockerService) createContainerLogSymlink(containerID string) error {
if err != nil {
return fmt.Errorf("failed to get container %q log path: %v", containerID, err)
}
if path != "" {
if path != "" && realPath != "" {
// Only create the symlink when container log path is specified.
if err = ds.os.Symlink(realPath, path); err != nil {
return fmt.Errorf("failed to create symbolic link %q to the container log file %q for container %q: %v",

View File

@ -23,7 +23,9 @@ import (
"testing"
"time"
dockertypes "github.com/docker/engine-api/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
@ -101,14 +103,15 @@ func TestContainerStatus(t *testing.T) {
sConfig := makeSandboxConfig("foo", "bar", "1", 0)
labels := map[string]string{"abc.xyz": "foo"}
annotations := map[string]string{"foo.bar.baz": "abc"}
config := makeContainerConfig(sConfig, "pause", "iamimage", 0, labels, annotations)
imageName := "iamimage"
config := makeContainerConfig(sConfig, "pause", imageName, 0, labels, annotations)
var defaultTime time.Time
dt := defaultTime.UnixNano()
ct, st, ft := dt, dt, dt
state := runtimeapi.ContainerState_CONTAINER_CREATED
imageRef := DockerImageIDPrefix + imageName
// The following variables are not set in FakeDockerClient.
imageRef := DockerImageIDPrefix + ""
exitCode := int32(0)
var reason, message string
@ -128,11 +131,14 @@ func TestContainerStatus(t *testing.T) {
Annotations: config.Annotations,
}
fDocker.InjectImages([]dockertypes.Image{{ID: imageName}})
// Create the container.
fClock.SetTime(time.Now().Add(-1 * time.Hour))
expected.CreatedAt = fClock.Now().UnixNano()
const sandboxId = "sandboxid"
id, err := ds.CreateContainer(sandboxId, config, sConfig)
assert.NoError(t, err)
// Check internal labels
c, err := fDocker.InspectContainer(id)
@ -242,23 +248,20 @@ func TestContainerCreationConflict(t *testing.T) {
expectFields: 6,
},
"random create error": {
createError: randomError,
expectError: randomError,
expectCalls: []string{"create"},
expectFields: 1,
createError: randomError,
expectError: randomError,
expectCalls: []string{"create"},
},
"conflict create error with successful remove": {
createError: conflictError,
expectError: conflictError,
expectCalls: []string{"create", "remove"},
expectFields: 1,
createError: conflictError,
expectError: conflictError,
expectCalls: []string{"create", "remove"},
},
"conflict create error with random remove error": {
createError: conflictError,
removeError: randomError,
expectError: conflictError,
expectCalls: []string{"create", "remove"},
expectFields: 1,
createError: conflictError,
removeError: randomError,
expectError: conflictError,
expectCalls: []string{"create", "remove"},
},
"conflict create error with no such container remove error": {
createError: conflictError,
@ -276,9 +279,13 @@ func TestContainerCreationConflict(t *testing.T) {
if test.removeError != nil {
fDocker.InjectError("remove", test.removeError)
}
name, err := ds.CreateContainer(sandboxId, config, sConfig)
assert.Equal(t, test.expectError, err)
id, err := ds.CreateContainer(sandboxId, config, sConfig)
require.Equal(t, test.expectError, err)
assert.NoError(t, fDocker.AssertCalls(test.expectCalls))
assert.Len(t, strings.Split(name, nameDelimiter), test.expectFields)
if err == nil {
c, err := fDocker.InspectContainer(id)
assert.NoError(t, err)
assert.Len(t, strings.Split(c.Name, nameDelimiter), test.expectFields)
}
}
}

View File

@ -28,7 +28,7 @@ import (
func TestRemoveImage(t *testing.T) {
ds, fakeDocker, _ := newTestDockerService()
id := "1111"
fakeDocker.Image = &dockertypes.ImageInspect{ID: id, RepoTags: []string{"foo"}}
fakeDocker.InjectImageInspects([]dockertypes.ImageInspect{{ID: id, RepoTags: []string{"foo"}}})
ds.RemoveImage(&runtimeapi.ImageSpec{Image: id})
fakeDocker.AssertCallDetails(dockertools.NewCalledDetail("inspect_image", nil),
dockertools.NewCalledDetail("remove_image", []interface{}{id, dockertypes.ImageRemoveOptions{PruneChildren: true}}))
@ -37,7 +37,7 @@ func TestRemoveImage(t *testing.T) {
func TestRemoveImageWithMultipleTags(t *testing.T) {
ds, fakeDocker, _ := newTestDockerService()
id := "1111"
fakeDocker.Image = &dockertypes.ImageInspect{ID: id, RepoTags: []string{"foo", "bar"}}
fakeDocker.InjectImageInspects([]dockertypes.ImageInspect{{ID: id, RepoTags: []string{"foo", "bar"}}})
ds.RemoveImage(&runtimeapi.ImageSpec{Image: id})
fakeDocker.AssertCallDetails(dockertools.NewCalledDetail("inspect_image", nil),
dockertools.NewCalledDetail("remove_image", []interface{}{"foo", dockertypes.ImageRemoveOptions{PruneChildren: true}}),

View File

@ -26,6 +26,7 @@ import (
runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/dockertools"
"k8s.io/kubernetes/pkg/kubelet/network"
"k8s.io/kubernetes/pkg/kubelet/types"
)
@ -157,7 +158,7 @@ func TestNetworkPluginInvocation(t *testing.T) {
map[string]string{"label": name},
map[string]string{"annotation": ns},
)
cID := kubecontainer.ContainerID{Type: runtimeName, ID: fmt.Sprintf("/%v", makeSandboxName(c))}
cID := kubecontainer.ContainerID{Type: runtimeName, ID: dockertools.GetFakeContainerID(fmt.Sprintf("/%v", makeSandboxName(c)))}
mockPlugin.EXPECT().Name().Return("mockNetworkPlugin").AnyTimes()
setup := mockPlugin.EXPECT().SetUpPod(ns, name, cID)
@ -195,7 +196,7 @@ func TestHostNetworkPluginInvocation(t *testing.T) {
},
},
}
cID := kubecontainer.ContainerID{Type: runtimeName, ID: fmt.Sprintf("/%v", makeSandboxName(c))}
cID := kubecontainer.ContainerID{Type: runtimeName, ID: dockertools.GetFakeContainerID(fmt.Sprintf("/%v", makeSandboxName(c)))}
// No calls to network plugin are expected
_, err := ds.RunPodSandbox(c)

View File

@ -21,6 +21,7 @@ import (
"testing"
"github.com/blang/semver"
dockertypes "github.com/docker/engine-api/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -265,29 +266,37 @@ func TestGetSecurityOptSeparator(t *testing.T) {
}
func TestEnsureSandboxImageExists(t *testing.T) {
sandboxImage := "gcr.io/test/image"
for desc, test := range map[string]struct {
inject error
calls []string
err bool
injectImage bool
injectErr error
calls []string
err bool
}{
"should not pull image when it already exists": {
inject: nil,
calls: []string{"inspect_image"},
injectImage: true,
injectErr: nil,
calls: []string{"inspect_image"},
},
"should pull image when it doesn't exist": {
inject: dockertools.ImageNotFoundError{ID: "image_id"},
calls: []string{"inspect_image", "pull"},
injectImage: false,
injectErr: dockertools.ImageNotFoundError{ID: "image_id"},
calls: []string{"inspect_image", "pull"},
},
"should return error when inspect image fails": {
inject: fmt.Errorf("arbitrary error"),
calls: []string{"inspect_image"},
err: true,
injectImage: false,
injectErr: fmt.Errorf("arbitrary error"),
calls: []string{"inspect_image"},
err: true,
},
} {
t.Logf("TestCase: %q", desc)
_, fakeDocker, _ := newTestDockerService()
fakeDocker.InjectError("inspect_image", test.inject)
err := ensureSandboxImageExists(fakeDocker, "gcr.io/test/image")
if test.injectImage {
fakeDocker.InjectImages([]dockertypes.Image{{ID: sandboxImage}})
}
fakeDocker.InjectError("inspect_image", test.injectErr)
err := ensureSandboxImageExists(fakeDocker, sandboxImage)
assert.NoError(t, fakeDocker.AssertCalls(test.calls))
assert.Equal(t, test.err, err != nil)
}

View File

@ -118,13 +118,7 @@ func TestSeccompIsUnconfinedByDefaultWithDockerV110(t *testing.T) {
"create", "start", "inspect_container",
})
fakeDocker.Lock()
if len(fakeDocker.Created) != 2 ||
!matchString(t, "/k8s_POD\\.[a-f0-9]+_foo_new_", fakeDocker.Created[0]) ||
!matchString(t, "/k8s_bar\\.[a-f0-9]+_foo_new_", fakeDocker.Created[1]) {
t.Errorf("unexpected containers created %v", fakeDocker.Created)
}
fakeDocker.Unlock()
assert.NoError(t, fakeDocker.AssertCreatedByNameWithOrder([]string{"POD", "bar"}))
newContainer, err := fakeDocker.InspectContainer(fakeDocker.Created[1])
if err != nil {
@ -157,13 +151,7 @@ func TestUnconfinedSeccompProfileWithDockerV110(t *testing.T) {
"create", "start", "inspect_container",
})
fakeDocker.Lock()
if len(fakeDocker.Created) != 2 ||
!matchString(t, "/k8s_POD\\.[a-f0-9]+_foo4_new_", fakeDocker.Created[0]) ||
!matchString(t, "/k8s_bar4\\.[a-f0-9]+_foo4_new_", fakeDocker.Created[1]) {
t.Errorf("unexpected containers created %v", fakeDocker.Created)
}
fakeDocker.Unlock()
assert.NoError(t, fakeDocker.AssertCreatedByNameWithOrder([]string{"POD", "bar4"}))
newContainer, err := fakeDocker.InspectContainer(fakeDocker.Created[1])
if err != nil {
@ -192,13 +180,7 @@ func TestDefaultSeccompProfileWithDockerV110(t *testing.T) {
"create", "start", "inspect_container",
})
fakeDocker.Lock()
if len(fakeDocker.Created) != 2 ||
!matchString(t, "/k8s_POD\\.[a-f0-9]+_foo1_new_", fakeDocker.Created[0]) ||
!matchString(t, "/k8s_bar1\\.[a-f0-9]+_foo1_new_", fakeDocker.Created[1]) {
t.Errorf("unexpected containers created %v", fakeDocker.Created)
}
fakeDocker.Unlock()
assert.NoError(t, fakeDocker.AssertCreatedByNameWithOrder([]string{"POD", "bar1"}))
newContainer, err := fakeDocker.InspectContainer(fakeDocker.Created[1])
if err != nil {
@ -228,13 +210,7 @@ func TestSeccompContainerAnnotationTrumpsPod(t *testing.T) {
"create", "start", "inspect_container",
})
fakeDocker.Lock()
if len(fakeDocker.Created) != 2 ||
!matchString(t, "/k8s_POD\\.[a-f0-9]+_foo2_new_", fakeDocker.Created[0]) ||
!matchString(t, "/k8s_bar2\\.[a-f0-9]+_foo2_new_", fakeDocker.Created[1]) {
t.Errorf("unexpected containers created %v", fakeDocker.Created)
}
fakeDocker.Unlock()
assert.NoError(t, fakeDocker.AssertCreatedByNameWithOrder([]string{"POD", "bar2"}))
newContainer, err := fakeDocker.InspectContainer(fakeDocker.Created[1])
if err != nil {
@ -260,13 +236,7 @@ func TestSecurityOptsAreNilWithDockerV19(t *testing.T) {
"create", "start", "inspect_container",
})
fakeDocker.Lock()
if len(fakeDocker.Created) != 2 ||
!matchString(t, "/k8s_POD\\.[a-f0-9]+_foo_new_", fakeDocker.Created[0]) ||
!matchString(t, "/k8s_bar\\.[a-f0-9]+_foo_new_", fakeDocker.Created[1]) {
t.Errorf("unexpected containers created %v", fakeDocker.Created)
}
fakeDocker.Unlock()
assert.NoError(t, fakeDocker.AssertCreatedByNameWithOrder([]string{"POD", "bar"}))
newContainer, err := fakeDocker.InspectContainer(fakeDocker.Created[1])
if err != nil {
@ -306,13 +276,7 @@ func TestCreateAppArmorContanier(t *testing.T) {
"create", "start", "inspect_container",
})
fakeDocker.Lock()
if len(fakeDocker.Created) != 2 ||
!matchString(t, "/k8s_POD\\.[a-f0-9]+_foo_new_", fakeDocker.Created[0]) ||
!matchString(t, "/k8s_test\\.[a-f0-9]+_foo_new_", fakeDocker.Created[1]) {
t.Errorf("unexpected containers created %v", fakeDocker.Created)
}
fakeDocker.Unlock()
assert.NoError(t, fakeDocker.AssertCreatedByNameWithOrder([]string{"POD", "test"}))
// Verify security opts.
newContainer, err := fakeDocker.InspectContainer(fakeDocker.Created[1])
@ -384,13 +348,7 @@ func TestSeccompLocalhostProfileIsLoaded(t *testing.T) {
"create", "start", "inspect_container",
})
fakeDocker.Lock()
if len(fakeDocker.Created) != 2 ||
!matchString(t, "/k8s_POD\\.[a-f0-9]+_foo2_new_", fakeDocker.Created[0]) ||
!matchString(t, "/k8s_bar2\\.[a-f0-9]+_foo2_new_", fakeDocker.Created[1]) {
t.Errorf("unexpected containers created %v", fakeDocker.Created)
}
fakeDocker.Unlock()
assert.NoError(t, fakeDocker.AssertCreatedByNameWithOrder([]string{"POD", "bar2"}))
newContainer, err := fakeDocker.InspectContainer(fakeDocker.Created[1])
if err != nil {

View File

@ -398,7 +398,7 @@ func TestListImages(t *testing.T) {
func TestDeleteImage(t *testing.T) {
manager, fakeDocker := newTestDockerManager()
fakeDocker.Image = &dockertypes.ImageInspect{ID: "1111", RepoTags: []string{"foo"}}
fakeDocker.InjectImages([]dockertypes.Image{{ID: "1111", RepoTags: []string{"foo"}}})
manager.RemoveImage(kubecontainer.ImageSpec{Image: "1111"})
fakeDocker.AssertCallDetails(NewCalledDetail("inspect_image", nil), NewCalledDetail("remove_image",
[]interface{}{"1111", dockertypes.ImageRemoveOptions{PruneChildren: true}}))
@ -406,7 +406,7 @@ func TestDeleteImage(t *testing.T) {
func TestDeleteImageWithMultipleTags(t *testing.T) {
manager, fakeDocker := newTestDockerManager()
fakeDocker.Image = &dockertypes.ImageInspect{ID: "1111", RepoTags: []string{"foo", "bar"}}
fakeDocker.InjectImages([]dockertypes.Image{{ID: "1111", RepoTags: []string{"foo", "bar"}}})
manager.RemoveImage(kubecontainer.ImageSpec{Image: "1111"})
fakeDocker.AssertCallDetails(NewCalledDetail("inspect_image", nil),
NewCalledDetail("remove_image", []interface{}{"foo", dockertypes.ImageRemoveOptions{PruneChildren: true}}),
@ -601,21 +601,14 @@ func TestSyncPodCreateNetAndContainer(t *testing.T) {
if !found {
t.Errorf("Custom pod infra container not found: %v", fakeDocker.RunningContainerList)
}
if len(fakeDocker.Created) != 2 ||
!matchString(t, "/k8s_POD\\.[a-f0-9]+_foo_new_", fakeDocker.Created[0]) ||
!matchString(t, "/k8s_bar\\.[a-f0-9]+_foo_new_", fakeDocker.Created[1]) {
t.Errorf("unexpected containers created %v", fakeDocker.Created)
}
fakeDocker.Unlock()
assert.NoError(t, fakeDocker.AssertCreatedByNameWithOrder([]string{"POD", "bar"}))
}
func TestSyncPodCreatesNetAndContainerPullsImage(t *testing.T) {
dm, fakeDocker := newTestDockerManagerWithRealImageManager()
dm.podInfraContainerImage = "foo/infra_image:v1"
puller := dm.dockerPuller.(*FakeDockerPuller)
puller.HasImages = []string{}
dm.podInfraContainerImage = "foo/infra_image:v1"
pod := makePod("foo", &v1.PodSpec{
Containers: []v1.Container{
{Name: "bar", Image: "foo/something:v0", ImagePullPolicy: "IfNotPresent"},
@ -626,23 +619,13 @@ func TestSyncPodCreatesNetAndContainerPullsImage(t *testing.T) {
verifyCalls(t, fakeDocker, []string{
// Create pod infra container.
"inspect_image", "create", "start", "inspect_container", "inspect_container",
"inspect_image", "pull", "inspect_image", "create", "start", "inspect_container", "inspect_container",
// Create container.
"inspect_image", "create", "start", "inspect_container",
"inspect_image", "pull", "inspect_image", "create", "start", "inspect_container",
})
fakeDocker.Lock()
if !reflect.DeepEqual(puller.ImagesPulled, []string{"foo/infra_image:v1", "foo/something:v0"}) {
t.Errorf("unexpected pulled containers: %v", puller.ImagesPulled)
}
if len(fakeDocker.Created) != 2 ||
!matchString(t, "/k8s_POD\\.[a-f0-9]+_foo_new_", fakeDocker.Created[0]) ||
!matchString(t, "/k8s_bar\\.[a-f0-9]+_foo_new_", fakeDocker.Created[1]) {
t.Errorf("unexpected containers created %v", fakeDocker.Created)
}
fakeDocker.Unlock()
assert.NoError(t, fakeDocker.AssertImagesPulled([]string{"foo/infra_image:v1", "foo/something:v0"}))
assert.NoError(t, fakeDocker.AssertCreatedByNameWithOrder([]string{"POD", "bar"}))
}
func TestSyncPodWithPodInfraCreatesContainer(t *testing.T) {
@ -665,12 +648,7 @@ func TestSyncPodWithPodInfraCreatesContainer(t *testing.T) {
"create", "start", "inspect_container",
})
fakeDocker.Lock()
if len(fakeDocker.Created) != 1 ||
!matchString(t, "/k8s_bar\\.[a-f0-9]+_foo_new_", fakeDocker.Created[0]) {
t.Errorf("unexpected containers created %v", fakeDocker.Created)
}
fakeDocker.Unlock()
assert.NoError(t, fakeDocker.AssertCreatedByName([]string{"bar"}))
}
func TestSyncPodDeletesWithNoPodInfraContainer(t *testing.T) {
@ -696,16 +674,7 @@ func TestSyncPodDeletesWithNoPodInfraContainer(t *testing.T) {
"create", "start", "inspect_container",
})
// A map iteration is used to delete containers, so must not depend on
// order here.
expectedToStop := map[string]bool{
"1234": true,
}
fakeDocker.Lock()
if len(fakeDocker.Stopped) != 1 || !expectedToStop[fakeDocker.Stopped[0]] {
t.Errorf("Wrong containers were stopped: %v", fakeDocker.Stopped)
}
fakeDocker.Unlock()
assert.NoError(t, fakeDocker.AssertStopped([]string{"1234"}))
}
func TestSyncPodDeletesDuplicate(t *testing.T) {
@ -904,7 +873,7 @@ func TestSyncPodWithRestartPolicy(t *testing.T) {
// 'stop' is because the pod infra container is killed when no container is running.
verifyCalls(t, fakeDocker, tt.calls)
if err := fakeDocker.AssertCreated(tt.created); err != nil {
if err := fakeDocker.AssertCreatedByName(tt.created); err != nil {
t.Errorf("case [%d]: %v", i, err)
}
if err := fakeDocker.AssertStopped(tt.stopped); err != nil {
@ -1142,12 +1111,8 @@ func TestSyncPodWithPodInfraCreatesContainerCallsHandler(t *testing.T) {
"create", "start", "inspect_container",
})
fakeDocker.Lock()
if len(fakeDocker.Created) != 1 ||
!matchString(t, "/k8s_bar\\.[a-f0-9]+_foo_new_", fakeDocker.Created[0]) {
t.Errorf("unexpected containers created %v", fakeDocker.Created)
}
fakeDocker.Unlock()
assert.NoError(t, fakeDocker.AssertCreatedByName([]string{"bar"}))
if fakeHTTPClient.url != "http://foo:8080/bar" {
t.Errorf("unexpected handler: %q", fakeHTTPClient.url)
}
@ -1187,16 +1152,7 @@ func TestSyncPodEventHandlerFails(t *testing.T) {
"stop",
})
if len(fakeDocker.Stopped) != 1 {
t.Fatalf("Wrong containers were stopped: %v", fakeDocker.Stopped)
}
dockerName, _, err := ParseDockerName(fakeDocker.Stopped[0])
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if dockerName.ContainerName != "bar" {
t.Errorf("Wrong stopped container, expected: bar, get: %q", dockerName.ContainerName)
}
assert.NoError(t, fakeDocker.AssertStoppedByName([]string{"bar"}))
}
type fakeReadWriteCloser struct{}
@ -1252,13 +1208,9 @@ func TestSyncPodWithTerminationLog(t *testing.T) {
"create", "start", "inspect_container",
})
fakeDocker.Lock()
if len(fakeDocker.Created) != 2 ||
!matchString(t, "/k8s_POD\\.[a-f0-9]+_foo_new_", fakeDocker.Created[0]) ||
!matchString(t, "/k8s_bar\\.[a-f0-9]+_foo_new_", fakeDocker.Created[1]) {
t.Errorf("unexpected containers created %v", fakeDocker.Created)
}
fakeDocker.Unlock()
defer os.Remove(testPodContainerDir)
assert.NoError(t, fakeDocker.AssertCreatedByNameWithOrder([]string{"POD", "bar"}))
newContainer, err := fakeDocker.InspectContainer(fakeDocker.Created[1])
if err != nil {
t.Fatalf("unexpected error %v", err)
@ -1290,13 +1242,7 @@ func TestSyncPodWithHostNetwork(t *testing.T) {
"create", "start", "inspect_container",
})
fakeDocker.Lock()
if len(fakeDocker.Created) != 2 ||
!matchString(t, "/k8s_POD\\.[a-f0-9]+_foo_new_", fakeDocker.Created[0]) ||
!matchString(t, "/k8s_bar\\.[a-f0-9]+_foo_new_", fakeDocker.Created[1]) {
t.Errorf("unexpected containers created %v", fakeDocker.Created)
}
fakeDocker.Unlock()
assert.NoError(t, fakeDocker.AssertCreatedByNameWithOrder([]string{"POD", "bar"}))
newContainer, err := fakeDocker.InspectContainer(fakeDocker.Created[1])
if err != nil {
@ -1323,22 +1269,25 @@ func TestVerifyNonRoot(t *testing.T) {
// success cases
"non-root runAsUser": {
container: &v1.Container{
Image: "foobar",
SecurityContext: &v1.SecurityContext{
RunAsUser: &nonRootUid,
},
},
},
"numeric non-root image user": {
container: &v1.Container{},
container: &v1.Container{Image: "foobar"},
inspectImage: &dockertypes.ImageInspect{
ID: "foobar",
Config: &dockercontainer.Config{
User: "1",
},
},
},
"numeric non-root image user with gid": {
container: &v1.Container{},
container: &v1.Container{Image: "foobar"},
inspectImage: &dockertypes.ImageInspect{
ID: "foobar",
Config: &dockercontainer.Config{
User: "1:2",
},
@ -1348,6 +1297,7 @@ func TestVerifyNonRoot(t *testing.T) {
// failure cases
"root runAsUser": {
container: &v1.Container{
Image: "foobar",
SecurityContext: &v1.SecurityContext{
RunAsUser: &rootUid,
},
@ -1355,8 +1305,9 @@ func TestVerifyNonRoot(t *testing.T) {
expectedError: "container's runAsUser breaks non-root policy",
},
"non-numeric image user": {
container: &v1.Container{},
container: &v1.Container{Image: "foobar"},
inspectImage: &dockertypes.ImageInspect{
ID: "foobar",
Config: &dockercontainer.Config{
User: "foo",
},
@ -1364,8 +1315,9 @@ func TestVerifyNonRoot(t *testing.T) {
expectedError: "non-numeric user",
},
"numeric root image user": {
container: &v1.Container{},
container: &v1.Container{Image: "foobar"},
inspectImage: &dockertypes.ImageInspect{
ID: "foobar",
Config: &dockercontainer.Config{
User: "0",
},
@ -1373,8 +1325,9 @@ func TestVerifyNonRoot(t *testing.T) {
expectedError: "container has no runAsUser and image will run as root",
},
"numeric root image user with gid": {
container: &v1.Container{},
container: &v1.Container{Image: "foobar"},
inspectImage: &dockertypes.ImageInspect{
ID: "foobar",
Config: &dockercontainer.Config{
User: "0:1",
},
@ -1382,19 +1335,22 @@ func TestVerifyNonRoot(t *testing.T) {
expectedError: "container has no runAsUser and image will run as root",
},
"nil image in inspect": {
container: &v1.Container{},
container: &v1.Container{Image: "foobar"},
inspectImage: nil,
expectedError: "unable to inspect image",
expectedError: ImageNotFoundError{"foobar"}.Error(),
},
"nil config in image inspect": {
container: &v1.Container{},
inspectImage: &dockertypes.ImageInspect{},
container: &v1.Container{Image: "foobar"},
inspectImage: &dockertypes.ImageInspect{ID: "foobar"},
expectedError: "unable to inspect image",
},
}
for k, v := range tests {
fakeDocker.Image = v.inspectImage
fakeDocker.ResetImages()
if v.inspectImage != nil {
fakeDocker.InjectImageInspects([]dockertypes.ImageInspect{*v.inspectImage})
}
err := dm.verifyNonRoot(v.container)
if v.expectedError == "" && err != nil {
t.Errorf("case[%q]: unexpected error: %v", k, err)
@ -1481,8 +1437,8 @@ func TestGetIPCMode(t *testing.T) {
func TestSyncPodWithPullPolicy(t *testing.T) {
dm, fakeDocker := newTestDockerManagerWithRealImageManager()
puller := dm.dockerPuller.(*FakeDockerPuller)
puller.HasImages = []string{"foo/existing_one:v1", "foo/want:latest"}
fakeDocker.InjectImages([]dockertypes.Image{{ID: "foo/existing_one:v1"}, {ID: "foo/want:latest"}})
dm.podInfraContainerImage = "foo/infra_image:v1"
pod := makePod("foo", &v1.PodSpec{
@ -1511,13 +1467,11 @@ func TestSyncPodWithPullPolicy(t *testing.T) {
result := runSyncPod(t, dm, fakeDocker, pod, nil, true)
verifySyncResults(t, expectedResults, result)
assert.NoError(t, fakeDocker.AssertImagesPulled([]string{"foo/infra_image:v1", "foo/pull_always_image:v1", "foo/pull_if_not_present_image:v1"}))
fakeDocker.Lock()
defer fakeDocker.Unlock()
pulledImageSorted := puller.ImagesPulled[:]
sort.Strings(pulledImageSorted)
assert.Equal(t, []string{"foo/infra_image:v1", "foo/pull_always_image:v1", "foo/pull_if_not_present_image:v1"}, pulledImageSorted)
if len(fakeDocker.Created) != 5 {
t.Errorf("unexpected containers created %v", fakeDocker.Created)
}
@ -1531,33 +1485,28 @@ func TestSyncPodWithFailure(t *testing.T) {
tests := map[string]struct {
container v1.Container
dockerError map[string]error
pullerError []error
expected []*kubecontainer.SyncResult
}{
"PullImageFailure": {
v1.Container{Name: "bar", Image: "foo/real_image:v1", ImagePullPolicy: v1.PullAlways},
map[string]error{},
[]error{fmt.Errorf("can't pull image")},
map[string]error{"pull": fmt.Errorf("can't pull image")},
[]*kubecontainer.SyncResult{{kubecontainer.StartContainer, "bar", images.ErrImagePull, "can't pull image"}},
},
"CreateContainerFailure": {
v1.Container{Name: "bar", Image: "foo/already_present:v2"},
map[string]error{"create": fmt.Errorf("can't create container")},
[]error{},
[]*kubecontainer.SyncResult{{kubecontainer.StartContainer, "bar", kubecontainer.ErrRunContainer, "can't create container"}},
},
"StartContainerFailure": {
v1.Container{Name: "bar", Image: "foo/already_present:v2"},
map[string]error{"start": fmt.Errorf("can't start container")},
[]error{},
[]*kubecontainer.SyncResult{{kubecontainer.StartContainer, "bar", kubecontainer.ErrRunContainer, "can't start container"}},
},
}
for _, test := range tests {
dm, fakeDocker := newTestDockerManagerWithRealImageManager()
puller := dm.dockerPuller.(*FakeDockerPuller)
puller.HasImages = []string{test.container.Image}
fakeDocker.InjectImages([]dockertypes.Image{{ID: test.container.Image}})
// Pretend that the pod infra container has already been created, so that
// we can run the user containers.
fakeDocker.SetFakeRunningContainers([]*FakeContainer{{
@ -1565,7 +1514,6 @@ func TestSyncPodWithFailure(t *testing.T) {
Name: "/k8s_POD." + strconv.FormatUint(generatePodInfraContainerHash(pod), 16) + "_foo_new_12345678_0",
}})
fakeDocker.InjectErrors(test.dockerError)
puller.ErrorsToInject = test.pullerError
pod.Spec.Containers = []v1.Container{test.container}
result := runSyncPod(t, dm, fakeDocker, pod, nil, true)
verifySyncResults(t, test.expected, result)

View File

@ -19,18 +19,20 @@ package dockertools
import (
"encoding/json"
"fmt"
"hash/fnv"
"math/rand"
"os"
"reflect"
"sort"
"strconv"
"strings"
"sync"
"time"
dockertypes "github.com/docker/engine-api/types"
dockercontainer "github.com/docker/engine-api/types/container"
"k8s.io/client-go/util/clock"
"k8s.io/client-go/util/clock"
"k8s.io/kubernetes/pkg/api/v1"
)
@ -51,7 +53,7 @@ type FakeDockerClient struct {
RunningContainerList []dockertypes.Container
ExitedContainerList []dockertypes.Container
ContainerMap map[string]*dockertypes.ContainerJSON
Image *dockertypes.ImageInspect
ImageInspects map[string]*dockertypes.ImageInspect
Images []dockertypes.Image
Errors map[string]error
called []calledDetail
@ -59,10 +61,13 @@ type FakeDockerClient struct {
EnableTrace bool
// Created, Started, Stopped and Removed all contain container docker ID
Created []string
Started []string
Stopped []string
Removed []string
Created []string
Started []string
Stopped []string
Removed []string
// Images pulled by ref (name or ID).
ImagesPulled []string
VersionInfo dockertypes.Version
Information dockertypes.Info
ExecInspect *dockertypes.ContainerExecInspect
@ -71,9 +76,13 @@ type FakeDockerClient struct {
ImageHistoryMap map[string][]dockertypes.ImageHistory
}
// We don't check docker version now, just set the docker version of fake docker client to 1.8.1.
// Notice that if someday we also have minimum docker version requirement, this should also be updated.
const fakeDockerVersion = "1.8.1"
const (
// We don't check docker version now, just set the docker version of fake docker client to 1.8.1.
// Notice that if someday we also have minimum docker version requirement, this should also be updated.
fakeDockerVersion = "1.8.1"
fakeImageSize = 1024
)
func NewFakeDockerClient() *FakeDockerClient {
return &FakeDockerClient{
@ -81,10 +90,9 @@ func NewFakeDockerClient() *FakeDockerClient {
Errors: make(map[string]error),
ContainerMap: make(map[string]*dockertypes.ContainerJSON),
Clock: clock.RealClock{},
// default this to an empty result, so that we never have a nil non-error response from InspectImage
Image: &dockertypes.ImageInspect{},
// default this to true, so that we trace calls, image pulls and container lifecycle
EnableTrace: true,
EnableTrace: true,
ImageInspects: make(map[string]*dockertypes.ImageInspect),
}
}
@ -201,8 +209,9 @@ func convertFakeContainer(f *FakeContainer) *dockertypes.ContainerJSON {
}
return &dockertypes.ContainerJSON{
ContainerJSONBase: &dockertypes.ContainerJSONBase{
ID: f.ID,
Name: f.Name,
ID: f.ID,
Name: f.Name,
Image: f.Config.Image,
State: &dockertypes.ContainerState{
Running: f.Running,
ExitCode: f.ExitCode,
@ -273,55 +282,74 @@ func (f *FakeDockerClient) AssertCallDetails(calls ...calledDetail) (err error)
return
}
func (f *FakeDockerClient) AssertCreated(created []string) error {
// idsToNames converts container ids into names. The caller must hold the lock.
func (f *FakeDockerClient) idsToNames(ids []string) ([]string, error) {
names := []string{}
for _, id := range ids {
dockerName, _, err := ParseDockerName(f.ContainerMap[id].Name)
if err != nil {
return nil, fmt.Errorf("unexpected error: %v", err)
}
names = append(names, dockerName.ContainerName)
}
return names, nil
}
func (f *FakeDockerClient) AssertCreatedByNameWithOrder(created []string) error {
f.Lock()
defer f.Unlock()
actualCreated := []string{}
for _, c := range f.Created {
dockerName, _, err := ParseDockerName(c)
if err != nil {
return fmt.Errorf("unexpected error: %v", err)
}
actualCreated = append(actualCreated, dockerName.ContainerName)
actualCreated, err := f.idsToNames(f.Created)
if err != nil {
return err
}
sort.StringSlice(created).Sort()
sort.StringSlice(actualCreated).Sort()
if !reflect.DeepEqual(created, actualCreated) {
return fmt.Errorf("expected %#v, got %#v", created, actualCreated)
}
return nil
}
func (f *FakeDockerClient) AssertStarted(started []string) error {
func (f *FakeDockerClient) AssertCreatedByName(created []string) error {
f.Lock()
defer f.Unlock()
sort.StringSlice(started).Sort()
sort.StringSlice(f.Started).Sort()
if !reflect.DeepEqual(started, f.Started) {
return fmt.Errorf("expected %#v, got %#v", started, f.Started)
actualCreated, err := f.idsToNames(f.Created)
if err != nil {
return err
}
return nil
return sortedStringSlicesEqual(created, actualCreated)
}
func (f *FakeDockerClient) AssertStoppedByName(stopped []string) error {
f.Lock()
defer f.Unlock()
actualStopped, err := f.idsToNames(f.Stopped)
if err != nil {
return err
}
return sortedStringSlicesEqual(stopped, actualStopped)
}
func (f *FakeDockerClient) AssertStopped(stopped []string) error {
f.Lock()
defer f.Unlock()
sort.StringSlice(stopped).Sort()
sort.StringSlice(f.Stopped).Sort()
if !reflect.DeepEqual(stopped, f.Stopped) {
return fmt.Errorf("expected %#v, got %#v", stopped, f.Stopped)
}
return nil
// Copy stopped to avoid modifying it.
actualStopped := append([]string{}, f.Stopped...)
return sortedStringSlicesEqual(stopped, actualStopped)
}
func (f *FakeDockerClient) AssertRemoved(removed []string) error {
func (f *FakeDockerClient) AssertImagesPulled(pulled []string) error {
f.Lock()
defer f.Unlock()
sort.StringSlice(removed).Sort()
sort.StringSlice(f.Removed).Sort()
if !reflect.DeepEqual(removed, f.Removed) {
return fmt.Errorf("expected %#v, got %#v", removed, f.Removed)
// Copy pulled to avoid modifying it.
actualPulled := append([]string{}, f.ImagesPulled...)
return sortedStringSlicesEqual(pulled, actualPulled)
}
func sortedStringSlicesEqual(expected, actual []string) error {
sort.StringSlice(expected).Sort()
sort.StringSlice(actual).Sort()
if !reflect.DeepEqual(expected, actual) {
return fmt.Errorf("expected %#v, got %#v", expected, actual)
}
return nil
}
@ -353,30 +381,57 @@ func (f *FakeDockerClient) ListContainers(options dockertypes.ContainerListOptio
// TODO(random-liu): Is a fully sorted array needed?
containerList = append(containerList, f.ExitedContainerList...)
}
// TODO: Support other filters.
// Filter containers with id, only support 1 id.
idFilters := options.Filter.Get("id")
if len(idFilters) != 0 {
var filtered []dockertypes.Container
for _, container := range containerList {
for _, idFilter := range idFilters {
if container.ID == idFilter {
filtered = append(filtered, container)
break
}
}
}
containerList = filtered
}
// Filter containers with status, only support 1 status.
statusFilters := options.Filter.Get("status")
if len(statusFilters) == 1 {
var filtered []dockertypes.Container
for _, container := range containerList {
for _, statusFilter := range statusFilters {
if container.Status == statusFilter {
filtered = append(filtered, container)
break
}
}
}
containerList = filtered
}
// Filter containers with label filter.
labelFilters := options.Filter.Get("label")
if len(labelFilters) == 0 {
return containerList, err
}
var filtered []dockertypes.Container
for _, container := range containerList {
match := true
for _, labelFilter := range labelFilters {
kv := strings.Split(labelFilter, "=")
if len(kv) != 2 {
return nil, fmt.Errorf("invalid label filter %q", labelFilter)
if len(labelFilters) != 0 {
var filtered []dockertypes.Container
for _, container := range containerList {
match := true
for _, labelFilter := range labelFilters {
kv := strings.Split(labelFilter, "=")
if len(kv) != 2 {
return nil, fmt.Errorf("invalid label filter %q", labelFilter)
}
if container.Labels[kv[0]] != kv[1] {
match = false
break
}
}
if container.Labels[kv[0]] != kv[1] {
match = false
break
if match {
filtered = append(filtered, container)
}
}
if match {
filtered = append(filtered, container)
}
containerList = filtered
}
return filtered, err
return containerList, err
}
// InspectContainer is a test-spy implementation of DockerInterface.InspectContainer.
@ -402,8 +457,13 @@ func (f *FakeDockerClient) InspectImageByRef(name string) (*dockertypes.ImageIns
f.Lock()
defer f.Unlock()
f.appendCalled(calledDetail{name: "inspect_image"})
err := f.popError("inspect_image")
return f.Image, err
if err := f.popError("inspect_image"); err != nil {
return nil, err
}
if result, ok := f.ImageInspects[name]; ok {
return result, nil
}
return nil, ImageNotFoundError{name}
}
// InspectImageByID is a test-spy implementation of DockerInterface.InspectImageByID.
@ -412,8 +472,13 @@ func (f *FakeDockerClient) InspectImageByID(name string) (*dockertypes.ImageInsp
f.Lock()
defer f.Unlock()
f.appendCalled(calledDetail{name: "inspect_image"})
err := f.popError("inspect_image")
return f.Image, err
if err := f.popError("inspect_image"); err != nil {
return nil, err
}
if result, ok := f.ImageInspects[name]; ok {
return result, nil
}
return nil, ImageNotFoundError{name}
}
// Sleeps random amount of time with the normal distribution with given mean and stddev
@ -430,6 +495,13 @@ func (f *FakeDockerClient) normalSleep(mean, stdDev, cutOffMillis int) {
time.Sleep(delay)
}
// GetFakeContainerID generates a fake container id from container name with a hash.
func GetFakeContainerID(name string) string {
hash := fnv.New64a()
hash.Write([]byte(name))
return strconv.FormatUint(hash.Sum64(), 16)
}
// CreateContainer is a test-spy implementation of DockerInterface.CreateContainer.
// It adds an entry "create" to the internal method call record.
func (f *FakeDockerClient) CreateContainer(c dockertypes.ContainerCreateConfig) (*dockertypes.ContainerCreateResponse, error) {
@ -442,15 +514,17 @@ func (f *FakeDockerClient) CreateContainer(c dockertypes.ContainerCreateConfig)
// This is not a very good fake. We'll just add this container's name to the list.
// Docker likes to add a '/', so copy that behavior.
name := "/" + c.Name
id := name
f.appendContainerTrace("Created", name)
id := GetFakeContainerID(name)
f.appendContainerTrace("Created", id)
// The newest container should be in front, because we assume so in GetPodStatus()
f.RunningContainerList = append([]dockertypes.Container{
{ID: name, Names: []string{name}, Image: c.Config.Image, Labels: c.Config.Labels},
{ID: id, Names: []string{name}, Image: c.Config.Image, Labels: c.Config.Labels},
}, f.RunningContainerList...)
f.ContainerMap[name] = convertFakeContainer(&FakeContainer{
f.ContainerMap[id] = convertFakeContainer(&FakeContainer{
ID: id, Name: name, Config: c.Config, HostConfig: c.HostConfig, CreatedAt: f.Clock.Now()})
f.normalSleep(100, 25, 25)
return &dockertypes.ContainerCreateResponse{ID: id}, nil
}
@ -557,11 +631,11 @@ func (f *FakeDockerClient) PullImage(image string, auth dockertypes.AuthConfig,
err := f.popError("pull")
if err == nil {
authJson, _ := json.Marshal(auth)
f.Image = &dockertypes.ImageInspect{
ID: image,
RepoTags: []string{image},
}
inspect := createImageInspectFromRef(image)
f.ImageInspects[image] = inspect
f.appendPulled(fmt.Sprintf("%s using %s", image, string(authJson)))
f.Images = append(f.Images, *createImageFromImageInspect(*inspect))
f.ImagesPulled = append(f.ImagesPulled, image)
}
return err
}
@ -604,12 +678,16 @@ func (f *FakeDockerClient) InspectExec(id string) (*dockertypes.ContainerExecIns
}
func (f *FakeDockerClient) ListImages(opts dockertypes.ImageListOptions) ([]dockertypes.Image, error) {
f.Lock()
defer f.Unlock()
f.appendCalled(calledDetail{name: "list_images"})
err := f.popError("list_images")
return f.Images, err
}
func (f *FakeDockerClient) RemoveImage(image string, opts dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDelete, error) {
f.Lock()
defer f.Unlock()
f.appendCalled(calledDetail{name: "remove_image", arguments: []interface{}{image, opts}})
err := f.popError("remove_image")
if err == nil {
@ -627,6 +705,25 @@ func (f *FakeDockerClient) InjectImages(images []dockertypes.Image) {
f.Lock()
defer f.Unlock()
f.Images = append(f.Images, images...)
for _, i := range images {
f.ImageInspects[i.ID] = createImageInspectFromImage(i)
}
}
func (f *FakeDockerClient) ResetImages() {
f.Lock()
defer f.Unlock()
f.Images = []dockertypes.Image{}
f.ImageInspects = make(map[string]*dockertypes.ImageInspect)
}
func (f *FakeDockerClient) InjectImageInspects(inspects []dockertypes.ImageInspect) {
f.Lock()
defer f.Unlock()
for _, i := range inspects {
f.Images = append(f.Images, *createImageFromImageInspect(i))
f.ImageInspects[i.ID] = &i
}
}
func (f *FakeDockerClient) updateContainerStatus(id, status string) {
@ -651,44 +748,43 @@ func (f *FakeDockerClient) ResizeContainerTTY(id string, height, width int) erro
return nil
}
// FakeDockerPuller is a stub implementation of DockerPuller.
type FakeDockerPuller struct {
sync.Mutex
HasImages []string
ImagesPulled []string
// Every pull will return the first error here, and then reslice
// to remove it. Will give nil errors if this slice is empty.
ErrorsToInject []error
func createImageInspectFromRef(ref string) *dockertypes.ImageInspect {
return &dockertypes.ImageInspect{
ID: ref,
RepoTags: []string{ref},
// Image size is required to be non-zero for CRI integration.
VirtualSize: fakeImageSize,
Size: fakeImageSize,
Config: &dockercontainer.Config{},
}
}
// Pull records the image pull attempt, and optionally injects an error.
func (f *FakeDockerPuller) Pull(image string, secrets []v1.Secret) (err error) {
f.Lock()
defer f.Unlock()
f.ImagesPulled = append(f.ImagesPulled, image)
if len(f.ErrorsToInject) > 0 {
err = f.ErrorsToInject[0]
f.ErrorsToInject = f.ErrorsToInject[1:]
func createImageInspectFromImage(image dockertypes.Image) *dockertypes.ImageInspect {
return &dockertypes.ImageInspect{
ID: image.ID,
RepoTags: image.RepoTags,
// Image size is required to be non-zero for CRI integration.
VirtualSize: fakeImageSize,
Size: fakeImageSize,
Config: &dockercontainer.Config{},
}
return err
}
func (f *FakeDockerPuller) GetImageRef(name string) (string, error) {
f.Lock()
defer f.Unlock()
if f.HasImages == nil {
return name, nil
func createImageFromImageInspect(inspect dockertypes.ImageInspect) *dockertypes.Image {
return &dockertypes.Image{
ID: inspect.ID,
RepoTags: inspect.RepoTags,
// Image size is required to be non-zero for CRI integration.
VirtualSize: fakeImageSize,
Size: fakeImageSize,
}
for _, s := range f.HasImages {
if s == name {
return s, nil
}
}
return "", nil
}
// dockerTimestampToString converts the timestamp to string
func dockerTimestampToString(t time.Time) string {
return t.Format(time.RFC3339Nano)
}
func (f *FakeDockerClient) ImageHistory(id string) ([]dockertypes.ImageHistory, error) {
f.Lock()
defer f.Unlock()
@ -703,7 +799,20 @@ func (f *FakeDockerClient) InjectImageHistory(data map[string][]dockertypes.Imag
f.ImageHistoryMap = data
}
// dockerTimestampToString converts the timestamp to string
func dockerTimestampToString(t time.Time) string {
return t.Format(time.RFC3339Nano)
// FakeDockerPuller is meant to be a simple wrapper around FakeDockerClient.
// Please do not add more functionalities to it.
type FakeDockerPuller struct {
client DockerInterface
}
func (f *FakeDockerPuller) Pull(image string, _ []v1.Secret) error {
return f.client.PullImage(image, dockertypes.AuthConfig{}, dockertypes.ImagePullOptions{})
}
func (f *FakeDockerPuller) GetImageRef(image string) (string, error) {
_, err := f.client.InspectImageByRef(image)
if err != nil && IsImageNotFoundError(err) {
return "", nil
}
return image, err
}

View File

@ -52,7 +52,7 @@ func NewFakeDockerManager(
dm := NewDockerManager(client, recorder, livenessManager, containerRefManager, fakePodGetter, machineInfo, podInfraContainerImage, qps,
burst, containerLogsDir, osInterface, networkPlugin, runtimeHelper, httpClient, &NativeExecHandler{},
fakeOOMAdjuster, fakeProcFs, false, imageBackOff, false, false, true, "/var/lib/kubelet/seccomp")
dm.dockerPuller = &FakeDockerPuller{}
dm.dockerPuller = &FakeDockerPuller{client: client}
// ttl of version cache is set to 0 so we always call version api directly in tests.
dm.versionCache = cache.NewObjectCache(

View File

@ -159,8 +159,6 @@ func GetHollowKubeletConfig(
c.SerializeImagePulls = true
c.SystemCgroups = ""
c.ProtectKernelDefaults = false
// TODO: This is a temporary workaround until we fix CRI+kubemark properly.
c.EnableCRI = false
// TODO(mtaufen): Note that PodInfraContainerImage was being set to the empty value before,
// but this may not have been intentional. (previous code (SimpleKubelet)