github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #6711 from yujuhong/container_status 

Kubelet: surface the container creation/start failure reason
pull/6/head
Dawn Chen 2015-04-10 17:19:32 -07:00
parent 23c9b1a04b bb417e82d7
commit fff6cebb5f
5 changed files with 231 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/kubelet/dockertools/docker_test.go
View File

@ -243,7 +243,7 @@ func TestPull(t *testing.T) {
func TestDockerKeyringLookupFails(t *testing.T) {
fakeKeyring := &credentialprovider.FakeKeyring{}
fakeClient := &FakeDockerClient{
Err: fmt.Errorf("test error"),
Errors: map[string]error{"pull": fmt.Errorf("test error")},
}
dp := dockerPuller{
@ -394,7 +394,7 @@ func TestIsImagePresent(t *testing.T) {
}
func TestGetRunningContainers(t *testing.T) {
fakeDocker := &FakeDockerClient{}
fakeDocker := &FakeDockerClient{Errors: make(map[string]error)}
fakeRecorder := &record.FakeRecorder{}
containerManager := NewDockerManager(fakeDocker, fakeRecorder, PodInfraContainerImage)
tests := []struct {
@ -478,14 +478,16 @@ func TestGetRunningContainers(t *testing.T) {
}
for _, test := range tests {
fakeDocker.ContainerMap = test.containers
fakeDocker.Err = test.err
if test.err != nil {
fakeDocker.Errors["inspect_container"] = test.err
}
if results, err := containerManager.GetRunningContainers(test.inputIDs); err == nil {
resultIDs := []string{}
for _, result := range results {
resultIDs = append(resultIDs, result.ID)
}
if !reflect.DeepEqual(resultIDs, test.expectedIDs) {
t.Errorf("expected: %v, saw: %v", test.expectedIDs, resultIDs)
t.Errorf("expected: %#v, saw: %#v", test.expectedIDs, resultIDs)
}
if err != nil {
t.Errorf("unexpected error: %v", err)

120
pkg/kubelet/dockertools/fake_docker_client.go
View File

@ -38,7 +38,7 @@ type FakeDockerClient struct {
ContainerMap map[string]*docker.Container
Image *docker.Image
Images []docker.APIImages
Err error
Errors map[string]error
called []string
Stopped []string
pulled []string
@ -118,17 +118,30 @@ func (f *FakeDockerClient) AssertUnorderedCalls(calls []string) (err error) {
return
}
func (f *FakeDockerClient) popError(op string) error {
if f.Errors == nil {
return nil
}
err, ok := f.Errors[op]
if ok {
delete(f.Errors, op)
return err
} else {
return nil
}
}
// ListContainers is a test-spy implementation of DockerInterface.ListContainers.
// It adds an entry "list" to the internal method call record.
func (f *FakeDockerClient) ListContainers(options docker.ListContainersOptions) ([]docker.APIContainers, error) {
f.Lock()
defer f.Unlock()
f.called = append(f.called, "list")
err := f.popError("list")
if options.All {
return append(f.ContainerList, f.ExitedContainerList...), f.Err
return append(f.ContainerList, f.ExitedContainerList...), err
}
return f.ContainerList, f.Err
return f.ContainerList, err
}
// InspectContainer is a test-spy implementation of DockerInterface.InspectContainer.
@ -137,12 +150,13 @@ func (f *FakeDockerClient) InspectContainer(id string) (*docker.Container, error
f.Lock()
defer f.Unlock()
f.called = append(f.called, "inspect_container")
err := f.popError("inspect_container")
if f.ContainerMap != nil {
if container, ok := f.ContainerMap[id]; ok {
return container, f.Err
return container, err
}
}
return f.Container, f.Err
return f.Container, err
}
// InspectImage is a test-spy implementation of DockerInterface.InspectImage.
@ -151,7 +165,8 @@ func (f *FakeDockerClient) InspectImage(name string) (*docker.Image, error) {
f.Lock()
defer f.Unlock()
f.called = append(f.called, "inspect_image")
return f.Image, f.Err
err := f.popError("inspect_image")
return f.Image, err
}
// CreateContainer is a test-spy implementation of DockerInterface.CreateContainer.
@ -160,12 +175,16 @@ func (f *FakeDockerClient) CreateContainer(c docker.CreateContainerOptions) (*do
f.Lock()
defer f.Unlock()
f.called = append(f.called, "create")
f.Created = append(f.Created, c.Name)
// This is not a very good fake. We'll just add this container's name to the list.
// Docker likes to add a '/', so copy that behavior.
name := "/" + c.Name
f.ContainerList = append(f.ContainerList, docker.APIContainers{ID: name, Names: []string{name}, Image: c.Config.Image})
return &docker.Container{ID: name}, nil
err := f.popError("create")
if err == nil {
f.Created = append(f.Created, c.Name)
// This is not a very good fake. We'll just add this container's name to the list.
// Docker likes to add a '/', so copy that behavior.
name := "/" + c.Name
f.ContainerList = append(f.ContainerList, docker.APIContainers{ID: name, Names: []string{name}, Image: c.Config.Image})
return &docker.Container{ID: name}, nil
}
return nil, err
}
// StartContainer is a test-spy implementation of DockerInterface.StartContainer.
@ -174,18 +193,22 @@ func (f *FakeDockerClient) StartContainer(id string, hostConfig *docker.HostConf
f.Lock()
defer f.Unlock()
f.called = append(f.called, "start")
f.Container = &docker.Container{
ID: id,
Name: id, // For testing purpose, we set name to id
Config: &docker.Config{Image: "testimage"},
HostConfig: hostConfig,
State: docker.State{
Running: true,
Pid: os.Getpid(),
},
NetworkSettings: &docker.NetworkSettings{IPAddress: "1.2.3.4"},
err := f.popError("start")
if err == nil {
f.Container = &docker.Container{
ID: id,
Name: id, // For testing purpose, we set name to id
Config: &docker.Config{Image: "testimage"},
HostConfig: hostConfig,
State: docker.State{
Running: true,
Pid: os.Getpid(),
},
NetworkSettings: &docker.NetworkSettings{IPAddress: "1.2.3.4"},
}
}
return f.Err
return err
}
// StopContainer is a test-spy implementation of DockerInterface.StopContainer.
@ -194,23 +217,29 @@ func (f *FakeDockerClient) StopContainer(id string, timeout uint) error {
f.Lock()
defer f.Unlock()
f.called = append(f.called, "stop")
f.Stopped = append(f.Stopped, id)
var newList []docker.APIContainers
for _, container := range f.ContainerList {
if container.ID != id {
newList = append(newList, container)
err := f.popError("stop")
if err == nil {
f.Stopped = append(f.Stopped, id)
var newList []docker.APIContainers
for _, container := range f.ContainerList {
if container.ID != id {
newList = append(newList, container)
}
}
f.ContainerList = newList
}
f.ContainerList = newList
return f.Err
return err
}
func (f *FakeDockerClient) RemoveContainer(opts docker.RemoveContainerOptions) error {
f.Lock()
defer f.Unlock()
f.called = append(f.called, "remove")
f.Removed = append(f.Removed, opts.ID)
return f.Err
err := f.popError("remove")
if err == nil {
f.Removed = append(f.Removed, opts.ID)
}
return err
}
// Logs is a test-spy implementation of DockerInterface.Logs.
@ -219,7 +248,7 @@ func (f *FakeDockerClient) Logs(opts docker.LogsOptions) error {
f.Lock()
defer f.Unlock()
f.called = append(f.called, "logs")
return f.Err
return f.popError("logs")
}
// PullImage is a test-spy implementation of DockerInterface.StopContainer.
@ -228,12 +257,15 @@ func (f *FakeDockerClient) PullImage(opts docker.PullImageOptions, auth docker.A
f.Lock()
defer f.Unlock()
f.called = append(f.called, "pull")
registry := opts.Registry
if len(registry) != 0 {
registry = registry + "/"
err := f.popError("pull")
if err == nil {
registry := opts.Registry
if len(registry) != 0 {
registry = registry + "/"
}
f.pulled = append(f.pulled, fmt.Sprintf("%s%s:%s", registry, opts.Repository, opts.Tag))
}
f.pulled = append(f.pulled, fmt.Sprintf("%s%s:%s", registry, opts.Repository, opts.Tag))
return f.Err
return err
}
func (f *FakeDockerClient) Version() (*docker.Env, error) {
@ -249,12 +281,16 @@ func (f *FakeDockerClient) StartExec(_ string, _ docker.StartExecOptions) error
}
func (f *FakeDockerClient) ListImages(opts docker.ListImagesOptions) ([]docker.APIImages, error) {
return f.Images, f.Err
err := f.popError("list_images")
return f.Images, err
}
func (f *FakeDockerClient) RemoveImage(image string) error {
f.RemovedImages.Insert(image)
return f.Err
err := f.popError("remove_image")
if err == nil {
f.RemovedImages.Insert(image)
}
return err
}
// FakeDockerPuller is a stub implementation of DockerPuller.

121
pkg/kubelet/dockertools/manager.go
View File

@ -25,14 +25,21 @@ import (
"path"
"strconv"
"strings"
"sync"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
"github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities"
"github.com/GoogleCloudPlatform/kubernetes/pkg/client/record"
kubecontainer "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/container"
"github.com/GoogleCloudPlatform/kubernetes/pkg/types"
"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
"github.com/fsouza/go-dockerclient"
"github.com/golang/glog"
"github.com/golang/groupcache/lru"
)
const (
maxReasonCacheEntries = 200
)
// Implements kubecontainer.ContainerRunner.
@ -44,13 +51,60 @@ type DockerManager struct {
// TODO(yifan): PodInfraContainerImage can be unexported once
// we move createPodInfraContainer into dockertools.
PodInfraContainerImage string
// reasonCache stores the failure reason of the last container creation
// and/or start in a string, keyed by <pod_UID>_<container_name>. The goal
// is to propagate this reason to the container status. This endeavor is
// "best-effort" for two reasons:
// 1. The cache is not persisted.
// 2. We use an LRU cache to avoid extra garbage collection work. This
// means that some entries may be recycled before a pod has been
// deleted.
reasonCache stringCache
}
// Ensures DockerManager implements ConatinerRunner.
var _ kubecontainer.ContainerRunner = new(DockerManager)
func NewDockerManager(client DockerInterface, recorder record.EventRecorder, podInfraContainerImage string) *DockerManager {
return &DockerManager{client: client, recorder: recorder, PodInfraContainerImage: podInfraContainerImage}
reasonCache := stringCache{cache: lru.New(maxReasonCacheEntries)}
return &DockerManager{
client: client,
recorder: recorder,
PodInfraContainerImage: podInfraContainerImage,
reasonCache: reasonCache}
}
// A cache which stores strings keyed by <pod_UID>_<container_name>.
type stringCache struct {
lock sync.RWMutex
cache *lru.Cache
}
func (self *stringCache) composeKey(uid types.UID, name string) string {
return fmt.Sprintf("%s_%s", uid, name)
}
func (self *stringCache) Add(uid types.UID, name string, value string) {
self.lock.Lock()
defer self.lock.Unlock()
self.cache.Add(self.composeKey(uid, name), value)
}
func (self *stringCache) Remove(uid types.UID, name string) {
self.lock.Lock()
defer self.lock.Unlock()
self.cache.Remove(self.composeKey(uid, name))
}
func (self *stringCache) Get(uid types.UID, name string) (string, bool) {
self.lock.RLock()
defer self.lock.RUnlock()
value, ok := self.cache.Get(self.composeKey(uid, name))
if ok {
return value.(string), ok
} else {
return "", ok
}
}
// GetKubeletDockerContainerLogs returns logs of a specific container. By
@ -118,7 +172,6 @@ func (self *DockerManager) inspectContainer(dockerID, containerName, tPath strin
ContainerID: DockerPrefix + dockerID,
}
waiting := true
if inspectResult.State.Running {
result.status.State.Running = &api.ContainerStateRunning{
StartedAt: util.NewTime(inspectResult.State.StartedAt),
@ -126,7 +179,6 @@ func (self *DockerManager) inspectContainer(dockerID, containerName, tPath strin
if containerName == PodInfraContainerName && inspectResult.NetworkSettings != nil {
result.ip = inspectResult.NetworkSettings.IPAddress
}
waiting = false
} else if !inspectResult.State.FinishedAt.IsZero() {
reason := ""
// Note: An application might handle OOMKilled gracefully.
@ -155,13 +207,8 @@ func (self *DockerManager) inspectContainer(dockerID, containerName, tPath strin
}
}
}
waiting = false
}
if waiting {
} else {
// TODO(dchen1107): Separate issue docker/docker#8294 was filed
// TODO(dchen1107): Need to figure out why we are still waiting
// Check any issue to run container
result.status.State.Waiting = &api.ContainerStateWaiting{
Reason: ErrContainerCannotRun.Error(),
}
@ -233,31 +280,27 @@ func (self *DockerManager) GetPodStatus(pod *api.Pod) (*api.PodStatus, error) {
return nil, result.err
}
// Add user container information
if dockerContainerName == PodInfraContainerName &&
result.status.State.Running != nil {
if dockerContainerName == PodInfraContainerName {
// Found network container
podStatus.PodIP = result.ip
if result.status.State.Running != nil {
podStatus.PodIP = result.ip
}
} else {
// Add user container information.
statuses[dockerContainerName] = result.status
}
}
if len(statuses) == 0 && podStatus.PodIP == "" {
return nil, ErrNoContainersInPod
}
// Not all containers expected are created, check if there are
// image related issues
if len(statuses) < len(manifest.Containers) {
for _, container := range manifest.Containers {
var containerStatus api.ContainerStatus
for _, container := range manifest.Containers {
if _, found := statuses[container.Name]; found {
continue
}
if status, found := statuses[container.Name]; found {
containerStatus = status
} else {
// The container has not been created yet. Check image is ready on
// the node or not.
// TODO: If we integrate DockerPuller into DockerManager, we can
// record the pull failure and eliminate the image checking below.
image := container.Image
// Check image is ready on the node or not
// TODO(dchen1107): docker/docker/issues/8365 to figure out if the image exists
_, err := self.client.InspectImage(image)
if err == nil {
@ -268,14 +311,15 @@ func (self *DockerManager) GetPodStatus(pod *api.Pod) (*api.PodStatus, error) {
containerStatus.State.Waiting = &api.ContainerStateWaiting{
Reason: fmt.Sprintf("Image: %s is not ready on the node", image),
}
} else {
containerStatus.State.Waiting = &api.ContainerStateWaiting{
Reason: "",
}
}
statuses[container.Name] = containerStatus
}
if containerStatus.State.Waiting != nil {
// For containers in the waiting state, fill in a specific reason if it is recorded.
if reason, ok := self.reasonCache.Get(uid, container.Name); ok {
containerStatus.State.Waiting.Reason = reason
}
}
statuses[container.Name] = containerStatus
}
podStatus.ContainerStatuses = make([]api.ContainerStatus, 0)
@ -304,6 +348,19 @@ func (self *DockerManager) GetRunningContainers(ids []string) ([]*docker.Contain
}
func (self *DockerManager) RunContainer(pod *api.Pod, container *api.Container, opts *kubecontainer.RunContainerOptions) (string, error) {
dockerID, err := self.runContainer(pod, container, opts)
if err != nil {
errString := err.Error()
if errString != "" {
self.reasonCache.Add(pod.UID, container.Name, errString)
} else {
self.reasonCache.Remove(pod.UID, container.Name)
}
}
return dockerID, err
}
func (self *DockerManager) runContainer(pod *api.Pod, container *api.Container, opts *kubecontainer.RunContainerOptions) (string, error) {
ref, err := kubecontainer.GenerateContainerRef(pod, container)
if err != nil {
glog.Errorf("Couldn't make a ref to pod %v, container %v: '%v'", pod.Name, container.Name, err)

10
pkg/kubelet/kubelet.go
View File

@ -1698,18 +1698,16 @@ func (kl *Kubelet) validateContainerStatus(podStatus *api.PodStatus, containerNa
func (kl *Kubelet) GetKubeletContainerLogs(podFullName, containerName, tail string, follow bool, stdout, stderr io.Writer) error {
podStatus, err := kl.GetPodStatus(podFullName)
if err != nil {
if err == dockertools.ErrNoContainersInPod {
return fmt.Errorf("pod %q not found\n", podFullName)
} else {
return fmt.Errorf("failed to get status for pod %q - %v", podFullName, err)
}
return fmt.Errorf("failed to get status for pod %q - %v", podFullName, err)
}
if err := kl.validatePodPhase(&podStatus); err != nil {
// No log is available if pod is not in a "known" phase (e.g. Unknown).
return err
}
dockerContainerID, err := kl.validateContainerStatus(&podStatus, containerName)
if err != nil {
// No log is available if the container status is missing or is in the
// waiting state.
return err
}
return kl.containerManager.GetKubeletDockerContainerLogs(dockerContainerID, tail, follow, stdout, stderr)

66
pkg/kubelet/kubelet_test.go
View File

@ -69,7 +69,7 @@ type TestKubelet struct {
}
func newTestKubelet(t *testing.T) *TestKubelet {
fakeDocker := &dockertools.FakeDockerClient{RemovedImages: util.StringSet{}}
fakeDocker := &dockertools.FakeDockerClient{Errors: make(map[string]error), RemovedImages: util.StringSet{}}
fakeDockerCache := dockertools.NewFakeDockerCache(fakeDocker)
fakeRecorder := &record.FakeRecorder{}
fakeKubeClient := &testclient.Fake{}
@ -366,7 +366,7 @@ func TestKillContainerWithError(t *testing.T) {
},
}
fakeDocker := &dockertools.FakeDockerClient{
Err: fmt.Errorf("sample error"),
Errors: make(map[string]error),
ContainerList: append([]docker.APIContainers{}, containers...),
}
testKubelet := newTestKubelet(t)
@ -376,6 +376,7 @@ func TestKillContainerWithError(t *testing.T) {
}
kubelet.dockerClient = fakeDocker
c := apiContainerToContainer(fakeDocker.ContainerList[0])
fakeDocker.Errors["stop"] = fmt.Errorf("sample error")
err := kubelet.killContainer(&c)
if err == nil {
t.Errorf("expected error, found nil")
@ -512,7 +513,7 @@ func TestSyncPodsWithTerminationLog(t *testing.T) {
}
waitGroup.Wait()
verifyCalls(t, fakeDocker, []string{
"list", "list", "list", "create", "start", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"})
"list", "list", "list", "inspect_image", "create", "start", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"})
fakeDocker.Lock()
parts := strings.Split(fakeDocker.Container.HostConfig.Binds[0], ":")
@ -564,7 +565,7 @@ func TestSyncPodsCreatesNetAndContainer(t *testing.T) {
waitGroup.Wait()
verifyCalls(t, fakeDocker, []string{
"list", "list", "list", "create", "start", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"})
"list", "list", "list", "inspect_image", "create", "start", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"})
fakeDocker.Lock()
@ -619,7 +620,7 @@ func TestSyncPodsCreatesNetAndContainerPullsImage(t *testing.T) {
waitGroup.Wait()
verifyCalls(t, fakeDocker, []string{
"list", "list", "list", "create", "start", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"})
"list", "list", "list", "inspect_image", "create", "start", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"})
fakeDocker.Lock()
@ -671,7 +672,7 @@ func TestSyncPodsWithPodInfraCreatesContainer(t *testing.T) {
waitGroup.Wait()
verifyCalls(t, fakeDocker, []string{
"list", "list", "list", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"})
"list", "list", "list", "inspect_container", "inspect_image", "create", "start", "list", "inspect_container", "inspect_container"})
fakeDocker.Lock()
if len(fakeDocker.Created) != 1 ||
@ -730,7 +731,7 @@ func TestSyncPodsWithPodInfraCreatesContainerCallsHandler(t *testing.T) {
waitGroup.Wait()
verifyCalls(t, fakeDocker, []string{
"list", "list", "list", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"})
"list", "list", "list", "inspect_container", "inspect_image", "create", "start", "list", "inspect_container", "inspect_container"})
fakeDocker.Lock()
if len(fakeDocker.Created) != 1 ||
@ -948,7 +949,7 @@ func TestSyncPodDeletesDuplicate(t *testing.T) {
t.Errorf("unexpected error: %v", err)
}
verifyCalls(t, fakeDocker, []string{"list", "stop", "list"})
verifyCalls(t, fakeDocker, []string{"list", "inspect_image", "stop", "list", "inspect_image"})
// Expect one of the duplicates to be killed.
if len(fakeDocker.Stopped) != 1 || (fakeDocker.Stopped[0] != "1234" && fakeDocker.Stopped[0] != "4567") {
t.Errorf("Wrong containers were stopped: %v", fakeDocker.Stopped)
@ -990,8 +991,8 @@ func TestSyncPodBadHash(t *testing.T) {
t.Errorf("unexpected error: %v", err)
}
//verifyCalls(t, fakeDocker, []string{"list", "stop", "list", "create", "start", "stop", "create", "start", "inspect_container"})
verifyCalls(t, fakeDocker, []string{"list", "stop", "stop", "create", "start", "inspect_container", "create", "start", "list", "inspect_container", "inspect_container"})
verifyCalls(t, fakeDocker, []string{"list", "inspect_image", "stop", "stop", "create", "start",
"inspect_container", "create", "start", "list", "inspect_container", "inspect_container"})
// A map interation is used to delete containers, so must not depend on
// order here.
@ -1045,7 +1046,7 @@ func TestSyncPodUnhealthy(t *testing.T) {
t.Errorf("unexpected error: %v", err)
}
verifyCalls(t, fakeDocker, []string{"list", "stop", "create", "start", "list", "inspect_container"})
verifyCalls(t, fakeDocker, []string{"list", "inspect_image", "stop", "create", "start", "list", "inspect_container"})
// A map interation is used to delete containers, so must not depend on
// order here.
@ -1638,7 +1639,7 @@ func TestSyncPodEventHandlerFails(t *testing.T) {
t.Errorf("unexpected error: %v", err)
}
verifyCalls(t, fakeDocker, []string{"list", "create", "start", "stop", "list"})
verifyCalls(t, fakeDocker, []string{"list", "inspect_image", "create", "start", "stop", "list", "inspect_image"})
if len(fakeDocker.Stopped) != 1 {
t.Errorf("Wrong containers were stopped: %v", fakeDocker.Stopped)
@ -3792,3 +3793,44 @@ func TestGetPodStatusWithLastTermination(t *testing.T) {
}
}
}
func TestGetPodCreationFailureReason(t *testing.T) {
testKubelet := newTestKubelet(t)
kubelet := testKubelet.kubelet
fakeDocker := testKubelet.fakeDocker
failureReason := "creation failure"
fakeDocker.Errors["create"] = fmt.Errorf("%s", failureReason)
fakeDocker.ContainerList = []docker.APIContainers{}
pod := api.Pod{
ObjectMeta: api.ObjectMeta{
UID: "12345678",
Name: "bar",
Namespace: "new",
},
Spec: api.PodSpec{
Containers: []api.Container{
{Name: "foo"},
},
},
}
pods := []api.Pod{pod}
kubelet.podManager.SetPods(pods)
_, err := kubelet.runContainer(&pod, &pod.Spec.Containers[0], make(map[string]volume.Volume), "", "")
if err == nil {
t.Errorf("expected error, found nil")
}
status, err := kubelet.GetPodStatus(kubecontainer.GetPodFullName(&pod))
if err != nil {
t.Errorf("unexpected error %v", err)
}
if len(status.ContainerStatuses) < 1 {
t.Errorf("expected 1 container status, got %d", len(status.ContainerStatuses))
} else {
state := status.ContainerStatuses[0].State
if state.Waiting == nil {
t.Errorf("expected waiting state, got %#v", state)
} else if state.Waiting.Reason != failureReason {
t.Errorf("expected reason %q, got %q", state.Waiting.Reason)
}
}
}