Merge pull request #5395 from vmarmol/fix

Garbage collect unidentified Kubernetes containers.

pkg/kubelet/kubelet.go

@@ -466,12 +466,48 @@ func (kl *Kubelet) GarbageCollectContainers() error {
 	if err != nil {
 		return err
 	}
+
+	type unidentifiedContainer struct {
+		// Docker ID.
+		id string
+
+		// Docker container name
+		name string
+	}
+
+	unidentifiedContainers := make([]unidentifiedContainer, 0)
 	uidToIDMap := map[string][]string{}
 	for _, container := range containers {
 		_, uid, name, _ := dockertools.ParseDockerName(container.Names[0])
+		if uid == "" && name == "" {
+			unidentifiedContainers = append(unidentifiedContainers, unidentifiedContainer{
+				id:   container.ID,
+				name: container.Names[0],
+			})
+			continue
+		}
 		uidName := string(uid) + "." + name
 		uidToIDMap[uidName] = append(uidToIDMap[uidName], container.ID)
 	}
+
+	// Remove all non-running unidentified containers.
+	for _, container := range unidentifiedContainers {
+		data, err := kl.dockerClient.InspectContainer(container.id)
+		if err != nil {
+			return err
+		}
+		if data.State.Running {
+			continue
+		}
+
+		glog.Infof("Removing unidentified dead container %q with ID %q", container.name, container.id)
+		err = kl.dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: container.id})
+		if err != nil {
+			return err
+		}
+	}
+
+	// Evict dead containers according to our policies.
 	for _, list := range uidToIDMap {
 		if len(list) <= kl.maxContainerCount {
 			continue
@@ -480,6 +516,7 @@ func (kl *Kubelet) GarbageCollectContainers() error {
 			return err
 		}
 	}
+
 	return nil
 }
 

pkg/kubelet/kubelet_test.go

@@ -1741,6 +1741,43 @@ func TestKubeletGarbageCollection(t *testing.T) {
 			},
 			expectedRemoved: []string{"1706", "1876"},
 		},
+		// Remove non-running unidentified Kubernetes containers.
+		{
+			containers: []docker.APIContainers{
+				{
+					// Unidentified Kubernetes container.
+					Names: []string{"/k8s_unidentified"},
+					ID:    "1876",
+				},
+				{
+					// Unidentified (non-running) Kubernetes container.
+					Names: []string{"/k8s_unidentified"},
+					ID:    "2309",
+				},
+				{
+					// Regular Kubernetes container.
+					Names: []string{"/k8s_POD_foo_new_.deadbeef_42"},
+					ID:    "3876",
+				},
+			},
+			containerDetails: map[string]*docker.Container{
+				"1876": {
+					State: docker.State{
+						Running: false,
+					},
+					ID:      "1876",
+					Created: time.Now(),
+				},
+				"2309": {
+					State: docker.State{
+						Running: true,
+					},
+					ID:      "2309",
+					Created: time.Now(),
+				},
+			},
+			expectedRemoved: []string{"1876"},
+		},
 	}
 	for _, test := range tests {
 		kubelet, fakeDocker, _, _ := newTestKubelet(t)