Have the ContainerManager create a system container.

The system container is a resource-only container which contains all
non-kernel processes that are not already part of a container. This will
allow monitoring of their resource usage and limiting it (eventually).

pkg/kubelet/container_manager.go

@@ -20,5 +20,6 @@ package kubelet
 type containerManager interface {
 	// Runs the container manager's housekeeping.
 	// - Ensures that the Docker daemon is in a container.
+	// - Creates the system container where all non-containerized processes run.
 	Start() error
 }

pkg/kubelet/container_manager_linux.go

@@ -35,33 +35,60 @@ import (
 )
 
 type containerManagerImpl struct {
-	// Absolute name of the desired container that Docker should be in.
-	dockerContainerName string
+	// Whether to create and use the specified containers.
+	useDockerContainer bool
+	useSystemContainer bool
 
-	// The manager of the resource-only container Docker should be in.
-	manager           fs.Manager
+	// OOM score for the Docker container.
 	dockerOomScoreAdj int
+
+	// Managers for containers.
+	dockerContainer fs.Manager
+	systemContainer fs.Manager
+	rootContainer   fs.Manager
 }
 
 var _ containerManager = &containerManagerImpl{}
 
-// Takes the absolute name that the Docker daemon should be in.
-// Empty container name disables moving the Docker daemon.
-func newContainerManager(dockerDaemonContainer string) (containerManager, error) {
+// Takes the absolute name of the specified containers.
+// Empty container name disables use of the specified container.
+func newContainerManager(dockerDaemonContainer, systemContainer string) (containerManager, error) {
+	if systemContainer == "/" {
+		return nil, fmt.Errorf("system container cannot be root (\"/\")")
+	}
+
 	return &containerManagerImpl{
-		dockerContainerName: dockerDaemonContainer,
-		manager: fs.Manager{
+		useDockerContainer: dockerDaemonContainer != "",
+		useSystemContainer: systemContainer != "",
+		dockerOomScoreAdj:  -900,
+		dockerContainer: fs.Manager{
 			Cgroups: &configs.Cgroup{
 				Name:            dockerDaemonContainer,
 				AllowAllDevices: true,
 			},
 		},
-		dockerOomScoreAdj: -900,
+		systemContainer: fs.Manager{
+			Cgroups: &configs.Cgroup{
+				Name:            systemContainer,
+				AllowAllDevices: true,
+			},
+		},
+		rootContainer: fs.Manager{
+			Cgroups: &configs.Cgroup{
+				Name: "/",
+			},
+		},
 	}, nil
 }
 
 func (cm *containerManagerImpl) Start() error {
-	if cm.dockerContainerName != "" {
+	if cm.useSystemContainer {
+		err := cm.ensureSystemContainer()
+		if err != nil {
+			return err
+		}
+	}
+	if cm.useDockerContainer {
 		go util.Until(func() {
 			err := cm.ensureDockerInContainer()
 			if err != nil {
@@ -99,10 +126,10 @@ func (cm *containerManagerImpl) ensureDockerInContainer() error {
 			errs = append(errs, fmt.Errorf("failed to find container of PID %q: %v", pid, err))
 		}
 
-		if cont != cm.dockerContainerName {
-			err = cm.manager.Apply(pid)
+		if cont != cm.dockerContainer.Cgroups.Name {
+			err = cm.dockerContainer.Apply(pid)
 			if err != nil {
-				errs = append(errs, fmt.Errorf("failed to move PID %q (in %q) to %q", pid, cont, cm.dockerContainerName))
+				errs = append(errs, fmt.Errorf("failed to move PID %q (in %q) to %q", pid, cont, cm.dockerContainer.Cgroups.Name))
 			}
 		}
 
@@ -125,3 +152,60 @@ func getContainer(pid int) (string, error) {
 
 	return cgroups.ParseCgroupFile("cpu", f)
 }
+
+// Ensures the system container is created and all non-kernel processes without
+// a container are moved to it.
+func (cm *containerManagerImpl) ensureSystemContainer() error {
+	// Move non-kernel PIDs to the system container.
+	attemptsRemaining := 10
+	var errs []error
+	for attemptsRemaining >= 0 {
+		// Only keep errors on latest attempt.
+		errs = []error{}
+		attemptsRemaining--
+
+		allPids, err := cm.rootContainer.GetPids()
+		if err != nil {
+			errs = append(errs, fmt.Errorf("Failed to list PIDs for root: %v", err))
+			continue
+		}
+
+		// Remove kernel pids
+		pids := make([]int, 0, len(allPids))
+		for _, pid := range allPids {
+			if isKernelPid(pid) {
+				continue
+			}
+
+			pids = append(pids, pid)
+		}
+		glog.Infof("Found %d PIDs in root, %d of them are kernel related", len(allPids), len(allPids)-len(pids))
+
+		// Check if we moved all the non-kernel PIDs.
+		if len(pids) == 0 {
+			break
+		}
+
+		glog.Infof("Moving non-kernel threads: %v", pids)
+		for _, pid := range pids {
+			err := cm.systemContainer.Apply(pid)
+			if err != nil {
+				errs = append(errs, fmt.Errorf("failed to move PID %d into the system container %q: %v", pid, cm.systemContainer.Cgroups.Name, err))
+				continue
+			}
+		}
+
+	}
+	if attemptsRemaining < 0 {
+		errs = append(errs, fmt.Errorf("ran out of attempts to create system containers %q", cm.systemContainer.Cgroups.Name))
+	}
+
+	return errors.NewAggregate(errs)
+}
+
+// Determines whether the specified PID is a kernel PID.
+func isKernelPid(pid int) bool {
+	// Kernel threads have no associated executable.
+	_, err := os.Readlink(fmt.Sprintf("/proc/%d/exe", pid))
+	return err != nil
+}

pkg/kubelet/container_manager_unsupported.go

@@ -31,6 +31,6 @@ func (unsupportedContainerManager) Start() error {
 	return fmt.Errorf("Container Manager is unsupported in this build")
 }
 
-func newContainerManager(dockerDaemonContainer string) (containerManager, error) {
+func newContainerManager(dockerDaemonContainer, systemContainer string) (containerManager, error) {
 	return &unsupportedContainerManager{}, nil
 }

pkg/kubelet/kubelet.go

@@ -146,6 +146,9 @@ func NewMainKubelet(
 	if resyncInterval <= 0 {
 		return nil, fmt.Errorf("invalid sync frequency %d", resyncInterval)
 	}
+	if systemContainer != "" && cgroupRoot == "" {
+		return nil, fmt.Errorf("invalid configuration: system container was specified and cgroup root was not specified")
+	}
 	dockerClient = dockertools.NewInstrumentedDockerInterface(dockerClient)
 
 	serviceStore := cache.NewStore(cache.MetaNamespaceKeyFunc)
@@ -294,7 +297,10 @@ func NewMainKubelet(
 		return nil, fmt.Errorf("unsupported container runtime %q specified", containerRuntime)
 	}
 
-	containerManager, err := newContainerManager(dockerDaemonContainer)
+	// TODO(vmarmol): Make configurable.
+	// Setup container manager, can fail if the devices hierarchy is not mounted
+	// (it is required by Docker however).
+	containerManager, err := newContainerManager(dockerDaemonContainer, "/system")
 	if err != nil {
 		return nil, fmt.Errorf("failed to create the Container Manager: %v", err)
 	}