Merge pull request #25324 from jfrazelle/add-seccomp

Add Seccomp to Annotations

cmd/kubelet/app/options/options.go

@@ -19,6 +19,7 @@ package options
 
 import (
 	_ "net/http/pprof"
+	"path/filepath"
 	"runtime"
 	"time"
 
@@ -132,6 +133,7 @@ func NewKubeletServer() *KubeletServer {
 			RootDirectory:                    defaultRootDir,
 			RuntimeCgroups:                   "",
 			SerializeImagePulls:              true,
+			SeccompProfileRoot:               filepath.Join(defaultRootDir, "seccomp"),
 			StreamingConnectionIdleTimeout:   unversioned.Duration{Duration: 4 * time.Hour},
 			SyncFrequency:                    unversioned.Duration{Duration: 1 * time.Minute},
 			SystemCgroups:                    "",
@@ -171,6 +173,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.PodInfraContainerImage, "pod-infra-container-image", s.PodInfraContainerImage, "The image whose network/ipc namespaces containers in each pod will use.")
 	fs.StringVar(&s.DockerEndpoint, "docker-endpoint", s.DockerEndpoint, "If non-empty, use this for the docker endpoint to communicate with")
 	fs.StringVar(&s.RootDirectory, "root-dir", s.RootDirectory, "Directory path for managing kubelet files (volume mounts,etc).")
+	fs.StringVar(&s.SeccompProfileRoot, "seccomp-profile-root", s.SeccompProfileRoot, "Directory path for seccomp profiles.")
 	fs.BoolVar(&s.AllowPrivileged, "allow-privileged", s.AllowPrivileged, "If true, allow containers to request privileged mode. [default=false]")
 	fs.StringVar(&s.HostNetworkSources, "host-network-sources", s.HostNetworkSources, "Comma-separated list of sources from which the Kubelet allows pods to use of host network. [default=\"*\"]")
 	fs.StringVar(&s.HostPIDSources, "host-pid-sources", s.HostPIDSources, "Comma-separated list of sources from which the Kubelet allows pods to use the host pid namespace. [default=\"*\"]")

cmd/kubelet/app/server.go

@@ -261,6 +261,7 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) {
 		RktAPIEndpoint:                 s.RktAPIEndpoint,
 		RktStage1Image:                 s.RktStage1Image,
 		RootDirectory:                  s.RootDirectory,
+		SeccompProfileRoot:             s.SeccompProfileRoot,
 		Runonce:                        s.RunOnce,
 		SerializeImagePulls:            s.SerializeImagePulls,
 		StandaloneMode:                 (len(s.APIServerList) == 0),
@@ -831,6 +832,7 @@ type KubeletConfig struct {
 	RktStage1Image                 string
 	RootDirectory                  string
 	Runonce                        bool
+	SeccompProfileRoot             string
 	SerializeImagePulls            bool
 	StandaloneMode                 bool
 	StreamingConnectionIdleTimeout time.Duration
@@ -882,6 +884,7 @@ func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
 		kc.DockerClient,
 		kubeClient,
 		kc.RootDirectory,
+		kc.SeccompProfileRoot,
 		kc.PodInfraContainerImage,
 		kc.SyncFrequency,
 		float32(kc.RegistryPullQPS),

docs/admin/kubelet.md

@@ -149,6 +149,7 @@ kubelet
       --root-dir="/var/lib/kubelet": Directory path for managing kubelet files (volume mounts,etc).
       --runonce[=false]: If true, exit after spawning pods from local manifests or remote urls. Exclusive with --api-servers, and --enable-server
       --runtime-cgroups="": Optional absolute name of cgroups to create and run the runtime in.
+      --seccomp-profile-root="/var/lib/kubelet/seccomp": Directory path for seccomp profiles.
       --serialize-image-pulls[=true]: Pull images one at a time. We recommend *not* changing the default value on nodes that run docker daemon with version < 1.9 or an Aufs storage backend. Issue #10959 has more details. [default=true]
       --streaming-connection-idle-timeout=4h0m0s: Maximum time a streaming connection can be idle before the connection is automatically closed. 0 indicates no timeout. Example: '5m'
       --sync-frequency=1m0s: Max period between synchronizing running containers and config
@@ -160,7 +161,7 @@ kubelet
       --volume-stats-agg-period=1m0s: Specifies interval for kubelet to calculate and cache the volume disk usage for all pods and volumes.  To disable volume calculations, set to 0.  Default: '1m'
 ```
 
-###### Auto generated by spf13/cobra on 21-May-2016
+###### Auto generated by spf13/cobra on 24-May-2016
 
 
 <!-- BEGIN MUNGE: GENERATED_ANALYTICS -->

hack/verify-flags/known-flags.txt

@@ -385,6 +385,7 @@ save-config
 scheduler-config
 scheduler-name
 schema-cache-dir
+seccomp-profile-root
 secure-port
 serialize-image-pulls
 server-start-timeout

pkg/apis/componentconfig/deep_copy_generated.go

@@ -223,6 +223,7 @@ func DeepCopy_componentconfig_KubeletConfiguration(in KubeletConfiguration, out
 	out.PodInfraContainerImage = in.PodInfraContainerImage
 	out.DockerEndpoint = in.DockerEndpoint
 	out.RootDirectory = in.RootDirectory
+	out.SeccompProfileRoot = in.SeccompProfileRoot
 	out.AllowPrivileged = in.AllowPrivileged
 	out.HostNetworkSources = in.HostNetworkSources
 	out.HostPIDSources = in.HostPIDSources

pkg/apis/componentconfig/types.go

@@ -151,6 +151,8 @@ type KubeletConfiguration struct {
 	// rootDirectory is the directory path to place kubelet files (volume
 	// mounts,etc).
 	RootDirectory string `json:"rootDirectory"`
+	// seccompProfileRoot is the directory path for seccomp profiles.
+	SeccompProfileRoot string `json:"seccompProfileRoot"`
 	// allowPrivileged enables containers to request privileged mode.
 	// Defaults to false.
 	AllowPrivileged bool `json:"allowPrivileged"`

pkg/kubelet/dockertools/fake_manager.go

@@ -51,7 +51,7 @@ func NewFakeDockerManager(
 	fakePodGetter := &fakePodGetter{}
 	dm := NewDockerManager(client, recorder, livenessManager, containerRefManager, fakePodGetter, machineInfo, podInfraContainerImage, qps,
 		burst, containerLogsDir, osInterface, networkPlugin, runtimeHelper, httpClient, &NativeExecHandler{},
-		fakeOOMAdjuster, fakeProcFs, false, imageBackOff, false, false, true)
+		fakeOOMAdjuster, fakeProcFs, false, imageBackOff, false, false, true, "/var/lib/kubelet/seccomp")
 	dm.dockerPuller = &FakeDockerPuller{}
 
 	// ttl of version cache is set to 0 so we always call version api directly in tests.

pkg/kubelet/dockertools/manager.go

@@ -18,6 +18,7 @@ package dockertools
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -25,6 +26,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
@@ -102,8 +104,7 @@ var (
 	// TODO: make this a TTL based pull (if image older than X policy, pull)
 	podInfraContainerImagePullPolicy = api.PullIfNotPresent
 
-	// Default set of security options. Seccomp is disabled by default until
-	// github issue #20870 is resolved.
+	// Default set of security options.
 	defaultSecurityOpt = []string{"seccomp:unconfined"}
 )
 
@@ -172,6 +173,9 @@ type DockerManager struct {
 
 	// The version cache of docker daemon.
 	versionCache *cache.ObjectCache
+
+	// Directory to host local seccomp profiles.
+	seccompProfileRoot string
 }
 
 // A subset of the pod.Manager interface extracted for testing purposes.
@@ -214,6 +218,7 @@ func NewDockerManager(
 	serializeImagePulls bool,
 	enableCustomMetrics bool,
 	hairpinMode bool,
+	seccompProfileRoot string,
 	options ...kubecontainer.Option) *DockerManager {
 	// Wrap the docker client with instrumentedDockerInterface
 	client = newInstrumentedDockerInterface(client)
@@ -250,6 +255,7 @@ func NewDockerManager(
 		enableCustomMetrics:    enableCustomMetrics,
 		configureHairpinMode:   hairpinMode,
 		imageStatsProvider:     &imageStatsProvider{client},
+		seccompProfileRoot:     seccompProfileRoot,
 	}
 	dm.runner = lifecycle.NewHandlerRunner(httpClient, dm, dm)
 	if serializeImagePulls {
@@ -552,7 +558,7 @@ func (dm *DockerManager) runContainer(
 		ContainerName: container.Name,
 	}
 
-	securityOpts, err := dm.getDefaultSecurityOpt()
+	securityOpts, err := dm.getSecurityOpt(pod, container.Name)
 	if err != nil {
 		return kubecontainer.ContainerID{}, err
 	}
@@ -971,22 +977,59 @@ func (dm *DockerManager) checkVersionCompatibility() error {
 	return nil
 }
 
-func (dm *DockerManager) getDefaultSecurityOpt() ([]string, error) {
+func (dm *DockerManager) getSecurityOpt(pod *api.Pod, ctrName string) ([]string, error) {
 	version, err := dm.APIVersion()
 	if err != nil {
 		return nil, err
 	}
-	// seccomp is to be disabled on docker versions >= v1.10
+
+	// seccomp is only on docker versions >= v1.10
 	result, err := version.Compare(dockerV110APIVersion)
 	if err != nil {
 		return nil, err
 	}
-	if result >= 0 {
+	if result < 0 {
+		// return early for old versions
+		return nil, nil
+	}
+
+	profile, profileOK := pod.ObjectMeta.Annotations["security.alpha.kubernetes.io/seccomp/container/"+ctrName]
+	if !profileOK {
+		// try the pod profile
+		profile, profileOK = pod.ObjectMeta.Annotations["security.alpha.kubernetes.io/seccomp/pod"]
+		if !profileOK {
+			// return early the default
 			return defaultSecurityOpt, nil
 		}
+	}
+
+	if profile == "unconfined" {
+		// return early the default
+		return defaultSecurityOpt, nil
+	}
+
+	if profile == "docker/default" {
+		// return nil so docker will load the default seccomp profile
 		return nil, nil
 	}
 
+	if !strings.HasPrefix(profile, "localhost") {
+		return nil, fmt.Errorf("unknown seccomp profile option: %s", profile)
+	}
+
+	file, err := ioutil.ReadFile(filepath.Join(dm.seccompProfileRoot, strings.TrimPrefix(profile, "localhost/")))
+	if err != nil {
+		return nil, err
+	}
+
+	b := bytes.NewBuffer(nil)
+	if err := json.Compact(b, file); err != nil {
+		return nil, err
+	}
+
+	return []string{fmt.Sprintf("seccomp=%s", b.Bytes())}, nil
+}
+
 type dockerExitError struct {
 	Inspect *dockertypes.ContainerExecInspect
 }

pkg/kubelet/dockertools/manager_test.go

@@ -1714,7 +1714,7 @@ func verifySyncResults(t *testing.T, expectedResults []*kubecontainer.SyncResult
 	}
 }
 
-func TestSeccompIsDisabledWithDockerV110(t *testing.T) {
+func TestSeccompIsUnconfinedByDefaultWithDockerV110(t *testing.T) {
 	dm, fakeDocker := newTestDockerManagerWithVersion("1.10.1", "1.22")
 	pod := &api.Pod{
 		ObjectMeta: api.ObjectMeta{
@@ -1750,7 +1750,134 @@ func TestSeccompIsDisabledWithDockerV110(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected error %v", err)
 	}
-	assert.Contains(t, newContainer.HostConfig.SecurityOpt, "seccomp:unconfined", "Pods with Docker versions >= 1.10 must have seccomp disabled.")
+	assert.Contains(t, newContainer.HostConfig.SecurityOpt, "seccomp:unconfined", "Pods with Docker versions >= 1.10 must not have seccomp disabled by default")
+}
+
+func TestUnconfinedSeccompProfileWithDockerV110(t *testing.T) {
+	dm, fakeDocker := newTestDockerManagerWithVersion("1.10.1", "1.22")
+	pod := &api.Pod{
+		ObjectMeta: api.ObjectMeta{
+			UID:       "12345678",
+			Name:      "foo4",
+			Namespace: "new",
+			Annotations: map[string]string{
+				"security.alpha.kubernetes.io/seccomp/pod": "unconfined",
+			},
+		},
+		Spec: api.PodSpec{
+			Containers: []api.Container{
+				{Name: "bar4"},
+			},
+		},
+	}
+
+	runSyncPod(t, dm, fakeDocker, pod, nil, false)
+
+	verifyCalls(t, fakeDocker, []string{
+		// Create pod infra container.
+		"create", "start", "inspect_container", "inspect_container",
+		// Create container.
+		"create", "start", "inspect_container",
+	})
+
+	fakeDocker.Lock()
+	if len(fakeDocker.Created) != 2 ||
+		!matchString(t, "/k8s_POD\\.[a-f0-9]+_foo4_new_", fakeDocker.Created[0]) ||
+		!matchString(t, "/k8s_bar4\\.[a-f0-9]+_foo4_new_", fakeDocker.Created[1]) {
+		t.Errorf("unexpected containers created %v", fakeDocker.Created)
+	}
+	fakeDocker.Unlock()
+
+	newContainer, err := fakeDocker.InspectContainer(fakeDocker.Created[1])
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	assert.Contains(t, newContainer.HostConfig.SecurityOpt, "seccomp:unconfined", "Pods created with a secccomp annotation of unconfined should have seccomp:unconfined.")
+}
+
+func TestDefaultSeccompProfileWithDockerV110(t *testing.T) {
+	dm, fakeDocker := newTestDockerManagerWithVersion("1.10.1", "1.22")
+	pod := &api.Pod{
+		ObjectMeta: api.ObjectMeta{
+			UID:       "12345678",
+			Name:      "foo1",
+			Namespace: "new",
+			Annotations: map[string]string{
+				"security.alpha.kubernetes.io/seccomp/pod": "docker/default",
+			},
+		},
+		Spec: api.PodSpec{
+			Containers: []api.Container{
+				{Name: "bar1"},
+			},
+		},
+	}
+
+	runSyncPod(t, dm, fakeDocker, pod, nil, false)
+
+	verifyCalls(t, fakeDocker, []string{
+		// Create pod infra container.
+		"create", "start", "inspect_container", "inspect_container",
+		// Create container.
+		"create", "start", "inspect_container",
+	})
+
+	fakeDocker.Lock()
+	if len(fakeDocker.Created) != 2 ||
+		!matchString(t, "/k8s_POD\\.[a-f0-9]+_foo1_new_", fakeDocker.Created[0]) ||
+		!matchString(t, "/k8s_bar1\\.[a-f0-9]+_foo1_new_", fakeDocker.Created[1]) {
+		t.Errorf("unexpected containers created %v", fakeDocker.Created)
+	}
+	fakeDocker.Unlock()
+
+	newContainer, err := fakeDocker.InspectContainer(fakeDocker.Created[1])
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	assert.NotContains(t, newContainer.HostConfig.SecurityOpt, "seccomp:unconfined", "Pods created with a secccomp annotation of docker/default should have empty security opt.")
+}
+
+func TestSeccompContainerAnnotationTrumpsPod(t *testing.T) {
+	dm, fakeDocker := newTestDockerManagerWithVersion("1.10.1", "1.22")
+	pod := &api.Pod{
+		ObjectMeta: api.ObjectMeta{
+			UID:       "12345678",
+			Name:      "foo2",
+			Namespace: "new",
+			Annotations: map[string]string{
+				"security.alpha.kubernetes.io/seccomp/pod":            "unconfined",
+				"security.alpha.kubernetes.io/seccomp/container/bar2": "docker/default",
+			},
+		},
+		Spec: api.PodSpec{
+			Containers: []api.Container{
+				{Name: "bar2"},
+			},
+		},
+	}
+
+	runSyncPod(t, dm, fakeDocker, pod, nil, false)
+
+	verifyCalls(t, fakeDocker, []string{
+		// Create pod infra container.
+		"create", "start", "inspect_container", "inspect_container",
+		// Create container.
+		"create", "start", "inspect_container",
+	})
+
+	fakeDocker.Lock()
+	if len(fakeDocker.Created) != 2 ||
+		!matchString(t, "/k8s_POD\\.[a-f0-9]+_foo2_new_", fakeDocker.Created[0]) ||
+		!matchString(t, "/k8s_bar2\\.[a-f0-9]+_foo2_new_", fakeDocker.Created[1]) {
+		t.Errorf("unexpected containers created %v", fakeDocker.Created)
+	}
+	fakeDocker.Unlock()
+
+	newContainer, err := fakeDocker.InspectContainer(fakeDocker.Created[1])
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	assert.NotContains(t, newContainer.HostConfig.SecurityOpt, "seccomp:unconfined", "Container annotation should trump the pod annotation for seccomp.")
 }
 
 func TestSecurityOptsAreNilWithDockerV19(t *testing.T) {

pkg/kubelet/kubelet.go

@@ -177,6 +177,7 @@ func NewMainKubelet(
 	dockerClient dockertools.DockerInterface,
 	kubeClient clientset.Interface,
 	rootDirectory string,
+	seccompProfileRoot string,
 	podInfraContainerImage string,
 	resyncInterval time.Duration,
 	pullQPS float32,
@@ -423,6 +424,7 @@ func NewMainKubelet(
 			serializeImagePulls,
 			enableCustomMetrics,
 			klet.hairpinMode == componentconfig.HairpinVeth,
+			seccompProfileRoot,
 			containerRuntimeOptions...,
 		)
 	case "rkt":

test/e2e_node/image.go

@@ -47,7 +47,7 @@ func dockerRuntime() kubecontainer.Runtime {
 		nil, nil, nil, pm, nil,
 		"", 0, 0, "",
 		nil, nil, nil, nil, nil, nil, nil,
-		false, nil, true, false, false,
+		false, nil, true, false, false, "",
 	)
 
 	return dm