From c898ced795ac26d69adb5b7210620783e4a9057d Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Mon, 6 Aug 2018 15:22:58 -0400 Subject: [PATCH] Multi-arch images for apparmor-loader container Originally from: https://github.com/kubernetes/contrib/tree/master/apparmor/loader Moving the code here to prevent bit-rot and to be sure we can recreate or update the images on demand. Moving it here also ensures we can use the common harness to build the multi-arch manifests needed for running the apparmor e2e test can run on multiple architectures. Change-Id: Idece17c494fc944c0aaef64805d2f0e3c4d7fb28 --- test/images/BUILD | 1 + test/images/apparmor-loader/BASEIMAGE | 4 + test/images/apparmor-loader/BUILD | 29 ++ test/images/apparmor-loader/Dockerfile | 26 ++ test/images/apparmor-loader/Makefile | 25 ++ test/images/apparmor-loader/README.md | 66 +++++ test/images/apparmor-loader/VERSION | 1 + .../apparmor-loader/example-configmap.yaml | 76 +++++ .../apparmor-loader/example-daemon.yaml | 50 ++++ .../apparmor-loader/example-namespace.yaml | 6 + test/images/apparmor-loader/example-pod.yaml | 19 ++ test/images/apparmor-loader/loader.go | 260 ++++++++++++++++++ test/utils/image/manifest.go | 2 +- 13 files changed, 564 insertions(+), 1 deletion(-) create mode 100644 test/images/apparmor-loader/BASEIMAGE create mode 100644 test/images/apparmor-loader/BUILD create mode 100644 test/images/apparmor-loader/Dockerfile create mode 100644 test/images/apparmor-loader/Makefile create mode 100644 test/images/apparmor-loader/README.md create mode 100644 test/images/apparmor-loader/VERSION create mode 100644 test/images/apparmor-loader/example-configmap.yaml create mode 100644 test/images/apparmor-loader/example-daemon.yaml create mode 100644 test/images/apparmor-loader/example-namespace.yaml create mode 100644 test/images/apparmor-loader/example-pod.yaml create mode 100644 test/images/apparmor-loader/loader.go diff --git a/test/images/BUILD b/test/images/BUILD index 7295c900b0..e8a70a0930 100644 --- a/test/images/BUILD +++ b/test/images/BUILD @@ -11,6 +11,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//test/images/apparmor-loader:all-srcs", "//test/images/entrypoint-tester:all-srcs", "//test/images/fakegitserver:all-srcs", "//test/images/liveness:all-srcs", diff --git a/test/images/apparmor-loader/BASEIMAGE b/test/images/apparmor-loader/BASEIMAGE new file mode 100644 index 0000000000..960ad547f9 --- /dev/null +++ b/test/images/apparmor-loader/BASEIMAGE @@ -0,0 +1,4 @@ +amd64=alpine:3.8 +arm=arm32v6/alpine:3.8 +arm64=arm64v8/alpine:3.8 +ppc64le=ppc64le/alpine:3.8 diff --git a/test/images/apparmor-loader/BUILD b/test/images/apparmor-loader/BUILD new file mode 100644 index 0000000000..f033cc4ad8 --- /dev/null +++ b/test/images/apparmor-loader/BUILD @@ -0,0 +1,29 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["loader.go"], + importpath = "k8s.io/kubernetes/test/images/apparmor-loader", + visibility = ["//visibility:private"], + deps = ["//vendor/github.com/golang/glog:go_default_library"], +) + +go_binary( + name = "apparmor-loader", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/images/apparmor-loader/Dockerfile b/test/images/apparmor-loader/Dockerfile new file mode 100644 index 0000000000..0cad6e26ee --- /dev/null +++ b/test/images/apparmor-loader/Dockerfile @@ -0,0 +1,26 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM BASEIMAGE + +CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ + +RUN apk add apparmor libapparmor --update-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing/ --allow-untrusted + +ADD loader /usr/bin/loader + +ENTRYPOINT ["/usr/bin/loader", "-logtostderr", "-v=2"] + +# Default directory to watch. +CMD ["/profiles"] diff --git a/test/images/apparmor-loader/Makefile b/test/images/apparmor-loader/Makefile new file mode 100644 index 0000000000..807d3a2a48 --- /dev/null +++ b/test/images/apparmor-loader/Makefile @@ -0,0 +1,25 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SRCS=loader +ARCH ?= amd64 +TARGET ?= $(CURDIR) +GOLANG_VERSION ?= latest +SRC_DIR = $(notdir $(shell pwd)) +export + +bin: + ../image-util.sh bin $(SRCS) + +.PHONY: bin diff --git a/test/images/apparmor-loader/README.md b/test/images/apparmor-loader/README.md new file mode 100644 index 0000000000..97c9c370cb --- /dev/null +++ b/test/images/apparmor-loader/README.md @@ -0,0 +1,66 @@ +# AppArmor Profile Loader + +This is a small proof-of-concept daemon to demonstrate how AppArmor profiles can be loaded onto +nodes of a Kubernetes cluster. It is not considered production ready, nor will it be supported as a +long-term solution. + +## Running the AppArmor Profile Loader + +The [example-daemon.yaml](example-daemon.yaml) provides an example manifest for running the loader +as a cluster DaemonSet. In this example, the loader runs in a DaemonSet pod on each node in the +cluster, and periodically (every 30 seconds) polls for new profiles in the `apparmor-profiles` +configmap ([example manifest](example-configmap.yaml)). It is recommended to run the Daemon and +ConfigMap in a separate, restricted namespace: + + $ kubectl create -f example-namespace.yaml + $ kubectl create -f example-configmap.yaml # Includes the k8s-nginx profile + $ kubectl create -f example-daemon.yaml + +Check that the profile was loaded: + + $ POD=$(kubectl --namespace apparmor get pod -o jsonpath="{.items[0].metadata.name}") + $ kubectl --namespace apparmor logs $POD + I0829 22:48:24.917263 1 loader.go:139] Polling /profiles every 30s + I0829 22:48:24.954295 1 loader.go:196] Loading profiles from /profiles/k8s-nginx: + Addition succeeded for "k8s-nginx". + I0829 22:48:24.954328 1 loader.go:100] Successfully loaded profiles: [k8s-nginx] + +Trying running a pod with the loaded profile (requires Kubernetes >= v1.4): + + $ kubectl create -f example-pod.yaml + # Verify that it's running with the new profile: + $ kubectl exec nginx-apparmor cat /proc/1/attr/current + k8s-nginx (enforce) + $ kubectl exec nginx-apparmor touch /tmp/foo + touch: cannot touch '/tmp/foo': Permission denied + error: error executing remote command: command terminated with non-zero exit code: Error executing in Docker Container: 1 + + +### Standalone + +The loader go binary can also be run as a standalone binary on the host. It must be run with root +privileges: + + sudo loader -logtostderr /path/to/profile/dir + +Alternatively, it can be run with the supplied loader docker image: + + PROFILES_PATH=/path/to/profile/dir + sudo docker run \ + --privileged \ + --detach=true \ + --volume=/sys:/sys:ro \ + --volume=/etc/apparmor.d:/etc/apparmor.d:ro \ + --volume=$PROFILES_PATH:/profiles:ro \ + --name=aa-loader \ + google/apparmor-loader:latest + +## Build the loader + +The loader binary is a simple go program, and can be built with `make all-push WHAT=apparmor-loader` +(from test/images). + +## Limitations + +The loader will not unload profiles that are removed, and will not update profiles that are changed. +This is by design, since there are nuanced issues with changing profiles that are in use. diff --git a/test/images/apparmor-loader/VERSION b/test/images/apparmor-loader/VERSION new file mode 100644 index 0000000000..d3827e75a5 --- /dev/null +++ b/test/images/apparmor-loader/VERSION @@ -0,0 +1 @@ +1.0 diff --git a/test/images/apparmor-loader/example-configmap.yaml b/test/images/apparmor-loader/example-configmap.yaml new file mode 100644 index 0000000000..03ab295388 --- /dev/null +++ b/test/images/apparmor-loader/example-configmap.yaml @@ -0,0 +1,76 @@ +# An example ConfigMap demonstrating how profiles can be stored as Kubernetes objects, and loaded by +# the apparmor-loader DaemonSet. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: apparmor-profiles + namespace: apparmor +data: + # Filename k8s-nginx maps to the definition of the nginx profile. + k8s-nginx: |- + #include + + # From https://github.com/jfrazelle/bane/blob/master/docker-nginx-sample + profile k8s-nginx flags=(attach_disconnected,mediate_deleted) { + #include + + network inet tcp, + network inet udp, + network inet icmp, + + deny network raw, + + deny network packet, + + file, + umount, + + deny /bin/** wl, + deny /boot/** wl, + deny /dev/** wl, + deny /etc/** wl, + deny /home/** wl, + deny /lib/** wl, + deny /lib64/** wl, + deny /media/** wl, + deny /mnt/** wl, + deny /opt/** wl, + deny /proc/** wl, + deny /root/** wl, + deny /sbin/** wl, + deny /srv/** wl, + deny /tmp/** wl, + deny /sys/** wl, + deny /usr/** wl, + + audit /** w, + + /var/run/nginx.pid w, + + /usr/sbin/nginx ix, + + deny /bin/dash mrwklx, + deny /bin/sh mrwklx, + deny /usr/bin/top mrwklx, + + capability chown, + capability dac_override, + capability setuid, + capability setgid, + capability net_bind_service, + + deny @{PROC}/{*,**^[0-9*],sys/kernel/shm*} wkx, + deny @{PROC}/sysrq-trigger rwklx, + deny @{PROC}/mem rwklx, + deny @{PROC}/kmem rwklx, + deny @{PROC}/kcore rwklx, + deny mount, + deny /sys/[^f]*/** wklx, + deny /sys/f[^s]*/** wklx, + deny /sys/fs/[^c]*/** wklx, + deny /sys/fs/c[^g]*/** wklx, + deny /sys/fs/cg[^r]*/** wklx, + deny /sys/firmware/efi/efivars/** rwklx, + deny /sys/kernel/security/** rwklx, + } diff --git a/test/images/apparmor-loader/example-daemon.yaml b/test/images/apparmor-loader/example-daemon.yaml new file mode 100644 index 0000000000..f7df03804b --- /dev/null +++ b/test/images/apparmor-loader/example-daemon.yaml @@ -0,0 +1,50 @@ +# The example DaemonSet demonstrating how the profile loader can be deployed onto a cluster to +# automatically load AppArmor profiles from a ConfigMap. + +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: apparmor-loader + # Namespace must match that of the ConfigMap. + namespace: apparmor +spec: + template: + metadata: + name: apparmor-loader + labels: + daemon: apparmor-loader + spec: + containers: + - name: apparmor-loader + image: google/apparmor-loader:latest + args: + # Tell the loader to pull the /profiles directory every 30 seconds. + - -poll + - 30s + - /profiles + securityContext: + # The loader requires root permissions to actually load the profiles. + privileged: true + volumeMounts: + - name: sys + mountPath: /sys + readOnly: true + - name: apparmor-includes + mountPath: /etc/apparmor.d + readOnly: true + - name: profiles + mountPath: /profiles + readOnly: true + volumes: + # The /sys directory must be mounted to interact with the AppArmor module. + - name: sys + hostPath: + path: /sys + # The /etc/apparmor.d directory is required for most apparmor include templates. + - name: apparmor-includes + hostPath: + path: /etc/apparmor.d + # Map in the profile data. + - name: profiles + configMap: + name: apparmor-profiles diff --git a/test/images/apparmor-loader/example-namespace.yaml b/test/images/apparmor-loader/example-namespace.yaml new file mode 100644 index 0000000000..29ef3c13f3 --- /dev/null +++ b/test/images/apparmor-loader/example-namespace.yaml @@ -0,0 +1,6 @@ +# The example Namespace used by other example objects. + +apiVersion: v1 +kind: Namespace +metadata: + name: apparmor diff --git a/test/images/apparmor-loader/example-pod.yaml b/test/images/apparmor-loader/example-pod.yaml new file mode 100644 index 0000000000..253dd2ff68 --- /dev/null +++ b/test/images/apparmor-loader/example-pod.yaml @@ -0,0 +1,19 @@ +# The example Pod utilizing the profile loaded by the sample daemon. + +apiVersion: v1 +kind: Pod +metadata: + name: nginx-apparmor + # Note that the Pod does not need to be in the same namespace as the loader. + labels: + app: nginx + annotations: + # Tell Kubernetes to apply the AppArmor profile "k8s-nginx". + # Note that this is ignored if the Kubernetes node is not running version 1.4 or greater. + container.apparmor.security.beta.kubernetes.io/nginx: localhost/k8s-nginx +spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 diff --git a/test/images/apparmor-loader/loader.go b/test/images/apparmor-loader/loader.go new file mode 100644 index 0000000000..5f1e2095e6 --- /dev/null +++ b/test/images/apparmor-loader/loader.go @@ -0,0 +1,260 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bufio" + "bytes" + "flag" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "time" + + "github.com/golang/glog" +) + +var ( + // The directories to load profiles from. + dirs []string + poll = flag.Duration("poll", -1, "Poll the directories for new profiles with this interval. Values < 0 disable polling, and exit after loading the profiles.") +) + +const ( + parser = "apparmor_parser" + apparmorfs = "/sys/kernel/security/apparmor" +) + +func main() { + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s [FLAG]... [PROFILE_DIR]...\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Load the AppArmor profiles specified in the PROFILE_DIR directories.\n") + flag.PrintDefaults() + } + flag.Parse() + + dirs = flag.Args() + if len(dirs) == 0 { + glog.Errorf("Must specify at least one directory.") + flag.Usage() + os.Exit(1) + } + + // Check that the required parser binary is found. + if _, err := exec.LookPath(parser); err != nil { + glog.Exitf("Required binary %s not found in PATH", parser) + } + + // Check that loaded profiles can be read. + if _, err := getLoadedProfiles(); err != nil { + glog.Exitf("Unable to access apparmor profiles: %v", err) + } + + if *poll < 0 { + runOnce() + } else { + pollForever() + } +} + +// No polling: run once and exit. +func runOnce() { + if success, newProfiles := loadNewProfiles(); !success { + if len(newProfiles) > 0 { + glog.Exitf("Not all profiles were successfully loaded. Loaded: %v", newProfiles) + } else { + glog.Exit("Error loading profiles.") + } + } else { + if len(newProfiles) > 0 { + glog.Infof("Successfully loaded profiles: %v", newProfiles) + } else { + glog.Warning("No new profiles found.") + } + } +} + +// Poll the directories indefinitely. +func pollForever() { + glog.V(2).Infof("Polling %s every %s", strings.Join(dirs, ", "), poll.String()) + pollFn := func() { + _, newProfiles := loadNewProfiles() + if len(newProfiles) > 0 { + glog.V(2).Infof("Successfully loaded profiles: %v", newProfiles) + } + } + pollFn() // Run immediately. + ticker := time.NewTicker(*poll) + for range ticker.C { + pollFn() + } +} + +func loadNewProfiles() (success bool, newProfiles []string) { + loadedProfiles, err := getLoadedProfiles() + if err != nil { + glog.Errorf("Error reading loaded profiles: %v", err) + return false, nil + } + + success = true + for _, dir := range dirs { + infos, err := ioutil.ReadDir(dir) + if err != nil { + glog.Warningf("Error reading %s: %v", dir, err) + success = false + continue + } + + for _, info := range infos { + path := filepath.Join(dir, info.Name()) + // If directory, or symlink to a directory, skip it. + resolvedInfo, err := resolveSymlink(dir, info) + if err != nil { + glog.Warningf("Error resolving symlink: %v", err) + continue + } + if resolvedInfo.IsDir() { + // Directory listing is shallow. + glog.V(4).Infof("Skipping directory %s", path) + continue + } + + glog.V(4).Infof("Scanning %s for new profiles", path) + profiles, err := getProfileNames(path) + if err != nil { + glog.Warningf("Error reading %s: %v", path, err) + success = false + continue + } + + if unloadedProfiles(loadedProfiles, profiles) { + if err := loadProfiles(path); err != nil { + glog.Errorf("Could not load profiles: %v", err) + success = false + continue + } + // Add new profiles to list of loaded profiles. + newProfiles = append(newProfiles, profiles...) + for _, profile := range profiles { + loadedProfiles[profile] = true + } + } + } + } + + return success, newProfiles +} + +func getProfileNames(path string) ([]string, error) { + cmd := exec.Command(parser, "--names", path) + stderr := &bytes.Buffer{} + cmd.Stderr = stderr + out, err := cmd.Output() + if err != nil { + if stderr.Len() > 0 { + glog.Warning(stderr.String()) + } + return nil, fmt.Errorf("error reading profiles from %s: %v", path, err) + } + + trimmed := strings.TrimSpace(string(out)) // Remove trailing \n + return strings.Split(trimmed, "\n"), nil +} + +func unloadedProfiles(loadedProfiles map[string]bool, profiles []string) bool { + for _, profile := range profiles { + if !loadedProfiles[profile] { + return true + } + } + return false +} + +func loadProfiles(path string) error { + cmd := exec.Command(parser, "--verbose", path) + stderr := &bytes.Buffer{} + cmd.Stderr = stderr + out, err := cmd.Output() + glog.V(2).Infof("Loading profiles from %s:\n%s", path, out) + if err != nil { + if stderr.Len() > 0 { + glog.Warning(stderr.String()) + } + return fmt.Errorf("error loading profiles from %s: %v", path, err) + } + return nil +} + +// If the given fileinfo is a symlink, return the FileInfo of the target. Otherwise, return the +// given fileinfo. +func resolveSymlink(basePath string, info os.FileInfo) (os.FileInfo, error) { + if info.Mode()&os.ModeSymlink == 0 { + // Not a symlink. + return info, nil + } + fpath := filepath.Join(basePath, info.Name()) + resolvedName, err := filepath.EvalSymlinks(fpath) + if err != nil { + return nil, fmt.Errorf("error resolving symlink %s: %v", fpath, err) + } + resolvedInfo, err := os.Stat(resolvedName) + if err != nil { + return nil, fmt.Errorf("error calling stat on %s: %v", resolvedName, err) + } + return resolvedInfo, nil +} + +// TODO: This is copied from k8s.io/kubernetes/pkg/security/apparmor.getLoadedProfiles. +// Refactor that method to expose it in a reusable way, and delete this version. +func getLoadedProfiles() (map[string]bool, error) { + profilesPath := path.Join(apparmorfs, "profiles") + profilesFile, err := os.Open(profilesPath) + if err != nil { + return nil, fmt.Errorf("failed to open %s: %v", profilesPath, err) + } + defer profilesFile.Close() + + profiles := map[string]bool{} + scanner := bufio.NewScanner(profilesFile) + for scanner.Scan() { + profileName := parseProfileName(scanner.Text()) + if profileName == "" { + // Unknown line format; skip it. + continue + } + profiles[profileName] = true + } + return profiles, nil +} + +// The profiles file is formatted with one profile per line, matching a form: +// namespace://profile-name (mode) +// profile-name (mode) +// Where mode is {enforce, complain, kill}. The "namespace://" is only included for namespaced +// profiles. For the purposes of Kubernetes, we consider the namespace part of the profile name. +func parseProfileName(profileLine string) string { + modeIndex := strings.IndexRune(profileLine, '(') + if modeIndex < 0 { + return "" + } + return strings.TrimSpace(profileLine[:modeIndex]) +} diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index 31c601ebe9..52e29a65ba 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -51,7 +51,7 @@ func (i *ImageConfig) SetVersion(version string) { var ( AdmissionWebhook = ImageConfig{e2eRegistry, "webhook", "1.12v2", false} APIServer = ImageConfig{e2eRegistry, "sample-apiserver", "1.0", false} - AppArmorLoader = ImageConfig{gcRegistry, "apparmor-loader", "0.1", false} + AppArmorLoader = ImageConfig{e2eRegistry, "apparmor-loader", "1.0", false} BusyBox = ImageConfig{dockerHubRegistry, "busybox", "1.29", false} CheckMetadataConcealment = ImageConfig{gcRegistry, "check-metadata-concealment", "v0.0.3", false} CudaVectorAdd = ImageConfig{e2eRegistry, "cuda-vector-add", "1.0", false}