Merge pull request #55509 from tallclair/psp-addons

Automatic merge from submit-queue (batch tested with PRs 54602, 54877, 55243, 55509, 55128). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

PodSecurityPolicies for addons

**What this PR does / why we need it**:

1. Colocate addon PodSecurityPolicy config with the addons (in a `podsecuritypolicies` subdirectory). 
2. Add policies for addons that are currently missing policies (not in the default GCE suite)
3. Remove HostPath SSL certs from several heapster deployments, so that heapster doesn't require a special PSP

**Which issue(s) this PR fixes**:
#43538

**Release note**:
```release-note
- Add PodSecurityPolicies for cluster addons
- Remove SSL cert HostPath volumes from heapster addons
```

cluster/addons/calico-policy-controller/podsecuritypolicies/calico-node-psp-binding.yaml

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gce:podsecuritypolicy:calico
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/cluster-service: "true"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: gce:podsecuritypolicy:privileged
+subjects:
+- kind: ServiceAccount
+  name: calico
+  namespace: kube-system

cluster/addons/cluster-monitoring/google/heapster-controller.yaml

@@ -58,26 +58,12 @@ spec:
             - /heapster
             - --source=kubernetes.summary_api:''
             - --sink=gcm
-          volumeMounts:
-            - name: ssl-certs
-              mountPath: /etc/ssl/certs
-              readOnly: true
-            - name: usr-ca-certs
-              mountPath: /usr/share/ca-certificates
-              readOnly: true
         - image: gcr.io/google_containers/heapster-amd64:v1.5.0-beta.0
           name: eventer
           command:
             - /eventer
             - --source=kubernetes:''
             - --sink=gcl
-          volumeMounts:
-            - name: ssl-certs
-              mountPath: /etc/ssl/certs
-              readOnly: true
-            - name: usr-ca-certs
-              mountPath: /usr/share/ca-certificates
-              readOnly: true
         - image: gcr.io/google_containers/addon-resizer:1.7
           name: heapster-nanny
           resources:
@@ -136,13 +122,6 @@ spec:
             - --container=eventer
             - --poll-period=300000
             - --estimator=exponential
-      volumes:
-        - name: ssl-certs
-          hostPath:
-            path: "/etc/ssl/certs"
-        - name: usr-ca-certs
-          hostPath:
-            path: "/usr/share/ca-certificates"
       serviceAccountName: heapster
       tolerations:
         - key: "CriticalAddonsOnly"

cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml

@@ -59,26 +59,12 @@ spec:
             - --source=kubernetes.summary_api:''
             - --sink=influxdb:http://monitoring-influxdb:8086
             - --sink=gcm:?metrics=autoscaling
-          volumeMounts:
-            - name: ssl-certs
-              mountPath: /etc/ssl/certs
-              readOnly: true
-            - name: usr-ca-certs
-              mountPath: /usr/share/ca-certificates
-              readOnly: true
         - image: gcr.io/google_containers/heapster-amd64:v1.5.0-beta.0
           name: eventer
           command:
             - /eventer
             - --source=kubernetes:''
             - --sink=gcl
-          volumeMounts:
-            - name: ssl-certs
-              mountPath: /etc/ssl/certs
-              readOnly: true
-            - name: usr-ca-certs
-              mountPath: /usr/share/ca-certificates
-              readOnly: true
         - image: gcr.io/google_containers/addon-resizer:1.7
           name: heapster-nanny
           resources:
@@ -137,13 +123,6 @@ spec:
             - --container=eventer
             - --poll-period=300000
             - --estimator=exponential
-      volumes:
-        - name: ssl-certs
-          hostPath:
-            path: "/etc/ssl/certs"
-        - name: usr-ca-certs
-          hostPath:
-            path: "/usr/share/ca-certificates"
       serviceAccountName: heapster
       tolerations:
         - key: "CriticalAddonsOnly"

cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml

@@ -56,13 +56,6 @@ spec:
             - /heapster
             - --source=kubernetes.summary_api:''
             - --sink=stackdriver:?cluster_name={{ cluster_name }}&min_interval_sec=100&batch_export_timeout_sec=110
-          volumeMounts:
-            - name: ssl-certs
-              mountPath: /etc/ssl/certs
-              readOnly: true
-            - name: usr-ca-certs
-              mountPath: /usr/share/ca-certificates
-              readOnly: true
         # BEGIN_PROMETHEUS_TO_SD
         - name: prom-to-sd
           image: gcr.io/google-containers/prometheus-to-sd:v0.2.2
@@ -112,13 +105,6 @@ spec:
             - --container=heapster
             - --poll-period=300000
             - --estimator=exponential
-      volumes:
-        - name: ssl-certs
-          hostPath:
-            path: "/etc/ssl/certs"
-        - name: usr-ca-certs
-          hostPath:
-            path: "/usr/share/ca-certificates"
       serviceAccountName: heapster
       tolerations:
         - key: "CriticalAddonsOnly"

cluster/addons/etcd-empty-dir-cleanup/etcd-empty-dir-cleanup.yaml

@@ -1,4 +1,14 @@
 apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: etcd-empty-dir-cleanup
+  namespace: kube-system
+  labels:
+    k8s-app: etcd-empty-dir-cleanup
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+---
+apiVersion: v1
 kind: Pod
 metadata:
   name: etcd-empty-dir-cleanup
@@ -8,6 +18,7 @@ metadata:
   labels:
     k8s-app: etcd-empty-dir-cleanup
 spec:
+  serviceAccountName: etcd-empty-dir-cleanup
   hostNetwork: true
   dnsPolicy: Default
   containers:

cluster/addons/etcd-empty-dir-cleanup/podsecuritypolicies/etcd-empty-dir-cleanup-psp-binding.yaml

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gce:podsecuritypolicy:etcd-empty-dir-cleanup
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/cluster-service: "true"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: gce:podsecuritypolicy:etcd-empty-dir-cleanup
+subjects:
+- kind: ServiceAccount
+  name: etcd-empty-dir-cleanup
+  namespace: kube-system

cluster/addons/etcd-empty-dir-cleanup/podsecuritypolicies/etcd-empty-dir-cleanup-psp-role.yaml

@@ -0,0 +1,17 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: gce:podsecuritypolicy:etcd-empty-dir-cleanup
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups:
+  - extensions
+  resourceNames:
+  - gce.etcd-empty-dir-cleanup
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use

cluster/addons/etcd-empty-dir-cleanup/podsecuritypolicies/etcd-empty-dir-cleanup-psp.yaml

@@ -0,0 +1,31 @@
+apiVersion: extensions/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: gce.etcd-empty-dir-cleanup
+  annotations:
+    kubernetes.io/description: 'Policy used by the etcd-empty-dir-cleanup addon.'
+    # TODO: etcd-empty-dir-cleanup should run with the default seccomp profile
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*'
+    # 'runtime/default' is already the default, but must be filled in on the
+    # pod to pass admission.
+    apparmor.security.beta.kubernetes.io/defaultProfileName:  'runtime/default'
+    apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
+  labels:
+    kubernetes.io/cluster-service: 'true'
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  privileged: false
+  volumes:
+  - 'secret'
+  hostNetwork: true
+  hostIPC: false
+  hostPID: false
+  runAsUser:
+    rule: 'RunAsAny'
+  seLinux:
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'RunAsAny'
+  fsGroup:
+    rule: 'RunAsAny'
+  readOnlyRootFilesystem: false

cluster/addons/fluentd-elasticsearch/podsecuritypolicies/es-psp-binding.yaml

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gce:podsecuritypolicy:elasticsearch-logging
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/cluster-service: "true"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: gce:podsecuritypolicy:privileged
+subjects:
+- kind: ServiceAccount
+  name: elasticsearch-logging
+  namespace: kube-system

cluster/addons/ip-masq-agent/ip-masq-agent.yaml

@@ -1,3 +1,13 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ip-masq-agent
+  namespace: kube-system
+  labels:
+    k8s-app: ip-masq-agent
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+---
 # https://github.com/kubernetes-incubator/ip-masq-agent/blob/v2.0.0/README.md
 apiVersion: extensions/v1beta1
 kind: DaemonSet
@@ -14,6 +24,7 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+      serviceAccountName: ip-masq-agent
       hostNetwork: true
       containers:
       - name: ip-masq-agent

cluster/addons/ip-masq-agent/podsecuritypolicies/ip-masq-agent-psp-binding.yaml

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gce:podsecuritypolicy:ip-masq-agent
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/cluster-service: "true"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: gce:podsecuritypolicy:privileged
+subjects:
+- kind: ServiceAccount
+  name: ip-masq-agent
+  namespace: kube-system

cluster/gce/gci/configure-helper.sh

@@ -1702,14 +1702,35 @@ function start-cluster-autoscaler {
   fi
 }
 
-# A helper function for copying addon manifests and set dir/files
-# permissions.
+# A helper function for setting up addon manifests.
 #
 # $1: addon category under /etc/kubernetes
 # $2: manifest source dir
+# $3: (optional) auxilary manifest source dir
 function setup-addon-manifests {
-  local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/$2"
+  local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty"
   local -r dst_dir="/etc/kubernetes/$1/$2"
+
+  copy-manifests "${src_dir}/$2" "${dst_dir}"
+
+  # If the PodSecurityPolicy admission controller is enabled,
+  # set up the corresponding addon policies.
+  if [[ "${ENABLE_POD_SECURITY_POLICY:-}" == "true" ]]; then
+    local -r psp_dir="${src_dir}/${3:-$2}/podsecuritypolicies"
+    if [[ -d "${psp_dir}" ]]; then
+      copy-manifests "${psp_dir}" "${dst_dir}"
+    fi
+  fi
+}
+
+# A helper function for copying manifests and setting dir/files
+# permissions.
+#
+# $1: absolute source dir
+# $2: absolute destination dir
+function copy-manifests {
+  local -r src_dir="$1"
+  local -r dst_dir="$2"
   if [[ ! -d "${dst_dir}" ]]; then
     mkdir -p "${dst_dir}"
   fi
@@ -1780,7 +1801,7 @@ function start-kube-addons {
   fi
 
   if [[ "${ENABLE_POD_SECURITY_POLICY:-}" == "true" ]]; then
-      setup-addon-manifests "addons" "podsecuritypolicies"
+    setup-addon-manifests "addons" "podsecuritypolicies"
   fi
 
   # Set up manifests of other addons.
@@ -1892,7 +1913,7 @@ EOF
   fi
   if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
     # Setup role binding for standalone node problem detector.
-    setup-addon-manifests "addons" "node-problem-detector/standalone"
+    setup-addon-manifests "addons" "node-problem-detector/standalone" "node-problem-detector"
   fi
   if echo "${ADMISSION_CONTROL:-}" | grep -q "LimitRanger"; then
     setup-addon-manifests "admission-controls" "limit-range"