From a604d34c2105af5b32275da5dc4280cf12d42340 Mon Sep 17 00:00:00 2001 From: "Tim St. Clair" Date: Thu, 1 Jun 2017 17:42:26 -0700 Subject: [PATCH] Add GCE configuration parameter for webhook audit logging --- cluster/common.sh | 1 + cluster/gce/gci/configure-helper.sh | 71 ++++++++++++++----- .../kube-apiserver/kube-apiserver.manifest | 16 ++++- 3 files changed, 71 insertions(+), 17 deletions(-) diff --git a/cluster/common.sh b/cluster/common.sh index 590a369432..60ff964eb9 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -660,6 +660,7 @@ ENABLE_DEFAULT_STORAGE_CLASS: $(yaml-quote ${ENABLE_DEFAULT_STORAGE_CLASS:-}) ENABLE_APISERVER_BASIC_AUDIT: $(yaml-quote ${ENABLE_APISERVER_BASIC_AUDIT:-}) ENABLE_APISERVER_ADVANCED_AUDIT: $(yaml-quote ${ENABLE_APISERVER_ADVANCED_AUDIT:-}) ENABLE_CACHE_MUTATION_DETECTOR: $(yaml-quote ${ENABLE_CACHE_MUTATION_DETECTOR:-false}) +ADVANCED_AUDIT_BACKEND: $(yaml-quote ${ADVANCED_AUDIT_BACKEND:-log}) EOF if [ -n "${KUBELET_PORT:-}" ]; then cat >>$file <"${path}" +clusters: + - name: gcp-audit-server + cluster: + server: ${GCP_AUDIT_URL} +users: + - name: kube-apiserver + user: + auth-provider: + name: gcp +current-context: webhook +contexts: +- context: + cluster: gcp-audit-server + user: kube-apiserver + name: webhook +EOF + fi +} + function create-kubelet-kubeconfig { echo "Creating kubelet kubeconfig file" cat </var/lib/kubelet/bootstrap-kubeconfig @@ -1167,6 +1193,8 @@ function start-kube-apiserver { local audit_policy_config_mount="" local audit_policy_config_volume="" + local audit_webhook_config_mount="" + local audit_webhook_config_volume="" if [[ "${ENABLE_APISERVER_BASIC_AUDIT:-}" == "true" ]]; then # We currently only support enabling with a fixed path and with built-in log # rotation "disabled" (large value) so it behaves like kube-apiserver.log. @@ -1181,26 +1209,35 @@ function start-kube-apiserver { # never restarts. Please manually restart apiserver before this time. params+=" --audit-log-maxsize=2000000000" elif [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then - # We currently only support enabling with a fixed path and with built-in log - # rotation "disabled" (large value) so it behaves like kube-apiserver.log. - # External log rotation should be set up the same as for kube-apiserver.log. - params+=" --audit-log-path=/var/log/kube-apiserver-audit.log" - params+=" --audit-log-maxage=0" - params+=" --audit-log-maxbackup=0" - # Lumberjack doesn't offer any way to disable size-based rotation. It also - # has an in-memory counter that doesn't notice if you truncate the file. - # 2000000000 (in MiB) is a large number that fits in 31 bits. If the log - # grows at 10MiB/s (~30K QPS), it will rotate after ~6 years if apiserver - # never restarts. Please manually restart apiserver before this time. - params+=" --audit-log-maxsize=2000000000" - - local audit_policy_file="/etc/audit_policy.config" + local -r audit_policy_file="/etc/audit_policy.config" params+=" --audit-policy-file=${audit_policy_file}" - # Create the audit policy file, and mount it into the apiserver pod. create-master-audit-policy "${audit_policy_file}" - audit_policy_config_mount="{\"name\": \"auditpolicyconfigmount\",\"mountPath\": \"${audit_policy_file}\", \"readOnly\": false}," + audit_policy_config_mount="{\"name\": \"auditpolicyconfigmount\",\"mountPath\": \"${audit_policy_file}\", \"readOnly\": true}," audit_policy_config_volume="{\"name\": \"auditpolicyconfigmount\",\"hostPath\": {\"path\": \"${audit_policy_file}\"}}," + + if [[ "${ADVANCED_AUDIT_BACKEND:-log}" == *"log"* ]]; then + # The advanced audit log backend config matches the basic audit log config. + params+=" --audit-log-path=/var/log/kube-apiserver-audit.log" + params+=" --audit-log-maxage=0" + params+=" --audit-log-maxbackup=0" + # Lumberjack doesn't offer any way to disable size-based rotation. It also + # has an in-memory counter that doesn't notice if you truncate the file. + # 2000000000 (in MiB) is a large number that fits in 31 bits. If the log + # grows at 10MiB/s (~30K QPS), it will rotate after ~6 years if apiserver + # never restarts. Please manually restart apiserver before this time. + params+=" --audit-log-maxsize=2000000000" + fi + if [[ "${ADVANCED_AUDIT_BACKEND:-}" == *"webhook"* ]]; then + params+=" --audit-webhook-mode=batch" + + # Create the audit webhook config file, and mount it into the apiserver pod. + local -r audit_webhook_config_file="/etc/audit_webhook.config" + params+=" --audit-webhook-config-file=${audit_webhook_config_file}" + create-master-audit-webhook-config "${audit_webhook_config_file}" + audit_webhook_config_mount="{\"name\": \"auditwebhookconfigmount\",\"mountPath\": \"${audit_webhook_config_file}\", \"readOnly\": true}," + audit_webhook_config_volume="{\"name\": \"auditwebhookconfigmount\",\"hostPath\": {\"path\": \"${audit_webhook_config_file}\"}}," + fi fi if [[ "${ENABLE_APISERVER_LOGS_HANDLER:-}" == "false" ]]; then @@ -1311,6 +1348,8 @@ function start-kube-apiserver { sed -i -e "s@{{webhook_config_volume}}@${webhook_config_volume}@g" "${src_file}" sed -i -e "s@{{audit_policy_config_mount}}@${audit_policy_config_mount}@g" "${src_file}" sed -i -e "s@{{audit_policy_config_volume}}@${audit_policy_config_volume}@g" "${src_file}" + sed -i -e "s@{{audit_webhook_config_mount}}@${audit_webhook_config_mount}@g" "${src_file}" + sed -i -e "s@{{audit_webhook_config_volume}}@${audit_webhook_config_volume}@g" "${src_file}" sed -i -e "s@{{admission_controller_config_mount}}@${admission_controller_config_mount}@g" "${src_file}" sed -i -e "s@{{admission_controller_config_volume}}@${admission_controller_config_volume}@g" "${src_file}" sed -i -e "s@{{image_policy_webhook_config_mount}}@${image_policy_webhook_config_mount}@g" "${src_file}" diff --git a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest index d8c7c96213..5ac6d5fcd4 100644 --- a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest +++ b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest @@ -178,10 +178,22 @@ {% set audit_log = "" -%} {% set audit_policy_config_mount = "" -%} {% set audit_policy_config_volume = "" -%} +{% set audit_webhook_config_mount = "" -%} +{% set audit_webhook_config_volume = "" -%} {% if pillar['enable_apiserver_basic_audit'] is defined and pillar['enable_apiserver_basic_audit'] in ['true'] -%} {% set audit_log = "--audit-log-path=/var/log/kube-apiserver-audit.log --audit-log-maxage=0 --audit-log-maxbackup=0 --audit-log-maxsize=2000000000" -%} {% elif pillar['enable_apiserver_advanced_audit'] is defined and pillar['enable_apiserver_advanced_audit'] in ['true'] -%} - {% set audit_log = "--audit-log-path=/var/log/kube-apiserver-audit.log --audit-log-maxage=0 --audit-log-maxbackup=0 --audit-log-maxsize=2000000000 --audit-policy-file=/etc/audit_policy.config" -%} + {% set audit_log = "--audit-policy-file=/etc/audit_policy.config" -%} + {% set audit_policy_config_mount = "{\"name\": \"auditpolicyconfigmount\",\"mountPath\": \"/etc/audit_policy.config\", \"readOnly\": true}," -%} + {% set audit_policy_config_volume = "{\"name\": \"auditpolicyconfigmount\",\"hostPath\": {\"path\": \"/etc/audit_policy.config\"}}," -%} + {% if pillar['advanced_audit_backend'] is defined and 'log' in pillar['advanced_audit_backend'] -%} + {% set audit_log = audit_log + " --audit-log-path=/var/log/kube-apiserver-audit.log --audit-log-maxage=0 --audit-log-maxbackup=0 --audit-log-maxsize=2000000000" -%} + {% endif %} + {% if pillar['advanced_audit_backend'] is defined and 'webhook' in pillar['advanced_audit_backend'] -%} + {% set audit_log = audit_log + " --audit-webhook-mode=batch" -%} + {% set audit_webhook_config_mount = "{\"name\": \"auditwebhookconfigmount\",\"mountPath\": \"/etc/audit_webhook.config\", \"readOnly\": true}," -%} + {% set audit_webhook_config_volume = "{\"name\": \"auditwebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/audit_webhook.config\"}}," -%} + {% endif %} {% endif -%} {% set params = address + " " + storage_backend + " " + storage_media_type + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + feature_gates + " " + admission_control + " " + max_requests_inflight + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector + " " + etcd_quorum_read + " " + audit_log -%} @@ -245,6 +257,7 @@ {{webhook_config_mount}} {{webhook_authn_config_mount}} {{audit_policy_config_mount}} + {{audit_webhook_config_mount}} {{admission_controller_config_mount}} {{image_policy_webhook_config_mount}} { "name": "srvkube", @@ -283,6 +296,7 @@ {{webhook_config_volume}} {{webhook_authn_config_volume}} {{audit_policy_config_volume}} + {{audit_webhook_config_volume}} {{admission_controller_config_volume}} {{image_policy_webhook_config_volume}} { "name": "srvkube",