From cbc428395c5de098681c760c9fa0da66ce0e7348 Mon Sep 17 00:00:00 2001 From: immutablet Date: Wed, 28 Mar 2018 13:53:01 -0700 Subject: [PATCH] Enable CloudKMS Plugin deployment. --- cluster/gce/BUILD | 1 + cluster/gce/gci/configure-helper.sh | 79 ++++++++++++++++++- .../manifests/kms-plugin-container.manifest | 8 ++ cluster/gce/manifests/kube-apiserver.manifest | 5 ++ 4 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 cluster/gce/manifests/kms-plugin-container.manifest diff --git a/cluster/gce/BUILD b/cluster/gce/BUILD index 83e25e586b..f14a7382b1 100644 --- a/cluster/gce/BUILD +++ b/cluster/gce/BUILD @@ -51,6 +51,7 @@ pkg_tar( "manifests/e2e-image-puller.manifest", "manifests/etcd.manifest", "manifests/glbc.manifest", + "manifests/kms-plugin-container.manifest", "manifests/kube-addon-manager.yaml", "manifests/kube-apiserver.manifest", "manifests/kube-controller-manager.manifest", diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 5f1552106f..08d03a0471 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1680,6 +1680,22 @@ function start-kube-apiserver { container_env="\"env\":[{${container_env}}]," fi + if [[ -n "${ETCD_KMS_KEY_ID:-}" ]]; then + ENCRYPTION_PROVIDER_CONFIG=$(cat << EOM | base64 | tr -d '\r\n' +kind: EncryptionConfig +apiVersion: v1 +resources: + - resources: + - secrets + providers: + - kms: + name: grpc-kms-provider + cachesize: 1000 + endpoint: unix:///var/run/kmsplugin/socket.sock +EOM +) + fi + if [[ -n "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then local encryption_provider_config_path="/etc/srv/kubernetes/encryption-provider-config.yml" echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}" @@ -1715,6 +1731,67 @@ function start-kube-apiserver { sed -i -e "s@{{admission_controller_config_volume}}@${admission_controller_config_volume}@g" "${src_file}" sed -i -e "s@{{image_policy_webhook_config_mount}}@${image_policy_webhook_config_mount}@g" "${src_file}" sed -i -e "s@{{image_policy_webhook_config_volume}}@${image_policy_webhook_config_volume}@g" "${src_file}" + + if [[ -z "${ETCD_KMS_KEY_ID:-}" ]]; then + # Removing KMS related placeholders. + sed -i -e " { + s@{{kms_plugin_container}}@@ + + s@{{kms_socket_mount}}@@ + s@{{encryption_provider_mount}}@@ + + s@{{kms_socket_volume}}@@ + s@{{encryption_provider_volume}}@@ + } " "${src_file}" + else + local kms_plugin_src_file="${src_dir}/kms-plugin-container.manifest" + + if [[ ! -f "${kms_plugin_src_file}" ]]; then + echo "Error: KMS Integration was requested, but "${kms_plugin_src_file}" is missing." + exit 1 + fi + + if [[ ! -f "${encryption_provider_config_path}" ]]; then + echo "Error: KMS Integration was requested, but "${encryption_provider_config_path}" is missing." + exit 1 + fi + + # TODO: Validate that the encryption config is for KMS. + + local kms_socket_dir="/var/run/kmsplugin" + + # kms_socket_mnt is used by both kms_plugin and kube-apiserver - this is how these containers talk. + local kms_socket_mnt="{ \"name\": \"kmssocket\", \"mountPath\": \"${kms_socket_dir}\", \"readOnly\": false}" + + local kms_socket_vol="{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}" + local kms_path_to_socket="${kms_socket_dir}/socket.sock" + + local encryption_provider_mnt="{ \"name\": \"encryptionconfig\", \"mountPath\": \"${encryption_provider_config_path}\", \"readOnly\": true}" + local encryption_provider_vol="{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${encryption_provider_config_path}\", \"type\": \"File\"}}" + + # TODO these are used in other places, convert to global. + local gce_conf_path="/etc/gce.conf" + local cloud_config_mount="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true}" + + local kms_plugin_container=$(echo $(sed " { + s@{{kms_key_uri}}@${ETCD_KMS_KEY_ID}@ + s@{{gce_conf_path}}@${gce_conf_path}@ + s@{{kms_path_to_socket}}@${kms_path_to_socket}@ + s@{{kms_socket_mount}}@${kms_socket_mnt}@ + s@{{cloud_config_mount}}@${cloud_config_mount}@ + } " "${kms_plugin_src_file}") | tr "\n" "\\n") + + sed -i -e " { + s@{{kms_plugin_container}}@${kms_plugin_container},@ + + s@{{kms_socket_mount}}@${kms_socket_mnt},@ + s@{{encryption_provider_mount}}@${encryption_provider_mnt},@ + + s@{{kms_socket_volume}}@${kms_socket_vol},@ + s@{{encryption_provider_volume}}@${encryption_provider_vol},@ + } " "${src_file}" + fi + cp "${src_file}" /etc/kubernetes/manifests } @@ -2451,4 +2528,4 @@ fi reset-motd prepare-mounter-rootfs modprobe configs -echo "Done for the configuration for kubernetes" +echo "Done for the configuration for kubernetes" \ No newline at end of file diff --git a/cluster/gce/manifests/kms-plugin-container.manifest b/cluster/gce/manifests/kms-plugin-container.manifest new file mode 100644 index 0000000000..cb90c9fef3 --- /dev/null +++ b/cluster/gce/manifests/kms-plugin-container.manifest @@ -0,0 +1,8 @@ +{ + "name": "kms-plugin", + "image": "gcr.io/google-containers/k8s-cloud-kms-plugin:v0.1.1", + "command": ["/k8s-cloud-kms-plugin", "--key-uri={{kms_key_uri}}", "--path-to-unix-socket={{kms_path_to_socket}}", "--gce-config={{gce_conf_path}}", "--logtostderr", "2>\&1"], + "livenessProbe": { "httpGet": {"host": "127.0.0.1", "port": 8081, "path": "/healthz"}, "initialDelaySeconds": 3, "timeoutSeconds": 3}, + "ports":[{ "name": "healthz", "containerPort": 8081, "hostPort": 8081}, { "name": "metrics", "containerPort": 8082, "hostPort": 8082}], + "volumeMounts": [{{cloud_config_mount}}, {{kms_socket_mount}}] +} \ No newline at end of file diff --git a/cluster/gce/manifests/kube-apiserver.manifest b/cluster/gce/manifests/kube-apiserver.manifest index 188a10b815..d13dbed70e 100644 --- a/cluster/gce/manifests/kube-apiserver.manifest +++ b/cluster/gce/manifests/kube-apiserver.manifest @@ -15,6 +15,7 @@ "spec":{ "hostNetwork": true, "containers":[ + {{kms_plugin_container}} { "name": "kube-apiserver", "image": "{{pillar['kube_docker_registry']}}/kube-apiserver:{{pillar['kube-apiserver_docker_tag']}}", @@ -47,6 +48,8 @@ "hostPort": 8080} ], "volumeMounts": [ + {{kms_socket_mount}} + {{encryption_provider_mount}} {{cloud_config_mount}} {{additional_cloud_config_mount}} {{webhook_config_mount}} @@ -86,6 +89,8 @@ } ], "volumes":[ + {{kms_socket_volume}} + {{encryption_provider_volume}} {{cloud_config_volume}} {{additional_cloud_config_volume}} {{webhook_config_volume}}