Merge pull request #61862 from immutableT/kms-plugin-deploy-cherry-pick

Automatic merge from submit-queue (batch tested with PRs 59636, 62429, 61862). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Inject CloudKMS Plugin container into Kube-APIServer pod.

**What this PR does / why we need it**:
Inject CloudKMS Plugin container into Kube-APIServer pod when etcd level encryption via CloudKMS Plugin is requested.
**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE

```

cluster/gce/BUILD

@@ -51,6 +51,7 @@ pkg_tar(
         "manifests/e2e-image-puller.manifest",
         "manifests/etcd.manifest",
         "manifests/glbc.manifest",
+        "manifests/kms-plugin-container.manifest",
         "manifests/kube-addon-manager.yaml",
         "manifests/kube-apiserver.manifest",
         "manifests/kube-controller-manager.manifest",

cluster/gce/gci/configure-helper.sh

@@ -1680,6 +1680,22 @@ function start-kube-apiserver {
     container_env="\"env\":[{${container_env}}],"
   fi
 
+  if [[ -n "${ETCD_KMS_KEY_ID:-}" ]]; then
+    ENCRYPTION_PROVIDER_CONFIG=$(cat << EOM | base64 | tr -d '\r\n'
+kind: EncryptionConfig
+apiVersion: v1
+resources:
+  - resources:
+    - secrets
+    providers:
+    - kms:
+       name: grpc-kms-provider
+       cachesize: 1000
+       endpoint: unix:///var/run/kmsplugin/socket.sock
+EOM
+)
+  fi
+
   if [[ -n "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then
     local encryption_provider_config_path="/etc/srv/kubernetes/encryption-provider-config.yml"
     echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}"
@@ -1715,6 +1731,67 @@ function start-kube-apiserver {
   sed -i -e "s@{{admission_controller_config_volume}}@${admission_controller_config_volume}@g" "${src_file}"
   sed -i -e "s@{{image_policy_webhook_config_mount}}@${image_policy_webhook_config_mount}@g" "${src_file}"
   sed -i -e "s@{{image_policy_webhook_config_volume}}@${image_policy_webhook_config_volume}@g" "${src_file}"
+
+  if [[ -z "${ETCD_KMS_KEY_ID:-}" ]]; then
+    # Removing KMS related placeholders.
+    sed -i -e " {
+      s@{{kms_plugin_container}}@@
+
+      s@{{kms_socket_mount}}@@
+      s@{{encryption_provider_mount}}@@
+
+      s@{{kms_socket_volume}}@@
+      s@{{encryption_provider_volume}}@@
+    } " "${src_file}"
+  else
+    local kms_plugin_src_file="${src_dir}/kms-plugin-container.manifest"
+
+    if [[ ! -f "${kms_plugin_src_file}" ]]; then
+      echo "Error: KMS Integration was requested, but "${kms_plugin_src_file}" is missing."
+      exit 1
+    fi
+
+    if [[ ! -f "${encryption_provider_config_path}" ]]; then
+      echo "Error: KMS Integration was requested, but "${encryption_provider_config_path}" is missing."
+      exit 1
+    fi
+
+    # TODO: Validate that the encryption config is for KMS.
+
+    local kms_socket_dir="/var/run/kmsplugin"
+
+    # kms_socket_mnt is used by both kms_plugin and kube-apiserver - this is how these containers talk.
+    local kms_socket_mnt="{ \"name\": \"kmssocket\", \"mountPath\": \"${kms_socket_dir}\", \"readOnly\": false}"
+
+    local kms_socket_vol="{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}"
+    local kms_path_to_socket="${kms_socket_dir}/socket.sock"
+
+    local encryption_provider_mnt="{ \"name\": \"encryptionconfig\", \"mountPath\": \"${encryption_provider_config_path}\", \"readOnly\": true}"
+    local encryption_provider_vol="{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${encryption_provider_config_path}\", \"type\": \"File\"}}"
+
+    # TODO these are used in other places, convert to global.
+    local gce_conf_path="/etc/gce.conf"
+    local cloud_config_mount="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true}"
+
+    local kms_plugin_container=$(echo $(sed " {
+      s@{{kms_key_uri}}@${ETCD_KMS_KEY_ID}@
+      s@{{gce_conf_path}}@${gce_conf_path}@
+      s@{{kms_path_to_socket}}@${kms_path_to_socket}@
+      s@{{kms_socket_mount}}@${kms_socket_mnt}@
+      s@{{cloud_config_mount}}@${cloud_config_mount}@
+    } " "${kms_plugin_src_file}") | tr "\n" "\\n")
+
+    sed -i -e " {
+      s@{{kms_plugin_container}}@${kms_plugin_container},@
+
+      s@{{kms_socket_mount}}@${kms_socket_mnt},@
+      s@{{encryption_provider_mount}}@${encryption_provider_mnt},@
+
+      s@{{kms_socket_volume}}@${kms_socket_vol},@
+      s@{{encryption_provider_volume}}@${encryption_provider_vol},@
+    } " "${src_file}"
+  fi
+
   cp "${src_file}" /etc/kubernetes/manifests
 }
 

cluster/gce/manifests/kms-plugin-container.manifest

@@ -0,0 +1,8 @@
+{
+  "name": "kms-plugin",
+  "image": "gcr.io/google-containers/k8s-cloud-kms-plugin:v0.1.1",
+  "command": ["/k8s-cloud-kms-plugin", "--key-uri={{kms_key_uri}}", "--path-to-unix-socket={{kms_path_to_socket}}", "--gce-config={{gce_conf_path}}", "--logtostderr", "2>\&1"],
+  "livenessProbe": { "httpGet": {"host": "127.0.0.1", "port": 8081, "path": "/healthz"}, "initialDelaySeconds": 3, "timeoutSeconds": 3},
+  "ports":[{ "name": "healthz", "containerPort": 8081, "hostPort": 8081}, { "name": "metrics", "containerPort": 8082, "hostPort": 8082}],
+  "volumeMounts": [{{cloud_config_mount}}, {{kms_socket_mount}}]
+}

cluster/gce/manifests/kube-apiserver.manifest

@@ -15,6 +15,7 @@
 "spec":{
 "hostNetwork": true,
 "containers":[
+    {{kms_plugin_container}}
     {
     "name": "kube-apiserver",
     "image": "{{pillar['kube_docker_registry']}}/kube-apiserver:{{pillar['kube-apiserver_docker_tag']}}",
@@ -47,6 +48,8 @@
         "hostPort": 8080}
         ],
     "volumeMounts": [
+        {{kms_socket_mount}}
+        {{encryption_provider_mount}}
         {{cloud_config_mount}}
         {{additional_cloud_config_mount}}
         {{webhook_config_mount}}
@@ -86,6 +89,8 @@
     }
 ],
 "volumes":[
+  {{kms_socket_volume}}
+  {{encryption_provider_volume}}
   {{cloud_config_volume}}
   {{additional_cloud_config_volume}}
   {{webhook_config_volume}}