Merge pull request #57581 from verult/NoPath-FlexDirExec

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Updated Flexvolume setup mechanisms for COS instance image.

- If REMOUNT_VOLUME_PLUGIN_DIR is set to true, VOLUME_PLUGIN_DIR is remounted with `exec` option during cluster startup. This allows any writable location to be used as the plugin directory.
- New HostPath added to controller-manager deployment to enable access to volume plugin directory.
- Improved how the default directory is passed to master and node setup.



**What this PR does / why we need it**: Part of the fix for #57353

/release-note-none
/sig storage
/assign @saad-ali @roberthbailey 
/cc @chakri-nelluri @wongma7

cluster/gce/gci/configure-helper.sh

@@ -1452,6 +1452,8 @@ function start-etcd-servers {
 #   CLOUD_CONFIG_VOLUME
 #   CLOUD_CONFIG_MOUNT
 #   DOCKER_REGISTRY
+#   FLEXVOLUME_HOSTPATH_MOUNT
+#   FLEXVOLUME_HOSTPATH_VOLUME
 function compute-master-manifest-variables {
   CLOUD_CONFIG_OPT=""
   CLOUD_CONFIG_VOLUME=""
@@ -1465,6 +1467,13 @@ function compute-master-manifest-variables {
   if [[ -n "${KUBE_DOCKER_REGISTRY:-}" ]]; then
     DOCKER_REGISTRY="${KUBE_DOCKER_REGISTRY}"
   fi
+
+  FLEXVOLUME_HOSTPATH_MOUNT=""
+  FLEXVOLUME_HOSTPATH_VOLUME=""
+  if [[ -n "${VOLUME_PLUGIN_DIR:-}" ]]; then
+    FLEXVOLUME_HOSTPATH_MOUNT="{ \"name\": \"flexvolumedir\", \"mountPath\": \"${VOLUME_PLUGIN_DIR}\", \"readOnly\": true},"
+    FLEXVOLUME_HOSTPATH_VOLUME="{ \"name\": \"flexvolumedir\", \"hostPath\": {\"path\": \"${VOLUME_PLUGIN_DIR}\"}},"
+  fi
 }
 
 # A helper function that bind mounts kubelet dirs for running mount in a chroot
@@ -1867,6 +1876,9 @@ function start-kube-controller-manager {
   sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}"
   sed -i -e "s@{{pv_recycler_mount}}@${PV_RECYCLER_MOUNT}@g" "${src_file}"
   sed -i -e "s@{{pv_recycler_volume}}@${PV_RECYCLER_VOLUME}@g" "${src_file}"
+  sed -i -e "s@{{flexvolume_hostpath_mount}}@${FLEXVOLUME_HOSTPATH_MOUNT}@g" "${src_file}"
+  sed -i -e "s@{{flexvolume_hostpath}}@${FLEXVOLUME_HOSTPATH_VOLUME}@g" "${src_file}"
+
   cp "${src_file}" /etc/kubernetes/manifests
 }
 

cluster/gce/gci/configure.sh

@@ -138,6 +138,13 @@ function split-commas {
   echo $1 | tr "," "\n"
 }
 
+function remount-flexvolume-directory {
+  local -r flexvolume_plugin_dir=$1
+  mkdir -p $flexvolume_plugin_dir
+  mount --bind $flexvolume_plugin_dir $flexvolume_plugin_dir
+  mount -o remount,exec $flexvolume_plugin_dir
+}
+
 function install-gci-mounter-tools {
   CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
   local -r mounter_tar_sha="${DEFAULT_MOUNTER_TAR_SHA}"
@@ -336,6 +343,11 @@ function install-kube-binary-config {
   # Install gci mounter related artifacts to allow mounting storage volumes in GCI
   install-gci-mounter-tools
 
+  # Remount the Flexvolume directory with the "exec" option, if needed.
+  if [[ "${REMOUNT_VOLUME_PLUGIN_DIR:-}" == "true" && -n "${VOLUME_PLUGIN_DIR:-}" ]]; then
+    remount-flexvolume-directory "${VOLUME_PLUGIN_DIR}"
+  fi
+
   # Clean up.
   rm -rf "${KUBE_HOME}/kubernetes"
   rm -f "${KUBE_HOME}/${server_binary_tar}"

cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest

@@ -37,6 +37,7 @@
 {% set pv_recycler_mount = "" -%}
 {% set pv_recycler_volume = "" -%}
 {% set srv_kube_path = "/srv/kubernetes" -%}
+{% flex_vol_plugin_dir = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec" -%}
 
 {% if grains.cloud is defined -%}
   {% if grains.cloud not in ['vagrant', 'photon-controller', 'azure-legacy'] -%}
@@ -137,6 +138,7 @@
         { "name": "srvkube",
         "mountPath": "{{srv_kube_path}}",
         "readOnly": true},
+        {{flexvolume_hostpath_mount}}
         { "name": "logfile",
         "mountPath": "/var/log/kube-controller-manager.log",
         "readOnly": false},
@@ -166,6 +168,7 @@
     "hostPath": {
         "path": "{{srv_kube_path}}"}
   },
+  {{flexvolume_hostpath}}
   { "name": "logfile",
     "hostPath": {
         "path": "/var/log/kube-controller-manager.log",