Merge pull request #51069 from MrHohn/kube-proxy-pod-priority

Automatic merge from submit-queue (batch tested with PRs 51553, 51538, 51663, 51069, 51737)

Allow enable pod priority feature gate for GCE and configure priority for kube-proxy

**What this PR does / why we need it**:
From #23225, this PR adds an option for user to enable pod priority feature gate using GCE startup scripts, and configure pod priority for kube-proxy when enabled.

The setup `priorityClassName: system` derives from: ce1485c626/staging/src/k8s.io/api/core/v1/types.go (L2536-L2542)

The plan is to configure pod priority for kube-proxy daemonset (https://github.com/kubernetes/kubernetes/pull/50705) in the same way.

**Special notes for your reviewer**:
cc @bsalamat @davidopp @thockin 

**Release note**:

```release-note
When using kube-up.sh on GCE, user could set env `ENABLE_POD_PRIORITY=true` to enable pod priority feature gate.
```

cluster/addons/kube-proxy/kube-proxy-ds.yaml

@@ -24,6 +24,7 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+      {{pod_priority}}
       hostNetwork: true
       nodeSelector:
         beta.kubernetes.io/kube-proxy-ds-ready: "true"

cluster/common.sh

@@ -672,6 +672,7 @@ GCE_API_ENDPOINT: $(yaml-quote ${GCE_API_ENDPOINT:-})
 PROMETHEUS_TO_SD_ENDPOINT: $(yaml-quote ${PROMETHEUS_TO_SD_ENDPOINT:-})
 PROMETHEUS_TO_SD_PREFIX: $(yaml-quote ${PROMETHEUS_TO_SD_PREFIX:-})
 ENABLE_PROMETHEUS_TO_SD: $(yaml-quote ${ENABLE_PROMETHEUS_TO_SD:-false})
+ENABLE_POD_PRIORITY: $(yaml-quote ${ENABLE_POD_PRIORITY:-})
 EOF
   if [ -n "${KUBELET_PORT:-}" ]; then
     cat >>$file <<EOF

cluster/gce/config-default.sh

@@ -301,3 +301,9 @@ ENABLE_PROMETHEUS_TO_SD="${ENABLE_PROMETHEUS_TO_SD:-false}"
 # TODO(#51292): Make kube-proxy Daemonset default and remove the configuration here.
 # Optional: Run kube-proxy as a DaemonSet if set to true, run as static pods otherwise.
 KUBE_PROXY_DAEMONSET="${KUBE_PROXY_DAEMONSET:-false}" # true, false
+
+# Optional: enable pod priority
+ENABLE_POD_PRIORITY="${ENABLE_POD_PRIORITY:-}"
+if [[ "${ENABLE_POD_PRIORITY}" == "true" ]]; then
+    FEATURE_GATES="${FEATURE_GATES},PodPriority=true"
+fi

cluster/gce/config-test.sh

@@ -360,3 +360,9 @@ ENABLE_PROMETHEUS_TO_SD="${ENABLE_PROMETHEUS_TO_SD:-true}"
 # TODO(#51292): Make kube-proxy Daemonset default and remove the configuration here.
 # Optional: Run kube-proxy as a DaemonSet if set to true, run as static pods otherwise.
 KUBE_PROXY_DAEMONSET="${KUBE_PROXY_DAEMONSET:-false}" # true, false
+
+# Optional: enable pod priority
+ENABLE_POD_PRIORITY="${ENABLE_POD_PRIORITY:-}"
+if [[ "${ENABLE_POD_PRIORITY}" == "true" ]]; then
+    FEATURE_GATES="${FEATURE_GATES},PodPriority=true"
+fi

cluster/gce/configure-vm.sh

@@ -447,6 +447,7 @@ initial_etcd_cluster: '$(echo "${INITIAL_ETCD_CLUSTER:-}" | sed -e "s/'/''/g")'
 initial_etcd_cluster_state: '$(echo "${INITIAL_ETCD_CLUSTER_STATE:-}" | sed -e "s/'/''/g")'
 ca_cert_bundle_path: '$(echo "${CA_CERT_BUNDLE_PATH:-}" | sed -e "s/'/''/g")'
 hostname: $(hostname -s)
+enable_pod_priority: '$(echo "$ENABLE_POD_PRIORITY" | sed -e "s/'/''/g")'
 enable_default_storage_class: '$(echo "$ENABLE_DEFAULT_STORAGE_CLASS" | sed -e "s/'/''/g")'
 kube_proxy_daemonset: '$(echo "$KUBE_PROXY_DAEMONSET" | sed -e "s/'/''/g")'
 EOF

cluster/gce/container-linux/configure-helper.sh

@@ -735,6 +735,10 @@ function prepare-kube-proxy-manifest-variables {
     kube_cache_mutation_detector_env_name="- name: KUBE_CACHE_MUTATION_DETECTOR"
     kube_cache_mutation_detector_env_value="value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\""
   fi
+  local pod_priority=""
+  if [[ "${ENABLE_POD_PRIORITY}" == "true" ]]; then
+    pod_priority="priorityClassName: system-node-critical"
+  fi
   sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" ${src_file}
   sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" ${src_file}
   sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" ${src_file}
@@ -742,6 +746,7 @@ function prepare-kube-proxy-manifest-variables {
   sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file}
   sed -i -e "s@{{kube_cache_mutation_detector_env_name}}@${kube_cache_mutation_detector_env_name}@g" ${src_file}
   sed -i -e "s@{{kube_cache_mutation_detector_env_value}}@${kube_cache_mutation_detector_env_value}@g" ${src_file}
+  sed -i -e "s@{{pod_priority}}@${pod_priority}@g" ${src_file}
   sed -i -e "s@{{ cpurequest }}@100m@g" ${src_file}
   sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${src_file}
   if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then

cluster/gce/gci/configure-helper.sh

@@ -1071,6 +1071,10 @@ function prepare-kube-proxy-manifest-variables {
     kube_cache_mutation_detector_env_name="- name: KUBE_CACHE_MUTATION_DETECTOR"
     kube_cache_mutation_detector_env_value="value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\""
   fi
+  local pod_priority=""
+  if [[ "${ENABLE_POD_PRIORITY}" == "true" ]]; then
+    pod_priority="priorityClassName: system-node-critical"
+  fi
   sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" ${src_file}
   sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" ${src_file}
   sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" ${src_file}
@@ -1078,6 +1082,7 @@ function prepare-kube-proxy-manifest-variables {
   sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file}
   sed -i -e "s@{{kube_cache_mutation_detector_env_name}}@${kube_cache_mutation_detector_env_name}@g" ${src_file}
   sed -i -e "s@{{kube_cache_mutation_detector_env_value}}@${kube_cache_mutation_detector_env_value}@g" ${src_file}
+  sed -i -e "s@{{pod_priority}}@${pod_priority}@g" ${src_file}
   sed -i -e "s@{{ cpurequest }}@100m@g" ${src_file}
   sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${src_file}
   if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then

cluster/saltbase/salt/kube-proxy/kube-proxy.manifest

@@ -34,6 +34,11 @@
 
 {% set throttles = "--iptables-sync-period=1m --iptables-min-sync-period=10s" -%}
 
+{% set pod_priority = "" -%}
+{% if pillar.get('enable_pod_priority', '').lower() == 'true' -%}
+  {% set pod_priority = "priorityClassName: system-node-critical" -%}
+{% endif -%}
+
 # test_args should always go last to overwrite prior configuration
 {% set params = log_level + " " + throttles + " " + feature_gates + " " + test_args -%}
 
@@ -58,6 +63,7 @@ metadata:
     tier: node
     component: kube-proxy
 spec:
+  {{pod_priority}}
   hostNetwork: true
   initContainers:
   - name: touch-lock