Merge pull request #41132 from kubernetes/revert-40893-kubelet-auth

Revert "remove second CA used for kubelet auth in favor of webhook auth"

cluster/addons/rbac/apiserver-node-proxy-binding.yaml

@@ -1,14 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRoleBinding
-metadata:
-  name: apiserver-node-proxy
-  labels:
-    kubernetes.io/cluster-service: "true"
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: node-proxy
-subjects:
-- apiVersion: rbac/v1beta1
-  kind: User
-  name: kube-apiserver

cluster/addons/rbac/node-proxy-role.yaml

@@ -1,23 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRole
-metadata:
-  name: node-proxy
-  labels:
-    kubernetes.io/cluster-service: "true"
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - nodes/proxy
-  verbs:
-  - create
-  - get
-- apiGroups:
-  - ""
-  resources:
-  - nodes/log
-  - nodes/stats
-  - nodes/metrics
-  - nodes/spec
-  verbs:
-  - get

cluster/common.sh

@@ -585,6 +585,7 @@ function build-kube-master-certs {
   cat >$file <<EOF
 KUBEAPISERVER_CERT: $(yaml-quote ${KUBEAPISERVER_CERT_BASE64:-})
 KUBEAPISERVER_KEY: $(yaml-quote ${KUBEAPISERVER_KEY_BASE64:-})
+KUBELET_AUTH_CA_CERT: $(yaml-quote ${KUBELET_AUTH_CA_CERT_BASE64:-})
 CA_KEY: $(yaml-quote ${CA_KEY_BASE64:-})
 EOF
 }
@@ -801,6 +802,7 @@ EOF
 KUBERNETES_MASTER: $(yaml-quote "false")
 ZONE: $(yaml-quote ${ZONE})
 EXTRA_DOCKER_OPTS: $(yaml-quote ${EXTRA_DOCKER_OPTS:-})
+KUBELET_AUTH_CA_CERT: $(yaml-quote ${KUBELET_AUTH_CA_CERT_BASE64:-})
 EOF
     if [ -n "${KUBEPROXY_TEST_ARGS:-}" ]; then
       cat >>$file <<EOF
@@ -968,8 +970,9 @@ function create-certs {
   KUBELET_KEY_BASE64=$(cat "${CERT_DIR}/pki/private/kubelet.key" | base64 | tr -d '\r\n')
   KUBECFG_CERT_BASE64=$(cat "${CERT_DIR}/pki/issued/kubecfg.crt" | base64 | tr -d '\r\n')
   KUBECFG_KEY_BASE64=$(cat "${CERT_DIR}/pki/private/kubecfg.key" | base64 | tr -d '\r\n')
-  KUBEAPISERVER_CERT_BASE64=$(cat "${CERT_DIR}/pki/issued/kube-apiserver.crt" | base64 | tr -d '\r\n')
-  KUBEAPISERVER_KEY_BASE64=$(cat "${CERT_DIR}/pki/private/kube-apiserver.key" | base64 | tr -d '\r\n')
+  KUBELET_AUTH_CA_CERT_BASE64=$(cat "${KUBE_TEMP}/easy-rsa-master/kubelet/pki/ca.crt" | base64 | tr -d '\r\n')
+  KUBEAPISERVER_CERT_BASE64=$(cat "${KUBE_TEMP}/easy-rsa-master/kubelet/pki/issued/kube-apiserver.crt" | base64 | tr -d '\r\n')
+  KUBEAPISERVER_KEY_BASE64=$(cat "${KUBE_TEMP}/easy-rsa-master/kubelet/pki/private/kube-apiserver.key" | base64 | tr -d '\r\n')
 }
 
 # Runs the easy RSA commands to generate certificate files.
@@ -996,7 +999,6 @@ function generate-certs {
     # this puts the cert into pki/ca.crt and the key into pki/private/ca.key
     ./easyrsa --batch "--req-cn=${PRIMARY_CN}@$(date +%s)" build-ca nopass
     ./easyrsa --subject-alt-name="${SANS}" build-server-full "${MASTER_NAME}" nopass
-    ./easyrsa build-client-full kube-apiserver nopass
 
     download-cfssl
 
@@ -1012,7 +1014,12 @@ function generate-certs {
     ./easyrsa --dn-mode=org \
       --req-cn=kubecfg --req-org=system:masters \
       --req-c= --req-st= --req-city= --req-email= --req-ou= \
-      build-client-full kubecfg nopass) &>${cert_create_debug_output} || {
+      build-client-full kubecfg nopass
+
+    cd ../kubelet
+    ./easyrsa init-pki
+    ./easyrsa --batch "--req-cn=kubelet@$(date +%s)" build-ca nopass
+    ./easyrsa build-client-full kube-apiserver nopass) &>${cert_create_debug_output} || {
     # If there was an error in the subshell, just die.
     # TODO(roberthbailey): add better error handling here
     cat "${cert_create_debug_output}" >&2

cluster/gce/configure-vm.sh

@@ -630,6 +630,11 @@ EOF
     if [ -n "${SCHEDULING_ALGORITHM_PROVIDER:-}" ]; then
       cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls
 scheduling_algorithm_provider: '$(echo "${SCHEDULING_ALGORITHM_PROVIDER}" | sed -e "s/'/''/g")'
+EOF
+    fi
+    if [ -n "${KUBELET_AUTH_CA_CERT:-}" ]; then
+      cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls
+kubelet_auth_ca_cert: /var/lib/kubelet/kubelet_auth_ca.crt
 EOF
     fi
 }
@@ -750,9 +755,11 @@ current-context: service-account-context
 EOF
 )
   fi
-  local -r client_ca_file="/srv/salt-overlay/salt/kubelet/ca.crt"
-  (umask 077;
-    echo "${KUBELET_CA_CERT}" | base64 --decode > "${client_ca_file}")
+  local -r kubelet_auth_ca_file="/srv/salt-overlay/salt/kubelet/kubelet_auth_ca.crt"
+  if [ ! -e "${kubelet_auth_ca_file}" ] && [[ ! -z "${KUBELET_AUTH_CA_CERT:-}" ]]; then
+    (umask 077;
+      echo "${KUBELET_AUTH_CA_CERT}" | base64 --decode > "${kubelet_auth_ca_file}")
+  fi
 }
 
 # This should happen both on cluster initialization and node upgrades.

cluster/gce/gci/configure-helper.sh

@@ -369,7 +369,12 @@ contexts:
   name: service-account-context
 current-context: service-account-context
 EOF
-  echo "${KUBELET_CA_CERT}" | base64 -d > /var/lib/kubelet/ca.crt
+}
+
+function create-kubelet-auth-ca {
+  if [[ -n "${KUBELET_AUTH_CA_CERT:-}" ]]; then
+    echo "${KUBELET_AUTH_CA_CERT}" | base64 --decode > "/var/lib/kubelet/kubelet_auth_ca.crt"
+  fi
 }
 
 # Uses KUBELET_CA_CERT (falling back to CA_CERT), KUBELET_CERT, and KUBELET_KEY
@@ -383,6 +388,7 @@ function create-master-kubelet-auth {
     REGISTER_MASTER_KUBELET="true"
     create-kubelet-kubeconfig
   fi
+  
 }
 
 function create-kubeproxy-kubeconfig {
@@ -576,7 +582,9 @@ function start-kubelet {
        [[ "${HAIRPIN_MODE:-}" == "none" ]]; then
       flags+=" --hairpin-mode=${HAIRPIN_MODE}"
     fi
-    flags+=" --anonymous-auth=false --authorization-mode=Webhook --client-ca-file=/var/lib/kubelet/ca.crt"
+    if [ -n "${KUBELET_AUTH_CA_CERT:-}" ]; then
+      flags+=" --anonymous-auth=false --client-ca-file=/var/lib/kubelet/kubelet_auth_ca.crt"
+    fi
   fi
   # Network plugin
   if [[ -n "${NETWORK_PROVIDER:-}" ]]; then
@@ -821,10 +829,8 @@ function start-kube-apiserver {
   params+=" --secure-port=443"
   params+=" --tls-cert-file=/etc/srv/kubernetes/server.cert"
   params+=" --tls-private-key-file=/etc/srv/kubernetes/server.key"
-  if [[ -e /etc/srv/kubernetes/kubeapiserver.cert ]] && [[ -e /etc/srv/kubernetes/kubeapiserver.key ]]; then
-    params+=" --kubelet-client-certificate=/etc/srv/kubernetes/kubeapiserver.cert"
-    params+=" --kubelet-client-key=/etc/srv/kubernetes/kubeapiserver.key"
-  fi
+  params+=" --kubelet-client-certificate=/etc/srv/kubernetes/kubeapiserver.cert"
+  params+=" --kubelet-client-key=/etc/srv/kubernetes/kubeapiserver.key"
   params+=" --token-auth-file=/etc/srv/kubernetes/known_tokens.csv"
   if [[ -n "${KUBE_PASSWORD:-}" && -n "${KUBE_USER:-}" ]]; then
     params+=" --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv"
@@ -1093,13 +1099,9 @@ function start-kube-addons {
   local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty"
   local -r dst_dir="/etc/kubernetes/addons"
 
-  # TODO(mikedanese): only enable these in e2e
   # prep the additional bindings that are particular to e2e users and groups
   setup-addon-manifests "addons" "e2e-rbac-bindings"
 
-  # prep addition kube-up specific rbac objects
-  setup-addon-manifests "addons" "rbac"
-
   # Set up manifests of other addons.
   if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "influxdb" ]] || \
      [[ "${ENABLE_CLUSTER_MONITORING:-}" == "google" ]] || \
@@ -1343,6 +1345,7 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
   create-master-etcd-auth
 else
   create-kubelet-kubeconfig
+  create-kubelet-auth-ca
   create-kubeproxy-kubeconfig
 fi
 

cluster/gce/upgrade.sh

@@ -233,6 +233,9 @@ function prepare-node-upgrade() {
   KUBELET_CERT_BASE64=$(get-env-val "${node_env}" "KUBELET_CERT")
   KUBELET_KEY_BASE64=$(get-env-val "${node_env}" "KUBELET_KEY")
 
+  local master_env=$(get-master-env)
+  KUBELET_AUTH_CA_CERT_BASE64=$(get-env-val "${master_env}" "KUBELET_AUTH_CA_CERT")
+
   # TODO(zmerlynn): How do we ensure kube-env is written in a ${version}-
   #                 compatible way?
   write-node-env

cluster/saltbase/salt/kubelet/default

@@ -188,7 +188,10 @@
   {% set eviction_hard="--eviction-hard=" + pillar['eviction_hard'] %}
 {% endif -%}
 
-{% set kubelet_auth = "--anonymous-auth=false --authorization-mode=Webhook --client-ca-file=/var/lib/kubelet/ca.crt" %}
+{% set kubelet_auth_ca_cert = "" %}
+{% if pillar['kubelet_auth_ca_cert'] is defined -%}
+  {% set kubelet_auth_ca_cert="--anonymous-auth=false --client-ca-file=" + pillar['kubelet_auth_ca_cert'] %}
+{% endif -%}
 
 # test_args has to be kept at the end, so they'll overwrite any prior configuration
-DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{cloud_config}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}}  {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{babysit_daemons}} {{eviction_hard}} {{kubelet_auth}} {{feature_gates}} {{test_args}}"
+DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{cloud_config}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}}  {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{babysit_daemons}} {{eviction_hard}} {{kubelet_auth_ca_cert}} {{feature_gates}} {{test_args}}"

cluster/saltbase/salt/kubelet/init.sls

@@ -31,13 +31,15 @@
     - mode: 400
     - makedirs: true
 
-/var/lib/kubelet/ca.crt:
+{% if pillar['kubelet_auth_ca_cert'] is defined %}
+/var/lib/kubelet/kubelet_auth_ca.crt:
   file.managed:
-    - source: salt://kubelet/ca.crt
+    - source: salt://kubelet/kubelet_auth_ca.crt
     - user: root
     - group: root
     - mode: 400
     - makedirs: true
+{% endif %}
 
 {% if pillar.get('is_systemd') %}
 
@@ -59,7 +61,7 @@ fix-service-kubelet:
       - file: {{ pillar.get('systemd_system_path') }}/kubelet.service
       - file: {{ environment_file }}
       - file: /var/lib/kubelet/kubeconfig
-      - file: /var/lib/kubelet/ca.crt
+      - file: /var/lib/kubelet/kubelet_auth_ca.crt
 
 {% else %}
 
@@ -87,7 +89,9 @@ kubelet:
 {% endif %}
       - file: {{ environment_file }}
       - file: /var/lib/kubelet/kubeconfig
-      - file: /var/lib/kubelet/ca.crt
+{% if pillar['kubelet_auth_ca_cert'] is defined  %}
+      - file: /var/lib/kubelet/kubelet_auth_ca.crt
+{% endif %}
 {% if pillar.get('is_systemd') %}
     - provider:
       - service: systemd

hack/verify-flags/exceptions.txt

@@ -14,7 +14,6 @@ cluster/gce/configure-vm.sh:  cloud_config: ${CLOUD_CONFIG}
 cluster/gce/configure-vm.sh:  env-to-grains "feature_gates"
 cluster/gce/configure-vm.sh:  env-to-grains "runtime_config"
 cluster/gce/configure-vm.sh:  kubelet_api_servers: '${KUBELET_APISERVER}'
-cluster/gce/configure-vm.sh:  local -r client_ca_file="/srv/salt-overlay/salt/kubelet/ca.crt"
 cluster/gce/container-linux/configure-helper.sh:    authorization_mode+=",ABAC"
 cluster/gce/container-linux/configure-helper.sh:    authorization_mode+=",Webhook"
 cluster/gce/container-linux/configure-helper.sh:  local api_servers="--master=https://${KUBERNETES_MASTER_NAME}"