From 1841e5b2e0de0b5e3ff8b7e1fe46e93a47bacde9 Mon Sep 17 00:00:00 2001 From: Aleksandra Malinowska Date: Wed, 8 Feb 2017 13:22:10 +0100 Subject: [PATCH] Revert "remove second CA used for kubelet auth in favor of webhook auth" --- .../rbac/apiserver-node-proxy-binding.yaml | 14 ----------- cluster/addons/rbac/node-proxy-role.yaml | 23 ------------------- cluster/common.sh | 15 ++++++++---- cluster/gce/configure-vm.sh | 13 ++++++++--- cluster/gce/gci/configure-helper.sh | 23 +++++++++++-------- cluster/gce/upgrade.sh | 3 +++ cluster/saltbase/salt/kubelet/default | 7 ++++-- cluster/saltbase/salt/kubelet/init.sls | 12 ++++++---- hack/verify-flags/exceptions.txt | 1 - 9 files changed, 50 insertions(+), 61 deletions(-) delete mode 100644 cluster/addons/rbac/apiserver-node-proxy-binding.yaml delete mode 100644 cluster/addons/rbac/node-proxy-role.yaml diff --git a/cluster/addons/rbac/apiserver-node-proxy-binding.yaml b/cluster/addons/rbac/apiserver-node-proxy-binding.yaml deleted file mode 100644 index 1dac4e4c58..0000000000 --- a/cluster/addons/rbac/apiserver-node-proxy-binding.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: apiserver-node-proxy - labels: - kubernetes.io/cluster-service: "true" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: node-proxy -subjects: -- apiVersion: rbac/v1beta1 - kind: User - name: kube-apiserver diff --git a/cluster/addons/rbac/node-proxy-role.yaml b/cluster/addons/rbac/node-proxy-role.yaml deleted file mode 100644 index d3684c9933..0000000000 --- a/cluster/addons/rbac/node-proxy-role.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: node-proxy - labels: - kubernetes.io/cluster-service: "true" -rules: -- apiGroups: - - "" - resources: - - nodes/proxy - verbs: - - create - - get -- apiGroups: - - "" - resources: - - nodes/log - - nodes/stats - - nodes/metrics - - nodes/spec - verbs: - - get diff --git a/cluster/common.sh b/cluster/common.sh index 6c64946399..39cb0f0e21 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -585,6 +585,7 @@ function build-kube-master-certs { cat >$file <>$file <${cert_create_debug_output} || { + build-client-full kubecfg nopass + + cd ../kubelet + ./easyrsa init-pki + ./easyrsa --batch "--req-cn=kubelet@$(date +%s)" build-ca nopass + ./easyrsa build-client-full kube-apiserver nopass) &>${cert_create_debug_output} || { # If there was an error in the subshell, just die. # TODO(roberthbailey): add better error handling here cat "${cert_create_debug_output}" >&2 diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index 542f105445..26e3b802fe 100755 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -630,6 +630,11 @@ EOF if [ -n "${SCHEDULING_ALGORITHM_PROVIDER:-}" ]; then cat <>/srv/salt-overlay/pillar/cluster-params.sls scheduling_algorithm_provider: '$(echo "${SCHEDULING_ALGORITHM_PROVIDER}" | sed -e "s/'/''/g")' +EOF + fi + if [ -n "${KUBELET_AUTH_CA_CERT:-}" ]; then + cat <>/srv/salt-overlay/pillar/cluster-params.sls +kubelet_auth_ca_cert: /var/lib/kubelet/kubelet_auth_ca.crt EOF fi } @@ -750,9 +755,11 @@ current-context: service-account-context EOF ) fi - local -r client_ca_file="/srv/salt-overlay/salt/kubelet/ca.crt" - (umask 077; - echo "${KUBELET_CA_CERT}" | base64 --decode > "${client_ca_file}") + local -r kubelet_auth_ca_file="/srv/salt-overlay/salt/kubelet/kubelet_auth_ca.crt" + if [ ! -e "${kubelet_auth_ca_file}" ] && [[ ! -z "${KUBELET_AUTH_CA_CERT:-}" ]]; then + (umask 077; + echo "${KUBELET_AUTH_CA_CERT}" | base64 --decode > "${kubelet_auth_ca_file}") + fi } # This should happen both on cluster initialization and node upgrades. diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 3bd19735dc..af7d732372 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -369,7 +369,12 @@ contexts: name: service-account-context current-context: service-account-context EOF - echo "${KUBELET_CA_CERT}" | base64 -d > /var/lib/kubelet/ca.crt +} + +function create-kubelet-auth-ca { + if [[ -n "${KUBELET_AUTH_CA_CERT:-}" ]]; then + echo "${KUBELET_AUTH_CA_CERT}" | base64 --decode > "/var/lib/kubelet/kubelet_auth_ca.crt" + fi } # Uses KUBELET_CA_CERT (falling back to CA_CERT), KUBELET_CERT, and KUBELET_KEY @@ -383,6 +388,7 @@ function create-master-kubelet-auth { REGISTER_MASTER_KUBELET="true" create-kubelet-kubeconfig fi + } function create-kubeproxy-kubeconfig { @@ -576,7 +582,9 @@ function start-kubelet { [[ "${HAIRPIN_MODE:-}" == "none" ]]; then flags+=" --hairpin-mode=${HAIRPIN_MODE}" fi - flags+=" --anonymous-auth=false --authorization-mode=Webhook --client-ca-file=/var/lib/kubelet/ca.crt" + if [ -n "${KUBELET_AUTH_CA_CERT:-}" ]; then + flags+=" --anonymous-auth=false --client-ca-file=/var/lib/kubelet/kubelet_auth_ca.crt" + fi fi # Network plugin if [[ -n "${NETWORK_PROVIDER:-}" ]]; then @@ -821,10 +829,8 @@ function start-kube-apiserver { params+=" --secure-port=443" params+=" --tls-cert-file=/etc/srv/kubernetes/server.cert" params+=" --tls-private-key-file=/etc/srv/kubernetes/server.key" - if [[ -e /etc/srv/kubernetes/kubeapiserver.cert ]] && [[ -e /etc/srv/kubernetes/kubeapiserver.key ]]; then - params+=" --kubelet-client-certificate=/etc/srv/kubernetes/kubeapiserver.cert" - params+=" --kubelet-client-key=/etc/srv/kubernetes/kubeapiserver.key" - fi + params+=" --kubelet-client-certificate=/etc/srv/kubernetes/kubeapiserver.cert" + params+=" --kubelet-client-key=/etc/srv/kubernetes/kubeapiserver.key" params+=" --token-auth-file=/etc/srv/kubernetes/known_tokens.csv" if [[ -n "${KUBE_PASSWORD:-}" && -n "${KUBE_USER:-}" ]]; then params+=" --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv" @@ -1093,13 +1099,9 @@ function start-kube-addons { local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty" local -r dst_dir="/etc/kubernetes/addons" - # TODO(mikedanese): only enable these in e2e # prep the additional bindings that are particular to e2e users and groups setup-addon-manifests "addons" "e2e-rbac-bindings" - # prep addition kube-up specific rbac objects - setup-addon-manifests "addons" "rbac" - # Set up manifests of other addons. if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "influxdb" ]] || \ [[ "${ENABLE_CLUSTER_MONITORING:-}" == "google" ]] || \ @@ -1343,6 +1345,7 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then create-master-etcd-auth else create-kubelet-kubeconfig + create-kubelet-auth-ca create-kubeproxy-kubeconfig fi diff --git a/cluster/gce/upgrade.sh b/cluster/gce/upgrade.sh index a3c8e77bdd..83b8dadc1c 100755 --- a/cluster/gce/upgrade.sh +++ b/cluster/gce/upgrade.sh @@ -233,6 +233,9 @@ function prepare-node-upgrade() { KUBELET_CERT_BASE64=$(get-env-val "${node_env}" "KUBELET_CERT") KUBELET_KEY_BASE64=$(get-env-val "${node_env}" "KUBELET_KEY") + local master_env=$(get-master-env) + KUBELET_AUTH_CA_CERT_BASE64=$(get-env-val "${master_env}" "KUBELET_AUTH_CA_CERT") + # TODO(zmerlynn): How do we ensure kube-env is written in a ${version}- # compatible way? write-node-env diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default index 167aaf4783..8aa9c03901 100644 --- a/cluster/saltbase/salt/kubelet/default +++ b/cluster/saltbase/salt/kubelet/default @@ -188,7 +188,10 @@ {% set eviction_hard="--eviction-hard=" + pillar['eviction_hard'] %} {% endif -%} -{% set kubelet_auth = "--anonymous-auth=false --authorization-mode=Webhook --client-ca-file=/var/lib/kubelet/ca.crt" %} +{% set kubelet_auth_ca_cert = "" %} +{% if pillar['kubelet_auth_ca_cert'] is defined -%} + {% set kubelet_auth_ca_cert="--anonymous-auth=false --client-ca-file=" + pillar['kubelet_auth_ca_cert'] %} +{% endif -%} # test_args has to be kept at the end, so they'll overwrite any prior configuration -DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{cloud_config}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{babysit_daemons}} {{eviction_hard}} {{kubelet_auth}} {{feature_gates}} {{test_args}}" +DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{cloud_config}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{babysit_daemons}} {{eviction_hard}} {{kubelet_auth_ca_cert}} {{feature_gates}} {{test_args}}" diff --git a/cluster/saltbase/salt/kubelet/init.sls b/cluster/saltbase/salt/kubelet/init.sls index eb31fd6f13..f5b3eee14d 100644 --- a/cluster/saltbase/salt/kubelet/init.sls +++ b/cluster/saltbase/salt/kubelet/init.sls @@ -31,13 +31,15 @@ - mode: 400 - makedirs: true -/var/lib/kubelet/ca.crt: +{% if pillar['kubelet_auth_ca_cert'] is defined %} +/var/lib/kubelet/kubelet_auth_ca.crt: file.managed: - - source: salt://kubelet/ca.crt + - source: salt://kubelet/kubelet_auth_ca.crt - user: root - group: root - mode: 400 - makedirs: true +{% endif %} {% if pillar.get('is_systemd') %} @@ -59,7 +61,7 @@ fix-service-kubelet: - file: {{ pillar.get('systemd_system_path') }}/kubelet.service - file: {{ environment_file }} - file: /var/lib/kubelet/kubeconfig - - file: /var/lib/kubelet/ca.crt + - file: /var/lib/kubelet/kubelet_auth_ca.crt {% else %} @@ -87,7 +89,9 @@ kubelet: {% endif %} - file: {{ environment_file }} - file: /var/lib/kubelet/kubeconfig - - file: /var/lib/kubelet/ca.crt +{% if pillar['kubelet_auth_ca_cert'] is defined %} + - file: /var/lib/kubelet/kubelet_auth_ca.crt +{% endif %} {% if pillar.get('is_systemd') %} - provider: - service: systemd diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index 8a79e23033..600b2ed24d 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -14,7 +14,6 @@ cluster/gce/configure-vm.sh: cloud_config: ${CLOUD_CONFIG} cluster/gce/configure-vm.sh: env-to-grains "feature_gates" cluster/gce/configure-vm.sh: env-to-grains "runtime_config" cluster/gce/configure-vm.sh: kubelet_api_servers: '${KUBELET_APISERVER}' -cluster/gce/configure-vm.sh: local -r client_ca_file="/srv/salt-overlay/salt/kubelet/ca.crt" cluster/gce/container-linux/configure-helper.sh: authorization_mode+=",ABAC" cluster/gce/container-linux/configure-helper.sh: authorization_mode+=",Webhook" cluster/gce/container-linux/configure-helper.sh: local api_servers="--master=https://${KUBERNETES_MASTER_NAME}"