From 8bd0b45eaed7c0b56af720d8d47d0f15c2e80cf5 Mon Sep 17 00:00:00 2001 From: Jake Sanders Date: Fri, 26 Apr 2019 13:16:17 -0700 Subject: [PATCH] use static token to authenticate glbc --- .../cluster-loadbalancing/glbc/roles.yaml | 55 +++++++++++++++++++ .../glbc/user-rolebindings.yaml | 28 ++++++++++ cluster/gce/gci/configure-helper.sh | 32 +++++++++++ cluster/gce/manifests/glbc.manifest | 8 ++- 4 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 cluster/addons/rbac/cluster-loadbalancing/glbc/roles.yaml create mode 100644 cluster/addons/rbac/cluster-loadbalancing/glbc/user-rolebindings.yaml diff --git a/cluster/addons/rbac/cluster-loadbalancing/glbc/roles.yaml b/cluster/addons/rbac/cluster-loadbalancing/glbc/roles.yaml new file mode 100644 index 0000000000..c14cc17e3f --- /dev/null +++ b/cluster/addons/rbac/cluster-loadbalancing/glbc/roles.yaml @@ -0,0 +1,55 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: system:controller:glbc + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "update", "create", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:controller:glbc + labels: + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] +- apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "update", "create", "patch"] +- apiGroups: [""] + resources: ["endpoints", "services", "pods", "nodes", "namespaces"] + verbs: ["get", "list", "watch"] +# TODO: switch to patch services/status +# https://github.com/kubernetes/ingress-gce/blob/4918eb2f0f484f09ac9e5a975907a9b16ed2b344/pkg/neg/controller.go#L339-L342 +# https://github.com/kubernetes/ingress-gce/blob/4918eb2f0f484f09ac9e5a975907a9b16ed2b344/pkg/neg/controller.go#L359-L361 +- apiGroups: [""] + resources: ["services"] + verbs: ["update", "patch"] +- apiGroups: ["extensions", "networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get", "list", "watch"] +# For now, GLBC annotates ingress resources with various state and statuses: +# https://github.com/kubernetes/ingress-gce/blob/50d49b077d9ab4362a02fae05f94e433cd3f08dc/pkg/controller/controller.go#L579 +# TODO(rramkumar1): Remove unnecessary `update` permission once statuses are propagated through `ingresses/status` +- apiGroups: ["extensions", "networking.k8s.io"] + resources: ["ingresses"] + verbs: ["update"] +- apiGroups: ["extensions", "networking.k8s.io"] + resources: ["ingresses/status"] + verbs: ["update"] +# GLBC ensures that the `cloud.google.com/backendconfigs` CRD exists in a desired state: +# https://github.com/kubernetes/ingress-gce/blob/4918eb2f0f484f09ac9e5a975907a9b16ed2b344/cmd/glbc/main.go#L93 +# TODO(rramkumar1): https://github.com/kubernetes/ingress-gce/issues/744 +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list", "watch", "update", "create", "patch"] +- apiGroups: ["cloud.google.com"] + resources: ["backendconfigs"] + verbs: ["get", "list", "watch", "update", "create", "patch"] diff --git a/cluster/addons/rbac/cluster-loadbalancing/glbc/user-rolebindings.yaml b/cluster/addons/rbac/cluster-loadbalancing/glbc/user-rolebindings.yaml new file mode 100644 index 0000000000..7196e2fedf --- /dev/null +++ b/cluster/addons/rbac/cluster-loadbalancing/glbc/user-rolebindings.yaml @@ -0,0 +1,28 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system:controller:glbc + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: system:controller:glbc +subjects: +- kind: User + name: system:controller:glbc +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:controller:glbc + labels: + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:controller:glbc +subjects: +- kind: User + name: system:controller:glbc diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index f16d156f8b..9a31610496 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -594,6 +594,9 @@ function create-master-auth { if [[ -n "${NODE_PROBLEM_DETECTOR_TOKEN:-}" ]]; then append_or_replace_prefixed_line "${known_tokens_csv}" "${NODE_PROBLEM_DETECTOR_TOKEN}," "system:node-problem-detector,uid:node-problem-detector" fi + if [[ -n "${GCE_GLBC_TOKEN:-}" ]]; then + append_or_replace_prefixed_line "${known_tokens_csv}" "${GCE_GLBC_TOKEN}," "system:controller:glbc,uid:system:controller:glbc" + fi local use_cloud_config="false" cat </etc/gce.conf [global] @@ -1048,6 +1051,30 @@ current-context: service-account-context EOF } +function create-l7-lb-controller-kubeconfig { + echo "Creating l7-lb-controller kubeconfig file" + mkdir -p /etc/srv/kubernetes/l7-lb-controller + cat </etc/srv/kubernetes/l7-lb-controller/kubeconfig +apiVersion: v1 +kind: Config +users: +- name: l7-lb-controller + user: + token: ${GCE_GLBC_TOKEN} +clusters: +- name: local + cluster: + insecure-skip-tls-verify: true + server: https://localhost:443 +contexts: +- context: + cluster: local + user: l7-lb-controller + name: l7-lb-controller +current-context: l7-lb-controller +EOF +} + function create-kubescheduler-kubeconfig { echo "Creating kube-scheduler kubeconfig file" mkdir -p /etc/srv/kubernetes/kube-scheduler @@ -2737,6 +2764,8 @@ function start-lb-controller { echo "Start GCE L7 pod" prepare-log-file /var/log/glbc.log setup-addon-manifests "addons" "cluster-loadbalancing/glbc" + setup-addon-manifests "addons" "rbac/cluster-loadbalancing/glbc" + create-l7-lb-controller-kubeconfig local -r src_manifest="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/glbc.manifest" local -r dest_manifest="/etc/kubernetes/manifests/glbc.manifest" @@ -2906,6 +2935,9 @@ function main() { KUBE_CONTROLLER_MANAGER_TOKEN="$(secure_random 32)" KUBE_SCHEDULER_TOKEN="$(secure_random 32)" KUBE_CLUSTER_AUTOSCALER_TOKEN="$(secure_random 32)" + if [[ "${ENABLE_L7_LOADBALANCING:-}" == "glbc" ]]; then + GCE_GLBC_TOKEN="$(secure_random 32)" + fi setup-os-params config-ip-firewall diff --git a/cluster/gce/manifests/glbc.manifest b/cluster/gce/manifests/glbc.manifest index d7643791c2..319037d0ef 100644 --- a/cluster/gce/manifests/glbc.manifest +++ b/cluster/gce/manifests/glbc.manifest @@ -34,6 +34,9 @@ spec: - mountPath: /var/log/glbc.log name: logfile readOnly: false + - name: srvkube + mountPath: /etc/srv/kubernetes/l7-lb-controller + readOnly: true resources: # Request is set to accommodate this pod alongside the other # master components on a single core master. @@ -45,7 +48,7 @@ spec: # TODO: split this out into args when we no longer need to pipe stdout to a file #6428 - sh - -c - - 'exec /glbc --gce-ratelimit=ga.Operations.Get,qps,10,100 --gce-ratelimit=alpha.Operations.Get,qps,10,100 --gce-ratelimit=ga.BackendServices.Get,qps,1.8,1 --gce-ratelimit=ga.HealthChecks.Get,qps,1.8,1 --gce-ratelimit=alpha.HealthChecks.Get,qps,1.8,1 --gce-ratelimit=beta.NetworkEndpointGroups.Get,qps,1.8,1 --gce-ratelimit=beta.NetworkEndpointGroups.AttachNetworkEndpoints,qps,1.8,1 --gce-ratelimit=beta.NetworkEndpointGroups.DetachNetworkEndpoints,qps,1.8,1 --gce-ratelimit=beta.NetworkEndpointGroups.ListNetworkEndpoints,qps,1.8,1 --verbose --apiserver-host=http://localhost:8080 --default-backend-service=kube-system/default-http-backend --sync-period=600s --running-in-cluster=false --use-real-cloud=true --config-file-path=/etc/gce.conf --healthz-port=8086 1>>/var/log/glbc.log 2>&1' + - 'exec /glbc --gce-ratelimit=ga.Operations.Get,qps,10,100 --gce-ratelimit=alpha.Operations.Get,qps,10,100 --gce-ratelimit=ga.BackendServices.Get,qps,1.8,1 --gce-ratelimit=ga.HealthChecks.Get,qps,1.8,1 --gce-ratelimit=alpha.HealthChecks.Get,qps,1.8,1 --gce-ratelimit=beta.NetworkEndpointGroups.Get,qps,1.8,1 --gce-ratelimit=beta.NetworkEndpointGroups.AttachNetworkEndpoints,qps,1.8,1 --gce-ratelimit=beta.NetworkEndpointGroups.DetachNetworkEndpoints,qps,1.8,1 --gce-ratelimit=beta.NetworkEndpointGroups.ListNetworkEndpoints,qps,1.8,1 --verbose --kubeconfig=/etc/srv/kubernetes/l7-lb-controller/kubeconfig --default-backend-service=kube-system/default-http-backend --sync-period=600s --running-in-cluster=false --use-real-cloud=true --config-file-path=/etc/gce.conf --healthz-port=8086 1>>/var/log/glbc.log 2>&1' volumes: - hostPath: path: /etc/gce.conf @@ -55,3 +58,6 @@ spec: path: /var/log/glbc.log type: FileOrCreate name: logfile + - name: srvkube + hostPath: + path: /etc/srv/kubernetes/l7-lb-controller