From 20d1818d78746247785f327b89028a35f5ca6caa Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Thu, 6 Oct 2016 15:59:14 -0400 Subject: [PATCH] Enable local-up-cluster.sh to start with the secured port Generate a kubeconfig for use with controller-manager, kubelet, scheduler etc. This kubeconfig should use the secure https port of the api server with appropriate ca cert for the components to talk to api server. With this change, one can set API_PORT=0 to completely switch off insecure access for testing admission controllers etc. Fixes #33375 --- hack/lib/util.sh | 2 +- hack/local-up-cluster.sh | 71 ++++++++++++++----- .../options/server_run_options.go | 8 ++- 3 files changed, 63 insertions(+), 18 deletions(-) diff --git a/hack/lib/util.sh b/hack/lib/util.sh index 1e4aba5881..eb35b36e06 100755 --- a/hack/lib/util.sh +++ b/hack/lib/util.sh @@ -32,7 +32,7 @@ kube::util::wait_for_url() { local i for i in $(seq 1 $times); do local out - if out=$(curl -gfs $url 2>/dev/null); then + if out=$(curl -gkfs $url 2>/dev/null); then kube::log::status "On try ${i}, ${prefix}: ${out}" return 0 fi diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index 2c11ddc680..bffd9f4dca 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -140,8 +140,9 @@ function test_openssl_installed { set +e API_PORT=${API_PORT:-8080} -API_HOST=${API_HOST:-127.0.0.1} -API_HOST_IP=${API_HOST_IP:-${API_HOST}} +API_SECURE_PORT=${API_SECURE_PORT:-6443} +API_HOST=${API_HOST:-localhost} +API_HOST_IP=${API_HOST_IP:-"127.0.0.1"} API_BIND_ADDR=${API_BIND_ADDR:-"0.0.0.0"} KUBELET_HOST=${KUBELET_HOST:-"127.0.0.1"} # By default only allow CORS for requests on localhost @@ -158,15 +159,28 @@ CPU_CFS_QUOTA=${CPU_CFS_QUOTA:-true} ENABLE_HOSTPATH_PROVISIONER=${ENABLE_HOSTPATH_PROVISIONER:-"false"} CLAIM_BINDER_SYNC_PERIOD=${CLAIM_BINDER_SYNC_PERIOD:-"15s"} # current k8s default ENABLE_CONTROLLER_ATTACH_DETACH=${ENABLE_CONTROLLER_ATTACH_DETACH:-"true"} # current default +CERT_DIR=${CERT_DIR:-"/var/run/kubernetes"} +ROOT_CA_FILE=$CERT_DIR/apiserver.crt + function test_apiserver_off { # For the common local scenario, fail fast if server is already running. # this can happen if you run local-up-cluster.sh twice and kill etcd in between. - curl --silent -g $API_HOST:$API_PORT + if [[ "${API_PORT}" -gt "0" ]]; then + curl --silent -g $API_HOST:$API_PORT + if [ ! $? -eq 0 ]; then + echo "API SERVER insecure port is free, proceeding..." + else + echo "ERROR starting API SERVER, exiting. Some process on $API_HOST is serving already on $API_PORT" + exit 1 + fi + fi + + curl --silent -k -g $API_HOST:$API_SECURE_PORT if [ ! $? -eq 0 ]; then - echo "API SERVER port is free, proceeding..." + echo "API SERVER secure port is free, proceeding..." else - echo "ERROR starting API SERVER, exiting. Some host on $API_HOST is serving already on $API_PORT" + echo "ERROR starting API SERVER, exiting. Some process on $API_HOST is serving already on $API_SECURE_PORT" exit 1 fi } @@ -297,8 +311,6 @@ function start_apiserver { fi # This is the default dir and filename where the apiserver will generate a self-signed cert # which should be able to be used as the CA to verify itself - CERT_DIR=/var/run/kubernetes - ROOT_CA_FILE=$CERT_DIR/apiserver.crt anytoken_arg="" if [[ -n "${ALLOW_ANY_TOKEN:-}" ]]; then @@ -324,6 +336,10 @@ function start_apiserver { advertise_address="--advertise_address=${API_HOST_IP}" fi + # Ensure CERT_DIR is created for auto-generated crt/key and kubeconfig + sudo mkdir -p "${CERT_DIR}" + + APISERVER_LOG=/tmp/kube-apiserver.log sudo -E "${GO_OUT}/hyperkube" apiserver ${anytoken_arg} ${authorizer_arg} ${priv_arg} ${runtime_config}\ ${advertise_address} \ @@ -333,6 +349,8 @@ function start_apiserver { --service-account-lookup="${SERVICE_ACCOUNT_LOOKUP}" \ --admission-control="${ADMISSION_CONTROL}" \ --bind-address="${API_BIND_ADDR}" \ + --secure-port="${API_SECURE_PORT}" \ + --tls-ca-file="${ROOT_CA_FILE}" \ --insecure-bind-address="${API_HOST_IP}" \ --insecure-port="${API_PORT}" \ --etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}" \ @@ -343,9 +361,25 @@ function start_apiserver { --cors-allowed-origins="${API_CORS_ALLOWED_ORIGINS}" >"${APISERVER_LOG}" 2>&1 & APISERVER_PID=$! + # We created a kubeconfig that uses the apiserver.crt + cat < /dev/null +apiVersion: v1 +kind: Config +clusters: + - cluster: + certificate-authority: ${ROOT_CA_FILE} + server: https://${API_HOST}:${API_SECURE_PORT}/ + name: local-up-cluster +contexts: + - context: + cluster: local-up-cluster + name: service-to-apiserver +current-context: service-to-apiserver +EOF + # Wait for kube-apiserver to come up before launching the rest of the components. echo "Waiting for apiserver to come up" - kube::util::wait_for_url "http://${API_HOST}:${API_PORT}/api/v1/pods" "apiserver: " 1 ${WAIT_FOR_URL_API_SERVER} || exit 1 + kube::util::wait_for_url "https://${API_HOST}:${API_SECURE_PORT}/api/v1/pods" "apiserver: " 1 ${WAIT_FOR_URL_API_SERVER} || exit 1 } function start_controller_manager { @@ -365,7 +399,8 @@ function start_controller_manager { --feature-gates="${FEATURE_GATES}" \ --cloud-provider="${CLOUD_PROVIDER}" \ --cloud-config="${CLOUD_CONFIG}" \ - --master="${API_HOST}:${API_PORT}" >"${CTLRMGR_LOG}" 2>&1 & + --kubeconfig "$CERT_DIR"/kubeconfig \ + --master="https://${API_HOST}:${API_SECURE_PORT}" >"${CTLRMGR_LOG}" 2>&1 & CTLRMGR_PID=$! } @@ -436,7 +471,8 @@ function start_kubelet { --cloud-provider="${CLOUD_PROVIDER}" \ --cloud-config="${CLOUD_CONFIG}" \ --address="${KUBELET_HOST}" \ - --api-servers="${API_HOST}:${API_PORT}" \ + --require-kubeconfig \ + --kubeconfig "$CERT_DIR"/kubeconfig \ --feature-gates="${FEATURE_GATES}" \ --cpu-cfs-quota=${CPU_CFS_QUOTA} \ --enable-controller-attach-detach="${ENABLE_CONTROLLER_ATTACH_DETACH}" \ @@ -482,7 +518,7 @@ function start_kubelet { -i \ --cidfile=$KUBELET_CIDFILE \ gcr.io/google_containers/kubelet \ - /kubelet --v=${LOG_LEVEL} --containerized ${priv_arg}--chaos-chance="${CHAOS_CHANCE}" --hostname-override="${HOSTNAME_OVERRIDE}" --cloud-provider="${CLOUD_PROVIDER}" --cloud-config="${CLOUD_CONFIG}" \ --address="127.0.0.1" --api-servers="${API_HOST}:${API_PORT}" --port="$KUBELET_PORT" --enable-controller-attach-detach="${ENABLE_CONTROLLER_ATTACH_DETACH}" &> $KUBELET_LOG & + /kubelet --v=${LOG_LEVEL} --containerized ${priv_arg}--chaos-chance="${CHAOS_CHANCE}" --hostname-override="${HOSTNAME_OVERRIDE}" --cloud-provider="${CLOUD_PROVIDER}" --cloud-config="${CLOUD_CONFIG}" \ --address="127.0.0.1" --require-kubeconfig --kubeconfig "$CERT_DIR"/kubeconfig --api-servers="https://${API_HOST}:${API_SECURE_PORT}" --port="$KUBELET_PORT" --enable-controller-attach-detach="${ENABLE_CONTROLLER_ATTACH_DETACH}" &> $KUBELET_LOG & fi } @@ -492,13 +528,15 @@ function start_kubeproxy { --v=${LOG_LEVEL} \ --hostname-override="${HOSTNAME_OVERRIDE}" \ --feature-gates="${FEATURE_GATES}" \ - --master="http://${API_HOST}:${API_PORT}" >"${PROXY_LOG}" 2>&1 & + --kubeconfig "$CERT_DIR"/kubeconfig \ + --master="https://${API_HOST}:${API_SECURE_PORT}" >"${PROXY_LOG}" 2>&1 & PROXY_PID=$! SCHEDULER_LOG=/tmp/kube-scheduler.log sudo -E "${GO_OUT}/hyperkube" scheduler \ --v=${LOG_LEVEL} \ - --master="http://${API_HOST}:${API_PORT}" >"${SCHEDULER_LOG}" 2>&1 & + --kubeconfig "$CERT_DIR"/kubeconfig \ + --master="https://${API_HOST}:${API_SECURE_PORT}" >"${SCHEDULER_LOG}" 2>&1 & SCHEDULER_PID=$! } @@ -527,7 +565,7 @@ kind: Namespace metadata: name: kube-system EOF - ${KUBECTL} config set-cluster local --server=http://${API_HOST}:${API_PORT} --insecure-skip-tls-verify=true + ${KUBECTL} config set-cluster local --server=https://${API_HOST}:${API_SECURE_PORT} --certificate-authority=$(ROOT_CA_FILE) ${KUBECTL} config set-context local --cluster=local ${KUBECTL} config use-context local @@ -568,8 +606,9 @@ To start using your cluster, open up another terminal/tab and run: export KUBERNETES_PROVIDER=local - cluster/kubectl.sh config set-cluster local --server=http://${API_HOST}:${API_PORT} --insecure-skip-tls-verify=true - cluster/kubectl.sh config set-context local --cluster=local + cluster/kubectl.sh config set-cluster local --server=https://${API_HOST}:${API_SECURE_PORT} --certificate-authority=${ROOT_CA_FILE} + cluster/kubectl.sh config set-credentials myself --username=admin --password=admin + cluster/kubectl.sh config set-context local --cluster=local --user=myself cluster/kubectl.sh config use-context local cluster/kubectl.sh EOF diff --git a/pkg/genericapiserver/options/server_run_options.go b/pkg/genericapiserver/options/server_run_options.go index c0267e1423..c858efefd6 100644 --- a/pkg/genericapiserver/options/server_run_options.go +++ b/pkg/genericapiserver/options/server_run_options.go @@ -225,8 +225,14 @@ func (s *ServerRunOptions) NewSelfClientConfig(token string) (*restclient.Config QPS: 50, Burst: 100, } + + // Use secure port if the TLSCAFile is specified if s.SecurePort > 0 && len(s.TLSCAFile) > 0 { - clientConfig.Host = "https://" + net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.SecurePort)) + host := s.BindAddress.String() + if host == "0.0.0.0" { + host = "localhost" + } + clientConfig.Host = "https://" + net.JoinHostPort(host, strconv.Itoa(s.SecurePort)) clientConfig.CAFile = s.TLSCAFile clientConfig.BearerToken = token } else if s.InsecurePort > 0 {