From e077ebf87463323279974d1c0097ac7ccbb1023d Mon Sep 17 00:00:00 2001 From: Andy Zheng Date: Thu, 3 Mar 2016 15:26:43 -0800 Subject: [PATCH] Fix a breakage of kubelet in Trusty The kubelet flag "nosystem" was removed recently, which breaks kubelet in Trusty. This changes remove the flag usage accordingly. It also revises several aspects of Trusty support to make it in the same page as running on ContainerVM, such as new flags in kubelet and new logic in api-server and etcd pods. --- cluster/gce/trusty/configure.sh | 56 +++++++++++++++++++++++++++------ cluster/gce/trusty/master.yaml | 35 +++++++-------------- cluster/gce/trusty/node.yaml | 24 +++++++------- 3 files changed, 71 insertions(+), 44 deletions(-) diff --git a/cluster/gce/trusty/configure.sh b/cluster/gce/trusty/configure.sh index 4504c77941..00d79c8326 100644 --- a/cluster/gce/trusty/configure.sh +++ b/cluster/gce/trusty/configure.sh @@ -227,6 +227,47 @@ install_kube_binary_config() { rm "/run/kube-manifests/${manifests_tar}" } +# Assembles kubelet command line flags. +# It should be called by master and nodes before running kubelet process. The caller +# needs to source the config file /etc/kube-env. This function sets the following +# variable that will be used in kubelet command line. +# KUBELET_CMD_FLAGS +assemble_kubelet_flags() { + KUBELET_CMD_FLAGS="--v=2" + if [ -n "${KUBELET_TEST_LOG_LEVEL:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_TEST_LOG_LEVEL}" + fi + if [ -n "${KUBELET_PORT:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --port=${KUBELET_PORT}" + fi + if [ -n "${KUBELET_TEST_ARGS:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} ${KUBELET_TEST_ARGS}" + fi + if [ ! -z "${KUBELET_APISERVER:-}" ] && [ ! -z "${KUBELET_CERT:-}" ] && [ ! -z "${KUBELET_KEY:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --api-servers=https://${KUBELET_APISERVER}" + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --register-schedulable=false --reconcile-cidr=false" + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --pod-cidr=10.123.45.0/30" + else + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --pod-cidr=${MASTER_IP_RANGE}" + fi + if [ "${ENABLE_MANIFEST_URL:-}" = "true" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --manifest-url=${MANIFEST_URL} --manifest-url-header=${MANIFEST_URL_HEADER}" + fi + if [ "${KUBERNETES_MASTER:-}" = "true" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=none" + elif [ "${HAIRPIN_MODE:-}" = "promiscuous-bridge" ] || \ + [ "${HAIRPIN_MODE:-}" = "hairpin-veth" ] || \ + [ "${HAIRPIN_MODE:-}" = "none" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --hairpin-mode=${HAIRPIN_MODE}" + fi + if [ -n "${ENABLE_CUSTOM_METRICS:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --enable-custom-metrics=${ENABLE_CUSTOM_METRICS}" + fi + if [ -n "${NODE_LABELS:-}" ]; then + KUBELET_CMD_FLAGS="${KUBELET_CMD_FLAGS} --node-labels=${NODE_LABELS}" + fi +} + restart_docker_daemon() { # Assemble docker deamon options DOCKER_OPTS="-p /var/run/docker.pid --bridge=cbr0 --iptables=false --ip-masq=false" @@ -317,7 +358,7 @@ mount_master_pd() { # $2: token file add_token_entry() { current_token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) - echo "${tcurrent_token},$1,$1" >> $2 + echo "${current_token},$1,$1" >> $2 } # After the first boot and on upgrade, these files exists on the master-pd @@ -388,7 +429,7 @@ create_master_kubelet_auth() { # $4: value for variable 'cpulimit' # $5: pod name, which should be either etcd or etcd-events prepare_etcd_manifest() { - readonly etcd_temp_file="/tmp/$5" + etcd_temp_file="/tmp/$5" cp /run/kube-manifests/kubernetes/trusty/etcd.manifest "${etcd_temp_file}" sed -i -e "s@{{ *suffix *}}@$1@g" "${etcd_temp_file}" sed -i -e "s@{{ *port *}}@$2@g" "${etcd_temp_file}" @@ -418,11 +459,8 @@ start_etcd_servers() { prepare_log_file /var/log/etcd.log prepare_etcd_manifest "" "4001" "2380" "200m" "etcd.manifest" - # Switch on the second etcd instance if there are more than 50 nodes. - if [ -n "${NUM_NODES:-}" ] && [ "${NUM_NODES}" -gt 50 ]; then - prepare_log_file /var/log/etcd-events.log - prepare_etcd_manifest "-events" "4002" "2381" "100m" "etcd-events.manifest" - fi + prepare_log_file /var/log/etcd-events.log + prepare_etcd_manifest "-events" "4002" "2381" "100m" "etcd-events.manifest" } # Calculates the following variables based on env variables, which will be used @@ -459,9 +497,7 @@ start_kube_apiserver() { # Calculate variables and assemble the command line. params="--cloud-provider=gce --address=127.0.0.1 --etcd-servers=http://127.0.0.1:4001 --tls-cert-file=/etc/srv/kubernetes/server.cert --tls-private-key-file=/etc/srv/kubernetes/server.key --secure-port=443 --client-ca-file=/etc/srv/kubernetes/ca.crt --token-auth-file=/etc/srv/kubernetes/known_tokens.csv --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv --allow-privileged=true" - if [ -n "${NUM_NODES:-}" ] && [ "${NUM_NODES}" -gt 50 ]; then - params="${params} --etcd-servers-overrides=/events#http://127.0.0.1:4002" - fi + params="${params} --etcd-servers-overrides=/events#http://127.0.0.1:4002" if [ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]; then params="${params} --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" fi diff --git a/cluster/gce/trusty/master.yaml b/cluster/gce/trusty/master.yaml index 990e86f692..120fcd2aa3 100644 --- a/cluster/gce/trusty/master.yaml +++ b/cluster/gce/trusty/master.yaml @@ -1,4 +1,4 @@ -From nobody Tue Feb 1 11:33:00 2016 +From nobody Thu Mar 3 15:33:00 2016 Content-Type: multipart/mixed; boundary="====================================" MIME-Version: 1.0 @@ -122,30 +122,15 @@ script set -o nounset echo "Start kubelet upstart job" - + . /etc/kube-configure.sh . /etc/kube-env BINARY_PATH="/usr/bin/kubelet" if [ "${TEST_CLUSTER:-}" = "true" ]; then BINARY_PATH="/usr/local/bin/kubelet" fi - # Assemble command line flags based on env variables. - ARGS="--v=2" - if [ -n "${KUBELET_TEST_LOG_LEVEL:-}" ]; then - ARGS="${KUBELET_TEST_LOG_LEVEL}" - fi - if [ -n "${KUBELET_TEST_ARGS:-}" ]; then - ARGS="${ARGS} ${KUBELET_TEST_ARGS}" - fi - if [ ! -z "${KUBELET_APISERVER:-}" ] && [ ! -z "${KUBELET_CERT:-}" ] && [ ! -z "${KUBELET_KEY:-}" ]; then - ARGS="${ARGS} --api-servers=https://${KUBELET_APISERVER}" - ARGS="${ARGS} --register-schedulable=false --reconcile-cidr=false" - ARGS="${ARGS} --pod-cidr=10.123.45.0/30" - else - ARGS="${ARGS} --pod-cidr=${MASTER_IP_RANGE}" - fi - if [ "${ENABLE_MANIFEST_URL:-}" = "true" ]; then - ARGS="${ARGS} --manifest-url=${MANIFEST_URL} --manifest-url-header=${MANIFEST_URL_HEADER}" - fi + # Assemble command line flags based on env variables, which will put the string + # of flags in variable KUBELET_CMD_FLAGS + assemble_kubelet_flags ${BINARY_PATH} \ --enable-debugging-handlers=false \ @@ -159,8 +144,8 @@ script --system-cgroups=/system \ --runtime-cgroups=/docker-daemon \ --kubelet-cgroups=/kubelet \ - --nosystemd=true \ - ${ARGS} 1>>/var/log/kubelet.log 2>&1 + --babysit-daemons=true \ + ${KUBELET_CMD_FLAGS} 1>>/var/log/kubelet.log 2>&1 end script # Wait for 10s to start kubelet again. @@ -236,7 +221,6 @@ script set -o nounset . /etc/kube-env - export HOME="/root" if [ "${TEST_CLUSTER:-}" = "true" ]; then export KUBECTL_BIN="/usr/local/bin/kubectl" @@ -265,6 +249,8 @@ description "Kubenetes master health monitoring" start on stopped kube-docker +respawn + script set -o errexit set -o nounset @@ -274,4 +260,7 @@ script health_monitoring end script +# Wait for 10s to start it again. +post-stop exec sleep 10 + --====================================-- diff --git a/cluster/gce/trusty/node.yaml b/cluster/gce/trusty/node.yaml index 364b2e1cbc..e21f42d19d 100644 --- a/cluster/gce/trusty/node.yaml +++ b/cluster/gce/trusty/node.yaml @@ -1,4 +1,4 @@ -From nobody Tue Feb 1 11:33:00 2016 +From nobody Thu Mar 3 15:33:00 2016 Content-Type: multipart/mixed; boundary="====================================" MIME-Version: 1.0 @@ -119,18 +119,17 @@ script set -o errexit set -o nounset + echo "Start kubelet upstart job" + . /etc/kube-configure.sh . /etc/kube-env - ARGS="--v=2" - if [ -n "${KUBELET_TEST_LOG_LEVEL:-}" ]; then - ARGS="${KUBELET_TEST_LOG_LEVEL}" - fi - if [ -n "${KUBELET_TEST_ARGS:-}" ]; then - ARGS="${ARGS} ${KUBELET_TEST_ARGS}" - fi BINARY_PATH="/usr/bin/kubelet" if [ "${TEST_CLUSTER:-}" = "true" ]; then BINARY_PATH="/usr/local/bin/kubelet" fi + # Assemble command line flags based on env variables, which will put the string + # of flags in variable KUBELET_CMD_FLAGS. + assemble_kubelet_flags + ${BINARY_PATH} \ --api-servers=https://${KUBERNETES_MASTER_NAME} \ --enable-debugging-handlers=true \ @@ -144,8 +143,8 @@ script --system-cgroups=/system \ --runtime-cgroups=/docker-daemon \ --kubelet-cgroups=/kubelet \ - --nosystemd=true \ - ${ARGS} 1>>/var/log/kubelet.log 2>&1 + --babysit-daemons=true \ + ${KUBELET_CMD_FLAGS} 1>>/var/log/kubelet.log 2>&1 end script # Wait for 10s to start kubelet again. @@ -272,14 +271,17 @@ respawn script set -o nounset + set -o errexit # Wait for a minute to let docker, kubelet, and kube-proxy processes finish initialization. # TODO(andyzheng0831): replace it with a more reliable method if possible. sleep 60 - . /etc/kube-configure.sh . /etc/kube-env health_monitoring end script +# Wait for 10s to start it again. +post-stop exec sleep 10 + --====================================--