From fef1ede240a77656a7116a7cd8ad4c1221a255c8 Mon Sep 17 00:00:00 2001 From: Andy Zheng Date: Mon, 10 Aug 2015 16:59:00 -0700 Subject: [PATCH] Add config to run minions on GCE using Ubuntu. It is for running nodes on Ubuntu image upto 14.04 LTS (Trusty). The change for running master on Ubuntu will be added later. The configuration consists of several upstart jobs, which is passed to node instances through GCE metadata and parsed by cloud-init. --- cluster/gce/trusty/helper.sh | 155 +++++++++++++++++++ cluster/gce/trusty/node.yaml | 286 +++++++++++++++++++++++++++++++++++ cluster/gce/util.sh | 8 +- 3 files changed, 447 insertions(+), 2 deletions(-) create mode 100644 cluster/gce/trusty/helper.sh create mode 100644 cluster/gce/trusty/node.yaml diff --git a/cluster/gce/trusty/helper.sh b/cluster/gce/trusty/helper.sh new file mode 100644 index 0000000000..6ad5f79c72 --- /dev/null +++ b/cluster/gce/trusty/helper.sh @@ -0,0 +1,155 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A library of helper functions and constant for ubuntu os distro + +# The code and configuration is for running node instances on Ubuntu images. +# The master is still on Debian. In addition, the configuration is based on +# upstart, which is in Ubuntu upto 14.04 LTS (Trusty). Ubuntu 15.04 and above +# replaced upstart with systemd as the init system. Consequently, the +# configuration cannot work on these images. + +# $1: if 'true', we're building a master yaml, else a node +function build-kube-env { + local master=$1 + local file=$2 + + rm -f ${file} + # TODO(andyzheng0831): master node is still running with Debian image. Switch it + # to Ubuntu trusty. + if [[ "${master}" == "true" ]]; then + cat >$file <>$file < /dev/null; then + echo "Add rules to accpet all inbound TCP/UDP packets" + iptables -A INPUT -w -p TCP -j ACCEPT + iptables -A INPUT -w -p UDP -j ACCEPT + fi + if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then + echo "Add rules to accpet all forwarded TCP/UDP packets" + iptables -A FORWARD -w -p TCP -j ACCEPT + iptables -A FORWARD -w -p UDP -j ACCEPT + fi + # Create required directories. + mkdir -p /var/lib/kubelet + mkdir -p /var/lib/kube-proxy + mkdir -p /etc/kubernetes/manifests + + # Fetch kube-env from GCE metadata server. + curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /etc/kube-env \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env + + # Create the kubelet kubeconfig file. + . /etc/kube-env + cat > /var/lib/kubelet/kubeconfig << EOF +apiVersion: v1 +kind: Config +users: +- name: kubelet + user: + token: ${KUBELET_TOKEN} +clusters: +- name: local + cluster: + insecure-skip-tls-verify: true +contexts: +- context: + cluster: local + user: kubelet + name: service-account-context +current-context: service-account-context +EOF + + # Create the kube-proxy config file. + cat > /var/lib/kube-proxy/kubeconfig << EOF +apiVersion: v1 +kind: Config +users: +- name: kube-proxy + user: + token: ${KUBE_PROXY_TOKEN} +clusters: +- name: local + cluster: + insecure-skip-tls-verify: true +contexts: +- context: + cluster: local + user: kube-proxy + name: service-account-context +current-context: service-account-context +EOF +end script + +--===============6024533374511606659== +MIME-Version: 1.0 +Content-Type: text/upstart-job; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="kube-install-packages.conf" + +#upstart-job + +description "Install packages needed to run kubernetes" + +start on cloud-config + +script + apt-get update + # Install docker, brctl, and socat if they are not in the image. + if ! which docker > /dev/null; then + echo "Do not find docker. Install it." + # We should install the docker that passes qualification. At present, it is version 1.7.1. + curl -sSL https://get.docker.com/ubuntu/ | DOCKER_VERSION=1.7.1 sh + fi + if ! which brctl > /dev/null; then + echo "Do not find brctl. Install it." + apt-get install --yes bridge-utils + fi + if ! which socat > /dev/null; then + echo "Do not find socat. Install it." + apt-get install --yes socat + fi +end script + +--===============6024533374511606659== +MIME-Version: 1.0 +Content-Type: text/upstart-job; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="kube-install-minion.conf" + +#upstart-job + +description "Download and install k8s binaries and configurations" + +start on stopped kube-env + +script + . /etc/kube-env + # If kubelet or kube-proxy is not stalled in the image, pull release binaries and put them in /usr/bin. + if ! which kubelet > /dev/null || ! which kube-proxy > /dev/null; then + cd /tmp + k8s_sha1="${SERVER_BINARY_TAR_URL##*/}.sha1" + echo "Downloading k8s tar sha1 file ${k8s_sha1}" + curl -Lo "${k8s_sha1}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SERVER_BINARY_TAR_URL}.sha1" + k8s_tar="${SERVER_BINARY_TAR_URL##*/}" + echo "Downloading k8s tar file ${k8s_tar}" + curl -Lo "${k8s_tar}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SERVER_BINARY_TAR_URL}" + # Validate hash. + actual=$(sha1sum ${k8s_tar} | awk '{ print $1 }') || true + if [ "${actual}" != "${SERVER_BINARY_TAR_HASH}" ]; then + echo "== ${k8s_tar} corrupted, sha1 ${actual} doesn't match expected ${SERVER_BINARY_TAR_HASH} ==" + else + echo "Validated ${SERVER_BINARY_TAR_URL} SHA1 = ${SERVER_BINARY_TAR_HASH}" + fi + tar xzf "/tmp/${k8s_tar}" -C /tmp/ --overwrite + cp /tmp/kubernetes/server/bin/kubelet /usr/bin/ + cp /tmp/kubernetes/server/bin/kube-proxy /usr/bin/ + rm -rf "/tmp/kubernetes" + rm "/tmp/${k8s_tar}" + rm "/tmp/${k8s_sha1}" + fi + + # Put saltbase configuration files in /etc/saltbase. We will use the add-on yaml files. + mkdir -p /etc/saltbase + cd /etc/saltbase + salt_sha1="${SALT_TAR_URL##*/}.sha1" + echo "Downloading Salt tar sha1 file ${salt_sha1}" + curl -Lo "${salt_sha1}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SALT_TAR_URL}.sha1" + salt_tar="${SALT_TAR_URL##*/}" + echo "Downloading Salt tar file ${salt_tar}" + curl -Lo "${salt_tar}" --connect-timeout 20 --retry 6 --retry-delay 2 "${SALT_TAR_URL}" + # Validate hash. + actual=$(sha1sum ${salt_tar} | awk '{ print $1 }') || true + if [ "${actual}" != "${SALT_TAR_HASH}" ]; then + echo "== ${salt_tar} corrupted, sha1 ${actual} doesn't match expected ${SALT_TAR_HASH} ==" + else + echo "Validated ${SALT_TAR_URL} SHA1 = ${SALT_TAR_HASH}" + fi + tar xzf "/etc/saltbase/${salt_tar}" -C /etc/saltbase/ --overwrite + rm "/etc/saltbase/${salt_sha1}" + rm "/etc/saltbase/${salt_tar}" +end script + +--===============6024533374511606659== +MIME-Version: 1.0 +Content-Type: text/upstart-job; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="kubelet.conf" + +#upstart-job + +description "Run kubelet service" + +start on stopped kube-install-minion and stopped kube-install-packages + +respawn + +script + # TODO(andyzheng0831): Add health check functionality. + . /etc/kube-env + /usr/bin/kubelet \ + --api_servers=https://${KUBERNETES_MASTER_NAME} \ + --enable-debugging-handlers=true \ + --cloud_provider=gce \ + --config=/etc/kubernetes/manifests \ + --allow_privileged=false \ + --v=2 \ + --cluster_dns=10.0.0.10 \ + --cluster_domain=cluster.local \ + --configure-cbr0=true \ + --cgroup_root=/ \ + --system-container=/system +end script + +# Wait for 10s to start kubelet again. +post-stop exec sleep 10 + +--===============6024533374511606659== +MIME-Version: 1.0 +Content-Type: text/upstart-job; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="kube-proxy.conf" + +#upstart-job + +description "Start kube-proxy service" + +start on stopped kube-install-minion and stopped kube-install-packages + +respawn + +script + . /etc/kube-env + /usr/bin/kube-proxy \ + --master=https://${KUBERNETES_MASTER_NAME} \ + --kubeconfig=/var/lib/kube-proxy/kubeconfig \ + --v=2 +end script + +# Wait for 10s to start kube-proxy again. +post-stop exec sleep 10 + +--===============6024533374511606659== +MIME-Version: 1.0 +Content-Type: text/upstart-job; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="kube-docker.conf" + +#upstart-job + +description "Restart docker daemon" + +start on started kubelet + +script + . /etc/kube-env + # Assemble docker deamon options + echo "DOCKER_OPTS=\"-p /var/run/docker.pid ${EXTRA_DOCKER_OPTS} --log-level=\"debug\" --bridge cbr0 --iptables=false --ip-masq=false\"" > /etc/default/docker + # Make sure the network interface cbr0 is created before restarting docker daemon + while ! [ -L /sys/class/net/cbr0 ]; do + echo "Sleep 1 second to wait for cbr0" + sleep 1 + done + initctl restart docker + # Remove docker0 + ifconfig docker0 down + brctl delbr docker0 +end script + +--===============6024533374511606659== +MIME-Version: 1.0 +Content-Type: text/upstart-job; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="kube-addons.conf" + +#upstart-job + +description "Install kubelet add-on manifest files" + +start on stopped kube-docker + +script + # Configuration files are located at /etc/saltbase. + . /etc/kube-env + if [ "${ENABLE_NODE_LOGGING}" = "true" ]; then + if [ "${LOGGING_DESTINATION}" = "gcp" ]; then + cp /etc/saltbase/kubernetes/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml /etc/kubernetes/manifests/ + elif [ "${LOGGING_DESTINATION}" = "elasticsearch" ]; then + cp /etc/saltbase/kubernetes/saltbase/salt/fluentd-es/fluentd-es.yaml /etc/kubernetes/manifests/ + fi + fi +end script + +--===============6024533374511606659==-- + diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 61f3835b79..dd14ec56f8 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -22,7 +22,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. source "${KUBE_ROOT}/cluster/gce/${KUBE_CONFIG_FILE-"config-default.sh"}" source "${KUBE_ROOT}/cluster/common.sh" -if [[ "${OS_DISTRIBUTION}" == "debian" || "${OS_DISTRIBUTION}" == "coreos" ]]; then +if [[ "${OS_DISTRIBUTION}" == "debian" || "${OS_DISTRIBUTION}" == "coreos" || "${OS_DISTRIBUTION}" == "trusty" ]]; then source "${KUBE_ROOT}/cluster/gce/${OS_DISTRIBUTION}/helper.sh" else echo "Cannot operate on cluster using os distro: ${OS_DISTRIBUTION}" >&2 @@ -1196,7 +1196,11 @@ function ssh-to-node { # Restart the kube-proxy on a node ($1) function restart-kube-proxy { - ssh-to-node "$1" "sudo /etc/init.d/kube-proxy restart" + if [[ "${OS_DISTRIBUTION}" == "trusty" ]]; then + ssh-to-node "$1" "sudo initctl restart kube-proxy" + else + ssh-to-node "$1" "sudo /etc/init.d/kube-proxy restart" + fi } # Restart the kube-apiserver on a node ($1)