From cf23d4c939fee32bed17497ccf7d4f5640011db4 Mon Sep 17 00:00:00 2001 From: Chris Glass Date: Tue, 18 Apr 2017 20:47:23 +0200 Subject: [PATCH] Add Ubuntu as a platform for e2e tests on GCE Using Ubuntu on GCE to run cluster e2e tests requires slightly different node.yaml and master.yaml files than GCI, because Ubuntu uses systemd as PID 1, wheras GCI uses upstart with a systemd delegate. Therefore the e2e tests fail using those files since the kubernetes services are not brought back up after a node/master reboot. --- cluster/common.sh | 12 +-- cluster/gce/ubuntu/README.md | 12 +++ cluster/gce/ubuntu/master-helper.sh | 111 ++++++++++++++++++++++++ cluster/gce/ubuntu/master.yaml | 128 ++++++++++++++++++++++++++++ cluster/gce/ubuntu/node-helper.sh | 32 +++++++ cluster/gce/ubuntu/node.yaml | 128 ++++++++++++++++++++++++++++ cluster/gce/util.sh | 16 ++-- 7 files changed, 429 insertions(+), 10 deletions(-) create mode 100644 cluster/gce/ubuntu/README.md create mode 100644 cluster/gce/ubuntu/master-helper.sh create mode 100644 cluster/gce/ubuntu/master.yaml create mode 100644 cluster/gce/ubuntu/node-helper.sh create mode 100644 cluster/gce/ubuntu/node.yaml diff --git a/cluster/common.sh b/cluster/common.sh index b89fdd5345..58e77d3834 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -450,8 +450,8 @@ function find-release-tars() { # This tarball is used by GCI, Ubuntu Trusty, and Container Linux. KUBE_MANIFESTS_TAR= - if [[ "${MASTER_OS_DISTRIBUTION:-}" == "trusty" || "${MASTER_OS_DISTRIBUTION:-}" == "gci" || "${MASTER_OS_DISTRIBUTION:-}" == "container-linux" ]] || \ - [[ "${NODE_OS_DISTRIBUTION:-}" == "trusty" || "${NODE_OS_DISTRIBUTION:-}" == "gci" || "${NODE_OS_DISTRIBUTION:-}" == "container-linux" ]] ; then + if [[ "${MASTER_OS_DISTRIBUTION:-}" == "trusty" || "${MASTER_OS_DISTRIBUTION:-}" == "gci" || "${MASTER_OS_DISTRIBUTION:-}" == "container-linux" || "${MASTER_OS_DISTRIBUTION:-}" == "ubuntu" ]] || \ + [[ "${NODE_OS_DISTRIBUTION:-}" == "trusty" || "${NODE_OS_DISTRIBUTION:-}" == "gci" || "${NODE_OS_DISTRIBUTION:-}" == "container-linux" || "${NODE_OS_DISTRIBUTION:-}" == "ubuntu" ]] ; then KUBE_MANIFESTS_TAR=$(find-tar kubernetes-manifests.tar.gz) fi } @@ -587,7 +587,9 @@ function build-kube-env { local salt_tar_url=$SALT_TAR_URL local kube_manifests_tar_url="${KUBE_MANIFESTS_TAR_URL:-}" if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "container-linux" ]] || \ - [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "container-linux" ]] ; then + [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "container-linux" ]] || \ + [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]] || \ + [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]] ; then # TODO: Support fallback .tar.gz settings on Container Linux server_binary_tar_url=$(split_csv "${SERVER_BINARY_TAR_URL}") salt_tar_url=$(split_csv "${SALT_TAR_URL}") @@ -671,8 +673,8 @@ EOF TERMINATED_POD_GC_THRESHOLD: $(yaml-quote ${TERMINATED_POD_GC_THRESHOLD}) EOF fi - if [[ "${master}" == "true" && ("${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "container-linux") ]] || \ - [[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "container-linux") ]] ; then + if [[ "${master}" == "true" && ("${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "container-linux") || "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]] || \ + [[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "container-linux") || "${NODE_OS_DISTRIBUTION}" = "ubuntu" ]] ; then cat >>$file < ${KUBE_TEMP}/master-kube-env.yaml + get-metadata "${existing_master_zone}" "${existing_master_name}" cluster-name > "${KUBE_TEMP}/cluster-name.txt" + get-metadata "${existing_master_zone}" "${existing_master_name}" gci-update-strategy > "${KUBE_TEMP}/gci-update.txt" + get-metadata "${existing_master_zone}" "${existing_master_name}" gci-ensure-gke-docker > "${KUBE_TEMP}/gci-ensure-gke-docker.txt" + get-metadata "${existing_master_zone}" "${existing_master_name}" gci-docker-version > "${KUBE_TEMP}/gci-docker-version.txt" + get-metadata "${existing_master_zone}" "${existing_master_name}" kube-master-certs > "${KUBE_TEMP}/kube-master-certs.yaml" + + create-master-instance-internal "${REPLICA_NAME}" +} + + +function create-master-instance-internal() { + local -r master_name="${1}" + local -r address_option="${2:-}" + + local preemptible_master="" + if [[ "${PREEMPTIBLE_MASTER:-}" == "true" ]]; then + preemptible_master="--preemptible --maintenance-policy TERMINATE" + fi + + gcloud compute instances create "${master_name}" \ + ${address_option} \ + --project "${PROJECT}" \ + --zone "${ZONE}" \ + --machine-type "${MASTER_SIZE}" \ + --image-project="${MASTER_IMAGE_PROJECT}" \ + --image "${MASTER_IMAGE}" \ + --tags "${MASTER_TAG}" \ + --network "${NETWORK}" \ + --scopes "storage-ro,compute-rw,monitoring,logging-write" \ + --can-ip-forward \ + --metadata-from-file \ + "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/ubuntu/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt,gci-update-strategy=${KUBE_TEMP}/gci-update.txt,gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt,gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt,kube-master-certs=${KUBE_TEMP}/kube-master-certs.yaml" \ + --disk "name=${master_name}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \ + --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \ + ${preemptible_master} +} + +function get-metadata() { + local zone="${1}" + local name="${2}" + local key="${3}" + gcloud compute ssh "${name}" \ + --project "${PROJECT}" \ + --zone "${zone}" \ + --command "curl \"http://metadata.google.internal/computeMetadata/v1/instance/attributes/${key}\" -H \"Metadata-Flavor: Google\"" 2>/dev/null +} diff --git a/cluster/gce/ubuntu/master.yaml b/cluster/gce/ubuntu/master.yaml new file mode 100644 index 0000000000..07d7edc308 --- /dev/null +++ b/cluster/gce/ubuntu/master.yaml @@ -0,0 +1,128 @@ +#cloud-config + +write_files: + - path: /etc/systemd/system/kube-master-installation.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Download and install k8s binaries and configurations + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/kubernetes/bin + ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin + ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh + ExecStart=/home/kubernetes/bin/configure.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-master-configuration.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Configure kubernetes master + After=kube-master-installation.service + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh + ExecStart=/home/kubernetes/bin/configure-helper.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-docker-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for docker + After=kube-master-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh docker + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubelet-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for kubelet + After=kube-master-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.timer + permissions: 0644 + owner: root + content: | + [Unit] + Description=Hourly kube-logrotate invocation + + [Timer] + OnCalendar=hourly + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes log rotation + After=kube-master-configuration.service + + [Service] + Type=oneshot + ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubernetes.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes + + [Install] + WantedBy=multi-user.target + After=docker.service + +runcmd: + - systemctl daemon-reload + - systemctl enable kube-master-installation.service + - systemctl enable kube-master-configuration.service + - systemctl enable kube-docker-monitor.service + - systemctl enable kubelet-monitor.service + - systemctl enable kube-logrotate.timer + - systemctl enable kube-logrotate.service + - systemctl enable kubernetes.target + - systemctl start kubernetes.target diff --git a/cluster/gce/ubuntu/node-helper.sh b/cluster/gce/ubuntu/node-helper.sh new file mode 100644 index 0000000000..38fa0d7981 --- /dev/null +++ b/cluster/gce/ubuntu/node-helper.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A library of helper functions and constant for GCI distro +source "${KUBE_ROOT}/cluster/gce/gci/helper.sh" + +# $1: template name (required). +function create-node-instance-template { + local template_name="$1" + ensure-gci-metadata-files + create-node-template "$template_name" "${scope_flags[*]}" \ + "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \ + "user-data=${KUBE_ROOT}/cluster/gce/ubuntu/node.yaml" \ + "configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh" \ + "cluster-name=${KUBE_TEMP}/cluster-name.txt" \ + "gci-update-strategy=${KUBE_TEMP}/gci-update.txt" \ + "gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt" \ + "gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt" +} diff --git a/cluster/gce/ubuntu/node.yaml b/cluster/gce/ubuntu/node.yaml new file mode 100644 index 0000000000..ede0943323 --- /dev/null +++ b/cluster/gce/ubuntu/node.yaml @@ -0,0 +1,128 @@ +#cloud-config + +write_files: + - path: /etc/systemd/system/kube-node-installation.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Download and install k8s binaries and configurations + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/kubernetes/bin + ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin + ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh + ExecStart=/home/kubernetes/bin/configure.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-node-configuration.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Configure kubernetes node + After=kube-node-installation.service + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh + ExecStart=/home/kubernetes/bin/configure-helper.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-docker-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for docker + After=kube-node-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh docker + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubelet-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for kubelet + After=kube-node-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.timer + permissions: 0644 + owner: root + content: | + [Unit] + Description=Hourly kube-logrotate invocation + + [Timer] + OnCalendar=hourly + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes log rotation + After=kube-node-configuration.service + + [Service] + Type=oneshot + ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubernetes.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes + + [Install] + WantedBy=multi-user.target + After=docker.service + +runcmd: + - systemctl daemon-reload + - systemctl enable kube-node-installation.service + - systemctl enable kube-node-configuration.service + - systemctl enable kube-docker-monitor.service + - systemctl enable kubelet-monitor.service + - systemctl enable kube-logrotate.timer + - systemctl enable kube-logrotate.service + - systemctl enable kubernetes.target + - systemctl start kubernetes.target diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 6a7cfc7464..4099430b3d 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -23,14 +23,14 @@ source "${KUBE_ROOT}/cluster/gce/${KUBE_CONFIG_FILE-"config-default.sh"}" source "${KUBE_ROOT}/cluster/common.sh" source "${KUBE_ROOT}/hack/lib/util.sh" -if [[ "${NODE_OS_DISTRIBUTION}" == "debian" || "${NODE_OS_DISTRIBUTION}" == "container-linux" || "${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then +if [[ "${NODE_OS_DISTRIBUTION}" == "debian" || "${NODE_OS_DISTRIBUTION}" == "container-linux" || "${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then source "${KUBE_ROOT}/cluster/gce/${NODE_OS_DISTRIBUTION}/node-helper.sh" else echo "Cannot operate on cluster using node os distro: ${NODE_OS_DISTRIBUTION}" >&2 exit 1 fi -if [[ "${MASTER_OS_DISTRIBUTION}" == "container-linux" || "${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then +if [[ "${MASTER_OS_DISTRIBUTION}" == "container-linux" || "${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then source "${KUBE_ROOT}/cluster/gce/${MASTER_OS_DISTRIBUTION}/master-helper.sh" else echo "Cannot operate on cluster using master os distro: ${MASTER_OS_DISTRIBUTION}" >&2 @@ -45,10 +45,13 @@ if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then elif [[ "${MASTER_OS_DISTRIBUTION}" == "debian" ]]; then MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-${CVM_VERSION}} MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-google-containers} +elif [[ "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then + MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE} + MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT} fi # Sets node image based on the specified os distro. Currently this function only -# supports gci and debian. +# supports gci, debian, and ubuntu. function set-node-image() { if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then # If the node image is not set, we use the latest GCI image. @@ -58,6 +61,9 @@ function set-node-image() { elif [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${CVM_VERSION}} NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-google-containers} + elif [[ "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then + NODE_IMAGE=${KUBE_GCE_NODE_IMAGE} + NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT} fi } @@ -643,8 +649,8 @@ function kube-up() { create-subnetwork create-nodes elif [[ ${KUBE_REPLICATE_EXISTING_MASTER:-} == "true" ]]; then - if [[ "${MASTER_OS_DISTRIBUTION}" != "gci" && "${MASTER_OS_DISTRIBUTION}" != "debian" ]]; then - echo "Master replication supported only for gci and debian" + if [[ "${MASTER_OS_DISTRIBUTION}" != "gci" && "${MASTER_OS_DISTRIBUTION}" != "debian" && "${MASTER_OS_DISTRIBUTION}" != "ubuntu" ]]; then + echo "Master replication supported only for gci, debian, and ubuntu" return 1 fi create-loadbalancer