Add Ubuntu as a platform for e2e tests on GCE

Using Ubuntu on GCE to run cluster e2e tests requires slightly different
node.yaml and master.yaml files than GCI, because Ubuntu uses systemd as
PID 1, wheras GCI uses upstart with a systemd delegate. Therefore the
e2e tests fail using those files since the kubernetes services are not
brought back up after a node/master reboot.
pull/6/head
Chris Glass 2017-04-18 20:47:23 +02:00 committed by Yang Guo
parent 3153cd6841
commit cf23d4c939
7 changed files with 429 additions and 10 deletions

View File

@ -450,8 +450,8 @@ function find-release-tars() {
# This tarball is used by GCI, Ubuntu Trusty, and Container Linux.
KUBE_MANIFESTS_TAR=
if [[ "${MASTER_OS_DISTRIBUTION:-}" == "trusty" || "${MASTER_OS_DISTRIBUTION:-}" == "gci" || "${MASTER_OS_DISTRIBUTION:-}" == "container-linux" ]] || \
[[ "${NODE_OS_DISTRIBUTION:-}" == "trusty" || "${NODE_OS_DISTRIBUTION:-}" == "gci" || "${NODE_OS_DISTRIBUTION:-}" == "container-linux" ]] ; then
if [[ "${MASTER_OS_DISTRIBUTION:-}" == "trusty" || "${MASTER_OS_DISTRIBUTION:-}" == "gci" || "${MASTER_OS_DISTRIBUTION:-}" == "container-linux" || "${MASTER_OS_DISTRIBUTION:-}" == "ubuntu" ]] || \
[[ "${NODE_OS_DISTRIBUTION:-}" == "trusty" || "${NODE_OS_DISTRIBUTION:-}" == "gci" || "${NODE_OS_DISTRIBUTION:-}" == "container-linux" || "${NODE_OS_DISTRIBUTION:-}" == "ubuntu" ]] ; then
KUBE_MANIFESTS_TAR=$(find-tar kubernetes-manifests.tar.gz)
fi
}
@ -587,7 +587,9 @@ function build-kube-env {
local salt_tar_url=$SALT_TAR_URL
local kube_manifests_tar_url="${KUBE_MANIFESTS_TAR_URL:-}"
if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "container-linux" ]] || \
[[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "container-linux" ]] ; then
[[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "container-linux" ]] || \
[[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]] || \
[[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]] ; then
# TODO: Support fallback .tar.gz settings on Container Linux
server_binary_tar_url=$(split_csv "${SERVER_BINARY_TAR_URL}")
salt_tar_url=$(split_csv "${SALT_TAR_URL}")
@ -671,8 +673,8 @@ EOF
TERMINATED_POD_GC_THRESHOLD: $(yaml-quote ${TERMINATED_POD_GC_THRESHOLD})
EOF
fi
if [[ "${master}" == "true" && ("${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "container-linux") ]] || \
[[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "container-linux") ]] ; then
if [[ "${master}" == "true" && ("${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "container-linux") || "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]] || \
[[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "container-linux") || "${NODE_OS_DISTRIBUTION}" = "ubuntu" ]] ; then
cat >>$file <<EOF
KUBE_MANIFESTS_TAR_URL: $(yaml-quote ${kube_manifests_tar_url})
KUBE_MANIFESTS_TAR_HASH: $(yaml-quote ${KUBE_MANIFESTS_TAR_HASH})

View File

@ -0,0 +1,12 @@
# Ubuntu Image
The Ubuntu GKE image is an image optimized to run Kubernetes on the Google
Cloud Platform.
The image is currently made to behave as much as possible like the GCI image,
but since it uses "real" systemd (instead of upstart with a delegate), the systemd
targets need to be changed slightly to ensure services are always started on
boot.
In practice this means that for now the only changes are in master.yaml and node.yaml,
as well as their injection points in node-helper.sh and master-helper.sh.

View File

@ -0,0 +1,111 @@
#!/bin/bash
# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# A library of helper functions and constant for GCI distro
source "${KUBE_ROOT}/cluster/gce/gci/helper.sh"
# create-master-instance creates the master instance. If called with
# an argument, the argument is used as the name to a reserved IP
# address for the master. (In the case of upgrade/repair, we re-use
# the same IP.)
#
# It requires a whole slew of assumed variables, partially due to to
# the call to write-master-env. Listing them would be rather
# futile. Instead, we list the required calls to ensure any additional
#
# variables are set:
# ensure-temp-dir
# detect-project
# get-bearer-token
function create-master-instance {
local address_opt=""
[[ -n ${1:-} ]] && address_opt="--address ${1}"
write-master-env
ensure-gci-metadata-files
create-master-instance-internal "${MASTER_NAME}" "${address_opt}"
}
function replicate-master-instance() {
local existing_master_zone="${1}"
local existing_master_name="${2}"
local existing_master_replicas="${3}"
local kube_env="$(get-metadata "${existing_master_zone}" "${existing_master_name}" kube-env)"
# Substitute INITIAL_ETCD_CLUSTER to enable etcd clustering.
kube_env="$(echo "${kube_env}" | grep -v "INITIAL_ETCD_CLUSTER")"
kube_env="$(echo -e "${kube_env}\nINITIAL_ETCD_CLUSTER: '${existing_master_replicas},${REPLICA_NAME}'")"
# Substitute INITIAL_ETCD_CLUSTER_STATE
kube_env="$(echo "${kube_env}" | grep -v "INITIAL_ETCD_CLUSTER_STATE")"
kube_env="$(echo -e "${kube_env}\nINITIAL_ETCD_CLUSTER_STATE: 'existing'")"
ETCD_CA_KEY="$(echo "${kube_env}" | grep "ETCD_CA_KEY" | sed "s/^.*: '//" | sed "s/'$//")"
ETCD_CA_CERT="$(echo "${kube_env}" | grep "ETCD_CA_CERT" | sed "s/^.*: '//" | sed "s/'$//")"
create-etcd-certs "${REPLICA_NAME}" "${ETCD_CA_CERT}" "${ETCD_CA_KEY}"
kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_KEY")"
kube_env="$(echo -e "${kube_env}\nETCD_PEER_KEY: '${ETCD_PEER_KEY_BASE64}'")"
kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_CERT")"
kube_env="$(echo -e "${kube_env}\nETCD_PEER_CERT: '${ETCD_PEER_CERT_BASE64}'")"
echo "${kube_env}" > ${KUBE_TEMP}/master-kube-env.yaml
get-metadata "${existing_master_zone}" "${existing_master_name}" cluster-name > "${KUBE_TEMP}/cluster-name.txt"
get-metadata "${existing_master_zone}" "${existing_master_name}" gci-update-strategy > "${KUBE_TEMP}/gci-update.txt"
get-metadata "${existing_master_zone}" "${existing_master_name}" gci-ensure-gke-docker > "${KUBE_TEMP}/gci-ensure-gke-docker.txt"
get-metadata "${existing_master_zone}" "${existing_master_name}" gci-docker-version > "${KUBE_TEMP}/gci-docker-version.txt"
get-metadata "${existing_master_zone}" "${existing_master_name}" kube-master-certs > "${KUBE_TEMP}/kube-master-certs.yaml"
create-master-instance-internal "${REPLICA_NAME}"
}
function create-master-instance-internal() {
local -r master_name="${1}"
local -r address_option="${2:-}"
local preemptible_master=""
if [[ "${PREEMPTIBLE_MASTER:-}" == "true" ]]; then
preemptible_master="--preemptible --maintenance-policy TERMINATE"
fi
gcloud compute instances create "${master_name}" \
${address_option} \
--project "${PROJECT}" \
--zone "${ZONE}" \
--machine-type "${MASTER_SIZE}" \
--image-project="${MASTER_IMAGE_PROJECT}" \
--image "${MASTER_IMAGE}" \
--tags "${MASTER_TAG}" \
--network "${NETWORK}" \
--scopes "storage-ro,compute-rw,monitoring,logging-write" \
--can-ip-forward \
--metadata-from-file \
"kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/ubuntu/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt,gci-update-strategy=${KUBE_TEMP}/gci-update.txt,gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt,gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt,kube-master-certs=${KUBE_TEMP}/kube-master-certs.yaml" \
--disk "name=${master_name}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \
--boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \
${preemptible_master}
}
function get-metadata() {
local zone="${1}"
local name="${2}"
local key="${3}"
gcloud compute ssh "${name}" \
--project "${PROJECT}" \
--zone "${zone}" \
--command "curl \"http://metadata.google.internal/computeMetadata/v1/instance/attributes/${key}\" -H \"Metadata-Flavor: Google\"" 2>/dev/null
}

View File

@ -0,0 +1,128 @@
#cloud-config
write_files:
- path: /etc/systemd/system/kube-master-installation.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Download and install k8s binaries and configurations
After=network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStartPre=/bin/mkdir -p /home/kubernetes/bin
ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin
ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh
ExecStart=/home/kubernetes/bin/configure.sh
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kube-master-configuration.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Configure kubernetes master
After=kube-master-installation.service
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
ExecStart=/home/kubernetes/bin/configure-helper.sh
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kube-docker-monitor.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Kubernetes health monitoring for docker
After=kube-master-configuration.service
[Service]
Restart=always
RestartSec=10
RemainAfterExit=yes
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
ExecStart=/home/kubernetes/bin/health-monitor.sh docker
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kubelet-monitor.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Kubernetes health monitoring for kubelet
After=kube-master-configuration.service
[Service]
Restart=always
RestartSec=10
RemainAfterExit=yes
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kube-logrotate.timer
permissions: 0644
owner: root
content: |
[Unit]
Description=Hourly kube-logrotate invocation
[Timer]
OnCalendar=hourly
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kube-logrotate.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Kubernetes log rotation
After=kube-master-configuration.service
[Service]
Type=oneshot
ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kubernetes.target
permissions: 0644
owner: root
content: |
[Unit]
Description=Kubernetes
[Install]
WantedBy=multi-user.target
After=docker.service
runcmd:
- systemctl daemon-reload
- systemctl enable kube-master-installation.service
- systemctl enable kube-master-configuration.service
- systemctl enable kube-docker-monitor.service
- systemctl enable kubelet-monitor.service
- systemctl enable kube-logrotate.timer
- systemctl enable kube-logrotate.service
- systemctl enable kubernetes.target
- systemctl start kubernetes.target

View File

@ -0,0 +1,32 @@
#!/bin/bash
# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# A library of helper functions and constant for GCI distro
source "${KUBE_ROOT}/cluster/gce/gci/helper.sh"
# $1: template name (required).
function create-node-instance-template {
local template_name="$1"
ensure-gci-metadata-files
create-node-template "$template_name" "${scope_flags[*]}" \
"kube-env=${KUBE_TEMP}/node-kube-env.yaml" \
"user-data=${KUBE_ROOT}/cluster/gce/ubuntu/node.yaml" \
"configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh" \
"cluster-name=${KUBE_TEMP}/cluster-name.txt" \
"gci-update-strategy=${KUBE_TEMP}/gci-update.txt" \
"gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt" \
"gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt"
}

View File

@ -0,0 +1,128 @@
#cloud-config
write_files:
- path: /etc/systemd/system/kube-node-installation.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Download and install k8s binaries and configurations
After=network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStartPre=/bin/mkdir -p /home/kubernetes/bin
ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin
ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin
ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh
ExecStart=/home/kubernetes/bin/configure.sh
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kube-node-configuration.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Configure kubernetes node
After=kube-node-installation.service
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
ExecStart=/home/kubernetes/bin/configure-helper.sh
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kube-docker-monitor.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Kubernetes health monitoring for docker
After=kube-node-configuration.service
[Service]
Restart=always
RestartSec=10
RemainAfterExit=yes
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
ExecStart=/home/kubernetes/bin/health-monitor.sh docker
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kubelet-monitor.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Kubernetes health monitoring for kubelet
After=kube-node-configuration.service
[Service]
Restart=always
RestartSec=10
RemainAfterExit=yes
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kube-logrotate.timer
permissions: 0644
owner: root
content: |
[Unit]
Description=Hourly kube-logrotate invocation
[Timer]
OnCalendar=hourly
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kube-logrotate.service
permissions: 0644
owner: root
content: |
[Unit]
Description=Kubernetes log rotation
After=kube-node-configuration.service
[Service]
Type=oneshot
ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf
[Install]
WantedBy=kubernetes.target
- path: /etc/systemd/system/kubernetes.target
permissions: 0644
owner: root
content: |
[Unit]
Description=Kubernetes
[Install]
WantedBy=multi-user.target
After=docker.service
runcmd:
- systemctl daemon-reload
- systemctl enable kube-node-installation.service
- systemctl enable kube-node-configuration.service
- systemctl enable kube-docker-monitor.service
- systemctl enable kubelet-monitor.service
- systemctl enable kube-logrotate.timer
- systemctl enable kube-logrotate.service
- systemctl enable kubernetes.target
- systemctl start kubernetes.target

View File

@ -23,14 +23,14 @@ source "${KUBE_ROOT}/cluster/gce/${KUBE_CONFIG_FILE-"config-default.sh"}"
source "${KUBE_ROOT}/cluster/common.sh"
source "${KUBE_ROOT}/hack/lib/util.sh"
if [[ "${NODE_OS_DISTRIBUTION}" == "debian" || "${NODE_OS_DISTRIBUTION}" == "container-linux" || "${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then
if [[ "${NODE_OS_DISTRIBUTION}" == "debian" || "${NODE_OS_DISTRIBUTION}" == "container-linux" || "${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then
source "${KUBE_ROOT}/cluster/gce/${NODE_OS_DISTRIBUTION}/node-helper.sh"
else
echo "Cannot operate on cluster using node os distro: ${NODE_OS_DISTRIBUTION}" >&2
exit 1
fi
if [[ "${MASTER_OS_DISTRIBUTION}" == "container-linux" || "${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then
if [[ "${MASTER_OS_DISTRIBUTION}" == "container-linux" || "${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then
source "${KUBE_ROOT}/cluster/gce/${MASTER_OS_DISTRIBUTION}/master-helper.sh"
else
echo "Cannot operate on cluster using master os distro: ${MASTER_OS_DISTRIBUTION}" >&2
@ -45,10 +45,13 @@ if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then
elif [[ "${MASTER_OS_DISTRIBUTION}" == "debian" ]]; then
MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-${CVM_VERSION}}
MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-google-containers}
elif [[ "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then
MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE}
MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT}
fi
# Sets node image based on the specified os distro. Currently this function only
# supports gci and debian.
# supports gci, debian, and ubuntu.
function set-node-image() {
if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then
# If the node image is not set, we use the latest GCI image.
@ -58,6 +61,9 @@ function set-node-image() {
elif [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then
NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${CVM_VERSION}}
NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-google-containers}
elif [[ "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then
NODE_IMAGE=${KUBE_GCE_NODE_IMAGE}
NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT}
fi
}
@ -643,8 +649,8 @@ function kube-up() {
create-subnetwork
create-nodes
elif [[ ${KUBE_REPLICATE_EXISTING_MASTER:-} == "true" ]]; then
if [[ "${MASTER_OS_DISTRIBUTION}" != "gci" && "${MASTER_OS_DISTRIBUTION}" != "debian" ]]; then
echo "Master replication supported only for gci and debian"
if [[ "${MASTER_OS_DISTRIBUTION}" != "gci" && "${MASTER_OS_DISTRIBUTION}" != "debian" && "${MASTER_OS_DISTRIBUTION}" != "ubuntu" ]]; then
echo "Master replication supported only for gci, debian, and ubuntu"
return 1
fi
create-loadbalancer