diff --git a/cluster/common.sh b/cluster/common.sh index b89fdd5345..58e77d3834 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -450,8 +450,8 @@ function find-release-tars() { # This tarball is used by GCI, Ubuntu Trusty, and Container Linux. KUBE_MANIFESTS_TAR= - if [[ "${MASTER_OS_DISTRIBUTION:-}" == "trusty" || "${MASTER_OS_DISTRIBUTION:-}" == "gci" || "${MASTER_OS_DISTRIBUTION:-}" == "container-linux" ]] || \ - [[ "${NODE_OS_DISTRIBUTION:-}" == "trusty" || "${NODE_OS_DISTRIBUTION:-}" == "gci" || "${NODE_OS_DISTRIBUTION:-}" == "container-linux" ]] ; then + if [[ "${MASTER_OS_DISTRIBUTION:-}" == "trusty" || "${MASTER_OS_DISTRIBUTION:-}" == "gci" || "${MASTER_OS_DISTRIBUTION:-}" == "container-linux" || "${MASTER_OS_DISTRIBUTION:-}" == "ubuntu" ]] || \ + [[ "${NODE_OS_DISTRIBUTION:-}" == "trusty" || "${NODE_OS_DISTRIBUTION:-}" == "gci" || "${NODE_OS_DISTRIBUTION:-}" == "container-linux" || "${NODE_OS_DISTRIBUTION:-}" == "ubuntu" ]] ; then KUBE_MANIFESTS_TAR=$(find-tar kubernetes-manifests.tar.gz) fi } @@ -587,7 +587,9 @@ function build-kube-env { local salt_tar_url=$SALT_TAR_URL local kube_manifests_tar_url="${KUBE_MANIFESTS_TAR_URL:-}" if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "container-linux" ]] || \ - [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "container-linux" ]] ; then + [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "container-linux" ]] || \ + [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]] || \ + [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]] ; then # TODO: Support fallback .tar.gz settings on Container Linux server_binary_tar_url=$(split_csv "${SERVER_BINARY_TAR_URL}") salt_tar_url=$(split_csv "${SALT_TAR_URL}") @@ -671,8 +673,8 @@ EOF TERMINATED_POD_GC_THRESHOLD: $(yaml-quote ${TERMINATED_POD_GC_THRESHOLD}) EOF fi - if [[ "${master}" == "true" && ("${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "container-linux") ]] || \ - [[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "container-linux") ]] ; then + if [[ "${master}" == "true" && ("${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "container-linux") || "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]] || \ + [[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "container-linux") || "${NODE_OS_DISTRIBUTION}" = "ubuntu" ]] ; then cat >>$file < ${KUBE_TEMP}/master-kube-env.yaml + get-metadata "${existing_master_zone}" "${existing_master_name}" cluster-name > "${KUBE_TEMP}/cluster-name.txt" + get-metadata "${existing_master_zone}" "${existing_master_name}" gci-update-strategy > "${KUBE_TEMP}/gci-update.txt" + get-metadata "${existing_master_zone}" "${existing_master_name}" gci-ensure-gke-docker > "${KUBE_TEMP}/gci-ensure-gke-docker.txt" + get-metadata "${existing_master_zone}" "${existing_master_name}" gci-docker-version > "${KUBE_TEMP}/gci-docker-version.txt" + get-metadata "${existing_master_zone}" "${existing_master_name}" kube-master-certs > "${KUBE_TEMP}/kube-master-certs.yaml" + + create-master-instance-internal "${REPLICA_NAME}" +} + + +function create-master-instance-internal() { + local -r master_name="${1}" + local -r address_option="${2:-}" + + local preemptible_master="" + if [[ "${PREEMPTIBLE_MASTER:-}" == "true" ]]; then + preemptible_master="--preemptible --maintenance-policy TERMINATE" + fi + + gcloud compute instances create "${master_name}" \ + ${address_option} \ + --project "${PROJECT}" \ + --zone "${ZONE}" \ + --machine-type "${MASTER_SIZE}" \ + --image-project="${MASTER_IMAGE_PROJECT}" \ + --image "${MASTER_IMAGE}" \ + --tags "${MASTER_TAG}" \ + --network "${NETWORK}" \ + --scopes "storage-ro,compute-rw,monitoring,logging-write" \ + --can-ip-forward \ + --metadata-from-file \ + "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/ubuntu/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt,gci-update-strategy=${KUBE_TEMP}/gci-update.txt,gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt,gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt,kube-master-certs=${KUBE_TEMP}/kube-master-certs.yaml" \ + --disk "name=${master_name}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \ + --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \ + ${preemptible_master} +} + +function get-metadata() { + local zone="${1}" + local name="${2}" + local key="${3}" + gcloud compute ssh "${name}" \ + --project "${PROJECT}" \ + --zone "${zone}" \ + --command "curl \"http://metadata.google.internal/computeMetadata/v1/instance/attributes/${key}\" -H \"Metadata-Flavor: Google\"" 2>/dev/null +} diff --git a/cluster/gce/ubuntu/master.yaml b/cluster/gce/ubuntu/master.yaml new file mode 100644 index 0000000000..07d7edc308 --- /dev/null +++ b/cluster/gce/ubuntu/master.yaml @@ -0,0 +1,128 @@ +#cloud-config + +write_files: + - path: /etc/systemd/system/kube-master-installation.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Download and install k8s binaries and configurations + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/kubernetes/bin + ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin + ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh + ExecStart=/home/kubernetes/bin/configure.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-master-configuration.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Configure kubernetes master + After=kube-master-installation.service + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh + ExecStart=/home/kubernetes/bin/configure-helper.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-docker-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for docker + After=kube-master-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh docker + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubelet-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for kubelet + After=kube-master-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.timer + permissions: 0644 + owner: root + content: | + [Unit] + Description=Hourly kube-logrotate invocation + + [Timer] + OnCalendar=hourly + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes log rotation + After=kube-master-configuration.service + + [Service] + Type=oneshot + ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubernetes.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes + + [Install] + WantedBy=multi-user.target + After=docker.service + +runcmd: + - systemctl daemon-reload + - systemctl enable kube-master-installation.service + - systemctl enable kube-master-configuration.service + - systemctl enable kube-docker-monitor.service + - systemctl enable kubelet-monitor.service + - systemctl enable kube-logrotate.timer + - systemctl enable kube-logrotate.service + - systemctl enable kubernetes.target + - systemctl start kubernetes.target diff --git a/cluster/gce/ubuntu/node-helper.sh b/cluster/gce/ubuntu/node-helper.sh new file mode 100644 index 0000000000..38fa0d7981 --- /dev/null +++ b/cluster/gce/ubuntu/node-helper.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A library of helper functions and constant for GCI distro +source "${KUBE_ROOT}/cluster/gce/gci/helper.sh" + +# $1: template name (required). +function create-node-instance-template { + local template_name="$1" + ensure-gci-metadata-files + create-node-template "$template_name" "${scope_flags[*]}" \ + "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \ + "user-data=${KUBE_ROOT}/cluster/gce/ubuntu/node.yaml" \ + "configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh" \ + "cluster-name=${KUBE_TEMP}/cluster-name.txt" \ + "gci-update-strategy=${KUBE_TEMP}/gci-update.txt" \ + "gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt" \ + "gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt" +} diff --git a/cluster/gce/ubuntu/node.yaml b/cluster/gce/ubuntu/node.yaml new file mode 100644 index 0000000000..ede0943323 --- /dev/null +++ b/cluster/gce/ubuntu/node.yaml @@ -0,0 +1,128 @@ +#cloud-config + +write_files: + - path: /etc/systemd/system/kube-node-installation.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Download and install k8s binaries and configurations + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/mkdir -p /home/kubernetes/bin + ExecStartPre=/bin/mount --bind /home/kubernetes/bin /home/kubernetes/bin + ExecStartPre=/bin/mount -o remount,exec /home/kubernetes/bin + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/kubernetes/bin/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-sh + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure.sh + ExecStart=/home/kubernetes/bin/configure.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-node-configuration.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Configure kubernetes node + After=kube-node-installation.service + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh + ExecStart=/home/kubernetes/bin/configure-helper.sh + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-docker-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for docker + After=kube-node-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh docker + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubelet-monitor.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes health monitoring for kubelet + After=kube-node-configuration.service + + [Service] + Restart=always + RestartSec=10 + RemainAfterExit=yes + RemainAfterExit=yes + ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh + ExecStart=/home/kubernetes/bin/health-monitor.sh kubelet + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.timer + permissions: 0644 + owner: root + content: | + [Unit] + Description=Hourly kube-logrotate invocation + + [Timer] + OnCalendar=hourly + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kube-logrotate.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes log rotation + After=kube-node-configuration.service + + [Service] + Type=oneshot + ExecStart=-/usr/sbin/logrotate /etc/logrotate.conf + + [Install] + WantedBy=kubernetes.target + + - path: /etc/systemd/system/kubernetes.target + permissions: 0644 + owner: root + content: | + [Unit] + Description=Kubernetes + + [Install] + WantedBy=multi-user.target + After=docker.service + +runcmd: + - systemctl daemon-reload + - systemctl enable kube-node-installation.service + - systemctl enable kube-node-configuration.service + - systemctl enable kube-docker-monitor.service + - systemctl enable kubelet-monitor.service + - systemctl enable kube-logrotate.timer + - systemctl enable kube-logrotate.service + - systemctl enable kubernetes.target + - systemctl start kubernetes.target diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 6a7cfc7464..4099430b3d 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -23,14 +23,14 @@ source "${KUBE_ROOT}/cluster/gce/${KUBE_CONFIG_FILE-"config-default.sh"}" source "${KUBE_ROOT}/cluster/common.sh" source "${KUBE_ROOT}/hack/lib/util.sh" -if [[ "${NODE_OS_DISTRIBUTION}" == "debian" || "${NODE_OS_DISTRIBUTION}" == "container-linux" || "${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then +if [[ "${NODE_OS_DISTRIBUTION}" == "debian" || "${NODE_OS_DISTRIBUTION}" == "container-linux" || "${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then source "${KUBE_ROOT}/cluster/gce/${NODE_OS_DISTRIBUTION}/node-helper.sh" else echo "Cannot operate on cluster using node os distro: ${NODE_OS_DISTRIBUTION}" >&2 exit 1 fi -if [[ "${MASTER_OS_DISTRIBUTION}" == "container-linux" || "${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then +if [[ "${MASTER_OS_DISTRIBUTION}" == "container-linux" || "${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then source "${KUBE_ROOT}/cluster/gce/${MASTER_OS_DISTRIBUTION}/master-helper.sh" else echo "Cannot operate on cluster using master os distro: ${MASTER_OS_DISTRIBUTION}" >&2 @@ -45,10 +45,13 @@ if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then elif [[ "${MASTER_OS_DISTRIBUTION}" == "debian" ]]; then MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-${CVM_VERSION}} MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-google-containers} +elif [[ "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then + MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE} + MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT} fi # Sets node image based on the specified os distro. Currently this function only -# supports gci and debian. +# supports gci, debian, and ubuntu. function set-node-image() { if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then # If the node image is not set, we use the latest GCI image. @@ -58,6 +61,9 @@ function set-node-image() { elif [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${CVM_VERSION}} NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-google-containers} + elif [[ "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then + NODE_IMAGE=${KUBE_GCE_NODE_IMAGE} + NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT} fi } @@ -643,8 +649,8 @@ function kube-up() { create-subnetwork create-nodes elif [[ ${KUBE_REPLICATE_EXISTING_MASTER:-} == "true" ]]; then - if [[ "${MASTER_OS_DISTRIBUTION}" != "gci" && "${MASTER_OS_DISTRIBUTION}" != "debian" ]]; then - echo "Master replication supported only for gci and debian" + if [[ "${MASTER_OS_DISTRIBUTION}" != "gci" && "${MASTER_OS_DISTRIBUTION}" != "debian" && "${MASTER_OS_DISTRIBUTION}" != "ubuntu" ]]; then + echo "Master replication supported only for gci, debian, and ubuntu" return 1 fi create-loadbalancer