github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #24893 from andyzheng0831/gci 

Automatic merge from submit-queue

Add support for running clusters on GCI

Google Container-VM Image (GCI) is the next revision of Container-VM. See documentation at https://cloud.google.com/compute/docs/containers/vm-image/. This change adds support for starting a Kubernetes cluster using GCI.

With this change, users can start a kubernetes cluster using the latest kubelet and kubectl release binary built in the GCI image by running:

	$ KUBE_OS_DISTRIBUTION="gci" cluster/kube-up.sh

Or run a testing cluster on GCI by running:

	$ KUBE_OS_DISTRIBUTION="gci" go run hack/e2e.go -v --up

The commands above will choose the latest GCI image by default.
pull/6/head
k8s-merge-robot 2016-04-27 23:46:26 -07:00
parent 4a0e0826e5 93929c3853
commit 19408e5575
7 changed files with 48 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
build/common.sh
View File

@ -885,35 +885,36 @@ function kube::release::package_salt_tarball() {
}
# This will pack kube-system manifests files for distros without using salt
# such as Ubuntu Trusty. For Trusty, we directly copy manifests from cluster/addons
# and cluster/saltbase/salt. The script of cluster initialization will remove
# the salt configuration and evaluate the variables in the manifests.
# such as GCI and Ubuntu Trusty. We directly copy manifests from
# cluster/addons and cluster/saltbase/salt. The script of cluster initialization
# will remove the salt configuration and evaluate the variables in the manifests.
function kube::release::package_kube_manifests_tarball() {
kube::log::status "Building tarball: manifests"
local release_stage="${RELEASE_STAGE}/manifests/kubernetes"
rm -rf "${release_stage}"
mkdir -p "${release_stage}/trusty"
local dst_dir="${release_stage}/gci-trusty"
mkdir -p "${dst_dir}"
local salt_dir="${KUBE_ROOT}/cluster/saltbase/salt"
cp "${salt_dir}/fluentd-es/fluentd-es.yaml" "${release_stage}/"
cp "${salt_dir}/fluentd-gcp/fluentd-gcp.yaml" "${release_stage}/"
cp "${salt_dir}/kube-registry-proxy/kube-registry-proxy.yaml" "${release_stage}/"
cp "${salt_dir}/kube-proxy/kube-proxy.manifest" "${release_stage}/"
cp "${salt_dir}/etcd/etcd.manifest" "${release_stage}/trusty"
cp "${salt_dir}/kube-scheduler/kube-scheduler.manifest" "${release_stage}/trusty"
cp "${salt_dir}/kube-apiserver/kube-apiserver.manifest" "${release_stage}/trusty"
cp "${salt_dir}/kube-controller-manager/kube-controller-manager.manifest" "${release_stage}/trusty"
cp "${salt_dir}/kube-addons/namespace.yaml" "${release_stage}/trusty"
cp "${salt_dir}/kube-addons/kube-addons.sh" "${release_stage}/trusty"
cp "${salt_dir}/kube-addons/kube-addon-update.sh" "${release_stage}/trusty"
cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${release_stage}/trusty"
cp -r "${salt_dir}/kube-admission-controls/limit-range" "${release_stage}/trusty"
cp "${salt_dir}/etcd/etcd.manifest" "${dst_dir}"
cp "${salt_dir}/kube-scheduler/kube-scheduler.manifest" "${dst_dir}"
cp "${salt_dir}/kube-apiserver/kube-apiserver.manifest" "${dst_dir}"
cp "${salt_dir}/kube-controller-manager/kube-controller-manager.manifest" "${dst_dir}"
cp "${salt_dir}/kube-addons/namespace.yaml" "${dst_dir}"
cp "${salt_dir}/kube-addons/kube-addons.sh" "${dst_dir}"
cp "${salt_dir}/kube-addons/kube-addon-update.sh" "${dst_dir}"
cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${dst_dir}"
cp -r "${salt_dir}/kube-admission-controls/limit-range" "${dst_dir}"
local objects
objects=$(cd "${KUBE_ROOT}/cluster/addons" && find . \( -name \*.yaml -or -name \*.yaml.in -or -name \*.json \) | grep -v demo)
tar c -C "${KUBE_ROOT}/cluster/addons" ${objects} | tar x -C "${release_stage}/trusty"
tar c -C "${KUBE_ROOT}/cluster/addons" ${objects} | tar x -C "${dst_dir}"
# This is for coreos only. ContainerVM or Trusty does not use it.
# This is for coreos only. ContainerVM, GCI, or Trusty does not use it.
cp -r "${KUBE_ROOT}/cluster/gce/coreos/kube-manifests"/* "${release_stage}/"
kube::release::clean_cruft
@ -1104,8 +1105,9 @@ function kube::release::gcs::copy_release_artifacts() {
# Stage everything in release directory
kube::release::gcs::stage_and_hash "${RELEASE_DIR}"/* . || return 1
# Having the configure-vm.sh script and and trusty code from the GCE cluster
# Having the configure-vm.sh script and trusty code from the GCE cluster
# deploy hosted with the release is useful for GKE.
# TODO(andyzheng0831): Replace the trusty path with GCI after finshing the GCI code.
kube::release::gcs::stage_and_hash "${RELEASE_STAGE}/full/kubernetes/cluster/gce/configure-vm.sh" extra/gce || return 1
kube::release::gcs::stage_and_hash "${RELEASE_STAGE}/full/kubernetes/cluster/gce/trusty/node.yaml" extra/gce || return 1
kube::release::gcs::stage_and_hash "${RELEASE_STAGE}/full/kubernetes/cluster/gce/trusty/master.yaml" extra/gce || return 1

6
cluster/common.sh
View File

@ -335,9 +335,9 @@ function find-release-tars() {
exit 1
fi
# This tarball is only used by Ubuntu Trusty.
# This tarball is used by GCI, Ubuntu Trusty, and CoreOS.
KUBE_MANIFESTS_TAR=
if [[ "${KUBE_OS_DISTRIBUTION:-}" == "trusty" || "${KUBE_OS_DISTRIBUTION:-}" == "coreos" ]]; then
if [[ "${KUBE_OS_DISTRIBUTION:-}" == "trusty" || "${KUBE_OS_DISTRIBUTION:-}" == "gci" || "${KUBE_OS_DISTRIBUTION:-}" == "coreos" ]]; then
KUBE_MANIFESTS_TAR="${KUBE_ROOT}/server/kubernetes-manifests.tar.gz"
if [[ ! -f "${KUBE_MANIFESTS_TAR}" ]]; then
KUBE_MANIFESTS_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-manifests.tar.gz"
@ -519,7 +519,7 @@ EOF
TERMINATED_POD_GC_THRESHOLD: $(yaml-quote ${TERMINATED_POD_GC_THRESHOLD})
EOF
fi
if [[ "${OS_DISTRIBUTION}" == "trusty" ]]; then
if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "gci" ]]; then
cat >>$file <<EOF
KUBE_MANIFESTS_TAR_URL: $(yaml-quote ${KUBE_MANIFESTS_TAR_URL})
KUBE_MANIFESTS_TAR_HASH: $(yaml-quote ${KUBE_MANIFESTS_TAR_HASH})

2
cluster/gce/config-test.sh
View File

@ -137,7 +137,7 @@ KUBE_UP_AUTOMATIC_CLEANUP=${KUBE_UP_AUTOMATIC_CLEANUP:-false}
# Optional: setting it to true denotes this is a testing cluster,
# so that we can use pulled kubernetes binaries, even if binaries
# are pre-installed in the image. Note that currently this logic
# is only supported in trusty nodes.
# is only supported in trusty or GCI.
TEST_CLUSTER="${TEST_CLUSTER:-true}"
# OpenContrail networking plugin specific settings

12
cluster/gce/trusty/configure-helper.sh
View File

@ -357,7 +357,7 @@ create_master_kubelet_auth() {
# $5: pod name, which should be either etcd or etcd-events
prepare_etcd_manifest() {
etcd_temp_file="/tmp/$5"
cp /home/kubernetes/kube-manifests/kubernetes/trusty/etcd.manifest "${etcd_temp_file}"
cp /home/kubernetes/kube-manifests/kubernetes/gci-trusty/etcd.manifest "${etcd_temp_file}"
sed -i -e "s@{{ *suffix *}}@$1@g" "${etcd_temp_file}"
sed -i -e "s@{{ *port *}}@$2@g" "${etcd_temp_file}"
sed -i -e "s@{{ *server_port *}}@$3@g" "${etcd_temp_file}"
@ -462,7 +462,7 @@ start_kube_apiserver() {
fi
readonly kube_apiserver_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)
src_file="/home/kubernetes/kube-manifests/kubernetes/trusty/kube-apiserver.manifest"
src_file="/home/kubernetes/kube-manifests/kubernetes/gci-trusty/kube-apiserver.manifest"
remove_salt_config_comments "${src_file}"
# Evaluate variables
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
@ -521,7 +521,7 @@ start_kube_controller_manager() {
fi
readonly kube_rc_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-controller-manager.docker_tag)
src_file="/home/kubernetes/kube-manifests/kubernetes/trusty/kube-controller-manager.manifest"
src_file="/home/kubernetes/kube-manifests/kubernetes/gci-trusty/kube-controller-manager.manifest"
remove_salt_config_comments "${src_file}"
# Evaluate variables
sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
@ -561,7 +561,7 @@ start_kube_scheduler() {
readonly kube_scheduler_docker_tag=$(cat "${kube_home}/kube-docker-files/kube-scheduler.docker_tag")
# Remove salt comments and replace variables with values
src_file="${kube_home}/kube-manifests/kubernetes/trusty/kube-scheduler.manifest"
src_file="${kube_home}/kube-manifests/kubernetes/gci-trusty/kube-scheduler.manifest"
remove_salt_config_comments "${src_file}"
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
@ -585,7 +585,7 @@ start_fluentd() {
# $1: addon category under /etc/kubernetes
# $2: manifest source dir
setup_addon_manifests() {
src_dir="/home/kubernetes/kube-manifests/kubernetes/trusty/$2"
src_dir="/home/kubernetes/kube-manifests/kubernetes/gci-trusty/$2"
dst_dir="/etc/kubernetes/$1/$2"
if [ ! -d "${dst_dir}" ]; then
mkdir -p "${dst_dir}"
@ -609,7 +609,7 @@ setup_addon_manifests() {
# Prepares the manifests of k8s addons static pods.
prepare_kube_addons() {
addon_src_dir="/home/kubernetes/kube-manifests/kubernetes/trusty"
addon_src_dir="/home/kubernetes/kube-manifests/kubernetes/gci-trusty"
addon_dst_dir="/etc/kubernetes/addons"
# Set up manifests of other addons.
if [ "${ENABLE_CLUSTER_MONITORING:-}" = "influxdb" ] || \

2
cluster/gce/trusty/configure.sh
View File

@ -157,7 +157,7 @@ install_kube_binary_config() {
find "${dst_dir}" -name \*.manifest -or -name \*.json | \
xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@"
fi
cp "${dst_dir}/kubernetes/trusty/configure-helper.sh" /etc/kube-configure-helper.sh
cp "${dst_dir}/kubernetes/gci-trusty/configure-helper.sh" /etc/kube-configure-helper.sh
# Clean up.
rm -rf "${kube_home}/kubernetes"

24
cluster/gce/util.sh
View File

@ -25,6 +25,18 @@ source "${KUBE_ROOT}/cluster/lib/util.sh"
if [[ "${OS_DISTRIBUTION}" == "debian" || "${OS_DISTRIBUTION}" == "coreos" || "${OS_DISTRIBUTION}" == "trusty" ]]; then
source "${KUBE_ROOT}/cluster/gce/${OS_DISTRIBUTION}/helper.sh"
elif [[ "${OS_DISTRIBUTION}" == "gci" ]]; then
# TODO(andyzheng0831): Switch to use the GCI specific code.
source "${KUBE_ROOT}/cluster/gce/trusty/helper.sh"
MASTER_IMAGE_PROJECT="google-containers"
# If choosing "gci" disto, at least the cluster master needs to run on GCI image.
# If the user does not set a GCI image for master, we run both master and nodes
# using the latest GCI dev image.
if [[ "${MASTER_IMAGE}" != gci* ]]; then
MASTER_IMAGE=$(gcloud compute images list | grep "gci-dev" | cut -d ' ' -f1)
NODE_IMAGE="${MASTER_IMAGE}"
NODE_IMAGE_PROJECT="${MASTER_IMAGE_PROJECT}"
fi
else
echo "Cannot operate on cluster using os distro: ${OS_DISTRIBUTION}" >&2
exit 1
@ -169,8 +181,6 @@ function set-preferred-region() {
# Take the local tar files and upload them to Google Storage. They will then be
# downloaded by the master as part of the start up script for the master.
# If running on Ubuntu trusty, we also pack the dir cluster/gce/trusty/kube-manifest
# and upload it to Google Storage.
#
# Assumed vars:
# PROJECT
@ -208,7 +218,7 @@ function upload-server-tars() {
SERVER_BINARY_TAR_HASH=$(sha1sum-file "${SERVER_BINARY_TAR}")
SALT_TAR_HASH=$(sha1sum-file "${SALT_TAR}")
if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "gci" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${KUBE_MANIFESTS_TAR}")
fi
@ -241,7 +251,7 @@ function upload-server-tars() {
server_binary_tar_urls+=("${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}")
salt_tar_urls+=("${salt_gs_url/gs:\/\//https://storage.googleapis.com/}")
if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "gci" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}"
copy-to-staging "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}"
# Convert from gs:// URL to an https:// URL
@ -257,7 +267,7 @@ function upload-server-tars() {
else
SERVER_BINARY_TAR_URL=$(join_csv "${server_binary_tar_urls[@]}")
SALT_TAR_URL=$(join_csv "${salt_tar_urls[@]}")
if [[ "${OS_DISTRIBUTION}" == "trusty" ]]; then
if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "gci" ]]; then
KUBE_MANIFESTS_TAR_URL=$(join_csv "${kube_manifests_tar_urls[@]}")
fi
fi
@ -695,8 +705,8 @@ function create-nodes-template() {
local template_name="${NODE_INSTANCE_PREFIX}-template"
# For master on trusty, we support running nodes on ContainerVM or trusty.
if [[ "${OS_DISTRIBUTION}" == "trusty" ]] && \
# For master on GCI or trusty, we support the hybrid mode with nodes on ContainerVM.
if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "gci" ]] && \
[[ "${NODE_IMAGE}" == container* ]]; then
source "${KUBE_ROOT}/cluster/gce/debian/helper.sh"
fi

4
test/e2e/addon_update.go
View File

@ -214,7 +214,7 @@ var _ = framework.KubeDescribe("Addon update", func() {
switch framework.TestContext.OSDistro {
case "debian":
sshExecAndVerify(sshClient, "sudo TEST_ADDON_CHECK_INTERVAL_SEC=1 /etc/init.d/kube-addons restart")
case "trusty":
case "trusty", "gci":
sshExecAndVerify(sshClient, "sudo initctl restart kube-addons TEST_ADDON_CHECK_INTERVAL_SEC=1")
case "coreos":
sshExecAndVerify(sshClient, "sudo systemctl set-environment TEST_ADDON_CHECK_INTERVAL_SEC=1")
@ -230,7 +230,7 @@ var _ = framework.KubeDescribe("Addon update", func() {
switch framework.TestContext.OSDistro {
case "debian":
sshExec(sshClient, "sudo /etc/init.d/kube-addons restart")
case "trusty":
case "trusty", "gci":
sshExec(sshClient, "sudo initctl restart kube-addons")
case "coreos":
sshExec(sshClient, "sudo systemctl unset-environment TEST_ADDON_CHECK_INTERVAL_SEC")