From d2fadbe30fb4749b5ce8cb0117d271ce287a4869 Mon Sep 17 00:00:00 2001 From: Shyam Jeedigunta Date: Thu, 19 Jan 2017 14:55:16 +0100 Subject: [PATCH] Refactored kubemark code into provider-specific and provider-independent parts --- cluster/kubemark/{ => gce}/config-default.sh | 2 +- cluster/kubemark/util.sh | 5 +- test/kubemark/cloud-provider-config.sh | 17 + test/kubemark/common.sh | 71 --- test/kubemark/configure-kubectl.sh | 4 +- test/kubemark/gce/util.sh | 140 +++++ .../kubemark/resources/heapster_template.json | 2 +- .../resources/hollow-node_template.json | 4 +- test/kubemark/run-e2e-tests.sh | 11 +- test/kubemark/skeleton/util.sh | 63 +++ test/kubemark/start-kubemark.sh | 518 ++++++++++-------- test/kubemark/stop-kubemark.sh | 50 +- 12 files changed, 524 insertions(+), 363 deletions(-) rename cluster/kubemark/{ => gce}/config-default.sh (99%) create mode 100644 test/kubemark/cloud-provider-config.sh delete mode 100644 test/kubemark/common.sh create mode 100644 test/kubemark/gce/util.sh create mode 100644 test/kubemark/skeleton/util.sh diff --git a/cluster/kubemark/config-default.sh b/cluster/kubemark/gce/config-default.sh similarity index 99% rename from cluster/kubemark/config-default.sh rename to cluster/kubemark/gce/config-default.sh index 60306151ef..ffdfe6d51d 100644 --- a/cluster/kubemark/config-default.sh +++ b/cluster/kubemark/gce/config-default.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2015 The Kubernetes Authors. +# Copyright 2017 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/cluster/kubemark/util.sh b/cluster/kubemark/util.sh index a252c5cfcd..3b48f87a25 100644 --- a/cluster/kubemark/util.sh +++ b/cluster/kubemark/util.sh @@ -15,5 +15,6 @@ # limitations under the License. KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. -source ${KUBE_ROOT}/cluster/gce/util.sh -source ${KUBE_ROOT}/cluster/kubemark/config-default.sh +source ${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh +source ${KUBE_ROOT}/cluster/${CLOUD_PROVIDER}/util.sh +source ${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh diff --git a/test/kubemark/cloud-provider-config.sh b/test/kubemark/cloud-provider-config.sh new file mode 100644 index 0000000000..9b3475548d --- /dev/null +++ b/test/kubemark/cloud-provider-config.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CLOUD_PROVIDER="${CLOUD_PROVIDER:-gce}" diff --git a/test/kubemark/common.sh b/test/kubemark/common.sh deleted file mode 100644 index 7d2dd1552e..0000000000 --- a/test/kubemark/common.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source "${KUBE_ROOT}/cluster/kubemark/config-default.sh" -source "${KUBE_ROOT}/cluster/kubemark/util.sh" -source "${KUBE_ROOT}/cluster/lib/util.sh" - -# hack/lib/init.sh will ovewrite ETCD_VERSION if this is unset -# what what is default in hack/lib/etcd.sh -# To avoid it, if it is empty, we set it to 'avoid-overwrite' and -# clean it after that. -if [ -z "${ETCD_VERSION:-}" ]; then - ETCD_VERSION="avoid-overwrite" -fi -source "${KUBE_ROOT}/hack/lib/init.sh" -if [ "${ETCD_VERSION:-}" == "avoid-overwrite" ]; then - ETCD_VERSION="" -fi - -detect-project &> /dev/null -export PROJECT -find-release-tars - -MASTER_NAME="${INSTANCE_PREFIX}-kubemark-master" -MASTER_TAG="kubemark-master" -EVENT_STORE_NAME="${INSTANCE_PREFIX}-event-store" - -RETRIES=3 - -export KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh" -export KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark" -export RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" - -# Runs gcloud compute command with the given parameters. Up to $RETRIES will be made -# to execute the command. -# arguments: -# $@: all stuff that goes after 'gcloud compute ' -function run-gcloud-compute-with-retries { - echo "" > /tmp/gcloud_retries - for attempt in $(seq 1 ${RETRIES}); do - if ! gcloud compute $@ &> /tmp/gcloud_retries; then - if [[ $(grep -c "already exists" /tmp/gcloud_retries) -gt 0 ]]; then - if [[ "${attempt}" == 1 ]]; then - echo -e "${color_red} Failed to $1 $2 $3 as the resource hasn't been deleted from a previous run.${color_norm}" >& 2 - exit 1 - fi - echo -e "${color_yellow}Succeeded to $1 $2 $3 in the previous attempt, but status response wasn't received.${color_norm}" - return 0 - fi - echo -e "${color_yellow}Attempt $(($attempt+1)) failed to $1 $2 $3. Retrying.${color_norm}" >& 2 - sleep $(($attempt * 5)) - else - return 0 - fi - done - echo -e "${color_red} Failed to $1 $2 $3.${color_norm}" >& 2 - exit 1 -} diff --git a/test/kubemark/configure-kubectl.sh b/test/kubemark/configure-kubectl.sh index dac4a892b1..3a55de67d3 100644 --- a/test/kubemark/configure-kubectl.sh +++ b/test/kubemark/configure-kubectl.sh @@ -14,9 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -curl https://sdk.cloud.google.com 2> /dev/null | bash -sudo gcloud components update kubectl -q -sudo ln -s /usr/local/share/google/google-cloud-sdk/bin/kubectl /bin/ +# This script assumes that kubectl binary is present in PATH. kubectl config set-cluster hollow-cluster --server=http://localhost:8080 --insecure-skip-tls-verify=true kubectl config set-credentials $(whoami) kubectl config set-context hollow-context --cluster=hollow-cluster --user=$(whoami) diff --git a/test/kubemark/gce/util.sh b/test/kubemark/gce/util.sh new file mode 100644 index 0000000000..148cf93b5d --- /dev/null +++ b/test/kubemark/gce/util.sh @@ -0,0 +1,140 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source "${KUBE_ROOT}/cluster/gce/util.sh" + +detect-project &> /dev/null +export PROJECT + +RETRIES=3 +# Runs gcloud compute command with the given parameters. Up to $RETRIES will be made +# to execute the command. +# arguments: +# $@: all stuff that goes after 'gcloud compute' +function run-gcloud-compute-with-retries { + for attempt in $(seq 1 ${RETRIES}); do + local -r gcloud_cmd_hash=`echo "gcloud compute $@" | md5sum | cut -f1 -d" "` + local -r gcloud_logfile="/tmp/gcloud_${gcloud_cmd_hash}.log" + echo "" > ${gcloud_logfile} + if ! gcloud compute "$@" |& tee ${gcloud_logfile}; then + if [[ $(grep -c "already exists" ${gcloud_logfile}) -gt 0 ]]; then + if [[ "${attempt}" == 1 ]]; then + echo -e "${color_red} Failed to $1 $2 $3 as the resource hasn't been deleted from a previous run.${color_norm}" >& 2 + exit 1 + fi + echo -e "${color_yellow}Succeeded to $1 $2 $3 in the previous attempt, but status response wasn't received.${color_norm}" + return 0 + fi + echo -e "${color_yellow}Attempt $attempt failed to $1 $2 $3. Retrying.${color_norm}" >& 2 + sleep $(($attempt * 5)) + else + echo -e "${color_green}Succeeded to gcloud compute $1 $2 $3.${color_norm}" + return 0 + fi + done + echo -e "${color_red} Failed to $1 $2 $3.${color_norm}" >& 2 + exit 1 +} + +function create-master-instance-with-resources { + GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE}" + + run-gcloud-compute-with-retries disks create "${MASTER_NAME}-pd" \ + ${GCLOUD_COMMON_ARGS} \ + --type "${MASTER_DISK_TYPE}" \ + --size "${MASTER_DISK_SIZE}" + + if [ "${EVENT_PD:-false}" == "true" ]; then + run-gcloud-compute-with-retries disks create "${MASTER_NAME}-event-pd" \ + ${GCLOUD_COMMON_ARGS} \ + --type "${MASTER_DISK_TYPE}" \ + --size "${MASTER_DISK_SIZE}" + fi + + run-gcloud-compute-with-retries addresses create "${MASTER_NAME}-ip" \ + --project "${PROJECT}" \ + --region "${REGION}" -q + + MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \ + --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') + + run-gcloud-compute-with-retries instances create "${MASTER_NAME}" \ + ${GCLOUD_COMMON_ARGS} \ + --address "${MASTER_IP}" \ + --machine-type "${MASTER_SIZE}" \ + --image-project="${MASTER_IMAGE_PROJECT}" \ + --image "${MASTER_IMAGE}" \ + --tags "${MASTER_TAG}" \ + --network "${NETWORK}" \ + --scopes "storage-ro,compute-rw,logging-write" \ + --boot-disk-size "${MASTER_ROOT_DISK_SIZE}" \ + --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" + + if [ "${EVENT_PD:-false}" == "true" ]; then + echo "Attaching ${MASTER_NAME}-event-pd to ${MASTER_NAME}" + run-gcloud-compute-with-retries instances attach-disk "${MASTER_NAME}" \ + ${GCLOUD_COMMON_ARGS} \ + --disk "${MASTER_NAME}-event-pd" \ + --device-name="master-event-pd" + fi + + run-gcloud-compute-with-retries firewall-rules create "${MASTER_NAME}-https" \ + --project "${PROJECT}" \ + --network "${NETWORK}" \ + --source-ranges "0.0.0.0/0" \ + --target-tags "${MASTER_TAG}" \ + --allow "tcp:443" +} + +# Command to be executed is '$1'. +# No. of retries is '$2' (if provided) or 1 (default). +function execute-cmd-on-master-with-retries() { + RETRIES="${2:-1}" run-gcloud-compute-with-retries ssh "${MASTER_NAME}" --zone="${ZONE}" --project="${PROJECT}" --command="$1" +} + +function copy-files() { + run-gcloud-compute-with-retries copy-files --zone="${ZONE}" --project="${PROJECT}" $@ +} + +function delete-master-instance-and-resources { + GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE} --quiet" + + gcloud compute instances delete "${MASTER_NAME}" \ + ${GCLOUD_COMMON_ARGS} || true + + gcloud compute disks delete "${MASTER_NAME}-pd" \ + ${GCLOUD_COMMON_ARGS} || true + + gcloud compute disks delete "${MASTER_NAME}-event-pd" \ + ${GCLOUD_COMMON_ARGS} &> /dev/null || true + + gcloud compute addresses delete "${MASTER_NAME}-ip" \ + --project "${PROJECT}" \ + --region "${REGION}" \ + --quiet || true + + gcloud compute firewall-rules delete "${MASTER_NAME}-https" \ + --project "${PROJECT}" \ + --quiet || true + + if [ "${SEPARATE_EVENT_MACHINE:-false}" == "true" ]; then + gcloud compute instances delete "${EVENT_STORE_NAME}" \ + ${GCLOUD_COMMON_ARGS} || true + + gcloud compute disks delete "${EVENT_STORE_NAME}-pd" \ + ${GCLOUD_COMMON_ARGS} || true + fi +} diff --git a/test/kubemark/resources/heapster_template.json b/test/kubemark/resources/heapster_template.json index f1021cea08..cfedce9d0d 100644 --- a/test/kubemark/resources/heapster_template.json +++ b/test/kubemark/resources/heapster_template.json @@ -66,7 +66,7 @@ "/eventer" ], "args": [ - "--source=kubernetes:https://104.197.233.84:443?inClusterConfig=0&useServiceAccount=0&auth=/kubeconfig/heapster.kubeconfig" + "--source=kubernetes:https://{{MASTER_IP}}:443?inClusterConfig=0&useServiceAccount=0&auth=/kubeconfig/heapster.kubeconfig" ], "volumeMounts": [ { diff --git a/test/kubemark/resources/hollow-node_template.json b/test/kubemark/resources/hollow-node_template.json index f998b17b58..784dc9b0ea 100644 --- a/test/kubemark/resources/hollow-node_template.json +++ b/test/kubemark/resources/hollow-node_template.json @@ -80,7 +80,7 @@ "command": [ "/bin/sh", "-c", - "./kubemark.sh --morph=kubelet --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/logs/kubelet_$(MY_POD_NAME).log 2>&1" + "./kubemark.sh --morph=kubelet --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/logs/kubelet_$(NODE_NAME).log 2>&1" ], "volumeMounts": [ { @@ -129,7 +129,7 @@ "command": [ "/bin/sh", "-c", - "./kubemark.sh --morph=proxy --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/logs/kube_proxy_$(MY_POD_NAME).log 2>&1" + "./kubemark.sh --morph=proxy --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/logs/kubeproxy_$(NODE_NAME).log 2>&1" ], "volumeMounts": [ { diff --git a/test/kubemark/run-e2e-tests.sh b/test/kubemark/run-e2e-tests.sh index 3ae027087b..6083dabd61 100755 --- a/test/kubemark/run-e2e-tests.sh +++ b/test/kubemark/run-e2e-tests.sh @@ -14,17 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. + +source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh" +source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh" +source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh" + export KUBERNETES_PROVIDER="kubemark" export KUBE_CONFIG_FILE="config-default.sh" -KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. - # We need an absolute path to KUBE_ROOT ABSOLUTE_ROOT=$(readlink -f ${KUBE_ROOT}) -source ${KUBE_ROOT}/cluster/kubemark/util.sh -source ${KUBE_ROOT}/cluster/kubemark/config-default.sh - echo "Kubemark master name: ${MASTER_NAME}" detect-master diff --git a/test/kubemark/skeleton/util.sh b/test/kubemark/skeleton/util.sh new file mode 100644 index 0000000000..351b90379d --- /dev/null +++ b/test/kubemark/skeleton/util.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script contains the helper functions that each provider hosting +# Kubermark must implement to use test/kubemark/start-kubemark.sh and +# test/kubemark/stop-kubemark.sh scripts. + + +# This function should create a machine instance for the master along +# with any/all of the following resources: +# - Attach a PD to the master (optionally 1 more for storing events) +# - A public IP address for the master ($MASTER_IP) +# - A network firewall rule allowing all TCP traffic on port 443 in master +# Note: This step is compulsory in order for kubemark to work +# +# ENV vars that should be defined by the end of this function: +# 1. MASTER_IP +# 2. MASTER_NAME +# +# Recommended for this function to include retrying logic for the above +# operations in case of failures. +function create-master-instance-with-resources { + echo "MASTER_IP: $MASTER_IP" 1>&2 + echo "MASTER_NAME: $MASTER_NAME" 1>&2 +} + +# This function should execute the command('$1') on the master machine +# (possibly through SSH), retrying in case of failure. The allowed number of +# retries would be '$2' (if not provided, default to single try). +function execute-cmd-on-master-with-retries() { + echo "Executing command on the master" 1>&2 +} + +# This function should act as an scp for the kubemark cluster, which copies +# the files given by the first n-1 arguments to the remote location given +# by the n^th argument. +# +# Recommended for this function to include retrying logic in case of failures. +function copy-files() { + echo "Copying files" 1>&2 +} + +# This function should delete the master instance along with all the +# resources that have been allocated inside the function +# 'create-master-instance-with-resources' above. +# +# Recommended for this function to include retrying logic in case of failures. +function delete-master-instance-and-resources { + echo "Deleting master instance and its allocated resources" 1>&2 +} diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh index 3e9c0aebdb..20c2d8d12e 100755 --- a/test/kubemark/start-kubemark.sh +++ b/test/kubemark/start-kubemark.sh @@ -14,15 +14,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Script that creates a Kubemark cluster with Master running on GCE. +# Script that creates a Kubemark cluster for any given cloud provider. -# Hack to make it work for OS X. Ugh... TMP_ROOT="$(dirname "${BASH_SOURCE}")/../.." KUBE_ROOT=$(readlink -e ${TMP_ROOT} 2> /dev/null || perl -MCwd -e 'print Cwd::abs_path shift' ${TMP_ROOT}) +KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh" +KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark" +RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" -source "${KUBE_ROOT}/test/kubemark/common.sh" +source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh" +source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh" +source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh" +source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh" -function writeEnvironmentFile() { +# hack/lib/init.sh will ovewrite ETCD_VERSION if this is unset +# to what is default in hack/lib/etcd.sh +# To avoid it, if it is empty, we set it to 'avoid-overwrite' and +# clean it after that. +if [ -z "${ETCD_VERSION:-}" ]; then + ETCD_VERSION="avoid-overwrite" +fi +source "${KUBE_ROOT}/hack/lib/init.sh" +if [ "${ETCD_VERSION:-}" == "avoid-overwrite" ]; then + ETCD_VERSION="" +fi + +# Write all environment variables that we need to pass to the kubemark master, +# locally to the file ${RESOURCE_DIRECTORY}/kubemark-master-env.sh. +function create-master-environment-file { cat > "${RESOURCE_DIRECTORY}/kubemark-master-env.sh" </dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + NODE_PROBLEM_DETECTOR_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + HEAPSTER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + echo "Generated PKI authentication data for kubemark." +} -GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE}" +# Wait for the master to be reachable for executing commands on it. We do this by +# trying to run the bash noop(:) on the master, with 10 retries. +function wait-for-master-reachability { + execute-cmd-on-master-with-retries ":" 10 + echo "Checked master reachability for remote command execution." +} -run-gcloud-compute-with-retries disks create "${MASTER_NAME}-pd" \ - ${GCLOUD_COMMON_ARGS} \ - --type "${MASTER_DISK_TYPE}" \ - --size "${MASTER_DISK_SIZE}" - -if [ "${EVENT_PD:-false}" == "true" ]; then - run-gcloud-compute-with-retries disks create "${MASTER_NAME}-event-pd" \ - ${GCLOUD_COMMON_ARGS} \ - --type "${MASTER_DISK_TYPE}" \ - --size "${MASTER_DISK_SIZE}" -fi - -run-gcloud-compute-with-retries addresses create "${MASTER_NAME}-ip" \ - --project "${PROJECT}" \ - --region "${REGION}" -q - -MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \ - --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') - -run-gcloud-compute-with-retries instances create "${MASTER_NAME}" \ - ${GCLOUD_COMMON_ARGS} \ - --address "${MASTER_IP}" \ - --machine-type "${MASTER_SIZE}" \ - --image-project="${MASTER_IMAGE_PROJECT}" \ - --image "${MASTER_IMAGE}" \ - --tags "${MASTER_TAG}" \ - --network "${NETWORK}" \ - --scopes "storage-ro,compute-rw,logging-write" \ - --boot-disk-size "${MASTER_ROOT_DISK_SIZE}" \ - --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" - -if [ "${EVENT_PD:-false}" == "true" ]; then - echo "Attaching ${MASTER_NAME}-event-pd to ${MASTER_NAME}" - run-gcloud-compute-with-retries instances attach-disk "${MASTER_NAME}" \ - ${GCLOUD_COMMON_ARGS} \ - --disk "${MASTER_NAME}-event-pd" \ - --device-name="master-event-pd" -fi - -run-gcloud-compute-with-retries firewall-rules create "${INSTANCE_PREFIX}-kubemark-master-https" \ - --project "${PROJECT}" \ - --network "${NETWORK}" \ - --source-ranges "0.0.0.0/0" \ - --target-tags "${MASTER_TAG}" \ - --allow "tcp:443" - -ensure-temp-dir -gen-kube-bearertoken -create-certs ${MASTER_IP} -KUBELET_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) -KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) -HEAPSTER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) -NODE_PROBLEM_DETECTOR_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) - -until gcloud compute ssh --zone="${ZONE}" --project="${PROJECT}" "${MASTER_NAME}" --command="ls" &> /dev/null; do - sleep 1 -done - -password=$(python -c 'import string,random; print("".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16)))') - -run-gcloud-compute-with-retries ssh --zone="${ZONE}" --project="${PROJECT}" "${MASTER_NAME}" \ - --command="sudo mkdir /home/kubernetes -p && sudo mkdir /etc/srv/kubernetes -p && \ +# Write all the relevant certs/keys/tokens to the master. +function write-pki-config-to-master { + PKI_SETUP_CMD="sudo mkdir /home/kubernetes -p && sudo mkdir /etc/srv/kubernetes -p && \ sudo bash -c \"echo ${CA_CERT_BASE64} | base64 --decode > /etc/srv/kubernetes/ca.crt\" && \ sudo bash -c \"echo ${MASTER_CERT_BASE64} | base64 --decode > /etc/srv/kubernetes/server.cert\" && \ sudo bash -c \"echo ${MASTER_KEY_BASE64} | base64 --decode > /etc/srv/kubernetes/server.key\" && \ @@ -123,11 +101,16 @@ run-gcloud-compute-with-retries ssh --zone="${ZONE}" --project="${PROJECT}" "${M sudo bash -c \"echo \"${KUBE_BEARER_TOKEN},admin,admin\" > /etc/srv/kubernetes/known_tokens.csv\" && \ sudo bash -c \"echo \"${KUBELET_TOKEN},system:node:node-name,uid:kubelet,system:nodes\" >> /etc/srv/kubernetes/known_tokens.csv\" && \ sudo bash -c \"echo \"${KUBE_PROXY_TOKEN},system:kube-proxy,uid:kube_proxy\" >> /etc/srv/kubernetes/known_tokens.csv\" && \ - sudo bash -c \"echo \"${HEAPSTER_TOKEN},system:heapster,uid:heapster\" >> /etc/srv/kubernetes/known_tokens.csv\" && \ sudo bash -c \"echo \"${NODE_PROBLEM_DETECTOR_TOKEN},system:node-problem-detector,uid:system:node-problem-detector\" >> /etc/srv/kubernetes/known_tokens.csv\" && \ - sudo bash -c \"echo ${password},admin,admin > /etc/srv/kubernetes/basic_auth.csv\"" + sudo bash -c \"echo \"${HEAPSTER_TOKEN},system:heapster,uid:heapster\" >> /etc/srv/kubernetes/known_tokens.csv\" && \ + sudo bash -c \"echo ${KUBE_PASSWORD},admin,admin > /etc/srv/kubernetes/basic_auth.csv\"" + execute-cmd-on-master-with-retries "${PKI_SETUP_CMD}" 3 + echo "Wrote PKI certs, keys, tokens and admin password to master." +} -run-gcloud-compute-with-retries copy-files --zone="${ZONE}" --project="${PROJECT}" \ +# Copy all the necessary resource files (scripts/configs/manifests) to the master. +function copy-resource-files-to-master { + copy-files \ "${SERVER_BINARY_TAR}" \ "${RESOURCE_DIRECTORY}/kubemark-master-env.sh" \ "${RESOURCE_DIRECTORY}/start-kubemark-master.sh" \ @@ -140,120 +123,24 @@ run-gcloud-compute-with-retries copy-files --zone="${ZONE}" --project="${PROJECT "${RESOURCE_DIRECTORY}/manifests/kube-addon-manager.yaml" \ "${RESOURCE_DIRECTORY}/manifests/addons/kubemark-rbac-bindings" \ "kubernetes@${MASTER_NAME}":/home/kubernetes/ + echo "Copied server binary, master startup scripts, configs and resource manifests to master." +} -gcloud compute ssh "${MASTER_NAME}" --zone="${ZONE}" --project="${PROJECT}" \ - --command="sudo chmod a+x /home/kubernetes/configure-kubectl.sh && \ +# Make startup scripts executable and run start-kubemark-master.sh. +function start-master-components { + echo "" + MASTER_STARTUP_CMD="sudo chmod a+x /home/kubernetes/configure-kubectl.sh && \ sudo chmod a+x /home/kubernetes/start-kubemark-master.sh && \ sudo bash /home/kubernetes/start-kubemark-master.sh" + execute-cmd-on-master-with-retries "${MASTER_STARTUP_CMD}" + echo "The master has started and is now live." +} -# Setup the docker image for kubemark hollow-node. -MAKE_DIR="${KUBE_ROOT}/cluster/images/kubemark" -KUBEMARK_BIN="$(kube::util::find-binary-for-platform kubemark linux/amd64)" -if [[ -z "${KUBEMARK_BIN}" ]]; then - echo 'Cannot find cmd/kubemark binary' - exit 1 -fi - -echo "Copying kubemark to ${MAKE_DIR}" -cp "${KUBEMARK_BIN}" "${MAKE_DIR}" -CURR_DIR=`pwd` -cd "${MAKE_DIR}" -RETRIES=3 -for attempt in $(seq 1 ${RETRIES}); do - if ! make; then - if [[ $((attempt)) -eq "${RETRIES}" ]]; then - echo "${color_red}Make failed. Exiting.${color_norm}" - exit 1 - fi - echo -e "${color_yellow}Make attempt $(($attempt)) failed. Retrying.${color_norm}" >& 2 - sleep $(($attempt * 5)) - else - break - fi -done -rm kubemark -cd $CURR_DIR - -# Create kubeconfig for Kubelet. -KUBELET_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1 -kind: Config -users: -- name: kubelet - user: - client-certificate-data: "${KUBELET_CERT_BASE64}" - client-key-data: "${KUBELET_KEY_BASE64}" -clusters: -- name: kubemark - cluster: - certificate-authority-data: "${CA_CERT_BASE64}" - server: https://${MASTER_IP} -contexts: -- context: - cluster: kubemark - user: kubelet - name: kubemark-context -current-context: kubemark-context") - -# Create kubeconfig for Kubeproxy. -KUBEPROXY_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1 -kind: Config -users: -- name: kube-proxy - user: - token: ${KUBE_PROXY_TOKEN} -clusters: -- name: kubemark - cluster: - insecure-skip-tls-verify: true - server: https://${MASTER_IP} -contexts: -- context: - cluster: kubemark - user: kube-proxy - name: kubemark-context -current-context: kubemark-context") - -# Create kubeconfig for Heapster. -HEAPSTER_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1 -kind: Config -users: -- name: heapster - user: - token: ${HEAPSTER_TOKEN} -clusters: -- name: kubemark - cluster: - insecure-skip-tls-verify: true - server: https://${MASTER_IP} -contexts: -- context: - cluster: kubemark - user: heapster - name: kubemark-context -current-context: kubemark-context") - -# Create kubeconfig for NodeProblemDetector. -NPD_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1 -kind: Config -users: -- name: node-problem-detector - user: - token: ${NODE_PROBLEM_DETECTOR_TOKEN} -clusters: -- name: kubemark - cluster: - insecure-skip-tls-verify: true - server: https://${MASTER_IP} -contexts: -- context: - cluster: kubemark - user: node-problem-detector - name: kubemark-npd-context -current-context: kubemark-npd-context") - -# Create kubeconfig for local kubectl. -LOCAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark" -cat > "${LOCAL_KUBECONFIG}" << EOF +# Write kubeconfig to ${RESOURCE_DIRECTORY}/kubeconfig.kubemark in order to +# use kubectl locally. +function write-local-kubeconfig { + LOCAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark" + cat > "${LOCAL_KUBECONFIG}" << EOF apiVersion: v1 kind: Config users: @@ -275,70 +162,217 @@ contexts: name: kubemark-context current-context: kubemark-context EOF + echo "Kubeconfig file for kubemark master written to ${LOCAL_KUBECONFIG}." +} -sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.json" > "${RESOURCE_DIRECTORY}/hollow-node.json" -sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" -sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" - -mkdir "${RESOURCE_DIRECTORY}/addons" || true - -sed "s/{{MASTER_IP}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json" -metrics_mem_per_node=4 -metrics_mem=$((200 + ${metrics_mem_per_node}*${NUM_NODES:-10})) -sed -i'' -e "s/{{METRICS_MEM}}/${metrics_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" -eventer_mem_per_node=500 -eventer_mem=$((200 * 1024 + ${eventer_mem_per_node}*${NUM_NODES:-10})) -sed -i'' -e "s/{{EVENTER_MEM}}/${eventer_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" - -# Create kubemark namespace. -"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" -# Create configmap for configuring hollow- kubelet, proxy and npd. -"${KUBECTL}" create configmap "node-configmap" --namespace="kubemark" \ - --from-literal=content.type="${TEST_CLUSTER_API_CONTENT_TYPE}" \ - --from-file=kernel.monitor="${RESOURCE_DIRECTORY}/kernel-monitor.json" -# Create secret for passing kubeconfigs to kubelet, kubeproxy and npd. -"${KUBECTL}" create secret generic "kubeconfig" --type=Opaque --namespace="kubemark" \ - --from-literal=kubelet.kubeconfig="${KUBELET_KUBECONFIG_CONTENTS}" \ - --from-literal=kubeproxy.kubeconfig="${KUBEPROXY_KUBECONFIG_CONTENTS}" \ - --from-literal=heapster.kubeconfig="${HEAPSTER_KUBECONFIG_CONTENTS}" \ - --from-literal=npd.kubeconfig="${NPD_KUBECONFIG_CONTENTS}" -# Create addon pods. -"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark" -# Create the replication controller for hollow-nodes. -"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.json" --namespace="kubemark" - -echo "Waiting for all HollowNodes to become Running..." -start=$(date +%s) -nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true -ready=$(($(echo "${nodes}" | grep -v "NotReady" | wc -l) - 1)) - -until [[ "${ready}" -ge "${NUM_NODES}" ]]; do - echo -n . - sleep 1 - now=$(date +%s) - # Fail it if it already took more than 30 minutes. - if [ $((now - start)) -gt 1800 ]; then - echo "" - echo "Timeout waiting for all HollowNodes to become Running" - # Try listing nodes again - if it fails it means that API server is not responding - if "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node &> /dev/null; then - echo "Found only ${ready} ready Nodes while waiting for ${NUM_NODES}." - else - echo "Got error while trying to list Nodes. Probably API server is down." - fi - pods=$("${KUBECTL}" get pods --namespace=kubemark) || true - running=$(($(echo "${pods}" | grep "Running" | wc -l))) - echo "${running} HollowNode pods are reported as 'Running'" - not_running=$(($(echo "${pods}" | grep -v "Running" | wc -l) - 1)) - echo "${not_running} HollowNode pods are reported as NOT 'Running'" - echo $(echo "${pods}" | grep -v "Running") +# Finds the right kubemark binary for 'linux/amd64' platform and uses it to +# create a docker image for hollow-node and upload it to the appropriate +# docker container registry for the cloud provider. +# TODO(shyamjvs): Make the image upload URL and makefile variable w.r.t. provider. +function create-and-upload-hollow-node-image { + MAKE_DIR="${KUBE_ROOT}/cluster/images/kubemark" + KUBEMARK_BIN="$(kube::util::find-binary-for-platform kubemark linux/amd64)" + if [[ -z "${KUBEMARK_BIN}" ]]; then + echo 'Cannot find cmd/kubemark binary' exit 1 fi - nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true - ready=$(($(echo "${nodes}" | grep -v "NotReady" | wc -l) - 1)) -done -echo "" + + echo "Copying kubemark binary to ${MAKE_DIR}" + cp "${KUBEMARK_BIN}" "${MAKE_DIR}" + CURR_DIR=`pwd` + cd "${MAKE_DIR}" + RETRIES=3 + for attempt in $(seq 1 ${RETRIES}); do + if ! make; then + if [[ $((attempt)) -eq "${RETRIES}" ]]; then + echo "${color_red}Make failed. Exiting.${color_norm}" + exit 1 + fi + echo -e "${color_yellow}Make attempt $(($attempt)) failed. Retrying.${color_norm}" >& 2 + sleep $(($attempt * 5)) + else + break + fi + done + rm kubemark + cd $CURR_DIR + echo "Created and uploaded the kubemark hollow-node image to docker registry." +} +# Generate secret and configMap for the hollow-node pods to work, prepare +# manifests of the hollow-node and heapster replication controllers from +# templates, and finally create these resources through kubectl. +function create-kube-hollow-node-resources { + # Create kubeconfig for Kubelet. + KUBELET_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1 +kind: Config +users: +- name: kubelet + user: + client-certificate-data: "${KUBELET_CERT_BASE64}" + client-key-data: "${KUBELET_KEY_BASE64}" +clusters: +- name: kubemark + cluster: + certificate-authority-data: "${CA_CERT_BASE64}" + server: https://${MASTER_IP} +contexts: +- context: + cluster: kubemark + user: kubelet + name: kubemark-context +current-context: kubemark-context") + + # Create kubeconfig for Kubeproxy. + KUBEPROXY_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1 +kind: Config +users: +- name: kube-proxy + user: + token: ${KUBE_PROXY_TOKEN} +clusters: +- name: kubemark + cluster: + insecure-skip-tls-verify: true + server: https://${MASTER_IP} +contexts: +- context: + cluster: kubemark + user: kube-proxy + name: kubemark-context +current-context: kubemark-context") + + # Create kubeconfig for NodeProblemDetector. + NPD_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1 +kind: Config +users: +- name: node-problem-detector + user: + token: ${NODE_PROBLEM_DETECTOR_TOKEN} +clusters: +- name: kubemark + cluster: + insecure-skip-tls-verify: true + server: https://${MASTER_IP} +contexts: +- context: + cluster: kubemark + user: node-problem-detector + name: kubemark-context +current-context: kubemark-context") + + # Create kubeconfig for Heapster. + HEAPSTER_KUBECONFIG_CONTENTS=$(echo "apiVersion: v1 +kind: Config +users: +- name: heapster + user: + token: ${HEAPSTER_TOKEN} +clusters: +- name: kubemark + cluster: + insecure-skip-tls-verify: true + server: https://${MASTER_IP} +contexts: +- context: + cluster: kubemark + user: heapster + name: kubemark-context +current-context: kubemark-context") + + # Create kubemark namespace. + "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" + + # Create configmap for configuring hollow- kubelet, proxy and npd. + "${KUBECTL}" create configmap "node-configmap" --namespace="kubemark" \ + --from-literal=content.type="${TEST_CLUSTER_API_CONTENT_TYPE}" \ + --from-file=kernel.monitor="${RESOURCE_DIRECTORY}/kernel-monitor.json" + + # Create secret for passing kubeconfigs to kubelet, kubeproxy and npd. + "${KUBECTL}" create secret generic "kubeconfig" --type=Opaque --namespace="kubemark" \ + --from-literal=kubelet.kubeconfig="${KUBELET_KUBECONFIG_CONTENTS}" \ + --from-literal=kubeproxy.kubeconfig="${KUBEPROXY_KUBECONFIG_CONTENTS}" \ + --from-literal=heapster.kubeconfig="${HEAPSTER_KUBECONFIG_CONTENTS}" \ + --from-literal=npd.kubeconfig="${NPD_KUBECONFIG_CONTENTS}" + + # Create addon pods. + # TODO(shyamjvs): Make path to docker image variable in heapster_template.json. + mkdir -p "${RESOURCE_DIRECTORY}/addons" + sed "s/{{MASTER_IP}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json" + metrics_mem_per_node=4 + metrics_mem=$((200 + ${metrics_mem_per_node}*${NUM_NODES:-10})) + sed -i'' -e "s/{{METRICS_MEM}}/${metrics_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" + eventer_mem_per_node=500 + eventer_mem=$((200 * 1024 + ${eventer_mem_per_node}*${NUM_NODES:-10})) + sed -i'' -e "s/{{EVENTER_MEM}}/${eventer_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" + "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark" + + # Create the replication controller for hollow-nodes. + # TODO(shyamjvs): Make path to docker image variable in hollow-node_template.json. + sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.json" > "${RESOURCE_DIRECTORY}/hollow-node.json" + sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" + sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" + "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.json" --namespace="kubemark" + + echo "Created secrets, configMaps, replication-controllers required for hollow-nodes." +} + +# Wait until all hollow-nodes are running or there is a timeout. +function wait-for-hollow-nodes-to-run-or-timeout { + echo -n "Waiting for all hollow-nodes to become Running" + start=$(date +%s) + nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node) || true + ready=$(($(echo "${nodes}" | grep -v "NotReady" | wc -l) - 1)) + + until [[ "${ready}" -ge "${NUM_NODES}" ]]; do + echo -n "." + sleep 1 + now=$(date +%s) + # Fail it if it already took more than 30 minutes. + if [ $((now - start)) -gt 1800 ]; then + echo "" + echo -e "${color_red} Timeout waiting for all hollow-nodes to become Running. ${color_norm}" + # Try listing nodes again - if it fails it means that API server is not responding + if "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node &> /dev/null; then + echo "Found only ${ready} ready hollow-nodes while waiting for ${NUM_NODES}." + else + echo "Got error while trying to list hollow-nodes. Probably API server is down." + fi + pods=$("${KUBECTL}" get pods --namespace=kubemark) || true + running=$(($(echo "${pods}" | grep "Running" | wc -l))) + echo "${running} hollow-nodes are reported as 'Running'" + not_running=$(($(echo "${pods}" | grep -v "Running" | wc -l) - 1)) + echo "${not_running} hollow-nodes are reported as NOT 'Running'" + echo $(echo "${pods}" | grep -v "Running") + exit 1 + fi + nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node) || true + ready=$(($(echo "${nodes}" | grep -v "NotReady" | wc -l) - 1)) + done + echo -e "${color_green} Done!${color_norm}" +} + +############################### Main Function ######################################## +# Setup for master. +echo -e "${color_yellow}STARTING SETUP FOR MASTER${color_norm}" +find-release-tars +create-master-environment-file +create-master-instance-with-resources +generate-pki-config +wait-for-master-reachability +write-pki-config-to-master +copy-resource-files-to-master +start-master-components + +# Setup for hollow-nodes. +echo "" +echo -e "${color_yellow}STARTING SETUP FOR HOLLOW-NODES${color_norm}" +write-local-kubeconfig +create-and-upload-hollow-node-image +create-kube-hollow-node-resources +wait-for-hollow-nodes-to-run-or-timeout + +echo "" echo "Master IP: ${MASTER_IP}" -echo "Password to kubemark master: ${password}" +echo "Password to kubemark master: ${KUBE_PASSWORD}" echo "Kubeconfig for kubemark master is written in ${LOCAL_KUBECONFIG}" diff --git a/test/kubemark/stop-kubemark.sh b/test/kubemark/stop-kubemark.sh index 0ee2477def..94ed33fa0f 100755 --- a/test/kubemark/stop-kubemark.sh +++ b/test/kubemark/stop-kubemark.sh @@ -14,47 +14,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Script that destroys Kubemark clusters and deletes all GCE resources created for Master +# Script that destroys Kubemark cluster and deletes all master resources. + KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. -source "${KUBE_ROOT}/test/kubemark/common.sh" +source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh" +source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh" +source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh" +source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh" + +KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh" +RESOURCE_DIRECTORY="${KUBE_ROOT}/test/kubemark/resources" -"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/hollow-kubelet.json" &> /dev/null || true "${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/addons" &> /dev/null || true +"${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/hollow-node.json" &> /dev/null || true "${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" &> /dev/null || true -rm -rf "${RESOURCE_DIRECTORY}/addons" -GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE} --quiet" - -gcloud compute instances delete "${MASTER_NAME}" \ - ${GCLOUD_COMMON_ARGS} || true - -gcloud compute disks delete "${MASTER_NAME}-pd" \ - ${GCLOUD_COMMON_ARGS} || true - -gcloud compute disks delete "${MASTER_NAME}-event-pd" \ - ${GCLOUD_COMMON_ARGS} &> /dev/null || true - -gcloud compute addresses delete "${MASTER_NAME}-ip" \ - --project "${PROJECT}" \ - --region "${REGION}" \ - --quiet || true - -gcloud compute firewall-rules delete "${INSTANCE_PREFIX}-kubemark-master-https" \ - --project "${PROJECT}" \ - --quiet || true - -if [ "${SEPARATE_EVENT_MACHINE:-false}" == "true" ]; then - gcloud compute instances delete "${EVENT_STORE_NAME}" \ - ${GCLOUD_COMMON_ARGS} || true - - gcloud compute disks delete "${EVENT_STORE_NAME}-pd" \ - ${GCLOUD_COMMON_ARGS} || true -fi - -rm -rf "${RESOURCE_DIRECTORY}/addons" "${RESOURCE_DIRECTORY}/kubeconfig.kubemark" &> /dev/null || true -rm "${RESOURCE_DIRECTORY}/ca.crt" \ - "${RESOURCE_DIRECTORY}/kubecfg.crt" \ - "${RESOURCE_DIRECTORY}/kubecfg.key" \ +rm -rf "${RESOURCE_DIRECTORY}/addons" \ + "${RESOURCE_DIRECTORY}/kubeconfig.kubemark" \ "${RESOURCE_DIRECTORY}/hollow-node.json" \ "${RESOURCE_DIRECTORY}/kubemark-master-env.sh" &> /dev/null || true + +delete-master-instance-and-resources