From be20deaf96522f6200038dd659d0366f7cc586ed Mon Sep 17 00:00:00 2001 From: Shyam Jeedigunta Date: Wed, 22 Aug 2018 15:22:45 +0200 Subject: [PATCH] Parallelize kubemark cluster creation --- test/kubemark/gce/util.sh | 55 ++++++++++++++++++++------------- test/kubemark/skeleton/util.sh | 12 +++++-- test/kubemark/start-kubemark.sh | 39 ++++++++++++++--------- 3 files changed, 68 insertions(+), 38 deletions(-) diff --git a/test/kubemark/gce/util.sh b/test/kubemark/gce/util.sh index e6a7c9c95a..51de5a9aa6 100644 --- a/test/kubemark/gce/util.sh +++ b/test/kubemark/gce/util.sh @@ -31,28 +31,23 @@ function authenticate-docker { gcloud beta auth configure-docker -q } -function create-master-instance-with-resources { - GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE}" +# This function isn't too robust to race, but that should be ok given its one-off usage during setup. +function get-or-create-master-ip { + MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \ + --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') 2>/dev/null || true - run-gcloud-compute-with-retries disks create "${MASTER_NAME}-pd" \ - ${GCLOUD_COMMON_ARGS} \ - --type "${MASTER_DISK_TYPE}" \ - --size "${MASTER_DISK_SIZE}" - - if [ "${EVENT_PD:-}" == "true" ]; then - run-gcloud-compute-with-retries disks create "${MASTER_NAME}-event-pd" \ - ${GCLOUD_COMMON_ARGS} \ - --type "${MASTER_DISK_TYPE}" \ - --size "${MASTER_DISK_SIZE}" - fi - - run-gcloud-compute-with-retries addresses create "${MASTER_NAME}-ip" \ + if [[ -z "${MASTER_IP:-}" ]]; then + run-gcloud-compute-with-retries addresses create "${MASTER_NAME}-ip" \ --project "${PROJECT}" \ --region "${REGION}" -q - - MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \ - --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') + MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \ + --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') + fi +} + +function create-master-instance-with-resources { + GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE}" # Override the master image project to cos-cloud for COS images staring with `cos` string prefix. DEFAULT_GCI_PROJECT=google-containers if [[ "${GCI_VERSION}" == "cos"* ]]; then @@ -60,6 +55,22 @@ function create-master-instance-with-resources { fi MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-${DEFAULT_GCI_PROJECT}} + run-gcloud-compute-with-retries disks create "${MASTER_NAME}-pd" \ + ${GCLOUD_COMMON_ARGS} \ + --type "${MASTER_DISK_TYPE}" \ + --size "${MASTER_DISK_SIZE}" & + + if [ "${EVENT_PD:-}" == "true" ]; then + run-gcloud-compute-with-retries disks create "${MASTER_NAME}-event-pd" \ + ${GCLOUD_COMMON_ARGS} \ + --type "${MASTER_DISK_TYPE}" \ + --size "${MASTER_DISK_SIZE}" & + fi + + get-or-create-master-ip & + + wait + run-gcloud-compute-with-retries instances create "${MASTER_NAME}" \ ${GCLOUD_COMMON_ARGS} \ --address "${MASTER_IP}" \ @@ -74,14 +85,14 @@ function create-master-instance-with-resources { run-gcloud-compute-with-retries instances add-metadata "${MASTER_NAME}" \ ${GCLOUD_COMMON_ARGS} \ - --metadata-from-file startup-script="${KUBE_ROOT}/test/kubemark/resources/start-kubemark-master.sh" + --metadata-from-file startup-script="${KUBE_ROOT}/test/kubemark/resources/start-kubemark-master.sh" & if [ "${EVENT_PD:-}" == "true" ]; then echo "Attaching ${MASTER_NAME}-event-pd to ${MASTER_NAME}" run-gcloud-compute-with-retries instances attach-disk "${MASTER_NAME}" \ ${GCLOUD_COMMON_ARGS} \ --disk "${MASTER_NAME}-event-pd" \ - --device-name="master-event-pd" + --device-name="master-event-pd" & fi run-gcloud-compute-with-retries firewall-rules create "${MASTER_NAME}-https" \ @@ -89,7 +100,9 @@ function create-master-instance-with-resources { --network "${NETWORK}" \ --source-ranges "0.0.0.0/0" \ --target-tags "${MASTER_TAG}" \ - --allow "tcp:443" + --allow "tcp:443" & + + wait } # Command to be executed is '$1'. diff --git a/test/kubemark/skeleton/util.sh b/test/kubemark/skeleton/util.sh index fdd0c1c293..144f947372 100644 --- a/test/kubemark/skeleton/util.sh +++ b/test/kubemark/skeleton/util.sh @@ -24,6 +24,15 @@ function authenticate-docker { echo "Configuring registry authentication" 1>&2 } +# This function should get master IP address (creating one if needed). +# ENV vars that should be defined by the end of this function: +# - MASTER_IP +# +# Recommended for this function to include retrying logic in case of failures. +function get-or-create-master-ip { + echo "MASTER_IP: $MASTER_IP" 1>&2 +} + # This function should create a machine instance for the master along # with any/all of the following resources: # - Attach a PD to the master (optionally 1 more for storing events) @@ -32,8 +41,7 @@ function authenticate-docker { # Note: This step is compulsory in order for kubemark to work # # ENV vars that should be defined by the end of this function: -# 1. MASTER_IP -# 2. MASTER_NAME +# - MASTER_NAME # # Recommended for this function to include retrying logic for the above # operations in case of failures. diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh index 7ce9f46786..f4eaf59ca1 100755 --- a/test/kubemark/start-kubemark.sh +++ b/test/kubemark/start-kubemark.sh @@ -456,26 +456,35 @@ function wait-for-hollow-nodes-to-run-or-timeout { ############################### Main Function ######################################## detect-project &> /dev/null +find-release-tars + +# We need master IP to generate PKI and kubeconfig for cluster. +get-or-create-master-ip +generate-pki-config +write-local-kubeconfig # Setup for master. -echo -e "${color_yellow}STARTING SETUP FOR MASTER${color_norm}" -find-release-tars -create-master-environment-file -create-master-instance-with-resources -generate-pki-config -wait-for-master-reachability -write-pki-config-to-master -write-local-kubeconfig -copy-resource-files-to-master -start-master-components +function start-master { + echo -e "${color_yellow}STARTING SETUP FOR MASTER${color_norm}" + create-master-environment-file + create-master-instance-with-resources + wait-for-master-reachability + write-pki-config-to-master + copy-resource-files-to-master + start-master-components +} +start-master & # Setup for hollow-nodes. -echo "" -echo -e "${color_yellow}STARTING SETUP FOR HOLLOW-NODES${color_norm}" -create-and-upload-hollow-node-image -create-kube-hollow-node-resources -wait-for-hollow-nodes-to-run-or-timeout +function start-hollow-nodes { + echo -e "${color_yellow}STARTING SETUP FOR HOLLOW-NODES${color_norm}" + create-and-upload-hollow-node-image + create-kube-hollow-node-resources + wait-for-hollow-nodes-to-run-or-timeout +} +start-hollow-nodes & +wait echo "" echo "Master IP: ${MASTER_IP}" echo "Password to kubemark master: ${KUBE_PASSWORD}"