Parallelize kubemark cluster creation

test/kubemark/gce/util.sh

@@ -31,28 +31,23 @@ function authenticate-docker {
   gcloud beta auth configure-docker -q
 }
 
-function create-master-instance-with-resources {
-  GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE}"
+# This function isn't too robust to race, but that should be ok given its one-off usage during setup.
+function get-or-create-master-ip {
+  MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \
+    --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') 2>/dev/null || true
 
-  run-gcloud-compute-with-retries disks create "${MASTER_NAME}-pd" \
-    ${GCLOUD_COMMON_ARGS} \
-    --type "${MASTER_DISK_TYPE}" \
-    --size "${MASTER_DISK_SIZE}"
-  
-  if [ "${EVENT_PD:-}" == "true" ]; then
-    run-gcloud-compute-with-retries disks create "${MASTER_NAME}-event-pd" \
-      ${GCLOUD_COMMON_ARGS} \
-      --type "${MASTER_DISK_TYPE}" \
-      --size "${MASTER_DISK_SIZE}"
-  fi
-  
-  run-gcloud-compute-with-retries addresses create "${MASTER_NAME}-ip" \
+  if [[ -z "${MASTER_IP:-}" ]]; then
+    run-gcloud-compute-with-retries addresses create "${MASTER_NAME}-ip" \
     --project "${PROJECT}" \
     --region "${REGION}" -q
-  
-  MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \
-    --project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
 
+    MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \
+    --project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
+  fi
+}
+
+function create-master-instance-with-resources {
+  GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE}"
   # Override the master image project to cos-cloud for COS images staring with `cos` string prefix.
   DEFAULT_GCI_PROJECT=google-containers
   if [[ "${GCI_VERSION}" == "cos"* ]]; then
@@ -60,6 +55,22 @@ function create-master-instance-with-resources {
   fi
   MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-${DEFAULT_GCI_PROJECT}}
 
+  run-gcloud-compute-with-retries disks create "${MASTER_NAME}-pd" \
+    ${GCLOUD_COMMON_ARGS} \
+    --type "${MASTER_DISK_TYPE}" \
+    --size "${MASTER_DISK_SIZE}" &
+  
+  if [ "${EVENT_PD:-}" == "true" ]; then
+    run-gcloud-compute-with-retries disks create "${MASTER_NAME}-event-pd" \
+      ${GCLOUD_COMMON_ARGS} \
+      --type "${MASTER_DISK_TYPE}" \
+      --size "${MASTER_DISK_SIZE}" &
+  fi
+
+  get-or-create-master-ip &
+
+  wait
+
   run-gcloud-compute-with-retries instances create "${MASTER_NAME}" \
     ${GCLOUD_COMMON_ARGS} \
     --address "${MASTER_IP}" \
@@ -74,14 +85,14 @@ function create-master-instance-with-resources {
 
   run-gcloud-compute-with-retries instances add-metadata "${MASTER_NAME}" \
     ${GCLOUD_COMMON_ARGS} \
-    --metadata-from-file startup-script="${KUBE_ROOT}/test/kubemark/resources/start-kubemark-master.sh"
+    --metadata-from-file startup-script="${KUBE_ROOT}/test/kubemark/resources/start-kubemark-master.sh" &
   
   if [ "${EVENT_PD:-}" == "true" ]; then
     echo "Attaching ${MASTER_NAME}-event-pd to ${MASTER_NAME}"
     run-gcloud-compute-with-retries instances attach-disk "${MASTER_NAME}" \
     ${GCLOUD_COMMON_ARGS} \
     --disk "${MASTER_NAME}-event-pd" \
-    --device-name="master-event-pd"
+    --device-name="master-event-pd" &
   fi
   
   run-gcloud-compute-with-retries firewall-rules create "${MASTER_NAME}-https" \
@@ -89,7 +100,9 @@ function create-master-instance-with-resources {
     --network "${NETWORK}" \
     --source-ranges "0.0.0.0/0" \
     --target-tags "${MASTER_TAG}" \
-    --allow "tcp:443"
+    --allow "tcp:443" &
+
+  wait
 }
 
 # Command to be executed is '$1'.

test/kubemark/skeleton/util.sh

@@ -24,6 +24,15 @@ function authenticate-docker {
 	echo "Configuring registry authentication" 1>&2
 }
 
+# This function should get master IP address (creating one if needed).
+# ENV vars that should be defined by the end of this function:
+# - MASTER_IP
+#
+# Recommended for this function to include retrying logic in case of failures.
+function get-or-create-master-ip {
+	echo "MASTER_IP: $MASTER_IP" 1>&2
+}
+
 # This function should create a machine instance for the master along
 # with any/all of the following resources:
 # - Attach a PD to the master (optionally 1 more for storing events)
@@ -32,8 +41,7 @@ function authenticate-docker {
 #   Note: This step is compulsory in order for kubemark to work
 #
 # ENV vars that should be defined by the end of this function:
-# 1. MASTER_IP
-# 2. MASTER_NAME
+# - MASTER_NAME
 #
 # Recommended for this function to include retrying logic for the above
 # operations in case of failures.

test/kubemark/start-kubemark.sh

@@ -456,26 +456,35 @@ function wait-for-hollow-nodes-to-run-or-timeout {
 
 ############################### Main Function ########################################
 detect-project &> /dev/null
+find-release-tars
+
+# We need master IP to generate PKI and kubeconfig for cluster.
+get-or-create-master-ip
+generate-pki-config
+write-local-kubeconfig
 
 # Setup for master.
-echo -e "${color_yellow}STARTING SETUP FOR MASTER${color_norm}"
-find-release-tars
-create-master-environment-file
-create-master-instance-with-resources
-generate-pki-config
-wait-for-master-reachability
-write-pki-config-to-master
-write-local-kubeconfig
-copy-resource-files-to-master
-start-master-components
+function start-master {
+  echo -e "${color_yellow}STARTING SETUP FOR MASTER${color_norm}"
+  create-master-environment-file
+  create-master-instance-with-resources
+  wait-for-master-reachability
+  write-pki-config-to-master
+  copy-resource-files-to-master
+  start-master-components
+}
+start-master &
 
 # Setup for hollow-nodes.
-echo ""
-echo -e "${color_yellow}STARTING SETUP FOR HOLLOW-NODES${color_norm}"
-create-and-upload-hollow-node-image
-create-kube-hollow-node-resources
-wait-for-hollow-nodes-to-run-or-timeout
+function start-hollow-nodes {
+  echo -e "${color_yellow}STARTING SETUP FOR HOLLOW-NODES${color_norm}"
+  create-and-upload-hollow-node-image
+  create-kube-hollow-node-resources
+  wait-for-hollow-nodes-to-run-or-timeout
+}
+start-hollow-nodes &
 
+wait
 echo ""
 echo "Master IP: ${MASTER_IP}"
 echo "Password to kubemark master: ${KUBE_PASSWORD}"