Merge pull request #25173 from ixdy/gcloud-grep-awk-cut

Automatic merge from submit-queue Use --format='value(name)' with gcloud instead of grep/awk/cut Fixing our fragile parsing of `gcloud` is getting old (#24746, #25159, maybe others?). Instead, let's just get the proper output out of `gcloud` in the first place.
2016-05-15 07:46:28 -07:00 · 2016-05-15 07:46:28 -07:00 · 87403cfe7b
parent 9cf6642531 bc7922c374
commit 87403cfe7b
4 changed files with 27 additions and 48 deletions
--- a/cluster/gce/delete-stranded-load-balancers.sh
+++ b/cluster/gce/delete-stranded-load-balancers.sh
@ -18,7 +18,7 @@
 PROJECT=${PROJECT:-kubernetes-jenkins}
 REGION=${REGION:-us-central1}

-LIST=$(gcloud --project=${PROJECT} compute target-pools list | sed 1d | awk '{print $1}')
+LIST=$(gcloud --project=${PROJECT} compute target-pools list --format='value(name)')

 result=0
 for x in ${LIST}; do
--- a/cluster/gce/util.sh
+++ b/cluster/gce/util.sh
@ -33,7 +33,8 @@ elif [[ "${OS_DISTRIBUTION}" == "gci" ]]; then
  # If the user does not set a GCI image for master, we run both master and nodes
  # using the latest GCI dev image.
  if [[ "${MASTER_IMAGE}" != gci* ]]; then
-    gci_images=( $(gcloud compute images list --project google-containers | grep "gci-dev" | cut -d ' ' -f1) )
+    gci_images=( $(gcloud compute images list --project google-containers \
+      --regexp='gci-dev.*' --format='value(name)') )
    MASTER_IMAGE="${gci_images[0]}"
    NODE_IMAGE="${MASTER_IMAGE}"
    NODE_IMAGE_PROJECT="${MASTER_IMAGE_PROJECT}"
@ -113,7 +114,7 @@ function ensure-temp-dir {
 #   PROJECT_REPORTED
 function detect-project () {
  if [[ -z "${PROJECT-}" ]]; then
-    PROJECT=$(gcloud config list project | tail -n 1 | cut -f 3 -d ' ')
+    PROJECT=$(gcloud config list project --format 'value(core.project)')
  fi

  if [[ -z "${PROJECT-}" ]]; then
@ -663,8 +664,7 @@ function create-master() {
  local REGION=${ZONE%-*}
  create-static-ip "${MASTER_NAME}-ip" "${REGION}"
  MASTER_RESERVED_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \
-    --project "${PROJECT}" \
-    --region "${REGION}" -q --format yaml | awk '/^address:/ { print $2 }')
+    --project "${PROJECT}" --region "${REGION}" -q --format='value(address)')

  create-certs "${MASTER_RESERVED_IP}"

@ -881,9 +881,9 @@ function kube-down {

  # Delete autoscaler for nodes if present. We assume that all or none instance groups have an autoscaler
  local autoscaler
-  autoscaler=( $(gcloud compute instance-groups managed list --zone "${ZONE}" --project "${PROJECT}" \
-                 | grep "${NODE_INSTANCE_PREFIX}-group" \
-                 | awk '{print $7}') )
+  autoscaler=( $(gcloud compute instance-groups managed list \
+    --zone "${ZONE}" --project "${PROJECT}" --regexp="${NODE_INSTANCE_PREFIX}-.+" \
+    --format='value(autoscaled)') )
  if [[ "${autoscaler:-}" == "yes" ]]; then
    for group in ${INSTANCE_GROUPS[@]:-}; do
      gcloud compute instance-groups managed stop-autoscaling "${group}" --zone "${ZONE}" --project "${PROJECT}"
@ -895,28 +895,13 @@ function kube-down {
  # change during a cluster upgrade.)
  local template=$(get-template "${PROJECT}")

-  # The gcloud APIs don't return machine parseable error codes/retry information. Therefore the best we can
-  # do is parse the output and special case particular responses we are interested in.
  for group in ${INSTANCE_GROUPS[@]:-}; do
    if gcloud compute instance-groups managed describe "${group}" --project "${PROJECT}" --zone "${ZONE}" &>/dev/null; then
-      deleteCmdOutput=$(gcloud compute instance-groups managed delete --zone "${ZONE}" \
+      gcloud compute instance-groups managed delete \
        --project "${PROJECT}" \
        --quiet \
-        "${group}")
-      if [[ "$deleteCmdOutput" != ""  ]]; then
-        # Managed instance group deletion is done asynchronously, we must wait for it to complete, or subsequent steps fail
-        deleteCmdOperationId=$(echo $deleteCmdOutput | grep "Operation:" | sed "s/.*Operation:[[:space:]]*\([^[:space:]]*\).*/\1/g")
-        if [[ "$deleteCmdOperationId" != ""  ]]; then
-          deleteCmdStatus="PENDING"
-          while [[ "$deleteCmdStatus" != "DONE" ]]
-          do
-            sleep 5
-            deleteCmdOperationOutput=$(gcloud compute instance-groups managed --zone "${ZONE}" --project "${PROJECT}" get-operation $deleteCmdOperationId)
-            deleteCmdStatus=$(echo $deleteCmdOperationOutput | grep -i "status:" | sed "s/.*status:[[:space:]]*\([^[:space:]]*\).*/\1/g")
-            echo "Waiting for MIG deletion to complete. Current status: " $deleteCmdStatus
-          done
-        fi
-      fi
+        --zone "${ZONE}" \
+        "${group}"
    fi
  done

@ -962,7 +947,7 @@ function kube-down {
  minions=( $(gcloud compute instances list \
                --project "${PROJECT}" --zone "${ZONE}" \
                --regexp "${NODE_INSTANCE_PREFIX}-.+" \
-                | awk 'NR >= 2 { print $1 }') )
+                --format='value(name)') )
  # If any minions are running, delete them in batches.
  while (( "${#minions[@]}" > 0 )); do
    echo Deleting nodes "${minions[*]::10}"
@ -1000,7 +985,8 @@ function kube-down {
  # first allows the master to cleanup routes itself.
  local TRUNCATED_PREFIX="${INSTANCE_PREFIX:0:26}"
  routes=( $(gcloud compute routes list --project "${PROJECT}" \
-    --regexp "${TRUNCATED_PREFIX}-.{8}-.{4}-.{4}-.{4}-.{12}" | awk 'NR >= 2 { print $1 }') )
+    --regexp "${TRUNCATED_PREFIX}-.{8}-.{4}-.{4}-.{4}-.{12}"  \
+    --format='value(name)') )
  while (( "${#routes[@]}" > 0 )); do
    echo Deleting routes "${routes[*]::10}"
    gcloud compute routes delete \
@ -1031,16 +1017,11 @@ function kube-down {
 #   NODE_INSTANCE_PREFIX
 #
 # $1: project
-# $2: zone
 function get-template {
-  local template=""
-  if [[ -n $(gcloud compute instance-templates list "${NODE_INSTANCE_PREFIX}"-template --project="${1}" | grep template) ]]; then
-    template="${NODE_INSTANCE_PREFIX}"-template
-  fi
-  echo "${template}"
+  gcloud compute instance-templates list "${NODE_INSTANCE_PREFIX}-template" \
+    --project="${1}" --format='value(name)'
 }

-
 # Checks if there are any present resources related kubernetes cluster.
 #
 # Assumed vars:
@ -1086,7 +1067,7 @@ function check-resources {
  minions=( $(gcloud compute instances list \
                --project "${PROJECT}" --zone "${ZONE}" \
                --regexp "${NODE_INSTANCE_PREFIX}-.+" \
-                | awk 'NR >= 2 { print $1 }') )
+                --format='value(name)') )
  if (( "${#minions[@]}" > 0 )); then
    KUBE_RESOURCE_FOUND="${#minions[@]} matching matching ${NODE_INSTANCE_PREFIX}-.+"
    return 1
@ -1104,7 +1085,7 @@ function check-resources {

  local -a routes
  routes=( $(gcloud compute routes list --project "${PROJECT}" \
-    --regexp "${INSTANCE_PREFIX}-minion-.{4}" | awk 'NR >= 2 { print $1 }') )
+    --regexp "${INSTANCE_PREFIX}-minion-.{4}" --format='value(name)') )
  if (( "${#routes[@]}" > 0 )); then
    KUBE_RESOURCE_FOUND="${#routes[@]} routes matching ${INSTANCE_PREFIX}-minion-.{4}"
    return 1
--- a/cluster/gke/util.sh
+++ b/cluster/gke/util.sh
@ -50,7 +50,7 @@ function prepare-e2e() {
 function detect-project() {
  echo "... in gke:detect-project()" >&2
  if [[ -z "${PROJECT:-}" ]]; then
-    export PROJECT=$("${GCLOUD}" config list project | tail -n 1 | cut -f 3 -d ' ')
+    export PROJECT=$("${GCLOUD}" config list project --format 'value(core.project)')
    echo "... Using project: ${PROJECT}" >&2
  fi
  if [[ -z "${PROJECT:-}" ]]; then
@ -202,7 +202,7 @@ function test-setup() {
  detect-nodes >&2

  # At this point, CLUSTER_NAME should have been used, so its value is final.
-  NODE_TAG=$($GCLOUD compute instances describe ${NODE_NAMES[0]} --project="${PROJECT}" --zone="${ZONE}" | grep -o "gke-${CLUSTER_NAME}-.\{8\}-node" | head -1)
+  NODE_TAG=$($GCLOUD compute instances describe ${NODE_NAMES[0]} --project="${PROJECT}" --zone="${ZONE}" --format='value(tags.items)' | grep -o "gke-${CLUSTER_NAME}-.\{8\}-node")
  OLD_NODE_TAG="k8s-${CLUSTER_NAME}-node"

  # Open up port 80 & 8080 so common containers on minions can be reached.
@ -233,8 +233,8 @@ function detect-master() {
  echo "... in gke:detect-master()" >&2
  detect-project >&2
  KUBE_MASTER_IP=$("${GCLOUD}" ${CMD_GROUP:-} container clusters describe \
-    --project="${PROJECT}" --zone="${ZONE}" "${CLUSTER_NAME}" \
-    | grep endpoint | cut -f 2 -d ' ')
+    --project="${PROJECT}" --zone="${ZONE}" --format='value(endpoint)' \
+    "${CLUSTER_NAME}")
 }

 # Assumed vars:
@ -280,9 +280,10 @@ function detect-node-names {
 #   NODE_INSTANCE_GROUP
 function detect-node-instance-group {
  echo "... in gke:detect-node-instance-group()" >&2
-  NODE_INSTANCE_GROUP=$("${GCLOUD}" ${CMD_GROUP:-} container clusters describe \
-    --project="${PROJECT}" --zone="${ZONE}" "${CLUSTER_NAME}" \
-    | grep instanceGroupManagers | grep "${ZONE}" | cut -d '/' -f 11)
+  local url=$("${GCLOUD}" ${CMD_GROUP:-} container clusters describe \
+    --project="${PROJECT}" --zone="${ZONE}" \
+    --format='value(instanceGroupUrls)' "${CLUSTER_NAME}")
+  NODE_INSTANCE_GROUP="${url##*/}"
 }

 # SSH to a node by name ($1) and run a command ($2).
--- a/hack/jenkins/e2e-runner.sh
+++ b/hack/jenkins/e2e-runner.sh
@ -34,10 +34,7 @@ function fetch_output_tars() {
 }

 function fetch_server_version_tars() {
-    local -r msg=$(gcloud ${CMD_GROUP:-} container get-server-config --project=${PROJECT} --zone=${ZONE} | grep defaultClusterVersion)
-    # msg will look like "defaultClusterVersion: 1.0.1". Strip
-    # everything up to, including ": "
-    local -r build_version="v${msg##*: }"
+    local -r build_version="v$(gcloud ${CMD_GROUP:-} container get-server-config --project=${PROJECT} --zone=${ZONE}  --format='value(defaultClusterVersion)')"
    fetch_tars_from_gcs "release" "${build_version}"
    unpack_binaries
 }