Refactored kubemark code into provider-specific and provider-independent parts [Part-3]

2017-02-07 13:07:23 +01:00 · 2017-02-07 13:07:23 +01:00 · 3ac0e22f62
parent 844407e6ca
commit 3ac0e22f62
9 changed files with 67 additions and 133 deletions
--- a/cluster/kubemark/gce/config-default.sh
+++ b/cluster/kubemark/gce/config-default.sh
@ -43,6 +43,7 @@ NETWORK=${KUBE_GCE_NETWORK:-default}
 INSTANCE_PREFIX="${INSTANCE_PREFIX:-"default"}"
 MASTER_NAME="${INSTANCE_PREFIX}-kubemark-master"
 MASTER_TAG="kubemark-master"
+EVENT_STORE_NAME="${INSTANCE_PREFIX}-event-store"
 MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
 CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.240.0.0/11}"
 RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}"
--- a/test/kubemark/cloud-provider-config.sh
+++ b/test/kubemark/cloud-provider-config.sh
@ -15,3 +15,6 @@
 # limitations under the License.

 CLOUD_PROVIDER="${CLOUD_PROVIDER:-gce}"
+CONTAINER_REGISTRY="${CONTAINER_REGISTRY:-gcr.io}"
+PROJECT="${PROJECT:-}"
+KUBEMARK_IMAGE_MAKE_TARGET="${KUBEMARK_IMAGE_MAKE_TARGET:-gcloudpush}"
--- a/test/kubemark/common.sh
+++ b/test/kubemark/common.sh
@ -1,74 +0,0 @@
-#!/bin/bash
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh"
-source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh"
-source "${KUBE_ROOT}/cluster/kubemark/util.sh"
-source "${KUBE_ROOT}/cluster/lib/util.sh"
-
-# hack/lib/init.sh will ovewrite ETCD_VERSION if this is unset
-# what what is default in hack/lib/etcd.sh
-# To avoid it, if it is empty, we set it to 'avoid-overwrite' and
-# clean it after that.
-if [ -z "${ETCD_VERSION:-}" ]; then
-  ETCD_VERSION="avoid-overwrite"
-fi
-source "${KUBE_ROOT}/hack/lib/init.sh"
-if [ "${ETCD_VERSION:-}" == "avoid-overwrite" ]; then
-  ETCD_VERSION=""
-fi
-
-detect-project &> /dev/null
-export PROJECT
-find-release-tars
-
-MASTER_NAME="${INSTANCE_PREFIX}-kubemark-master"
-MASTER_TAG="kubemark-master"
-EVENT_STORE_NAME="${INSTANCE_PREFIX}-event-store"
-
-export KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh"
-export KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
-export RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"
-
-# Runs gcloud compute command with the given parameters. Up to $RETRIES will be made
-# to execute the command.
-# arguments:
-# $@: all stuff that goes after 'gcloud compute'
-function run-gcloud-compute-with-retries {
-  RETRIES="${RETRIES:-3}"
-  for attempt in $(seq 1 ${RETRIES}); do
-    local -r gcloud_result=$(gcloud compute "$@" 2>&1)
-    local -r ret_val="$?"
-    echo "${gcloud_result}"
-    if [[ "${ret_val}" -ne "0" ]]; then
-      if [[ $(echo "${gcloud_result}" | grep -c "already exists") -gt 0 ]]; then
-        if [[ "${attempt}" == 1 ]]; then
-          echo -e "${color_red}Failed to $1 $2 $3 as the resource hasn't been deleted from a previous run.${color_norm}" >& 2
-          exit 1
-        fi
-        echo -e "${color_yellow}Succeeded to $1 $2 $3 in the previous attempt, but status response wasn't received.${color_norm}"
-        return 0
-      fi
-      echo -e "${color_yellow}Attempt $attempt failed to $1 $2 $3. Retrying.${color_norm}" >& 2
-      sleep $(($attempt * 5))
-    else
-      echo -e "${color_green}Succeeded to gcloud compute $1 $2 $3.${color_norm}"
-      return 0
-    fi
-  done
-  echo -e "${color_red}Failed to $1 $2 $3.${color_norm}" >& 2
-  exit 1
-}
--- a/test/kubemark/gce/util.sh
+++ b/test/kubemark/gce/util.sh
@ -14,6 +14,35 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+# Wrapper for gcloud compute, running it $RETRIES times in case of failures.
+# Args:
+# $@: all stuff that goes after 'gcloud compute'
+function run-gcloud-compute-with-retries {
+  RETRIES="${RETRIES:-3}"
+  for attempt in $(seq 1 ${RETRIES}); do
+    local -r gcloud_result=$(gcloud compute "$@" 2>&1)
+    local -r ret_val="$?"
+    echo "${gcloud_result}"
+    if [[ "${ret_val}" -ne "0" ]]; then
+      if [[ $(echo "${gcloud_result}" | grep -c "already exists") -gt 0 ]]; then
+        if [[ "${attempt}" == 1 ]]; then
+          echo -e "${color_red}Failed to $1 $2 $3 as the resource hasn't been deleted from a previous run.${color_norm}" >& 2
+          exit 1
+        fi
+        echo -e "${color_yellow}Succeeded to $1 $2 $3 in the previous attempt, but status response wasn't received.${color_norm}"
+        return 0
+      fi
+      echo -e "${color_yellow}Attempt $attempt failed to $1 $2 $3. Retrying.${color_norm}" >& 2
+      sleep $(($attempt * 5))
+    else
+      echo -e "${color_green}Succeeded to gcloud compute $1 $2 $3.${color_norm}"
+      return 0
+    fi
+  done
+  echo -e "${color_red}Failed to $1 $2 $3.${color_norm}" >& 2
+  exit 1
+}
+
 function create-master-instance-with-resources {
  GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE}"

@ -103,33 +132,3 @@ function delete-master-instance-and-resources {
    	  ${GCLOUD_COMMON_ARGS} || true
  fi
 }
-
-function delete-master-instance-and-resources {
-  GCLOUD_COMMON_ARGS="--project ${PROJECT} --zone ${ZONE} --quiet"
-
-  gcloud compute instances delete "${MASTER_NAME}" \
-      ${GCLOUD_COMMON_ARGS} || true
-  
-  gcloud compute disks delete "${MASTER_NAME}-pd" \
-      ${GCLOUD_COMMON_ARGS} || true
-  
-  gcloud compute disks delete "${MASTER_NAME}-event-pd" \
-      ${GCLOUD_COMMON_ARGS} &> /dev/null || true
-  
-  gcloud compute addresses delete "${MASTER_NAME}-ip" \
-      --project "${PROJECT}" \
-      --region "${REGION}" \
-      --quiet || true
-  
-  gcloud compute firewall-rules delete "${MASTER_NAME}-https" \
-    --project "${PROJECT}" \
-    --quiet || true
-  
-  if [ "${SEPARATE_EVENT_MACHINE:-false}" == "true" ]; then
-    gcloud compute instances delete "${EVENT_STORE_NAME}" \
-        ${GCLOUD_COMMON_ARGS} || true
-  
-    gcloud compute disks delete "${EVENT_STORE_NAME}-pd" \
-        ${GCLOUD_COMMON_ARGS} || true
-  fi
-}
--- a/test/kubemark/get-real-pod-for-hollow-node.sh
+++ b/test/kubemark/get-real-pod-for-hollow-node.sh
@ -1,18 +0,0 @@
-#!/bin/bash
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-kubectl get pod --namespace=kubemark -o json | grep -C100 $1 | grep hollow | head -n1 | cut -d: -f2 | cut -d, -f1 | tr -d \" | tr -d '[[:space:]]'
-echo ""
--- a/test/kubemark/resources/hollow-node_template.json
+++ b/test/kubemark/resources/hollow-node_template.json
@ -52,7 +52,7 @@
 				"containers": [
 				{
 					"name": "hollow-kubelet",
-					"image": "gcr.io/{{project}}/kubemark:latest",
+					"image": "{{registry}}/{{project}}/kubemark:latest",
 					"ports": [
 						{"containerPort": 4194},
 						{"containerPort": 10250},
@ -106,7 +106,7 @@
 				},
 				{
 					"name": "hollow-proxy",
-					"image": "gcr.io/{{project}}/kubemark:latest",
+					"image": "{{registry}}/{{project}}/kubemark:latest",
 					"env": [
 						{
 							"name": "CONTENT_TYPE",
--- a/test/kubemark/run-e2e-tests.sh
+++ b/test/kubemark/run-e2e-tests.sh
@ -22,9 +22,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 # We need an absolute path to KUBE_ROOT
 ABSOLUTE_ROOT=$(readlink -f ${KUBE_ROOT})

-source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh"
 source "${KUBE_ROOT}/cluster/kubemark/util.sh"
-source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh"

 echo "Kubemark master name: ${MASTER_NAME}"

@ -35,7 +33,7 @@ export KUBECONFIG="${ABSOLUTE_ROOT}/test/kubemark/resources/kubeconfig.kubemark"
 export E2E_MIN_STARTUP_PODS=0

 if [[ -z "$@" ]]; then
-	ARGS='--ginkgo.focus=should\sallow\sstarting\s30\spods\sper\snode'
+	ARGS='--ginkgo.focus=\[Feature:performance\]'
 else
 	ARGS=$@
 fi
--- a/test/kubemark/start-kubemark.sh
+++ b/test/kubemark/start-kubemark.sh
@ -19,10 +19,27 @@
 TMP_ROOT="$(dirname "${BASH_SOURCE}")/../.."
 KUBE_ROOT=$(readlink -e ${TMP_ROOT} 2> /dev/null || perl -MCwd -e 'print Cwd::abs_path shift' ${TMP_ROOT})

-source "${KUBE_ROOT}/test/kubemark/common.sh"
 source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh"
 source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh"
 source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh"
+source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh"
+source "${KUBE_ROOT}/cluster/kubemark/util.sh"
+
+# hack/lib/init.sh will ovewrite ETCD_VERSION if this is unset
+# what what is default in hack/lib/etcd.sh
+# To avoid it, if it is empty, we set it to 'avoid-overwrite' and
+# clean it after that.
+if [ -z "${ETCD_VERSION:-}" ]; then
+  ETCD_VERSION="avoid-overwrite"
+fi
+source "${KUBE_ROOT}/hack/lib/init.sh"
+if [ "${ETCD_VERSION:-}" == "avoid-overwrite" ]; then
+  ETCD_VERSION=""
+fi
+
+KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh"
+KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
+RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"

 # Write all environment variables that we need to pass to the kubemark master,
 # locally to the file ${RESOURCE_DIRECTORY}/kubemark-master-env.sh.
@ -153,7 +170,6 @@ EOF
 # Finds the right kubemark binary for 'linux/amd64' platform and uses it to
 # create a docker image for hollow-node and upload it to the appropriate
 # docker container registry for the cloud provider.
-# TODO(shyamjvs): Make the image upload URL and makefile variable w.r.t. provider.
 function create-and-upload-hollow-node-image {
  MAKE_DIR="${KUBE_ROOT}/cluster/images/kubemark"
  KUBEMARK_BIN="$(kube::util::find-binary-for-platform kubemark linux/amd64)"
@ -168,7 +184,7 @@ function create-and-upload-hollow-node-image {
  cd "${MAKE_DIR}"
  RETRIES=3
  for attempt in $(seq 1 ${RETRIES}); do
-    if ! make; then
+    if ! REGISTRY="${CONTAINER_REGISTRY}" PROJECT="${PROJECT}" make "${KUBEMARK_IMAGE_MAKE_TARGET}"; then
      if [[ $((attempt)) -eq "${RETRIES}" ]]; then
        echo "${color_red}Make failed. Exiting.${color_norm}"
        exit 1
@ -281,7 +297,6 @@ current-context: kubemark-context")
    --from-literal=npd.kubeconfig="${NPD_KUBECONFIG_CONTENTS}"

  # Create addon pods.
-  # TODO(shyamjvs): Make path to docker image variable in heapster_template.json.
  mkdir -p "${RESOURCE_DIRECTORY}/addons"
  sed "s/{{MASTER_IP}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json"
  metrics_mem_per_node=4
@ -293,8 +308,8 @@ current-context: kubemark-context")
  "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark"

  # Create the replication controller for hollow-nodes.
-  # TODO(shyamjvs): Make path to docker image variable in hollow-node_template.json.
  sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.json" > "${RESOURCE_DIRECTORY}/hollow-node.json"
+  sed -i'' -e "s/{{registry}}/${CONTAINER_REGISTRY}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
  sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
  sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
  "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.json" --namespace="kubemark"
@ -338,8 +353,11 @@ function wait-for-hollow-nodes-to-run-or-timeout {
 }

 ############################### Main Function ########################################
+detect-project &> /dev/null
+
 # Setup for master.
 echo -e "${color_yellow}STARTING SETUP FOR MASTER${color_norm}"
+find-release-tars
 create-master-environment-file
 create-master-instance-with-resources
 generate-pki-config
--- a/test/kubemark/stop-kubemark.sh
+++ b/test/kubemark/stop-kubemark.sh
@ -18,10 +18,17 @@

 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..

-source "${KUBE_ROOT}/test/kubemark/common.sh"
 source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh"
 source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh"
 source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh"
+source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh"
+source "${KUBE_ROOT}/cluster/kubemark/util.sh"
+
+KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh"
+KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
+RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"
+
+detect-project &> /dev/null

 "${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/addons" &> /dev/null || true
 "${KUBECTL}" delete -f "${RESOURCE_DIRECTORY}/hollow-node.json" &> /dev/null || true