diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index e724a28088..370939b7c9 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -44,6 +44,7 @@ PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false} PREEMPTIBLE_MASTER=${PREEMPTIBLE_MASTER:-false} KUBE_DELETE_NODES=${KUBE_DELETE_NODES:-true} KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-false} +CREATE_CUSTOM_NETWORK=${CREATE_CUSTOM_NETWORK:-false} MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}} NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}} @@ -83,6 +84,9 @@ RKT_VERSION=${KUBE_RKT_VERSION:-1.23.0} RKT_STAGE1_IMAGE=${KUBE_RKT_STAGE1_IMAGE:-coreos.com/rkt/stage1-coreos} NETWORK=${KUBE_GCE_NETWORK:-default} +if [[ "${CREATE_CUSTOM_NETWORK}" == true ]]; then + SUBNETWORK="${SUBNETWORK:-${NETWORK}-custom-subnet}" +fi INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX:-kubernetes}" CLUSTER_NAME="${CLUSTER_NAME:-${INSTANCE_PREFIX}}" MASTER_NAME="${INSTANCE_PREFIX}-master" @@ -94,6 +98,9 @@ NODE_TAG="${INSTANCE_PREFIX}-minion" CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-$(get-cluster-ip-range)}" MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}" +# NODE_IP_RANGE is used when ENABLE_IP_ALIASES=true or CREATE_CUSTOM_NETWORK=true. +# It is the primary range in the subnet and is the range used for node instance IPs. +NODE_IP_RANGE="$(get-node-ip-range)" if [[ "${FEDERATION:-}" == true ]]; then NODE_SCOPES="${NODE_SCOPES:-monitoring,logging-write,storage-ro,https://www.googleapis.com/auth/ndev.clouddns.readwrite}" @@ -233,9 +240,6 @@ if [ ${ENABLE_IP_ALIASES} = true ]; then IP_ALIAS_SUBNETWORK=${KUBE_GCE_IP_ALIAS_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-default} # Reserve the services IP space to avoid being allocated for other GCP resources. SERVICE_CLUSTER_IP_SUBNETWORK=${KUBE_GCE_SERVICE_CLUSTER_IP_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-services} - # NODE_IP_RANGE is used when ENABLE_IP_ALIASES=true. It is the primary range in - # the subnet and is the range used for node instance IPs. - NODE_IP_RANGE="$(get-node-ip-range)" # Add to the provider custom variables. PROVIDER_VARS="${PROVIDER_VARS:-} ENABLE_IP_ALIASES" fi diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index c6b87e4130..eb34efd367 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -43,6 +43,7 @@ PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false} PREEMPTIBLE_MASTER=${PREEMPTIBLE_MASTER:-false} KUBE_DELETE_NODES=${KUBE_DELETE_NODES:-true} KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-true} +CREATE_CUSTOM_NETWORK=${CREATE_CUSTOM_NETWORK:-false} MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}} NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}} @@ -83,6 +84,9 @@ RKT_VERSION=${KUBE_RKT_VERSION:-1.23.0} RKT_STAGE1_IMAGE=${KUBE_RKT_STAGE1_IMAGE:-coreos.com/rkt/stage1-coreos} NETWORK=${KUBE_GCE_NETWORK:-e2e-test-${USER}} +if [[ "${CREATE_CUSTOM_NETWORK}" == true ]]; then + SUBNETWORK="${SUBNETWORK:-${NETWORK}-custom-subnet}" +fi INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX:-e2e-test-${USER}}" CLUSTER_NAME="${CLUSTER_NAME:-${INSTANCE_PREFIX}}" MASTER_NAME="${INSTANCE_PREFIX}-master" @@ -94,8 +98,8 @@ NODE_TAG="${INSTANCE_PREFIX}-minion" CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-$(get-cluster-ip-range)}" MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}" -# NODE_IP_RANGE is used when ENABLE_IP_ALIASES=true. It is the primary range in -# the subnet and is the range used for node instance IPs. +# NODE_IP_RANGE is used when ENABLE_IP_ALIASES=true or CREATE_CUSTOM_NETWORK=true. +# It is the primary range in the subnet and is the range used for node instance IPs. NODE_IP_RANGE="$(get-node-ip-range)" RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}" @@ -273,9 +277,6 @@ if [ ${ENABLE_IP_ALIASES} = true ]; then IP_ALIAS_SUBNETWORK=${KUBE_GCE_IP_ALIAS_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-default} # Reserve the services IP space to avoid being allocated for other GCP resources. SERVICE_CLUSTER_IP_SUBNETWORK=${KUBE_GCE_SERVICE_CLUSTER_IP_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-services} - # NODE_IP_RANGE is used when ENABLE_IP_ALIASES=true. It is the primary range in - # the subnet and is the range used for node instance IPs. - NODE_IP_RANGE="${NODE_IP_RANGE:-10.40.0.0/22}" # Add to the provider custom variables. PROVIDER_VARS="${PROVIDER_VARS:-} ENABLE_IP_ALIASES" fi diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index e62f362e54..f6d8954e04 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -773,10 +773,14 @@ function check-existing() { function create-network() { if ! gcloud compute networks --project "${NETWORK_PROJECT}" describe "${NETWORK}" &>/dev/null; then - echo "Creating new network: ${NETWORK}" # The network needs to be created synchronously or we have a race. The # firewalls can be added concurrent with instance creation. - gcloud compute networks create --project "${NETWORK_PROJECT}" "${NETWORK}" --mode=auto + local network_mode="auto" + if [[ "${CREATE_CUSTOM_NETWORK:-}" == "true" ]]; then + network_mode="custom" + fi + echo "Creating new ${network_mode} network: ${NETWORK}" + gcloud compute networks create --project "${NETWORK_PROJECT}" "${NETWORK}" --mode="${network_mode}" else PREEXISTING_NETWORK=true PREEXISTING_NETWORK_MODE="$(gcloud compute networks list ${NETWORK} --project ${NETWORK_PROJECT} --format='value(x_gcloud_mode)' || true)" @@ -833,6 +837,8 @@ function create-subnetworks() { else echo "${color_yellow}Using pre-existing network ${NETWORK}, subnets won't be expanded to /19!${color_norm}" fi + elif [[ "${CREATE_CUSTOM_NETWORK:-}" == "true" && "${PREEXISTING_NETWORK}" != "true" ]]; then + gcloud compute networks subnets create "${SUBNETWORK}" --project "${NETWORK_PROJECT}" --region "${REGION}" --network "${NETWORK}" --range "${NODE_IP_RANGE}" fi return;; *) echo "${color_red}Invalid argument to ENABLE_IP_ALIASES${color_norm}" @@ -852,11 +858,6 @@ function create-subnetworks() { exit 1 fi - if [[ -z ${NODE_IP_RANGE:-} ]]; then - echo "${color_red}NODE_IP_RANGE must be specified{color_norm}" - exit 1 - fi - echo "Creating subnet ${NETWORK}:${IP_ALIAS_SUBNETWORK}" gcloud beta compute networks subnets create \ ${IP_ALIAS_SUBNETWORK} \ @@ -935,15 +936,18 @@ function delete-network() { function delete-subnetworks() { if [[ ${ENABLE_IP_ALIASES:-} != "true" ]]; then - if [[ "${ENABLE_BIG_CLUSTER_SUBNETS}" = "true" ]]; then - # If running in custom mode network we need to delete subnets - mode="$(gcloud compute networks list ${NETWORK} --project ${NETWORK_PROJECT} --format='value(x_gcloud_mode)' || true)" - if [[ "${mode}" == "custom" ]]; then + # If running in custom mode network we need to delete subnets + mode="$(gcloud compute networks list ${NETWORK} --project ${NETWORK_PROJECT} --format='value(x_gcloud_mode)' || true)" + if [[ "${mode}" == "custom" ]]; then + if [[ "${ENABLE_BIG_CLUSTER_SUBNETS}" = "true" ]]; then echo "Deleting default subnets..." # This value should be kept in sync with number of regions. local parallelism=9 gcloud compute networks subnets list --network="${NETWORK}" --project "${NETWORK_PROJECT}" --format='value(region.basename())' | \ xargs -i -P ${parallelism} gcloud --quiet compute networks subnets delete "${NETWORK}" --project "${NETWORK_PROJECT}" --region="{}" || true + elif [[ "${CREATE_CUSTOM_NETWORK:-}" == "true" ]]; then + echo "Deleting custom subnet..." + gcloud --quiet compute networks subnets delete "${SUBNETWORK}" --project "${NETWORK_PROJECT}" --region="${REGION}" || true fi fi return