From 345c65847fb877dbf19e60892fe0d926a92ffc67 Mon Sep 17 00:00:00 2001 From: Bowei Du Date: Mon, 27 Feb 2017 00:31:13 -0800 Subject: [PATCH] Add KUBE_GCE_ENABLE_IP_ALIASES flag to the cluster turn up scripts. KUBE_GCE_ENABLE_IP_ALIASES=true will enable allocation of PodCIDR ips using the ip alias mechanism rather than using routes. NODE_IP_RANGE will control the node instance IP cidr KUBE_GCE_IP_ALIAS_SIZE controls the size of each podCIDR IP_ALIAS_SUBNETWORK controls the name of the subnet created for the cluster --- cluster/common.sh | 11 ++ cluster/gce/config-default.sh | 24 +++- cluster/gce/config-test.sh | 24 ++++ cluster/gce/configure-vm.sh | 5 + .../gce/container-linux/configure-helper.sh | 6 +- cluster/gce/container-linux/master-helper.sh | 46 +++++-- cluster/gce/gci/configure-helper.sh | 8 +- cluster/gce/gci/master-helper.sh | 45 +++++-- cluster/gce/util.sh | 116 +++++++++++++++++- cluster/kube-up.sh | 1 - cluster/kube-util.sh | 6 + 11 files changed, 257 insertions(+), 35 deletions(-) diff --git a/cluster/common.sh b/cluster/common.sh index b43b4adc73..e8e51897f4 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -734,6 +734,17 @@ EOF FEATURE_GATES: $(yaml-quote ${FEATURE_GATES}) EOF fi + + if [ -n "${PROVIDER_VARS:-}" ]; then + local var_name + local var_value + + for var_name in ${PROVIDER_VARS}; do + eval "local var_value=\$(yaml-quote \${${var_name}})" + echo "${var_name}: ${var_value}" >>$file + done + fi + if [[ "${master}" == "true" ]]; then # Master-only env vars. cat >>$file <>/srv/salt-overlay/pillar/cluster-params.sls scheduling_algorithm_provider: '$(echo "${SCHEDULING_ALGORITHM_PROVIDER}" | sed -e "s/'/''/g")' +EOF + fi + if [ -n "${ENABLE_IP_ALIASES:-}" ]; then + cat <>/srv/salt-overlay/pillar/cluster-params.sls +enable_ip_aliases: '$(echo "$ENABLE_IP_ALIASES" | sed -e "s/'/''/g")' EOF fi } diff --git a/cluster/gce/container-linux/configure-helper.sh b/cluster/gce/container-linux/configure-helper.sh index 5e3043b45f..0d893622fb 100755 --- a/cluster/gce/container-linux/configure-helper.sh +++ b/cluster/gce/container-linux/configure-helper.sh @@ -896,7 +896,7 @@ function start-kube-apiserver { local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty" # Enable ABAC mode unless the user explicitly opts out with ENABLE_LEGACY_ABAC=false - if [[ "${ENABLE_LEGACY_ABAC:-}" != "false" ]]; then + if [[ "${ENABLE_LEGACY_ABAC:-}" != "false" ]]; then echo "Warning: Enabling legacy ABAC policy. All service accounts will have superuser API access. Set ENABLE_LEGACY_ABAC=false to disable this." # Create the ABAC file if it doesn't exist yet, or if we have a KUBE_USER set (to ensure the right user is given permissions) if [[ -n "${KUBE_USER:-}" || ! -e /etc/srv/kubernetes/abac-authz-policy.jsonl ]]; then @@ -997,6 +997,10 @@ function start-kube-controller-manager { if [[ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]]; then params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}" fi + if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then + params+=" --cidr-allocator-type=CloudAllocator" + params+=" --configure-cloud-routes=false" + fi if [[ -n "${FEATURE_GATES:-}" ]]; then params+=" --feature-gates=${FEATURE_GATES}" fi diff --git a/cluster/gce/container-linux/master-helper.sh b/cluster/gce/container-linux/master-helper.sh index 1d250d5ef2..5e0adbc359 100755 --- a/cluster/gce/container-linux/master-helper.sh +++ b/cluster/gce/container-linux/master-helper.sh @@ -31,11 +31,11 @@ source "${KUBE_ROOT}/cluster/gce/container-linux/helper.sh" # detect-project # get-bearer-token function create-master-instance { - local address_opt="" - [[ -n ${1:-} ]] && address_opt="--address ${1}" + local address="" + [[ -n ${1:-} ]] && address="${1}" write-master-env - create-master-instance-internal "${MASTER_NAME}" "${address_opt}" + create-master-instance-internal "${MASTER_NAME}" "${address}" } function replicate-master-instance() { @@ -65,38 +65,58 @@ function replicate-master-instance() { function create-master-instance-internal() { + local gcloud="gcloud" + if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then + gcloud="gcloud alpha" + fi + local -r master_name="${1}" - local -r address_option="${2:-}" + local -r address="${2:-}" local preemptible_master="" if [[ "${PREEMPTIBLE_MASTER:-}" == "true" ]]; then preemptible_master="--preemptible --maintenance-policy TERMINATE" fi - gcloud compute instances create "${master_name}" \ - ${address_option} \ + local network=$(make-gcloud-network-argument \ + "${NETWORK}" "${address:-}" \ + "${ENABLE_IP_ALIASES:-}" "${IP_ALIAS_SUBNETWORK:-}" "${IP_ALIAS_SIZE:-}") + + local metadata="kube-env=${KUBE_TEMP}/master-kube-env.yaml" + metadata="${metadata},user-data=${KUBE_ROOT}/cluster/gce/container-linux/master.yaml" + metadata="${metadata},configure-sh=${KUBE_ROOT}/cluster/gce/container-linux/configure.sh" + metadata="${metadata},cluster-name=${KUBE_TEMP}/cluster-name.txt" + + local disk="name=${master_name}-pd" + disk="${disk},device-name=master-pd" + disk="${disk},mode=rw" + disk="${disk},boot=no" + disk="${disk},auto-delete=no" + + ${gcloud} compute instances create "${master_name}" \ --project "${PROJECT}" \ --zone "${ZONE}" \ --machine-type "${MASTER_SIZE}" \ --image-project="${MASTER_IMAGE_PROJECT}" \ --image "${MASTER_IMAGE}" \ --tags "${MASTER_TAG}" \ - --network "${NETWORK}" \ --scopes "storage-ro,compute-rw,monitoring,logging-write" \ - --can-ip-forward \ - --metadata-from-file \ - "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/container-linux/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/container-linux/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \ - --disk "name=${master_name}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \ + --metadata-from-file "${metadata}" \ + --disk "${disk}" \ --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-30}" \ - ${preemptible_master} + ${preemptible_master} \ + ${network} } function get-metadata() { local zone="${1}" local name="${2}" local key="${3}" + + local metadata_url="http://metadata.google.internal/computeMetadata/v1/instance/attributes/${key}" + gcloud compute ssh "${name}" \ --project "${PROJECT}" \ --zone "${zone}" \ - --command "curl \"http://metadata.google.internal/computeMetadata/v1/instance/attributes/${key}\" -H \"Metadata-Flavor: Google\"" 2>/dev/null + --command "curl '${metadata_url}' -H 'Metadata-Flavor: Google'" 2>/dev/null } diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 7732af0002..2d95bc5a35 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -280,7 +280,7 @@ function create-master-pki { # and should never be touched again (except perhaps an additional service # account, see NB below.) One exception is if METADATA_CLOBBERS_CONFIG is # enabled. In that case the basic_auth.csv file will be rewritten to make -# sure it matches the metadata source of truth. +# sure it matches the metadata source of truth. function create-master-auth { echo "Creating master auth files" local -r auth_dir="/etc/srv/kubernetes" @@ -1100,7 +1100,7 @@ function start-kube-apiserver { local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty" # Enable ABAC mode unless the user explicitly opts out with ENABLE_LEGACY_ABAC=false - if [[ "${ENABLE_LEGACY_ABAC:-}" != "false" ]]; then + if [[ "${ENABLE_LEGACY_ABAC:-}" != "false" ]]; then echo "Warning: Enabling legacy ABAC policy. All service accounts will have superuser API access. Set ENABLE_LEGACY_ABAC=false to disable this." # Create the ABAC file if it doesn't exist yet, or if we have a KUBE_USER set (to ensure the right user is given permissions) if [[ -n "${KUBE_USER:-}" || ! -e /etc/srv/kubernetes/abac-authz-policy.jsonl ]]; then @@ -1205,6 +1205,10 @@ function start-kube-controller-manager { if [[ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]]; then params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}" fi + if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then + params+=" --cidr-allocator-type=CloudAllocator" + params+=" --configure-cloud-routes=false" + fi if [[ -n "${FEATURE_GATES:-}" ]]; then params+=" --feature-gates=${FEATURE_GATES}" fi diff --git a/cluster/gce/gci/master-helper.sh b/cluster/gce/gci/master-helper.sh index 8dbc854929..6357202c16 100755 --- a/cluster/gce/gci/master-helper.sh +++ b/cluster/gce/gci/master-helper.sh @@ -31,12 +31,12 @@ source "${KUBE_ROOT}/cluster/gce/gci/helper.sh" # detect-project # get-bearer-token function create-master-instance { - local address_opt="" - [[ -n ${1:-} ]] && address_opt="--address ${1}" + local address="" + [[ -n ${1:-} ]] && address="${1}" write-master-env ensure-gci-metadata-files - create-master-instance-internal "${MASTER_NAME}" "${address_opt}" + create-master-instance-internal "${MASTER_NAME}" "${address}" } function replicate-master-instance() { @@ -74,30 +74,51 @@ function replicate-master-instance() { function create-master-instance-internal() { + local gcloud="gcloud" + if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then + gcloud="gcloud alpha" + fi + local -r master_name="${1}" - local -r address_option="${2:-}" + local -r address="${2:-}" local preemptible_master="" if [[ "${PREEMPTIBLE_MASTER:-}" == "true" ]]; then preemptible_master="--preemptible --maintenance-policy TERMINATE" fi - gcloud compute instances create "${master_name}" \ - ${address_option} \ + local network=$(make-gcloud-network-argument \ + "${NETWORK}" "${address:-}" \ + "${ENABLE_IP_ALIASES:-}" "${IP_ALIAS_SUBNETWORK:-}" "${IP_ALIAS_SIZE:-}") + + local metadata="kube-env=${KUBE_TEMP}/master-kube-env.yaml" + metadata="${metadata},user-data=${KUBE_ROOT}/cluster/gce/gci/master.yaml" + metadata="${metadata},configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh" + metadata="${metadata},cluster-name=${KUBE_TEMP}/cluster-name.txt" + metadata="${metadata},gci-update-strategy=${KUBE_TEMP}/gci-update.txt" + metadata="${metadata},gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt" + metadata="${metadata},gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt" + metadata="${metadata},kube-master-certs=${KUBE_TEMP}/kube-master-certs.yaml" + + local disk="name=${master_name}-pd" + disk="${disk},device-name=master-pd" + disk="${disk},mode=rw" + disk="${disk},boot=no" + disk="${disk},auto-delete=no" + + ${gcloud} compute instances create "${master_name}" \ --project "${PROJECT}" \ --zone "${ZONE}" \ --machine-type "${MASTER_SIZE}" \ --image-project="${MASTER_IMAGE_PROJECT}" \ --image "${MASTER_IMAGE}" \ --tags "${MASTER_TAG}" \ - --network "${NETWORK}" \ --scopes "storage-ro,compute-rw,monitoring,logging-write" \ - --can-ip-forward \ - --metadata-from-file \ - "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/gci/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt,gci-update-strategy=${KUBE_TEMP}/gci-update.txt,gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt,gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt,kube-master-certs=${KUBE_TEMP}/kube-master-certs.yaml" \ - --disk "name=${master_name}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \ + --metadata-from-file "${metadata}" \ + --disk "${disk}" \ --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \ - ${preemptible_master} + ${preemptible_master} \ + ${network} } function get-metadata() { diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 39b9e5bc76..51a3fcbdd7 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -449,6 +449,35 @@ function create-firewall-rule() { done } +# Format the string argument for gcloud network. +function make-gcloud-network-argument() { + local network="$1" + local address="$2" # optional + local enable_ip_alias="$3" # optional + local alias_subnetwork="$4" # optional + local alias_size="$5" # optional + + local ret="" + + if [[ "${enable_ip_alias}" == 'true' ]]; then + ret="--network-interface" + ret="${ret} network=${network}" + # If address is omitted, instance will not receive an external IP. + ret="${ret},address=${address:-}" + ret="${ret},subnet=${alias_subnetwork}" + ret="${ret},aliases=pods-default:${alias_size}" + ret="${ret} --no-can-ip-forward" + else + ret="--network ${network}" + ret="${ret} --can-ip-forward" + if [[ -n ${address:-} ]]; then + ret="${ret} --address ${address}" + fi + fi + + echo "${ret}" +} + # $1: version (required) function get-template-name-from-version() { # trim template name to pass gce name validation @@ -475,20 +504,34 @@ function create-node-template() { fi fi - local attempt=1 + local gcloud="gcloud" + if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then + gcloud="gcloud alpha" + fi + local preemptible_minions="" if [[ "${PREEMPTIBLE_NODE}" == "true" ]]; then preemptible_minions="--preemptible --maintenance-policy TERMINATE" fi + local local_ssds="" if [ ! -z ${NODE_LOCAL_SSDS+x} ]; then for i in $(seq ${NODE_LOCAL_SSDS}); do local_ssds="$local_ssds--local-ssd=interface=SCSI " done fi + + local network=$(make-gcloud-network-argument \ + "${NETWORK}" "" \ + "${ENABLE_IP_ALIASES:-}" \ + "${IP_ALIAS_SUBNETWORK:-}" \ + "${IP_ALIAS_SIZE:-}") + + local attempt=1 while true; do echo "Attempt ${attempt} to create ${1}" >&2 - if ! gcloud compute instance-templates create "$template_name" \ + if ! ${gcloud} compute instance-templates create \ + "$template_name" \ --project "${PROJECT}" \ --machine-type "${NODE_SIZE}" \ --boot-disk-type "${NODE_DISK_TYPE}" \ @@ -496,11 +539,11 @@ function create-node-template() { --image-project="${NODE_IMAGE_PROJECT}" \ --image "${NODE_IMAGE}" \ --tags "${NODE_TAG}" \ - --network "${NETWORK}" \ ${local_ssds} \ + --region "${REGION}" \ + ${network} \ ${preemptible_minions} \ $2 \ - --can-ip-forward \ --metadata-from-file $(echo ${@:3} | tr ' ' ',') >&2; then if (( attempt > 5 )); then echo -e "${color_red}Failed to create instance template $template_name ${color_norm}" >&2 @@ -597,6 +640,7 @@ function kube-up() { if [[ ${KUBE_USE_EXISTING_MASTER:-} == "true" ]]; then detect-master parse-master-env + create-subnetwork create-nodes elif [[ ${KUBE_REPLICATE_EXISTING_MASTER:-} == "true" ]]; then if [[ "${MASTER_OS_DISTRIBUTION}" != "gci" && "${MASTER_OS_DISTRIBUTION}" != "debian" ]]; then @@ -612,6 +656,7 @@ function kube-up() { else check-existing create-network + create-subnetwork write-cluster-name create-autoscaler-config create-master @@ -680,6 +725,48 @@ function create-network() { fi } +function create-subnetwork() { + case ${ENABLE_IP_ALIASES} in + true) ;; + false) return;; + *) echo "${color_red}Invalid argument to ENABLE_IP_ALIASES${color_norm}" + exit 1;; + esac + + # Look for the subnet, it must exist and have a secondary range + # configured. + local subnet=$(gcloud alpha compute networks subnets describe \ + --region ${REGION} ${IP_ALIAS_SUBNETWORK} 2>/dev/null) + if [[ -z ${subnet} ]]; then + # Only allow auto-creation for default subnets + if [[ ${IP_ALIAS_SUBNETWORK} != ${INSTANCE_PREFIX}-subnet-default ]]; then + echo "${color_red}Subnetwork ${NETWORK}:${IP_ALIAS_SUBNETWORK} does not exist${color_norm}" + exit 1 + fi + + if [ -z ${NODE_IP_RANGE:-} ]; then + echo "${color_red}NODE_IP_RANGE must be specified{color_norm}" + exit 1 + fi + + echo "Creating subnet ${NETWORK}:${IP_ALIAS_SUBNETWORK}" + gcloud alpha compute networks subnets create \ + ${IP_ALIAS_SUBNETWORK} \ + --description "Automatically generated subnet for ${INSTANCE_PREFIX} cluster. This will be removed on cluster teardown." \ + --network ${NETWORK} \ + --region ${REGION} \ + --range ${NODE_IP_RANGE} \ + --secondary-range "name=pods-default,range=${CLUSTER_IP_RANGE}" + + echo "Created subnetwork ${IP_ALIAS_SUBNETWORK}" + else + if ! echo ${subnet} | grep --quiet secondaryIpRanges ${subnet}; then + echo "${color_red}Subnet ${IP_ALIAS_SUBNETWORK} does not have a secondary range${color_norm}" + exit 1 + fi + fi +} + function delete-firewall-rules() { for fw in $@; do if [[ -n $(gcloud compute firewall-rules --project "${PROJECT}" describe "${fw}" --format='value(name)' 2>/dev/null || true) ]]; then @@ -701,6 +788,24 @@ function delete-network() { fi } +function delete-subnetwork() { + if [[ ${ENABLE_IP_ALIASES:-} != "true" ]]; then + return + fi + + # Only delete automatically created subnets. + if [[ ${IP_ALIAS_SUBNETWORK} != ${INSTANCE_PREFIX}-subnet-default ]]; then + return + fi + + echo "Removing auto-created subnet ${NETWORK}:${IP_ALIAS_SUBNETWORK}" + if [[ -n $(gcloud alpha compute networks subnets describe \ + --region ${REGION} ${IP_ALIAS_SUBNETWORK} 2>/dev/null) ]]; then + gcloud alpha --quiet compute networks subnets delete \ + --region ${REGION} ${IP_ALIAS_SUBNETWORK} + fi +} + # Assumes: # NUM_NODES # Sets: @@ -1414,6 +1519,9 @@ function kube-down() { "${CLUSTER_NAME}-default-internal-node" \ "${NETWORK}-default-ssh" \ "${NETWORK}-default-internal" # Pre-1.5 clusters + + delete-subnetwork + if [[ "${KUBE_DELETE_NETWORK}" == "true" ]]; then delete-network || true # might fail if there are leaked firewall rules fi diff --git a/cluster/kube-up.sh b/cluster/kube-up.sh index a6eb618888..7877fb90e1 100755 --- a/cluster/kube-up.sh +++ b/cluster/kube-up.sh @@ -32,7 +32,6 @@ fi source "${KUBE_ROOT}/cluster/kube-util.sh" - if [ -z "${ZONE-}" ]; then echo "... Starting cluster using provider: ${KUBERNETES_PROVIDER}" >&2 else diff --git a/cluster/kube-util.sh b/cluster/kube-util.sh index 4520838314..ae9c686b21 100644 --- a/cluster/kube-util.sh +++ b/cluster/kube-util.sh @@ -28,6 +28,12 @@ else KUBERNETES_PROVIDER="${KUBERNETES_PROVIDER:-gce}" fi +# PROVIDER_VARS is a list of cloud provider specific variables. Note: +# this is a list of the _names_ of the variables, not the value of the +# variables. Providers can add variables to be appended to kube-env. +# (see `build-kube-env`). +PROVIDER_VARS="" + PROVIDER_UTILS="${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" if [ -f ${PROVIDER_UTILS} ]; then source "${PROVIDER_UTILS}"