From 551ffbe7bfd8bacfcebdb2a0db0450d3b204d580 Mon Sep 17 00:00:00 2001 From: Jing Ai Date: Mon, 20 Nov 2017 21:11:50 -0800 Subject: [PATCH] Implement upgrade-aliases.sh to migrate a route-based k8s cluster to use IP aliases in GCE. --- cluster/gce/config-default.sh | 5 + .../gce/container-linux/configure-helper.sh | 2 +- cluster/gce/container-linux/master-helper.sh | 9 +- cluster/gce/gci/configure-helper.sh | 2 +- cluster/gce/upgrade-aliases.sh | 165 ++++++++++++++++++ pkg/controller/node/ipam/sync/sync.go | 8 +- 6 files changed, 182 insertions(+), 9 deletions(-) create mode 100755 cluster/gce/upgrade-aliases.sh diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 3c9cb99af9..cf2f1f71da 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -253,14 +253,19 @@ ENABLE_RESCHEDULER="${KUBE_ENABLE_RESCHEDULER:-true}" # IP_ALIAS_SUBNETWORK is the subnetwork to allocate from. If empty, a # new subnetwork will be created for the cluster. ENABLE_IP_ALIASES=${KUBE_GCE_ENABLE_IP_ALIASES:-false} +NODE_IPAM_MODE=${KUBE_GCE_NODE_IPAM_MODE:-RangeAllocator} if [ ${ENABLE_IP_ALIASES} = true ]; then # Size of ranges allocated to each node. Currently supports only /32 and /24. IP_ALIAS_SIZE=${KUBE_GCE_IP_ALIAS_SIZE:-/24} IP_ALIAS_SUBNETWORK=${KUBE_GCE_IP_ALIAS_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-default} # Reserve the services IP space to avoid being allocated for other GCP resources. SERVICE_CLUSTER_IP_SUBNETWORK=${KUBE_GCE_SERVICE_CLUSTER_IP_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-services} + NODE_IPAM_MODE=${KUBE_GCE_NODE_IPAM_MODE:-CloudAllocator} + SECONDARY_RANGE_NAME=${SECONDARY_RANGE_NAME:-} # Add to the provider custom variables. PROVIDER_VARS="${PROVIDER_VARS:-} ENABLE_IP_ALIASES" + PROVIDER_VARS="${PROVIDER_VARS:-} NODE_IPAM_MODE" + PROVIDER_VARS="${PROVIDER_VARS:-} SECONDARY_RANGE_NAME" fi # Enable GCE Alpha features. diff --git a/cluster/gce/container-linux/configure-helper.sh b/cluster/gce/container-linux/configure-helper.sh index ac54e662f3..8e90a695e8 100755 --- a/cluster/gce/container-linux/configure-helper.sh +++ b/cluster/gce/container-linux/configure-helper.sh @@ -1104,7 +1104,7 @@ function start-kube-controller-manager { params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}" fi if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then - params+=" --cidr-allocator-type=CloudAllocator" + params+=" --cidr-allocator-type=${NODE_IPAM_MODE}" params+=" --configure-cloud-routes=false" fi if [[ -n "${FEATURE_GATES:-}" ]]; then diff --git a/cluster/gce/container-linux/master-helper.sh b/cluster/gce/container-linux/master-helper.sh index e727349f2c..f8177fd308 100755 --- a/cluster/gce/container-linux/master-helper.sh +++ b/cluster/gce/container-linux/master-helper.sh @@ -79,9 +79,16 @@ function create-master-instance-internal() { preemptible_master="--preemptible --maintenance-policy TERMINATE" fi + local enable_ip_aliases + if [[ "${NODE_IPAM_MODE:-}" == "CloudAllocator" ]]; then + enable_ip_aliases=true + else + enable_ip_aliases=false + fi + local network=$(make-gcloud-network-argument \ "${NETWORK_PROJECT}" "${REGION}" "${NETWORK}" "${SUBNETWORK:-}" \ - "${address:-}" "${ENABLE_IP_ALIASES:-}" "${IP_ALIAS_SIZE:-}") + "${address:-}" "${enable_ip_aliases:-}" "${IP_ALIAS_SIZE:-}") local metadata="kube-env=${KUBE_TEMP}/master-kube-env.yaml" metadata="${metadata},user-data=${KUBE_ROOT}/cluster/gce/container-linux/master.yaml" diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index f5407936ec..00bcf556d0 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1601,7 +1601,7 @@ function start-kube-controller-manager { params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}" fi if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then - params+=" --cidr-allocator-type=CloudAllocator" + params+=" --cidr-allocator-type=${NODE_IPAM_MODE}" params+=" --configure-cloud-routes=false" fi if [[ -n "${FEATURE_GATES:-}" ]]; then diff --git a/cluster/gce/upgrade-aliases.sh b/cluster/gce/upgrade-aliases.sh new file mode 100755 index 0000000000..311f629d4a --- /dev/null +++ b/cluster/gce/upgrade-aliases.sh @@ -0,0 +1,165 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# !!!EXPERIMENTAL!!! Upgrade a k8s cluster from routes to IP aliases for +# node connectivity on GCE. This is only for migration. + +set -o errexit +set -o nounset +set -o pipefail + +if [[ "${KUBERNETES_PROVIDER:-gce}" != "gce" ]]; then + echo "!!! ${1} only works on GCE" >&2 + exit 1 +fi + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. +source "${KUBE_ROOT}/hack/lib/util.sh" +source "${KUBE_ROOT}/cluster/kube-util.sh" + +# Print the number of routes used for k8s cluster node connectivity. +# +# Assumed vars: +# PROJECT +function get-k8s-node-routes-count() { + local k8s_node_routes_count=$(gcloud alpha compute routes list \ + --project=${PROJECT} --filter='description=k8s-node-route' \ + --format='value(name)' | wc -l) + echo -n "${k8s_node_routes_count}" +} + +# Detect the subnetwork where the k8s cluster resides. +# +# Assumed vars: +# KUBE_MASTER +# PROJECT +# ZONE +# Vars set: +# IP_ALIAS_SUBNETWORK +function detect-k8s-subnetwork() { + local subnetwork_url=$(gcloud alpha compute instances describe \ + ${KUBE_MASTER} --project=${PROJECT} --zone=${ZONE} \ + --format='value(networkInterfaces[0].subnetwork)') + if [ -n ${subnetwork_url} ]; then + IP_ALIAS_SUBNETWORK=$(echo ${subnetwork_url##*/}) + fi +} + +# Set IP_ALIAS_SUBNETWORK's allowSubnetCidrRoutesOverlap to a boolean value. +# $1: true or false for the desired allowSubnetCidrRoutesOverlap. +# +# Assumed vars: +# IP_ALIAS_SUBNETWORK +# GCE_API_ENDPOINT +# PROJECT +# REGION +function set-allow-subnet-cidr-routes-overlap() { + local allow_subnet_cidr_routes_overlap + allow_subnet_cidr_routes_overlap=$(gcloud alpha compute networks subnets \ + describe ${IP_ALIAS_SUBNETWORK} --project=${PROJECT} --region=${REGION} \ + --format='value(allowSubnetCidrRoutesOverlap)') + if [ ${allow_subnet_cidr_routes_overlap,,} = $1 ]; then + echo "Subnet ${IP_ALIAS_SUBNETWORK}'s allowSubnetCidrRoutesOverlap is already set as $1" + return + fi + + echo "Setting subnet \"${IP_ALIAS_SUBNETWORK}\" allowSubnetCidrRoutesOverlap to $1" + local fingerprint=$(gcloud alpha compute networks subnets describe \ + ${IP_ALIAS_SUBNETWORK} --project=${PROJECT} --region=${REGION} \ + --format='value(fingerprint)') + local access_token=$(gcloud auth print-access-token) + local request="{\"allowSubnetCidrRoutesOverlap\":$1, \"fingerprint\":\"${fingerprint}\"}" + local subnetwork_url="${GCE_API_ENDPOINT}projects/${PROJECT}/regions/${REGION}/subnetworks/${IP_ALIAS_SUBNETWORK}" + until curl --header "Content-Type: application/json" --header "Authorization: Bearer ${access_token}" \ + -X PATCH -d "${request}" "${subnetwork_url}" --output /dev/null; do + printf "." + sleep 1 + done +} + +# Add secondary ranges to k8s subnet. +# +# Assumed vars: +# IP_ALIAS_SUBNETWORK +# PROJECT +# REGION +# CLUSTER_IP_RANGE +# SERVICE_CLUSTER_IP_RANGE +function add-k8s-subnet-secondary-ranges() { + local secondary_ranges=$(gcloud alpha compute networks subnets describe "${IP_ALIAS_SUBNETWORK}" \ + --project="${PROJECT}" --region="${REGION}" \ + --format='value(secondaryIpRanges)') + if [[ "${secondary_ranges}" =~ "pods-default" && "${secondary_ranges}" =~ "services-default" ]]; then + echo "${secondary_ranges} already contains both pods-default and services-default secondary ranges" + return + fi + + echo "Adding secondary ranges: pods-default (${CLUSTER_IP_RANGE}), services-default (${SERVICE_CLUSTER_IP_RANGE})" + gcloud alpha compute networks subnets update ${IP_ALIAS_SUBNETWORK} \ + --project=${PROJECT} --region=${REGION} \ + --add-secondary-ranges="pods-default=${CLUSTER_IP_RANGE},services-default=${SERVICE_CLUSTER_IP_RANGE}" +} + +# Delete all k8s node routes. +# +# Assumed vars: +# PROJECT +function delete-k8s-node-routes() { + local -a routes + local -r batch=200 + routes=( $(gcloud alpha compute routes list \ + --project=${PROJECT} --filter='description=k8s-node-route' \ + --format='value(name)') ) + while (( "${#routes[@]}" > 0 )); do + echo Deleting k8s node routes "${routes[*]::${batch}}" + gcloud compute routes delete --project "${PROJECT}" --quiet "${routes[@]::${batch}}" + routes=( "${routes[@]:${batch}}" ) + done +} + +detect-project +detect-master + +k8s_node_routes_count=$(get-k8s-node-routes-count) +if [[ "${k8s_node_routes_count}" -eq 0 ]]; then + echo "No k8s node routes found and IP alias should already be enabled. Exiting..." + exit 0 +fi +echo "Found ${k8s_node_routes_count} k8s node routes. Proceeding to upgrade them to IP aliases based connectivity..." + +detect-k8s-subnetwork +if [ -z ${IP_ALIAS_SUBNETWORK} ]; then + echo "No k8s cluster subnetwork found. Exiting..." + exit 1 +fi +echo "k8s cluster sits on subnetwork \"${IP_ALIAS_SUBNETWORK}\"" + +set-allow-subnet-cidr-routes-overlap true +add-k8s-subnet-secondary-ranges + +echo "Changing k8s master envs and restarting..." +export KUBE_GCE_IP_ALIAS_SUBNETWORK=${IP_ALIAS_SUBNETWORK} +export KUBE_GCE_NODE_IPAM_MODE="IPAMFromCluster" +export KUBE_GCE_ENABLE_IP_ALIASES=true +export SECONDARY_RANGE_NAME="pods-default" +export STORAGE_BACKEND="etcd3" +export STORAGE_MEDIA_TYPE="application/vnd.kubernetes.protobuf" + +# Upgrade master with updated kube envs +${KUBE_ROOT}/cluster/gce/upgrade.sh -M -l + +delete-k8s-node-routes +set-allow-subnet-cidr-routes-overlap false diff --git a/pkg/controller/node/ipam/sync/sync.go b/pkg/controller/node/ipam/sync/sync.go index 4995f42554..737826547c 100644 --- a/pkg/controller/node/ipam/sync/sync.go +++ b/pkg/controller/node/ipam/sync/sync.go @@ -244,9 +244,7 @@ func (op *updateOp) validateRange(ctx context.Context, sync *NodeSync, node *v1. // alias. func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, node *v1.Node, aliasRange *net.IPNet) error { if sync.mode != SyncFromCloud { - sync.kubeAPI.EmitNodeWarningEvent(node.Name, InvalidModeEvent, - "Cannot sync from cloud in mode %q", sync.mode) - return fmt.Errorf("cannot sync from cloud in mode %q", sync.mode) + fmt.Errorf("Detect mode %q while expect mode %q when syncing from cloud", sync.mode, SyncFromCloud) } glog.V(2).Infof("Updating node spec with alias range, node.PodCIDR = %v", aliasRange) @@ -276,9 +274,7 @@ func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, nod // updateAliasFromNode updates the cloud alias given the node allocation. func (op *updateOp) updateAliasFromNode(ctx context.Context, sync *NodeSync, node *v1.Node) error { if sync.mode != SyncFromCluster { - sync.kubeAPI.EmitNodeWarningEvent( - node.Name, InvalidModeEvent, "Cannot sync to cloud in mode %q", sync.mode) - return fmt.Errorf("cannot sync to cloud in mode %q", sync.mode) + fmt.Errorf("Detect mode %q while expect mode %q when syncing from cluster", sync.mode, SyncFromCluster) } _, aliasRange, err := net.ParseCIDR(node.Spec.PodCIDR)