Implement upgrade-aliases.sh to migrate a route-based k8s cluster to use IP aliases in GCE.

cluster/gce/config-default.sh

@@ -253,14 +253,19 @@ ENABLE_RESCHEDULER="${KUBE_ENABLE_RESCHEDULER:-true}"
 # IP_ALIAS_SUBNETWORK is the subnetwork to allocate from. If empty, a
 #   new subnetwork will be created for the cluster.
 ENABLE_IP_ALIASES=${KUBE_GCE_ENABLE_IP_ALIASES:-false}
+NODE_IPAM_MODE=${KUBE_GCE_NODE_IPAM_MODE:-RangeAllocator}
 if [ ${ENABLE_IP_ALIASES} = true ]; then
   # Size of ranges allocated to each node. Currently supports only /32 and /24.
   IP_ALIAS_SIZE=${KUBE_GCE_IP_ALIAS_SIZE:-/24}
   IP_ALIAS_SUBNETWORK=${KUBE_GCE_IP_ALIAS_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-default}
   # Reserve the services IP space to avoid being allocated for other GCP resources.
   SERVICE_CLUSTER_IP_SUBNETWORK=${KUBE_GCE_SERVICE_CLUSTER_IP_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-services}
+  NODE_IPAM_MODE=${KUBE_GCE_NODE_IPAM_MODE:-CloudAllocator}
+  SECONDARY_RANGE_NAME=${SECONDARY_RANGE_NAME:-}
   # Add to the provider custom variables.
   PROVIDER_VARS="${PROVIDER_VARS:-} ENABLE_IP_ALIASES"
+  PROVIDER_VARS="${PROVIDER_VARS:-} NODE_IPAM_MODE"
+  PROVIDER_VARS="${PROVIDER_VARS:-} SECONDARY_RANGE_NAME"
 fi
 
 # Enable GCE Alpha features.

cluster/gce/container-linux/configure-helper.sh

@@ -1104,7 +1104,7 @@ function start-kube-controller-manager {
     params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}"
   fi
   if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
-    params+=" --cidr-allocator-type=CloudAllocator"
+    params+=" --cidr-allocator-type=${NODE_IPAM_MODE}"
     params+=" --configure-cloud-routes=false"
   fi
   if [[ -n "${FEATURE_GATES:-}" ]]; then

cluster/gce/container-linux/master-helper.sh

@@ -79,9 +79,16 @@ function create-master-instance-internal() {
     preemptible_master="--preemptible --maintenance-policy TERMINATE"
   fi
 
+  local enable_ip_aliases
+  if [[ "${NODE_IPAM_MODE:-}" == "CloudAllocator" ]]; then
+    enable_ip_aliases=true
+  else
+    enable_ip_aliases=false
+  fi
+
   local network=$(make-gcloud-network-argument \
     "${NETWORK_PROJECT}" "${REGION}" "${NETWORK}" "${SUBNETWORK:-}" \
-    "${address:-}" "${ENABLE_IP_ALIASES:-}" "${IP_ALIAS_SIZE:-}")
+    "${address:-}" "${enable_ip_aliases:-}" "${IP_ALIAS_SIZE:-}")
 
   local metadata="kube-env=${KUBE_TEMP}/master-kube-env.yaml"
   metadata="${metadata},user-data=${KUBE_ROOT}/cluster/gce/container-linux/master.yaml"

cluster/gce/gci/configure-helper.sh

@@ -1601,7 +1601,7 @@ function start-kube-controller-manager {
     params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}"
   fi
   if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
-    params+=" --cidr-allocator-type=CloudAllocator"
+    params+=" --cidr-allocator-type=${NODE_IPAM_MODE}"
     params+=" --configure-cloud-routes=false"
   fi
   if [[ -n "${FEATURE_GATES:-}" ]]; then

cluster/gce/upgrade-aliases.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# !!!EXPERIMENTAL!!! Upgrade a k8s cluster from routes to IP aliases for
+# node connectivity on GCE. This is only for migration.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [[ "${KUBERNETES_PROVIDER:-gce}" != "gce" ]]; then
+  echo "!!! ${1} only works on GCE" >&2
+  exit 1
+fi
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
+source "${KUBE_ROOT}/hack/lib/util.sh"
+source "${KUBE_ROOT}/cluster/kube-util.sh"
+
+# Print the number of routes used for k8s cluster node connectivity.
+#
+# Assumed vars:
+#   PROJECT
+function get-k8s-node-routes-count() {
+  local k8s_node_routes_count=$(gcloud alpha compute routes list \
+    --project=${PROJECT} --filter='description=k8s-node-route' \
+    --format='value(name)' | wc -l)
+  echo -n "${k8s_node_routes_count}"
+}
+
+# Detect the subnetwork where the k8s cluster resides.
+#
+# Assumed vars:
+#  KUBE_MASTER
+#  PROJECT
+#  ZONE
+# Vars set:
+#  IP_ALIAS_SUBNETWORK
+function detect-k8s-subnetwork() {
+  local subnetwork_url=$(gcloud alpha compute instances describe \
+    ${KUBE_MASTER} --project=${PROJECT} --zone=${ZONE} \
+    --format='value(networkInterfaces[0].subnetwork)')
+  if [ -n ${subnetwork_url} ]; then
+    IP_ALIAS_SUBNETWORK=$(echo ${subnetwork_url##*/})
+  fi
+}
+
+# Set IP_ALIAS_SUBNETWORK's allowSubnetCidrRoutesOverlap to a boolean value.
+# $1: true or false for the desired allowSubnetCidrRoutesOverlap.
+#
+# Assumed vars:
+#   IP_ALIAS_SUBNETWORK
+#   GCE_API_ENDPOINT
+#   PROJECT
+#   REGION
+function set-allow-subnet-cidr-routes-overlap() {
+  local allow_subnet_cidr_routes_overlap
+  allow_subnet_cidr_routes_overlap=$(gcloud alpha compute networks subnets \
+    describe ${IP_ALIAS_SUBNETWORK} --project=${PROJECT} --region=${REGION} \
+    --format='value(allowSubnetCidrRoutesOverlap)')
+  if [ ${allow_subnet_cidr_routes_overlap,,} = $1 ]; then
+    echo "Subnet ${IP_ALIAS_SUBNETWORK}'s allowSubnetCidrRoutesOverlap is already set as $1"
+    return
+  fi
+
+  echo "Setting subnet \"${IP_ALIAS_SUBNETWORK}\" allowSubnetCidrRoutesOverlap to $1"
+  local fingerprint=$(gcloud alpha compute networks subnets describe \
+    ${IP_ALIAS_SUBNETWORK} --project=${PROJECT} --region=${REGION} \
+    --format='value(fingerprint)')
+  local access_token=$(gcloud auth print-access-token)
+  local request="{\"allowSubnetCidrRoutesOverlap\":$1, \"fingerprint\":\"${fingerprint}\"}"
+  local subnetwork_url="${GCE_API_ENDPOINT}projects/${PROJECT}/regions/${REGION}/subnetworks/${IP_ALIAS_SUBNETWORK}"
+  until curl --header "Content-Type: application/json" --header "Authorization: Bearer ${access_token}" \
+    -X PATCH -d "${request}" "${subnetwork_url}" --output /dev/null; do
+    printf "."
+    sleep 1
+  done
+}
+
+# Add secondary ranges to k8s subnet.
+#
+# Assumed vars:
+#   IP_ALIAS_SUBNETWORK
+#   PROJECT
+#   REGION
+#   CLUSTER_IP_RANGE
+#   SERVICE_CLUSTER_IP_RANGE
+function add-k8s-subnet-secondary-ranges() {
+  local secondary_ranges=$(gcloud alpha compute networks subnets describe "${IP_ALIAS_SUBNETWORK}" \
+    --project="${PROJECT}" --region="${REGION}" \
+    --format='value(secondaryIpRanges)')
+  if [[ "${secondary_ranges}" =~ "pods-default" && "${secondary_ranges}" =~ "services-default" ]]; then
+    echo "${secondary_ranges} already contains both pods-default and services-default secondary ranges"
+    return
+  fi
+
+  echo "Adding secondary ranges: pods-default (${CLUSTER_IP_RANGE}), services-default (${SERVICE_CLUSTER_IP_RANGE})"
+  gcloud alpha compute networks subnets update ${IP_ALIAS_SUBNETWORK} \
+    --project=${PROJECT} --region=${REGION} \
+    --add-secondary-ranges="pods-default=${CLUSTER_IP_RANGE},services-default=${SERVICE_CLUSTER_IP_RANGE}"
+}
+
+# Delete all k8s node routes.
+#
+# Assumed vars:
+#   PROJECT
+function delete-k8s-node-routes() {
+  local -a routes
+  local -r batch=200
+  routes=( $(gcloud alpha compute routes list \
+    --project=${PROJECT} --filter='description=k8s-node-route' \
+    --format='value(name)') )
+  while (( "${#routes[@]}" > 0 )); do
+      echo Deleting k8s node routes "${routes[*]::${batch}}"
+      gcloud compute routes delete --project "${PROJECT}" --quiet "${routes[@]::${batch}}"
+      routes=( "${routes[@]:${batch}}" )
+  done
+}
+
+detect-project
+detect-master
+
+k8s_node_routes_count=$(get-k8s-node-routes-count)
+if [[ "${k8s_node_routes_count}" -eq 0 ]]; then
+  echo "No k8s node routes found and IP alias should already be enabled. Exiting..."
+  exit 0
+fi
+echo "Found ${k8s_node_routes_count} k8s node routes. Proceeding to upgrade them to IP aliases based connectivity..."
+
+detect-k8s-subnetwork
+if [ -z ${IP_ALIAS_SUBNETWORK} ]; then
+  echo "No k8s cluster subnetwork found. Exiting..."
+  exit 1
+fi
+echo "k8s cluster sits on subnetwork \"${IP_ALIAS_SUBNETWORK}\""
+
+set-allow-subnet-cidr-routes-overlap true
+add-k8s-subnet-secondary-ranges
+
+echo "Changing k8s master envs and restarting..."
+export KUBE_GCE_IP_ALIAS_SUBNETWORK=${IP_ALIAS_SUBNETWORK}
+export KUBE_GCE_NODE_IPAM_MODE="IPAMFromCluster"
+export KUBE_GCE_ENABLE_IP_ALIASES=true
+export SECONDARY_RANGE_NAME="pods-default"
+export STORAGE_BACKEND="etcd3"
+export STORAGE_MEDIA_TYPE="application/vnd.kubernetes.protobuf"
+
+# Upgrade master with updated kube envs
+${KUBE_ROOT}/cluster/gce/upgrade.sh -M -l
+
+delete-k8s-node-routes
+set-allow-subnet-cidr-routes-overlap false

pkg/controller/node/ipam/sync/sync.go

@@ -244,9 +244,7 @@ func (op *updateOp) validateRange(ctx context.Context, sync *NodeSync, node *v1.
 // alias.
 func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, node *v1.Node, aliasRange *net.IPNet) error {
 	if sync.mode != SyncFromCloud {
-		sync.kubeAPI.EmitNodeWarningEvent(node.Name, InvalidModeEvent,
+		fmt.Errorf("Detect mode %q while expect mode %q when syncing from cloud", sync.mode, SyncFromCloud)
-			"Cannot sync from cloud in mode %q", sync.mode)
-		return fmt.Errorf("cannot sync from cloud in mode %q", sync.mode)
 	}
 
 	glog.V(2).Infof("Updating node spec with alias range, node.PodCIDR = %v", aliasRange)
@@ -276,9 +274,7 @@ func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, nod
 // updateAliasFromNode updates the cloud alias given the node allocation.
 func (op *updateOp) updateAliasFromNode(ctx context.Context, sync *NodeSync, node *v1.Node) error {
 	if sync.mode != SyncFromCluster {
-		sync.kubeAPI.EmitNodeWarningEvent(
+		fmt.Errorf("Detect mode %q while expect mode %q when syncing from cluster", sync.mode, SyncFromCluster)
-			node.Name, InvalidModeEvent, "Cannot sync to cloud in mode %q", sync.mode)
-		return fmt.Errorf("cannot sync to cloud in mode %q", sync.mode)
 	}
 
 	_, aliasRange, err := net.ParseCIDR(node.Spec.PodCIDR)