mirror of https://github.com/k3s-io/k3s
Implement upgrade-aliases.sh to migrate a route-based k8s cluster to use IP aliases in GCE.
Jing Ai
parent
40e7101844
commit
551ffbe7bf
|
|
@ -253,14 +253,19 @@ ENABLE_RESCHEDULER="${KUBE_ENABLE_RESCHEDULER:-true}"
|
|||
# IP_ALIAS_SUBNETWORK is the subnetwork to allocate from. If empty, a
|
||||
# new subnetwork will be created for the cluster.
|
||||
ENABLE_IP_ALIASES=${KUBE_GCE_ENABLE_IP_ALIASES:-false}
|
||||
NODE_IPAM_MODE=${KUBE_GCE_NODE_IPAM_MODE:-RangeAllocator}
|
||||
if [ ${ENABLE_IP_ALIASES} = true ]; then
|
||||
# Size of ranges allocated to each node. Currently supports only /32 and /24.
|
||||
IP_ALIAS_SIZE=${KUBE_GCE_IP_ALIAS_SIZE:-/24}
|
||||
IP_ALIAS_SUBNETWORK=${KUBE_GCE_IP_ALIAS_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-default}
|
||||
# Reserve the services IP space to avoid being allocated for other GCP resources.
|
||||
SERVICE_CLUSTER_IP_SUBNETWORK=${KUBE_GCE_SERVICE_CLUSTER_IP_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-services}
|
||||
NODE_IPAM_MODE=${KUBE_GCE_NODE_IPAM_MODE:-CloudAllocator}
|
||||
SECONDARY_RANGE_NAME=${SECONDARY_RANGE_NAME:-}
|
||||
# Add to the provider custom variables.
|
||||
PROVIDER_VARS="${PROVIDER_VARS:-} ENABLE_IP_ALIASES"
|
||||
PROVIDER_VARS="${PROVIDER_VARS:-} NODE_IPAM_MODE"
|
||||
PROVIDER_VARS="${PROVIDER_VARS:-} SECONDARY_RANGE_NAME"
|
||||
fi
|
||||
|
||||
# Enable GCE Alpha features.
|
||||
|
|
|
|||
|
|
@ -1104,7 +1104,7 @@ function start-kube-controller-manager {
|
|||
params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}"
|
||||
fi
|
||||
if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
|
||||
params+=" --cidr-allocator-type=CloudAllocator"
|
||||
params+=" --cidr-allocator-type=${NODE_IPAM_MODE}"
|
||||
params+=" --configure-cloud-routes=false"
|
||||
fi
|
||||
if [[ -n "${FEATURE_GATES:-}" ]]; then
|
||||
|
|
|
|||
|
|
@ -79,9 +79,16 @@ function create-master-instance-internal() {
|
|||
preemptible_master="--preemptible --maintenance-policy TERMINATE"
|
||||
fi
|
||||
|
||||
local enable_ip_aliases
|
||||
if [[ "${NODE_IPAM_MODE:-}" == "CloudAllocator" ]]; then
|
||||
enable_ip_aliases=true
|
||||
else
|
||||
enable_ip_aliases=false
|
||||
fi
|
||||
|
||||
local network=$(make-gcloud-network-argument \
|
||||
"${NETWORK_PROJECT}" "${REGION}" "${NETWORK}" "${SUBNETWORK:-}" \
|
||||
"${address:-}" "${ENABLE_IP_ALIASES:-}" "${IP_ALIAS_SIZE:-}")
|
||||
"${address:-}" "${enable_ip_aliases:-}" "${IP_ALIAS_SIZE:-}")
|
||||
|
||||
local metadata="kube-env=${KUBE_TEMP}/master-kube-env.yaml"
|
||||
metadata="${metadata},user-data=${KUBE_ROOT}/cluster/gce/container-linux/master.yaml"
|
||||
|
|
|
|||
|
|
@ -1601,7 +1601,7 @@ function start-kube-controller-manager {
|
|||
params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}"
|
||||
fi
|
||||
if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
|
||||
params+=" --cidr-allocator-type=CloudAllocator"
|
||||
params+=" --cidr-allocator-type=${NODE_IPAM_MODE}"
|
||||
params+=" --configure-cloud-routes=false"
|
||||
fi
|
||||
if [[ -n "${FEATURE_GATES:-}" ]]; then
|
||||
|
|
|
|||
|
|
@ -0,0 +1,165 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2017 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# !!!EXPERIMENTAL!!! Upgrade a k8s cluster from routes to IP aliases for
|
||||
# node connectivity on GCE. This is only for migration.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
if [[ "${KUBERNETES_PROVIDER:-gce}" != "gce" ]]; then
|
||||
echo "!!! ${1} only works on GCE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
|
||||
source "${KUBE_ROOT}/hack/lib/util.sh"
|
||||
source "${KUBE_ROOT}/cluster/kube-util.sh"
|
||||
|
||||
# Print the number of routes used for k8s cluster node connectivity.
|
||||
#
|
||||
# Assumed vars:
|
||||
# PROJECT
|
||||
function get-k8s-node-routes-count() {
|
||||
local k8s_node_routes_count=$(gcloud alpha compute routes list \
|
||||
--project=${PROJECT} --filter='description=k8s-node-route' \
|
||||
--format='value(name)' | wc -l)
|
||||
echo -n "${k8s_node_routes_count}"
|
||||
}
|
||||
|
||||
# Detect the subnetwork where the k8s cluster resides.
|
||||
#
|
||||
# Assumed vars:
|
||||
# KUBE_MASTER
|
||||
# PROJECT
|
||||
# ZONE
|
||||
# Vars set:
|
||||
# IP_ALIAS_SUBNETWORK
|
||||
function detect-k8s-subnetwork() {
|
||||
local subnetwork_url=$(gcloud alpha compute instances describe \
|
||||
${KUBE_MASTER} --project=${PROJECT} --zone=${ZONE} \
|
||||
--format='value(networkInterfaces[0].subnetwork)')
|
||||
if [ -n ${subnetwork_url} ]; then
|
||||
IP_ALIAS_SUBNETWORK=$(echo ${subnetwork_url##*/})
|
||||
fi
|
||||
}
|
||||
|
||||
# Set IP_ALIAS_SUBNETWORK's allowSubnetCidrRoutesOverlap to a boolean value.
|
||||
# $1: true or false for the desired allowSubnetCidrRoutesOverlap.
|
||||
#
|
||||
# Assumed vars:
|
||||
# IP_ALIAS_SUBNETWORK
|
||||
# GCE_API_ENDPOINT
|
||||
# PROJECT
|
||||
# REGION
|
||||
function set-allow-subnet-cidr-routes-overlap() {
|
||||
local allow_subnet_cidr_routes_overlap
|
||||
allow_subnet_cidr_routes_overlap=$(gcloud alpha compute networks subnets \
|
||||
describe ${IP_ALIAS_SUBNETWORK} --project=${PROJECT} --region=${REGION} \
|
||||
--format='value(allowSubnetCidrRoutesOverlap)')
|
||||
if [ ${allow_subnet_cidr_routes_overlap,,} = $1 ]; then
|
||||
echo "Subnet ${IP_ALIAS_SUBNETWORK}'s allowSubnetCidrRoutesOverlap is already set as $1"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "Setting subnet \"${IP_ALIAS_SUBNETWORK}\" allowSubnetCidrRoutesOverlap to $1"
|
||||
local fingerprint=$(gcloud alpha compute networks subnets describe \
|
||||
${IP_ALIAS_SUBNETWORK} --project=${PROJECT} --region=${REGION} \
|
||||
--format='value(fingerprint)')
|
||||
local access_token=$(gcloud auth print-access-token)
|
||||
local request="{\"allowSubnetCidrRoutesOverlap\":$1, \"fingerprint\":\"${fingerprint}\"}"
|
||||
local subnetwork_url="${GCE_API_ENDPOINT}projects/${PROJECT}/regions/${REGION}/subnetworks/${IP_ALIAS_SUBNETWORK}"
|
||||
until curl --header "Content-Type: application/json" --header "Authorization: Bearer ${access_token}" \
|
||||
-X PATCH -d "${request}" "${subnetwork_url}" --output /dev/null; do
|
||||
printf "."
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
# Add secondary ranges to k8s subnet.
|
||||
#
|
||||
# Assumed vars:
|
||||
# IP_ALIAS_SUBNETWORK
|
||||
# PROJECT
|
||||
# REGION
|
||||
# CLUSTER_IP_RANGE
|
||||
# SERVICE_CLUSTER_IP_RANGE
|
||||
function add-k8s-subnet-secondary-ranges() {
|
||||
local secondary_ranges=$(gcloud alpha compute networks subnets describe "${IP_ALIAS_SUBNETWORK}" \
|
||||
--project="${PROJECT}" --region="${REGION}" \
|
||||
--format='value(secondaryIpRanges)')
|
||||
if [[ "${secondary_ranges}" =~ "pods-default" && "${secondary_ranges}" =~ "services-default" ]]; then
|
||||
echo "${secondary_ranges} already contains both pods-default and services-default secondary ranges"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "Adding secondary ranges: pods-default (${CLUSTER_IP_RANGE}), services-default (${SERVICE_CLUSTER_IP_RANGE})"
|
||||
gcloud alpha compute networks subnets update ${IP_ALIAS_SUBNETWORK} \
|
||||
--project=${PROJECT} --region=${REGION} \
|
||||
--add-secondary-ranges="pods-default=${CLUSTER_IP_RANGE},services-default=${SERVICE_CLUSTER_IP_RANGE}"
|
||||
}
|
||||
|
||||
# Delete all k8s node routes.
|
||||
#
|
||||
# Assumed vars:
|
||||
# PROJECT
|
||||
function delete-k8s-node-routes() {
|
||||
local -a routes
|
||||
local -r batch=200
|
||||
routes=( $(gcloud alpha compute routes list \
|
||||
--project=${PROJECT} --filter='description=k8s-node-route' \
|
||||
--format='value(name)') )
|
||||
while (( "${#routes[@]}" > 0 )); do
|
||||
echo Deleting k8s node routes "${routes[*]::${batch}}"
|
||||
gcloud compute routes delete --project "${PROJECT}" --quiet "${routes[@]::${batch}}"
|
||||
routes=( "${routes[@]:${batch}}" )
|
||||
done
|
||||
}
|
||||
|
||||
detect-project
|
||||
detect-master
|
||||
|
||||
k8s_node_routes_count=$(get-k8s-node-routes-count)
|
||||
if [[ "${k8s_node_routes_count}" -eq 0 ]]; then
|
||||
echo "No k8s node routes found and IP alias should already be enabled. Exiting..."
|
||||
exit 0
|
||||
fi
|
||||
echo "Found ${k8s_node_routes_count} k8s node routes. Proceeding to upgrade them to IP aliases based connectivity..."
|
||||
|
||||
detect-k8s-subnetwork
|
||||
if [ -z ${IP_ALIAS_SUBNETWORK} ]; then
|
||||
echo "No k8s cluster subnetwork found. Exiting..."
|
||||
exit 1
|
||||
fi
|
||||
echo "k8s cluster sits on subnetwork \"${IP_ALIAS_SUBNETWORK}\""
|
||||
|
||||
set-allow-subnet-cidr-routes-overlap true
|
||||
add-k8s-subnet-secondary-ranges
|
||||
|
||||
echo "Changing k8s master envs and restarting..."
|
||||
export KUBE_GCE_IP_ALIAS_SUBNETWORK=${IP_ALIAS_SUBNETWORK}
|
||||
export KUBE_GCE_NODE_IPAM_MODE="IPAMFromCluster"
|
||||
export KUBE_GCE_ENABLE_IP_ALIASES=true
|
||||
export SECONDARY_RANGE_NAME="pods-default"
|
||||
export STORAGE_BACKEND="etcd3"
|
||||
export STORAGE_MEDIA_TYPE="application/vnd.kubernetes.protobuf"
|
||||
|
||||
# Upgrade master with updated kube envs
|
||||
${KUBE_ROOT}/cluster/gce/upgrade.sh -M -l
|
||||
|
||||
delete-k8s-node-routes
|
||||
set-allow-subnet-cidr-routes-overlap false
|
||||
|
|
@ -244,9 +244,7 @@ func (op *updateOp) validateRange(ctx context.Context, sync *NodeSync, node *v1.
|
|||
// alias.
|
||||
func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, node *v1.Node, aliasRange *net.IPNet) error {
|
||||
if sync.mode != SyncFromCloud {
|
||||
sync.kubeAPI.EmitNodeWarningEvent(node.Name, InvalidModeEvent,
|
||||
"Cannot sync from cloud in mode %q", sync.mode)
|
||||
return fmt.Errorf("cannot sync from cloud in mode %q", sync.mode)
|
||||
fmt.Errorf("Detect mode %q while expect mode %q when syncing from cloud", sync.mode, SyncFromCloud)
|
||||
}
|
||||
|
||||
glog.V(2).Infof("Updating node spec with alias range, node.PodCIDR = %v", aliasRange)
|
||||
|
|
@ -276,9 +274,7 @@ func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, nod
|
|||
// updateAliasFromNode updates the cloud alias given the node allocation.
|
||||
func (op *updateOp) updateAliasFromNode(ctx context.Context, sync *NodeSync, node *v1.Node) error {
|
||||
if sync.mode != SyncFromCluster {
|
||||
sync.kubeAPI.EmitNodeWarningEvent(
|
||||
node.Name, InvalidModeEvent, "Cannot sync to cloud in mode %q", sync.mode)
|
||||
return fmt.Errorf("cannot sync to cloud in mode %q", sync.mode)
|
||||
fmt.Errorf("Detect mode %q while expect mode %q when syncing from cluster", sync.mode, SyncFromCluster)
|
||||
}
|
||||
|
||||
_, aliasRange, err := net.ParseCIDR(node.Spec.PodCIDR)
|
||||
|
|
|
|||
Loading…
Reference in New Issue