mirror of https://github.com/k3s-io/k3s
Delete all firewall rules and network on GCE/GKE test cluster teardown
Jeff Grafton
parent
0357341fd5
commit
661d6bde6f
|
|
@ -35,6 +35,7 @@ REGISTER_MASTER_KUBELET=${REGISTER_MASTER:-true}
|
|||
PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false}
|
||||
PREEMPTIBLE_MASTER=${PREEMPTIBLE_MASTER:-false}
|
||||
KUBE_DELETE_NODES=${KUBE_DELETE_NODES:-true}
|
||||
KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-false}
|
||||
|
||||
MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
|
||||
NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ KUBE_APISERVER_REQUEST_TIMEOUT=300
|
|||
PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false}
|
||||
PREEMPTIBLE_MASTER=${PREEMPTIBLE_MASTER:-false}
|
||||
KUBE_DELETE_NODES=${KUBE_DELETE_NODES:-true}
|
||||
KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-true}
|
||||
|
||||
MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
|
||||
NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-${KUBE_OS_DISTRIBUTION:-gci}}
|
||||
|
|
|
|||
|
|
@ -696,6 +696,27 @@ function create-network() {
|
|||
fi
|
||||
}
|
||||
|
||||
function delete-firewall-rules() {
|
||||
for fw in $@; do
|
||||
if [[ -n $(gcloud compute firewall-rules --project "${PROJECT}" describe "${fw}" --format='value(name)' 2>/dev/null || true) ]]; then
|
||||
gcloud compute firewall-rules delete --project "${PROJECT}" --quiet "${fw}" &
|
||||
fi
|
||||
done
|
||||
kube::util::wait-for-jobs || {
|
||||
echo -e "${color_red}Failed to delete firewall rules.${color_norm}" >&2
|
||||
}
|
||||
}
|
||||
|
||||
function delete-network() {
|
||||
if [[ -n $(gcloud compute networks --project "${PROJECT}" describe "${NETWORK}" --format='value(name)' 2>/dev/null || true) ]]; then
|
||||
if ! gcloud compute networks delete --project "${PROJECT}" --quiet "${NETWORK}"; then
|
||||
echo "Failed to delete network '${NETWORK}'. Listing firewall-rules:"
|
||||
gcloud compute firewall-rules --project "${PROJECT}" list --filter="network=${NETWORK}"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Assumes:
|
||||
# NUM_NODES
|
||||
# Sets:
|
||||
|
|
@ -1273,13 +1294,8 @@ function kube-down() {
|
|||
|
||||
# If there are no more remaining master replicas, we should delete all remaining network resources.
|
||||
if [[ "${REMAINING_MASTER_COUNT}" == "0" ]]; then
|
||||
# Delete firewall rule for the master.
|
||||
if gcloud compute firewall-rules describe --project "${PROJECT}" "${MASTER_NAME}-https" &>/dev/null; then
|
||||
gcloud compute firewall-rules delete \
|
||||
--project "${PROJECT}" \
|
||||
--quiet \
|
||||
"${MASTER_NAME}-https"
|
||||
fi
|
||||
# Delete firewall rule for the master, etcd servers, and nodes.
|
||||
delete-firewall-rules "${MASTER_NAME}-https" "${MASTER_NAME}-etcd" "${NODE_TAG}-all"
|
||||
# Delete the master's reserved IP
|
||||
if gcloud compute addresses describe "${MASTER_NAME}-ip" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then
|
||||
gcloud compute addresses delete \
|
||||
|
|
@ -1288,20 +1304,6 @@ function kube-down() {
|
|||
--quiet \
|
||||
"${MASTER_NAME}-ip"
|
||||
fi
|
||||
# Delete firewall rule for minions.
|
||||
if gcloud compute firewall-rules describe --project "${PROJECT}" "${NODE_TAG}-all" &>/dev/null; then
|
||||
gcloud compute firewall-rules delete \
|
||||
--project "${PROJECT}" \
|
||||
--quiet \
|
||||
"${NODE_TAG}-all"
|
||||
fi
|
||||
# Delete firewall rule for etcd servers.
|
||||
if gcloud compute firewall-rules --project "${PROJECT}" describe "${MASTER_NAME}-etcd" &>/dev/null; then
|
||||
gcloud compute firewall-rules delete \
|
||||
--project "${PROJECT}" \
|
||||
--quiet \
|
||||
"${MASTER_NAME}-etcd"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "${KUBE_DELETE_NODES:-}" != "false" ]]; then
|
||||
|
|
@ -1353,6 +1355,16 @@ function kube-down() {
|
|||
"${INSTANCE_PREFIX}"-influxdb-pd
|
||||
fi
|
||||
|
||||
# Delete all remaining firewall rules and network.
|
||||
delete-firewall-rules \
|
||||
"${NETWORK}-default-internal-master" \
|
||||
"${NETWORK}-default-internal-node" \
|
||||
"${NETWORK}-default-ssh" \
|
||||
"${NETWORK}-default-internal" # Pre-1.5 clusters
|
||||
if [[ "${KUBE_DELETE_NETWORK}" == "true" ]]; then
|
||||
delete-network
|
||||
fi
|
||||
|
||||
# If there are no more remaining master replicas, we should update kubeconfig.
|
||||
if [[ "${REMAINING_MASTER_COUNT}" == "0" ]]; then
|
||||
export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}"
|
||||
|
|
@ -1721,14 +1733,9 @@ function test-setup() {
|
|||
function test-teardown() {
|
||||
detect-project
|
||||
echo "Shutting down test cluster in background."
|
||||
gcloud compute firewall-rules delete \
|
||||
--project "${PROJECT}" \
|
||||
--quiet \
|
||||
"${NODE_TAG}-${INSTANCE_PREFIX}-http-alt" || true
|
||||
gcloud compute firewall-rules delete \
|
||||
--project "${PROJECT}" \
|
||||
--quiet \
|
||||
"${NODE_TAG}-${INSTANCE_PREFIX}-nodeports" || true
|
||||
delete-firewall-rules \
|
||||
"${NODE_TAG}-${INSTANCE_PREFIX}-http-alt" \
|
||||
"${NODE_TAG}-${INSTANCE_PREFIX}-nodeports"
|
||||
if [[ ${MULTIZONE:-} == "true" ]]; then
|
||||
local zones=( ${E2E_ZONES} )
|
||||
# tear them down in reverse order, finally tearing down the master too.
|
||||
|
|
|
|||
|
|
@ -40,3 +40,5 @@ ENABLE_L7_LOADBALANCING="${KUBE_ENABLE_L7_LOADBALANCING:-glbc}"
|
|||
# google - Heapster, Google Cloud Monitoring, and Google Cloud Logging
|
||||
# standalone - Heapster only. Metrics available via Heapster REST API.
|
||||
ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-standalone}"
|
||||
|
||||
KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-false}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ NETWORK=${KUBE_GKE_NETWORK:-e2e}
|
|||
NODE_TAG="k8s-${CLUSTER_NAME}-node"
|
||||
IMAGE_TYPE="${KUBE_GKE_IMAGE_TYPE:-container_vm}"
|
||||
|
||||
KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-true}
|
||||
|
||||
# For ease of maintenance, extract any pieces that do not vary between default
|
||||
# and test in a common config.
|
||||
|
|
|
|||
|
|
@ -370,13 +370,27 @@ function test-teardown() {
|
|||
# instances, but we can safely delete the cluster before the firewall.
|
||||
#
|
||||
# NOTE: Keep in sync with names above in test-setup.
|
||||
"${GCLOUD}" compute firewall-rules delete "${CLUSTER_NAME}-http-alt" \
|
||||
--project="${PROJECT}" &
|
||||
"${GCLOUD}" compute firewall-rules delete "${CLUSTER_NAME}-nodeports" \
|
||||
--project="${PROJECT}" &
|
||||
for fw in "${CLUSTER_NAME}-http-alt" "${CLUSTER_NAME}-nodeports" "${FIREWALL_SSH}"; do
|
||||
if [[ -n $("${GCLOUD}" compute firewall-rules --project "${PROJECT}" describe "${fw}" --format='value(name)' 2>/dev/null || true) ]]; then
|
||||
"${GCLOUD}" compute firewall-rules delete "${fw}" --project="${PROJECT}" --quiet &
|
||||
fi
|
||||
done
|
||||
|
||||
# Wait for firewall rule teardown.
|
||||
kube::util::wait-for-jobs || true
|
||||
|
||||
# It's unfortunate that the $FIREWALL_SSH rule and network are created in
|
||||
# kube-up, but we can only really delete them in test-teardown. So much for
|
||||
# symmetry.
|
||||
if [[ "${KUBE_DELETE_NETWORK}" == "true" ]]; then
|
||||
if [[ -n $("${GCLOUD}" compute networks --project "${PROJECT}" describe "${NETWORK}" --format='value(name)' 2>/dev/null || true) ]]; then
|
||||
if ! "${GCLOUD}" compute networks delete --project "${PROJECT}" --quiet "${NETWORK}"; then
|
||||
echo "Failed to delete network '${NETWORK}'. Listing firewall-rules:"
|
||||
"${GCLOUD}" compute firewall-rules --project "${PROJECT}" list --filter="network=${NETWORK}"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Actually take down the cluster. This is called from test-teardown.
|
||||
|
|
|
|||
Loading…
Reference in New Issue