From a092fd2dd4091b735cc2389015ce4bef951abfab Mon Sep 17 00:00:00 2001 From: Colin Hom Date: Tue, 10 May 2016 14:44:45 -0700 Subject: [PATCH] Federation build and e2e test integration Federation components are now buildable and e2e-testable via e2e.go. --- build/common.sh | 9 + build/push-ci-build.sh | 29 ++- build/release.sh | 8 + build/util.sh | 32 +++ cluster/common.sh | 17 +- cluster/kube-util.sh | 30 +++ cluster/vagrant/provision-master.sh | 2 +- cluster/vagrant/provision-node.sh | 2 + federation/cluster/common.sh | 189 ++++++++++++++++++ federation/cluster/federated-down.sh | 27 +++ federation/cluster/federated-push.sh | 32 +++ federation/cluster/federated-up.sh | 25 +++ federation/cluster/template.go | 76 +++++++ federation/manifests/.gitignore | 1 + .../federation-apiserver-deployment.yaml | 42 ++++ .../federation-apiserver-lb-service.yaml | 17 ++ ...federation-apiserver-nodeport-service.yaml | 17 ++ .../federation-apiserver-secrets.yaml | 9 + federation/manifests/federation-ns.yaml | 4 + hack/e2e-internal/e2e-cluster-size.sh | 1 + hack/e2e-internal/e2e-down.sh | 14 +- hack/e2e-internal/e2e-status.sh | 17 +- hack/e2e-internal/e2e-up.sh | 19 +- hack/e2e.go | 19 +- hack/federated-ginkgo-e2e.sh | 34 ++++ hack/lib/golang.sh | 2 + hack/verify-flags/known-flags.txt | 1 + test/e2e/cleanup/cleanup.go | 3 +- test/e2e/deployment.go | 7 +- test/e2e/federation-apiserver.go | 64 ++++++ test/e2e/framework/framework.go | 125 ++++++++++++ test/e2e/framework/test_context.go | 7 +- test/e2e/framework/util.go | 90 +++++++-- test/images/clusterapi-tester/main.go | 3 +- 34 files changed, 926 insertions(+), 48 deletions(-) create mode 100644 build/util.sh create mode 100644 federation/cluster/common.sh create mode 100755 federation/cluster/federated-down.sh create mode 100755 federation/cluster/federated-push.sh create mode 100755 federation/cluster/federated-up.sh create mode 100644 federation/cluster/template.go create mode 100644 federation/manifests/.gitignore create mode 100644 federation/manifests/federation-apiserver-deployment.yaml create mode 100644 federation/manifests/federation-apiserver-lb-service.yaml create mode 100644 federation/manifests/federation-apiserver-nodeport-service.yaml create mode 100644 federation/manifests/federation-apiserver-secrets.yaml create mode 100644 federation/manifests/federation-ns.yaml create mode 100755 hack/federated-ginkgo-e2e.sh create mode 100644 test/e2e/federation-apiserver.go diff --git a/build/common.sh b/build/common.sh index 4b9cca0520..016df783d5 100755 --- a/build/common.sh +++ b/build/common.sh @@ -98,6 +98,7 @@ kube::build::get_docker_wrapped_binaries() { kube-controller-manager,busybox kube-scheduler,busybox kube-proxy,gcr.io/google_containers/debian-iptables-amd64:v3 + federation-apiserver,busybox );; "arm") local targets=( @@ -105,6 +106,7 @@ kube::build::get_docker_wrapped_binaries() { kube-controller-manager,armel/busybox kube-scheduler,armel/busybox kube-proxy,gcr.io/google_containers/debian-iptables-arm:v3 + federation-apiserver,armel/busybox );; "arm64") local targets=( @@ -112,6 +114,7 @@ kube::build::get_docker_wrapped_binaries() { kube-controller-manager,aarch64/busybox kube-scheduler,aarch64/busybox kube-proxy,gcr.io/google_containers/debian-iptables-arm64:v3 + federation-apiserver,aarch64/busybox );; "ppc64le") local targets=( @@ -119,6 +122,7 @@ kube::build::get_docker_wrapped_binaries() { kube-controller-manager,ppc64le/busybox kube-scheduler,ppc64le/busybox kube-proxy,gcr.io/google_containers/debian-iptables-ppc64le:v3 + federation-apiserver,ppc64le/busybox );; esac @@ -1002,6 +1006,11 @@ function kube::release::package_full_tarball() { mkdir -p "${release_stage}/third_party" cp -R "${KUBE_ROOT}/third_party/htpasswd" "${release_stage}/third_party/htpasswd" + # Include only federation/cluster and federation/manifests + mkdir "${release_stage}/federation" + cp -R "${KUBE_ROOT}/federation/cluster" "${release_stage}/federation/" + cp -R "${KUBE_ROOT}/federation/manifests" "${release_stage}/federation/" + cp -R "${KUBE_ROOT}/examples" "${release_stage}/" cp -R "${KUBE_ROOT}/docs" "${release_stage}/" cp "${KUBE_ROOT}/README.md" "${release_stage}/" diff --git a/build/push-ci-build.sh b/build/push-ci-build.sh index d968d8e4f9..fb62081644 100755 --- a/build/push-ci-build.sh +++ b/build/push-ci-build.sh @@ -20,20 +20,10 @@ set -o errexit set -o nounset set -o pipefail -# TODO(zmerlynn): Blech, this belongs in build/common.sh, probably, -# but common.sh sets up its readonly variables when its sourced, so -# there's a chicken/egg issue getting it there and using it for -# KUBE_GCE_RELEASE_PREFIX. -function kube::release::semantic_version() { - # This takes: - # Client Version: version.Info{Major:"1", Minor:"1+", GitVersion:"v1.1.0-alpha.0.2328+3c0a05de4a38e3", GitCommit:"3c0a05de4a38e355d147dbfb4d85bad6d2d73bb9", GitTreeState:"clean"} - # and spits back the GitVersion piece in a way that is somewhat - # resilient to the other fields changing (we hope) - ${KUBE_ROOT}/cluster/kubectl.sh version -c | sed "s/, */\\ -/g" | egrep "^GitVersion:" | cut -f2 -d: | cut -f2 -d\" -} - KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. + +source "${KUBE_ROOT}/build/util.sh" + LATEST=$(kube::release::semantic_version) KUBE_GCS_NO_CACHING='n' @@ -44,7 +34,7 @@ KUBE_GCS_DELETE_EXISTING='y' KUBE_GCS_RELEASE_PREFIX="ci/${LATEST}" KUBE_GCS_PUBLISH_VERSION="${LATEST}" -source "$KUBE_ROOT/build/common.sh" +source "${KUBE_ROOT}/build/common.sh" MAX_ATTEMPTS=3 attempt=0 @@ -53,4 +43,13 @@ while [[ ${attempt} -lt ${MAX_ATTEMPTS} ]]; do attempt=$((attempt + 1)) sleep 5 done -[[ ${attempt} -lt ${MAX_ATTEMPTS} ]] || exit 1 +if [[ ! ${attempt} -lt ${MAX_ATTEMPTS} ]];then + kube::log::error "Max attempts reached. Will exit." + exit 1 +fi + +if [[ "${FEDERATION:-}" == "true" ]];then + source "${KUBE_ROOT}/federation/cluster/common.sh" + # Docker compatiblity + FEDERATION_IMAGE_TAG="$(kube::release::semantic_image_tag_version)" push-federated-images +fi diff --git a/build/release.sh b/build/release.sh index 0c1636adb6..def030c7d4 100755 --- a/build/release.sh +++ b/build/release.sh @@ -37,6 +37,14 @@ if [[ $KUBE_RELEASE_RUN_TESTS =~ ^[yY]$ ]]; then kube::build::run_build_command hack/test-integration.sh fi +if [[ "${FEDERATION:-}" == "true" ]];then + ( + source "${KUBE_ROOT}/build/util.sh" + # Write federated docker image tag to workspace + kube::release::semantic_image_tag_version > "${KUBE_ROOT}/federation/manifests/federated-image.tag" + ) +fi + kube::build::copy_output kube::release::package_tarballs kube::release::package_hyperkube diff --git a/build/util.sh b/build/util.sh new file mode 100644 index 0000000000..aa360225a8 --- /dev/null +++ b/build/util.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Common utility functions for build scripts + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. + +function kube::release::semantic_version() { + # This takes: + # Client Version: version.Info{Major:"1", Minor:"1+", GitVersion:"v1.1.0-alpha.0.2328+3c0a05de4a38e3", GitCommit:"3c0a05de4a38e355d147dbfb4d85bad6d2d73bb9", GitTreeState:"clean"} + # and spits back the GitVersion piece in a way that is somewhat + # resilient to the other fields changing (we hope) + ${KUBE_ROOT}/cluster/kubectl.sh version -c | sed "s/, */\\ +/g" | egrep "^GitVersion:" | cut -f2 -d: | cut -f2 -d\" +} + +function kube::release::semantic_image_tag_version() { + printf "$(kube::release::semantic_version)" | tr + _ +} diff --git a/cluster/common.sh b/cluster/common.sh index a70c89d0a3..1ca2736afd 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -38,7 +38,6 @@ KUBE_RELEASE_VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]* # kube::release::parse_and_validate_ci_version() KUBE_CI_VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)-(beta|alpha)\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*)\\+[-0-9a-z]*)?$" - # Generate kubeconfig data for the created cluster. # Assumed vars: # KUBE_USER @@ -50,12 +49,23 @@ KUBE_CI_VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)-(be # If the apiserver supports bearer auth, also provide: # KUBE_BEARER_TOKEN # +# If the kubeconfig context being created should NOT be set as the current context +# SECONDARY_KUBECONFIG=true +# +# To explicitly name the context being created, use OVERRIDE_CONTEXT +# # The following can be omitted for --insecure-skip-tls-verify # KUBE_CERT # KUBE_KEY # CA_CERT function create-kubeconfig() { local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" + SECONDARY_KUBECONFIG=${SECONDARY_KUBECONFIG:-} + OVERRIDE_CONTEXT=${OVERRIDE_CONTEXT:-} + + if [[ "$OVERRIDE_CONTEXT" != "" ]];then + CONTEXT=$OVERRIDE_CONTEXT + fi export KUBECONFIG=${KUBECONFIG:-$DEFAULT_KUBECONFIG} # KUBECONFIG determines the file we write to, but it may not exist yet @@ -99,7 +109,10 @@ function create-kubeconfig() { "${kubectl}" config set-credentials "${CONTEXT}" "${user_args[@]}" fi "${kubectl}" config set-context "${CONTEXT}" --cluster="${CONTEXT}" --user="${CONTEXT}" - "${kubectl}" config use-context "${CONTEXT}" --cluster="${CONTEXT}" + + if [[ "${SECONDARY_KUBECONFIG}" != "true" ]];then + "${kubectl}" config use-context "${CONTEXT}" --cluster="${CONTEXT}" + fi # If we have a bearer token, also create a credential entry with basic auth # so that it is easy to discover the basic auth password for your cluster diff --git a/cluster/kube-util.sh b/cluster/kube-util.sh index b3d0da5121..416e0cb4af 100644 --- a/cluster/kube-util.sh +++ b/cluster/kube-util.sh @@ -99,3 +99,33 @@ PROVIDER_UTILS="${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" if [ -f ${PROVIDER_UTILS} ]; then source "${PROVIDER_UTILS}" fi + +# Federation utils + +# Should NOT be called within the global scope, unless setting the desired global zone vars +# This function is currently NOT USED in the global scope +function set-federated-zone-vars { + zone="$1" + export OVERRIDE_CONTEXT="federation-e2e-${KUBERNETES_PROVIDER}-$zone" + echo "Setting zone vars to: $OVERRIDE_CONTEXT" + if [[ "$KUBERNETES_PROVIDER" == "gce" ]];then + + export KUBE_GCE_ZONE="$zone" + # gcloud has a 61 character limit, and for firewall rules this + # prefix gets appended to itslef, with some extra information + # need tot keep it short + export KUBE_GCE_INSTANCE_PREFIX="${USER}-${zone}" + + elif [[ "$KUBERNETES_PROVIDER" == "gke" ]];then + + export CLUSTER_NAME="${USER}-${zone}" + + elif [[ "$KUBERNETES_PROVIDER" == "aws" ]];then + + export KUBE_AWS_ZONE="$zone" + export KUBE_AWS_INSTANCE_PREFIX="${USER}-${zone}" + else + echo "Provider \"${KUBERNETES_PROVIDER}\" is not supported" + exit 1 + fi +} diff --git a/cluster/vagrant/provision-master.sh b/cluster/vagrant/provision-master.sh index eaffaa471e..99f1de04c7 100755 --- a/cluster/vagrant/provision-master.sh +++ b/cluster/vagrant/provision-master.sh @@ -118,7 +118,7 @@ if ! which /usr/libexec/cockpit-ws &>/dev/null; then pushd /etc/yum.repos.d curl -OL https://copr.fedorainfracloud.org/coprs/g/cockpit/cockpit-preview/repo/fedora-23/msuchy-cockpit-preview-fedora-23.repo - dnf install -y cockpit cockpit-kubernetes + dnf install -y cockpit cockpit-kubernetes socat ethtool dnf update -y docker popd diff --git a/cluster/vagrant/provision-node.sh b/cluster/vagrant/provision-node.sh index c7473255a0..d62536a5a4 100755 --- a/cluster/vagrant/provision-node.sh +++ b/cluster/vagrant/provision-node.sh @@ -83,3 +83,5 @@ add-volume-support run-salt +dnf install -y socat ethtool +dnf update -y docker diff --git a/federation/cluster/common.sh b/federation/cluster/common.sh new file mode 100644 index 0000000000..1c04b0ab45 --- /dev/null +++ b/federation/cluster/common.sh @@ -0,0 +1,189 @@ +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# required: +# KUBE_ROOT: path of the root of the Kubernetes reposiitory + +# optional override +# FEDERATION_IMAGE_REPO_BASE: repo which federated images are tagged under (default gcr.io/google_containers) +# FEDERATION_NAMESPACE: name of the namespace will created for the federated components in the underlying cluster. +# KUBE_PLATFORM +# KUBE_ARCH +# KUBE_BUILD_STAGE + +: "${KUBE_ROOT?Must set KUBE_ROOT env var}" + +FEDERATION_IMAGE_REPO_BASE=${FEDERATION_IMAGE_REPO_BASE:-'gcr.io/google_containers'} +FEDERATION_NAMESPACE=${FEDERATION_NAMESPACE:-federation-e2e} + +KUBE_PLATFORM=${KUBE_PLATFORM:-linux} +KUBE_ARCH=${KUBE_ARCH:-amd64} +KUBE_BUILD_STAGE=${KUBE_BUILD_STAGE:-release-stage} + +source "${KUBE_ROOT}/cluster/common.sh" + +host_kubectl="${KUBE_ROOT}/cluster/kubectl.sh --namespace=${FEDERATION_NAMESPACE}" + +# required: +# FEDERATION_PUSH_REPO_BASE: repo to which federated container images will be pushed + +# Optional +# FEDERATION_IMAGE_TAG: reference and pull all federated images with this tag. Used for ci testing +function create-federated-api-objects { +( + : "${FEDERATION_PUSH_REPO_BASE?Must set FEDERATION_PUSH_REPO_BASE env var}" + export FEDERATION_APISERVER_DEPLOYMENT_NAME="federation-apiserver" + export FEDERATION_APISERVER_IMAGE_REPO="${FEDERATION_PUSH_REPO_BASE}/federation-apiserver" + export FEDERATION_APISERVER_IMAGE_TAG="${FEDERATION_IMAGE_TAG:-$(cat ${KUBE_ROOT}/_output/${KUBE_BUILD_STAGE}/server/${KUBE_PLATFORM}-${KUBE_ARCH}/kubernetes/server/bin/federation-apiserver.docker_tag)}" + + export FEDERATION_SERVICE_CIDR=${FEDERATION_SERVICE_CIDR:-"10.10.0.0/24"} + + #Only used for providers that require a nodeport service (vagrant for now) + #We will use loadbalancer services where we can + export FEDERATION_API_NODEPORT=32111 + export FEDERATION_NAMESPACE + + template="go run ${KUBE_ROOT}/federation/cluster/template.go" + + FEDERATION_KUBECONFIG_PATH="${KUBE_ROOT}/federation/cluster/kubeconfig" + + federation_kubectl="${KUBE_ROOT}/cluster/kubectl.sh --context=federated-cluster --namespace=default" + + manifests_root="${KUBE_ROOT}/federation/manifests/" + + $template "${manifests_root}/federation-ns.yaml" | $host_kubectl apply -f - + + cleanup-federated-api-objects + + export FEDERATION_API_HOST="" + export KUBE_MASTER_IP="" + if [[ "$KUBERNETES_PROVIDER" == "vagrant" ]];then + # The vagrant approach is to use a nodeport service, and point kubectl at one of the nodes + $template "${manifests_root}/federation-apiserver-nodeport-service.yaml" | $host_kubectl create -f - + node_addresses=`$host_kubectl get nodes -o=jsonpath='{.items[*].status.addresses[?(@.type=="InternalIP")].address}'` + FEDERATION_API_HOST=`printf "$node_addresses" | cut -d " " -f1` + KUBE_MASTER_IP="${FEDERATION_API_HOST}:${FEDERATION_API_NODEPORT}" + elif [[ "$KUBERNETES_PROVIDER" == "gce" || "$KUBERNETES_PROVIDER" == "gke" || "$KUBERNETES_PROVIDER" == "aws" ]];then + # any capable providers should use a loadbalancer service + # we check for ingress.ip and ingress.hostname, so should work for any loadbalancer-providing provider + # allows 30x5 = 150 seconds for loadbalancer creation + $template "${manifests_root}/federation-apiserver-lb-service.yaml" | $host_kubectl create -f - + for i in {1..30};do + echo "attempting to get federation-apiserver loadbalancer hostname ($i / 30)" + for field in ip hostname;do + FEDERATION_API_HOST=`${host_kubectl} get -o=jsonpath svc/${FEDERATION_APISERVER_DEPLOYMENT_NAME} --template '{.status.loadBalancer.ingress[*].'"${field}}"` + if [[ ! -z "${FEDERATION_API_HOST// }" ]];then + break 2 + fi + done + if [[ $i -eq 30 ]];then + echo "Could not find ingress hostname for federation-apiserver loadbalancer service" + exit 1 + fi + sleep 5 + done + KUBE_MASTER_IP="${FEDERATION_API_HOST}:443" + else + echo "provider ${KUBERNETES_PROVIDER} is not (yet) supported for e2e testing" + exit 1 + fi + echo "Found federation-apiserver host at $FEDERATION_API_HOST" + + FEDERATION_API_TOKEN="$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)" + export FEDERATION_API_KNOWN_TOKENS="${FEDERATION_API_TOKEN},admin,admin" + + $template "${manifests_root}/federation-apiserver-"{deployment,secrets}".yaml" | $host_kubectl create -f - + + # Don't finish provisioning until federation-apiserver pod is running + for i in {1..30};do + #TODO(colhom): in the future this needs to scale out for N pods. This assumes just one pod + phase="$($host_kubectl get -o=jsonpath pods -lapp=federated-cluster,module=federation-apiserver --template '{.items[*].status.phase}')" + echo "Waiting for federation-apiserver to be running...(phase= $phase)" + if [[ "$phase" == "Running" ]];then + echo "federation-apiserver pod is running!" + break + fi + + if [[ $i -eq 30 ]];then + echo "federation-apiserver pod is not running! giving up." + exit 1 + fi + + sleep 4 + done + + CONTEXT=federated-cluster \ + KUBE_BEARER_TOKEN="$FEDERATION_API_TOKEN" \ + SECONDARY_KUBECONFIG=true \ + create-kubeconfig +) +} + +# Required +# FEDERATION_PUSH_REPO_BASE: the docker repo where federated images will be pushed + +# Optional +# FEDERATION_IMAGE_TAG: push all federated images with this tag. Used for ci testing +function push-federated-images { + : "${FEDERATION_PUSH_REPO_BASE?Must set FEDERATION_PUSH_REPO_BASE env var}" + local FEDERATION_BINARIES=${FEDERATION_BINARIES:-'federation-apiserver'} + + local imageFolder="${KUBE_ROOT}/_output/${KUBE_BUILD_STAGE}/server/${KUBE_PLATFORM}-${KUBE_ARCH}/kubernetes/server/bin" + + if [[ ! -d "$imageFolder" ]];then + echo "${imageFolder} does not exist! Run make quick-release or make release" + exit 1 + fi + + for binary in $FEDERATION_BINARIES;do + local imageFile="${imageFolder}/${binary}.tar" + + if [[ ! -f "$imageFile" ]];then + echo "${imageFile} does not exist!" + exit 1 + fi + + echo "Load: ${imageFile}" + # Load the image. Trust we know what it's called, as docker load provides no help there :( + docker load -q < "${imageFile}" + + local srcImageTag="$(cat ${imageFolder}/${binary}.docker_tag)" + local dstImageTag="${FEDERATION_IMAGE_TAG:-$srcImageTag}" + local srcImageName="${FEDERATION_IMAGE_REPO_BASE}/${binary}:${srcImageTag}" + local dstImageName="${FEDERATION_PUSH_REPO_BASE}/${binary}:${dstImageTag}" + + echo "Tag: ${srcImageName} --> ${dstImageName}" + docker tag "$srcImageName" "$dstImageName" + + echo "Push: $dstImageName" + if [[ "${FEDERATION_PUSH_REPO_BASE}" == "gcr.io/"* ]];then + echo " -> GCR repository detected. Using gcloud" + gcloud docker push "$dstImageName" + else + docker push "$dstImageName" + fi + + echo "Remove: $srcImageName" + docker rmi "$srcImageName" + + if [[ "$srcImageName" != "dstImageName" ]];then + echo "Remove: $dstImageName" + docker rmi "$dstImageName" + fi + + done +} +function cleanup-federated-api-objects { + $host_kubectl delete pods,svc,rc,deployment,secret -lapp=federated-cluster +} diff --git a/federation/cluster/federated-down.sh b/federation/cluster/federated-down.sh new file mode 100755 index 0000000000..92f1930795 --- /dev/null +++ b/federation/cluster/federated-down.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(readlink -m $(dirname "${BASH_SOURCE}")/../../) + +. ${KUBE_ROOT}/federation/cluster/common.sh + +cleanup-federated-api-objects + +$host_kubectl delete ns/${FEDERATION_NAMESPACE} diff --git a/federation/cluster/federated-push.sh b/federation/cluster/federated-push.sh new file mode 100755 index 0000000000..80d6664197 --- /dev/null +++ b/federation/cluster/federated-push.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Bring up a Kubernetes cluster. +# +# If the full release name (gs:///) is passed in then we take +# that directly. If not then we assume we are doing development stuff and take +# the defaults in the release config. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(readlink -m $(dirname "${BASH_SOURCE}")/../../) + +. ${KUBE_ROOT}/federation/cluster/common.sh + +push-federated-images + diff --git a/federation/cluster/federated-up.sh b/federation/cluster/federated-up.sh new file mode 100755 index 0000000000..750b907aaf --- /dev/null +++ b/federation/cluster/federated-up.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(readlink -m $(dirname "${BASH_SOURCE}")/../../) + +. ${KUBE_ROOT}/federation/cluster/common.sh + +create-federated-api-objects diff --git a/federation/cluster/template.go b/federation/cluster/template.go new file mode 100644 index 0000000000..9c947bc854 --- /dev/null +++ b/federation/cluster/template.go @@ -0,0 +1,76 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +This is a simple script that makes *every* environment variable available +as a go template field of the same name + +$ echo "hello world, MYVAR={{.MYVAR}}" > test.txt +$ MYVAR=foobar go run template.go test.txt +> hello world, MYVAR=foobar + +If you want the base64 version of any MYVAR, simple use {{.MYVAR_BASE64}} +*/ + +package main + +import ( + "encoding/base64" + "flag" + "fmt" + "io" + "os" + "path" + "strings" + "text/template" +) + +func main() { + flag.Parse() + env := make(map[string]string) + envList := os.Environ() + + for i := range envList { + pieces := strings.SplitN(envList[i], "=", 2) + if len(pieces) == 2 { + env[pieces[0]] = pieces[1] + env[pieces[0]+"_BASE64"] = base64.StdEncoding.EncodeToString([]byte(pieces[1])) + } else { + fmt.Fprintf(os.Stderr, "Invalid environ found: %s\n", envList[i]) + os.Exit(2) + } + } + + for i := 0; i < flag.NArg(); i++ { + inpath := flag.Arg(i) + + if err := templateYamlFile(env, inpath, os.Stdout); err != nil { + panic(err) + } + } +} + +func templateYamlFile(params map[string]string, inpath string, out io.Writer) error { + if tmpl, err := template.New(path.Base(inpath)).ParseFiles(inpath); err != nil { + return err + } else { + if err := tmpl.Execute(out, params); err != nil { + return err + } + } + _, err := out.Write([]byte("\n---\n")) + return err +} diff --git a/federation/manifests/.gitignore b/federation/manifests/.gitignore new file mode 100644 index 0000000000..b1280e87ff --- /dev/null +++ b/federation/manifests/.gitignore @@ -0,0 +1 @@ +/federated-image.tag diff --git a/federation/manifests/federation-apiserver-deployment.yaml b/federation/manifests/federation-apiserver-deployment.yaml new file mode 100644 index 0000000000..ad5e78dad0 --- /dev/null +++ b/federation/manifests/federation-apiserver-deployment.yaml @@ -0,0 +1,42 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{.FEDERATION_APISERVER_DEPLOYMENT_NAME}} + namespace: {{.FEDERATION_NAMESPACE}} + labels: + app: federated-cluster +spec: + template: + metadata: + name: federation-apiserver + labels: + app: federated-cluster + module: federation-apiserver + spec: + containers: + - name: apiserver + image: {{.FEDERATION_APISERVER_IMAGE_REPO}}:{{.FEDERATION_APISERVER_IMAGE_TAG}} + command: + - /usr/local/bin/federation-apiserver + - --bind-address=0.0.0.0 + - --etcd-servers=http://localhost:2379 + - --service-cluster-ip-range={{.FEDERATION_SERVICE_CIDR}} + - --secure-port=443 + - --advertise-address={{.FEDERATION_API_HOST}} + - --admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota + - --token-auth-file=/srv/kubernetes/known-tokens.csv + ports: + - containerPort: 443 + name: https + - containerPort: 8080 + name: local + volumeMounts: + - name: federation-apiserver-secrets + mountPath: /srv/kubernetes/ + readOnly: true + - name: etcd + image: quay.io/coreos/etcd:v2.3.3 + volumes: + - name: federation-apiserver-secrets + secret: + secretName: federation-apiserver-secrets diff --git a/federation/manifests/federation-apiserver-lb-service.yaml b/federation/manifests/federation-apiserver-lb-service.yaml new file mode 100644 index 0000000000..977df03ba7 --- /dev/null +++ b/federation/manifests/federation-apiserver-lb-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{.FEDERATION_APISERVER_DEPLOYMENT_NAME}} + namespace: {{.FEDERATION_NAMESPACE}} + labels: + app: federated-cluster +spec: + type: LoadBalancer + selector: + app: federated-cluster + module: federation-apiserver + ports: + - name: https + protocol: TCP + port: 443 + targetPort: 443 diff --git a/federation/manifests/federation-apiserver-nodeport-service.yaml b/federation/manifests/federation-apiserver-nodeport-service.yaml new file mode 100644 index 0000000000..f28856a92f --- /dev/null +++ b/federation/manifests/federation-apiserver-nodeport-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{.FEDERATION_APISERVER_DEPLOYMENT_NAME}} + namespace: {{.FEDERATION_NAMESPACE}} + labels: + app: federated-cluster +spec: + type: NodePort + selector: + app: federated-cluster + module: federation-apiserver + ports: + - name: https + protocol: TCP + nodePort: {{.FEDERATION_API_NODEPORT}} + port: 443 diff --git a/federation/manifests/federation-apiserver-secrets.yaml b/federation/manifests/federation-apiserver-secrets.yaml new file mode 100644 index 0000000000..13a8853d32 --- /dev/null +++ b/federation/manifests/federation-apiserver-secrets.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: federation-apiserver-secrets + labels: + app: federated-cluster +type: Opaque +data: + known-tokens.csv: {{.FEDERATION_API_KNOWN_TOKENS_BASE64}} diff --git a/federation/manifests/federation-ns.yaml b/federation/manifests/federation-ns.yaml new file mode 100644 index 0000000000..ae5a734959 --- /dev/null +++ b/federation/manifests/federation-ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{.FEDERATION_NAMESPACE}} diff --git a/hack/e2e-internal/e2e-cluster-size.sh b/hack/e2e-internal/e2e-cluster-size.sh index df4da5e203..66fe8c5f6a 100755 --- a/hack/e2e-internal/e2e-cluster-size.sh +++ b/hack/e2e-internal/e2e-cluster-size.sh @@ -29,4 +29,5 @@ source "${KUBE_ROOT}/cluster/kube-util.sh" prepare-e2e +#TODO(colhom): spec and implement federated version of this ${KUBECTL} get nodes --no-headers | wc -l diff --git a/hack/e2e-internal/e2e-down.sh b/hack/e2e-internal/e2e-down.sh index 4857e7ecfd..93dddd007b 100755 --- a/hack/e2e-internal/e2e-down.sh +++ b/hack/e2e-internal/e2e-down.sh @@ -29,4 +29,16 @@ source "${KUBE_ROOT}/cluster/kube-util.sh" prepare-e2e -test-teardown +if [[ "${FEDERATION:-}" == "true" ]];then + source "${KUBE_ROOT}/federation/cluster/common.sh" + for zone in ${E2E_ZONES};do + # bring up e2e cluster + ( + set-federated-zone-vars "$zone" + cleanup-federated-api-objects || echo "Couldn't cleanup federated api objects" + test-teardown + ) + done +else + test-teardown +fi diff --git a/hack/e2e-internal/e2e-status.sh b/hack/e2e-internal/e2e-status.sh index 61eb6313c8..f6e37dc0e8 100755 --- a/hack/e2e-internal/e2e-status.sh +++ b/hack/e2e-internal/e2e-status.sh @@ -29,4 +29,19 @@ source "${KUBE_ROOT}/cluster/kube-util.sh" prepare-e2e -${KUBECTL} version +if [[ "${FEDERATION:-}" == "true" ]];then + FEDERATION_NAMESPACE=${FEDERATION_NAMESPACE:-federation-e2e} + #TODO(colhom): the last cluster that was created in the loop above is the current context. + # Hence, it will be the cluster that hosts the federated components. + # In the future, we will want to loop through the all the federated contexts, + # select each one and call federated-up + for zone in ${E2E_ZONES};do + ( + set-federated-zone-vars "$zone" + printf "\n\tChecking version for $OVERRIDE_CONTEXT\n" + ${KUBECTL} --context="$OVERRIDE_CONTEXT" version + ) + done +else + ${KUBECTL} version +fi diff --git a/hack/e2e-internal/e2e-up.sh b/hack/e2e-internal/e2e-up.sh index 7a4a62fea4..4d5dadc4e1 100755 --- a/hack/e2e-internal/e2e-up.sh +++ b/hack/e2e-internal/e2e-up.sh @@ -29,4 +29,21 @@ source "${KUBE_ROOT}/cluster/kube-util.sh" prepare-e2e -test-setup +if [[ "${FEDERATION:-}" == "true" ]];then + #TODO(colhom): the last cluster that was created in the loop above is the current context. + # Hence, it will be the cluster that hosts the federated components. + # In the future, we will want to loop through the all the federated contexts, + # select each one and call federated-up + for zone in ${E2E_ZONES};do + ( + set-federated-zone-vars "$zone" + test-setup + ) + done + if [[ -f "${KUBE_ROOT}/federation/manifests/federated-image.tag" ]];then + export FEDERATION_IMAGE_TAG="$(cat "${KUBE_ROOT}/federation/manifests/federated-image.tag")" + fi + "${KUBE_ROOT}/federation/cluster/federated-up.sh" +else + test-setup +fi diff --git a/hack/e2e.go b/hack/e2e.go index 6f83febce9..ee2b3388c1 100644 --- a/hack/e2e.go +++ b/hack/e2e.go @@ -155,6 +155,10 @@ func Up() bool { // Ensure that the cluster is large engough to run the e2e tests. func ValidateClusterSize() { + if os.Getenv("FEDERATION") == "true" { + //TODO(colhom): federated equivalent of ValidateClusterSize + return + } // Check that there are at least minNodeCount nodes running cmd := exec.Command(path.Join(*root, "hack/e2e-internal/e2e-cluster-size.sh")) if *verbose { @@ -189,7 +193,15 @@ func Test() bool { ValidateClusterSize() } - return finishRunning("Ginkgo tests", exec.Command(filepath.Join(*root, "hack/ginkgo-e2e.sh"), strings.Fields(*testArgs)...)) + if os.Getenv("FEDERATION") == "" { + return finishRunning("Ginkgo tests", exec.Command(filepath.Join(*root, "hack/ginkgo-e2e.sh"), strings.Fields(*testArgs)...)) + } else { + + if *testArgs == "" { + *testArgs = "--ginkgo.focus=\\[Feature:Federation\\]" + } + return finishRunning("Federated Ginkgo tests", exec.Command(filepath.Join(*root, "hack/federated-ginkgo-e2e.sh"), strings.Fields(*testArgs)...)) + } } func finishRunning(stepName string, cmd *exec.Cmd) bool { @@ -212,8 +224,9 @@ func finishRunning(stepName string, cmd *exec.Cmd) bool { // returns either "", or a list of args intended for appending with the // kubectl command (beginning with a space). func kubectlArgs() string { + args := []string{""} if *checkVersionSkew { - return " --match-server-version" + args = append(args, "--match-server-version") } - return "" + return strings.Join(args, " ") } diff --git a/hack/federated-ginkgo-e2e.sh b/hack/federated-ginkgo-e2e.sh new file mode 100755 index 0000000000..b147e71991 --- /dev/null +++ b/hack/federated-ginkgo-e2e.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. + +source "${KUBE_ROOT}/cluster/kube-util.sh" + +#A little hack to get the last zone. we always deploy federated cluster to the last zone. +#TODO(colhom): deploy federated control plane to multiple underlying clusters in robust way +lastZone="" +for zone in ${E2E_ZONES};do + lastZone="$zone" +done +( + set-federated-zone-vars "$zone" + "${KUBE_ROOT}/hack/ginkgo-e2e.sh" $@ +) diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index 0ff6e972b2..f5f1ab9a3b 100755 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -136,6 +136,7 @@ readonly KUBE_TEST_PORTABLE=( hack/e2e-internal hack/get-build.sh hack/ginkgo-e2e.sh + hack/federated-ginkgo-e2e.sh hack/lib ) @@ -164,6 +165,7 @@ readonly KUBE_STATIC_LIBRARIES=( kube-scheduler kube-proxy kubectl + federation-apiserver ) kube::golang::is_statically_linked_library() { diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index c722de3cf4..6bbbe43a5d 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -143,6 +143,7 @@ failure-domains fake-clientset federated-api-burst federated-api-qps +federated-kube-context file-check-frequency file-suffix file_content_in_loop diff --git a/test/e2e/cleanup/cleanup.go b/test/e2e/cleanup/cleanup.go index 7fa15b6a4a..e8e5ac3f04 100644 --- a/test/e2e/cleanup/cleanup.go +++ b/test/e2e/cleanup/cleanup.go @@ -17,11 +17,12 @@ limitations under the License. package main import ( - flag "github.com/spf13/pflag" "log" "os" "strings" + flag "github.com/spf13/pflag" + "k8s.io/kubernetes/test/e2e" ) diff --git a/test/e2e/deployment.go b/test/e2e/deployment.go index b32b74a068..81b7c7c928 100644 --- a/test/e2e/deployment.go +++ b/test/e2e/deployment.go @@ -20,7 +20,10 @@ import ( "fmt" "time" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/annotations" "k8s.io/kubernetes/pkg/api/errors" "k8s.io/kubernetes/pkg/api/unversioned" "k8s.io/kubernetes/pkg/apis/extensions" @@ -35,10 +38,6 @@ import ( "k8s.io/kubernetes/pkg/util/wait" "k8s.io/kubernetes/pkg/watch" "k8s.io/kubernetes/test/e2e/framework" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "k8s.io/kubernetes/pkg/api/annotations" ) const ( diff --git a/test/e2e/federation-apiserver.go b/test/e2e/federation-apiserver.go new file mode 100644 index 0000000000..892ee2651b --- /dev/null +++ b/test/e2e/federation-apiserver.go @@ -0,0 +1,64 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + federationapi "k8s.io/kubernetes/federation/apis/federation" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/test/e2e/framework" +) + +// Create/delete cluster api objects +var _ = framework.KubeDescribe("Federation apiserver [Feature:Federation]", func() { + f := framework.NewDefaultFederatedFramework("federated-cluster") + It("should allow creation of cluster api objects", func() { + framework.SkipUnlessFederated() + + contexts := f.GetUnderlyingFederatedContexts() + + for _, context := range contexts { + framework.Logf("Creating cluster object: %s (%s)", context.Name, context.Cluster.Cluster.Server) + cluster := federationapi.Cluster{ + ObjectMeta: api.ObjectMeta{ + Name: context.Name, + }, + Spec: federationapi.ClusterSpec{ + ServerAddressByClientCIDRs: []federationapi.ServerAddressByClientCIDR{ + { + ClientCIDR: "0.0.0.0/0", + ServerAddress: context.Cluster.Cluster.Server, + }, + }, + //TODO(colhom): add SecretRef when #26132 lands + }, + } + _, err := f.FederationClient.Clusters().Create(&cluster) + framework.ExpectNoError(err, fmt.Sprintf("creating cluster: %+v", err)) + } + + for _, context := range contexts { + c, err := f.FederationClient.Clusters().Get(context.Name) + framework.ExpectNoError(err, fmt.Sprintf("get cluster: %+v", err)) + if c.ObjectMeta.Name != context.Name { + framework.Failf("cluster name does not match input context: actual=%+v, expected=%+v", c, context) + } + } + }) +}) diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 9a8b8d7d2b..a83c71b015 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -19,11 +19,13 @@ package framework import ( "bytes" "fmt" + "io/ioutil" "reflect" "strings" "sync" "time" + unversionedfederation "k8s.io/kubernetes/federation/client/clientset_generated/federation_internalclientset/typed/federation/unversioned" "k8s.io/kubernetes/pkg/api" apierrs "k8s.io/kubernetes/pkg/api/errors" "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_2" @@ -39,6 +41,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + yaml "gopkg.in/yaml.v2" ) const ( @@ -54,6 +57,8 @@ type Framework struct { Clientset_1_2 *release_1_2.Clientset Clientset_1_3 *release_1_3.Clientset + FederationClient *unversionedfederation.FederationClient + Namespace *api.Namespace // Every test has at least one namespace namespacesToDelete []*api.Namespace // Some tests have more than one. NamespaceDeletionTimeout time.Duration @@ -75,6 +80,9 @@ type Framework struct { // configuration for framework's client options FrameworkOptions + + // will this framework exercise a federated cluster as well + federated bool } type TestDataSummary interface { @@ -97,6 +105,12 @@ func NewDefaultFramework(baseName string) *Framework { return NewFramework(baseName, options, nil) } +func NewDefaultFederatedFramework(baseName string) *Framework { + f := NewDefaultFramework(baseName) + f.federated = true + return f +} + func NewFramework(baseName string, options FrameworkOptions, client *client.Client) *Framework { f := &Framework{ BaseName: baseName, @@ -130,9 +144,17 @@ func (f *Framework) BeforeEach() { Expect(err).NotTo(HaveOccurred()) f.Client = c } + f.Clientset_1_2 = adapter_1_2.FromUnversionedClient(f.Client) f.Clientset_1_3 = adapter_1_3.FromUnversionedClient(f.Client) + if f.federated && f.FederationClient == nil { + By("Creating a federated kubernetes client") + var err error + f.FederationClient, err = LoadFederationClient() + Expect(err).NotTo(HaveOccurred()) + } + By("Building a namespace api object") namespace, err := f.CreateNamespace(f.BaseName, map[string]string{ "e2e-framework": f.BaseName, @@ -206,6 +228,18 @@ func (f *Framework) AfterEach() { f.Client = nil }() + if f.federated { + defer func() { + if f.FederationClient == nil { + Logf("Warning: framework is marked federated, but has no FederationClient") + return + } + if err := f.FederationClient.Clusters().DeleteCollection(nil, api.ListOptions{}); err != nil { + Logf("Error: failed to delete Clusters: %+v", err) + } + }() + } + // Print events if the test failed. if CurrentGinkgoTestDescription().Failed && TestContext.DumpLogsOnFailure { DumpAllNamespaceInfo(f.Client, f.Namespace.Name) @@ -465,6 +499,97 @@ func (f *Framework) CreatePodsPerNodeForSimpleApp(appName string, podSpec func(n return labels } +type KubeUser struct { + Name string `yaml:"name"` + User struct { + Username string `yaml:"username"` + Password string `yaml:"password"` + Token string `yaml:"token"` + } `yaml:"user"` +} + +type KubeCluster struct { + Name string `yaml:"name"` + Cluster struct { + CertificateAuthorityData string `yaml:"certificate-authority-data"` + Server string `yaml:"server"` + } `yaml:"cluster"` +} + +type KubeConfig struct { + Contexts []struct { + Name string `yaml:"name"` + Context struct { + Cluster string `yaml:"cluster"` + User string + } `yaml:"context"` + } `yaml:"contexts"` + + Clusters []KubeCluster `yaml:"clusters"` + + Users []KubeUser `yaml:"users"` +} + +func (kc *KubeConfig) findUser(name string) *KubeUser { + for _, user := range kc.Users { + if user.Name == name { + return &user + } + } + return nil +} + +func (kc *KubeConfig) findCluster(name string) *KubeCluster { + for _, cluster := range kc.Clusters { + if cluster.Name == name { + return &cluster + } + } + return nil +} + +type E2EContext struct { + Name string `yaml:"name"` + Cluster *KubeCluster `yaml:"cluster"` + User *KubeUser `yaml:"user"` +} + +func (f *Framework) GetUnderlyingFederatedContexts() []E2EContext { + if !f.federated { + Failf("geUnderlyingFederatedContexts called on non-federated framework") + } + + kubeconfig := KubeConfig{} + configBytes, err := ioutil.ReadFile(TestContext.KubeConfig) + ExpectNoError(err) + err = yaml.Unmarshal(configBytes, &kubeconfig) + ExpectNoError(err) + + e2eContexts := []E2EContext{} + for _, context := range kubeconfig.Contexts { + if strings.HasPrefix(context.Name, "federation-e2e") { + + user := kubeconfig.findUser(context.Context.User) + if user == nil { + Failf("Could not find user for context %+v", context) + } + + cluster := kubeconfig.findCluster(context.Context.Cluster) + if cluster == nil { + Failf("Could not find cluster for context %+v", context) + } + + e2eContexts = append(e2eContexts, E2EContext{ + Name: context.Name, + Cluster: cluster, + User: user, + }) + } + } + + return e2eContexts +} + func kubectlExecWithRetry(namespace string, podName, containerName string, args ...string) ([]byte, []byte, error) { for numRetries := 0; numRetries < maxKubectlExecRetries; numRetries++ { if numRetries > 0 { diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index f327163f5d..128cc07c17 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -77,10 +77,7 @@ type CloudConfig struct { } var TestContext TestContextType - -func SetTestContext(t TestContextType) { - TestContext = t -} +var federatedKubeContext string func RegisterFlags() { // Turn on verbose by default to get spec names @@ -95,6 +92,8 @@ func RegisterFlags() { flag.StringVar(&TestContext.KubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.") flag.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'") flag.StringVar(&TestContext.KubeAPIContentType, "kube-api-content-type", "", "ContentType used to communicate with apiserver") + flag.StringVar(&federatedKubeContext, "federated-kube-context", "federated-cluster", "kubeconfig context for federated-cluster.") + flag.StringVar(&TestContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.") flag.StringVar(&TestContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.") flag.StringVar(&TestContext.Host, "host", "", "The host, or apiserver, to connect to") diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 99140300b6..5c6047872a 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -36,6 +36,7 @@ import ( "sync" "time" + unversionedfederation "k8s.io/kubernetes/federation/client/clientset_generated/federation_internalclientset/typed/federation/unversioned" "k8s.io/kubernetes/pkg/api" apierrs "k8s.io/kubernetes/pkg/api/errors" "k8s.io/kubernetes/pkg/api/resource" @@ -325,6 +326,28 @@ func SkipUnlessServerVersionGTE(v semver.Version, c discovery.ServerVersionInter } } +// Detects whether the federation namespace exists in the underlying cluster +func SkipUnlessFederated() { + c, err := LoadClient() + if err != nil { + Failf("Unable to load client: %v", err) + } + + federationNS := os.Getenv("FEDERATION_NAMESPACE") + if federationNS == "" { + federationNS = "federation-e2e" + } + + _, err = c.Namespaces().Get(federationNS) + if err != nil { + if apierrs.IsNotFound(err) { + Skipf("Could not find federation namespace %s: skipping federated test", federationNS) + } else { + Failf("Unexpected error getting namespace: %v", err) + } + } +} + // ProvidersWithSSH are those providers where each node is accessible with SSH var ProvidersWithSSH = []string{"gce", "gke", "aws"} @@ -1521,22 +1544,42 @@ func ServiceResponding(c *client.Client, ns, name string) error { }) } -func LoadConfig() (*restclient.Config, error) { - switch { - case TestContext.KubeConfig != "": - Logf(">>> TestContext.KubeConfig: %s\n", TestContext.KubeConfig) - c, err := clientcmd.LoadFromFile(TestContext.KubeConfig) - if err != nil { - return nil, fmt.Errorf("error loading KubeConfig: %v", err.Error()) - } - if TestContext.KubeContext != "" { - Logf(">>> TestContext.KubeContext: %s\n", TestContext.KubeContext) - c.CurrentContext = TestContext.KubeContext - } - return clientcmd.NewDefaultClientConfig(*c, &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: TestContext.Host}}).ClientConfig() - default: +func restclientConfig(kubeContext string) (*clientcmdapi.Config, error) { + Logf(">>> kubeConfig: %s\n", TestContext.KubeConfig) + if TestContext.KubeConfig == "" { return nil, fmt.Errorf("KubeConfig must be specified to load client config") } + c, err := clientcmd.LoadFromFile(TestContext.KubeConfig) + if err != nil { + return nil, fmt.Errorf("error loading KubeConfig: %v", err.Error()) + } + if kubeContext != "" { + Logf(">>> kubeContext: %s\n", kubeContext) + c.CurrentContext = kubeContext + } + return c, nil +} + +func LoadConfig() (*restclient.Config, error) { + c, err := restclientConfig(TestContext.KubeContext) + if err != nil { + return nil, err + } + + return clientcmd.NewDefaultClientConfig(*c, &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: TestContext.Host}}).ClientConfig() +} + +func LoadFederatedConfig() (*restclient.Config, error) { + c, err := restclientConfig(federatedKubeContext) + if err != nil { + return nil, err + } + cfg, err := clientcmd.NewDefaultClientConfig(*c, &clientcmd.ConfigOverrides{}).ClientConfig() + if cfg != nil { + //TODO(colhom): this is only here because https://github.com/kubernetes/kubernetes/issues/25422 + cfg.NegotiatedSerializer = api.Codecs + } + return cfg, err } func loadClientFromConfig(config *restclient.Config) (*client.Client, error) { @@ -1550,6 +1593,25 @@ func loadClientFromConfig(config *restclient.Config) (*client.Client, error) { return c, nil } +func loadFederationClientFromConfig(config *restclient.Config) (*unversionedfederation.FederationClient, error) { + c, err := unversionedfederation.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("error creating client: %v", err.Error()) + } + if c.Client.Timeout == 0 { + c.Client.Timeout = SingleCallTimeout + } + return c, nil +} + +func LoadFederationClient() (*unversionedfederation.FederationClient, error) { + config, err := LoadFederatedConfig() + if err != nil { + return nil, fmt.Errorf("error creating client: %v", err.Error()) + } + return loadFederationClientFromConfig(config) +} + func LoadClient() (*client.Client, error) { config, err := LoadConfig() if err != nil { diff --git a/test/images/clusterapi-tester/main.go b/test/images/clusterapi-tester/main.go index 58c7a43270..b38e3986b4 100644 --- a/test/images/clusterapi-tester/main.go +++ b/test/images/clusterapi-tester/main.go @@ -22,11 +22,12 @@ import ( "log" "fmt" + "net/http" + "k8s.io/kubernetes/pkg/api" client "k8s.io/kubernetes/pkg/client/unversioned" "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/labels" - "net/http" ) func main() {