diff --git a/hack/conformance-test.sh b/hack/conformance-test.sh index de774c353e..8ae519c43a 100755 --- a/hack/conformance-test.sh +++ b/hack/conformance-test.sh @@ -14,26 +14,97 @@ # See the License for the specific language governing permissions and # limitations under the License. -# The conformance test is provided to let users run an e2e test -# against an already-setup cluster for which there is no automated -# setup, teardown, and other cluster/... scripts. +# The conformance test checks whether a kubernetes cluster supports +# a minimum set of features to be called "Kubernetes". It is similar +# to `hack/e2e-test.sh` but it differs in that: +# - hack/e2e-test.sh is intended to test a cluster with binaries built at HEAD, +# while this conformance test does not care what version the binaries are. +# - this means the user needs to setup a cluster first. +# - this means the user does not need to write any cluster/... scripts. Custom +# clusters can be tested. +# - hack/e2e-test.sh is intended to run e2e tests built at HEAD, while +# this conformance test is intended to be run e2e tests built at a particular +# version. This ensures that all conformance testees run the same set of tests, +# regardless of when they test for conformance. +# - it excludes certain e2e tests: +# - tests that are specific to certain cloud providers +# - tests of optional features, such as volume types. +# - tests of performance, scale, or reliability +# - known flaky tests. + +# The conformance test should be run from a github repository at +# commit TBDCOMMITNUMBER. Otherwise, it may not include the right +# set of tests. +# e.g.: +# cd /new/directory +# git clone git://github.com/GoogleCloudPlatform/kubernetes.git +# cd kubernetes +# git checkout TBDCOMMITNUMBER. +# The working tree will be in a "detached HEAD" state. # -# The user must export these environment variables: -# KUBE_MASTER_IP to the ip address of the master. -# AUTH_CONFIG to the argument of the "--auth_config=" flag. -# If certs required, set CERT_DIR. -# -# Example to test against a local vagrant cluster: -# declare -x AUTH_CONFIG="$HOME/.kubernetes_vagrant_auth" -# declare -x KUBE_MASTER_IP=10.245.1.2 -# hack/conformance-test.sh -if [[ -z "KUBE_MASTER_IP" ]]; then - echo "Must set KUBE_MASTER_IP before running conformance test." - exit 1 -fi -if [[ -z "AUTH_CONFIG" ]]; then - echo "Must set AUTH_CONFIG before running conformance test." - exit 1 -fi +# When run as described above, the conformance test tests whether a cluster is +# supports key features for Kubernetes version 1.0. +# +# TODO: when preparing to release a new major or minor version of Kubernetes, +# then update above commit number, reevaluate the set of e2e tests, +# update documentation at docs/getting-started-guides/README.md to have +# a new column for conformance at that new version, and notify +# community. + +# Instructions: +# - Setup a Kubernetes cluster with $NUM_MINIONS nodes (defined below). +# - Provide a Kubeconfig file whose current context is set to the +# cluster to be tested, and with suitable auth setting. +# - Specify the location of that kubeconfig with, e.g.: +# declare -x KUBECONFIG="$HOME/.kube/config" +# - Specify the location of the master with, e.g.: +# declare -x KUBE_MASTER_IP="1.2.3.4" +# - Make sure only essential pods are running and there are no failed/pending pods. +# - Make binaries needed by e2e, e.g.: +# make clean +# make quick-release +# - Run the test and capture output: +# hack/conformance-test.sh 2>&1 | tee conformance.$(date +%FT%T%z).log + +: ${KUBECONFIG:?"Must set KUBECONFIG before running conformance test."} +: ${KUBE_MASTER_IP:?"Must set KUBE_MASTER_IP before running conformance test."} +echo "Conformance test using ${KUBECONFIG} against master at ${KUBE_MASTER_IP}" +echo -n "Conformance test run date:" +date +echo -n "Conformance test SHA:" +HEAD_SHA=$(git rev-parse HEAD) +echo $HEAD_SHA +echo "Conformance test version tag(s):" +git show-ref | grep $HEAD_SHA | grep refs/tags +echo +echo "Conformance test checking conformance with Kubernetes version 1.0" + +# It runs a whitelist of tests. This whitelist was assembled at commit +# b70b7084c93d4ce80b7463f48c23d5ac04edb2b1 starting from this list of tests: +# grep -h 'It(\|Describe(' -R test +# +# List of test name patterns not included and why not included: +# Cadvisor: impl detail how stats gotten from containers. +# MasterCerts: GKE/GCE specific +# Density: performance +# Cluster level logging...: optional feature +# Etcd failure: reliability +# Load Capacity: performance +# Monitoring: optional feature. +# Namespaces.*seconds: performance. +# Pod disks: uses GCE specific feature. +# Reboot: node management +# Nodes: node management. +# Restart: node management. +# Scale: performance +# Services.*load balancer: not all cloud providers have a load balancer. +# Services.*NodePort: flaky +# Shell: replies on optional ssh access to nodes. +# SSH: optional feature. +# Volumes: contained only skipped tests. +export CONFORMANCE_TEST_SKIP_REGEX="Cadvisor|MasterCerts|Density|Cluster\slevel\slogging.*|Etcd\sfailure.*|Load\sCapacity|Monitoring|Namespaces.*seconds|Pod\sdisks|Reboot|Restart|Nodes|Scale|Services.*load\sbalancer|Services.*NodePort|Shell|SSH|Volumes" + +declare -x KUBERNETES_CONFORMANCE_TEST="1" +declare -x NUM_MINIONS=4 hack/ginkgo-e2e.sh exit $? diff --git a/hack/ginkgo-e2e.sh b/hack/ginkgo-e2e.sh index 8332f898fb..7479a44f7b 100755 --- a/hack/ginkgo-e2e.sh +++ b/hack/ginkgo-e2e.sh @@ -42,7 +42,13 @@ export KUBECTL KUBE_CONFIG_FILE source "${KUBE_ROOT}/cluster/kube-env.sh" # ---- Do cloud-provider-specific setup -if [[ -z "${AUTH_CONFIG:-}" ]]; then +if [[ -n "${KUBERNETES_CONFORMANCE_TEST:-}" ]]; then + echo "Conformance test: not doing test setup." + KUBERNETES_PROVIDER="" + auth_config=( + "--kubeconfig=${KUBECONFIG}" + ) +else echo "Setting up for KUBERNETES_PROVIDER=\"${KUBERNETES_PROVIDER}\"." source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" @@ -51,23 +57,9 @@ if [[ -z "${AUTH_CONFIG:-}" ]]; then detect-master >/dev/null - if [[ "${KUBERNETES_PROVIDER}" == "conformance_test" ]]; then - auth_config=( - "--auth-config=${KUBERNETES_CONFORMANCE_TEST_AUTH_CONFIG:-}" - "--cert-dir=${KUBERNETES_CONFORMANCE_TEST_CERT_DIR:-}" - ) - else - auth_config=( - "--kubeconfig=${KUBECONFIG:-$DEFAULT_KUBECONFIG}" - ) - fi -else - echo "Conformance Test. No cloud-provider-specific preparation." - KUBERNETES_PROVIDER="" - auth_config=( - "--auth-config=${AUTH_CONFIG:-}" - "--cert-dir=${CERT_DIR:-}" - ) + auth_config=( + "--kubeconfig=${KUBECONFIG:-$DEFAULT_KUBECONFIG}" + ) fi if [[ -n "${NODE_INSTANCE_PREFIX:-}" ]]; then @@ -81,6 +73,9 @@ if [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then fi ginkgo_args=() +if [[ -n ${CONFORMANCE_TEST_SKIP_REGEX:-} ]]; then + ginkgo_args+=("--skip=\"${CONFORMANCE_TEST_SKIP_REGEX}\"") +fi if [[ ${GINKGO_PARALLEL} =~ ^[yY]$ ]]; then ginkgo_args+=("-p") fi diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 22c88dbdf9..98df0876b5 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -103,9 +103,8 @@ func TestE2E(t *testing.T) { util.InitLogs() defer util.FlushLogs() - // TODO: possibly clean up or refactor this functionality. if testContext.Provider == "" { - glog.Fatal("The --provider flag is not set. Treating as a conformance test. Some tests may not be run.") + glog.Info("The --provider flag is not set. Treating as a conformance test. Some tests may not be run.") } if testContext.Provider == "aws" { diff --git a/test/e2e/util.go b/test/e2e/util.go index 250d528492..dc2883b2ff 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -183,9 +183,6 @@ func Failf(format string, a ...interface{}) { } func providerIs(providers ...string) bool { - if testContext.Provider == "" { - Fail("testContext.Provider is not defined") - } for _, provider := range providers { if strings.ToLower(provider) == strings.ToLower(testContext.Provider) { return true