mirror of https://github.com/k3s-io/k3s
Revive conformance test.
Use KUBE_CONFIG_FILE instead of AUTH_CONFIG (and CERT_DIR). Pass GINKGO_TEST_ARGS for a subset of e2e tests which @erictune has deemed initially sufficient for conformance. Allow GINKGO_TEST_ARGS to pass through hack/ginkgo-e2e.sh. Set NUM_MINIONS (need better way to handle this). Remove use of "KUBERNETES_CONFORMANCE_TEST" variable and use of KUBERNETES_PROVIDER="conformance_test" convention, both of which have no apparent purpose. Allow unset testContext.provider in test/e2e/e2e_test.go Allow testContext.Provider to be unset in providerIs().pull/6/head
Eric Tune
parent
ebeb104493
commit
c5efb12a62
|
|
@ -14,26 +14,97 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# The conformance test is provided to let users run an e2e test
|
||||
# against an already-setup cluster for which there is no automated
|
||||
# setup, teardown, and other cluster/... scripts.
|
||||
# The conformance test checks whether a kubernetes cluster supports
|
||||
# a minimum set of features to be called "Kubernetes". It is similar
|
||||
# to `hack/e2e-test.sh` but it differs in that:
|
||||
# - hack/e2e-test.sh is intended to test a cluster with binaries built at HEAD,
|
||||
# while this conformance test does not care what version the binaries are.
|
||||
# - this means the user needs to setup a cluster first.
|
||||
# - this means the user does not need to write any cluster/... scripts. Custom
|
||||
# clusters can be tested.
|
||||
# - hack/e2e-test.sh is intended to run e2e tests built at HEAD, while
|
||||
# this conformance test is intended to be run e2e tests built at a particular
|
||||
# version. This ensures that all conformance testees run the same set of tests,
|
||||
# regardless of when they test for conformance.
|
||||
# - it excludes certain e2e tests:
|
||||
# - tests that are specific to certain cloud providers
|
||||
# - tests of optional features, such as volume types.
|
||||
# - tests of performance, scale, or reliability
|
||||
# - known flaky tests.
|
||||
|
||||
# The conformance test should be run from a github repository at
|
||||
# commit TBDCOMMITNUMBER. Otherwise, it may not include the right
|
||||
# set of tests.
|
||||
# e.g.:
|
||||
# cd /new/directory
|
||||
# git clone git://github.com/GoogleCloudPlatform/kubernetes.git
|
||||
# cd kubernetes
|
||||
# git checkout TBDCOMMITNUMBER.
|
||||
# The working tree will be in a "detached HEAD" state.
|
||||
#
|
||||
# The user must export these environment variables:
|
||||
# KUBE_MASTER_IP to the ip address of the master.
|
||||
# AUTH_CONFIG to the argument of the "--auth_config=" flag.
|
||||
# If certs required, set CERT_DIR.
|
||||
#
|
||||
# Example to test against a local vagrant cluster:
|
||||
# declare -x AUTH_CONFIG="$HOME/.kubernetes_vagrant_auth"
|
||||
# declare -x KUBE_MASTER_IP=10.245.1.2
|
||||
# hack/conformance-test.sh
|
||||
if [[ -z "KUBE_MASTER_IP" ]]; then
|
||||
echo "Must set KUBE_MASTER_IP before running conformance test."
|
||||
exit 1
|
||||
fi
|
||||
if [[ -z "AUTH_CONFIG" ]]; then
|
||||
echo "Must set AUTH_CONFIG before running conformance test."
|
||||
exit 1
|
||||
fi
|
||||
# When run as described above, the conformance test tests whether a cluster is
|
||||
# supports key features for Kubernetes version 1.0.
|
||||
#
|
||||
# TODO: when preparing to release a new major or minor version of Kubernetes,
|
||||
# then update above commit number, reevaluate the set of e2e tests,
|
||||
# update documentation at docs/getting-started-guides/README.md to have
|
||||
# a new column for conformance at that new version, and notify
|
||||
# community.
|
||||
|
||||
# Instructions:
|
||||
# - Setup a Kubernetes cluster with $NUM_MINIONS nodes (defined below).
|
||||
# - Provide a Kubeconfig file whose current context is set to the
|
||||
# cluster to be tested, and with suitable auth setting.
|
||||
# - Specify the location of that kubeconfig with, e.g.:
|
||||
# declare -x KUBECONFIG="$HOME/.kube/config"
|
||||
# - Specify the location of the master with, e.g.:
|
||||
# declare -x KUBE_MASTER_IP="1.2.3.4"
|
||||
# - Make sure only essential pods are running and there are no failed/pending pods.
|
||||
# - Make binaries needed by e2e, e.g.:
|
||||
# make clean
|
||||
# make quick-release
|
||||
# - Run the test and capture output:
|
||||
# hack/conformance-test.sh 2>&1 | tee conformance.$(date +%FT%T%z).log
|
||||
|
||||
: ${KUBECONFIG:?"Must set KUBECONFIG before running conformance test."}
|
||||
: ${KUBE_MASTER_IP:?"Must set KUBE_MASTER_IP before running conformance test."}
|
||||
echo "Conformance test using ${KUBECONFIG} against master at ${KUBE_MASTER_IP}"
|
||||
echo -n "Conformance test run date:"
|
||||
date
|
||||
echo -n "Conformance test SHA:"
|
||||
HEAD_SHA=$(git rev-parse HEAD)
|
||||
echo $HEAD_SHA
|
||||
echo "Conformance test version tag(s):"
|
||||
git show-ref | grep $HEAD_SHA | grep refs/tags
|
||||
echo
|
||||
echo "Conformance test checking conformance with Kubernetes version 1.0"
|
||||
|
||||
# It runs a whitelist of tests. This whitelist was assembled at commit
|
||||
# b70b7084c93d4ce80b7463f48c23d5ac04edb2b1 starting from this list of tests:
|
||||
# grep -h 'It(\|Describe(' -R test
|
||||
#
|
||||
# List of test name patterns not included and why not included:
|
||||
# Cadvisor: impl detail how stats gotten from containers.
|
||||
# MasterCerts: GKE/GCE specific
|
||||
# Density: performance
|
||||
# Cluster level logging...: optional feature
|
||||
# Etcd failure: reliability
|
||||
# Load Capacity: performance
|
||||
# Monitoring: optional feature.
|
||||
# Namespaces.*seconds: performance.
|
||||
# Pod disks: uses GCE specific feature.
|
||||
# Reboot: node management
|
||||
# Nodes: node management.
|
||||
# Restart: node management.
|
||||
# Scale: performance
|
||||
# Services.*load balancer: not all cloud providers have a load balancer.
|
||||
# Services.*NodePort: flaky
|
||||
# Shell: replies on optional ssh access to nodes.
|
||||
# SSH: optional feature.
|
||||
# Volumes: contained only skipped tests.
|
||||
export CONFORMANCE_TEST_SKIP_REGEX="Cadvisor|MasterCerts|Density|Cluster\slevel\slogging.*|Etcd\sfailure.*|Load\sCapacity|Monitoring|Namespaces.*seconds|Pod\sdisks|Reboot|Restart|Nodes|Scale|Services.*load\sbalancer|Services.*NodePort|Shell|SSH|Volumes"
|
||||
|
||||
declare -x KUBERNETES_CONFORMANCE_TEST="1"
|
||||
declare -x NUM_MINIONS=4
|
||||
hack/ginkgo-e2e.sh
|
||||
exit $?
|
||||
|
|
|
|||
|
|
@ -42,7 +42,13 @@ export KUBECTL KUBE_CONFIG_FILE
|
|||
source "${KUBE_ROOT}/cluster/kube-env.sh"
|
||||
|
||||
# ---- Do cloud-provider-specific setup
|
||||
if [[ -z "${AUTH_CONFIG:-}" ]]; then
|
||||
if [[ -n "${KUBERNETES_CONFORMANCE_TEST:-}" ]]; then
|
||||
echo "Conformance test: not doing test setup."
|
||||
KUBERNETES_PROVIDER=""
|
||||
auth_config=(
|
||||
"--kubeconfig=${KUBECONFIG}"
|
||||
)
|
||||
else
|
||||
echo "Setting up for KUBERNETES_PROVIDER=\"${KUBERNETES_PROVIDER}\"."
|
||||
|
||||
source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh"
|
||||
|
|
@ -51,23 +57,9 @@ if [[ -z "${AUTH_CONFIG:-}" ]]; then
|
|||
|
||||
detect-master >/dev/null
|
||||
|
||||
if [[ "${KUBERNETES_PROVIDER}" == "conformance_test" ]]; then
|
||||
auth_config=(
|
||||
"--auth-config=${KUBERNETES_CONFORMANCE_TEST_AUTH_CONFIG:-}"
|
||||
"--cert-dir=${KUBERNETES_CONFORMANCE_TEST_CERT_DIR:-}"
|
||||
)
|
||||
else
|
||||
auth_config=(
|
||||
"--kubeconfig=${KUBECONFIG:-$DEFAULT_KUBECONFIG}"
|
||||
)
|
||||
fi
|
||||
else
|
||||
echo "Conformance Test. No cloud-provider-specific preparation."
|
||||
KUBERNETES_PROVIDER=""
|
||||
auth_config=(
|
||||
"--auth-config=${AUTH_CONFIG:-}"
|
||||
"--cert-dir=${CERT_DIR:-}"
|
||||
)
|
||||
auth_config=(
|
||||
"--kubeconfig=${KUBECONFIG:-$DEFAULT_KUBECONFIG}"
|
||||
)
|
||||
fi
|
||||
|
||||
if [[ -n "${NODE_INSTANCE_PREFIX:-}" ]]; then
|
||||
|
|
@ -81,6 +73,9 @@ if [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then
|
|||
fi
|
||||
|
||||
ginkgo_args=()
|
||||
if [[ -n ${CONFORMANCE_TEST_SKIP_REGEX:-} ]]; then
|
||||
ginkgo_args+=("--skip=\"${CONFORMANCE_TEST_SKIP_REGEX}\"")
|
||||
fi
|
||||
if [[ ${GINKGO_PARALLEL} =~ ^[yY]$ ]]; then
|
||||
ginkgo_args+=("-p")
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -103,9 +103,8 @@ func TestE2E(t *testing.T) {
|
|||
util.InitLogs()
|
||||
defer util.FlushLogs()
|
||||
|
||||
// TODO: possibly clean up or refactor this functionality.
|
||||
if testContext.Provider == "" {
|
||||
glog.Fatal("The --provider flag is not set. Treating as a conformance test. Some tests may not be run.")
|
||||
glog.Info("The --provider flag is not set. Treating as a conformance test. Some tests may not be run.")
|
||||
}
|
||||
|
||||
if testContext.Provider == "aws" {
|
||||
|
|
|
|||
|
|
@ -183,9 +183,6 @@ func Failf(format string, a ...interface{}) {
|
|||
}
|
||||
|
||||
func providerIs(providers ...string) bool {
|
||||
if testContext.Provider == "" {
|
||||
Fail("testContext.Provider is not defined")
|
||||
}
|
||||
for _, provider := range providers {
|
||||
if strings.ToLower(provider) == strings.ToLower(testContext.Provider) {
|
||||
return true
|
||||
|
|
|
|||
Loading…
Reference in New Issue