From c09da9e03cfcce34adc49ed5f4024d7e9e4354f1 Mon Sep 17 00:00:00 2001 From: Isaac Hollander McCreery Date: Tue, 9 Feb 2016 16:06:46 -0800 Subject: [PATCH 1/5] Remove [Skipped] as a label for tests. --- docs/devel/e2e-tests.md | 1 - hack/conformance-test.sh | 2 +- hack/jenkins/e2e.sh | 39 ++++++++++++--------------------------- test/e2e/e2e.go | 3 +-- 4 files changed, 14 insertions(+), 31 deletions(-) diff --git a/docs/devel/e2e-tests.md b/docs/devel/e2e-tests.md index 5016381533..a9c440c73e 100644 --- a/docs/devel/e2e-tests.md +++ b/docs/devel/e2e-tests.md @@ -185,7 +185,6 @@ We are working on implementing clearer partitioning of our e2e tests to make run - `[Serial]`: If a test cannot be run in parallel with other tests (e.g. it takes too many resources or restarts nodes), it is labeled `[Serial]`, and should be run in serial as part of a separate suite. - `[Disruptive]`: If a test restarts components that might cause other tests to fail or break the cluster completely, it is labeled `[Disruptive]`. Any `[Disruptive]` test is also assumed to qualify for the `[Serial]` label, but need not be labeled as both. These tests are not run against soak clusters to avoid restarting components. - `[Flaky]`: If a test is found to be flaky and we have decided that it's too hard to fix in the short term (e.g. it's going to take a full engineer-week), it receives the `[Flaky]` label until it is fixed. The `[Flaky]` label should be used very sparingly, and should be accompanied with a reference to the issue for de-flaking the test, because while a test remains labeled `[Flaky]`, it is not monitored closely in CI. `[Flaky]` tests are by default not run, unless a `focus` or `skip` argument is explicitly given. -- `[Skipped]`: `[Skipped]` is a legacy label that we're phasing out. If a test is marked `[Skipped]`, there should be an issue open to label it properly. `[Skipped]` tests are by default not run, unless a `focus` or `skip` argument is explicitly given. - `[Feature:.+]`: If a test has non-default requirements to run or targets some non-core functionality, and thus should not be run as part of the standard suite, it receives a `[Feature:.+]` label, e.g. `[Feature:Performance]` or `[Feature:Ingress]`. `[Feature:.+]` tests are not run in our core suites, instead running in custom suites. If a feature is experimental or alpha and is not enabled by default due to being incomplete or potentially subject to breaking changes, it does *not* block the merge-queue, and thus should run in some separate test suites owned by the feature owner(s) (see #continuous_integration below). ### Conformance tests diff --git a/hack/conformance-test.sh b/hack/conformance-test.sh index 641c810c87..17453200fe 100755 --- a/hack/conformance-test.sh +++ b/hack/conformance-test.sh @@ -84,4 +84,4 @@ echo "Conformance test checking conformance with Kubernetes version 1.0" declare -x KUBERNETES_CONFORMANCE_TEST="y" declare -x NUM_NODES=4 -exec hack/ginkgo-e2e.sh -ginkgo.focus='\[Conformance\]' -ginkgo.skip='\[Skipped\]' ${TEST_ARGS} +exec hack/ginkgo-e2e.sh -ginkgo.focus='\[Conformance\]' ${TEST_ARGS} diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index 729b8ba055..47fcdd7a3d 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -225,8 +225,7 @@ case ${JOB_NAME} in : ${E2E_NETWORK:="e2e-gce-${NODE_NAME}-${EXECUTOR_NUMBER}"} : ${GINKGO_PARALLEL:="y"} # This list should match the list in kubernetes-e2e-gce. - # TODO(ihmccreery) remove [Skipped] once tests are relabeled - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} : ${KUBE_GCE_INSTANCE_PREFIX:="e2e-gce-${NODE_NAME}-${EXECUTOR_NUMBER}"} : ${PROJECT:="kubernetes-jenkins-pull"} # Override GCE defaults @@ -241,8 +240,7 @@ case ${JOB_NAME} in : ${E2E_PUBLISH_GREEN_VERSION:="true"} : ${E2E_NETWORK:="e2e-gce"} # This list should match the list in kubernetes-pull-build-test-e2e-gce. - # TODO(ihmccreery) remove [Skipped] once tests are relabeled - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} : ${GINKGO_PARALLEL:="y"} : ${KUBE_GCE_INSTANCE_PREFIX="e2e-gce"} : ${PROJECT:="k8s-jkns-e2e-gce"} @@ -253,9 +251,8 @@ case ${JOB_NAME} in kubernetes-e2e-gce-slow) : ${E2E_CLUSTER_NAME:="jenkins-gce-e2e-slow"} : ${E2E_NETWORK:="e2e-slow"} - # TODO(ihmccreery) remove [Skipped] once tests are relabeled : ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Slow\] \ - --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} : ${GINKGO_PARALLEL:="y"} : ${KUBE_GCE_INSTANCE_PREFIX:="e2e-slow"} : ${PROJECT:="k8s-jkns-e2e-gce-slow"} @@ -304,9 +301,8 @@ case ${JOB_NAME} in kubernetes-e2e-gce-flaky) : ${E2E_CLUSTER_NAME:="jenkins-gce-e2e-flaky"} : ${E2E_NETWORK:="e2e-flaky"} - # TODO(ihmccreery) remove [Skipped] once tests are relabeled : ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Flaky\] \ - --ginkgo.skip=\[Feature:.+\]|\[Skipped\]"} + --ginkgo.skip=\[Feature:.+\]"} : ${KUBE_GCE_INSTANCE_PREFIX:="e2e-flaky"} : ${PROJECT:="k8s-jkns-e2e-gce-flaky"} : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} @@ -318,9 +314,8 @@ case ${JOB_NAME} in : ${E2E_CLUSTER_NAME:="parallel-flaky"} : ${E2E_NETWORK:="e2e-parallel-flaky"} : ${GINKGO_PARALLEL:="y"} - # TODO(ihmccreery) remove [Skipped] once tests are relabeled : ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Flaky\] \ - --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Feature:.+\]|\[Skipped\]"} + --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Feature:.+\]"} : ${KUBE_GCE_INSTANCE_PREFIX:="parallel-flaky"} : ${PROJECT:="k8s-jkns-e2e-gce-prl-flaky"} : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} @@ -337,8 +332,7 @@ case ${JOB_NAME} in : ${E2E_SET_CLUSTER_API_VERSION:=y} : ${PROJECT:="k8s-jkns-e2e-gke-ci"} : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} - # TODO(ihmccreery) remove [Skipped] once tests are relabeled - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} : ${GINKGO_PARALLEL:="y"} ;; @@ -348,9 +342,8 @@ case ${JOB_NAME} in : ${E2E_SET_CLUSTER_API_VERSION:=y} : ${PROJECT:="k8s-jkns-e2e-gke-slow"} : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} - # TODO(ihmccreery) remove [Skipped] once tests are relabeled : ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Slow\] \ - --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} : ${GINKGO_PARALLEL:="y"} ;; @@ -371,9 +364,8 @@ case ${JOB_NAME} in : ${E2E_SET_CLUSTER_API_VERSION:=y} : ${PROJECT:="k8s-jkns-e2e-gke-ci-flaky"} : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} - # TODO(ihmccreery) remove [Skipped] once tests are relabeled : ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Flaky\] \ - --ginkgo.skip=\[Feature:.+\]|\[Skipped\]"} + --ginkgo.skip=\[Feature:.+\]"} ;; # AWS core jobs @@ -385,8 +377,7 @@ case ${JOB_NAME} in : ${E2E_ZONE:="us-west-2a"} : ${ZONE:="us-west-2a"} : ${E2E_NETWORK:="e2e-aws"} - # TODO(ihmccreery) remove [Skipped] once tests are relabeled - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} : ${GINKGO_PARALLEL:="y"} : ${KUBE_GCE_INSTANCE_PREFIX="e2e-aws"} : ${PROJECT:="k8s-jkns-e2e-aws"} @@ -619,9 +610,7 @@ case ${JOB_NAME} in # We should be testing the reliability of a long-running cluster. The # [Disruptive] tests kill/restart components or nodes in the cluster, # defeating the purpose of a soak cluster. (#15722) - # - # TODO(ihmccreery) remove [Skipped] once tests are relabeled - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} : ${KUBE_GCE_INSTANCE_PREFIX:="gce-soak-weekly"} : ${HAIRPIN_MODE:="false"} : ${PROJECT:="kubernetes-jenkins"} @@ -649,9 +638,7 @@ case ${JOB_NAME} in # We should be testing the reliability of a long-running cluster. The # [Disruptive] tests kill/restart components or nodes in the cluster, # defeating the purpose of a soak cluster. (#15722) - # - # TODO(ihmccreery) remove [Skipped] once tests are relabeled - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} : ${KUBE_GCE_INSTANCE_PREFIX:="gce-soak-weekly-2"} : ${PROJECT:="kubernetes-jenkins"} ;; @@ -683,9 +670,7 @@ case ${JOB_NAME} in # We should be testing the reliability of a long-running cluster. The # [Disruptive] tests kill/restart components or nodes in the cluster, # defeating the purpose of a soak cluster. (#15722) - # - # TODO(ihmccreery) remove [Skipped] once tests are relabeled - : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"} + : ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"} ;; # Upgrade jobs diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 58a010eea5..b76d63d716 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -221,8 +221,7 @@ func RunE2ETests(t *testing.T) { gomega.RegisterFailHandler(ginkgo.Fail) // Disable skipped tests unless they are explicitly requested. if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" { - // TODO(ihmccreery) Remove [Skipped] once all [Skipped] labels have been reclassified. - config.GinkgoConfig.SkipString = `\[Flaky\]|\[Skipped\]|\[Feature:.+\]` + config.GinkgoConfig.SkipString = `\[Flaky\]|\[Feature:.+\]` } // Run tests through the Ginkgo runner with output to console + JUnit for Jenkins From 3cec929f1cd93d6b9c4649b57e2dff5f6a9434a7 Mon Sep 17 00:00:00 2001 From: Isaac Hollander McCreery Date: Tue, 9 Feb 2016 16:31:10 -0800 Subject: [PATCH 2/5] Remove old GCE upgrade jobs; no longer useful --- hack/jenkins/e2e.sh | 103 -------------------------------------------- 1 file changed, 103 deletions(-) diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index 47fcdd7a3d..f57f9e6396 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -740,109 +740,6 @@ case ${JOB_NAME} in kubernetes-upgrade-gce-1.1-master-step7-e2e-new) configure_upgrade_step 'configured-in-release-1.1' 'ci/latest' 'upgrade-gce-1-1-master' 'k8s-jkns-gce-upgrade' ;; - - # kubernetes-upgrade-gce-1.0-current-release - # - # This suite: - # - # 1. launches a cluster at ci/latest-1.0, - # 2. upgrades the master to CURRENT_RELEASE_PUBLISHED_VERSION - # 3. runs ci/latest-1.0 e2es, - # 4. upgrades the rest of the cluster, - # 5. runs ci/latest-1.0 e2es again, then - # 6. runs CURRENT_RELEASE_PUBLISHED_VERSION e2es and tears down the cluster. - - kubernetes-upgrade-1.0-current-release-gce-step1-deploy) - : ${E2E_CLUSTER_NAME:="gce-upgrade-1-0"} - : ${E2E_NETWORK:="gce-upgrade-1-0"} - : ${JENKINS_PUBLISHED_VERSION:="ci/latest-1.0"} - : ${PROJECT:="k8s-jkns-gce-upgrade"} - : ${E2E_UP:="true"} - : ${E2E_TEST:="false"} - : ${E2E_DOWN:="false"} - : ${KUBE_GCE_INSTANCE_PREFIX:="e2e-upgrade-1-0"} - : ${NUM_NODES:=5} - ;; - - kubernetes-upgrade-1.0-current-release-gce-step2-upgrade-master) - : ${E2E_CLUSTER_NAME:="gce-upgrade-1-0"} - : ${E2E_NETWORK:="gce-upgrade-1-0"} - : ${E2E_OPT:="--check_version_skew=false"} - # Use upgrade logic of version we're upgrading to. - : ${JENKINS_PUBLISHED_VERSION:="${CURRENT_RELEASE_PUBLISHED_VERSION}"} - : ${JENKINS_FORCE_GET_TARS:=y} - : ${PROJECT:="k8s-jkns-gce-upgrade"} - : ${E2E_UP:="false"} - : ${E2E_TEST:="true"} - : ${E2E_DOWN:="false"} - : ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Feature:Upgrade\].*upgrade-master --upgrade-target=${CURRENT_RELEASE_PUBLISHED_VERSION}"} - : ${KUBE_GCE_INSTANCE_PREFIX:="e2e-upgrade-1-0"} - : ${NUM_NODES:=5} - : ${KUBE_ENABLE_DAEMONSETS:=true} - ;; - - kubernetes-upgrade-1.0-current-release-gce-step3-e2e-old) - : ${E2E_CLUSTER_NAME:="gce-upgrade-1-0"} - : ${E2E_NETWORK:="gce-upgrade-1-0"} - : ${E2E_OPT:="--check_version_skew=false"} - : ${JENKINS_FORCE_GET_TARS:=y} - # Run old e2es - : ${JENKINS_PUBLISHED_VERSION:="ci/latest-1.0"} - : ${PROJECT:="k8s-jkns-gce-upgrade"} - : ${E2E_UP:="false"} - : ${E2E_TEST:="true"} - : ${E2E_DOWN:="false"} - : ${KUBE_GCE_INSTANCE_PREFIX:="e2e-upgrade-1-0"} - : ${NUM_NODES:=5} - ;; - - kubernetes-upgrade-1.0-current-release-gce-step4-upgrade-cluster) - : ${E2E_CLUSTER_NAME:="gce-upgrade-1-0"} - : ${E2E_NETWORK:="gce-upgrade-1-0"} - : ${E2E_OPT:="--check_version_skew=false"} - # Use upgrade logic of version we're upgrading to. - : ${JENKINS_PUBLISHED_VERSION:="${CURRENT_RELEASE_PUBLISHED_VERSION}"} - : ${JENKINS_FORCE_GET_TARS:=y} - : ${PROJECT:="k8s-jkns-gce-upgrade"} - : ${E2E_UP:="false"} - : ${E2E_TEST:="true"} - : ${E2E_DOWN:="false"} - : ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Feature:Upgrade\].*upgrade-cluster --upgrade-target=${CURRENT_RELEASE_PUBLISHED_VERSION}"} - : ${KUBE_GCE_INSTANCE_PREFIX:="e2e-upgrade-1-0"} - : ${NUM_NODES:=5} - : ${KUBE_ENABLE_DAEMONSETS:=true} - ;; - - kubernetes-upgrade-1.0-current-release-gce-step5-e2e-old) - : ${E2E_CLUSTER_NAME:="gce-upgrade-1-0"} - : ${E2E_NETWORK:="gce-upgrade-1-0"} - : ${E2E_OPT:="--check_version_skew=false"} - : ${JENKINS_FORCE_GET_TARS:=y} - # Run old e2es - : ${JENKINS_PUBLISHED_VERSION:="ci/latest-1.0"} - : ${PROJECT:="k8s-jkns-gce-upgrade"} - : ${E2E_UP:="false"} - : ${E2E_TEST:="true"} - : ${E2E_DOWN:="false"} - : ${KUBE_GCE_INSTANCE_PREFIX:="e2e-upgrade-1-0"} - : ${NUM_NODES:=5} - ;; - - kubernetes-upgrade-1.0-current-release-gce-step6-e2e-new) - : ${E2E_CLUSTER_NAME:="gce-upgrade-1-0"} - : ${E2E_NETWORK:="gce-upgrade-1-0"} - # TODO(15011): these really shouldn't be (very) version skewed, but because - # we have to get CURRENT_RELEASE_PUBLISHED_VERSION again, it could get slightly out of whack. - : ${E2E_OPT:="--check_version_skew=false"} - : ${JENKINS_FORCE_GET_TARS:=y} - : ${JENKINS_PUBLISHED_VERSION:="${CURRENT_RELEASE_PUBLISHED_VERSION}"} - : ${PROJECT:="k8s-jkns-gce-upgrade"} - : ${E2E_UP:="false"} - : ${E2E_TEST:="true"} - : ${E2E_DOWN:="true"} - : ${KUBE_GCE_INSTANCE_PREFIX:="e2e-upgrade-1-0"} - : ${NUM_NODES:=5} - ;; esac # Skip gcloud update checking From ec5b9abba894fbac8b9a0e066a015a8a6088a4d3 Mon Sep 17 00:00:00 2001 From: Isaac Hollander McCreery Date: Tue, 9 Feb 2016 17:08:27 -0800 Subject: [PATCH 3/5] Add instructions and tooling for munging test infra for a new release series --- docs/devel/releasing.md | 30 +++++++++++++++++++++++------- hack/list-feature-tests.sh | 23 +++++++++++++++++++++++ 2 files changed, 46 insertions(+), 7 deletions(-) create mode 100755 hack/list-feature-tests.sh diff --git a/docs/devel/releasing.md b/docs/devel/releasing.md index 7c5ad9d9ba..643827d7f8 100644 --- a/docs/devel/releasing.md +++ b/docs/devel/releasing.md @@ -217,14 +217,30 @@ Finally, from a clone of upstream/master, *make sure* you still have `RELEASE_VERSION` set correctly, and run `./build/mark-stable-release.sh ${RELEASE_VERSION}`. -### Updating the master branch +### Manual tasks for new release series -If you are cutting a new release series, please also update the master branch: -change the `latestReleaseBranch` in `cmd/mungedocs/mungedocs.go` to the new -release branch (`release-X.Y`), run `hack/update-generated-docs.sh`. This will -let the unversioned warning in docs point to the latest release series. Please -send the changes as a PR titled "Update the latestReleaseBranch to release-X.Y -in the munger". +*TODO(#20946) Burn this list down.* + +If you are cutting a new release series, there are a few tasks that haven't yet +been automated that need to happen after the branch has been cut: + +1. Update the master branch constant for doc generation: change the + `latestReleaseBranch` in `cmd/mungedocs/mungedocs.go` to the new release + branch (`release-X.Y`), run `hack/update-generated-docs.sh`. This will let + the unversioned warning in docs point to the latest release series. Please + send the changes as a PR titled "Update the latestReleaseBranch to + release-X.Y in the munger". +1. Add test jobs for the new branch. See [End-2-End Testing in + Kubernetes](e2e-tests.md) for the test jobs that run in CI, which are under + version control in `hack/jenkins/e2e.sh` (on the release branch) and + `hack/jenkins/job-configs/kubernetes-e2e.yaml` (in `master`). You'll want + to duplicate/munge these for the release branch so that, as we cherry-pick + fixes onto the branch, we know that it builds, etc. +1. Make sure all features that are supposed to be GA are covered by tests. You + can use `hack/list-feature-tests.sh` to see a list of tests labeled as + `[Feature:.+]`; make sure that these are all either covered in CI jobs or + are experimental features. (The answer should already be 'yes', but this is + a good time to reconcile.) ## Injecting Version into Binaries diff --git a/hack/list-feature-tests.sh b/hack/list-feature-tests.sh new file mode 100755 index 0000000000..42eeda77ad --- /dev/null +++ b/hack/list-feature-tests.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A single script that lists all of the [Feature:.+] tests in our e2e suite. +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. +grep "\[Feature:(.+?)\]" "${KUBE_ROOT}"/test/e2e/*.go -Poh | sort | uniq From 0ad9b2552bafecf130b5cf9dc70eaf66dc972ca4 Mon Sep 17 00:00:00 2001 From: Isaac Hollander McCreery Date: Wed, 10 Feb 2016 10:30:10 -0800 Subject: [PATCH 4/5] Add docs for branching e2e jobs --- docs/devel/releasing.md | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/docs/devel/releasing.md b/docs/devel/releasing.md index 643827d7f8..27b0e906f9 100644 --- a/docs/devel/releasing.md +++ b/docs/devel/releasing.md @@ -230,17 +230,23 @@ been automated that need to happen after the branch has been cut: the unversioned warning in docs point to the latest release series. Please send the changes as a PR titled "Update the latestReleaseBranch to release-X.Y in the munger". -1. Add test jobs for the new branch. See [End-2-End Testing in - Kubernetes](e2e-tests.md) for the test jobs that run in CI, which are under - version control in `hack/jenkins/e2e.sh` (on the release branch) and - `hack/jenkins/job-configs/kubernetes-e2e.yaml` (in `master`). You'll want - to duplicate/munge these for the release branch so that, as we cherry-pick - fixes onto the branch, we know that it builds, etc. -1. Make sure all features that are supposed to be GA are covered by tests. You - can use `hack/list-feature-tests.sh` to see a list of tests labeled as - `[Feature:.+]`; make sure that these are all either covered in CI jobs or - are experimental features. (The answer should already be 'yes', but this is - a good time to reconcile.) +1. Add test jobs for the new branch. + 1. See [End-2-End Testing in Kubernetes](e2e-tests.md) for the test jobs + that should be running in CI, which are under version control in + `hack/jenkins/e2e.sh` (on the release branch) and + `hack/jenkins/job-configs/kubernetes-e2e.yaml` (in `master`). You'll + want to munge these for the release branch so that, as we cherry-pick + fixes onto the branch, we know that it builds, etc. (Talk with + @ihmccreery for more details.) + 1. Make sure all features that are supposed to be GA are covered by tests, + but remove feature tests on the release branch for features that aren't + GA. You can use `hack/list-feature-tests.sh` to see a list of tests + labeled as `[Feature:.+]`; make sure that these are all either covered in + CI jobs on the release branch or are experimental features. (The answer + should already be 'yes', but this is a good time to reconcile.) + 1. Make a dashboard in Jenkins that contains all of the jobs for this + release cycle, and also add them to Critical Builds. (Don't add them to + the merge-bot blockers; see kubernetes/contrib#156.) ## Injecting Version into Binaries From cc9b092e94d1d157f6ad2177e59b5b8a8386485d Mon Sep 17 00:00:00 2001 From: Isaac Hollander McCreery Date: Wed, 10 Feb 2016 10:32:25 -0800 Subject: [PATCH 5/5] Remove parallel-flaky suite --- hack/jenkins/e2e.sh | 14 -------------- hack/jenkins/job-configs/kubernetes-e2e.yaml | 3 --- 2 files changed, 17 deletions(-) diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index f57f9e6396..1048e79448 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -309,20 +309,6 @@ case ${JOB_NAME} in : ${E2E_DOWN:="true"} ;; - # Runs the flaky tests on GCE in parallel. - kubernetes-e2e-gce-parallel-flaky) - : ${E2E_CLUSTER_NAME:="parallel-flaky"} - : ${E2E_NETWORK:="e2e-parallel-flaky"} - : ${GINKGO_PARALLEL:="y"} - : ${GINKGO_TEST_ARGS:="--ginkgo.focus=\[Flaky\] \ - --ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Feature:.+\]"} - : ${KUBE_GCE_INSTANCE_PREFIX:="parallel-flaky"} - : ${PROJECT:="k8s-jkns-e2e-gce-prl-flaky"} - : ${FAIL_ON_GCP_RESOURCE_LEAK:="true"} - # Override GCE defaults. - NUM_NODES=${NUM_NODES_PARALLEL} - ;; - # GKE core jobs # Runs all non-slow, non-serial, non-flaky, tests on GKE in parallel. diff --git a/hack/jenkins/job-configs/kubernetes-e2e.yaml b/hack/jenkins/job-configs/kubernetes-e2e.yaml index 001e7c0fae..61e0151b62 100644 --- a/hack/jenkins/job-configs/kubernetes-e2e.yaml +++ b/hack/jenkins/job-configs/kubernetes-e2e.yaml @@ -54,9 +54,6 @@ - 'gce-flaky': description: 'Run E2E tests on GCE using the latest successful build. Limit to known-flaky tests.' timeout: 180 - - 'gce-parallel-flaky': - description: 'Run E2E tests using Ginkgo''s parallel test runner on GCE using the latest successful build. Limit to known-flaky tests.' - timeout: 90 - 'gce-scalability': description: 'Run scalability E2E tests on GCE using the latest successful build.' timeout: 120