From 22f985470b742fa08bdca0fbc20fccfb2a4d6ba9 Mon Sep 17 00:00:00 2001 From: Dan Bond Date: Tue, 9 May 2023 09:13:38 -0700 Subject: [PATCH] Revert "ci: remove test splitting for compatibility tests (#17166)" This reverts commit 861a8151d50377315c6c391833fef85b71b54d18. --- .github/workflows/test-integrations.yml | 45 +++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml index 3615e15548..33d1ccc9e2 100644 --- a/.github/workflows/test-integrations.yml +++ b/.github/workflows/test-integrations.yml @@ -237,11 +237,47 @@ jobs: name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} path: ${{ env.TEST_RESULTS_DIR }} + generate-compatibility-job-matrices: + needs: [setup] + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + name: Generate Compatibility Job Matrices + outputs: + compatibility-matrix: ${{ steps.set-matrix.outputs.compatibility-matrix }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - name: Generate Compatibility Job Matrix + id: set-matrix + env: + TOTAL_RUNNERS: 6 + JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' + run: | + cd ./test/integration/consul-container + NUM_RUNNERS=$TOTAL_RUNNERS + NUM_DIRS=$(find ./test -mindepth 1 -maxdepth 2 -type d | wc -l) + + if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then + echo "TOTAL_RUNNERS is larger than the number of tests/packages to split." + NUM_RUNNERS=$((NUM_DIRS-1)) + fi + # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS. + NUM_RUNNERS=$((NUM_RUNNERS-1)) + { + echo -n "compatibility-matrix=" + find ./test -maxdepth 2 -type d -print0 | xargs -0 -n 1 \ + | grep -v util | grep -v upgrade \ + | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \ + | jq --compact-output 'map(join(" "))' + } >> "$GITHUB_OUTPUT" compatibility-integration-test: runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} needs: - setup - dev-build + - generate-compatibility-job-matrices + strategy: + fail-fast: false + matrix: + test-cases: ${{ fromJSON(needs.generate-compatibility-job-matrices.outputs.compatibility-matrix) }} steps: - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 @@ -271,9 +307,12 @@ jobs: mkdir -p "/tmp/test-results" cd ./test/integration/consul-container docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version + echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" + # shellcheck disable=SC2001 + sed 's, ,\n,g' <<< "${{ matrix.test-cases }}" go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \ --raw-command \ - --format=standard-verbose \ + --format=short-verbose \ --debug \ --rerun-fails=3 \ -- \ @@ -282,7 +321,7 @@ jobs: -tags "${{ env.GOTAGS }}" \ -timeout=30m \ -json \ - `go list ./... | grep -v upgrade` \ + ${{ matrix.test-cases }} \ --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \ --target-version local \ --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \ @@ -383,6 +422,7 @@ jobs: --raw-command \ --format=short-verbose \ --debug \ + --rerun-fails=3 \ --packages="./..." \ -- \ go test \ @@ -416,6 +456,7 @@ jobs: - vault-integration-test - generate-envoy-job-matrices - envoy-integration-test + - generate-compatibility-job-matrices - compatibility-integration-test - generate-upgrade-job-matrices - upgrade-integration-test