diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index e4041c3b50..705aa5f672 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -21,7 +21,6 @@ permissions: env: TEST_RESULTS: /tmp/test-results - GOTESTSUM_VERSION: 1.8.2 jobs: setup: @@ -186,37 +185,38 @@ jobs: secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} - dev-build-arm64: - # only run on enterprise because GHA does not have arm64 runners in OSS - if: ${{ endsWith(github.repository, '-enterprise') }} - needs: - - setup - uses: ./.github/workflows/reusable-dev-build.yml - with: - uploaded-binary-name: 'consul-bin-arm64' - runs-on: ${{ needs.setup.outputs.compute-xl }} - go-arch: "arm64" - repository-name: ${{ github.repository }} - secrets: - elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} - - go-test-arm64: - # only run on enterprise because GHA does not have arm64 runners in OSS - if: ${{ endsWith(github.repository, '-enterprise') }} - needs: - - setup - - dev-build-arm64 - uses: ./.github/workflows/reusable-unit-split.yml - with: - directory: . - uploaded-binary-name: 'consul-bin-arm64' - runner-count: 12 - runs-on: "['self-hosted', 'ondemand', 'os=macos-arm', 'arm64']" - go-test-flags: 'if ! [[ "$GITHUB_REF_NAME" =~ ^main$|^release/ ]]; then export GO_TEST_FLAGS="-short"; fi' - repository-name: ${{ github.repository }} - secrets: - elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} - consul-license: ${{secrets.CONSUL_LICENSE}} + # dev-build-arm64: + # # only run on enterprise because GHA does not have arm64 runners in OSS + # if: ${{ endsWith(github.repository, '-enterprise') }} + # needs: + # - setup + # uses: ./.github/workflows/reusable-dev-build.yml + # with: + # uploaded-binary-name: 'consul-bin-arm64' + # runs-on: ${{ needs.setup.outputs.compute-xl }} + # go-arch: "arm64" + # repository-name: ${{ github.repository }} + # secrets: + # elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + + # go-test-arm64: + # # only run on enterprise because GHA does not have arm64 runners in OSS + # if: ${{ endsWith(github.repository, '-enterprise') }} + # needs: + # - setup + # - dev-build-arm64 + # uses: ./.github/workflows/reusable-unit-split.yml + # with: + # directory: . + # uploaded-binary-name: 'consul-bin-arm64' + # runner-count: 12 + # runs-on: "['self-hosted', 'ondemand', 'os=macos-arm', 'arm64']" + # go-test-flags: 'if ! [[ "$GITHUB_REF_NAME" =~ ^main$|^release/ ]]; then export GO_TEST_FLAGS="-short"; fi' + # repository-name: ${{ github.repository }} + # secrets: + # elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + # consul-license: ${{secrets.CONSUL_LICENSE}} + # datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-oss: needs: @@ -229,9 +229,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-enterprise: if: ${{ endsWith(github.repository, '-enterprise') }} @@ -245,9 +249,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-race: needs: @@ -261,9 +269,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-32bit: needs: @@ -277,9 +289,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-envoyextensions: needs: @@ -291,9 +307,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-troubleshoot: needs: @@ -305,9 +325,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-api-1-19: needs: @@ -319,9 +343,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-api-1-20: needs: @@ -333,9 +361,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-sdk-1-19: needs: @@ -347,9 +379,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" go-test-sdk-1-20: needs: @@ -361,9 +397,13 @@ jobs: runs-on: ${{ needs.setup.outputs.compute-xl }} repository-name: ${{ github.repository }} go-tags: "${{ github.event.repository.name == 'consul-enterprise' && 'consulent consulprem consuldev' || '' }}" + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read secrets: elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} consul-license: ${{secrets.CONSUL_LICENSE}} + datadog-api-key: "${{ !endsWith(github.repository, '-enterprise') && secrets.DATADOG_API_KEY || '' }}" noop: runs-on: ubuntu-latest diff --git a/.github/workflows/reusable-unit-split.yml b/.github/workflows/reusable-unit-split.yml index 0131582b0b..f8088f4a80 100644 --- a/.github/workflows/reusable-unit-split.yml +++ b/.github/workflows/reusable-unit-split.yml @@ -42,6 +42,8 @@ on: required: true consul-license: required: true + datadog-api-key: + required: true env: TEST_RESULTS: /tmp/test-results GOTESTSUM_VERSION: 1.8.2 @@ -49,6 +51,7 @@ env: TOTAL_RUNNERS: ${{inputs.runner-count}} CONSUL_LICENSE: ${{secrets.consul-license}} GOTAGS: ${{ inputs.go-tags}} + DATADOG_API_KEY: ${{secrets.datadog-api-key}} jobs: set-test-package-matrix: @@ -128,6 +131,36 @@ jobs: -tags="${{env.GOTAGS}}" -p 2 \ ${GO_TEST_FLAGS-} \ -cover -coverprofile=coverage.txt + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + env: + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" ${{env.TEST_RESULTS}}/gotestsum-report.xml + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 with: name: test-results diff --git a/.github/workflows/reusable-unit.yml b/.github/workflows/reusable-unit.yml index fde340bd5e..5fd47339e1 100644 --- a/.github/workflows/reusable-unit.yml +++ b/.github/workflows/reusable-unit.yml @@ -38,12 +38,15 @@ on: required: true consul-license: required: true + datadog-api-key: + required: true env: TEST_RESULTS: /tmp/test-results GOTESTSUM_VERSION: 1.8.2 GOARCH: ${{inputs.go-arch}} CONSUL_LICENSE: ${{secrets.consul-license}} GOTAGS: ${{ inputs.go-tags}} + DATADOG_API_KEY: ${{secrets.datadog-api-key}} jobs: go-test: @@ -96,6 +99,36 @@ jobs: -tags="${{env.GOTAGS}}" \ ${GO_TEST_FLAGS-} \ -cover -coverprofile=coverage.txt + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + env: + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" ${{env.TEST_RESULTS}}/gotestsum-report.xml + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2 with: name: test-results diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml index d5b6c97a82..d6e53817d6 100644 --- a/.github/workflows/test-integrations.yml +++ b/.github/workflows/test-integrations.yml @@ -54,6 +54,9 @@ jobs: needs: - setup - dev-build + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read strategy: matrix: nomad-version: ['v1.3.3', 'v1.2.10', 'v1.1.16'] @@ -92,12 +95,45 @@ jobs: --packages="./command/agent/consul" \ --junitfile $TEST_RESULTS_DIR/results.xml -- \ -run TestConsul + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml vault-integration-test: runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} needs: - setup - dev-build + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read strategy: matrix: vault-version: ["1.13.1", "1.12.5", "1.11.9", "1.10.11"] @@ -139,6 +175,48 @@ jobs: --junitfile "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-agent.xml" \ -- -tags "${{ env.GOTAGS }}" -cover -coverprofile=coverage-agent.txt -run Vault ./agent + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" "${{ env.TEST_RESULTS_DIR }}/gotestsum-report.xml" + + - name: upload leader coverage + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-leader.xml" + + - name: upload agent coverage + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" "${{ env.TEST_RESULTS_DIR }}/gotestsum-report-agent.xml" + generate-envoy-job-matrices: needs: [setup] runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} @@ -181,6 +259,9 @@ jobs: - setup - generate-envoy-job-matrices - dev-build + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read strategy: fail-fast: false matrix: @@ -232,10 +313,35 @@ jobs: --packages=./test/integration/connect/envoy \ -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})" - - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 with: - name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} - path: ${{ env.TEST_RESULTS_DIR }} + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml generate-compatibility-job-matrices: needs: [setup] @@ -274,6 +380,9 @@ jobs: - setup - dev-build - generate-compatibility-job-matrices + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read strategy: fail-fast: false matrix: @@ -335,10 +444,35 @@ jobs: # tput complains if this isn't set to something. TERM: ansi - - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 with: - name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} - path: ${{ env.TEST_RESULTS_DIR }} + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/datadog apikey | DATADOG_API_KEY; + + - name: prepare datadog-ci + if: ${{ !endsWith(github.repository, '-enterprise') }} + run: | + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" + chmod +x /usr/local/bin/datadog-ci + + - name: upload coverage + env: + DATADOG_API_KEY: "${{ endsWith(github.repository, '-enterprise') && env.DATADOG_API_KEY || secrets.DATADOG_API_KEY }}" + DD_ENV: ci + run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml generate-upgrade-job-matrices: needs: [setup]