From fa2dda7bb3d85d651d5e8ebfcd79e4d946c80941 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Fri, 7 Apr 2023 15:06:38 -0400 Subject: [PATCH] Backport of ci: increase ENT runner size for xl to match OSS. have build-distros use xl to match CircleCI into release/1.15.x (#16924) * no-op commit due to failed cherry-picking * increase ENT runner size for xl to match OSS. have guild-distros use xl to match CircleCI (#16920) * ci: Add success jobs. make go-test-enterprise conditional. build-distros and go-tests trigger on push to main and release branches (#16905) * Add go-tests-success job and make go-test-enterprise conditional * fixing lint-32bit reference * fixing reference to -go-test-troubleshoot * add all jobs that fan out. * fixing success job to need set up * add echo to success job * adding success jobs to build-artifacts, build-distros, and frontend. * changing the name of the job in verify ci to be consistent with other workflows * enable go-tests, build-distros, and verify-ci to run on merge to main and release branches because they currently do not with just the pull_request trigger --------- Co-authored-by: temp Co-authored-by: John Murret --- .github/scripts/get_runner_classes.sh | 3 +- .github/workflows/build-artifacts.yml | 123 ++++++++++++++++++++++++++ .github/workflows/build-distros.yml | 41 ++++++++- .github/workflows/frontend.yml | 28 ++++++ .github/workflows/go-tests.yml | 5 ++ .github/workflows/verify-ci.yml | 12 ++- 6 files changed, 205 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/build-artifacts.yml diff --git a/.github/scripts/get_runner_classes.sh b/.github/scripts/get_runner_classes.sh index 4834e9e929..b722ba7687 100755 --- a/.github/scripts/get_runner_classes.sh +++ b/.github/scripts/get_runner_classes.sh @@ -10,7 +10,8 @@ case "$GITHUB_REPOSITORY" in echo "compute-small=['self-hosted', 'linux', 'small']" >> "$GITHUB_OUTPUT" echo "compute-medium=['self-hosted', 'linux', 'medium']" >> "$GITHUB_OUTPUT" echo "compute-large=['self-hosted', 'linux', 'large']" >> "$GITHUB_OUTPUT" - echo "compute-xl=['self-hosted', 'ondemand', 'linux', 'type=m5.2xlarge']" >> "$GITHUB_OUTPUT" + # m5d.8xlarge is equivalent to our xl custom runner in OSS + echo "compute-xl=['self-hosted', 'ondemand', 'linux', 'type=m5d.8xlarge']" >> "$GITHUB_OUTPUT" ;; *) # shellcheck disable=SC2129 diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 0000000000..57d23d52f5 --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,123 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# This workflow builds a dev binary and distributes a Docker image on every push to the main branch. +name: build-artifacts + +on: + push: + branches: + - main + +permissions: + contents: read + +env: + GOPRIVATE: github.com/hashicorp + +jobs: + setup: + name: Setup + runs-on: ubuntu-latest + outputs: + compute-small: ${{ steps.setup-outputs.outputs.compute-small }} + compute-medium: ${{ steps.setup-outputs.outputs.compute-medium }} + compute-large: ${{ steps.setup-outputs.outputs.compute-large }} + compute-xl: ${{ steps.setup-outputs.outputs.compute-xl }} + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + - id: setup-outputs + name: Setup outputs + run: ./.github/scripts/get_runner_classes.sh + + dev-build-push: + needs: setup + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + steps: + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/dockerhub username | DOCKERHUB_USERNAME; + kv/data/github/${{ github.repository }}/dockerhub token | DOCKERHUB_TOKEN; + + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 + + # NOTE: ENT specific step as we need to set elevated GitHub permissions. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # pin@v3.5.0 + with: + go-version-file: 'go.mod' + + - name: Build dev binary + run: make dev + + - name: Set env vars + run: | + echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + echo "GITHUB_BUILD_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # pin@v2.4.1 + + # NOTE: conditional specific logic as we store secrets in Vault in ENT and use GHA secrets in OSS. + - name: Login to Docker Hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # pin@v2.1.0 + with: + username: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_USERNAME || secrets.DOCKERHUB_USERNAME }} + password: ${{ endsWith(github.repository, '-enterprise') && steps.secrets.outputs.DOCKERHUB_TOKEN || secrets.DOCKERHUB_TOKEN }} + + - name: Docker build and push + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # pin@v4.0.0 + with: + context: ./bin + file: ./build-support/docker/Consul-Dev.dockerfile + labels: COMMIT_SHA=${{ github.sha }},GITHUB_BUILD_URL=${{ env.GITHUB_BUILD_URL }} + push: true + tags: | + hashicorpdev/${{ github.event.repository.name }}:${{ env.SHORT_SHA }} + hashicorpdev/${{ github.event.repository.name }}:latest + + # This is job is required for branch protection as a required gihub check + # because GitHub actions show up as checks at the job level and not the + # workflow level. This is currently a feature request: + # https://github.com/orgs/community/discussions/12395 + # + # This job must: + # - be placed after the fanout of a workflow so that everything fans back in + # to this job. + # - "need" any job that is part of the fan out / fan in + # - implement the if logic because we have conditional jobs + # (go-test-enteprise) that this job needs and this would potentially get + # skipped if a previous job got skipped. So we use the if clause to make + # sure it does not get skipped. + + build-artifacts-success: + needs: + - setup + - dev-build-push + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + if: | + (always() && ! cancelled()) && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') + steps: + - run: echo "build-artifacts succeeded" diff --git a/.github/workflows/build-distros.yml b/.github/workflows/build-distros.yml index 8b6c6d9930..c8036a7505 100644 --- a/.github/workflows/build-distros.yml +++ b/.github/workflows/build-distros.yml @@ -2,7 +2,13 @@ # It is aimed at checking new commits don't introduce any breaking build changes. name: build-distros -on: [pull_request] +on: + pull_request: + push: + branches: + # Push events on the main branch + - main + - release/** permissions: contents: read @@ -38,7 +44,7 @@ jobs: - check-go-mod env: XC_OS: "freebsd linux windows" - runs-on: ${{ fromJSON(needs.setup.outputs.compute-medium) }} + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 @@ -62,7 +68,7 @@ jobs: - check-go-mod env: XC_OS: "darwin freebsd linux solaris windows" - runs-on: ${{ fromJSON(needs.setup.outputs.compute-medium) }} + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # pin@v3.3.0 @@ -106,3 +112,32 @@ jobs: - run: CC=arm-linux-gnueabi-gcc GOARCH=arm GOARM=5 go build - run: CC=arm-linux-gnueabihf-gcc GOARCH=arm GOARM=6 go build - run: CC=aarch64-linux-gnu-gcc GOARCH=arm64 go build + + # This is job is required for branch protection as a required gihub check + # because GitHub actions show up as checks at the job level and not the + # workflow level. This is currently a feature request: + # https://github.com/orgs/community/discussions/12395 + # + # This job must: + # - be placed after the fanout of a workflow so that everything fans back in + # to this job. + # - "need" any job that is part of the fan out / fan in + # - implement the if logic because we have conditional jobs + # (go-test-enteprise) that this job needs and this would potentially get + # skipped if a previous job got skipped. So we use the if clause to make + # sure it does not get skipped. + + build-distros-success: + needs: + - setup + - check-go-mod + - build-386 + - build-amd64 + - build-arm + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + if: | + (always() && ! cancelled()) && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') + steps: + - run: echo "build-distros succeeded" diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 094d86ad55..b6451a378c 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -104,3 +104,31 @@ jobs: - working-directory: ui/packages/consul-ui run: make test-coverage-ci + + # This is job is required for branch protection as a required gihub check + # because GitHub actions show up as checks at the job level and not the + # workflow level. This is currently a feature request: + # https://github.com/orgs/community/discussions/12395 + # + # This job must: + # - be placed after the fanout of a workflow so that everything fans back in + # to this job. + # - "need" any job that is part of the fan out / fan in + # - implement the if logic because we have conditional jobs + # (go-test-enteprise) that this job needs and this would potentially get + # skipped if a previous job got skipped. So we use the if clause to make + # sure it does not get skipped. + + frontend-success: + needs: + - setup + - workspace-tests + - node-tests + - ember-build-test + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + if: | + (always() && ! cancelled()) && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') + steps: + - run: echo "frontend succeeded" diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 83c37d3b32..4d8f7b04bc 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -10,6 +10,11 @@ on: - 'backport/docs/**' - 'backport/ui/**' - 'backport/mktg-**' + push: + branches: + # Push events on the main branch + - main + - release/** permissions: contents: read diff --git a/.github/workflows/verify-ci.yml b/.github/workflows/verify-ci.yml index 16fb4db25b..4bd4536add 100644 --- a/.github/workflows/verify-ci.yml +++ b/.github/workflows/verify-ci.yml @@ -8,10 +8,16 @@ name: verify-ci permissions: contents: read -on: [pull_request] +on: + pull_request: + push: + branches: + # Push events on the main branch + - main + - release/** jobs: - noop: + verify-ci-success: runs-on: ubuntu-latest steps: - - run: echo "ok" + - run: echo "verify-ci succeeded"