From 0c03802bff8484660f3d16c62a8d03d4dbe96899 Mon Sep 17 00:00:00 2001 From: Frank Lee Date: Mon, 6 Feb 2023 16:29:04 +0800 Subject: [PATCH] [workflow] hooked pypi release with lark (#2596) --- .github/workflows/auto_example_check.yml | 143 ------------------ .github/workflows/auto_release_bdist.yml | 70 --------- ...draft_github_release_post_after_merge.yml} | 0 .../workflows/release_nightly_on_schedule.yml | 36 +++++ .../workflows/release_pypi_after_merge.yml | 36 +++++ 5 files changed, 72 insertions(+), 213 deletions(-) delete mode 100644 .github/workflows/auto_example_check.yml delete mode 100644 .github/workflows/auto_release_bdist.yml rename .github/workflows/{draft_github_release_post.yml => draft_github_release_post_after_merge.yml} (100%) diff --git a/.github/workflows/auto_example_check.yml b/.github/workflows/auto_example_check.yml deleted file mode 100644 index df413f646..000000000 --- a/.github/workflows/auto_example_check.yml +++ /dev/null @@ -1,143 +0,0 @@ -name: Test Example -on: - pull_request: - # any change in the examples folder will trigger check for the corresponding example. - paths: - - 'examples/**' - # run at 00:00 of every Sunday(singapore time) so here is UTC time Saturday 16:00 - schedule: - - cron: '0 16 * * 6' - -jobs: - # This is for changed example files detect and output a matrix containing all the corresponding directory name. - detect-changed-example: - if: | - github.event.pull_request.draft == false && - github.base_ref == 'main' && - github.event.pull_request.base.repo.full_name == 'hpcaitech/ColossalAI' && github.event_name == 'pull_request' - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.setup-matrix.outputs.matrix }} - anyChanged: ${{ steps.setup-matrix.outputs.anyChanged }} - name: Detect changed example files - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - - name: Locate base commit - id: locate-base-sha - run: | - curBranch=$(git rev-parse --abbrev-ref HEAD) - commonCommit=$(git merge-base origin/main $curBranch) - echo $commonCommit - echo "baseSHA=$commonCommit" >> $GITHUB_OUTPUT - - - name: Get all changed example files - id: changed-files - uses: tj-actions/changed-files@v35 - with: - base_sha: ${{ steps.locate-base-sha.outputs.baseSHA }} - - - name: setup matrix - id: setup-matrix - run: | - changedFileName="" - for file in ${{ steps.changed-files.outputs.all_changed_files }}; do - changedFileName="${file}:${changedFileName}" - done - echo "$changedFileName was changed" - res=`python .github/workflows/scripts/example_checks/detect_changed_example.py --fileNameList $changedFileName` - echo "All changed examples are $res" - - if [ "$res" = "[]" ]; then - echo "anyChanged=false" >> $GITHUB_OUTPUT - echo "matrix=null" >> $GITHUB_OUTPUT - else - dirs=$( IFS=',' ; echo "${res[*]}" ) - echo "anyChanged=true" >> $GITHUB_OUTPUT - echo "matrix={\"directory\":$(echo "$dirs")}" >> $GITHUB_OUTPUT - fi - - # If no file is changed, it will prompt an error and shows the matrix do not have value. - check-changed-example: - # Add this condition to avoid executing this job if the trigger event is workflow_dispatch. - if: | - github.event.pull_request.draft == false && - github.base_ref == 'main' && - github.event.pull_request.base.repo.full_name == 'hpcaitech/ColossalAI' && github.event_name == 'pull_request' && - needs.detect-changed-example.outputs.anyChanged == 'true' - name: Test the changed example - needs: detect-changed-example - runs-on: [self-hosted, gpu] - strategy: - matrix: ${{fromJson(needs.detect-changed-example.outputs.matrix)}} - container: - image: hpcaitech/pytorch-cuda:1.12.0-11.3.0 - options: --gpus all --rm -v /data/scratch/examples-data:/data/ - timeout-minutes: 10 - steps: - - uses: actions/checkout@v3 - - - name: Install Colossal-AI - run: | - pip install -v . - - - name: Test the example - run: | - example_dir=${{ matrix.directory }} - cd "${PWD}/examples/${example_dir}" - bash test_ci.sh - env: - NCCL_SHM_DISABLE: 1 - - # This is for all files' weekly check. Specifically, this job is to find all the directories. - matrix_preparation: - if: | - github.repository == 'hpcaitech/ColossalAI' && - github.event_name == 'schedule' - name: Prepare matrix for weekly check - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.setup-matrix.outputs.matrix }} - steps: - - name: 📚 Checkout - uses: actions/checkout@v3 - - - name: setup matrix - id: setup-matrix - run: | - res=`python .github/workflows/scripts/example_checks/check_example_weekly.py` - all_loc=$( IFS=',' ; echo "${res[*]}" ) - echo "Found the examples: $all_loc" - echo "matrix={\"directory\":$(echo "$all_loc")}" >> $GITHUB_OUTPUT - - weekly_check: - if: | - github.repository == 'hpcaitech/ColossalAI' && - github.event_name == 'schedule' - name: Weekly check all examples - needs: matrix_preparation - runs-on: [self-hosted, gpu] - strategy: - matrix: ${{fromJson(needs.matrix_preparation.outputs.matrix)}} - container: - image: hpcaitech/pytorch-cuda:1.12.0-11.3.0 - timeout-minutes: 10 - steps: - - name: 📚 Checkout - uses: actions/checkout@v3 - - - name: Install Colossal-AI - run: | - pip install -v . - - - name: Traverse all files - run: | - example_dir=${{ matrix.diretory }} - echo "Testing ${example_dir} now" - cd "${PWD}/examples/${example_dir}" - bash test_ci.sh - env: - NCCL_SHM_DISABLE: 1 diff --git a/.github/workflows/auto_release_bdist.yml b/.github/workflows/auto_release_bdist.yml deleted file mode 100644 index 56a3036f8..000000000 --- a/.github/workflows/auto_release_bdist.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: Auto Release bdist wheel - -on: - workflow_dispatch: - pull_request: - paths: - - 'version.txt' - types: - - closed - -jobs: - matrix_preparation: - name: Prepare Container List - if: ( github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true ) && github.repository == 'hpcaitech/ColossalAI' - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v3 - - id: set-matrix - run: | - bdist=$(cat .bdist.json | tr '\n' ' ') - echo "matrix=${bdist}" >> $GITHUB_OUTPUT - - build: - name: Release bdist wheels - needs: matrix_preparation - runs-on: [self-hosted, gpu] - strategy: - fail-fast: false - matrix: ${{fromJson(needs.matrix_preparation.outputs.matrix)}} - container: - image: ${{ matrix.build.cuda_image }} - options: --gpus all --rm - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - # cub is for cuda 10.2 - - name: Copy scripts - run: | - cp -r ./.github/workflows/scripts/* ./ - - # link the cache diretories to current path - ln -s /github/home/conda_pkgs ./conda_pkgs - ln -s /github/home/pip_wheels ./pip_wheels - - # set the conda package path - echo "pkgs_dirs:\n - $PWD/conda_pkgs" > ~/.condarc - - # set safe directory - git config --global --add safe.directory /__w/ColossalAI/ColossalAI - - # get cub package for cuda 10.2 - wget https://github.com/NVIDIA/cub/archive/refs/tags/1.8.0.zip - unzip 1.8.0.zip - - name: Build bdist wheel - run: | - pip install beautifulsoup4 requests packaging - python ./build_colossalai_wheel.py --torch_version $TORCH_VERSIONS - env: - TORCH_VERSIONS: ${{ matrix.build.torch_version }} - - name: 🚀 Deploy - uses: garygrossgarten/github-action-scp@release - with: - local: all_dist - remote: ${{ secrets.PRIVATE_PYPI_DIR }} - host: ${{ secrets.PRIVATE_PYPI_HOST }} - username: ${{ secrets.PRIVATE_PYPI_USER }} - password: ${{ secrets.PRIVATE_PYPI_PASSWD }} diff --git a/.github/workflows/draft_github_release_post.yml b/.github/workflows/draft_github_release_post_after_merge.yml similarity index 100% rename from .github/workflows/draft_github_release_post.yml rename to .github/workflows/draft_github_release_post_after_merge.yml diff --git a/.github/workflows/release_nightly_on_schedule.yml b/.github/workflows/release_nightly_on_schedule.yml index 8aa48b8ed..aab42e1d7 100644 --- a/.github/workflows/release_nightly_on_schedule.yml +++ b/.github/workflows/release_nightly_on_schedule.yml @@ -23,7 +23,43 @@ jobs: # publish to PyPI if executed on the main branch - name: Publish package to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + id: publish with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} verbose: true + + notify: + name: Notify Lark via webhook + needs: release + runs-on: ubuntu-latest + if: ${{ always() }} + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + with: + python-version: '3.8.14' + + - name: Install requests + run: pip install requests + + - name: Notify Lark + id: message-preparation + run: | + url=$SERVER_URL/$REPO/actions/runs/$RUN_ID + + if [ $STATUS == 'success' ] + then + msg="The Colossal-AI nightly version has been successfully released to PyPI." + else + msg="Failed to release Colossal-AI nightly version to PyPI, please visit $url for details." + fi + echo $msg + python .github/workflows/scripts/send_message_to_lark.py -m "$msg" -u $WEBHOOK_URL + env: + SERVER_URL: ${{github.server_url }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + WEBHOOK_URL: ${{ secrets.LARK_NOTIFICATION_WEBHOOK_URL }} + STATUS: ${{ steps.publish.outcome }} diff --git a/.github/workflows/release_pypi_after_merge.yml b/.github/workflows/release_pypi_after_merge.yml index 797cd69f3..7fa4bdd03 100644 --- a/.github/workflows/release_pypi_after_merge.yml +++ b/.github/workflows/release_pypi_after_merge.yml @@ -25,8 +25,44 @@ jobs: # publish to PyPI if executed on the main branch - name: Publish package to PyPI + id: publish uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} verbose: true + + notify: + name: Notify Lark via webhook + needs: release + runs-on: ubuntu-latest + if: ${{ always() }} + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + with: + python-version: '3.8.14' + + - name: Install requests + run: pip install requests + + - name: Notify Lark + id: message-preparation + run: | + url=$SERVER_URL/$REPO/actions/runs/$RUN_ID + + if [ $STATUS == 'success' ] + then + msg="The Colossal-AI latest version has been successfully released to PyPI." + else + msg="Failed to release Colossal-AI to PyPI, please visit $url for details." + fi + echo $msg + python .github/workflows/scripts/send_message_to_lark.py -m "$msg" -u $WEBHOOK_URL + env: + SERVER_URL: ${{github.server_url }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + WEBHOOK_URL: ${{ secrets.LARK_NOTIFICATION_WEBHOOK_URL }} + STATUS: ${{ steps.publish.outcome }}