[workflow] hooked pypi release with lark (#2596)

.github/workflows/auto_example_check.yml

@@ -1,143 +0,0 @@
-name: Test Example
-on:
-  pull_request:
-    # any change in the examples folder will trigger check for the corresponding example.
-    paths:
-      - 'examples/**'
-  # run at 00:00 of every Sunday(singapore time) so here is UTC time Saturday 16:00
-  schedule:
-    - cron:  '0 16 * * 6'
-
-jobs:
-  # This is for changed example files detect and output a matrix containing all the corresponding directory name.
-  detect-changed-example:
-    if: |
-        github.event.pull_request.draft == false &&
-        github.base_ref == 'main' &&
-        github.event.pull_request.base.repo.full_name == 'hpcaitech/ColossalAI' && github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.setup-matrix.outputs.matrix }}
-      anyChanged: ${{ steps.setup-matrix.outputs.anyChanged }}
-    name: Detect changed example files
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      - name: Locate base commit
-        id: locate-base-sha
-        run: |
-            curBranch=$(git rev-parse --abbrev-ref HEAD)
-            commonCommit=$(git merge-base origin/main $curBranch)
-            echo $commonCommit
-            echo "baseSHA=$commonCommit" >> $GITHUB_OUTPUT
-
-      - name: Get all changed example files
-        id: changed-files
-        uses: tj-actions/changed-files@v35
-        with:
-          base_sha: ${{ steps.locate-base-sha.outputs.baseSHA }}
-
-      - name: setup matrix
-        id: setup-matrix
-        run: |
-          changedFileName=""
-          for file in ${{ steps.changed-files.outputs.all_changed_files  }}; do
-            changedFileName="${file}:${changedFileName}"
-          done
-          echo "$changedFileName was changed"
-          res=`python .github/workflows/scripts/example_checks/detect_changed_example.py --fileNameList $changedFileName`
-          echo "All changed examples are $res"
-
-          if [ "$res" = "[]" ]; then
-            echo "anyChanged=false" >> $GITHUB_OUTPUT
-            echo "matrix=null" >> $GITHUB_OUTPUT
-          else
-            dirs=$( IFS=',' ; echo "${res[*]}" )
-            echo "anyChanged=true" >> $GITHUB_OUTPUT
-            echo "matrix={\"directory\":$(echo "$dirs")}" >> $GITHUB_OUTPUT
-          fi
-
-  # If no file is changed, it will prompt an error and shows the matrix do not have value.
-  check-changed-example:
-    # Add this condition to avoid executing this job if the trigger event is workflow_dispatch.
-    if: |
-        github.event.pull_request.draft == false &&
-        github.base_ref == 'main' &&
-        github.event.pull_request.base.repo.full_name == 'hpcaitech/ColossalAI' && github.event_name == 'pull_request' &&
-        needs.detect-changed-example.outputs.anyChanged == 'true'
-    name: Test the changed example
-    needs: detect-changed-example
-    runs-on: [self-hosted, gpu]
-    strategy:
-      matrix: ${{fromJson(needs.detect-changed-example.outputs.matrix)}}
-    container:
-      image: hpcaitech/pytorch-cuda:1.12.0-11.3.0
-      options: --gpus all --rm -v /data/scratch/examples-data:/data/
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Colossal-AI
-        run: |
-          pip install -v .
-
-      - name: Test the example
-        run: |
-          example_dir=${{ matrix.directory }}
-          cd "${PWD}/examples/${example_dir}"
-          bash test_ci.sh
-        env:
-          NCCL_SHM_DISABLE: 1
-
-  # This is for all files' weekly check. Specifically, this job is to find all the directories.
-  matrix_preparation:
-    if: |
-        github.repository == 'hpcaitech/ColossalAI' &&
-        github.event_name == 'schedule'
-    name: Prepare matrix for weekly check
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.setup-matrix.outputs.matrix }}
-    steps:
-    - name: 📚 Checkout
-      uses: actions/checkout@v3
-
-    - name: setup matrix
-      id: setup-matrix
-      run: |
-        res=`python .github/workflows/scripts/example_checks/check_example_weekly.py`
-        all_loc=$( IFS=',' ; echo "${res[*]}" )
-        echo "Found the examples: $all_loc"
-        echo "matrix={\"directory\":$(echo "$all_loc")}" >> $GITHUB_OUTPUT
-
-  weekly_check:
-    if: |
-        github.repository == 'hpcaitech/ColossalAI' &&
-        github.event_name == 'schedule'
-    name: Weekly check all examples
-    needs: matrix_preparation
-    runs-on: [self-hosted, gpu]
-    strategy:
-      matrix: ${{fromJson(needs.matrix_preparation.outputs.matrix)}}
-    container:
-      image: hpcaitech/pytorch-cuda:1.12.0-11.3.0
-    timeout-minutes: 10
-    steps:
-      - name: 📚 Checkout
-        uses: actions/checkout@v3
-
-      - name: Install Colossal-AI
-        run: |
-          pip install -v .
-
-      - name: Traverse all files
-        run: |
-          example_dir=${{ matrix.diretory }}
-          echo "Testing ${example_dir} now"
-          cd "${PWD}/examples/${example_dir}"
-          bash test_ci.sh
-        env:
-          NCCL_SHM_DISABLE: 1

.github/workflows/auto_release_bdist.yml

@@ -1,70 +0,0 @@
-name: Auto Release bdist wheel
-
-on:
-  workflow_dispatch:
-  pull_request:
-    paths:
-      - 'version.txt'
-    types:
-      - closed
-
-jobs:
-  matrix_preparation:
-    name: Prepare Container List
-    if: ( github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true ) && github.repository == 'hpcaitech/ColossalAI'
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-    steps:
-      - uses: actions/checkout@v3
-      - id: set-matrix
-        run: |
-          bdist=$(cat .bdist.json | tr '\n' ' ')
-          echo "matrix=${bdist}" >> $GITHUB_OUTPUT
-
-  build:
-    name: Release bdist wheels
-    needs: matrix_preparation
-    runs-on: [self-hosted, gpu]
-    strategy:
-      fail-fast: false
-      matrix: ${{fromJson(needs.matrix_preparation.outputs.matrix)}}
-    container:
-      image: ${{ matrix.build.cuda_image }}
-      options: --gpus all --rm
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-      # cub is for cuda 10.2
-      - name: Copy scripts
-        run: |
-          cp -r ./.github/workflows/scripts/* ./
-
-          # link the cache diretories to current path
-          ln -s /github/home/conda_pkgs ./conda_pkgs
-          ln -s /github/home/pip_wheels ./pip_wheels
-
-          # set the conda package path
-          echo "pkgs_dirs:\n  - $PWD/conda_pkgs" > ~/.condarc
-
-          # set safe directory
-          git config --global --add safe.directory /__w/ColossalAI/ColossalAI
-
-          # get cub package for cuda 10.2
-          wget https://github.com/NVIDIA/cub/archive/refs/tags/1.8.0.zip
-          unzip 1.8.0.zip
-      - name: Build bdist wheel
-        run: |
-          pip install beautifulsoup4 requests packaging
-          python ./build_colossalai_wheel.py --torch_version $TORCH_VERSIONS
-        env:
-          TORCH_VERSIONS: ${{ matrix.build.torch_version }}
-      - name: 🚀 Deploy
-        uses: garygrossgarten/github-action-scp@release
-        with:
-          local: all_dist
-          remote: ${{ secrets.PRIVATE_PYPI_DIR }}
-          host: ${{ secrets.PRIVATE_PYPI_HOST }}
-          username: ${{ secrets.PRIVATE_PYPI_USER }}
-          password: ${{ secrets.PRIVATE_PYPI_PASSWD }}

.github/workflows/release_nightly_on_schedule.yml

@@ -23,7 +23,43 @@ jobs:
     # publish to PyPI if executed on the main branch
     - name: Publish package to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
+      id: publish
       with:
         user: __token__
         password: ${{ secrets.PYPI_API_TOKEN }}
         verbose: true
+
+  notify:
+    name: Notify Lark via webhook
+    needs: release
+    runs-on: ubuntu-latest
+    if: ${{ always() }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.8.14'
+
+      - name: Install requests
+        run: pip install requests
+
+      - name: Notify Lark
+        id: message-preparation
+        run: |
+          url=$SERVER_URL/$REPO/actions/runs/$RUN_ID
+
+          if [ $STATUS == 'success' ]
+          then
+            msg="The Colossal-AI nightly version has been successfully released to PyPI."
+          else
+            msg="Failed to release Colossal-AI nightly version to PyPI, please visit $url for details."
+          fi
+          echo $msg
+          python .github/workflows/scripts/send_message_to_lark.py -m "$msg" -u $WEBHOOK_URL
+        env:
+          SERVER_URL: ${{github.server_url }}
+          REPO: ${{ github.repository }}
+          RUN_ID: ${{ github.run_id }}
+          WEBHOOK_URL: ${{ secrets.LARK_NOTIFICATION_WEBHOOK_URL }}
+          STATUS: ${{ steps.publish.outcome }}

.github/workflows/release_pypi_after_merge.yml

@@ -25,8 +25,44 @@ jobs:
 
     # publish to PyPI if executed on the main branch
     - name: Publish package to PyPI
+      id: publish
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         user: __token__
         password: ${{ secrets.PYPI_API_TOKEN }}
         verbose: true
+
+  notify:
+    name: Notify Lark via webhook
+    needs: release
+    runs-on: ubuntu-latest
+    if: ${{ always() }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.8.14'
+
+      - name: Install requests
+        run: pip install requests
+
+      - name: Notify Lark
+        id: message-preparation
+        run: |
+          url=$SERVER_URL/$REPO/actions/runs/$RUN_ID
+
+          if [ $STATUS == 'success' ]
+          then
+            msg="The Colossal-AI latest version has been successfully released to PyPI."
+          else
+            msg="Failed to release Colossal-AI to PyPI, please visit $url for details."
+          fi
+          echo $msg
+          python .github/workflows/scripts/send_message_to_lark.py -m "$msg" -u $WEBHOOK_URL
+        env:
+          SERVER_URL: ${{github.server_url }}
+          REPO: ${{ github.repository }}
+          RUN_ID: ${{ github.run_id }}
+          WEBHOOK_URL: ${{ secrets.LARK_NOTIFICATION_WEBHOOK_URL }}
+          STATUS: ${{ steps.publish.outcome }}