test(ci_scripts): add install requirements automaticlly,trigger event about lint check and other optimizations (#174)

* add pull_request in lint check

* use default variables in ci_scripts

* fix format

* check and install requirements automaticlly

* fix format

---------

Co-authored-by: qa-caif-cicd <qa-caif-cicd@pjlab.org.cn>

.github/workflows/demo_in_readme.yaml

@@ -3,13 +3,33 @@ on:
   pull_request:
     branches:
       - "main"
-      - "develop"
     paths-ignore:
       - "docs/**"
       - "**.md"
-     
+
 jobs:
+  check-requirements:
+    runs-on: [lmtest]
+    steps:
+    - uses: actions/checkout@v3
+      with:
+         fetch-depth: 2
+    - name: check-requirements
+      run: |
+        source activate internlm-env-test
+        changed_files=$(git diff --name-only -r HEAD^1 HEAD)
+        echo $changed_files
+        if [[ $changed_files =~ "runtime.txt" ]]; then
+          pip install -r requirements/runtime.txt
+        fi
+
+        if [[ $changed_files =~ "torch.txt"  ]]; then
+          pip install -r requirements/torch.txt
+        fi
+
   dataset-preparation:
+    if: ${{ always() }}
+    needs: check-requirements
     runs-on: [lmtest]
     steps:
     - uses: actions/checkout@v3
@@ -24,8 +44,9 @@ jobs:
         source activate internlm-env-test
         sh ./ci_scripts/data/tokenizer_alpaca.sh
 
-
   train:
+    if: ${{ always() }}
+    needs: check-requirements
     runs-on: [lmtest]
     steps:
     - uses: actions/checkout@v3
@@ -43,6 +64,8 @@ jobs:
         rm -rf $GITHUB_WORKSPACE/llm_ckpts
 
   convert-model-then-load:
+    if: ${{ always() }}
+    needs: check-requirements
     runs-on: [lmtest]
     steps:
     - uses: actions/checkout@v3
@@ -58,6 +81,8 @@ jobs:
         rm -rf $GITHUB_WORKSPACE/hf_ckpt
   
   load-chat-model-in-hf:
+    if: ${{ always() }}
+    needs: check-requirements
     runs-on: [lmtest]
     steps:
     - uses: actions/checkout@v3

.github/workflows/lint_check.yaml

@@ -1,6 +1,11 @@
 name: lint-check
 
-on: [push]
+on: 
+  push:
+  pull_request:
+    branches:
+      - "main"
+      - "develop"
 
 jobs:
   # lint check can be auto-executed by the workflow
@@ -35,4 +40,4 @@ jobs:
         pip install pylint==v2.17.2
         PYLINT_DISABLE_LIST="C0114,C0415,W0212,W0235,W0238,W0621,C0103,R1735,C2801,E0402,C0412,W0719,R1728,W1514,W0718,W0105,W0707,C0209,W0703,W1203"
         pylint --rcfile .pylintrc --disable=$PYLINT_DISABLE_LIST ./internlm/*
-        pylint --rcfile .pylintrc --disable=$PYLINT_DISABLE_LIST ./train.py
+        pylint --rcfile .pylintrc --disable=$PYLINT_DISABLE_LIST ./train.py

ci_scripts/common/variables.sh

@@ -1,4 +1,3 @@
 #!/bin/bash
 
-# TODO: variable defination should be in repo configures. 
-readonly DATA_VOLUME=${DATA_VOLUME:-"/mnt/petrelfs/qa-caif-cicd/data"}
+readonly DATA_VOLUME=$(echo $GITHUB_WORKSPACE | cut -d '/' -f 1-4)/data