test(ci_scripts): add timeout settings and clean work after the slurm job (#185)

* restore pr test on develop branch

* add mask

* add post action to cancel slurm job

* remove readonly attribute on job log

* add debug info

* debug job log

* try stdin

* use stdin

* set default value avoid error

* try setting readonly on job log

* performance echo

* remove debug info

* use squeue to check slurm job status

* restore the lossed parm

* litmit retry times

* use exclusive to avoid port already in use

* optimize loop body

* remove partition

* add {} for variables

* set env variable for slurm partition

---------

Co-authored-by: qa-caif-cicd <qa-caif-cicd@pjlab.org.cn>

.github/workflows/demo_in_readme.yaml

@@ -3,14 +3,21 @@ on:
   pull_request:
     branches:
       - "main"
+      - "develop"
     paths-ignore:
       - "docs/**"
       - "**.md"
+env:
+  WORKSPACE_PREFIX: $(echo $GITHUB_WORKSPACE |cut -d '/' -f 1-4)
+  SLURM_PARTITION: llm
 
 jobs:
   check-requirements:
     runs-on: [lmtest]
     steps:
+    - name: mask env
+      run: |
+        echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
     - uses: actions/checkout@v3
       with:
          fetch-depth: 2
@@ -32,12 +39,15 @@ jobs:
     needs: check-requirements
     runs-on: [lmtest]
     steps:
+    - name: mask env 
+      run: |
+        echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
     - uses: actions/checkout@v3
 
     - name: raw-chinese-data
       run: |
         source activate internlm-env-test
-        sh ./ci_scripts/data/tokenizer_chinese.sh
+        sh ./ci_scripts/data/tokenizer_chinese.sh ${GITHUB_RUN_ID}-${GITHUB_JOB}
 
     - name: alpaca-data
       run: |
@@ -48,19 +58,23 @@ jobs:
     if: ${{ always() }}
     needs: check-requirements
     runs-on: [lmtest]
+    timeout-minutes: 30
     steps:
+    - name: mask env 
+      run: |
+        echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
     - uses: actions/checkout@v3
 
     - name: slurm-train
       run: |
         source activate internlm-env-test
-        sh ./ci_scripts/train/slurm_train.sh
+        sh ./ci_scripts/train/slurm_train.sh ${GITHUB_RUN_ID}-${GITHUB_JOB}
         rm -rf $GITHUB_WORKSPACE/llm_ckpts
 
     - name: torchrun-train
       run: |
         source activate internlm-env-test
-        sh ./ci_scripts/train/torchrun.sh
+        sh ./ci_scripts/train/torchrun.sh ${GITHUB_RUN_ID}-${GITHUB_JOB}
         rm -rf $GITHUB_WORKSPACE/llm_ckpts
 
   convert-model-then-load:
@@ -68,6 +82,9 @@ jobs:
     needs: check-requirements
     runs-on: [lmtest]
     steps:
+    - name: mask env
+      run: |
+        echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
     - uses: actions/checkout@v3
 
     - name: convert-model-then-load
@@ -76,7 +93,7 @@ jobs:
         export PYTHONPATH=$PWD:$PYTHONPATH
         sh ./ci_scripts/model/convert_to_hf.sh 
         cd ./hf_ckpt
-        srun -p llm python ../ci_scripts/model/loaded_as_transformer.py
+        srun -p ${SLURM_PARTITION} --job-name=${GITHUB_RUN_ID}-${GITHUB_JOB} --gpus-per-task=2 python ../ci_scripts/model/loaded_as_transformer.py
         cd ..
         rm -rf $GITHUB_WORKSPACE/hf_ckpt
   
@@ -85,9 +102,12 @@ jobs:
     needs: check-requirements
     runs-on: [lmtest]
     steps:
+    - name: mask env 
+      run: |
+        echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
     - uses: actions/checkout@v3
 
     - name: chat-model-in-hf
       run: |
         source activate internlm-env-test
-        srun -p llm python ./ci_scripts/model/demo_load_7B_chat_model.py
+        srun -p ${SLURM_PARTITION} --job-name=${GITHUB_RUN_ID}-${GITHUB_JOB} --gpus-per-task=2 python ./ci_scripts/model/demo_load_7B_chat_model.py

.github/workflows/lint_check.yaml

@@ -10,8 +10,7 @@ on:
 jobs:
   # lint check can be auto-executed by the workflow
   lint-check:
-    runs-on: [internlm]
+    runs-on: ubuntu-latest
-    if: github.repository == 'InternLM/InternLM'
     steps:
     - uses: actions/checkout@v3
 

ci_scripts/common/post_action.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -x
+
+retry_times=3
+for ((i=1;i<=$retry_times;i++));do
+    jobid=$(squeue -o "%A %j" -u $USER | grep ${GITHUB_RUN_ID}-${GITHUB_JOB} | awk '{print $1}')
+    if [[ -n "$jobid" ]];then
+        echo "The job $jobid will be canceled."
+        scancel $jobid
+        sleep 0.5
+    else
+        echo "There are no more jobs that need to be canceled."
+        break
+    fi
+done
+
+if [[ $i -gt $retry_times ]];then
+    echo "There have been tried $retry_times times. Please contact user $USER to confirm the job status."
+fi
+
+exit 0

ci_scripts/data/tokenizer_chinese.sh

@@ -22,7 +22,7 @@ if [[ ${num} -gt 0 ]]; then
     fi
 fi
 
-srun -p llm python tools/tokenizer.py --text_input_path ${DATA} --bin_output_path ${RESULT}
+srun -p ${SLURM_PARTITION} --job-name=$1 --gpus-per-task=1 python tools/tokenizer.py --text_input_path ${DATA} --bin_output_path ${RESULT}
 [[ $? -ne 0 ]] && { echo "test tokenizer.py failed.";  exit_code=$(($exit_code + 1)); }
 
 file_list=($RESULT $RESULT_META)

ci_scripts/train/slurm_train.sh

@@ -19,7 +19,7 @@ if [[ -d ${CKPTS20_PATH} ]]; then
     fi
 fi
 
-srun -p llm --quotatype=spot -n 8 --ntasks-per-node=8 --gpus-per-task=1 python train.py --config ./ci_scripts/train/ci_7B_sft.py
+srun -p ${SLURM_PARTITION} --exclusive --job-name=$1 -n 8 --ntasks-per-node=8 --gpus-per-task=1 python train.py --config ./ci_scripts/train/ci_7B_sft.py
 [[ $? -ne 0 ]] && { echo "test slurm training failed.";  exit_code=$(($exit_code + 1)); }
 
 num=$(num_files "${CKPTS20_OUTPUT}")

ci_scripts/train/torchrun.sh

@@ -19,7 +19,7 @@ if [[ -d ${CKPTS20_PATH} ]]; then
     fi
 fi
 
-srun -p llm -N 1 torchrun --nnodes=1 --nproc_per_node=8 --master_port=29501 train.py --config ./ci_scripts/train/ci_7B_sft.py --launcher torch
+srun -p ${SLURM_PARTITION} --exclusive --job-name=$1 -N 1 torchrun --nnodes=1 --nproc_per_node=8 --master_port=29501 train.py --config ./ci_scripts/train/ci_7B_sft.py --launcher torch
 [[ $? -ne 0 ]] && { echo "test torch training failed.";  exit_code=$(($exit_code + 1)); }
 
 num=$(num_files "${CKPTS_OUTPUT}")