test(ci_scripts): add timeout settings and clean work after the slurm job (#185)

* restore pr test on develop branch

* add mask

* add post action to cancel slurm job

* remove readonly attribute on job log

* add debug info

* debug job log

* try stdin

* use stdin

* set default value avoid error

* try setting readonly on job log

* performance echo

* remove debug info

* use squeue to check slurm job status

* restore the lossed parm

* litmit retry times

* use exclusive to avoid port already in use

* optimize loop body

* remove partition

* add {} for variables

* set env variable for slurm partition

---------

Co-authored-by: qa-caif-cicd <qa-caif-cicd@pjlab.org.cn>
pull/203/head
kkscilife 2023-08-11 17:09:56 +08:00 committed by GitHub
parent 7cfea534e7
commit ccb06a98e4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 50 additions and 10 deletions

View File

@ -3,14 +3,21 @@ on:
pull_request:
branches:
- "main"
- "develop"
paths-ignore:
- "docs/**"
- "**.md"
env:
WORKSPACE_PREFIX: $(echo $GITHUB_WORKSPACE |cut -d '/' -f 1-4)
SLURM_PARTITION: llm
jobs:
check-requirements:
runs-on: [lmtest]
steps:
- name: mask env
run: |
echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
- uses: actions/checkout@v3
with:
fetch-depth: 2
@ -32,12 +39,15 @@ jobs:
needs: check-requirements
runs-on: [lmtest]
steps:
- name: mask env
run: |
echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
- uses: actions/checkout@v3
- name: raw-chinese-data
run: |
source activate internlm-env-test
sh ./ci_scripts/data/tokenizer_chinese.sh
sh ./ci_scripts/data/tokenizer_chinese.sh ${GITHUB_RUN_ID}-${GITHUB_JOB}
- name: alpaca-data
run: |
@ -48,19 +58,23 @@ jobs:
if: ${{ always() }}
needs: check-requirements
runs-on: [lmtest]
timeout-minutes: 30
steps:
- name: mask env
run: |
echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
- uses: actions/checkout@v3
- name: slurm-train
run: |
source activate internlm-env-test
sh ./ci_scripts/train/slurm_train.sh
sh ./ci_scripts/train/slurm_train.sh ${GITHUB_RUN_ID}-${GITHUB_JOB}
rm -rf $GITHUB_WORKSPACE/llm_ckpts
- name: torchrun-train
run: |
source activate internlm-env-test
sh ./ci_scripts/train/torchrun.sh
sh ./ci_scripts/train/torchrun.sh ${GITHUB_RUN_ID}-${GITHUB_JOB}
rm -rf $GITHUB_WORKSPACE/llm_ckpts
convert-model-then-load:
@ -68,6 +82,9 @@ jobs:
needs: check-requirements
runs-on: [lmtest]
steps:
- name: mask env
run: |
echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
- uses: actions/checkout@v3
- name: convert-model-then-load
@ -76,7 +93,7 @@ jobs:
export PYTHONPATH=$PWD:$PYTHONPATH
sh ./ci_scripts/model/convert_to_hf.sh
cd ./hf_ckpt
srun -p llm python ../ci_scripts/model/loaded_as_transformer.py
srun -p ${SLURM_PARTITION} --job-name=${GITHUB_RUN_ID}-${GITHUB_JOB} --gpus-per-task=2 python ../ci_scripts/model/loaded_as_transformer.py
cd ..
rm -rf $GITHUB_WORKSPACE/hf_ckpt
@ -85,9 +102,12 @@ jobs:
needs: check-requirements
runs-on: [lmtest]
steps:
- name: mask env
run: |
echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
- uses: actions/checkout@v3
- name: chat-model-in-hf
run: |
source activate internlm-env-test
srun -p llm python ./ci_scripts/model/demo_load_7B_chat_model.py
srun -p ${SLURM_PARTITION} --job-name=${GITHUB_RUN_ID}-${GITHUB_JOB} --gpus-per-task=2 python ./ci_scripts/model/demo_load_7B_chat_model.py

View File

@ -10,8 +10,7 @@ on:
jobs:
# lint check can be auto-executed by the workflow
lint-check:
runs-on: [internlm]
if: github.repository == 'InternLM/InternLM'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3

View File

@ -0,0 +1,21 @@
#!/bin/bash
set -x
retry_times=3
for ((i=1;i<=$retry_times;i++));do
jobid=$(squeue -o "%A %j" -u $USER | grep ${GITHUB_RUN_ID}-${GITHUB_JOB} | awk '{print $1}')
if [[ -n "$jobid" ]];then
echo "The job $jobid will be canceled."
scancel $jobid
sleep 0.5
else
echo "There are no more jobs that need to be canceled."
break
fi
done
if [[ $i -gt $retry_times ]];then
echo "There have been tried $retry_times times. Please contact user $USER to confirm the job status."
fi
exit 0

View File

@ -22,7 +22,7 @@ if [[ ${num} -gt 0 ]]; then
fi
fi
srun -p llm python tools/tokenizer.py --text_input_path ${DATA} --bin_output_path ${RESULT}
srun -p ${SLURM_PARTITION} --job-name=$1 --gpus-per-task=1 python tools/tokenizer.py --text_input_path ${DATA} --bin_output_path ${RESULT}
[[ $? -ne 0 ]] && { echo "test tokenizer.py failed."; exit_code=$(($exit_code + 1)); }
file_list=($RESULT $RESULT_META)

View File

@ -19,7 +19,7 @@ if [[ -d ${CKPTS20_PATH} ]]; then
fi
fi
srun -p llm --quotatype=spot -n 8 --ntasks-per-node=8 --gpus-per-task=1 python train.py --config ./ci_scripts/train/ci_7B_sft.py
srun -p ${SLURM_PARTITION} --exclusive --job-name=$1 -n 8 --ntasks-per-node=8 --gpus-per-task=1 python train.py --config ./ci_scripts/train/ci_7B_sft.py
[[ $? -ne 0 ]] && { echo "test slurm training failed."; exit_code=$(($exit_code + 1)); }
num=$(num_files "${CKPTS20_OUTPUT}")

View File

@ -19,7 +19,7 @@ if [[ -d ${CKPTS20_PATH} ]]; then
fi
fi
srun -p llm -N 1 torchrun --nnodes=1 --nproc_per_node=8 --master_port=29501 train.py --config ./ci_scripts/train/ci_7B_sft.py --launcher torch
srun -p ${SLURM_PARTITION} --exclusive --job-name=$1 -N 1 torchrun --nnodes=1 --nproc_per_node=8 --master_port=29501 train.py --config ./ci_scripts/train/ci_7B_sft.py --launcher torch
[[ $? -ne 0 ]] && { echo "test torch training failed."; exit_code=$(($exit_code + 1)); }
num=$(num_files "${CKPTS_OUTPUT}")