github
/
InternLM
mirror of https://github.com/InternLM/InternLM
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

use llm partition (#159) 

Co-authored-by: qa-caif-cicd <qa-caif-cicd@pjlab.org.cn>
pull/165/head
kkscilife 2023-08-01 17:49:01 +08:00 committed by GitHub
parent fbe6ef1da5
commit 7fbf85eac9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/demo_in_readme.yaml vendored
View File

@ -53,7 +53,7 @@ jobs:
export PYTHONPATH=$PWD:$PYTHONPATH export PYTHONPATH=$PWD:$PYTHONPATH
sh ./ci_scripts/model/convert_to_hf.sh sh ./ci_scripts/model/convert_to_hf.sh
cd ./hf_ckpt cd ./hf_ckpt
srun -p llm2 python ../ci_scripts/model/loaded_as_transformer.py srun -p llm python ../ci_scripts/model/loaded_as_transformer.py
cd .. cd ..
rm -rf $GITHUB_WORKSPACE/hf_ckpt rm -rf $GITHUB_WORKSPACE/hf_ckpt
@ -65,4 +65,4 @@ jobs:
- name: chat-model-in-hf - name: chat-model-in-hf
run: | run: |
source activate internlm-env-test source activate internlm-env-test
srun -p llm2 python ./ci_scripts/model/demo_load_7B_chat_model.py srun -p llm python ./ci_scripts/model/demo_load_7B_chat_model.py

2
ci_scripts/data/tokenizer_chinese.sh
View File

@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
rm -rf /mnt/petrelfs/qa-caif-cicd/data/lm_data/cn_data/result.* rm -rf /mnt/petrelfs/qa-caif-cicd/data/lm_data/cn_data/result.*
srun -p llm2 python tools/tokenizer.py --text_input_path /mnt/petrelfs/qa-caif-cicd/data/lm_data/cn_data/raw_data.txt --bin_output_path /mnt/petrelfs/qa-caif-cicd/data/lm_data/cn_data/result.bin srun -p llm python tools/tokenizer.py --text_input_path /mnt/petrelfs/qa-caif-cicd/data/lm_data/cn_data/raw_data.txt --bin_output_path /mnt/petrelfs/qa-caif-cicd/data/lm_data/cn_data/result.bin
file_one="/mnt/petrelfs/qa-caif-cicd/data/lm_data/cn_data/result.bin" file_one="/mnt/petrelfs/qa-caif-cicd/data/lm_data/cn_data/result.bin"
file_two="/mnt/petrelfs/qa-caif-cicd/data/lm_data/cn_data/result.bin.meta" file_two="/mnt/petrelfs/qa-caif-cicd/data/lm_data/cn_data/result.bin.meta"

2
ci_scripts/train/slurm_train.sh
View File

@ -2,7 +2,7 @@
rm -rf $GITHUB_WORKSPACE/llm_ckpts/20 rm -rf $GITHUB_WORKSPACE/llm_ckpts/20
srun -p llm2 --quotatype=spot -n 8 --ntasks-per-node=8 --gpus-per-task=1 python train.py --config ./ci_scripts/train/ci_7B_sft.py srun -p llm --quotatype=spot -n 8 --ntasks-per-node=8 --gpus-per-task=1 python train.py --config ./ci_scripts/train/ci_7B_sft.py
file_dir="$GITHUB_WORKSPACE/llm_ckpts/20/*.pt" file_dir="$GITHUB_WORKSPACE/llm_ckpts/20/*.pt"
source ./ci_scripts/common/basic_func.sh source ./ci_scripts/common/basic_func.sh

2
ci_scripts/train/torchrun.sh
View File

@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
rm -rf $GITHUB_WORKSPACE/llm_ckpts/20 rm -rf $GITHUB_WORKSPACE/llm_ckpts/20
srun -p llm2 -N 1 torchrun --nnodes=1 --nproc_per_node=8 --master_port=29501 train.py --config ./ci_scripts/train/ci_7B_sft.py --launcher "torch" srun -p llm -N 1 torchrun --nnodes=1 --nproc_per_node=8 --master_port=29501 train.py --config ./ci_scripts/train/ci_7B_sft.py --launcher "torch"
file_dir="$GITHUB_WORKSPACE/llm_ckpts/20/*.pt" file_dir="$GITHUB_WORKSPACE/llm_ckpts/20/*.pt"
source ./ci_scripts/common/basic_func.sh source ./ci_scripts/common/basic_func.sh