github
/
ColossalAI
mirror of https://github.com/hpcaitech/ColossalAI
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

remove local data path

pull/5759/head
YeAnbang 2024-05-29 02:25:19 +00:00
parent bf57b13dda
commit 45195ac53d
2 changed files with 14 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
applications/ColossalChat/examples/data_preparation_scripts/prepare_sft_dataset.sh
View File

@ -1,22 +1,13 @@
SAVE_DIR="/home/yeanbang/data/experiment/dataset/alpaca/test/Yi-1.5-6B" SAVE_DIR=""
rm -rf $SAVE_DIR/cache rm -rf $SAVE_DIR/cache
rm -rf $SAVE_DIR/jsonl rm -rf $SAVE_DIR/jsonl
rm -rf $SAVE_DIR/arrow rm -rf $SAVE_DIR/arrow
# python prepare_dataset.py --type sft \
# --data_input_dirs /home/yeanbang/data/experiment/dataset/sft_data/test/sft-data \
# --conversation_template_config /home/yeanbang/data/ColossalAI/applications/ColossalChat/config/conversation_template/THUDM_chatglm3-6b.json \
# --tokenizer_dir "/mnt/jfs-hdd/home/data/models/ChatGlm-6B" \
# --data_cache_dir $SAVE_DIR/cache \
# --data_jsonl_output_dir $SAVE_DIR/jsonl \
# --data_arrow_output_dir $SAVE_DIR/arrow \
python prepare_dataset.py --type sft \ python prepare_dataset.py --type sft \
--data_input_dirs /home/yeanbang/data/experiment/dataset/sft_data/test/sft-data \ --data_input_dirs "PATH/TO/SFT/DATA" \
--conversation_template_config /home/yeanbang/data/ColossalAI/applications/ColossalChat/config/conversation_template/01-ai_Yi-1.5-9B-Chat.json \ --conversation_template_config /PATH/TO/CHAT/TEMPLATE/CONFIG.json \
--tokenizer_dir "/mnt/jfs-hdd/share/models/Yi-1.5-6B" \ --tokenizer_dir "" \
--data_cache_dir $SAVE_DIR/cache \ --data_cache_dir $SAVE_DIR/cache \
--data_jsonl_output_dir $SAVE_DIR/jsonl \ --data_jsonl_output_dir $SAVE_DIR/jsonl \
--data_arrow_output_dir $SAVE_DIR/arrow \ --data_arrow_output_dir $SAVE_DIR/arrow \

20
applications/ColossalChat/examples/training_scripts/train_sft.sh
View File

@ -23,16 +23,16 @@ PARENT_CONFIG_FILE="" # Path to a folder to save training config logs
PRETRAINED_MODEL_PATH="" # huggingface or local model path PRETRAINED_MODEL_PATH="" # huggingface or local model path
PRETRAINED_TOKENIZER_PATH="" # huggingface or local tokenizer path PRETRAINED_TOKENIZER_PATH="" # huggingface or local tokenizer path
declare -a dataset=( declare -a dataset=(
YOUR/PREFERENCE/DATA/DIR/arrow/part-00000 YOUR/SFT/DATA/DIR/arrow/part-00000
YOUR/PREFERENCE/DATA/DIR/arrow/part-00001 YOUR/SFT/DATA/DIR/arrow/part-00001
YOUR/PREFERENCE/DATA/DIR/arrow/part-00002 YOUR/SFT/DATA/DIR/arrow/part-00002
YOUR/PREFERENCE/DATA/DIR/arrow/part-00003 YOUR/SFT/DATA/DIR/arrow/part-00003
YOUR/PREFERENCE/DATA/DIR/arrow/part-00004 YOUR/SFT/DATA/DIR/arrow/part-00004
YOUR/PREFERENCE/DATA/DIR/arrow/part-00005 YOUR/SFT/DATA/DIR/arrow/part-00005
YOUR/PREFERENCE/DATA/DIR/arrow/part-00006 YOUR/SFT/DATA/DIR/arrow/part-00006
YOUR/PREFERENCE/DATA/DIR/arrow/part-00007 YOUR/SFT/DATA/DIR/arrow/part-00007
YOUR/PREFERENCE/DATA/DIR/arrow/part-00008 YOUR/SFT/DATA/DIR/arrow/part-00008
YOUR/PREFERENCE/DATA/DIR/arrow/part-00009 YOUR/SFT/DATA/DIR/arrow/part-00009
) )
TIMESTAMP=$(date +%Y-%m-%d-%H-%M-%S) TIMESTAMP=$(date +%Y-%m-%d-%H-%M-%S)