mirror of https://github.com/hpcaitech/ColossalAI
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
18 lines
488 B
18 lines
488 B
#!/bin/bash
|
|
set -x
|
|
|
|
pip install -r requirements.txt
|
|
|
|
FAIL_LIMIT=3
|
|
|
|
for plugin in "torch_ddp" "torch_ddp_fp16" "gemini" "low_level_zero" "hybrid_parallel"; do
|
|
for i in $(seq 1 $FAIL_LIMIT); do
|
|
torchrun --standalone --nproc_per_node 4 finetune.py --target_f1 0.86 --plugin $plugin --model_type "bert" && break
|
|
echo "Failed $i times"
|
|
if [ $i -eq $FAIL_LIMIT ]; then
|
|
echo "Failed $FAIL_LIMIT times, exiting"
|
|
exit 1
|
|
fi
|
|
done
|
|
done
|