diff --git a/configs/7B_sft.py b/configs/7B_sft.py index dd4104a..4c55fee 100644 --- a/configs/7B_sft.py +++ b/configs/7B_sft.py @@ -154,8 +154,8 @@ pipeline parallel (dict): tensor parallel: tensor parallel size, usually the number of GPUs per node. """ parallel = dict( - zero1=-1, - tensor=dict(size=2, mode='origin_tp'), # the mode should be 'origin_tp' or 'fstp'. if the mode is 'fstp', the sequence_parallel should be True + zero1=8, + tensor=dict(size=1, mode='origin_tp'), # the mode should be 'origin_tp' or 'fstp'. if the mode is 'fstp', the sequence_parallel should be True pipeline=dict(size=1, interleaved_overlap=True), sequence_parallel=False, )