github
/
InternLM
mirror of https://github.com/InternLM/InternLM
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix moe and zero1 check in args_sanity_check

pull/411/head
Qu Wenwen 2023-10-12 10:56:59 +08:00
parent 93bb5c2760
commit 816ecf8e04
2 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
configs/7B_MoE4_sft.py
View File

@ -150,7 +150,7 @@ pipeline parallel (dict):
tensor parallel: tensor parallel size, usually the number of GPUs per node. tensor parallel: tensor parallel size, usually the number of GPUs per node.
""" """
parallel = dict( parallel = dict(
zero1=-1, zero1=dict(size=-1, fsdp=False),
tensor=2, tensor=2,
pipeline=dict(size=1, interleaved_overlap=True), pipeline=dict(size=1, interleaved_overlap=True),
sequence_parallel=False, sequence_parallel=False,
@ -168,4 +168,4 @@ monitor = dict(
), ),
) )
model_type = "INTERNLM_MoE" model_type = "INTERNLM_MoE"

2
internlm/initialize/launch.py
View File

@ -349,7 +349,7 @@ def args_sanity_check():
assert ( assert (
not optim_ckpt.overlap_sync_grad & optim_ckpt.overlap_sync_param not optim_ckpt.overlap_sync_grad & optim_ckpt.overlap_sync_param
), "not support overlap and moe at the same time" ), "not support overlap and moe at the same time"
assert gpc.config.parallel.zero1 == -1, "moe only support zero1, set zero1=-1 can fix this" assert gpc.config.parallel.zero1.size == -1, "moe only support zero1, set zero1=dict(size=-1,...) can fix this"
def launch( def launch(