github
/
InternLM
mirror of https://github.com/InternLM/InternLM
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix moe zero mode bugs

pull/548/head
Qu Wenwen 2023-12-18 13:43:45 +08:00
parent bbb5651582
commit c801336732
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internlm/solver/optimizer/hybrid_zero_optim.py
View File

@ -150,7 +150,7 @@ class HybridZeroOptimizer(BaseOptimizer):
# if zero is used, expert dp group will use ParallelMode.EXPERT_DATA as the real zero mode # if zero is used, expert dp group will use ParallelMode.EXPERT_DATA as the real zero mode
zero_mode = ( zero_mode = (
ParallelMode.ZERO1 ParallelMode.ZERO1
if param_group["dp_mode"] == gpc.get_world_size(ParallelMode.ZERO1) == 1 or ParallelMode.DATA if gpc.get_world_size(ParallelMode.ZERO1) == 1 or param_group["dp_mode"] == ParallelMode.DATA
else ParallelMode.EXPERT_DATA else ParallelMode.EXPERT_DATA
) )
self._zero_local_rank.append(gpc.get_local_rank(zero_mode)) self._zero_local_rank.append(gpc.get_local_rank(zero_mode))