github
/
InternLM
mirror of https://github.com/InternLM/InternLM
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

modify moe config file

pull/182/head
Wenwen Qu 2023-09-26 21:01:06 +08:00
parent 54af6ba297
commit 8a63cb51ef
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
configs/7B_MoE4_sft.py
View File

@ -133,7 +133,7 @@ model = dict(
layer_norm_epsilon=1e-5, layer_norm_epsilon=1e-5,
use_flash_attn=True, use_flash_attn=True,
num_chunks=1, # if num_chunks > 1, interleaved pipeline scheduler is used. num_chunks=1, # if num_chunks > 1, interleaved pipeline scheduler is used.
num_experts=4, num_experts=2,
moe_use_residual=False, moe_use_residual=False,
moe_gate_k=2, moe_gate_k=2,
) )
@ -151,7 +151,7 @@ tensor parallel: tensor parallel size, usually the number of GPUs per node.
""" """
parallel = dict( parallel = dict(
zero1=-1, zero1=-1,
tensor=2, tensor=4,
pipeline=dict(size=1, interleaved_overlap=True), pipeline=dict(size=1, interleaved_overlap=True),
sequence_parallel=False, sequence_parallel=False,
) )