github
/
InternLM
mirror of https://github.com/InternLM/InternLM
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

add default setting for expert parallel size

pull/375/head
Wenwen Qu 2023-08-24 18:52:12 +08:00
parent 0e2eb90d22
commit 86bcda5ca9
2 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internlm/core/context/parallel_context.py
View File

@ -143,7 +143,7 @@ class ParallelContext(metaclass=SingletonMeta):
self.pipeline_parallel_size = 1 self.pipeline_parallel_size = 1
self.tensor_parallel_size = 1 self.tensor_parallel_size = 1
self.zero1_parallel_size = -1 self.zero1_parallel_size = -1
self.expert_parallel_size = -1 self.expert_parallel_size = 1
self.num_processes_on_current_node = -1 self.num_processes_on_current_node = -1
self.virtual_pipeline_parallel_size = None self.virtual_pipeline_parallel_size = None
self.virtual_pipeline_parallel_rank = None self.virtual_pipeline_parallel_rank = None

3
internlm/initialize/launch.py
View File

@ -60,6 +60,9 @@ def args_sanity_check():
if "tensor" not in gpc.config.parallel: if "tensor" not in gpc.config.parallel:
gpc.config.parallel._add_item("tensor", 1) gpc.config.parallel._add_item("tensor", 1)
if "expert" not in gpc.config.parallel:
gpc.config.parallel._add_item("expert", 1)
# processing the data config in gpc # processing the data config in gpc
data = gpc.config.data data = gpc.config.data