github
/
InternLM
mirror of https://github.com/InternLM/InternLM
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Update 7B_sft.py

pull/182/head
Wenwen Qu 2023-09-19 17:45:48 +08:00 committed by GitHub
parent b7ddc42dcd
commit bba9b01c0e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
configs/7B_sft.py
View File

@ -91,7 +91,6 @@ hybrid_zero_optimizer = dict(
loss = dict( loss = dict(
label_smoothing=0, label_smoothing=0,
moe_loss_coeff=1.0,
) )
adam = dict( adam = dict(
@ -133,7 +132,6 @@ model = dict(
layer_norm_epsilon=1e-5, layer_norm_epsilon=1e-5,
use_flash_attn=True, use_flash_attn=True,
num_chunks=1, # if num_chunks > 1, interleaved pipeline scheduler is used. num_chunks=1, # if num_chunks > 1, interleaved pipeline scheduler is used.
num_experts=8,
) )
""" """
zero1 parallel: zero1 parallel:
@ -149,6 +147,7 @@ tensor parallel: tensor parallel size, usually the number of GPUs per node.
""" """
parallel = dict( parallel = dict(
zero1=8, zero1=8,
tensor=1,
pipeline=dict(size=1, interleaved_overlap=True), pipeline=dict(size=1, interleaved_overlap=True),
sequence_parallel=False, sequence_parallel=False,
) )