Update 7B_sft.py

configs/7B_sft.py

@@ -91,7 +91,6 @@ hybrid_zero_optimizer = dict(
 
 loss = dict(
     label_smoothing=0,
-    moe_loss_coeff=1.0,
 )
 
 adam = dict(
@@ -133,7 +132,6 @@ model = dict(
     layer_norm_epsilon=1e-5,
     use_flash_attn=True,
     num_chunks=1,  # if num_chunks > 1, interleaved pipeline scheduler is used.
-    num_experts=8,
 )
 """
 zero1 parallel:
@@ -149,6 +147,7 @@ tensor parallel: tensor parallel size, usually the number of GPUs per node.
 """
 parallel = dict(
     zero1=8,
+    tensor=1,
     pipeline=dict(size=1, interleaved_overlap=True),
     sequence_parallel=False,
 )