From bba9b01c0ea683321a61f7e8bc23ca7640560b4c Mon Sep 17 00:00:00 2001
From: Wenwen Qu <vinny_qu@163.com>
Date: Tue, 19 Sep 2023 17:45:48 +0800
Subject: [PATCH] Update 7B_sft.py

---
 configs/7B_sft.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/configs/7B_sft.py b/configs/7B_sft.py
index 7f44533..25a98bf 100644
--- a/configs/7B_sft.py
+++ b/configs/7B_sft.py
@@ -91,7 +91,6 @@ hybrid_zero_optimizer = dict(
 
 loss = dict(
     label_smoothing=0,
-    moe_loss_coeff=1.0,
 )
 
 adam = dict(
@@ -133,7 +132,6 @@ model = dict(
     layer_norm_epsilon=1e-5,
     use_flash_attn=True,
     num_chunks=1,  # if num_chunks > 1, interleaved pipeline scheduler is used.
-    num_experts=8,
 )
 """
 zero1 parallel:
@@ -149,6 +147,7 @@ tensor parallel: tensor parallel size, usually the number of GPUs per node.
 """
 parallel = dict(
     zero1=8,
+    tensor=1,
     pipeline=dict(size=1, interleaved_overlap=True),
     sequence_parallel=False,
 )