From 742a21677b5b26c7b72b010c6b41499a59370dc8 Mon Sep 17 00:00:00 2001
From: Wenwen Qu <vinny_qu@163.com>
Date: Thu, 24 Aug 2023 18:54:00 +0800
Subject: [PATCH] Update 7B_sft.py

---
 configs/7B_sft.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/configs/7B_sft.py b/configs/7B_sft.py
index e0f62a5..a4dfbac 100644
--- a/configs/7B_sft.py
+++ b/configs/7B_sft.py
@@ -126,7 +126,6 @@ model = dict(
     layer_norm_epsilon=1e-5,
     use_flash_attn=True,
     num_chunks=1,  # if num_chunks > 1, interleaved pipeline scheduler is used.
-    sequence_parallel=False,
     num_experts=8,
 )
 """
@@ -144,6 +143,8 @@ tensor parallel: tensor parallel size, usually the number of GPUs per node.
 parallel = dict(
     zero1=8,
     pipeline=dict(size=1, interleaved_overlap=True),
+    sequence_parallel=False,
+    expert=2,
 )
 
 cudnn_deterministic = False