From 9aef11e89c2c2e221853447c9f5c3dda345994a6 Mon Sep 17 00:00:00 2001 From: ytxiong <45058324+yingtongxiong@users.noreply.github.com> Date: Mon, 9 Oct 2023 13:53:52 +0800 Subject: [PATCH] make seed in different tensor rank different (#405) --- internlm/core/context/parallel_context.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internlm/core/context/parallel_context.py b/internlm/core/context/parallel_context.py index 7f3e415..da6a0d7 100644 --- a/internlm/core/context/parallel_context.py +++ b/internlm/core/context/parallel_context.py @@ -568,7 +568,8 @@ class ParallelContext(metaclass=SingletonMeta): # during model construction), this is because the random state will be different in different tensor parallel # device of the same data parallel group. The underlying reason is that the device of tp_rank = 0 will perform # additional random operations during the RowParallelLinear module building process. - set_mode(ParallelMode.DUMMY) + # set_mode(ParallelMode.DUMMY) + set_mode(ParallelMode.TENSOR) seeds = get_seeds() seed_str = ", ".join([f"{k}: {v}" for k, v in seeds.items()])