make seed in different tensor rank different

2023-10-09 13:26:08 +08:00 · 2023-10-09 13:26:08 +08:00 · 0e26f52a89
parent 42851be36b
commit 0e26f52a89
1 changed files with 2 additions and 1 deletions
--- a/internlm/core/context/parallel_context.py
+++ b/internlm/core/context/parallel_context.py
@ -527,7 +527,8 @@ class ParallelContext(metaclass=SingletonMeta):
            tp_seed = seed + tp_rank + pipeline_offset * 1024
            add_seed(ParallelMode.TENSOR, tp_seed)

-        set_mode(ParallelMode.DATA)
+        # set_mode(ParallelMode.DATA)
+        set_mode(ParallelMode.TENSOR)

        seeds = get_seeds()
        seed_str = ", ".join([f"{k}: {v}" for k, v in seeds.items()])