Merge remote-tracking branch 'upstream/develop' into feat/fstp_refactor

merge develop
2023-10-26 17:41:42 +08:00 · 2023-10-26 17:41:42 +08:00 · 1aae39b667
parent d831ddcc1d aeee9fd2a9
commit 1aae39b667
1 changed files with 2 additions and 0 deletions
--- a/internlm/solver/optimizer/hybrid_zero_optim.py
+++ b/internlm/solver/optimizer/hybrid_zero_optim.py
@ -856,6 +856,8 @@ class HybridZeroOptimizer(BaseOptimizer):
        for handle in handles:
            handle.wait()

+        torch.cuda().synchronize()
+
    ##################
    # FP16 Utilities #
    ##################