use try-except to handle file error

2023-10-27 10:20:43 +08:00 · 2023-10-27 10:20:43 +08:00 · 2520edb795
parent a862f503b6
commit 2520edb795
1 changed files with 3 additions and 2 deletions
--- a/internlm/utils/storage_manager.py
+++ b/internlm/utils/storage_manager.py
@ -512,10 +512,11 @@ class LocalClient(StorageClient):
    def sync_upload_fileobj(fp: str, saved_obj=None, **kwargs):
        assert saved_obj is not None
        fp_dirname = os.path.dirname(fp)
-        if gpc.get_global_rank() == 0:
+        try:
            if not os.path.exists(fp_dirname):
                os.makedirs(fp_dirname, exist_ok=True)
-        dist.barrier()
+        except:
+            pass
        torch.save(saved_obj, fp, **kwargs)

    @staticmethod