github
/
ColossalAI
mirror of https://github.com/hpcaitech/ColossalAI
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

[hotfix] fix init context (#1543) 

* fix init context

* fix lazy init ctx
pull/1546/head
ver217 2022-09-06 11:45:08 +08:00 committed by GitHub
parent 64169f3e8f
commit a203b709d5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 15 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
colossalai/utils/model/lazy_init_context.py
View File

@ -138,14 +138,14 @@ class LazyInitContext():
cls.__orig_init__ = cls.__init__
cls.__init__ = self._wrap_module_init(cls.__init__)
substitute_init_recursively(self._torch_mod_cls, _activate_wrap_init)
substitute_init_recursively(self._torch_mod_cls, _activate_wrap_init, set())
def _unpatch_submodule_init(self):
def _recover_orig_init(cls):
cls.__init__ = cls.__orig_init__
substitute_init_recursively(self._torch_mod_cls, _recover_orig_init)
substitute_init_recursively(self._torch_mod_cls, _recover_orig_init, set())
def _patch_torch_tensor_funcs(self):
# patch tensor value-setting functions

10
colossalai/utils/model/utils.py
View File

@ -3,10 +3,12 @@ import functools
from typing import Optional
def substitute_init_recursively(cls, func):
def substitute_init_recursively(cls, func, visited: set):
for subcls in cls.__subclasses__():
substitute_init_recursively(subcls, func)
substitute_init_recursively(subcls, func, visited)
if subcls not in visited:
func(subcls)
visited.add(subcls)
def call_to_str(base, *args, **kwargs):
@ -64,7 +66,7 @@ class InsertPostInitMethodToModuleSubClasses(object):
# Replace .__init__() for all existing subclasses of torch.nn.Module
# Excution self._post_init_method after the default init function.
substitute_init_recursively(torch.nn.modules.module.Module, _enable_class)
substitute_init_recursively(torch.nn.modules.module.Module, _enable_class, set())
# holding on to the current __init__subclass__ for exit
torch.nn.modules.module.Module._old_init_subclass = (torch.nn.modules.module.Module.__init_subclass__)
@ -87,7 +89,7 @@ class InsertPostInitMethodToModuleSubClasses(object):
cls.__init__ = cls._old_init
# Replace .__init__() for all existing subclasses of torch.nn.Module
substitute_init_recursively(torch.nn.modules.module.Module, _disable_class)
substitute_init_recursively(torch.nn.modules.module.Module, _disable_class, set())
# Replace .__init__() for future subclasses of torch.nn.Module
torch.nn.modules.module.Module.__init_subclass__ = (torch.nn.modules.module.Module._old_init_subclass)