github
/
ColossalAI
mirror of https://github.com/hpcaitech/ColossalAI
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

[chatgpt] allow shard init and display warning (#2986)

pull/2993/head
ver217 2023-03-03 16:27:59 +08:00 committed by GitHub
parent f5ca0397dd
commit 0ff8406b00
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
applications/ChatGPT/chatgpt/trainer/strategies/colossalai.py
View File

@ -79,8 +79,9 @@ class ColossalAIStrategy(DDPStrategy):
self.stage = stage self.stage = stage
# TODO(ver217): support shard_init when using from_pretrained() # TODO(ver217): support shard_init when using from_pretrained()
if shard_init: if shard_init:
warnings.warn(f'Shard init is not supported yet. Ignore.') warnings.warn(
shard_init = False f'Shard init is not supported model.from_pretrained() yet. Please load weights after strategy.prepare()'
)
self.shard_init = shard_init self.shard_init = shard_init
self.gemini_config = dict(device=get_current_device(), self.gemini_config = dict(device=get_current_device(),
placement_policy=placement_policy, placement_policy=placement_policy,