2022-03-15 09:07:35 +00:00
|
|
|
from typing import Optional
|
|
|
|
|
2022-03-08 10:18:06 +00:00
|
|
|
import torch
|
2022-03-18 08:18:31 +00:00
|
|
|
import torch.distributed as dist
|
2022-03-08 10:18:06 +00:00
|
|
|
from colossalai.registry import OPHOOKS
|
2022-03-10 06:08:58 +00:00
|
|
|
from colossalai.utils import get_current_device
|
2022-03-15 09:07:35 +00:00
|
|
|
from colossalai.utils.memory_tracer.memstats_collector import MemStatsCollector
|
2022-03-14 06:48:32 +00:00
|
|
|
from colossalai.zero.shard_utils import BaseShardStrategy
|
2022-03-30 07:57:46 +00:00
|
|
|
from colossalai.zero.sharded_param.tensorful_state import TensorState
|
2022-03-14 06:48:32 +00:00
|
|
|
|
2022-03-08 10:18:06 +00:00
|
|
|
from ._base_ophook import BaseOpHook
|
2022-03-30 10:14:50 +00:00
|
|
|
|
|
|
|
from colossalai.utils.memory_utils.utils import \
|
|
|
|
colo_model_data_tensor_move_inline
|
2022-03-08 10:18:06 +00:00
|
|
|
|
|
|
|
|
|
|
|
@OPHOOKS.register_module
|
|
|
|
class ZeroHook(BaseOpHook):
|
|
|
|
"""
|
|
|
|
A hook to process sharded param for ZeRO method.
|
|
|
|
"""
|
|
|
|
|
2022-03-18 08:18:31 +00:00
|
|
|
def __init__(self,
|
|
|
|
shard_strategy: BaseShardStrategy,
|
|
|
|
memstarts_collector: Optional[MemStatsCollector],
|
|
|
|
process_group: Optional[dist.ProcessGroup] = None):
|
2022-03-08 10:18:06 +00:00
|
|
|
super().__init__()
|
|
|
|
self.shard_strategy = shard_strategy
|
2022-03-18 08:18:31 +00:00
|
|
|
self.process_group = process_group
|
2022-03-10 06:08:58 +00:00
|
|
|
# NOTE(jiaruifang) Now the computing device of FWD and BWD is always on GPU
|
|
|
|
self.computing_device = torch.device(f'cuda:{get_current_device()}')
|
2022-03-08 10:18:06 +00:00
|
|
|
|
2022-03-14 14:05:30 +00:00
|
|
|
self._memstarts_collector = memstarts_collector
|
|
|
|
|
2022-03-08 10:18:06 +00:00
|
|
|
def pre_fwd_exec(self, module: torch.nn.Module, *args):
|
2022-03-14 06:48:32 +00:00
|
|
|
tensor_list = []
|
2022-03-28 09:42:18 +00:00
|
|
|
for param in module.parameters(recurse=False):
|
2022-03-31 04:25:45 +00:00
|
|
|
assert hasattr(param, 'colo_attr')
|
|
|
|
tensor_list.append(param.colo_attr.sharded_data_tensor)
|
2022-03-18 08:18:31 +00:00
|
|
|
self.shard_strategy.gather(tensor_list, self.process_group)
|
2022-03-28 09:42:18 +00:00
|
|
|
for param in module.parameters(recurse=False):
|
2022-03-31 04:25:45 +00:00
|
|
|
colo_model_data_tensor_move_inline(param.colo_attr.sharded_data_tensor, self.computing_device)
|
|
|
|
param.data = param.colo_attr.sharded_data_tensor.payload
|
2022-03-08 10:18:06 +00:00
|
|
|
|
2022-03-14 14:05:30 +00:00
|
|
|
if self._memstarts_collector:
|
|
|
|
self._memstarts_collector.sample_memstats()
|
|
|
|
|
2022-03-30 07:57:46 +00:00
|
|
|
for param in module.parameters(recurse=False):
|
2022-03-31 04:25:45 +00:00
|
|
|
param.colo_attr.sharded_data_tensor.trans_state(TensorState.COMPUTE)
|
2022-03-30 07:57:46 +00:00
|
|
|
|
2022-03-08 10:18:06 +00:00
|
|
|
def post_fwd_exec(self, module: torch.nn.Module, *args):
|
2022-03-30 07:57:46 +00:00
|
|
|
for param in module.parameters(recurse=False):
|
2022-03-31 04:25:45 +00:00
|
|
|
param.colo_attr.sharded_data_tensor.trans_state(TensorState.HOLD_AFTER_FWD)
|
2022-03-30 07:57:46 +00:00
|
|
|
|
2022-03-14 06:48:32 +00:00
|
|
|
tensor_list = []
|
2022-03-28 09:42:18 +00:00
|
|
|
for param in module.parameters(recurse=False):
|
2022-03-31 04:25:45 +00:00
|
|
|
assert hasattr(param, 'colo_attr')
|
|
|
|
tensor_list.append(param.colo_attr.sharded_data_tensor)
|
2022-03-18 08:18:31 +00:00
|
|
|
self.shard_strategy.shard(tensor_list, self.process_group)
|
2022-03-28 09:42:18 +00:00
|
|
|
for param in module.parameters(recurse=False):
|
2022-03-31 04:25:45 +00:00
|
|
|
param.colo_attr.remove_torch_payload()
|
2022-03-08 10:18:06 +00:00
|
|
|
|
|
|
|
def pre_bwd_exec(self, module: torch.nn.Module, input, output):
|
2022-03-14 06:48:32 +00:00
|
|
|
tensor_list = []
|
2022-03-28 09:42:18 +00:00
|
|
|
for param in module.parameters(recurse=False):
|
2022-03-31 04:25:45 +00:00
|
|
|
assert hasattr(param, 'colo_attr')
|
|
|
|
tensor_list.append(param.colo_attr.sharded_data_tensor)
|
2022-03-18 08:18:31 +00:00
|
|
|
self.shard_strategy.gather(tensor_list, self.process_group)
|
2022-03-28 09:42:18 +00:00
|
|
|
for param in module.parameters(recurse=False):
|
2022-03-31 04:25:45 +00:00
|
|
|
colo_model_data_tensor_move_inline(param.colo_attr.sharded_data_tensor, self.computing_device)
|
|
|
|
param.data = param.colo_attr.sharded_data_tensor.payload
|
2022-03-14 14:05:30 +00:00
|
|
|
if self._memstarts_collector:
|
|
|
|
self._memstarts_collector.sample_memstats()
|
2022-03-08 10:18:06 +00:00
|
|
|
|
2022-03-30 07:57:46 +00:00
|
|
|
for param in module.parameters(recurse=False):
|
2022-03-31 04:25:45 +00:00
|
|
|
param.colo_attr.sharded_data_tensor.trans_state(TensorState.COMPUTE)
|
2022-03-30 07:57:46 +00:00
|
|
|
|
2022-03-08 10:18:06 +00:00
|
|
|
def post_bwd_exec(self, module: torch.nn.Module, input):
|
2022-03-30 07:57:46 +00:00
|
|
|
for param in module.parameters(recurse=False):
|
2022-03-31 04:25:45 +00:00
|
|
|
param.colo_attr.sharded_data_tensor.trans_state(TensorState.HOLD_AFTER_BWD)
|
2022-03-30 07:57:46 +00:00
|
|
|
|
2022-03-14 06:48:32 +00:00
|
|
|
tensor_list = []
|
2022-03-28 09:42:18 +00:00
|
|
|
for param in module.parameters(recurse=False):
|
2022-03-31 04:25:45 +00:00
|
|
|
assert hasattr(param, 'colo_attr')
|
|
|
|
tensor_list.append(param.colo_attr.sharded_data_tensor)
|
2022-03-18 08:18:31 +00:00
|
|
|
self.shard_strategy.shard(tensor_list, self.process_group)
|
2022-03-30 07:57:46 +00:00
|
|
|
|
2022-03-28 09:42:18 +00:00
|
|
|
for param in module.parameters(recurse=False):
|
2022-03-31 04:25:45 +00:00
|
|
|
param.colo_attr.remove_torch_payload()
|
2022-03-08 10:18:06 +00:00
|
|
|
|
|
|
|
def pre_iter(self):
|
|
|
|
pass
|
|
|
|
|
|
|
|
def post_iter(self):
|
|
|
|
pass
|