github
/
ColossalAI
mirror of https://github.com/hpcaitech/ColossalAI
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

[TP] Add gather_out arg to Linear (#541)

pull/552/head
Ziyue Jiang 2022-03-30 09:35:46 +08:00 committed by GitHub
parent 8c90d4df54
commit 763dc325f1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 14 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
colossalai/nn/layer/colossalai_layer/linear.py
View File

@ -1,4 +1,5 @@
import math import math
import inspect
from typing import Callable from typing import Callable
from colossalai.utils import get_current_device from colossalai.utils import get_current_device
@ -78,7 +79,11 @@ class Linear(nn.Module):
if self.layer.bias is not None: if self.layer.bias is not None:
bias_initializer(self.layer.bias, fan_in=in_features) bias_initializer(self.layer.bias, fan_in=in_features)
else: else:
self.layer = _parallel_linear[tensor_parallel]( linear_cls = _parallel_linear[tensor_parallel]
gather_output = kwargs.pop('gather_output', None)
if 'gather_output' in inspect.signature(linear_cls.__init__).parameters.keys(): # gather_out arg is available
kwargs['gather_output'] = gather_output
self.layer = linear_cls(
in_features, in_features,
out_features, out_features,
bias=bias, bias=bias,