[tensor]add assert for colo_tensor 1Drow (#846)

colossalai/tensor/_ops/linear.py

@@ -3,6 +3,8 @@ from colossalai.tensor.op_wrapper import colo_op_impl
 from colossalai.tensor.colo_tensor import ColoTensor
 from colossalai.context import ParallelMode
 from colossalai.nn.layer.parallel_1d._utils import split_forward_gather_backward, reduce_input
+from colossalai.nn.layer.utils import divide
+from colossalai.core import global_context as gpc
 from packaging import version
 
 @colo_op_impl(torch.nn.functional.linear)
@@ -29,10 +31,11 @@ def colo_linear(types, args, kwargs, pg):
         if weight.shard_spec == None:
             return torch.nn.functional.linear(input_tensor, weight.torch_tensor(), bias)
         elif weight.shard_spec == '1Drow':
-            """
+            # Input:S[1] x Weight:S[0] = Output:P
-            Input:S[1] x Weight:S[0] = Output:P
+            # All-Reduce(Output) + bias = res
-            All-Reduce(Output) + bias = res
+            assert divide(input_tensor.shape[-1], gpc.tensor_parallel_size) == weight.size[-1], \
-            """
+            'Invalid shapes in 1Drow forward: input={}, weight={}. Expected last dim of input {}.'.format(
+                input_tensor.shape, weight.size, weight.size[-1] * gpc.tensor_parallel_size)
             # Input:S[1]
             input_per_partition = split_forward_gather_backward(input_tensor, ParallelMode.PARALLEL_1D, dim=-1)
             # Output:P

colossalai/tensor/colo_tensor.py

@@ -1,7 +1,6 @@
 from numpy import product
 import torch
-from typing import Tuple
+from typing import Tuple, Optional
-import numpy
 from .op_wrapper import _COLOSSAL_OPS
 
 class ColoTensor(object):

tests/test_tensor/test_linear_tp.py

@@ -14,7 +14,7 @@ from colossalai.utils import free_port
 from colossalai.core import global_context as gpc
 import torch.distributed as dist
 
-from test_tensor_utils import check_equal, replace_parameter_add_grad, broadcast_tensor_chunk
+from _utils import check_equal, replace_parameter_add_grad, broadcast_tensor_chunk
 
 def run_linear_tp1d_row_test():
     device = get_current_device()