ColossalAI/colossalai/auto_parallel/tensor_shard/utils/broadcast.py

from enum import Enum, auto
from typing import List

import torch
from torch.fx.node import Node

from colossalai.auto_parallel.tensor_shard.sharding_strategy import (
    CommAction,
    CommType,
    OperationData,
    OperationDataType,
)
from colossalai.tensor.comm_spec import CollectiveCommPattern, CommSpec
from colossalai.tensor.sharding_spec import ShardingSpec

__all__ = [
    'BroadcastType', 'is_broadcastable', 'get_broadcast_shape', 'recover_sharding_spec_for_broadcast_shape',
    'comm_actions_for_oprands'
]


class BroadcastType(Enum):
    EQUAL = auto()
    PADDDING = auto()
    MULTIPLE = auto()


def is_broadcastable(shape1: torch.Size, shape2: torch.Size) -> bool:
    """
    Check if two shapes are broadcastable to each other.
    """
    for s1, s2 in zip(shape1[::-1], shape2[::-1]):
        if s1 == 1 or s2 == 1 or s1 == s2:
            pass
        else:
            return False
    return True


def get_broadcast_shape(shape1: torch.Size, shape2: torch.Size) -> List[int]:
    """
    Compute the broadcast shape given two shapes.
    """
    assert is_broadcastable(shape1, shape2), f'{shape1} and {shape2} are not broadcastable'
    shape1_reverse = shape1[::-1]
    shape2_reverse = shape2[::-1]
    min_common_dim = min(len(shape1), len(shape2))
    dims = []
    for s1, s2 in zip(shape1_reverse, shape2_reverse):
        dims.append(max(s1, s2))

    # append the remaining dims
    dims.extend(shape1_reverse[min_common_dim:])
    dims.extend(shape2_reverse[min_common_dim:])
    return dims[::-1]


def get_broadcast_dim_info(logical_shape, physical_shape):
    # get the number of dimensions
    logical_num_dims = len(logical_shape)
    physical_num_dims = len(physical_shape)

    assert logical_num_dims >= physical_num_dims, \
        'The number of dimensions in the logical shape is smaller than that of the physical shape, this tensor is not broadcast!'

    # track the dim and its broadcasting type
    logical_dim_broadcast_info = {}

    for i in range(logical_num_dims):
        # get the trailing dim size
        logical_dim_idx = logical_num_dims - i - 1
        phyiscal_dim_idx = physical_num_dims - i - 1
        logical_dim_size = logical_shape[logical_dim_idx]

        if phyiscal_dim_idx >= 0:
            physical_dim_size = physical_shape[phyiscal_dim_idx]

            if physical_dim_size == logical_dim_size:
                logical_dim_broadcast_info[logical_dim_idx] = BroadcastType.EQUAL
            elif physical_dim_size == 1 and physical_dim_size != logical_dim_size:
                logical_dim_broadcast_info[logical_dim_idx] = BroadcastType.MULTIPLE
        else:
            logical_dim_broadcast_info[logical_dim_idx] = BroadcastType.PADDDING

    return logical_dim_broadcast_info


def recover_sharding_spec_for_broadcast_shape(logical_sharding_spec: ShardingSpec, logical_shape: torch.Size,
                                              physical_shape: torch.Size) -> ShardingSpec:
    """
    This function computes the sharding spec for the physical shape of a broadcast tensor.

    Args:
        logical_sharding_spec (ShardingSpec): the sharding spec for the broadcast tensor
        logical_shape (torch.Size): logical shape is the broadcast shape of a tensor
        physical_shape (torch.Size): the shape of the tensor before broadcasting
    """
    # if the two shapes are the same, no broadcast occurs
    # we directly return the current sharding spec

    # recording the sharding dimensions removed during logical shape converting to physical one
    removed_dims = []
    if list(logical_shape) == list(physical_shape):
        return logical_sharding_spec, removed_dims

    # get the number of dimensions
    logical_num_dims = len(logical_shape)
    physical_num_dims = len(physical_shape)

    # get the broadcast info
    logical_dim_broadcast_info = get_broadcast_dim_info(logical_shape, physical_shape)

    # generate the sharding spec for the physical shape
    physical_dim_partition = {}
    logical_dim_partition = logical_sharding_spec.dim_partition_dict

    for shape_dim, mesh_dim in logical_dim_partition.items():
        logical_broadcast_type = logical_dim_broadcast_info[shape_dim]

        if logical_broadcast_type == BroadcastType.PADDDING or logical_broadcast_type == BroadcastType.MULTIPLE:
            removed_dims.extend(mesh_dim)
        else:
            # get the corresponding physical dim
            physical_dim = physical_num_dims - (logical_num_dims - shape_dim)
            physical_dim_partition[physical_dim] = mesh_dim

    physical_sharding_spec = ShardingSpec(device_mesh=logical_sharding_spec.device_mesh,
                                          entire_shape=physical_shape,
                                          dim_partition_dict=physical_dim_partition)

    return physical_sharding_spec, removed_dims


def comm_actions_for_oprands(node: Node, removed_dims: List[int], op_data: OperationData,
                             sharding_spec: ShardingSpec) -> CommAction:
    """
    This method is used to generate communication actions for oprands which lose information
    during convert logical shape to physical shape.
    """
    if len(removed_dims) == 1:
        # if list length is 1, extract element from list to avoid using flatten device mesh
        removed_dims = removed_dims[0]
    comm_spec = CommSpec(comm_pattern=CollectiveCommPattern.IDENTITY_FWD_ALLREDUCE_BWD,
                         sharding_spec=sharding_spec,
                         logical_process_axis=removed_dims)
    if op_data.type == OperationDataType.PARAM:
        comm_type = CommType.HOOK
    else:
        comm_type = CommType.BEFORE
    arg_index = -1
    for index, arg in enumerate(node.args):
        if op_data.name == str(arg):
            arg_index = index
    assert arg_index >= 0, f'op_data should be an argument of node.'
    comm_action = CommAction(
        comm_spec=comm_spec,
        comm_type=comm_type,
        arg_index=arg_index,
    )
    return comm_action