mirror of https://github.com/hpcaitech/ColossalAI
aibig-modeldata-parallelismdeep-learningdistributed-computingfoundation-modelsheterogeneous-traininghpcinferencelarge-scalemodel-parallelismpipeline-parallelism
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
32 lines
977 B
32 lines
977 B
#!/usr/bin/env python |
|
# -*- encoding: utf-8 -*- |
|
|
|
ALLOWED_MODES = [None, '1d', '2d', '2.5d', '3d', 'sequence'] |
|
TENSOR_PARALLEL_MODE = 'tensor_parallel_mode' |
|
|
|
# initializer |
|
INITIALIZER_MAPPING = { |
|
'data': 'Initializer_Data', |
|
'tensor': 'Initializer_Tensor', |
|
'pipeline': 'Initializer_Pipeline', |
|
'embedding': 'Initializer_Embedding', |
|
'1d': 'Initializer_1D', |
|
'2d': 'Initializer_2D', |
|
'2.5d': 'Initializer_2p5D', |
|
'3d': 'Initializer_3D', |
|
'sequence': 'Initializer_Sequence', |
|
'model': 'Initializer_Model', |
|
'moe': 'Initializer_Moe' |
|
} |
|
|
|
# 3D parallelism groups |
|
INPUT_GROUP_3D = 'input_group_3d' |
|
WEIGHT_GROUP_3D = 'weight_group_3d' |
|
OUTPUT_GROUP_3D = 'output_group_3d' |
|
INPUT_X_WEIGHT_3D = 'input_x_weight_group_3d' |
|
OUTPUT_X_WEIGHT_3D = 'output_x_weight_group_3d' |
|
|
|
# Attributes of tensor parallel parameters |
|
IS_TENSOR_PARALLEL = 'is_tensor_parallel' |
|
NUM_PARTITIONS = 'num_partitions' |
|
TENSOR_PARALLEL_ATTRIBUTES = [IS_TENSOR_PARALLEL, NUM_PARTITIONS]
|
|
|