diff --git a/colossalai/amp/__init__.py b/colossalai/amp/__init__.py index 1d35c80d8..790a83767 100644 --- a/colossalai/amp/__init__.py +++ b/colossalai/amp/__init__.py @@ -19,7 +19,7 @@ def convert_to_amp(model: nn.Module, optimizer: Optimizer, criterion: _Loss, mod optimizer (:class:`torch.optim.Optimizer`): your optimizer object. criterion (:class:`torch.nn.modules.loss._Loss`): your loss function object. mode (:class:`colossalai.amp.AMP_TYPE`): amp mode. - amp_config (:class:`colossalai.context.Config` or dict): configuration for different amp modes + amp_config (Union[:class:`colossalai.context.Config`, dict]): configuration for different amp modes. Returns: A tuple (model, optimizer, criterion). diff --git a/colossalai/amp/apex_amp/__init__.py b/colossalai/amp/apex_amp/__init__.py index 37aa16655..34e6bf091 100644 --- a/colossalai/amp/apex_amp/__init__.py +++ b/colossalai/amp/apex_amp/__init__.py @@ -9,7 +9,7 @@ def convert_to_apex_amp(model: nn.Module, optimizer: Optimizer, amp_config): Args: model (:class:`torch.nn.Module`): your model object. optimizer (:class:`torch.optim.Optimizer`): your optimizer object. - amp_config (:class: colossalai.context.Config or dict): configuration for initializing apex_amp. + amp_config (Union[:class:`colossalai.context.Config`, dict]): configuration for initializing apex_amp. The ``amp_config`` should include parameters below: :: diff --git a/colossalai/builder/builder.py b/colossalai/builder/builder.py index ee3001983..f4ccebfc7 100644 --- a/colossalai/builder/builder.py +++ b/colossalai/builder/builder.py @@ -29,21 +29,22 @@ def build_from_registry(config, registry: Registry): is specified by `registry`. Note: - the `config` is used to construct the return object such as `LAYERS`, - `OPTIMIZERS` and other support types in `registry`. The `config` should contain - all required parameters of corresponding object. The details of support - types in `registry` and the `mod_type` in `config` could be found in - `registry `_. + the `config` is used to construct the return object such as `LAYERS`, `OPTIMIZERS` + and other support types in `registry`. The `config` should contain + all required parameters of corresponding object. The details of support + types in `registry` and the `mod_type` in `config` could be found in + `registry `_. Args: config (dict or :class:`colossalai.context.colossalai.context.Config`): information used in the construction of the return object. registry (:class:`Registry`): A registry specifying the type of the return object - Returns: A Python object specified by `registry` + Returns: + A Python object specified by `registry`. Raises: - Exception: Raises an Exception if an error occurred when building from registry + Exception: Raises an Exception if an error occurred when building from registry. """ config_ = config.copy() # keep the original config untouched assert isinstance( diff --git a/colossalai/builder/pipeline.py b/colossalai/builder/pipeline.py index a1187d0d8..d6394be6c 100644 --- a/colossalai/builder/pipeline.py +++ b/colossalai/builder/pipeline.py @@ -163,17 +163,19 @@ def count_layer_params(layers): def build_pipeline_model_from_cfg(config, num_chunks: int = 1, partition_method: str = 'parameter', verbose: bool = False): - """An intializer to split the model into different stages for pipeline parallelism. + """An initializer to split the model into different stages for pipeline parallelism. An example for the model config is shown below. The class VisionTransformerFromConfig should inherit colossalai.nn.model.ModelFromConfig to allow this initializer to build model from a sequence of layer configurations. - model_config = dict( - type='VisionTransformerFromConfig', - embedding_cfg=dict(...), - ... - ) + :: + + model_config = dict( + type='VisionTransformerFromConfig', + embedding_cfg=dict(...), + ... + ) Args: config (dict): Configuration of the model. diff --git a/colossalai/communication/utils.py b/colossalai/communication/utils.py index b0337ed1b..3ac1bffc1 100644 --- a/colossalai/communication/utils.py +++ b/colossalai/communication/utils.py @@ -45,7 +45,7 @@ def recv_tensor_meta(tensor_shape, prev_rank=None): prev_rank (int): The rank of the source of the tensor. Returns: - torch.Size: The shape of the tensor to be received. + :class:`torch.Size`: The shape of the tensor to be received. """ if tensor_shape is None: if prev_rank is None: @@ -71,7 +71,7 @@ def split_tensor_into_1d_equal_chunks(tensor, new_buffer=False): new_buffer (bool, optional): Whether to use a new buffer to store sliced tensor. Returns: - torch.Tensor: The split tensor + :class:`torch.Size`: The split tensor """ partition_size = torch.numel(tensor) // gpc.get_world_size(ParallelMode.PARALLEL_1D) start_index = partition_size * gpc.get_local_rank(ParallelMode.PARALLEL_1D) @@ -92,7 +92,7 @@ def gather_split_1d_tensor(tensor): Args: tensor (torch.Tensor): Tensor to be gathered after communication. Returns: - gathered (torch.Tensor): The gathered tensor + :class:`torch.Size`: The gathered tensor. """ world_size = gpc.get_world_size(ParallelMode.PARALLEL_1D) numel = torch.numel(tensor) diff --git a/colossalai/context/parallel_context.py b/colossalai/context/parallel_context.py index ce40df22e..1148e905c 100644 --- a/colossalai/context/parallel_context.py +++ b/colossalai/context/parallel_context.py @@ -193,7 +193,7 @@ class ParallelContext(metaclass=SingletonMeta): Returns: bool: a boolean value indicating whether the current device is the first one - among its group for `parallel_mode`. + among its group for `parallel_mode`. """ rank = self.get_local_rank(parallel_mode) return rank == 0 @@ -211,7 +211,7 @@ class ParallelContext(metaclass=SingletonMeta): Returns: bool: a boolean value indicating whether the current device is the first one - among its group for `parallel_mode`. + among its group for `parallel_mode`. """ rank = self.get_local_rank(parallel_mode) world_size = self.get_world_size(parallel_mode) diff --git a/colossalai/context/random/seed_manager.py b/colossalai/context/random/seed_manager.py index c0cdc7972..3c84aaafc 100644 --- a/colossalai/context/random/seed_manager.py +++ b/colossalai/context/random/seed_manager.py @@ -34,6 +34,7 @@ class SeedManager: def set_state(self, parallel_mode: ParallelMode, state: Tensor): """Sets the state of the seed manager for `parallel_mode`. + Args: parallel_mode (:class:`colossalai.context.ParallelMode`): The chosen parallel mode. state (:class:`torch.Tensor`): the state to be set. @@ -66,9 +67,9 @@ class SeedManager: seed (int): The seed to be added. overwrtie (bool, optional): Whether allows to overwrite the seed that has been set already - Raises - AssertionError: Raises an AssertionError if `parallel_mode` is not an instance of - :class:`colossalai.context.ParallelMode` or the seed for `parallel_mode` has been added. + Raises: + AssertionError: Raises an AssertionError if `parallel_mode` is not an instance of :class:`colossalai.context.ParallelMode` + or the seed for `parallel_mode` has been added. """ assert isinstance(parallel_mode, ParallelMode), 'A valid ParallelMode must be provided' if overwrtie is False: diff --git a/colossalai/nn/layer/parallel_3d/_operation.py b/colossalai/nn/layer/parallel_3d/_operation.py index 86639f297..8cf95f519 100644 --- a/colossalai/nn/layer/parallel_3d/_operation.py +++ b/colossalai/nn/layer/parallel_3d/_operation.py @@ -264,7 +264,7 @@ def layernorm_3d(input_: Tensor, weight: Tensor, bias: Tensor, normalized_shape: def split_tensor_3d(tensor: Tensor, dim: int, parallel_mode: ParallelMode) -> Tensor: r"""Splits 3D parallel tensor in specified dimension. - Args: + Args: tensor (:class:`torch.tensor`): Input tensor. dim (int): Specified dimension in which to split. parallel_mode (:class:`colossalai.context.parallel_mode.ParallelMode`, optional): Parallel mode. diff --git a/colossalai/nn/loss/loss_2d.py b/colossalai/nn/loss/loss_2d.py index 8bc6ff68d..8e685bd3a 100644 --- a/colossalai/nn/loss/loss_2d.py +++ b/colossalai/nn/loss/loss_2d.py @@ -27,7 +27,7 @@ class CrossEntropyLoss2D(_Loss): reduce (bool, optional) label_smoothing (float, optional) - More details about args, kwargs and torch.nn.functional.cross_entropy could be found in + More details about ``args``, ``kwargs`` and ``torch.nn.functional.cross_entropy`` could be found in `Cross_entropy `_. """ diff --git a/colossalai/nn/loss/loss_2p5d.py b/colossalai/nn/loss/loss_2p5d.py index 317d47648..5aadcef1f 100644 --- a/colossalai/nn/loss/loss_2p5d.py +++ b/colossalai/nn/loss/loss_2p5d.py @@ -27,7 +27,7 @@ class CrossEntropyLoss2p5D(_Loss): reduce (bool, optional) label_smoothing (float, optional) - More details about args, kwargs and torch.nn.functional.cross_entropy could be found in + More details about ``args``, ``kwargs`` and ``torch.nn.functional.cross_entropy`` could be found in `Cross_entropy `_. """ def __init__(self, reduction=True, *args, **kwargs): diff --git a/colossalai/nn/loss/loss_3d.py b/colossalai/nn/loss/loss_3d.py index b2f01ebbe..e76439191 100644 --- a/colossalai/nn/loss/loss_3d.py +++ b/colossalai/nn/loss/loss_3d.py @@ -27,7 +27,7 @@ class CrossEntropyLoss3D(_Loss): reduce (bool, optional) label_smoothing (float, optional) - More details about args, kwargs and torch.nn.functional.cross_entropy could be found in + More details about ``args``, ``kwargs`` and ``torch.nn.functional.cross_entropy`` could be found in `Cross_entropy `_. """ diff --git a/colossalai/nn/loss/loss_moe.py b/colossalai/nn/loss/loss_moe.py index ffefe5375..a8b18a3e3 100644 --- a/colossalai/nn/loss/loss_moe.py +++ b/colossalai/nn/loss/loss_moe.py @@ -23,7 +23,7 @@ class MoeCrossEntropyLoss(_Loss): reduction (str, optional) label_smoothing (float, optional) - More details about args, kwargs and torch.nn.functional.cross_entropy could be found in + More details about ``args``, ``kwargs`` and ``torch.nn.functional.cross_entropy`` could be found in `Cross_entropy `_. """ @@ -40,7 +40,7 @@ class MoeCrossEntropyLoss(_Loss): input (:class:`torch.tensor`): Predicted unnormalized scores (often referred to as logits). target (:class:`torch.tensor`): Ground truth class indices or class probabilities. - More details about args, kwargs and torch.nn.functional.cross_entropy could be found in + More details about ``args``, ``kwargs`` and ``torch.nn.functional.cross_entropy`` could be found in `Cross_entropy `_. """ main_loss = self.loss(*args) diff --git a/colossalai/trainer/_trainer.py b/colossalai/trainer/_trainer.py index b1e512212..1ce139d59 100644 --- a/colossalai/trainer/_trainer.py +++ b/colossalai/trainer/_trainer.py @@ -307,8 +307,7 @@ class Trainer: max_steps (int, optional): Maximum number of running iterations. test_dataloader (:class:`torch.utils.data.DataLoader`, optional): DataLoader for validation. test_interval (int, optional): Interval of validation - hooks (list[`BaseHook `_], - optional): A list of hooks used in training. + hooks (list[BaseHook], optional): A list of hooks used in training. display_progress (bool, optional): If True, a progress bar will be displayed. """ diff --git a/colossalai/utils/memory_tracer/async_memtracer.py b/colossalai/utils/memory_tracer/async_memtracer.py index 69295a715..e9bde5b78 100644 --- a/colossalai/utils/memory_tracer/async_memtracer.py +++ b/colossalai/utils/memory_tracer/async_memtracer.py @@ -21,21 +21,22 @@ class AsyncMemoryMonitor: :type power: int Usage: + :: - ```python - async_mem_monitor = AsyncMemoryMonitor() - input = torch.randn(2, 20).cuda() - OP1 = torch.nn.Linear(20, 30).cuda() - OP2 = torch.nn.Linear(30, 40).cuda() + ```python + async_mem_monitor = AsyncMemoryMonitor() + input = torch.randn(2, 20).cuda() + OP1 = torch.nn.Linear(20, 30).cuda() + OP2 = torch.nn.Linear(30, 40).cuda() - async_mem_monitor.start() - output = OP1(input) - async_mem_monitor.finish() - async_mem_monitor.start() - output = OP2(output) - async_mem_monitor.finish() - async_mem_monitor.save('log.pkl') - ``` + async_mem_monitor.start() + output = OP1(input) + async_mem_monitor.finish() + async_mem_monitor.start() + output = OP2(output) + async_mem_monitor.finish() + async_mem_monitor.save('log.pkl') + ``` """ def __init__(self, power: int = 10): diff --git a/colossalai/utils/profiler/prof_utils.py b/colossalai/utils/profiler/prof_utils.py index 641a514cf..a7e35bc42 100644 --- a/colossalai/utils/profiler/prof_utils.py +++ b/colossalai/utils/profiler/prof_utils.py @@ -73,25 +73,26 @@ class ProfilerContext(object): """ Profiler context manager Usage: + :: - ```python - world_size = 4 - inputs = torch.randn(10, 10, dtype=torch.float32, device=get_current_device()) - outputs = torch.empty(world_size, 10, 10, dtype=torch.float32, device=get_current_device()) - outputs_list = list(torch.chunk(outputs, chunks=world_size, dim=0)) + ```python + world_size = 4 + inputs = torch.randn(10, 10, dtype=torch.float32, device=get_current_device()) + outputs = torch.empty(world_size, 10, 10, dtype=torch.float32, device=get_current_device()) + outputs_list = list(torch.chunk(outputs, chunks=world_size, dim=0)) - cc_prof = CommProfiler() + cc_prof = CommProfiler() - with ProfilerContext([cc_prof]) as prof: - op = dist.all_reduce(inputs, async_op=True) - dist.all_gather(outputs_list, inputs) - op.wait() - dist.reduce_scatter(inputs, outputs_list) - dist.broadcast(inputs, 0) - dist.reduce(inputs, 0) + with ProfilerContext([cc_prof]) as prof: + op = dist.all_reduce(inputs, async_op=True) + dist.all_gather(outputs_list, inputs) + op.wait() + dist.reduce_scatter(inputs, outputs_list) + dist.broadcast(inputs, 0) + dist.reduce(inputs, 0) - prof.show() - ``` + prof.show() + ``` """ def __init__(self, profilers: List[BaseProfiler] = None, enable: bool = True): diff --git a/docs/colossalai/colossalai.amp.apex_amp.apex_amp.rst b/docs/colossalai/colossalai.amp.apex_amp.apex_amp.rst deleted file mode 100644 index cba7e0062..000000000 --- a/docs/colossalai/colossalai.amp.apex_amp.apex_amp.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.amp.apex\_amp.apex\_amp -================================== - -.. automodule:: colossalai.amp.apex_amp.apex_amp - :members: diff --git a/docs/colossalai/colossalai.amp.apex_amp.rst b/docs/colossalai/colossalai.amp.apex_amp.rst index 7116a538b..0352ef9a6 100644 --- a/docs/colossalai/colossalai.amp.apex_amp.rst +++ b/docs/colossalai/colossalai.amp.apex_amp.rst @@ -3,9 +3,3 @@ colossalai.amp.apex\_amp .. automodule:: colossalai.amp.apex_amp :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.amp.apex_amp.apex_amp diff --git a/docs/colossalai/colossalai.amp.naive_amp.grad_scaler.base_grad_scaler.rst b/docs/colossalai/colossalai.amp.naive_amp.grad_scaler.base_grad_scaler.rst deleted file mode 100644 index a796d4947..000000000 --- a/docs/colossalai/colossalai.amp.naive_amp.grad_scaler.base_grad_scaler.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.amp.naive\_amp.grad\_scaler.base\_grad\_scaler -========================================================= - -.. automodule:: colossalai.amp.naive_amp.grad_scaler.base_grad_scaler - :members: diff --git a/docs/colossalai/colossalai.amp.naive_amp.grad_scaler.constant_grad_scaler.rst b/docs/colossalai/colossalai.amp.naive_amp.grad_scaler.constant_grad_scaler.rst deleted file mode 100644 index b506ca06d..000000000 --- a/docs/colossalai/colossalai.amp.naive_amp.grad_scaler.constant_grad_scaler.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.amp.naive\_amp.grad\_scaler.constant\_grad\_scaler -============================================================= - -.. automodule:: colossalai.amp.naive_amp.grad_scaler.constant_grad_scaler - :members: diff --git a/docs/colossalai/colossalai.amp.naive_amp.grad_scaler.dynamic_grad_scaler.rst b/docs/colossalai/colossalai.amp.naive_amp.grad_scaler.dynamic_grad_scaler.rst deleted file mode 100644 index 731c1185b..000000000 --- a/docs/colossalai/colossalai.amp.naive_amp.grad_scaler.dynamic_grad_scaler.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.amp.naive\_amp.grad\_scaler.dynamic\_grad\_scaler -============================================================ - -.. automodule:: colossalai.amp.naive_amp.grad_scaler.dynamic_grad_scaler - :members: diff --git a/docs/colossalai/colossalai.amp.naive_amp.grad_scaler.rst b/docs/colossalai/colossalai.amp.naive_amp.grad_scaler.rst deleted file mode 100644 index 4587e2909..000000000 --- a/docs/colossalai/colossalai.amp.naive_amp.grad_scaler.rst +++ /dev/null @@ -1,13 +0,0 @@ -colossalai.amp.naive\_amp.grad\_scaler -====================================== - -.. automodule:: colossalai.amp.naive_amp.grad_scaler - :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.amp.naive_amp.grad_scaler.base_grad_scaler - colossalai.amp.naive_amp.grad_scaler.constant_grad_scaler - colossalai.amp.naive_amp.grad_scaler.dynamic_grad_scaler diff --git a/docs/colossalai/colossalai.amp.naive_amp.naive_amp.rst b/docs/colossalai/colossalai.amp.naive_amp.naive_amp.rst deleted file mode 100644 index e20f22b2e..000000000 --- a/docs/colossalai/colossalai.amp.naive_amp.naive_amp.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.amp.naive\_amp.naive\_amp -==================================== - -.. automodule:: colossalai.amp.naive_amp.naive_amp - :members: diff --git a/docs/colossalai/colossalai.amp.naive_amp.rst b/docs/colossalai/colossalai.amp.naive_amp.rst index fd364c053..fac0cc3db 100644 --- a/docs/colossalai/colossalai.amp.naive_amp.rst +++ b/docs/colossalai/colossalai.amp.naive_amp.rst @@ -3,14 +3,3 @@ colossalai.amp.naive\_amp .. automodule:: colossalai.amp.naive_amp :members: - -.. toctree:: - :maxdepth: 2 - - colossalai.amp.naive_amp.grad_scaler - - -.. toctree:: - :maxdepth: 2 - - colossalai.amp.naive_amp.naive_amp diff --git a/docs/colossalai/colossalai.amp.rst b/docs/colossalai/colossalai.amp.rst index 5ef4f36c1..ffea8bbc7 100644 --- a/docs/colossalai/colossalai.amp.rst +++ b/docs/colossalai/colossalai.amp.rst @@ -1,8 +1,7 @@ colossalai.amp ============== -.. automodule:: colossalai.amp - :members: +*This module contains different amp models and optimizers.* .. toctree:: :maxdepth: 2 @@ -16,3 +15,6 @@ colossalai.amp :maxdepth: 2 colossalai.amp.amp_type + +.. automodule:: colossalai.amp + :members: convert_to_amp diff --git a/docs/colossalai/colossalai.amp.torch_amp.rst b/docs/colossalai/colossalai.amp.torch_amp.rst index f10095f13..62fe4741c 100644 --- a/docs/colossalai/colossalai.amp.torch_amp.rst +++ b/docs/colossalai/colossalai.amp.torch_amp.rst @@ -2,10 +2,4 @@ colossalai.amp.torch\_amp ========================= .. automodule:: colossalai.amp.torch_amp - :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.amp.torch_amp.torch_amp + :members: \ No newline at end of file diff --git a/docs/colossalai/colossalai.amp.torch_amp.torch_amp.rst b/docs/colossalai/colossalai.amp.torch_amp.torch_amp.rst deleted file mode 100644 index 5f1549cb8..000000000 --- a/docs/colossalai/colossalai.amp.torch_amp.torch_amp.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.amp.torch\_amp.torch\_amp -==================================== - -.. automodule:: colossalai.amp.torch_amp.torch_amp - :members: diff --git a/docs/colossalai/colossalai.builder.rst b/docs/colossalai/colossalai.builder.rst index 60b8501c8..c6641f742 100644 --- a/docs/colossalai/colossalai.builder.rst +++ b/docs/colossalai/colossalai.builder.rst @@ -1,9 +1,8 @@ colossalai.builder ================== -.. automodule:: colossalai.builder - :members: - +*This module contains different methods to build* ``schedule``, ``model``, ``optimizer``, +``layers`` *and other important classes.* .. toctree:: :maxdepth: 2 diff --git a/docs/colossalai/colossalai.communication.rst b/docs/colossalai/colossalai.communication.rst index 5086fa663..150c18687 100644 --- a/docs/colossalai/colossalai.communication.rst +++ b/docs/colossalai/colossalai.communication.rst @@ -1,9 +1,7 @@ colossalai.communication ======================== -.. automodule:: colossalai.communication - :members: - +*This module contains different methods for communication in parallel training.* .. toctree:: :maxdepth: 2 diff --git a/docs/colossalai/colossalai.constants.rst b/docs/colossalai/colossalai.constants.rst deleted file mode 100644 index 330b3e866..000000000 --- a/docs/colossalai/colossalai.constants.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.constants -==================== - -.. automodule:: colossalai.constants - :members: diff --git a/docs/colossalai/colossalai.context.moe_context.rst b/docs/colossalai/colossalai.context.moe_context.rst index 9027d19ff..b652c3df4 100644 --- a/docs/colossalai/colossalai.context.moe_context.rst +++ b/docs/colossalai/colossalai.context.moe_context.rst @@ -1,5 +1,5 @@ colossalai.context.moe\_context -=============================== +==================================== .. automodule:: colossalai.context.moe_context - :members: + :members: \ No newline at end of file diff --git a/docs/colossalai/colossalai.context.process_group_initializer.initializer_2d.rst b/docs/colossalai/colossalai.context.process_group_initializer.initializer_2d.rst index d99a2e1c3..883fb03dc 100644 --- a/docs/colossalai/colossalai.context.process_group_initializer.initializer_2d.rst +++ b/docs/colossalai/colossalai.context.process_group_initializer.initializer_2d.rst @@ -2,4 +2,4 @@ colossalai.context.process\_group\_initializer.initializer\_2d ============================================================== .. automodule:: colossalai.context.process_group_initializer.initializer_2d - :members: + :members: Initializer_2D_Row, Initializer_2D_Col diff --git a/docs/colossalai/colossalai.context.process_group_initializer.initializer_2p5d.rst b/docs/colossalai/colossalai.context.process_group_initializer.initializer_2p5d.rst index 73d80e443..9fcce069c 100644 --- a/docs/colossalai/colossalai.context.process_group_initializer.initializer_2p5d.rst +++ b/docs/colossalai/colossalai.context.process_group_initializer.initializer_2p5d.rst @@ -2,4 +2,4 @@ colossalai.context.process\_group\_initializer.initializer\_2p5d ================================================================ .. automodule:: colossalai.context.process_group_initializer.initializer_2p5d - :members: + :members: Initializer_2p5D_ROW, Initializer_2p5D_Col, Initializer_2p5D_Dep, Initializer_2p5D_XZ diff --git a/docs/colossalai/colossalai.context.process_group_initializer.initializer_3d.rst b/docs/colossalai/colossalai.context.process_group_initializer.initializer_3d.rst index 5cfba5ce0..e81678038 100644 --- a/docs/colossalai/colossalai.context.process_group_initializer.initializer_3d.rst +++ b/docs/colossalai/colossalai.context.process_group_initializer.initializer_3d.rst @@ -2,4 +2,4 @@ colossalai.context.process\_group\_initializer.initializer\_3d ============================================================== .. automodule:: colossalai.context.process_group_initializer.initializer_3d - :members: + :members: Initializer_3D_Input, Initializer_3D_Weight, Initializer_3D_Output diff --git a/docs/colossalai/colossalai.context.process_group_initializer.initializer_sequence.rst b/docs/colossalai/colossalai.context.process_group_initializer.initializer_sequence.rst index dab71cc3c..b4bc76bfa 100644 --- a/docs/colossalai/colossalai.context.process_group_initializer.initializer_sequence.rst +++ b/docs/colossalai/colossalai.context.process_group_initializer.initializer_sequence.rst @@ -2,4 +2,4 @@ colossalai.context.process\_group\_initializer.initializer\_sequence ==================================================================== .. automodule:: colossalai.context.process_group_initializer.initializer_sequence - :members: + :members: Initializer_Sequence_DP diff --git a/docs/colossalai/colossalai.context.process_group_initializer.rst b/docs/colossalai/colossalai.context.process_group_initializer.rst index 519337e9c..b5e261195 100644 --- a/docs/colossalai/colossalai.context.process_group_initializer.rst +++ b/docs/colossalai/colossalai.context.process_group_initializer.rst @@ -14,6 +14,7 @@ colossalai.context.process\_group\_initializer colossalai.context.process_group_initializer.initializer_3d colossalai.context.process_group_initializer.initializer_data colossalai.context.process_group_initializer.initializer_model + colossalai.context.process_group_initializer.initializer_moe colossalai.context.process_group_initializer.initializer_pipeline colossalai.context.process_group_initializer.initializer_sequence colossalai.context.process_group_initializer.initializer_tensor diff --git a/docs/colossalai/colossalai.context.rst b/docs/colossalai/colossalai.context.rst index 102a9e02e..b1311cab8 100644 --- a/docs/colossalai/colossalai.context.rst +++ b/docs/colossalai/colossalai.context.rst @@ -1,8 +1,8 @@ colossalai.context ================== -.. automodule:: colossalai.context - :members: +*This module is serving for receiving and passing user's configuration to all devices to +initialize and construct parallel training.* .. toctree:: :maxdepth: 2 @@ -15,7 +15,9 @@ colossalai.context :maxdepth: 2 colossalai.context.config - colossalai.context.moe_context colossalai.context.parallel_context + colossalai.context.moe_context colossalai.context.parallel_mode - colossalai.context.singleton_meta + +.. automodule:: colossalai.context + :members: diff --git a/docs/colossalai/colossalai.context.singleton_meta.rst b/docs/colossalai/colossalai.context.singleton_meta.rst deleted file mode 100644 index ae4ceb314..000000000 --- a/docs/colossalai/colossalai.context.singleton_meta.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.context.singleton\_meta -================================== - -.. automodule:: colossalai.context.singleton_meta - :members: diff --git a/docs/colossalai/colossalai.core.rst b/docs/colossalai/colossalai.core.rst deleted file mode 100644 index d9ddb76ed..000000000 --- a/docs/colossalai/colossalai.core.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.core -=============== - -.. automodule:: colossalai.core - :members: diff --git a/docs/colossalai/colossalai.engine.gradient_handler.rst b/docs/colossalai/colossalai.engine.gradient_handler.rst index 27eb2b56a..d7d1633a6 100644 --- a/docs/colossalai/colossalai.engine.gradient_handler.rst +++ b/docs/colossalai/colossalai.engine.gradient_handler.rst @@ -3,9 +3,3 @@ colossalai.engine.gradient\_handler .. automodule:: colossalai.engine.gradient_handler :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.engine.gradient_handler.utils diff --git a/docs/colossalai/colossalai.engine.gradient_handler.utils.rst b/docs/colossalai/colossalai.engine.gradient_handler.utils.rst deleted file mode 100644 index c8997e135..000000000 --- a/docs/colossalai/colossalai.engine.gradient_handler.utils.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.engine.gradient\_handler.utils -========================================= - -.. automodule:: colossalai.engine.gradient_handler.utils - :members: diff --git a/docs/colossalai/colossalai.engine.ophooks.rst b/docs/colossalai/colossalai.engine.ophooks.rst index f4b8a8396..a43ca7a6f 100644 --- a/docs/colossalai/colossalai.engine.ophooks.rst +++ b/docs/colossalai/colossalai.engine.ophooks.rst @@ -1,11 +1,5 @@ colossalai.engine.ophooks -========================= +=========================== .. automodule:: colossalai.engine.ophooks :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.engine.ophooks.zero_hook diff --git a/docs/colossalai/colossalai.engine.ophooks.zero_hook.rst b/docs/colossalai/colossalai.engine.ophooks.zero_hook.rst deleted file mode 100644 index 270d1839c..000000000 --- a/docs/colossalai/colossalai.engine.ophooks.zero_hook.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.engine.ophooks.zero\_hook -==================================== - -.. automodule:: colossalai.engine.ophooks.zero_hook - :members: diff --git a/docs/colossalai/colossalai.engine.paramhooks.rst b/docs/colossalai/colossalai.engine.paramhooks.rst index 2164bd91d..663f26ec6 100644 --- a/docs/colossalai/colossalai.engine.paramhooks.rst +++ b/docs/colossalai/colossalai.engine.paramhooks.rst @@ -1,5 +1,5 @@ colossalai.engine.paramhooks -============================ +============================== .. automodule:: colossalai.engine.paramhooks :members: diff --git a/docs/colossalai/colossalai.engine.rst b/docs/colossalai/colossalai.engine.rst index eaa8b03ea..87db7152f 100644 --- a/docs/colossalai/colossalai.engine.rst +++ b/docs/colossalai/colossalai.engine.rst @@ -8,6 +8,6 @@ colossalai.engine :maxdepth: 2 colossalai.engine.gradient_handler + colossalai.engine.schedule colossalai.engine.ophooks colossalai.engine.paramhooks - colossalai.engine.schedule diff --git a/docs/colossalai/colossalai.global_variables.rst b/docs/colossalai/colossalai.global_variables.rst deleted file mode 100644 index 1900c8835..000000000 --- a/docs/colossalai/colossalai.global_variables.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.global\_variables -============================ - -.. automodule:: colossalai.global_variables - :members: diff --git a/docs/colossalai/colossalai.logging.logger.rst b/docs/colossalai/colossalai.logging.logger.rst deleted file mode 100644 index 047deb8a1..000000000 --- a/docs/colossalai/colossalai.logging.logger.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.logging.logger -========================= - -.. automodule:: colossalai.logging.logger - :members: diff --git a/docs/colossalai/colossalai.logging.rst b/docs/colossalai/colossalai.logging.rst index bc593fc81..65492bbb7 100644 --- a/docs/colossalai/colossalai.logging.rst +++ b/docs/colossalai/colossalai.logging.rst @@ -3,9 +3,3 @@ colossalai.logging .. automodule:: colossalai.logging :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.logging.logger diff --git a/docs/colossalai/colossalai.nn.layer.colossalai_layer.dropout.rst b/docs/colossalai/colossalai.nn.layer.colossalai_layer.dropout.rst deleted file mode 100644 index ec1dfd395..000000000 --- a/docs/colossalai/colossalai.nn.layer.colossalai_layer.dropout.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.layer.colossalai\_layer.dropout -============================================= - -.. automodule:: colossalai.nn.layer.colossalai_layer.dropout - :members: diff --git a/docs/colossalai/colossalai.nn.layer.colossalai_layer.embedding.rst b/docs/colossalai/colossalai.nn.layer.colossalai_layer.embedding.rst deleted file mode 100644 index 8438b3a07..000000000 --- a/docs/colossalai/colossalai.nn.layer.colossalai_layer.embedding.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.layer.colossalai\_layer.embedding -=============================================== - -.. automodule:: colossalai.nn.layer.colossalai_layer.embedding - :members: diff --git a/docs/colossalai/colossalai.nn.layer.colossalai_layer.linear.rst b/docs/colossalai/colossalai.nn.layer.colossalai_layer.linear.rst deleted file mode 100644 index 321328254..000000000 --- a/docs/colossalai/colossalai.nn.layer.colossalai_layer.linear.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.layer.colossalai\_layer.linear -============================================ - -.. automodule:: colossalai.nn.layer.colossalai_layer.linear - :members: diff --git a/docs/colossalai/colossalai.nn.layer.colossalai_layer.normalization.rst b/docs/colossalai/colossalai.nn.layer.colossalai_layer.normalization.rst deleted file mode 100644 index f94dd27b8..000000000 --- a/docs/colossalai/colossalai.nn.layer.colossalai_layer.normalization.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.layer.colossalai\_layer.normalization -=================================================== - -.. automodule:: colossalai.nn.layer.colossalai_layer.normalization - :members: diff --git a/docs/colossalai/colossalai.nn.layer.colossalai_layer.rst b/docs/colossalai/colossalai.nn.layer.colossalai_layer.rst index 0f685e6c2..ae5af4c0e 100644 --- a/docs/colossalai/colossalai.nn.layer.colossalai_layer.rst +++ b/docs/colossalai/colossalai.nn.layer.colossalai_layer.rst @@ -3,12 +3,3 @@ colossalai.nn.layer.colossalai\_layer .. automodule:: colossalai.nn.layer.colossalai_layer :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.nn.layer.colossalai_layer.dropout - colossalai.nn.layer.colossalai_layer.embedding - colossalai.nn.layer.colossalai_layer.linear - colossalai.nn.layer.colossalai_layer.normalization diff --git a/docs/colossalai/colossalai.nn.layer.moe.experts.rst b/docs/colossalai/colossalai.nn.layer.moe.experts.rst deleted file mode 100644 index c05e763d5..000000000 --- a/docs/colossalai/colossalai.nn.layer.moe.experts.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.layer.moe.experts -=============================== - -.. automodule:: colossalai.nn.layer.moe.experts - :members: diff --git a/docs/colossalai/colossalai.nn.layer.moe.layers.rst b/docs/colossalai/colossalai.nn.layer.moe.layers.rst deleted file mode 100644 index d109d47b8..000000000 --- a/docs/colossalai/colossalai.nn.layer.moe.layers.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.layer.moe.layers -============================== - -.. automodule:: colossalai.nn.layer.moe.layers - :members: diff --git a/docs/colossalai/colossalai.nn.layer.moe.rst b/docs/colossalai/colossalai.nn.layer.moe.rst index f3106b98d..d13eb3649 100644 --- a/docs/colossalai/colossalai.nn.layer.moe.rst +++ b/docs/colossalai/colossalai.nn.layer.moe.rst @@ -3,11 +3,3 @@ colossalai.nn.layer.moe .. automodule:: colossalai.nn.layer.moe :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.nn.layer.moe.experts - colossalai.nn.layer.moe.layers - colossalai.nn.layer.moe.utils diff --git a/docs/colossalai/colossalai.nn.layer.moe.utils.rst b/docs/colossalai/colossalai.nn.layer.moe.utils.rst deleted file mode 100644 index fc085d136..000000000 --- a/docs/colossalai/colossalai.nn.layer.moe.utils.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.layer.moe.utils -============================= - -.. automodule:: colossalai.nn.layer.moe.utils - :members: diff --git a/docs/colossalai/colossalai.nn.layer.parallel_1d.layers.rst b/docs/colossalai/colossalai.nn.layer.parallel_1d.layers.rst deleted file mode 100644 index 380f6bf8d..000000000 --- a/docs/colossalai/colossalai.nn.layer.parallel_1d.layers.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.layer.parallel\_1d.layers -======================================= - -.. automodule:: colossalai.nn.layer.parallel_1d.layers - :members: diff --git a/docs/colossalai/colossalai.nn.layer.parallel_1d.rst b/docs/colossalai/colossalai.nn.layer.parallel_1d.rst index 3a8ed6206..e487563c9 100644 --- a/docs/colossalai/colossalai.nn.layer.parallel_1d.rst +++ b/docs/colossalai/colossalai.nn.layer.parallel_1d.rst @@ -3,9 +3,3 @@ colossalai.nn.layer.parallel\_1d .. automodule:: colossalai.nn.layer.parallel_1d :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.nn.layer.parallel_1d.layers diff --git a/docs/colossalai/colossalai.nn.layer.parallel_2d.layers.rst b/docs/colossalai/colossalai.nn.layer.parallel_2d.layers.rst deleted file mode 100644 index b64d402bd..000000000 --- a/docs/colossalai/colossalai.nn.layer.parallel_2d.layers.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.layer.parallel\_2d.layers -======================================= - -.. automodule:: colossalai.nn.layer.parallel_2d.layers - :members: diff --git a/docs/colossalai/colossalai.nn.layer.parallel_2d.rst b/docs/colossalai/colossalai.nn.layer.parallel_2d.rst index f5ad41a1b..b2609ad42 100644 --- a/docs/colossalai/colossalai.nn.layer.parallel_2d.rst +++ b/docs/colossalai/colossalai.nn.layer.parallel_2d.rst @@ -2,10 +2,4 @@ colossalai.nn.layer.parallel\_2d ================================ .. automodule:: colossalai.nn.layer.parallel_2d - :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.nn.layer.parallel_2d.layers + :members: \ No newline at end of file diff --git a/docs/colossalai/colossalai.nn.layer.parallel_2p5d.layers.rst b/docs/colossalai/colossalai.nn.layer.parallel_2p5d.layers.rst deleted file mode 100644 index ebc99d56c..000000000 --- a/docs/colossalai/colossalai.nn.layer.parallel_2p5d.layers.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.layer.parallel\_2p5d.layers -========================================= - -.. automodule:: colossalai.nn.layer.parallel_2p5d.layers - :members: diff --git a/docs/colossalai/colossalai.nn.layer.parallel_2p5d.rst b/docs/colossalai/colossalai.nn.layer.parallel_2p5d.rst index 5869bdee9..9a27f93e9 100644 --- a/docs/colossalai/colossalai.nn.layer.parallel_2p5d.rst +++ b/docs/colossalai/colossalai.nn.layer.parallel_2p5d.rst @@ -3,9 +3,3 @@ colossalai.nn.layer.parallel\_2p5d .. automodule:: colossalai.nn.layer.parallel_2p5d :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.nn.layer.parallel_2p5d.layers diff --git a/docs/colossalai/colossalai.nn.layer.parallel_3d.layers.rst b/docs/colossalai/colossalai.nn.layer.parallel_3d.layers.rst deleted file mode 100644 index a1702f1fc..000000000 --- a/docs/colossalai/colossalai.nn.layer.parallel_3d.layers.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.layer.parallel\_3d.layers -======================================= - -.. automodule:: colossalai.nn.layer.parallel_3d.layers - :members: diff --git a/docs/colossalai/colossalai.nn.layer.parallel_3d.rst b/docs/colossalai/colossalai.nn.layer.parallel_3d.rst index bb55a63e5..37af0c1e7 100644 --- a/docs/colossalai/colossalai.nn.layer.parallel_3d.rst +++ b/docs/colossalai/colossalai.nn.layer.parallel_3d.rst @@ -3,9 +3,3 @@ colossalai.nn.layer.parallel\_3d .. automodule:: colossalai.nn.layer.parallel_3d :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.nn.layer.parallel_3d.layers diff --git a/docs/colossalai/colossalai.nn.layer.parallel_sequence.layers.rst b/docs/colossalai/colossalai.nn.layer.parallel_sequence.layers.rst deleted file mode 100644 index 54929d2e7..000000000 --- a/docs/colossalai/colossalai.nn.layer.parallel_sequence.layers.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.layer.parallel\_sequence.layers -============================================= - -.. automodule:: colossalai.nn.layer.parallel_sequence.layers - :members: diff --git a/docs/colossalai/colossalai.nn.layer.parallel_sequence.rst b/docs/colossalai/colossalai.nn.layer.parallel_sequence.rst index 24e8941d4..9b252d459 100644 --- a/docs/colossalai/colossalai.nn.layer.parallel_sequence.rst +++ b/docs/colossalai/colossalai.nn.layer.parallel_sequence.rst @@ -3,9 +3,3 @@ colossalai.nn.layer.parallel\_sequence .. automodule:: colossalai.nn.layer.parallel_sequence :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.nn.layer.parallel_sequence.layers diff --git a/docs/colossalai/colossalai.nn.layer.utils.common.rst b/docs/colossalai/colossalai.nn.layer.utils.common.rst deleted file mode 100644 index 6a552830f..000000000 --- a/docs/colossalai/colossalai.nn.layer.utils.common.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.layer.utils.common -================================ - -.. automodule:: colossalai.nn.layer.utils.common - :members: diff --git a/docs/colossalai/colossalai.nn.layer.utils.rst b/docs/colossalai/colossalai.nn.layer.utils.rst index 16c3d7182..f241396a4 100644 --- a/docs/colossalai/colossalai.nn.layer.utils.rst +++ b/docs/colossalai/colossalai.nn.layer.utils.rst @@ -3,9 +3,3 @@ colossalai.nn.layer.utils .. automodule:: colossalai.nn.layer.utils :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.nn.layer.utils.common diff --git a/docs/colossalai/colossalai.nn.layer.vanilla.layers.rst b/docs/colossalai/colossalai.nn.layer.vanilla.layers.rst deleted file mode 100644 index f993b1f50..000000000 --- a/docs/colossalai/colossalai.nn.layer.vanilla.layers.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.layer.vanilla.layers -================================== - -.. automodule:: colossalai.nn.layer.vanilla.layers - :members: diff --git a/docs/colossalai/colossalai.nn.layer.vanilla.rst b/docs/colossalai/colossalai.nn.layer.vanilla.rst index fe1ea5c6c..1567788d5 100644 --- a/docs/colossalai/colossalai.nn.layer.vanilla.rst +++ b/docs/colossalai/colossalai.nn.layer.vanilla.rst @@ -3,9 +3,3 @@ colossalai.nn.layer.vanilla .. automodule:: colossalai.nn.layer.vanilla :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.nn.layer.vanilla.layers diff --git a/docs/colossalai/colossalai.nn.layer.wrapper.lambda_wrapper.rst b/docs/colossalai/colossalai.nn.layer.wrapper.lambda_wrapper.rst deleted file mode 100644 index f2ced6725..000000000 --- a/docs/colossalai/colossalai.nn.layer.wrapper.lambda_wrapper.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.layer.wrapper.lambda\_wrapper -=========================================== - -.. automodule:: colossalai.nn.layer.wrapper.lambda_wrapper - :members: diff --git a/docs/colossalai/colossalai.nn.layer.wrapper.pipeline_wrapper.rst b/docs/colossalai/colossalai.nn.layer.wrapper.pipeline_wrapper.rst deleted file mode 100644 index e5648873d..000000000 --- a/docs/colossalai/colossalai.nn.layer.wrapper.pipeline_wrapper.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.nn.layer.wrapper.pipeline\_wrapper -============================================= - -.. automodule:: colossalai.nn.layer.wrapper.pipeline_wrapper - :members: diff --git a/docs/colossalai/colossalai.nn.layer.wrapper.rst b/docs/colossalai/colossalai.nn.layer.wrapper.rst index 4e66651dc..0508c46ed 100644 --- a/docs/colossalai/colossalai.nn.layer.wrapper.rst +++ b/docs/colossalai/colossalai.nn.layer.wrapper.rst @@ -3,10 +3,3 @@ colossalai.nn.layer.wrapper .. automodule:: colossalai.nn.layer.wrapper :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.nn.layer.wrapper.lambda_wrapper - colossalai.nn.layer.wrapper.pipeline_wrapper diff --git a/docs/colossalai/colossalai.nn.lr_scheduler.rst b/docs/colossalai/colossalai.nn.lr_scheduler.rst index 427a3ee45..a6d5708c1 100644 --- a/docs/colossalai/colossalai.nn.lr_scheduler.rst +++ b/docs/colossalai/colossalai.nn.lr_scheduler.rst @@ -1,10 +1,6 @@ colossalai.nn.lr\_scheduler =========================== -.. automodule:: colossalai.nn.lr_scheduler - :members: - - .. toctree:: :maxdepth: 2 diff --git a/docs/colossalai/colossalai.nn.optimizer.rst b/docs/colossalai/colossalai.nn.optimizer.rst index d0063179d..6939c85d7 100644 --- a/docs/colossalai/colossalai.nn.optimizer.rst +++ b/docs/colossalai/colossalai.nn.optimizer.rst @@ -1,10 +1,6 @@ colossalai.nn.optimizer ======================= -.. automodule:: colossalai.nn.optimizer - :members: - - .. toctree:: :maxdepth: 2 diff --git a/docs/colossalai/colossalai.nn.rst b/docs/colossalai/colossalai.nn.rst index 32e5eae2f..6167ce5ba 100644 --- a/docs/colossalai/colossalai.nn.rst +++ b/docs/colossalai/colossalai.nn.rst @@ -1,21 +1,57 @@ colossalai.nn ============= -.. automodule:: colossalai.nn - :members: +*This part contains different colossalai layers for constructing your model. +You can easily use them as the way of using layers in torch.nn.* + +*Now colossalai support layer types below:* ``Linear``, ``Classifier``, ``Embedding``, +``PatchEmbedding``, ``LayerNorm``, ``Dropout`` *for different parallelisms.* .. toctree:: :maxdepth: 2 colossalai.nn.layer + +*This part contains different loss functions for different parallelisms.* + +.. toctree:: + :maxdepth: 2 + colossalai.nn.loss + +*This part contains different learning rate schedules to control your learning rate +in training process* + +.. toctree:: + :maxdepth: 2 + colossalai.nn.lr_scheduler + +*This part contains different metric to measure performance of your model.* + +.. toctree:: + :maxdepth: 2 + colossalai.nn.metric + +*This part contains some colossalai optimizers* + +.. toctree:: + :maxdepth: 2 + colossalai.nn.model + +.. toctree:: + :maxdepth: 2 + colossalai.nn.optimizer +*This part contains different methods to initialize weights.* .. toctree:: :maxdepth: 2 colossalai.nn.init + +.. automodule:: colossalai.nn + :members: diff --git a/docs/colossalai/colossalai.rst b/docs/colossalai/colossalai.rst index b86475b9e..08371f580 100644 --- a/docs/colossalai/colossalai.rst +++ b/docs/colossalai/colossalai.rst @@ -1,9 +1,6 @@ -colossalai +Colossal-AI ========== -.. automodule:: colossalai - :members: - .. toctree:: :maxdepth: 2 @@ -25,5 +22,7 @@ colossalai .. toctree:: :maxdepth: 2 - colossalai.global_variables colossalai.initialize + +.. automodule:: colossalai + :members: diff --git a/docs/colossalai/colossalai.testing.comparison.rst b/docs/colossalai/colossalai.testing.comparison.rst deleted file mode 100644 index bcfdf0598..000000000 --- a/docs/colossalai/colossalai.testing.comparison.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.testing.comparison -============================= - -.. automodule:: colossalai.testing.comparison - :members: diff --git a/docs/colossalai/colossalai.testing.rst b/docs/colossalai/colossalai.testing.rst index 1127aa52c..e35e82353 100644 --- a/docs/colossalai/colossalai.testing.rst +++ b/docs/colossalai/colossalai.testing.rst @@ -1,12 +1,5 @@ colossalai.testing -================== +======================== .. automodule:: colossalai.testing - :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.testing.comparison - colossalai.testing.utils + :members: \ No newline at end of file diff --git a/docs/colossalai/colossalai.testing.utils.rst b/docs/colossalai/colossalai.testing.utils.rst deleted file mode 100644 index d8c2edcce..000000000 --- a/docs/colossalai/colossalai.testing.utils.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.testing.utils -======================== - -.. automodule:: colossalai.testing.utils - :members: diff --git a/docs/colossalai/colossalai.utils.data_sampler.base_sampler.rst b/docs/colossalai/colossalai.utils.data_sampler.base_sampler.rst deleted file mode 100644 index 199e8fcf8..000000000 --- a/docs/colossalai/colossalai.utils.data_sampler.base_sampler.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.utils.data\_sampler.base\_sampler -============================================ - -.. automodule:: colossalai.utils.data_sampler.base_sampler - :members: diff --git a/docs/colossalai/colossalai.utils.data_sampler.data_parallel_sampler.rst b/docs/colossalai/colossalai.utils.data_sampler.data_parallel_sampler.rst deleted file mode 100644 index 85e1b121c..000000000 --- a/docs/colossalai/colossalai.utils.data_sampler.data_parallel_sampler.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.utils.data\_sampler.data\_parallel\_sampler -====================================================== - -.. automodule:: colossalai.utils.data_sampler.data_parallel_sampler - :members: diff --git a/docs/colossalai/colossalai.utils.data_sampler.rst b/docs/colossalai/colossalai.utils.data_sampler.rst index 61dde070b..b822aaa0b 100644 --- a/docs/colossalai/colossalai.utils.data_sampler.rst +++ b/docs/colossalai/colossalai.utils.data_sampler.rst @@ -3,10 +3,3 @@ colossalai.utils.data\_sampler .. automodule:: colossalai.utils.data_sampler :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.utils.data_sampler.base_sampler - colossalai.utils.data_sampler.data_parallel_sampler diff --git a/docs/colossalai/colossalai.utils.memory_tracer.async_memtracer.rst b/docs/colossalai/colossalai.utils.memory_tracer.async_memtracer.rst deleted file mode 100644 index 789d28aaf..000000000 --- a/docs/colossalai/colossalai.utils.memory_tracer.async_memtracer.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.utils.memory\_tracer.async\_memtracer -================================================ - -.. automodule:: colossalai.utils.memory_tracer.async_memtracer - :members: diff --git a/docs/colossalai/colossalai.utils.memory_tracer.memstats_collector.rst b/docs/colossalai/colossalai.utils.memory_tracer.memstats_collector.rst deleted file mode 100644 index b078807f9..000000000 --- a/docs/colossalai/colossalai.utils.memory_tracer.memstats_collector.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.utils.memory\_tracer.memstats\_collector -=================================================== - -.. automodule:: colossalai.utils.memory_tracer.memstats_collector - :members: diff --git a/docs/colossalai/colossalai.utils.memory_tracer.model_data_memtracer.rst b/docs/colossalai/colossalai.utils.memory_tracer.model_data_memtracer.rst deleted file mode 100644 index 51a7a986c..000000000 --- a/docs/colossalai/colossalai.utils.memory_tracer.model_data_memtracer.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.utils.memory\_tracer.model\_data\_memtracer -====================================================== - -.. automodule:: colossalai.utils.memory_tracer.model_data_memtracer - :members: diff --git a/docs/colossalai/colossalai.utils.memory_tracer.rst b/docs/colossalai/colossalai.utils.memory_tracer.rst index 1caf85aec..b67c2684d 100644 --- a/docs/colossalai/colossalai.utils.memory_tracer.rst +++ b/docs/colossalai/colossalai.utils.memory_tracer.rst @@ -1,15 +1,5 @@ colossalai.utils.memory\_tracer -=============================== +======================= .. automodule:: colossalai.utils.memory_tracer - :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.utils.memory_tracer.async_memtracer - colossalai.utils.memory_tracer.memstats_collector - colossalai.utils.memory_tracer.model_data_memtracer - colossalai.utils.memory_tracer.test_async_memtracer - colossalai.utils.memory_tracer.test_memstats_collector + :members: \ No newline at end of file diff --git a/docs/colossalai/colossalai.utils.memory_tracer.test_async_memtracer.rst b/docs/colossalai/colossalai.utils.memory_tracer.test_async_memtracer.rst deleted file mode 100644 index e9c379f33..000000000 --- a/docs/colossalai/colossalai.utils.memory_tracer.test_async_memtracer.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.utils.memory\_tracer.test\_async\_memtracer -====================================================== - -.. automodule:: colossalai.utils.memory_tracer.test_async_memtracer - :members: diff --git a/docs/colossalai/colossalai.utils.memory_tracer.test_memstats_collector.rst b/docs/colossalai/colossalai.utils.memory_tracer.test_memstats_collector.rst deleted file mode 100644 index 10681e18d..000000000 --- a/docs/colossalai/colossalai.utils.memory_tracer.test_memstats_collector.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.utils.memory\_tracer.test\_memstats\_collector -========================================================= - -.. automodule:: colossalai.utils.memory_tracer.test_memstats_collector - :members: diff --git a/docs/colossalai/colossalai.utils.memory_utils.bucket_tensor_copy.rst b/docs/colossalai/colossalai.utils.memory_utils.bucket_tensor_copy.rst deleted file mode 100644 index cccd52bb3..000000000 --- a/docs/colossalai/colossalai.utils.memory_utils.bucket_tensor_copy.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.utils.memory\_utils.bucket\_tensor\_copy -=================================================== - -.. automodule:: colossalai.utils.memory_utils.bucket_tensor_copy - :members: diff --git a/docs/colossalai/colossalai.utils.memory_utils.memory_monitor.rst b/docs/colossalai/colossalai.utils.memory_utils.memory_monitor.rst deleted file mode 100644 index 4b6d805ae..000000000 --- a/docs/colossalai/colossalai.utils.memory_utils.memory_monitor.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.utils.memory\_utils.memory\_monitor -============================================== - -.. automodule:: colossalai.utils.memory_utils.memory_monitor - :members: diff --git a/docs/colossalai/colossalai.utils.memory_utils.rst b/docs/colossalai/colossalai.utils.memory_utils.rst deleted file mode 100644 index 0087116e0..000000000 --- a/docs/colossalai/colossalai.utils.memory_utils.rst +++ /dev/null @@ -1,13 +0,0 @@ -colossalai.utils.memory\_utils -============================== - -.. automodule:: colossalai.utils.memory_utils - :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.utils.memory_utils.bucket_tensor_copy - colossalai.utils.memory_utils.memory_monitor - colossalai.utils.memory_utils.utils diff --git a/docs/colossalai/colossalai.utils.memory_utils.utils.rst b/docs/colossalai/colossalai.utils.memory_utils.utils.rst deleted file mode 100644 index a7bc1c607..000000000 --- a/docs/colossalai/colossalai.utils.memory_utils.utils.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.utils.memory\_utils.utils -==================================== - -.. automodule:: colossalai.utils.memory_utils.utils - :members: diff --git a/docs/colossalai/colossalai.utils.moe.rst b/docs/colossalai/colossalai.utils.moe.rst index b66ccdc8e..aa6e66367 100644 --- a/docs/colossalai/colossalai.utils.moe.rst +++ b/docs/colossalai/colossalai.utils.moe.rst @@ -1,5 +1,5 @@ colossalai.utils.moe -==================== +======================= .. automodule:: colossalai.utils.moe - :members: + :members: \ No newline at end of file diff --git a/docs/colossalai/colossalai.utils.profiler.comm_profiler.rst b/docs/colossalai/colossalai.utils.profiler.comm_profiler.rst deleted file mode 100644 index 88532eff2..000000000 --- a/docs/colossalai/colossalai.utils.profiler.comm_profiler.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.utils.profiler.comm\_profiler -======================================== - -.. automodule:: colossalai.utils.profiler.comm_profiler - :members: diff --git a/docs/colossalai/colossalai.utils.profiler.pcie_profiler.rst b/docs/colossalai/colossalai.utils.profiler.pcie_profiler.rst deleted file mode 100644 index fd2f4ea15..000000000 --- a/docs/colossalai/colossalai.utils.profiler.pcie_profiler.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.utils.profiler.pcie\_profiler -======================================== - -.. automodule:: colossalai.utils.profiler.pcie_profiler - :members: diff --git a/docs/colossalai/colossalai.utils.profiler.prof_utils.rst b/docs/colossalai/colossalai.utils.profiler.prof_utils.rst deleted file mode 100644 index 9b34a8505..000000000 --- a/docs/colossalai/colossalai.utils.profiler.prof_utils.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.utils.profiler.prof\_utils -===================================== - -.. automodule:: colossalai.utils.profiler.prof_utils - :members: diff --git a/docs/colossalai/colossalai.utils.profiler.rst b/docs/colossalai/colossalai.utils.profiler.rst index 1035dbced..d28ffe2bc 100644 --- a/docs/colossalai/colossalai.utils.profiler.rst +++ b/docs/colossalai/colossalai.utils.profiler.rst @@ -1,13 +1,5 @@ colossalai.utils.profiler -========================= +======================= .. automodule:: colossalai.utils.profiler - :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.utils.profiler.comm_profiler - colossalai.utils.profiler.pcie_profiler - colossalai.utils.profiler.prof_utils + :members: \ No newline at end of file diff --git a/docs/colossalai/colossalai.utils.rst b/docs/colossalai/colossalai.utils.rst index f6a74fe4e..92ac3222a 100644 --- a/docs/colossalai/colossalai.utils.rst +++ b/docs/colossalai/colossalai.utils.rst @@ -1,8 +1,7 @@ colossalai.utils ================ -.. automodule:: colossalai.utils - :members: +*This part contains useful utils for colossalai* .. toctree:: :maxdepth: 2 @@ -10,9 +9,8 @@ colossalai.utils colossalai.utils.data_sampler colossalai.utils.gradient_accumulation colossalai.utils.memory_tracer - colossalai.utils.memory_utils colossalai.utils.multi_tensor_apply - colossalai.utils.profiler + colossalai.utils.profile colossalai.utils.tensor_detector @@ -23,5 +21,6 @@ colossalai.utils colossalai.utils.checkpointing colossalai.utils.common colossalai.utils.cuda + colossalai.utils.memory colossalai.utils.moe colossalai.utils.timer diff --git a/docs/colossalai/colossalai.utils.tensor_detector.rst b/docs/colossalai/colossalai.utils.tensor_detector.rst index 807d67e3a..2708b5011 100644 --- a/docs/colossalai/colossalai.utils.tensor_detector.rst +++ b/docs/colossalai/colossalai.utils.tensor_detector.rst @@ -1,11 +1,5 @@ colossalai.utils.tensor\_detector -================================= +======================= -.. automodule:: colossalai.utils.tensor_detector - :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.utils.tensor_detector.tensor_detector +.. automodule:: colossalai.utils.tensor_detector.tensor_detector + :members: \ No newline at end of file diff --git a/docs/colossalai/colossalai.utils.tensor_detector.tensor_detector.rst b/docs/colossalai/colossalai.utils.tensor_detector.tensor_detector.rst deleted file mode 100644 index 991cea343..000000000 --- a/docs/colossalai/colossalai.utils.tensor_detector.tensor_detector.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.utils.tensor\_detector.tensor\_detector -================================================== - -.. automodule:: colossalai.utils.tensor_detector.tensor_detector - :members: diff --git a/docs/colossalai/colossalai.zero.init_ctx.init_context.rst b/docs/colossalai/colossalai.zero.init_ctx.init_context.rst deleted file mode 100644 index 1694074e8..000000000 --- a/docs/colossalai/colossalai.zero.init_ctx.init_context.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.init\_ctx.init\_context -======================================= - -.. automodule:: colossalai.zero.init_ctx.init_context - :members: diff --git a/docs/colossalai/colossalai.zero.init_ctx.rst b/docs/colossalai/colossalai.zero.init_ctx.rst index 88cf471df..46a459685 100644 --- a/docs/colossalai/colossalai.zero.init_ctx.rst +++ b/docs/colossalai/colossalai.zero.init_ctx.rst @@ -1,11 +1,8 @@ colossalai.zero.init\_ctx ========================= -.. automodule:: colossalai.zero.init_ctx +colossalai.zero.init\_ctx.init\_context +----------------------------------------- + +.. automodule:: colossalai.zero.init_ctx.init_context :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.zero.init_ctx.init_context diff --git a/docs/colossalai/colossalai.zero.shard_utils.base_shard_strategy.rst b/docs/colossalai/colossalai.zero.shard_utils.base_shard_strategy.rst deleted file mode 100644 index d5b59e06a..000000000 --- a/docs/colossalai/colossalai.zero.shard_utils.base_shard_strategy.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.shard\_utils.base\_shard\_strategy -================================================== - -.. automodule:: colossalai.zero.shard_utils.base_shard_strategy - :members: diff --git a/docs/colossalai/colossalai.zero.shard_utils.bucket_tensor_shard_strategy.rst b/docs/colossalai/colossalai.zero.shard_utils.bucket_tensor_shard_strategy.rst deleted file mode 100644 index 952c5bbdd..000000000 --- a/docs/colossalai/colossalai.zero.shard_utils.bucket_tensor_shard_strategy.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.shard\_utils.bucket\_tensor\_shard\_strategy -============================================================ - -.. automodule:: colossalai.zero.shard_utils.bucket_tensor_shard_strategy - :members: diff --git a/docs/colossalai/colossalai.zero.shard_utils.rst b/docs/colossalai/colossalai.zero.shard_utils.rst index 0a7899c64..10f5146c4 100644 --- a/docs/colossalai/colossalai.zero.shard_utils.rst +++ b/docs/colossalai/colossalai.zero.shard_utils.rst @@ -3,11 +3,3 @@ colossalai.zero.shard\_utils .. automodule:: colossalai.zero.shard_utils :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.zero.shard_utils.base_shard_strategy - colossalai.zero.shard_utils.bucket_tensor_shard_strategy - colossalai.zero.shard_utils.tensor_shard_strategy diff --git a/docs/colossalai/colossalai.zero.shard_utils.tensor_shard_strategy.rst b/docs/colossalai/colossalai.zero.shard_utils.tensor_shard_strategy.rst deleted file mode 100644 index 571b7bd7a..000000000 --- a/docs/colossalai/colossalai.zero.shard_utils.tensor_shard_strategy.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.shard\_utils.tensor\_shard\_strategy -==================================================== - -.. automodule:: colossalai.zero.shard_utils.tensor_shard_strategy - :members: diff --git a/docs/colossalai/colossalai.zero.sharded_model.param_manager.rst b/docs/colossalai/colossalai.zero.sharded_model.param_manager.rst deleted file mode 100644 index 7692f598d..000000000 --- a/docs/colossalai/colossalai.zero.sharded_model.param_manager.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.sharded\_model.param\_manager -============================================= - -.. automodule:: colossalai.zero.sharded_model.param_manager - :members: diff --git a/docs/colossalai/colossalai.zero.sharded_model.reduce_scatter.rst b/docs/colossalai/colossalai.zero.sharded_model.reduce_scatter.rst deleted file mode 100644 index cf861ee70..000000000 --- a/docs/colossalai/colossalai.zero.sharded_model.reduce_scatter.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.sharded\_model.reduce\_scatter -============================================== - -.. automodule:: colossalai.zero.sharded_model.reduce_scatter - :members: diff --git a/docs/colossalai/colossalai.zero.sharded_model.rst b/docs/colossalai/colossalai.zero.sharded_model.rst index 07660ff8b..c2a959e58 100644 --- a/docs/colossalai/colossalai.zero.sharded_model.rst +++ b/docs/colossalai/colossalai.zero.sharded_model.rst @@ -1,16 +1,39 @@ colossalai.zero.sharded\_model ============================== + +colossalai.zero.sharded\_model.sharded\_model\_v2 +-------------------------------------------------- + +.. automodule:: colossalai.zero.sharded_model.sharded_model_v2 + :members: + +colossalai.zero.sharded\_model.reduce\_scatter +----------------------------------------------- + +.. automodule:: colossalai.zero.sharded_model.reduce_scatter + :members: + +colossalai.zero.sharded\_model.utils +-------------------------------------- + +.. automodule:: colossalai.zero.sharded_model.utils + :members: + +colossalai.zero.sharded\_model.sharded\_model +-------------------------------------------------- + .. automodule:: colossalai.zero.sharded_model :members: +colossalai.zero.sharded\_model.sharded\_grad +-------------------------------------------------- -.. toctree:: - :maxdepth: 2 +.. automodule:: colossalai.zero.sharded_grad + :members: - colossalai.zero.sharded_model.param_manager - colossalai.zero.sharded_model.reduce_scatter - colossalai.zero.sharded_model.sharded_grad - colossalai.zero.sharded_model.sharded_model - colossalai.zero.sharded_model.sharded_model_v2 - colossalai.zero.sharded_model.utils +colossalai.zero.sharded\_model.param\_manager +---------------------------------------------- + +.. automodule:: colossalai.zero.sharded_model.param_manager + :members: \ No newline at end of file diff --git a/docs/colossalai/colossalai.zero.sharded_model.sharded_grad.rst b/docs/colossalai/colossalai.zero.sharded_model.sharded_grad.rst deleted file mode 100644 index 28607b68e..000000000 --- a/docs/colossalai/colossalai.zero.sharded_model.sharded_grad.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.sharded\_model.sharded\_grad -============================================ - -.. automodule:: colossalai.zero.sharded_model.sharded_grad - :members: diff --git a/docs/colossalai/colossalai.zero.sharded_model.sharded_model.rst b/docs/colossalai/colossalai.zero.sharded_model.sharded_model.rst deleted file mode 100644 index e61f6729f..000000000 --- a/docs/colossalai/colossalai.zero.sharded_model.sharded_model.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.sharded\_model.sharded\_model -============================================= - -.. automodule:: colossalai.zero.sharded_model.sharded_model - :members: diff --git a/docs/colossalai/colossalai.zero.sharded_model.sharded_model_v2.rst b/docs/colossalai/colossalai.zero.sharded_model.sharded_model_v2.rst deleted file mode 100644 index a0e191377..000000000 --- a/docs/colossalai/colossalai.zero.sharded_model.sharded_model_v2.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.sharded\_model.sharded\_model\_v2 -================================================= - -.. automodule:: colossalai.zero.sharded_model.sharded_model_v2 - :members: diff --git a/docs/colossalai/colossalai.zero.sharded_model.utils.rst b/docs/colossalai/colossalai.zero.sharded_model.utils.rst deleted file mode 100644 index 5e3767742..000000000 --- a/docs/colossalai/colossalai.zero.sharded_model.utils.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.sharded\_model.utils -==================================== - -.. automodule:: colossalai.zero.sharded_model.utils - :members: diff --git a/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.base_store.rst b/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.base_store.rst deleted file mode 100644 index 5010367b8..000000000 --- a/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.base_store.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.sharded\_optim.bookkeeping.base\_store -====================================================== - -.. automodule:: colossalai.zero.sharded_optim.bookkeeping.base_store - :members: diff --git a/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.bucket_store.rst b/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.bucket_store.rst deleted file mode 100644 index 7a89a615a..000000000 --- a/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.bucket_store.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.sharded\_optim.bookkeeping.bucket\_store -======================================================== - -.. automodule:: colossalai.zero.sharded_optim.bookkeeping.bucket_store - :members: diff --git a/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.gradient_store.rst b/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.gradient_store.rst deleted file mode 100644 index ef176e445..000000000 --- a/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.gradient_store.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.sharded\_optim.bookkeeping.gradient\_store -========================================================== - -.. automodule:: colossalai.zero.sharded_optim.bookkeeping.gradient_store - :members: diff --git a/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.parameter_store.rst b/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.parameter_store.rst deleted file mode 100644 index 86ffec727..000000000 --- a/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.parameter_store.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.sharded\_optim.bookkeeping.parameter\_store -=========================================================== - -.. automodule:: colossalai.zero.sharded_optim.bookkeeping.parameter_store - :members: diff --git a/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.rst b/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.rst deleted file mode 100644 index a0f0bf110..000000000 --- a/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.rst +++ /dev/null @@ -1,15 +0,0 @@ -colossalai.zero.sharded\_optim.bookkeeping -========================================== - -.. automodule:: colossalai.zero.sharded_optim.bookkeeping - :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.zero.sharded_optim.bookkeeping.base_store - colossalai.zero.sharded_optim.bookkeeping.bucket_store - colossalai.zero.sharded_optim.bookkeeping.gradient_store - colossalai.zero.sharded_optim.bookkeeping.parameter_store - colossalai.zero.sharded_optim.bookkeeping.tensor_bucket diff --git a/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.tensor_bucket.rst b/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.tensor_bucket.rst deleted file mode 100644 index b39ec95ac..000000000 --- a/docs/colossalai/colossalai.zero.sharded_optim.bookkeeping.tensor_bucket.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.sharded\_optim.bookkeeping.tensor\_bucket -========================================================= - -.. automodule:: colossalai.zero.sharded_optim.bookkeeping.tensor_bucket - :members: diff --git a/docs/colossalai/colossalai.zero.sharded_optim.rst b/docs/colossalai/colossalai.zero.sharded_optim.rst index 4b4a167c3..86e299260 100644 --- a/docs/colossalai/colossalai.zero.sharded_optim.rst +++ b/docs/colossalai/colossalai.zero.sharded_optim.rst @@ -1,17 +1,5 @@ colossalai.zero.sharded\_optim -============================== +=============================== .. automodule:: colossalai.zero.sharded_optim :members: - -.. toctree:: - :maxdepth: 2 - - colossalai.zero.sharded_optim.bookkeeping - - -.. toctree:: - :maxdepth: 2 - - colossalai.zero.sharded_optim.sharded_optim - colossalai.zero.sharded_optim.sharded_optim_v2 diff --git a/docs/colossalai/colossalai.zero.sharded_optim.sharded_optim.rst b/docs/colossalai/colossalai.zero.sharded_optim.sharded_optim.rst deleted file mode 100644 index 4ee9acade..000000000 --- a/docs/colossalai/colossalai.zero.sharded_optim.sharded_optim.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.sharded\_optim.sharded\_optim -============================================= - -.. automodule:: colossalai.zero.sharded_optim.sharded_optim - :members: diff --git a/docs/colossalai/colossalai.zero.sharded_optim.sharded_optim_v2.rst b/docs/colossalai/colossalai.zero.sharded_optim.sharded_optim_v2.rst deleted file mode 100644 index 01fbe0c4c..000000000 --- a/docs/colossalai/colossalai.zero.sharded_optim.sharded_optim_v2.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.sharded\_optim.sharded\_optim\_v2 -================================================= - -.. automodule:: colossalai.zero.sharded_optim.sharded_optim_v2 - :members: diff --git a/docs/colossalai/colossalai.zero.sharded_param.rst b/docs/colossalai/colossalai.zero.sharded_param.rst index 02e0fc6c2..ccbb98a3e 100644 --- a/docs/colossalai/colossalai.zero.sharded_param.rst +++ b/docs/colossalai/colossalai.zero.sharded_param.rst @@ -3,10 +3,3 @@ colossalai.zero.sharded\_param .. automodule:: colossalai.zero.sharded_param :members: - - -.. toctree:: - :maxdepth: 2 - - colossalai.zero.sharded_param.sharded_param - colossalai.zero.sharded_param.sharded_tensor diff --git a/docs/colossalai/colossalai.zero.sharded_param.sharded_param.rst b/docs/colossalai/colossalai.zero.sharded_param.sharded_param.rst deleted file mode 100644 index efa2f0de3..000000000 --- a/docs/colossalai/colossalai.zero.sharded_param.sharded_param.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.sharded\_param.sharded\_param -============================================= - -.. automodule:: colossalai.zero.sharded_param.sharded_param - :members: diff --git a/docs/colossalai/colossalai.zero.sharded_param.sharded_tensor.rst b/docs/colossalai/colossalai.zero.sharded_param.sharded_tensor.rst deleted file mode 100644 index 930c28de4..000000000 --- a/docs/colossalai/colossalai.zero.sharded_param.sharded_tensor.rst +++ /dev/null @@ -1,5 +0,0 @@ -colossalai.zero.sharded\_param.sharded\_tensor -============================================== - -.. automodule:: colossalai.zero.sharded_param.sharded_tensor - :members: diff --git a/docs/colossalai/colossalai.zero.sharded_utils.rst b/docs/colossalai/colossalai.zero.sharded_utils.rst new file mode 100644 index 000000000..ad375c24f --- /dev/null +++ b/docs/colossalai/colossalai.zero.sharded_utils.rst @@ -0,0 +1,5 @@ +colossalai.zero.sharded\_utils +=============== + +.. automodule:: colossalai.zero.sharded_utils + :members: \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index d0dedfc5c..893644f70 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,12 +16,6 @@ import sys sys.path.insert(0, os.path.abspath('..')) - -def get_version(): - with open('../version.txt') as f: - return f.read().strip() - - # -- Project information ----------------------------------------------------- project = 'Colossal-AI' @@ -29,7 +23,8 @@ copyright = f'{datetime.datetime.now().year}, HPC-AI Tech' author = 'HPC-AI Technology Inc.' # The full version, including alpha/beta/rc tags -release = get_version() +release = '0.0.1' + # -- General configuration --------------------------------------------------- @@ -40,6 +35,7 @@ extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.mathjax', 'sphinx.ext.napoleon', + 'sphinx.ext.linkcode', 'myst_parser', ] @@ -51,7 +47,9 @@ autodoc_typehints = 'none' # Enable overriding of function signatures in the first line of the docstring. autodoc_docstring_signature = True -autodoc_default_options = {'member-order': 'bysource'} +autodoc_default_options = { + 'member-order': 'bysource', +} # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -69,7 +67,7 @@ exclude_patterns = ['.build', 'Thumbs.db', '.DS_Store'] html_theme = 'sphinx_rtd_theme' html_show_sourcelink = False html_theme_options = { - 'navigation_depth': 2, + 'navigation_depth': 3, } html_context = { @@ -90,3 +88,48 @@ html_css_files = [ # -- Extension configuration ------------------------------------------------- source_suffix = ['.rst', '.md', '.MD'] + +import inspect +import colossalai +def linkcode_resolve(domain, info): + """ + Determine the URL corresponding to Python object + """ + if domain != 'py': + return None + + modname = info['module'] + fullname = info['fullname'] + + submod = sys.modules.get(modname) + if submod is None: + return None + + obj = submod + for part in fullname.split('.'): + try: + obj = getattr(obj, part) + except Exception: + return None + + try: + fn = inspect.getsourcefile(obj) + except Exception: + fn = None + if not fn: + return None + + try: + source, lineno = inspect.findsource(obj) + except Exception: + lineno = None + + if lineno: + linespec = "#L%d" % (lineno + 1) + else: + linespec = "" + + fn = os.path.relpath(fn, start=os.path.dirname(colossalai.__file__)) + + github = "https://github.com/hpcaitech/ColossalAI/blob/main/colossalai/{}{}" + return github.format(fn, linespec) diff --git a/docs/index.rst b/docs/index.rst index b29450f58..f275f7829 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,14 +5,23 @@ Colossal-AI API documentation ====================================== + .. toctree:: :maxdepth: 2 :caption: API REFERENCE colossalai/colossalai +.. toctree:: + :maxdepth: 2 + :caption: Useful links for Colossal-AI + + links/Colossalai examples + links/Colossalai benchmarks + links/Colossalai tutorial + Indices and tables -================== +-------------------- -* :ref:`genindex` \ No newline at end of file +* :ref:`genindex` diff --git a/docs/links/Colossalai Homepage.rst b/docs/links/Colossalai Homepage.rst new file mode 100644 index 000000000..38e223bd2 --- /dev/null +++ b/docs/links/Colossalai Homepage.rst @@ -0,0 +1,6 @@ +Colossal-AI Github Homepage +================================== + +*If you are looking for the Git homepage of Colossal-AI, please check* +`Colossal-AI Tutorial `_ +*for our source code.* \ No newline at end of file diff --git a/docs/links/Colossalai benchmarks.rst b/docs/links/Colossalai benchmarks.rst new file mode 100644 index 000000000..1835670a5 --- /dev/null +++ b/docs/links/Colossalai benchmarks.rst @@ -0,0 +1,6 @@ +Colossal-AI Benchmarks +================================== + +*If you are interested in the performance or the features of Colossal-AI, please check* +`Colossal-AI Benchmark `_. +*to get more details about our performance on CIFAR10, ImageNet1K or GPT2 ZeRO.* \ No newline at end of file diff --git a/docs/links/Colossalai examples.rst b/docs/links/Colossalai examples.rst new file mode 100644 index 000000000..c375f007a --- /dev/null +++ b/docs/links/Colossalai examples.rst @@ -0,0 +1,6 @@ +Colossal-AI Examples +================================== + +*If you are looking for the example code of using Colossal-AI in CV or NLP, please check* +`Colossal-AI Example `_ +*to get more details about using colossalai in Resnet, Moe, Vit, Bert and GPT* \ No newline at end of file diff --git a/docs/links/Colossalai tutorial.rst b/docs/links/Colossalai tutorial.rst new file mode 100644 index 000000000..a4ab7f5b9 --- /dev/null +++ b/docs/links/Colossalai tutorial.rst @@ -0,0 +1,7 @@ +Colossal-AI Tutorial +================================== + +*If you are looking for the tutorial of using Colossal-AI, please check* +`Colossal-AI Tutorial `_ +*to get more details about getting started, using TP (tensor parallel), PP (pipeline parallel) +and training with colossalai trainer or engine.* \ No newline at end of file