mirror of https://github.com/hpcaitech/ColossalAI
aibig-modeldata-parallelismdeep-learningdistributed-computingfoundation-modelsheterogeneous-traininghpcinferencelarge-scalemodel-parallelismpipeline-parallelism
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
37 lines
1.1 KiB
37 lines
1.1 KiB
10 months ago
|
from .cpu_adam import CpuAdamArmExtension, CpuAdamX86Extension
|
||
|
|
from .flash_attention import (
|
||
|
|
FlashAttentionDaoCudaExtension,
|
||
|
|
FlashAttentionNpuExtension,
|
||
|
|
FlashAttentionXformersCudaExtension,
|
||
|
|
)
|
||
|
|
from .layernorm import LayerNormCudaExtension
|
||
|
|
from .moe import MoeCudaExtension
|
||
|
|
from .optimizer import FusedOptimizerCudaExtension
|
||
|
|
from .softmax import ScaledMaskedSoftmaxCudaExtension, ScaledUpperTriangleMaskedSoftmaxCudaExtension
|
||
|
|
|
||
|
|
ALL_EXTENSIONS = [
|
||
|
|
CpuAdamArmExtension,
|
||
|
|
CpuAdamX86Extension,
|
||
|
|
LayerNormCudaExtension,
|
||
|
|
MoeCudaExtension,
|
||
|
|
FusedOptimizerCudaExtension,
|
||
|
|
ScaledMaskedSoftmaxCudaExtension,
|
||
|
|
ScaledUpperTriangleMaskedSoftmaxCudaExtension,
|
||
|
|
FlashAttentionDaoCudaExtension,
|
||
|
|
FlashAttentionXformersCudaExtension,
|
||
|
|
FlashAttentionNpuExtension,
|
||
|
|
]
|
||
|
|
|
||
|
|
__all__ = [
|
||
|
|
"CpuAdamArmExtension",
|
||
|
|
"CpuAdamX86Extension",
|
||
|
|
"LayerNormCudaExtension",
|
||
|
|
"MoeCudaExtension",
|
||
|
|
"FusedOptimizerCudaExtension",
|
||
|
|
"ScaledMaskedSoftmaxCudaExtension",
|
||
|
|
"ScaledUpperTriangleMaskedSoftmaxCudaExtension",
|
||
|
|
"FlashAttentionDaoCudaExtension",
|
||
|
|
"FlashAttentionXformersCudaExtension",
|
||
|
|
"FlashAttentionNpuExtension",
|
||
|
|
]
|