mirror of https://github.com/hpcaitech/ColossalAI
[Hotfix] hotfix normalization (#6163)
* [fix] hotfix normalization * [hotfix] force doc ci test * [hotfix] fallback docpull/6165/head
parent
130229fdcb
commit
fa9d0318e4
|
@ -76,6 +76,7 @@ if SUPPORT_NPU:
|
|||
|
||||
FusedRMSNormWithHook = NPUFusedRMSNormWithHook
|
||||
else:
|
||||
try:
|
||||
from apex.normalization import FusedRMSNorm as ApexFusedRMSNorm
|
||||
|
||||
class CUDAFusedRMSNormWithHook(ApexFusedRMSNorm):
|
||||
|
@ -88,6 +89,11 @@ else:
|
|||
return output
|
||||
|
||||
FusedRMSNormWithHook = CUDAFusedRMSNormWithHook
|
||||
except ImportError:
|
||||
warnings.warn(
|
||||
"Please install apex from source (https://github.com/NVIDIA/apex) to use the fused RMSNorm kernel"
|
||||
)
|
||||
|
||||
|
||||
FAST_LAYERNORM_SUPPORTED_SIZE = [
|
||||
1024,
|
||||
|
|
Loading…
Reference in New Issue