mirror of https://github.com/hpcaitech/ColossalAI
aibig-modeldata-parallelismdeep-learningdistributed-computingfoundation-modelsheterogeneous-traininghpcinferencelarge-scalemodel-parallelismpipeline-parallelism
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
22 lines
670 B
22 lines
670 B
import torch |
|
|
|
|
|
def bias_dropout_add(x, bias, residual, prob, training): |
|
# type: (Tensor, Tensor, Tensor, float, bool) -> Tensor |
|
out = torch.nn.functional.dropout(x + bias, p=prob, training=training) |
|
out = residual + out |
|
return out |
|
|
|
|
|
@torch.jit.script |
|
def bias_dropout_add_fused_train( |
|
x: torch.Tensor, bias: torch.Tensor, residual: torch.Tensor, prob: float |
|
) -> torch.Tensor: |
|
return bias_dropout_add(x, bias, residual, prob, True) |
|
|
|
|
|
@torch.jit.script |
|
def bias_dropout_add_fused_inference( |
|
x: torch.Tensor, bias: torch.Tensor, residual: torch.Tensor, prob: float |
|
) -> torch.Tensor: |
|
return bias_dropout_add(x, bias, residual, prob, False)
|
|
|