mirror of https://github.com/hpcaitech/ColossalAI
13 lines
431 B
Python
13 lines
431 B
Python
|
import torch
|
||
|
|
|
||
|
|
def bias_dropout_add(x, bias, residual, prob, training):
|
||
|
|
# type: (Tensor, Tensor, Tensor, float, bool) -> Tensor
|
||
|
|
out = torch.nn.functional.dropout(x + bias, p=prob, training=training)
|
||
|
|
out = residual + out
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def get_bias_dropout_add(training):
|
||
|
|
def _bias_dropout_add(x, bias, residual, prob):
|
||
|
|
return bias_dropout_add(x, bias, residual, prob, training)
|
||
|
|
return _bias_dropout_add
|