mirror of https://github.com/hpcaitech/ColossalAI
41 lines
1.3 KiB
Python
41 lines
1.3 KiB
Python
|
import torch
|
||
|
|
|
||
|
|
|
||
|
|
###### BIAS GELU FUSION/ NO AUTOGRAD ################
|
||
|
|
# 1/sqrt(2*pi)-> 0.3989423
|
||
|
|
# 1/sqrt(2) -> 0.70710678
|
||
|
|
# sqrt(2/pi) -> 0.79788456
|
||
|
|
# this function is tanh approximation of gelu
|
||
|
|
# actual gelu is:
|
||
|
|
# x * 0.5 * (1.0 + torch.erf(x * 0.70710678))
|
||
|
|
|
||
|
|
@torch.jit.script
|
||
|
|
def bias_gelu(bias, y):
|
||
|
|
x = bias + y
|
||
|
|
return x * 0.5 * (1.0 + torch.tanh(0.79788456 * x * (1 + 0.044715 * x * x)))
|
||
|
|
|
||
|
|
# gradient of tanh approximation of gelu
|
||
|
|
# gradient of actual gelu is:
|
||
|
|
# 0.5 * (1. + torch.erf(x * 0.70710678)) + 0.3989423 * x * torch.exp(-0.5 * x * x)
|
||
|
|
@torch.jit.script
|
||
|
|
def bias_gelu_back(g, bias, y):
|
||
|
|
x = bias + y
|
||
|
|
tanh_out = torch.tanh(0.79788456 * x * (1 + 0.044715 * x * x))
|
||
|
|
# sqrt(2/pi) * 3 * 0.044715 -> 0.1070322243
|
||
|
|
ff = 0.5 * x * ((1 - tanh_out * tanh_out) * (0.79788456 + 0.1070322243 * x * x)) + 0.5 * (1 + tanh_out)
|
||
|
|
return ff*g
|
||
|
|
|
||
|
|
class GeLUFunction(torch.autograd.Function):
|
||
|
|
@staticmethod
|
||
|
|
# bias is an optional argument
|
||
|
|
def forward(ctx, input, bias):
|
||
|
|
ctx.save_for_backward(input, bias)
|
||
|
|
return bias_gelu(bias, input)
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def backward(ctx, grad_output):
|
||
|
|
input, bias = ctx.saved_tensors
|
||
|
|
tmp = bias_gelu_back(grad_output, bias, input)
|
||
|
|
return tmp, tmp
|
||
|
|
|
||
|
|
bias_gelu_impl = GeLUFunction.apply
|