mirror of https://github.com/hpcaitech/ColossalAI
35 lines
1.2 KiB
Python
35 lines
1.2 KiB
Python
|
# adapted from Megatron-LM
|
||
|
|
# https://github.com/NVIDIA/Megatron-LM/blob/b31e1296354e979722627a6c4dedafe19b51fa97/megatron/model/fused_bias_gelu.py
|
||
|
|
|
||
|
|
import torch
|
||
|
|
|
||
|
|
@torch.jit.script
|
||
|
|
def bias_gelu(bias, y):
|
||
|
|
x = bias + y
|
||
|
|
return x * 0.5 * (1.0 + torch.tanh(0.79788456 * x * (1 + 0.044715 * x * x)))
|
||
|
|
|
||
|
|
# gradient of tanh approximation of gelu
|
||
|
|
# gradient of actual gelu is:
|
||
|
|
# 0.5 * (1. + torch.erf(x * 0.70710678)) + 0.3989423 * x * torch.exp(-0.5 * x * x)
|
||
|
|
@torch.jit.script
|
||
|
|
def bias_gelu_back(g, bias, y):
|
||
|
|
x = bias + y
|
||
|
|
tanh_out = torch.tanh(0.79788456 * x * (1 + 0.044715 * x * x))
|
||
|
|
# sqrt(2/pi) * 3 * 0.044715 -> 0.1070322243
|
||
|
|
ff = 0.5 * x * ((1 - tanh_out * tanh_out) * (0.79788456 + 0.1070322243 * x * x)) + 0.5 * (1 + tanh_out)
|
||
|
|
return ff*g
|
||
|
|
|
||
|
|
class GeLUFunction(torch.autograd.Function):
|
||
|
|
@staticmethod
|
||
|
|
# bias is an optional argument
|
||
|
|
def forward(ctx, input, bias):
|
||
|
|
ctx.save_for_backward(input, bias)
|
||
|
|
return bias_gelu(bias, input)
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def backward(ctx, grad_output):
|
||
|
|
input, bias = ctx.saved_tensors
|
||
|
|
tmp = bias_gelu_back(grad_output, bias, input)
|
||
|
|
return tmp, tmp
|
||
|
|
|
||
|
|
bias_gelu_impl = GeLUFunction.apply
|