import torch
from mindspeed_ops.utils import is_arch35
__all__ = ["LayerNormGatedFunction", "rms_norm_gated"]
class LayerNormGatedFunction(torch.autograd.Function):
@staticmethod
def forward(
ctx,
x: torch.Tensor,
g: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor,
activation: str,
residual: torch.Tensor | None = None,
eps: float = 1e-6,
prenorm: bool = False,
residual_in_fp32: bool = False,
is_rms_norm: bool = False,
):
if is_arch35():
raise NotImplementedError("rms_norm_gated is not supported on arch35")
from mindspeed_ops.arch32.triton.rmsnormgated import layer_norm_gated_fwd
x_shape_og = x.shape
g_shape_og = g.shape
x = x.reshape(-1, x.shape[-1])
g = g.reshape(-1, g.shape[-1])
if residual is not None:
assert residual.shape == x_shape_og
residual = residual.reshape(-1, residual.shape[-1])
residual_dtype = residual.dtype if residual is not None else (torch.float if residual_in_fp32 else None)
y, mean, rstd, residual_out = layer_norm_gated_fwd(
x=x,
g=g,
weight=weight,
bias=bias,
activation=activation,
eps=eps,
residual=residual,
residual_dtype=residual_dtype,
is_rms_norm=is_rms_norm,
)
ctx.save_for_backward(residual_out, g, weight, bias, mean, rstd)
ctx.x_shape_og = x_shape_og
ctx.g_shape_og = g_shape_og
ctx.activation = activation
ctx.eps = eps
ctx.is_rms_norm = is_rms_norm
ctx.has_residual = residual is not None
ctx.prenorm = prenorm
ctx.x_dtype = x.dtype
y = y.reshape(x_shape_og)
return y if not prenorm else (y, residual_out.reshape(x_shape_og))
@staticmethod
def backward(ctx, dy, *args):
if is_arch35():
raise NotImplementedError("rms_norm_gated is not supported on arch35")
from mindspeed_ops.arch32.triton.rmsnormgated import layer_norm_gated_bwd
x, g, weight, bias, mean, rstd = ctx.saved_tensors
dy = dy.reshape(-1, dy.shape[-1])
assert dy.shape == x.shape
if ctx.prenorm:
dresidual = args[0]
dresidual = dresidual.reshape(-1, dresidual.shape[-1])
assert dresidual.shape == x.shape
else:
dresidual = None
dx, dg, dw, db, dres_in = layer_norm_gated_bwd(
dy=dy,
x=x,
g=g,
weight=weight,
bias=bias,
activation=ctx.activation,
eps=ctx.eps,
mean=mean,
rstd=rstd,
dresidual=dresidual,
has_residual=ctx.has_residual,
is_rms_norm=ctx.is_rms_norm,
x_dtype=ctx.x_dtype,
)
return (
dx.reshape(ctx.x_shape_og),
dg.reshape(ctx.g_shape_og),
dw,
db,
None,
dres_in.reshape(ctx.x_shape_og) if ctx.has_residual else None,
None,
None,
None,
None,
)
def rms_norm_gated(
x: torch.Tensor,
g: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor,
activation: str = "swish",
residual: torch.Tensor | None = None,
prenorm: bool = False,
residual_in_fp32: bool = False,
eps: float = 1e-6,
):
return LayerNormGatedFunction.apply(
x,
g,
weight,
bias,
activation,
residual,
eps,
prenorm,
residual_in_fp32,
True,
)