Merged
norm: add LN goldens and align empty-shape semantics #13856
rkAdd on 16 days ago
norm: add LN goldens and align empty-shape semantics #13856
Merged
Merged fromwangpengbo26/ops-nnln_goldenintocann/ops-nnmaster
Total: 7 files changed+251-49
| @@ -860,11 +860,11 @@ ge::graphStatus AddLayerNormRegbaseTiling::DoOpTiling() | |||
| 860 | rowsPerCore_ = (rows_ + blockNum - 1) / blockNum; | 860 | rowsPerCore_ = (rows_ + blockNum - 1) / blockNum; |
| 861 | rowsPerTailCore_ = rows_ - (blockNum - 1) * rowsPerCore_; | 861 | rowsPerTailCore_ = rows_ - (blockNum - 1) * rowsPerCore_; |
| 862 | tailCoreStartIndex_ = blockNum - 1; | 862 | tailCoreStartIndex_ = blockNum - 1; |
| 863 | - int64_t perLoopMax = static_cast<int64_t>(ubSize_) / static_cast<int64_t>(sizeof(float)); | 863 | + int64_t perLoopMax = static_cast<int64_t>(ubSize_) / (OUTPUT_NUM * static_cast<int64_t>(sizeof(float))); |
| 864 | int64_t chunk = (perLoopMax < rowsPerCore_) ? perLoopMax : rowsPerCore_; | 864 | int64_t chunk = (perLoopMax < rowsPerCore_) ? perLoopMax : rowsPerCore_; |
| 865 | rowsPerLoop_ = chunk / elemsPerBlock * elemsPerBlock; | 865 | rowsPerLoop_ = chunk / elemsPerBlock * elemsPerBlock; |
| 866 | if (rowsPerLoop_ < 1) { | 866 | if (rowsPerLoop_ < 1) { |
| 867 | - rowsPerLoop_ = rowsPerCore_; | 867 | + rowsPerLoop_ = elemsPerBlock; |
| 868 | } | 868 | } |
| 869 | colsPerLoop_ = 0; | 869 | colsPerLoop_ = 0; |
| 870 | colsLoopCount_ = 0; | 870 | colsLoopCount_ = 0; |
| @@ -10,9 +10,9 @@ | |||
| 10 | 10 | ||
| 11 | /*! | 11 | /*! |
| 12 | * \file add_layer_norm_regbase_reduce_empty.h | 12 | * \file add_layer_norm_regbase_reduce_empty.h |
| 13 | - * \brief 空张量(reduce empty)兜底 kernel:合轴后 R==0 且 A>0 时,mean/rstd=[A,1] 填 NaN, | 13 | + * \brief 空张量(reduce empty)兜底 kernel:合轴后 R==0 且 A>0 时,mean=[A,1] 填 0、rstd=[A,1] 填 NaN, |
| 14 | * y/x=[A,0] 空不写。把 mean/rstd 当一维 A 个 fp32,host 多核切分+定每循环元素数, | 14 | * y/x=[A,0] 空不写。把 mean/rstd 当一维 A 个 fp32,host 多核切分+定每循环元素数, |
| 15 | - * kernel Duplicate NaN 一次、循环复用 buffer 搬出。 | 15 | + * 同一 buffer 分两段填充 0 和 NaN,循环搬出。 |
| 16 | */ | 16 | */ |
| 17 | 17 | ||
| 18 | #ifndef ADD_LAYER_NORM_REGBASE_REDUCE_EMPTY_H | 18 | #ifndef ADD_LAYER_NORM_REGBASE_REDUCE_EMPTY_H |
| @@ -45,7 +45,7 @@ public: | |||
| 45 | tailLen_ = rowsThisCore_ - (loopCount_ - 1) * rowsPerLoop_; | 45 | tailLen_ = rowsThisCore_ - (loopCount_ - 1) * rowsPerLoop_; |
| 46 | meanGm_.SetGlobalBuffer((__gm__ float*)mean + rowOffset_, rowsThisCore_); | 46 | meanGm_.SetGlobalBuffer((__gm__ float*)mean + rowOffset_, rowsThisCore_); |
| 47 | rstdGm_.SetGlobalBuffer((__gm__ float*)rstd + rowOffset_, rowsThisCore_); | 47 | rstdGm_.SetGlobalBuffer((__gm__ float*)rstd + rowOffset_, rowsThisCore_); |
| 48 | - pipe_.InitBuffer(outNanQueue_, 1, rowsPerLoop_ * sizeof(float)); | 48 | + pipe_.InitBuffer(outQueue_, 1, 2 * rowsPerLoop_ * sizeof(float)); |
| 49 | } | 49 | } |
| 50 | 50 | ||
| 51 | __aicore__ inline void Process() | 51 | __aicore__ inline void Process() |
| @@ -55,11 +55,12 @@ public: | |||
| 55 | return; | 55 | return; |
| 56 | } | 56 | } |
| 57 | 57 | ||
| 58 | - LocalTensor<float> nanLocal = outNanQueue_.AllocTensor<float>(); | 58 | + LocalTensor<float> outLocal = outQueue_.AllocTensor<float>(); |
| 59 | float nanVal = AscendC::NumericLimits<float>::QuietNaN(); | 59 | float nanVal = AscendC::NumericLimits<float>::QuietNaN(); |
| 60 | - Duplicate(nanLocal, nanVal, static_cast<int32_t>(rowsPerLoop_)); | 60 | + Duplicate(outLocal, 0.0f, static_cast<int32_t>(rowsPerLoop_)); |
| 61 | - outNanQueue_.EnQue(nanLocal); | 61 | + Duplicate(outLocal[rowsPerLoop_], nanVal, static_cast<int32_t>(rowsPerLoop_)); |
| 62 | - nanLocal = outNanQueue_.DeQue<float>(); | 62 | + outQueue_.EnQue(outLocal); |
| 63 | + outLocal = outQueue_.DeQue<float>(); | ||
| 63 | 64 | ||
| 64 | for (int64_t i = 0; i < loopCount_; i++) { | 65 | for (int64_t i = 0; i < loopCount_; i++) { |
| 65 | int64_t curLen = (i == loopCount_ - 1) ? tailLen_ : rowsPerLoop_; | 66 | int64_t curLen = (i == loopCount_ - 1) ? tailLen_ : rowsPerLoop_; |
| @@ -68,11 +69,11 @@ public: | |||
| 68 | copyParams.blockLen = static_cast<uint32_t>(curLen * sizeof(float)); | 69 | copyParams.blockLen = static_cast<uint32_t>(curLen * sizeof(float)); |
| 69 | copyParams.srcStride = 0; | 70 | copyParams.srcStride = 0; |
| 70 | copyParams.dstStride = 0; | 71 | copyParams.dstStride = 0; |
| 71 | - DataCopyPad(meanGm_[i * rowsPerLoop_], nanLocal, copyParams); | 72 | + DataCopyPad(meanGm_[i * rowsPerLoop_], outLocal, copyParams); |
| 72 | - DataCopyPad(rstdGm_[i * rowsPerLoop_], nanLocal, copyParams); | 73 | + DataCopyPad(rstdGm_[i * rowsPerLoop_], outLocal[rowsPerLoop_], copyParams); |
| 73 | } | 74 | } |
| 74 | 75 | ||
| 75 | - outNanQueue_.FreeTensor(nanLocal); | 76 | + outQueue_.FreeTensor(outLocal); |
| 76 | } | 77 | } |
| 77 | 78 | ||
| 78 | private: | 79 | private: |
| @@ -80,7 +81,7 @@ private: | |||
| 80 | const AddLayerNormRegbaseTilingData* tiling_; | 81 | const AddLayerNormRegbaseTilingData* tiling_; |
| 81 | GlobalTensor<float> meanGm_; | 82 | GlobalTensor<float> meanGm_; |
| 82 | GlobalTensor<float> rstdGm_; | 83 | GlobalTensor<float> rstdGm_; |
| 83 | - TQue<QuePosition::VECOUT, 1> outNanQueue_; | 84 | + TQue<QuePosition::VECOUT, 1> outQueue_; |
| 84 | int64_t usedCoreNum_{0}; | 85 | int64_t usedCoreNum_{0}; |
| 85 | int64_t rowsThisCore_{0}; | 86 | int64_t rowsThisCore_{0}; |
| 86 | int64_t rowOffset_{0}; | 87 | int64_t rowOffset_{0}; |
| @@ -15,6 +15,8 @@ import numpy as np | |||
| 15 | 15 | ||
| 16 | __golden__ = {"kernel": {"add_layer_norm": "add_layer_norm_golden"}} | 16 | __golden__ = {"kernel": {"add_layer_norm": "add_layer_norm_golden"}} |
| 17 | 17 | ||
| 18 | +__spec__ = {"add_layer_norm": "AddLayerNormTestSpec"} | ||
| 19 | + | ||
| 18 | 20 | ||
| 19 | def add_layer_norm_golden( | 21 | def add_layer_norm_golden( |
| 20 | x1, | 22 | x1, |
| @@ -40,42 +42,82 @@ def add_layer_norm_golden( | |||
| 40 | """ | 42 | """ |
| 41 | import torch | 43 | import torch |
| 42 | 44 | ||
| 43 | - x1_dtype = x1.dtype | 45 | + # FP32/FP64 输入用 FP64 累积,兼容框架升精度输入。 |
| 44 | - x1_x2_any_fp32 = x1.dtype.name == "float32" or x2.dtype.name == "float32" | 46 | + wide = x1.dtype.itemsize >= 4 or x2.dtype.itemsize >= 4 |
| 45 | - calc_dtype = np.float64 if x1_dtype.name == np.float32 else np.float32 | 47 | + calc_dtype = np.float64 if wide else np.float32 |
| 46 | - calc_dtype_torch = torch.float64 if x1_dtype.name == np.float32 else torch.float32 | 48 | + out_dtype = ( |
| 47 | - | 49 | + np.float64 |
| 48 | - x1 = x1.astype(calc_dtype) | 50 | + if x1.dtype == np.float64 or x2.dtype == np.float64 |
| 49 | - x2 = x2.astype(calc_dtype) | 51 | + else np.float32 |
| 50 | - gamma = gamma.astype(calc_dtype) | 52 | + if wide |
| 51 | - beta = beta.astype(calc_dtype) | 53 | + else x1.dtype |
| 52 | - | ||
| 53 | - x1 = torch.from_numpy(x1) | ||
| 54 | - x2 = torch.from_numpy(x2) | ||
| 55 | - gamma = torch.from_numpy(gamma) | ||
| 56 | - beta = torch.from_numpy(beta) | ||
| 57 | - | ||
| 58 | - epsilon = ( | ||
| 59 | - np.float64(epsilon) if x1_dtype.name == np.float32 else np.float32(epsilon) | ||
| 60 | ) | 54 | ) |
| 55 | + normalized_shape = tuple(gamma.shape) | ||
| 56 | + stat_shape = x1.shape[: x1.ndim - gamma.ndim] + (1,) * gamma.ndim | ||
| 61 | 57 | ||
| 58 | + x = x1.astype(calc_dtype) + x2.astype(calc_dtype) | ||
| 62 | if bias is not None: | 59 | if bias is not None: |
| 63 | - bias = bias.astype(calc_dtype) | 60 | + x = x + bias.astype(calc_dtype) |
| 64 | - bias = torch.from_numpy(bias) | 61 | + |
| 65 | - x = x1 + x2 + bias | 62 | + if 0 in normalized_shape: |
| 63 | + # 950 空归一化维度:mean=0,rstd=NaN。 | ||
| 64 | + y_out = np.empty(x1.shape, dtype=out_dtype) | ||
| 65 | + mean_out = np.zeros(stat_shape, dtype=np.float32) | ||
| 66 | + rstd_out = np.full(stat_shape, np.nan, dtype=np.float32) | ||
| 66 | else: | 67 | else: |
| 67 | - x = x1 + x2 | 68 | + y, mean, rstd = torch.ops.aten.native_layer_norm( |
| 68 | - | 69 | + torch.from_numpy(x), |
| 69 | - row_size = x1.shape[-1] | 70 | + list(normalized_shape), |
| 70 | - x = x.reshape(-1, row_size).to(calc_dtype_torch) | 71 | + weight=torch.from_numpy(gamma.astype(calc_dtype)), |
| 71 | - | 72 | + bias=torch.from_numpy(beta.astype(calc_dtype)), |
| 72 | - y, x_mean, x_rstd = torch.ops.aten.native_layer_norm( | 73 | + eps=float(calc_dtype(epsilon)), |
| 73 | - x, [row_size], weight=gamma, bias=beta, eps=epsilon | 74 | + ) |
| 74 | - ) | 75 | + y_out = y.numpy().astype(out_dtype) |
| 75 | - | 76 | + mean_out = mean.numpy().reshape(stat_shape).astype(np.float32) |
| 76 | - y_out = y.numpy().astype("float32" if x1_x2_any_fp32 else x1_dtype) | 77 | + rstd_out = rstd.numpy().reshape(stat_shape).astype(np.float32) |
| 77 | - mean_out = x_mean.numpy().astype("float32") | ||
| 78 | - rstd_out = x_rstd.numpy().astype("float32") | ||
| 79 | - x_out = x.numpy().astype("float32" if x1_x2_any_fp32 else x1_dtype) | ||
| 80 | 78 | ||
| 79 | + # 未启用的 x 输出用 None 跳过比较。 | ||
| 80 | + x_out = x.astype(out_dtype) if additional_output else None | ||
| 81 | return y_out, mean_out, rstd_out, x_out | 81 | return y_out, mean_out, rstd_out, x_out |
| 82 | + | ||
| 83 | + | ||
| 84 | +class AddLayerNormTorchThirdParty: | ||
| 85 | + """Torch 三方对照,低精度输入用 FP32 累积。""" | ||
| 86 | + | ||
| 87 | + def __call__( | ||
| 88 | + self, | ||
| 89 | + x1, | ||
| 90 | + x2, | ||
| 91 | + gamma, | ||
| 92 | + beta, | ||
| 93 | + bias=None, | ||
| 94 | + epsilon=1e-5, | ||
| 95 | + additional_output=False, | ||
| 96 | + **kwargs, | ||
| 97 | + ): | ||
| 98 | + import torch | ||
| 99 | + | ||
| 100 | + out_dtype = torch.promote_types(x1.dtype, x2.dtype) | ||
| 101 | + calc_dtype = torch.float64 if out_dtype == torch.float64 else torch.float32 | ||
| 102 | + x = x1.to(calc_dtype) + x2.to(calc_dtype) | ||
| 103 | + if bias is not None: | ||
| 104 | + x = x + bias.to(calc_dtype) | ||
| 105 | + y, mean, rstd = torch.ops.aten.native_layer_norm( | ||
| 106 | + x, | ||
| 107 | + list(gamma.shape), | ||
| 108 | + weight=gamma.to(calc_dtype), | ||
| 109 | + bias=beta.to(calc_dtype), | ||
| 110 | + eps=float(epsilon), | ||
| 111 | + ) | ||
| 112 | + # 保留输出槽位,关闭 x 输出时跳过比较。 | ||
| 113 | + return ( | ||
| 114 | + y.to(out_dtype), | ||
| 115 | + mean.to(torch.float32), | ||
| 116 | + rstd.to(torch.float32), | ||
| 117 | + x.to(out_dtype) if additional_output else None, | ||
| 118 | + ) | ||
| 119 | + | ||
| 120 | + | ||
| 121 | +class AddLayerNormTestSpec: | ||
| 122 | + golden = staticmethod(add_layer_norm_golden) | ||
| 123 | + third_party = {"torch": AddLayerNormTorchThirdParty} | ||
| @@ -45,9 +45,9 @@ def inplace_add_layer_norm_golden( | |||
| 45 | x = x1.astype(calc_dtype) + x2.astype(calc_dtype) | 45 | x = x1.astype(calc_dtype) + x2.astype(calc_dtype) |
| 46 | 46 | ||
| 47 | if 0 in normalized_shape: | 47 | if 0 in normalized_shape: |
| 48 | - # 950平台支持空tensor,mean/rstd 填 NaN、y/x 为空不写; | 48 | + # 950 空归一化维度:mean=0,rstd=NaN。 |
| 49 | y_out = np.zeros(x1.shape, dtype=yx_dtype) | 49 | y_out = np.zeros(x1.shape, dtype=yx_dtype) |
| 50 | - mean_out = np.full(stat_shape, np.nan).astype("float32") | 50 | + mean_out = np.zeros(stat_shape, dtype=np.float32) |
🟡 Medium Priority 改动点:第 50 行把空归一化维的 建议:在 TorchGpuThirdParty.call 内、调用 aten 之前加空维分支: ![]() ![]() Incorrect? | |||
| 51 | rstd_out = np.full(stat_shape, np.nan).astype("float32") | 51 | rstd_out = np.full(stat_shape, np.nan).astype("float32") |
| 52 | else: | 52 | else: |
| 53 | eps = np.float64(epsilon) if wide else np.float32(epsilon) | 53 | eps = np.float64(epsilon) if wide else np.float32(epsilon) |
| @@ -63,7 +63,7 @@ def inplace_add_layer_norm_golden( | |||
| 63 | rstd_out = rstd.numpy().reshape(stat_shape).astype("float32") | 63 | rstd_out = rstd.numpy().reshape(stat_shape).astype("float32") |
| 64 | 64 | ||
| 65 | if not additional_output: | 65 | if not additional_output: |
| 66 | - return y_out, mean_out, rstd_out | 66 | + return y_out, mean_out, rstd_out, None |
| 67 | return y_out, mean_out, rstd_out, x.astype(yx_dtype) | 67 | return y_out, mean_out, rstd_out, x.astype(yx_dtype) |
| 68 | 68 | ||
| 69 | 69 | ||
| @@ -13,6 +13,7 @@ | |||
| 13 | ## 使用约束 | 13 | ## 使用约束 |
| 14 | 14 | ||
| 15 | - 通用约束:仅限推理场景,不支持训练场景。 | 15 | - 通用约束:仅限推理场景,不支持训练场景。 |
| 16 | +- 当AddRmsNorm的输入x1或x2的直接前驱节点类型为Data时,不进行原地融合。 | ||
| 16 | - 融合成AddRmsNormCast约束: | 17 | - 融合成AddRmsNormCast约束: |
| 17 | - 仅支持输入x1为float16/bfloat16。 | 18 | - 仅支持输入x1为float16/bfloat16。 |
| 18 | - 仅支持cast从float16/bfloat16转float32。 | 19 | - 仅支持cast从float16/bfloat16转float32。 |
| @@ -0,0 +1,79 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +# -*- coding: utf-8 -*- | ||
| 3 | +# ---------------------------------------------------------------------------- | ||
| 4 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 5 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 6 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 7 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 8 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 9 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 10 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 11 | +# ---------------------------------------------------------------------------- | ||
| 12 | +"""LayerNorm references for Kernel/GEIR.""" | ||
| 13 | + | ||
| 14 | +import numpy as np | ||
| 15 | +import torch | ||
| 16 | + | ||
| 17 | +__golden__ = {"kernel": {"layer_norm": "layer_norm_golden"}} | ||
| 18 | + | ||
| 19 | +__spec__ = { | ||
| 20 | + "layer_norm": "LayerNormTestSpec", | ||
| 21 | +} | ||
| 22 | + | ||
| 23 | + | ||
| 24 | +def _check_axes_and_params(x, gamma, beta, begin_norm_axis, begin_params_axis): | ||
| 25 | + """校验轴范围和参数形状。""" | ||
| 26 | + for name, axis in ( | ||
| 27 | + ("begin_norm_axis", begin_norm_axis), | ||
| 28 | + ("begin_params_axis", begin_params_axis), | ||
| 29 | + ): | ||
| 30 | + if not -x.ndim <= axis < x.ndim: | ||
| 31 | + raise ValueError(f"{name} must be in [-x.ndim, x.ndim)") | ||
| 32 | + norm_axis = begin_norm_axis % x.ndim | ||
| 33 | + params_axis = begin_params_axis % x.ndim | ||
| 34 | + params_shape = tuple(x.shape[params_axis:]) | ||
| 35 | + if tuple(gamma.shape) != params_shape or tuple(beta.shape) != params_shape: | ||
| 36 | + raise ValueError("gamma and beta must match x.shape[begin_params_axis:]") | ||
| 37 | + return tuple(range(norm_axis, x.ndim)) | ||
| 38 | + | ||
| 39 | + | ||
| 40 | +def layer_norm_golden( | ||
| 41 | + x, gamma, beta, *, begin_norm_axis=0, begin_params_axis=0, epsilon=1e-12, **kwargs | ||
| 42 | +): | ||
| 43 | + """FP64 计算,统计量 dtype 跟随 gamma。""" | ||
| 44 | + axes = _check_axes_and_params(x, gamma, beta, begin_norm_axis, begin_params_axis) | ||
| 45 | + x_fp64 = x.astype(np.float64) | ||
| 46 | + mean = np.mean(x_fp64, axis=axes, keepdims=True) | ||
| 47 | + variance = np.mean(np.square(x_fp64 - mean), axis=axes, keepdims=True) | ||
| 48 | + rstd = np.reciprocal(np.sqrt(variance + float(epsilon))) | ||
| 49 | + y = (x_fp64 - mean) * rstd * gamma.astype(np.float64) + beta.astype(np.float64) | ||
| 50 | + return [y.astype(x.dtype), mean.astype(gamma.dtype), variance.astype(gamma.dtype)] | ||
| 51 | + | ||
| 52 | + | ||
| 53 | +class LayerNormTorchThirdParty: | ||
| 54 | + """Torch 三方对照,低精度输入用 FP32 累积。""" | ||
| 55 | + | ||
| 56 | + def __call__( | ||
| 57 | + self, | ||
| 58 | + x, | ||
| 59 | + gamma, | ||
| 60 | + beta, | ||
| 61 | + begin_norm_axis=0, | ||
| 62 | + begin_params_axis=0, | ||
| 63 | + epsilon=1e-12, | ||
| 64 | + **kwargs, | ||
| 65 | + ): | ||
| 66 | + axes = _check_axes_and_params( | ||
| 67 | + x, gamma, beta, begin_norm_axis, begin_params_axis | ||
| 68 | + ) | ||
| 69 | + dtype = torch.float32 if x.dtype in (torch.float16, torch.bfloat16) else x.dtype | ||
| 70 | + x_t = x.to(dtype) | ||
| 71 | + variance, mean = torch.var_mean(x_t, dim=axes, unbiased=False, keepdim=True) | ||
| 72 | + rstd = torch.rsqrt(variance + float(epsilon)) | ||
| 73 | + y = (x_t - mean) * rstd * gamma.to(dtype) + beta.to(dtype) | ||
| 74 | + return [y.to(x.dtype), mean.to(gamma.dtype), variance.to(gamma.dtype)] | ||
| 75 | + | ||
| 76 | + | ||
| 77 | +class LayerNormTestSpec: | ||
| 78 | + golden = staticmethod(layer_norm_golden) | ||
| 79 | + third_party = {"torch": LayerNormTorchThirdParty} | ||
| @@ -0,0 +1,79 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +# -*- coding: utf-8 -*- | ||
| 3 | +# ---------------------------------------------------------------------------- | ||
| 4 | +# Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 5 | +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 6 | +# CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 7 | +# Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 8 | +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 9 | +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 10 | +# See LICENSE in the root of the software repository for the full text of the License. | ||
| 11 | +# ---------------------------------------------------------------------------- | ||
| 12 | +"""LayerNormV3 references for Kernel/GEIR.""" | ||
| 13 | + | ||
| 14 | +import numpy as np | ||
| 15 | +import torch | ||
| 16 | + | ||
| 17 | +__golden__ = {"kernel": {"layer_norm_v3": "layer_norm_v3_golden"}} | ||
| 18 | + | ||
| 19 | +__spec__ = { | ||
| 20 | + "layer_norm_v3": "LayerNormV3TestSpec", | ||
| 21 | +} | ||
| 22 | + | ||
| 23 | + | ||
| 24 | +def _check_axes_and_params(x, gamma, beta, begin_norm_axis, begin_params_axis): | ||
| 25 | + """校验轴范围和参数形状。""" | ||
| 26 | + for name, axis in ( | ||
| 27 | + ("begin_norm_axis", begin_norm_axis), | ||
| 28 | + ("begin_params_axis", begin_params_axis), | ||
| 29 | + ): | ||
| 30 | + if not -x.ndim <= axis < x.ndim: | ||
| 31 | + raise ValueError(f"{name} must be in [-x.ndim, x.ndim)") | ||
| 32 | + norm_axis = begin_norm_axis % x.ndim | ||
| 33 | + params_axis = begin_params_axis % x.ndim | ||
| 34 | + params_shape = tuple(x.shape[params_axis:]) | ||
| 35 | + if tuple(gamma.shape) != params_shape or tuple(beta.shape) != params_shape: | ||
| 36 | + raise ValueError("gamma and beta must match x.shape[begin_params_axis:]") | ||
| 37 | + return tuple(range(norm_axis, x.ndim)) | ||
| 38 | + | ||
| 39 | + | ||
| 40 | +def layer_norm_v3_golden( | ||
| 41 | + x, gamma, beta, *, begin_norm_axis=0, begin_params_axis=0, epsilon=1e-5, **kwargs | ||
| 42 | +): | ||
| 43 | + """FP64 计算,统计量 dtype 跟随 gamma。""" | ||
| 44 | + axes = _check_axes_and_params(x, gamma, beta, begin_norm_axis, begin_params_axis) | ||
| 45 | + x_fp64 = x.astype(np.float64) | ||
| 46 | + mean = np.mean(x_fp64, axis=axes, keepdims=True) | ||
| 47 | + variance = np.mean(np.square(x_fp64 - mean), axis=axes, keepdims=True) | ||
| 48 | + rstd = np.reciprocal(np.sqrt(variance + float(epsilon))) | ||
| 49 | + y = (x_fp64 - mean) * rstd * gamma.astype(np.float64) + beta.astype(np.float64) | ||
| 50 | + return [y.astype(x.dtype), mean.astype(gamma.dtype), rstd.astype(gamma.dtype)] | ||
| 51 | + | ||
| 52 | + | ||
| 53 | +class LayerNormV3TorchThirdParty: | ||
| 54 | + """Torch 三方对照,低精度输入用 FP32 累积。""" | ||
| 55 | + | ||
| 56 | + def __call__( | ||
| 57 | + self, | ||
| 58 | + x, | ||
| 59 | + gamma, | ||
| 60 | + beta, | ||
| 61 | + begin_norm_axis=0, | ||
| 62 | + begin_params_axis=0, | ||
| 63 | + epsilon=1e-5, | ||
| 64 | + **kwargs, | ||
| 65 | + ): | ||
| 66 | + axes = _check_axes_and_params( | ||
| 67 | + x, gamma, beta, begin_norm_axis, begin_params_axis | ||
| 68 | + ) | ||
| 69 | + dtype = torch.float32 if x.dtype in (torch.float16, torch.bfloat16) else x.dtype | ||
| 70 | + x_t = x.to(dtype) | ||
| 71 | + variance, mean = torch.var_mean(x_t, dim=axes, unbiased=False, keepdim=True) | ||
| 72 | + rstd = torch.rsqrt(variance + float(epsilon)) | ||
| 73 | + y = (x_t - mean) * rstd * gamma.to(dtype) + beta.to(dtype) | ||
| 74 | + return [y.to(x.dtype), mean.to(gamma.dtype), rstd.to(gamma.dtype)] | ||
| 75 | + | ||
| 76 | + | ||
| 77 | +class LayerNormV3TestSpec: | ||
| 78 | + golden = staticmethod(layer_norm_v3_golden) | ||
| 79 | + third_party = {"torch": LayerNormV3TorchThirdParty} | ||


🟡 Medium Priority
改动点:本次 diff 在
add_layer_norm_golden中新增了空张量分支(第 62-66 行:0 in normalized_shape时 mean_out=0、rstd_out=NaN),同时新增了AddLayerNormTorchThirdParty(第 84-118 行),但该三方实现直接调用torch.ops.aten.native_layer_norm(第 105-111 行),没有任何空归一化维分支。 契约影响:golden(官方参考)与 third_party(cross_check 基准)对同一个空用例给出不同期望值——golden 的 mean=0,而 aten 对 R==0 的归约得到 mean=0/0=NaN;kernel 侧已同步改为 mean=0(add_layer_norm_regbase_reduce_empty.h第 60-61 行)。 失败模式:当用例集包含gamma.shape含 0 的 shape(例如 x1=(A,0)、gamma=(0,))且走 cross_check 比对时,kernel 输出的 mean(0) 与 third_party 的 mean(NaN) 不一致,判定失败;或 golden 与 third_party 互检时报不一致,出现红测。注意改动前该文件没有 third_party,此不一致由本次新增代码引入。建议:在 AddLayerNormTorchThirdParty.call 里在做 x1+x2 之后加入与 golden 相同的判断:
if 0 in tuple(gamma.shape): return (torch.empty(x.shape, dtype=out_dtype), torch.zeros(stat_shape, dtype=torch.float32), torch.full(stat_shape, float("nan"), dtype=torch.float32), x.to(out_dtype) if additional_output else None),stat_shape 与 golden 一致由 x1.shape/gamma.ndim 推出。