Merged
fix(cosh): 提升 arch35 大值精度并补齐 golden 注册与 E2E 兼容 #7643
yefeicodingAdd on 3 days ago
fix(cosh): 提升 arch35 大值精度并补齐 golden 注册与 E2E 兼容 #7643
Merged
yefeicodingAdd on 3 days ago
Total: 3 files changed+99-30
@@ -14,7 +14,11 @@ import numpy as np
14 14 
15__golden__ = {"kernel": {"diag_flat": "diag_flat_golden"}}15__golden__ = {"kernel": {"diag_flat": "diag_flat_golden"}}
16 16 
17-__spec__ = {"diag_flat": "DiagFlatKernelSpec", "aclnnDiagFlat": "AclnnDiagFlatSpec"}17+__spec__ = {
18+ "diag_flat": "DiagFlatKernelSpec",
19+ "DiagFlat": "DiagFlatKernelSpec", # GE 类型名双注册(REG_OP(DiagFlat)):GEIR op_name 为 DiagFlat 时同样命中
20+ "aclnnDiagFlat": "AclnnDiagFlatSpec",
21+}
18 22 
19 23 
20def _bf16_numpy_dtype():24def _bf16_numpy_dtype():
@@ -52,11 +56,27 @@ def diag_flat_golden(x, **kwargs):
52 return np.diagflat(x, k=diagonal)56 return np.diagflat(x, k=diagonal)
53 57 
54 58 
59+class _TorchDiagFlatReference:
60+ """torch.diagflat 三方对照:属性名 diagonal 需映射为 torch 的 offset。"""
61+ 
62+ def __call__(self, *args, **kwargs):
63+ import torch # 三方实现只在远端执行,本地 golden 保持 numpy-only
64+ 
65+ x = kwargs.get("x")
66+ if x is None:
67+ x = kwargs.get("self")
68+ if x is None and args:
69+ x = args[0]
70+ return [torch.diagflat(x, offset=int(kwargs.get("diagonal", 0)))]
71+ 
72+ 
55class DiagFlatKernelSpec:73class DiagFlatKernelSpec:
56 @staticmethod74 @staticmethod
57 def golden(x, **kwargs):75 def golden(x, **kwargs):
58 return [diag_flat_golden(x, **kwargs)]76 return [diag_flat_golden(x, **kwargs)]
59 77 
78+ third_party = {"torch": _TorchDiagFlatReference}
79+ 
60 tolerance = {80 tolerance = {
61 "float16": {"standard": "binary_equal"},81 "float16": {"standard": "binary_equal"},
62 "float32": {"standard": "binary_equal"},82 "float32": {"standard": "binary_equal"},
@@ -88,4 +108,6 @@ class AclnnDiagFlatSpec:
88 )108 )
89 return [np.diagflat(_as_numpy(args[0]), k=int(diagonal))]109 return [np.diagflat(_as_numpy(args[0]), k=int(diagonal))]
90 110 
111+ third_party = {"torch": _TorchDiagFlatReference}
112+ 
91 tolerance = DiagFlatKernelSpec.tolerance113 tolerance = DiagFlatKernelSpec.tolerance
@@ -18,17 +18,19 @@
18 * 数值流程:换底 + 范围缩减(Cody-Waite)+ 位操作 2^n:18 * 数值流程:换底 + 范围缩减(Cody-Waite)+ 位操作 2^n:
19 *19 *
20 * ① ax = |x|20 * ① ax = |x|
21- * ② n = trunc(ax · log2e) // 换底后取整,n ≥ 021+ * ② n = round(ax · log2e) // 四舍五入取整,n ≥ 0,使 |r| ≤ ln2/2
22- * ③ n = min(n, 126) // 钳位(copysign(126,n) 于 ax≥0 对应 min,在缩减之前)22+ * ③ n = min(n, 130) // 钳位(copysign(130,n) 于 ax≥0 对应 min,在缩减之前)
23 * ④⑤ r = ax − n·ln2 (Cody-Waite 双 FMA,单次舍入,用钳位后的 n)23 * ④⑤ r = ax − n·ln2 (Cody-Waite 双 FMA,单次舍入,用钳位后的 n)
24 * ⑥⑨ p2 = 2^(n−2) (n + 0x4B40007D 位重解释后 <<23,低 9 位 125+n 落入 IEEE754 指数字段)24 * ⑥⑨ p2 = 2^(n−2) (n + 0x4B40007D 位重解释后 <<23,低 9 位 125+n 落入 IEEE754 指数字段)
25- * ⑩⑪ er = exp(r) · 2^(n−2) = e^x / 4 // 用 base-e Exp(r)=e^r(无硬件 2^x 指令)25+ * ⑩ exp(r) = Σ r^k/k! (k≤7) // FMA Horner,|r|≤ln2/2 时相对误差 <1 ulp
26+ * ⑪ er = exp(r) · 2^(n−2) = e^x / 4
26 * ⑫⑮ y = 2·er + 0.125/er = e^x/2 + e^(−x)/2 = cosh(x)27 * ⑫⑮ y = 2·er + 0.125/er = e^x/2 + e^(−x)/2 = cosh(x)
27 * ⑯⑰ 溢出兜底:|x| ≥ 90 ⇒ +inf;[≈89.42, 90) 由 2·(e^x/4) 自然溢出到 inf28 * ⑯⑰ 溢出兜底:|x| ≥ 90 ⇒ +inf;[≈89.42, 90) 由 2·(e^x/4) 自然溢出到 inf
28 *29 *
29 * ⚠ 数值实现的关键点(易错,务必保持):30 * ⚠ 数值实现的关键点(易错,务必保持):
30- * 1. n 钳位 min(n,126) 必须在 Cody-Waite 之前:r 与 2^(n−2) 用同一 n 合成才等于 e^x/4;31+ * 1. n 钳位 min(n,130) 必须在 Cody-Waite 之前:r 与 2^(n−2) 用同一 n 合成才等于 e^x/4;
31- * 钳位到 126 使指数字段 125+n≤251<256,位构造不产生 inf/nan;被钳元素残差 r 变大由 Exp 吸收。32+ * 钳位到 130 使指数字段 125+n≤255;n=130 时 p2=2^128 位构造为 +inf(e^x/4 已溢出,正确)。
33+ * ax<90 时 n=round(ax·log2e)≤130,故不会触发钳位,|r|≤ln2/2;ax≥90 由溢出兜底置 +inf。
32 * 2. 2^(n−2) 位操作 magic=0x4B40007D(=1.5·2^23+125,含 +125 偏置):(n+magic)<<23 靠 32 位34 * 2. 2^(n−2) 位操作 magic=0x4B40007D(=1.5·2^23+125,含 +125 偏置):(n+magic)<<23 靠 32 位
33 * 逐位左移高位丢弃、低 9 位=125+n 落入指数字段。35 * 逐位左移高位丢弃、低 9 位=125+n 落入指数字段。
34 * 3. Cody-Waite 低位项符号为 +ln2_lo(+1.9046e-9,bit 0x3102E308)。36 * 3. Cody-Waite 低位项符号为 +ln2_lo(+1.9046e-9,bit 0x3102E308)。
@@ -74,11 +76,20 @@ class KernelCosh {
74 static constexpr float kExpMagic = 12583037.0f; // bit 0x4B40007D = 1.5·2^23 + 125(含 +125 偏置);76 static constexpr float kExpMagic = 12583037.0f; // bit 0x4B40007D = 1.5·2^23 + 125(含 +125 偏置);
75 // (n+magic) 位重解释后 <<23,低 9 位 125+n 落入指数字段 ⇒ 2^(n−2)77 // (n+magic) 位重解释后 <<23,低 9 位 125+n 落入指数字段 ⇒ 2^(n−2)
76 static constexpr int32_t kFp32MantissaBits = 23; // 左移量 = fp32 尾数位宽78 static constexpr int32_t kFp32MantissaBits = 23; // 左移量 = fp32 尾数位宽
77- static constexpr float kNClamp = 126.0f; // n 钳位上限(ax≥0 即 min(n,126));79+ static constexpr float kNClamp = 130.0f; // n 钳位上限(ax≥0 即 min(n,130));
78- // 保证 125+n≤251<256,位构造不越指数字段80+ // 125+n≤255;n=130 的 p2 位构造为 +inf(已溢出,正确)
79- static constexpr float kHalfInv8 = 0.125f; // e^(−x)/2 分子 0.125,bit 0x3E00000081+ static constexpr float kHalfInv8 = 0.125f; // e^(−x)/2 分子 0.125,bit 0x3E000000
80- static constexpr float kTwo = 2.0f; // e^x/4 → e^x/2 系数,bit 0x4000000082+ static constexpr float kTwo = 2.0f; // e^x/4 → e^x/2 系数,bit 0x40000000
81 static constexpr float kOvfThreshold = 90.0f; // |x|≥90 ⇒ +inf,bit 0x42B40000([≈89.42,90) 由自然溢出兜底)83 static constexpr float kOvfThreshold = 90.0f; // |x|≥90 ⇒ +inf,bit 0x42B40000([≈89.42,90) 由自然溢出兜底)
84+ // exp(r) 的 7 阶 Taylor 系数 1/k!(FMA Horner,|r|≤ln2/2)
85+ static constexpr float kExpC7 = 1.984126984e-4f; // 1/5040
86+ static constexpr float kExpC6 = 1.388888889e-3f; // 1/720
87+ static constexpr float kExpC5 = 8.333333333e-3f; // 1/120
88+ static constexpr float kExpC4 = 4.166666667e-2f; // 1/24
89+ static constexpr float kExpC3 = 1.666666667e-1f; // 1/6
90+ static constexpr float kExpC2 = 5.0e-1f; // 1/2
91+ static constexpr float kExpC1 = 1.0f;
92+ static constexpr float kExpC0 = 1.0f;
82 93 
83public:94public:
84 __aicore__ inline KernelCosh() {}95 __aicore__ inline KernelCosh() {}
@@ -201,12 +212,13 @@ __aicore__ inline void KernelCosh<T>::Compute(int64_t currentNum)
201 AscendC::Abs(ax, xLocal, cnt); // fp32 直通212 AscendC::Abs(ax, xLocal, cnt); // fp32 直通
202 }213 }
203 214 
204- // ── ② 换底取整 n = trunc(ax · log2e) ──215+ // ── ② 换底取整 n = round(ax · log2e):四舍五入使 |r| ≤ ln2/2,缩短多项式所需区间 ──
205 AscendC::Muls(n, ax, kLog2e, cnt);216 AscendC::Muls(n, ax, kLog2e, cnt);
206- AscendC::Trunc(n, n, cntU); // 高阶 API,count 形免 tmpBuffer 重载217+ AscendC::Adds(n, n, 0.5f, cnt); // ax≥0 ⇒ trunc(v+0.5) 即四舍五入
218+ AscendC::Trunc(n, n, cntU); // 高阶 API,count 形免 tmpBuffer 重载
207 219 
208- // ── ③ n 钳位 min(n,126)(钳位在 Cody-Waite 之前,使 r 与 2^(n−2) 用同一 n 合成)──220+ // ── ③ n 钳位 min(n,130)(钳位在 Cody-Waite 之前,使 r 与 2^(n−2) 用同一 n 合成)──
209- // ax≥0 ⇒ n≥0,copysign(126,n) 即 min(n,126);被钳元素残差 r 变大由 Exp 吸收,e^r·2^(n−2)=e^x/4 仍精确。221+ // ax≥0 ⇒ n≥0,copysign(130,n) 即 min(n,130)。ax<90 时 n≤130 不触发钳位;ax≥90 由溢出兜底置 +inf。
210 AscendC::Mins(n, n, kNClamp, cnt);222 AscendC::Mins(n, n, kNClamp, cnt);
211 223 
212 // ── ④⑤ Cody-Waite 双 FMA 范围缩减 r = ax − n·ln2(用钳位后的 n,单次舍入,dst≠src2 避免地址重叠)──224 // ── ④⑤ Cody-Waite 双 FMA 范围缩减 r = ax − n·ln2(用钳位后的 n,单次舍入,dst≠src2 避免地址重叠)──
@@ -214,16 +226,35 @@ __aicore__ inline void KernelCosh<T>::Compute(int64_t currentNum)
214 AscendC::Fma(er, n, cwLo, r, cntU); // er = n·(+ln2_lo) + r → 缩减后的 r226 AscendC::Fma(er, n, cwLo, r, cntU); // er = n·(+ln2_lo) + r → 缩减后的 r
215 227 
216 // ── ⑥⑨ 位操作构造 p2 = 2^(n−2)(magic 加 + 左移 23 构造 IEEE754 指数字段)──228 // ── ⑥⑨ 位操作构造 p2 = 2^(n−2)(magic 加 + 左移 23 构造 IEEE754 指数字段)──
217- // (n + 0x4B40007D) 位重解释后 <<23:32 位逐位左移,高位丢弃、低 9 位 (125+n)≤251 落入 IEEE754 指数字段;229+ // (n + 0x4B40007D) 位重解释后 <<23:32 位逐位左移,高位丢弃、低 9 位 (125+n)≤255 落入 IEEE754 指数字段;
218- // n≤126 保证指数字段有效(不产生 inf/nan)。230+ // n≤129 为有限 2^(n−2),n=130 位构造为 +inf(对应 e^x/4 已溢出),均不越指数字段。
219 AscendC::Adds(p2, n, kExpMagic, cnt);231 AscendC::Adds(p2, n, kExpMagic, cnt);
220 AscendC::LocalTensor<int32_t> p2i = p2.ReinterpretCast<int32_t>();232 AscendC::LocalTensor<int32_t> p2i = p2.ReinterpretCast<int32_t>();
221 AscendC::ShiftLeft(p2i, p2i, kFp32MantissaBits, cnt);233 AscendC::ShiftLeft(p2i, p2i, kFp32MantissaBits, cnt);
222 // p2(float 视图)现为 2^((125+n)−127) = 2^(n−2)234 // p2(float 视图)现为 2^((125+n)−127) = 2^(n−2)
223 235 
224- // ── ⑩⑪ 合成 e^x/4 = exp(r) · 2^(n−2) ──236+ // ── ⑩ 求 exp(r):FMA 单次舍入的 7 阶 Taylor(|r| ≤ ln2/2),
225- AscendC::Exp(er, er, cnt); // er = exp(r)(缩减小区间,高精度)237+ // 比单指令向量 Exp(~1–2 ulp)更接近正确舍入。nBuf 已不再需要,借作 ping-pong 暂存。──
226- AscendC::Mul(er, er, p2, cnt); // er = e^x / 4238+ AscendC::Duplicate(work, kExpC7, cnt);
239+ AscendC::Duplicate(cwHi, kExpC6, cnt);
240+ AscendC::Fma(n, work, er, cwHi, cntU); // n = work·r + c6
241+ AscendC::Duplicate(cwHi, kExpC5, cnt);
242+ AscendC::Fma(work, n, er, cwHi, cntU);
243+ AscendC::Duplicate(cwHi, kExpC4, cnt);
244+ AscendC::Fma(n, work, er, cwHi, cntU);
245+ AscendC::Duplicate(cwHi, kExpC3, cnt);
246+ AscendC::Fma(work, n, er, cwHi, cntU);
247+ AscendC::Duplicate(cwHi, kExpC2, cnt);
248+ AscendC::Fma(n, work, er, cwHi, cntU);
249+ AscendC::Duplicate(cwHi, kExpC1, cnt);
250+ AscendC::Fma(work, n, er, cwHi, cntU);
251+ AscendC::Duplicate(cwHi, kExpC0, cnt);
252+ AscendC::Fma(n, work, er, cwHi, cntU); // n = exp(r)
253+ // cwHiBuf_ 本轮被系数复用,恢复为 Cody-Waite 常量,供下一轮 Compute 使用
254+ AscendC::Duplicate(cwHi, kNegLn2Hi, cnt);
255+ 
256+ // ── ⑪ 合成 e^x/4 = exp(r) · 2^(n−2) ──
257+ AscendC::Mul(er, n, p2, cnt); // er = e^x / 4
227 258 
228 // ── ⑫⑮ cosh 合成:y = 2·(e^x/4) + 0.125/(e^x/4) = e^x/2 + e^(−x)/2 ──259 // ── ⑫⑮ cosh 合成:y = 2·(e^x/4) + 0.125/(e^x/4) = e^x/2 + e^(−x)/2 ──
229 AscendC::Reciprocal(work, er, cnt); // work = 4 / e^x260 AscendC::Reciprocal(work, er, cnt); // work = 4 / e^x
@@ -12,8 +12,15 @@
12"""TestSpec definitions for Cosh on GEIR/Kernel, ACLNN, Torch Single/Graph, and TensorFlow E2E paths."""12"""TestSpec definitions for Cosh on GEIR/Kernel, ACLNN, Torch Single/Graph, and TensorFlow E2E paths."""
13 13 
14import numpy as np14import numpy as np
15-import torch
16 15 
16+try:
17+ import torch
18+except ImportError: # TF-only Python (ttk geir): keep numpy/TF TestSpecs loadable
19+ torch = None
20+ 
21+_TORCH_TENSOR = torch.Tensor if torch is not None else ()
22+ 
23+# Static __spec__ index is AST-scanned by ttk; every key must stay in this literal.
17__spec__ = {24__spec__ = {
18 "cosh": "CoshKernelTestSpec",25 "cosh": "CoshKernelTestSpec",
19 "aclnnCosh": "AclnnCoshTestSpec",26 "aclnnCosh": "AclnnCoshTestSpec",
@@ -22,17 +29,18 @@ __spec__ = {
22 "tf.math.cosh": "CoshTensorFlowTestSpec",29 "tf.math.cosh": "CoshTensorFlowTestSpec",
23}30}
24 31 
32+if torch is not None:
25 33 
26-class CoshGraph(torch.nn.Module):34+ class CoshGraph(torch.nn.Module):
27- """Graph module consumed by TTK E2E torch.compile modes."""35+ """Graph module consumed by TTK E2E torch.compile modes."""
28 36 
29- def forward(self, input):37+ def forward(self, input):
30- return torch.cosh(input)38+ return torch.cosh(input)
31 39 
32 40 
33def _as_numpy(x):41def _as_numpy(x):
34 """Torch/TF/numpy -> numpy,供 CPU golden 统一计算。"""42 """Torch/TF/numpy -> numpy,供 CPU golden 统一计算。"""
35- if isinstance(x, torch.Tensor):43+ if _TORCH_TENSOR and isinstance(x, _TORCH_TENSOR):
36 if x.dtype == torch.bfloat16:44 if x.dtype == torch.bfloat16:
37 # numpy 无原生 bf16,torch 也不支持 bf16 张量直接 .numpy()45 # numpy 无原生 bf16,torch 也不支持 bf16 张量直接 .numpy()
38 # (TypeError: Got unsupported ScalarType BFloat16)。46 # (TypeError: Got unsupported ScalarType BFloat16)。
@@ -110,11 +118,18 @@ class _TensorFlowCosh:
110 return [tf.math.cosh(_get_tf_input(args, kwargs))]118 return [tf.math.cosh(_get_tf_input(args, kwargs))]
111 119 
112 120 
121+_THIRD_PARTY = (
122+ {"torch": "torch.cosh", "tf": _TensorFlowCosh}
123+ if torch is not None
124+ else {"tf": _TensorFlowCosh}
125+)
126+ 
127+ 
113class CoshKernelTestSpec:128class CoshKernelTestSpec:
114 """Shared GEIR/Kernel spec registered by CSV ``op_name``."""129 """Shared GEIR/Kernel spec registered by CSV ``op_name``."""
115 130 
116 golden = staticmethod(cosh_golden)131 golden = staticmethod(cosh_golden)
117- third_party = {"torch": "torch.cosh", "tf": _TensorFlowCosh}132+ third_party = _THIRD_PARTY
118 tolerance = {133 tolerance = {
119 "float16": {"standard": "cross_check", "level": "L0"},134 "float16": {"standard": "cross_check", "level": "L0"},
120 "float32": {"standard": "cross_check", "level": "L0"},135 "float32": {"standard": "cross_check", "level": "L0"},
@@ -126,7 +141,7 @@ class AclnnCoshTestSpec:
126 """ACLNN out-of-place spec registered by CSV ``api_name``."""141 """ACLNN out-of-place spec registered by CSV ``api_name``."""
127 142 
128 golden = staticmethod(aclnn_cosh_golden)143 golden = staticmethod(aclnn_cosh_golden)
129- third_party = {"torch": "torch.cosh", "tf": _TensorFlowCosh}144+ third_party = _THIRD_PARTY
130 tolerance = {145 tolerance = {
131 "float16": {"standard": "cross_check", "level": "L0"},146 "float16": {"standard": "cross_check", "level": "L0"},
132 "float32": {"standard": "cross_check", "level": "L0"},147 "float32": {"standard": "cross_check", "level": "L0"},
@@ -138,7 +153,7 @@ class AclnnInplaceCoshTestSpec:
138 """ACLNN inplace spec registered by CSV ``api_name``."""153 """ACLNN inplace spec registered by CSV ``api_name``."""
139 154 
140 golden = staticmethod(cosh_golden)155 golden = staticmethod(cosh_golden)
141- third_party = {"torch": "torch.cosh", "tf": _TensorFlowCosh}156+ third_party = _THIRD_PARTY
142 tolerance = {157 tolerance = {
143 "float16": {"standard": "cross_check", "level": "L0"},158 "float16": {"standard": "cross_check", "level": "L0"},
144 "float32": {"standard": "cross_check", "level": "L0"},159 "float32": {"standard": "cross_check", "level": "L0"},
@@ -150,8 +165,9 @@ class TorchCoshTestSpec:
150 """Shared Torch Single/Graph spec registered by CSV ``api_name``."""165 """Shared Torch Single/Graph spec registered by CSV ``api_name``."""
151 166 
152 golden = staticmethod(torch_cosh_golden)167 golden = staticmethod(torch_cosh_golden)
153- third_party = {"torch": "torch.cosh", "tf": _TensorFlowCosh}168+ third_party = _THIRD_PARTY
154- torch_graph = CoshGraph169+ if torch is not None:
170+ torch_graph = CoshGraph
155 tolerance = {171 tolerance = {
156 "float16": {"standard": "cross_check", "level": "L0"},172 "float16": {"standard": "cross_check", "level": "L0"},
157 "float32": {"standard": "cross_check", "level": "L0"},173 "float32": {"standard": "cross_check", "level": "L0"},