已合并
[docs][inductor]添加精度工具资料与阈值设置环境变量 #31423
AACAES创建于 3月5日
[docs][inductor]添加精度工具资料与阈值设置环境变量 #31423
已合并
共 5 个文件变更+101-6
| @@ -0,0 +1,45 @@ | |||
| 1 | +# INDUCTOR\_ASCEND\_CHECK\_ACCURACY | ||
| 2 | + | ||
M | |||
| 3 | +## 功能描述 | ||
| 4 | + | ||
| 5 | +INDUCTOR_ASCEND_CHECK_ACCURACY是Ascend Extension for PyTorch提供的精度校验工具，仅在torch.compile图编译后端为"Inductor"且模式为"Triton"时自动检测融合算子的数值精度。 | ||
| 6 | + | ||
| 7 | +该工具可捕获融合算子对应的FX子图，生成独立可执行的单算子测试用例，并在相同输入条件下比对eager与triton的输出差异。当差异超出预设阈值时，自动输出精度校验失败日志及诊断信息，辅助开发者快速定位精度问题。 | ||
| 8 | + | ||
| 9 | +## 配置示例 | ||
| 10 | + | ||
| 11 | +示例一：启用精度工具，使用默认精度阈值配置 | ||
| 12 | + | ||
| 13 | +``` | ||
| 14 | +export INDUCTOR_ASCEND_CHECK_ACCURACY=1 | ||
| 15 | +``` | ||
| 16 | + | ||
| 17 | +**表 1** 默认精度阈值 | ||
| 18 | + | ||
| 19 | +| 数据类型 | 相对误差rtol | 绝对误差atol | | ||
| 20 | +|:---:|:---:|:---:| | ||
| 21 | +| float32 | 1.3e-6 | 1e-5 | | ||
| 22 | +| float16 | 1e-3 | 1e-5 | | ||
| 23 | +| bfloat16 | 1.6e-2 | 1e-5 | | ||
| 24 | +| 其他 | 1.3e-6 | 1e-5 | | ||
| 25 | + | ||
| 26 | + | ||
| 27 | +示例二：启用精度工具，并设置精度对比阈值 | ||
| 28 | + | ||
| 29 | +``` | ||
| 30 | +export INDUCTOR_ASCEND_CHECK_ACCURACY=1 | ||
| 31 | +# 设置精度对比时相对误差阈值为1e-6，绝对误差阈值为1e-7 | ||
| 32 | +export INDUCTOR_ASCEND_CHECK_ACCURACY_RTOL_ATOL="rtol=1e-6,atol=1e-7" | ||
| 33 | +``` | ||
| 34 | + | ||
| 35 | +> [!CAUTION] 注意<br> | ||
| 36 | +如需根据不同数据类型（如float32、float16、bfloat16等）配置不同的精度阈值，请手动修改源码[config](../../../torch_npu/_inductor/config.py)中的acc_comp_tol字典。 | ||
| 37 | + | ||
| 38 | +## 使用约束 | ||
| 39 | + | ||
| 40 | +在torch.compile图编译后端为"Inductor"且模式为"Triton"（环境变量TORCHINDUCTOR_NPU_BACKEND为空或者"default"）时可使用此环境变量。 | ||
| 41 | + | ||
| 42 | +## 支持的型号 | ||
| 43 | + | ||
| 44 | +- <term>Atlas A2训练系列产品</term> | ||
| 45 | +- <term>Atlas A3训练系列产品</term> | ||
| @@ -41,4 +41,5 @@ | |||
| 41 | |设备管理|[STREAMS_PER_DEVICE](STREAMS_PER_DEVICE.md)|通过此环境变量可配置stream pool的最大流数。| | 41 | |设备管理|[STREAMS_PER_DEVICE](STREAMS_PER_DEVICE.md)|通过此环境变量可配置stream pool的最大流数。| |
| 42 | |设备管理|[TORCH_NPU_DEVICE_CAPABILITY](TORCH_NPU_DEVICE_CAPABILITY.md)|通过此环境变量可配置`torch_npu.npu.get_device_capability()`的返回值。| | 42 | |设备管理|[TORCH_NPU_DEVICE_CAPABILITY](TORCH_NPU_DEVICE_CAPABILITY.md)|通过此环境变量可配置`torch_npu.npu.get_device_capability()`的返回值。| |
| 43 | |设备管理|[TORCH_TRANSFER_TO_NPU](TORCH_TRANSFER_TO_NPU.md)|通过此环境变量可配置是否自动启用transfer_to_npu功能，将PyTorch的CUDA相关API自动替换为NPU对应API。| | 43 | |设备管理|[TORCH_TRANSFER_TO_NPU](TORCH_TRANSFER_TO_NPU.md)|通过此环境变量可配置是否自动启用transfer_to_npu功能，将PyTorch的CUDA相关API自动替换为NPU对应API。| |
| 44 | -|图模式|[TORCHINDUCTOR_NPU_BACKEND](TORCHINDUCTOR_NPU_BACKEND.md)|通过该环境变量可配置图模式下的优化模式，支持Triton、MLIR、DVM等优化模式。| | 44 | +|图模式|[TORCHINDUCTOR_NPU_BACKEND](TORCHINDUCTOR_NPU_BACKEND.md)|通过该环境变量可配置图模式下的优化模式，支持Triton、MLIR、DVM等优化模式。| |
| 45 | +|图模式|[INDUCTOR_ASCEND_CHECK_ACCURACY](INDUCTOR_ASCEND_CHECK_ACCURACY.md)|INDUCTOR_ASCEND_CHECK_ACCURACY是Ascend Extension for PyTorch提供的精度校验工具，仅在torch.compile图编译后端为"Inductor"且模式为"Triton"时自动检测融合算子的数值精度。| | ||
| @@ -1,3 +1,5 @@ | |||
| 1 | # 图模式 | 1 | # 图模式 |
| 2 | 2 | ||
| 3 | -- **[TORCHINDUCTOR\_NPU\_BACKEND](TORCHINDUCTOR_NPU_BACKEND.md)** | 3 | +- **[TORCHINDUCTOR\_NPU\_BACKEND](TORCHINDUCTOR_NPU_BACKEND.md)** |
| 4 | + | ||
| 5 | +- **[INDUCTOR\_ASCEND\_CHECK\_ACCURACY](INDUCTOR_ASCEND_CHECK_ACCURACY.md)** | ||
| @@ -1,5 +1,6 @@ | |||
| 1 | import logging | 1 | import logging |
| 2 | import os # noqa: C101 | 2 | import os # noqa: C101 |
| 3 | +import re | ||
| 3 | from typing import Any, Callable, Dict, Optional, TYPE_CHECKING | 4 | from typing import Any, Callable, Dict, Optional, TYPE_CHECKING |
| 4 | 5 | ||
| 5 | import torch | 6 | import torch |
| @@ -143,11 +144,56 @@ dump_fx_graph = os.environ.get("INDUCTOR_ASCEND_DUMP_FX_GRAPH", False) \ | |||
| 143 | # (2) [1, 2, 10] means try to fallback kernel like triton_xxx_1, triton_xxx_2 and triton_xxx_10 | 144 | # (2) [1, 2, 10] means try to fallback kernel like triton_xxx_1, triton_xxx_2 and triton_xxx_10 |
| 144 | force_fallback_kernel_id = [] | 145 | force_fallback_kernel_id = [] |
| 145 | 146 | ||
| 147 | + | ||
| 148 | +def parse_rtol_atol(env_str: str): | ||
| 149 | + rtol, atol = None, None | ||
| 150 | + if not env_str.strip(): | ||
| 151 | + return rtol, atol | ||
| 152 | + | ||
| 153 | + parts = [p.strip() for p in env_str.split(",") if p.strip()] | ||
| 154 | + for part in parts: | ||
| 155 | + match = re.match(r"^(rtol|atol)\s*=\s*([0-9.eE+-]+)$", part, re.IGNORECASE) | ||
| 156 | + if not match: | ||
| 157 | + logging.warning(f"INDUCTOR_ASCEND_CHECK_ACCURACY_RTOL_ATOL environment variable has invalid format: {part}. " | ||
| 158 | + f"It should be like 'rtol=1e-6,atol=1e-5'.") | ||
| 159 | + continue | ||
| 160 | + | ||
| 161 | + key, value_str = match.groups() | ||
| 162 | + try: | ||
| 163 | + value = float(value_str) | ||
| 164 | + if key.lower() == "rtol": | ||
| 165 | + rtol = value | ||
| 166 | + elif key.lower() == "atol": | ||
| 167 | + atol = value | ||
| 168 | + except ValueError: | ||
| 169 | + logging.warning(f"INDUCTOR_ASCEND_CHECK_ACCURACY_RTOL_ATOL environment variable has invalid value for {key}: {value_str}. " | ||
| 170 | + f"It should be a float number.") | ||
| 171 | + continue | ||
| 172 | + | ||
| 173 | + return rtol, atol | ||
| 174 | + | ||
| 175 | +# Default threshold | ||
| 176 | +rtol_f32 = 1.3e-6 | ||
| 177 | +rtol_f16 = 1e-3 | ||
| 178 | +rtol_bf16 = 1.6e-2 | ||
| 179 | +rtol_default = 1.3e-6 | ||
| 180 | +atol_default = 1e-5 | ||
| 181 | + | ||
| 182 | +if dump_fx_graph: | ||
| 183 | + # Configure accuracy comparison thresholds when check_accuracy is enabled | ||
| 184 | + ENV_TOL_STR = os.environ.get("INDUCTOR_ASCEND_CHECK_ACCURACY_RTOL_ATOL", "") | ||
| 185 | + rtol_custom, atol_custom = parse_rtol_atol(ENV_TOL_STR) | ||
| 186 | + | ||
| 187 | + if rtol_custom is not None: | ||
| 188 | + rtol_f32 = rtol_f16 = rtol_bf16 = rtol_default = rtol_custom | ||
| 189 | + if atol_custom is not None: | ||
| 190 | + atol_default = atol_custom | ||
| 191 | + | ||
| 146 | acc_comp_tol = { | 192 | acc_comp_tol = { |
| 147 | - torch.float32: {'rtol': 1.3e-6, 'atol': 1e-5}, | 193 | + torch.float32: {"rtol": rtol_f32, "atol": atol_default}, |
| 148 | - torch.float16: {'rtol': 1e-3, 'atol': 1e-5}, | 194 | + torch.float16: {"rtol": rtol_f16, "atol": atol_default}, |
| 149 | - torch.bfloat16: {'rtol': 1.6e-2, 'atol': 1e-5}, | 195 | + torch.bfloat16: {"rtol": rtol_bf16, "atol": atol_default}, |
| 150 | - "default": {'rtol': 1.3e-6, 'atol': 1e-5}, | 196 | + "default": {"rtol": rtol_default, "atol": atol_default}, |
| 151 | } | 197 | } |
| 152 | 198 | ||
| 153 | ub_size = 192 * 1024 | 199 | ub_size = 192 * 1024 |
docs/zh/environment_variable_reference/menu_env_variable_reference.md 中要增加此环境变量 在相应的所属模块也要增加这个的入口