已合并
refactor: decouple the check accuracy feature from triton, mlir, dvm. #35311
yvjc创建于 5月11日
refactor: decouple the check accuracy feature from triton, mlir, dvm. #35311
已合并
从已删除 :decouple_accuracy_v271合入到Ascend/pytorchv2.7.1
共 8 个文件变更+246-290
| @@ -2,9 +2,9 @@ | |||
| 2 | 2 | ||
| 3 | ## 功能描述 | 3 | ## 功能描述 |
| 4 | 4 | ||
| 5 | -INDUCTOR_ASCEND_CHECK_ACCURACY是Ascend Extension for PyTorch提供的精度校验工具，仅在torch.compile图编译后端为"Inductor"且模式为"Triton"时自动检测融合算子的数值精度。 | 5 | +INDUCTOR_ASCEND_CHECK_ACCURACY是Ascend Extension for PyTorch提供的精度校验工具，在torch.compile图编译后端为"Inductor"时自动检测融合算子的数值精度。 |
| 6 | 6 | ||
| 7 | -该工具可捕获融合算子对应的FX子图，生成独立可执行的单算子测试用例，并在相同输入条件下比对eager与Triton的输出差异。当差异超出预设阈值时，自动输出精度校验失败日志及诊断信息，辅助开发者快速定位精度问题。 | 7 | +该工具可捕获融合算子对应的FX子图，生成独立可执行的单算子测试用例，并在相同输入条件下比对eager与融合算子的输出差异。当差异超出预设阈值时，自动输出精度校验失败日志及诊断信息，辅助开发者快速定位精度问题。 |
| 8 | 8 | ||
| 9 | ## 配置示例 | 9 | ## 配置示例 |
| 10 | 10 | ||
| @@ -36,9 +36,9 @@ export INDUCTOR_ASCEND_CHECK_ACCURACY_RTOL_ATOL="rtol=1e-6,atol=1e-7" | |||
| 36 | 36 | ||
| 37 | ## 使用约束 | 37 | ## 使用约束 |
| 38 | 38 | ||
| 39 | -- 此环境变量仅可在PyTorch2.7.1和PyTorch2.9.0版本使用。 | 39 | +- 此环境变量仅可在PyTorch2.7.1，PyTorch2.9.0和PyTorch2.10版本使用。 |
| 40 | 40 | ||
| 41 | -- 在torch.compile图编译后端为"Inductor"且模式为"Triton"（环境变量TORCHINDUCTOR_NPU_BACKEND为空或者"default"）时可使用此环境变量。 | 41 | +- 在torch.compile图编译后端为"Inductor"时可使用此环境变量。 |
| 42 | 42 | ||
| 43 | ## 支持的型号 | 43 | ## 支持的型号 |
| 44 | 44 | ||
| @@ -1,6 +1,7 @@ | |||
| 1 | import os | 1 | import os |
| 2 | from unittest.mock import patch | 2 | from unittest.mock import patch |
| 3 | -from unittest import skip | 3 | + |
| 4 | +os.environ["INDUCTOR_ASCEND_CHECK_ACCURACY"] = "1" | ||
| 4 | 5 | ||
| 5 | import torch | 6 | import torch |
| 6 | import torch.nn.functional as F | 7 | import torch.nn.functional as F |
| @@ -9,7 +10,6 @@ from testutils import TestUtils | |||
| 9 | import torch_npu | 10 | import torch_npu |
| 10 | 11 | ||
| 11 | torch._inductor.config.fx_graph_cache = False | 12 | torch._inductor.config.fx_graph_cache = False |
| 12 | -os.environ["INDUCTOR_ASCEND_CHECK_ACCURACY"] = "1" | ||
| 13 | 13 | ||
| 14 | 14 | ||
| 15 | class TestCheckAccuracy(TestUtils): | 15 | class TestCheckAccuracy(TestUtils): |
| @@ -29,11 +29,11 @@ class TestCheckAccuracy(TestUtils): | |||
| 29 | nonlocal count_data_dump | 29 | nonlocal count_data_dump |
| 30 | count_data_dump += 1 | 30 | count_data_dump += 1 |
| 31 | return status | 31 | return status |
| 32 | - | ||
| 33 | - src_check_accuracy = NPUCachingAutotuner.check_accuracy | ||
| 34 | 32 | ||
| 35 | - def wrap_check_accuracy(self, *args, **kwargs): | 33 | + src_check_accuracy = torch_npu._inductor.npu_compare.check_accuracy_triton |
| 36 | - status = src_check_accuracy(self, *args, **kwargs) | 34 | + |
| 35 | + def wrap_check_accuracy(*args, **kwargs): | ||
| 36 | + status = src_check_accuracy(*args, **kwargs) | ||
| 37 | if status: | 37 | if status: |
| 38 | nonlocal count_check_accuracy | 38 | nonlocal count_check_accuracy |
| 39 | count_check_accuracy += 1 | 39 | count_check_accuracy += 1 |
| @@ -48,7 +48,7 @@ class TestCheckAccuracy(TestUtils): | |||
| 48 | _ = run(x, y) | 48 | _ = run(x, y) |
| 49 | 49 | ||
| 50 | with patch.object(NPUCachingAutotuner, "data_dump", wrap_data_dump), \ | 50 | with patch.object(NPUCachingAutotuner, "data_dump", wrap_data_dump), \ |
| 51 | - patch.object(NPUCachingAutotuner, "check_accuracy", wrap_check_accuracy): | 51 | + patch.object(torch_npu._inductor.runtime.triton_heuristics, "check_accuracy_triton", wrap_check_accuracy): |
| 52 | self.assertTrue(torch_npu._inductor.config.dump_fx_graph) | 52 | self.assertTrue(torch_npu._inductor.config.dump_fx_graph) |
| 53 | self.assertTrue(torch_npu._inductor.config.check_accuracy) | 53 | self.assertTrue(torch_npu._inductor.config.check_accuracy) |
| 54 | 54 | ||
| @@ -69,4 +69,4 @@ class TestCheckAccuracy(TestUtils): | |||
| 69 | 69 | ||
| 70 | 70 | ||
| 71 | if __name__ == "__main__": | 71 | if __name__ == "__main__": |
| 72 | - run_tests() | 72 | + run_tests() |
| @@ -576,6 +576,7 @@ class TestPublicBindings(TestCase): | |||
| 576 | "torch_npu._inductor.dependencies", | 576 | "torch_npu._inductor.dependencies", |
| 577 | "torch_npu._inductor.npu_device", | 577 | "torch_npu._inductor.npu_device", |
| 578 | "torch_npu._inductor.npu_static_kernel", | 578 | "torch_npu._inductor.npu_static_kernel", |
| 579 | + "torch_npu._inductor.npu_compare", | ||
| 579 | "torch_npu._inductor.utils", | 580 | "torch_npu._inductor.utils", |
| 580 | "torch_npu._inductor.codegen._sizevars", | 581 | "torch_npu._inductor.codegen._sizevars", |
| 581 | "torch_npu._inductor.codegen.common", | 582 | "torch_npu._inductor.codegen.common", |
| @@ -7,6 +7,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Iterator | |||
| 7 | 7 | ||
| 8 | import torch | 8 | import torch |
| 9 | from torch._inductor.compile_fx import clone_preserve_strides | 9 | from torch._inductor.compile_fx import clone_preserve_strides |
| 10 | +from torch_npu._inductor.npu_compare import check_accuracy_mlir | ||
| 10 | 11 | ||
| 11 | from .. import config as anir_config | 12 | from .. import config as anir_config |
| 12 | from .utils import replace_placeholders | 13 | from .utils import replace_placeholders |
| @@ -101,74 +102,23 @@ class MetaCompiler: | |||
| 101 | return failed_subgraph_dump_path | 102 | return failed_subgraph_dump_path |
| 102 | 103 | ||
| 103 | def acc_compare_and_dump(self, *args, **kwargs): | 104 | def acc_compare_and_dump(self, *args, **kwargs): |
| 104 | - from torch.testing._comparison import _make_mismatch_msg | ||
| 105 | self.register_fx_fallback(self.kernel_meta) | 105 | self.register_fx_fallback(self.kernel_meta) |
| 106 | - launcher_fx = self.launchers[1] | ||
| 107 | - launcher = self.launchers[0] | ||
| 108 | - | ||
| 109 | - fx_outputs = [clone_preserve_strides(arg).to(torch.float32) if arg.dtype == torch.bfloat16 \ | ||
| 110 | - else clone_preserve_strides(arg) for arg in args[-self.num_outputs:]] | ||
| 111 | - fx_inputs = [clone_preserve_strides(arg) if isinstance(arg, torch.Tensor) else arg for arg in args[:-self.num_outputs]] | ||
| 112 | - fx_inputs = [inp.float() if isinstance(inp, torch.Tensor) and inp.dtype == torch.bfloat16 else inp for inp in fx_inputs] | ||
| 113 | 106 | ||
| 114 | - fx_args = fx_inputs + fx_outputs | 107 | + output, has_acc_error = check_accuracy_mlir( |
| 115 | - launcher_fx(*fx_args, **kwargs) | 108 | + *args, |
| 116 | - | 109 | + kernel_name=self.kernel_name, |
| 117 | - if self.dynamic: | 110 | + launchers=self.launchers, |
| 118 | - args_new = self.prepare_runtime_args( | 111 | + num_outputs=self.num_outputs, |
| 119 | - list(args), | 112 | + dynamic=self.dynamic, |
| 120 | - ) | 113 | + **kwargs |
| 121 | - else: | 114 | + ) |
| 122 | - args_new = args | ||
| 123 | - | ||
| 124 | - output = launcher(*args_new, **kwargs) | ||
| 125 | - | ||
| 126 | - has_acc_error = False | ||
| 127 | - num_inputs = len(args) - self.num_outputs | ||
| 128 | - for idx, (actual, expected) in enumerate(zip(args[num_inputs:], fx_outputs)): | ||
| 129 | - if actual.dtype != expected.dtype: | ||
| 130 | - expected = expected.to(actual.dtype) | ||
| 131 | - acc_comp_tol = anir_config.acc_comp_tol.get(actual.dtype, anir_config.acc_comp_tol['default']) | ||
| 132 | - rtol = acc_comp_tol['rtol'] | ||
| 133 | - atol = acc_comp_tol['atol'] | ||
| 134 | - matches = torch.isclose( | ||
| 135 | - actual, expected, rtol=rtol, atol=atol, equal_nan=True | ||
| 136 | - ) | ||
| 137 | - if not matches.all(): | ||
| 138 | - abs_diff = abs(actual - expected) | ||
| 139 | - rel_diff = abs_diff / abs(expected) | ||
| 140 | - rel_diff.masked_fill_(matches, 0) | ||
| 141 | - number_of_elements = matches.numel() | ||
| 142 | - total_mismatches = number_of_elements - int(torch.sum(matches)) | ||
| 143 | - extra = ( | ||
| 144 | - f"Mismatched elements: {total_mismatches} / {number_of_elements} " | ||
| 145 | - f"({total_mismatches / number_of_elements:.1%})" | ||
| 146 | - ) | ||
| 147 | - msg = _make_mismatch_msg( | ||
| 148 | - default_identifier="Tensor-likes", | ||
| 149 | - identifier=None, | ||
| 150 | - extra=extra, | ||
| 151 | - abs_diff=abs_diff.max().item(), | ||
| 152 | - abs_diff_idx=None, | ||
| 153 | - atol=atol, | ||
| 154 | - rel_diff=rel_diff.max().item(), | ||
| 155 | - rel_diff_idx=None, | ||
| 156 | - rtol=rtol, | ||
| 157 | - ) | ||
| 158 | - print(f"Kernel Name: {self.kernel_name}\n{msg}", flush=True) | ||
| 159 | - has_acc_error = True | ||
| 160 | - | ||
| 161 | - del abs_diff | ||
| 162 | - del rel_diff | ||
| 163 | - del matches | ||
| 164 | - del expected | ||
| 165 | 115 | ||
| 166 | if anir_config.fx_subgraph_dump_path: | 116 | if anir_config.fx_subgraph_dump_path: |
| 167 | data = args | 117 | data = args |
| 168 | if has_acc_error: | 118 | if has_acc_error: |
| 169 | data_dump_path = self.fx_subgraph_dump('acc_failed') | 119 | data_dump_path = self.fx_subgraph_dump('acc_failed') |
| 170 | self.data_dump_fake(*data, dump_path=data_dump_path) | 120 | self.data_dump_fake(*data, dump_path=data_dump_path) |
| 171 | - del fx_inputs | 121 | + |
| 172 | torch.npu.synchronize() | 122 | torch.npu.synchronize() |
| 173 | self.launchers = [self.launchers[0]] | 123 | self.launchers = [self.launchers[0]] |
| 174 | self.is_fallback_kernels = [self.is_fallback_kernels[0]] | 124 | self.is_fallback_kernels = [self.is_fallback_kernels[0]] |
| @@ -12,7 +12,7 @@ from torch_npu._C.dvm import ( | |||
| 12 | TorchKernel as Kernel, | 12 | TorchKernel as Kernel, |
| 13 | ) | 13 | ) |
| 14 | 14 | ||
| 15 | -from .fx_test import _accuracy_check_run | 15 | +from torch_npu._inductor.npu_compare import check_accuracy_dvm |
| 16 | 16 | ||
| 17 | 17 | ||
| 18 | debug_mode = False | 18 | debug_mode = False |
| @@ -114,7 +114,7 @@ def kernel( | |||
| 114 | 114 | ||
| 115 | def run(*args, **kwargs): | 115 | def run(*args, **kwargs): |
| 116 | if fn._acc_meta is not None: | 116 | if fn._acc_meta is not None: |
| 117 | - _accuracy_check_run(kobj, fn._acc_meta, kernel_name, args) | 117 | + check_accuracy_dvm(kobj, fn._acc_meta, kernel_name, args) |
| 118 | else: | 118 | else: |
| 119 | kobj.run(*args) | 119 | kobj.run(*args) |
| 120 | if debug_mode: | 120 | if debug_mode: |
| @@ -1,6 +1,4 @@ | |||
| 1 | import os | 1 | import os |
| 2 | -import sys | ||
| 3 | -import importlib | ||
| 4 | import hashlib | 2 | import hashlib |
| 5 | import torch | 3 | import torch |
| 6 | 4 | ||
| @@ -141,86 +139,3 @@ if __name__ == "__main__": | |||
| 141 | 139 | ||
| 142 | print(f"[ok] generated: {file_path}") | 140 | print(f"[ok] generated: {file_path}") |
| 143 | return file_path | 141 | return file_path |
| 144 | - | ||
| 145 | - | ||
| 146 | -def _load_fx_model(acc_meta): | ||
| 147 | - """Load the traced FX GraphModule from disk for accuracy comparison.""" | ||
| 148 | - if acc_meta.get('_fx_model') is not None: | ||
| 149 | - return acc_meta['_fx_model'] | ||
| 150 | - dump_path = os.path.join( | ||
| 151 | - os.getenv("TORCHINDUCTOR_CACHE_DIR"), | ||
| 152 | - acc_meta['traced_graph_cache'], | ||
| 153 | - str(acc_meta['device_index']), | ||
| 154 | - acc_meta['traced_graph_hash'], | ||
| 155 | - ) | ||
| 156 | - sys.path.insert(0, dump_path) | ||
| 157 | - try: | ||
| 158 | - module = importlib.import_module(acc_meta['traced_graph_hash']) | ||
| 159 | - finally: | ||
| 160 | - sys.path.remove(dump_path) | ||
| 161 | - Model = getattr(module, acc_meta['traced_graph_hash']) | ||
| 162 | - model = Model() | ||
| 163 | - acc_meta['_fx_model'] = model | ||
| 164 | - return model | ||
| 165 | - | ||
| 166 | - | ||
| 167 | -def _accuracy_check_run(kobj, acc_meta, kernel_name, args): | ||
| 168 | - """Run DVM kernel then compare outputs against FX graph reference.""" | ||
| 169 | - from torch._inductor.compile_fx import clone_preserve_strides | ||
| 170 | - from torch_npu._inductor.ascend_npu_ir.ascend_npu_ir import ( | ||
| 171 | - config as anir_config, | ||
| 172 | - ) | ||
| 173 | - | ||
| 174 | - fx_model = _load_fx_model(acc_meta) | ||
| 175 | - | ||
| 176 | - num_outputs = acc_meta['num_outputs'] | ||
| 177 | - num_inputs = len(args) - num_outputs | ||
| 178 | - | ||
| 179 | - fx_inputs = [] | ||
| 180 | - for arg in args[:num_inputs]: | ||
| 181 | - if isinstance(arg, torch.Tensor): | ||
| 182 | - inp = clone_preserve_strides(arg) | ||
| 183 | - if arg.dtype == torch.bfloat16: | ||
| 184 | - inp = inp.float() | ||
| 185 | - fx_inputs.append(inp) | ||
| 186 | - else: | ||
| 187 | - fx_inputs.append(arg) | ||
| 188 | - | ||
| 189 | - fx_outputs = fx_model.forward(*fx_inputs) | ||
| 190 | - if not isinstance(fx_outputs, (tuple, list)): | ||
| 191 | - fx_outputs = (fx_outputs,) | ||
| 192 | - | ||
| 193 | - kobj.run(*args) | ||
| 194 | - | ||
| 195 | - for idx, (actual, expected) in enumerate( | ||
| 196 | - zip(args[num_inputs:], fx_outputs) | ||
| 197 | - ): | ||
| 198 | - if not isinstance(actual, torch.Tensor): | ||
| 199 | - continue | ||
| 200 | - if actual.dtype != expected.dtype: | ||
| 201 | - expected = expected.to(actual.dtype) | ||
| 202 | - tol = anir_config.acc_comp_tol.get( | ||
| 203 | - actual.dtype, anir_config.acc_comp_tol["default"] | ||
| 204 | - ) | ||
| 205 | - rtol, atol = tol["rtol"], tol["atol"] | ||
| 206 | - matches = torch.isclose( | ||
| 207 | - actual, expected, rtol=rtol, atol=atol, equal_nan=True | ||
| 208 | - ) | ||
| 209 | - if not matches.all(): | ||
| 210 | - abs_diff = torch.abs(actual - expected) | ||
| 211 | - rel_diff = abs_diff / torch.clamp(torch.abs(expected), min=1e-20) | ||
| 212 | - rel_diff.masked_fill_(matches, 0) | ||
| 213 | - num_el = matches.numel() | ||
| 214 | - num_mis = num_el - int(torch.sum(matches)) | ||
| 215 | - print( | ||
| 216 | - f"CHECK ACCURACY FAILED! " | ||
| 217 | - f"Kernel: {kernel_name}, " | ||
| 218 | - f"Output idx: {idx}, " | ||
| 219 | - f"Mismatched: {num_mis}/{num_el} ({num_mis / num_el:.1%}), " | ||
| 220 | - f"Greatest Rel Diff: {rel_diff.max().item()}, " | ||
| 221 | - f"Greatest Abs Diff: {abs_diff.max().item()}", | ||
| 222 | - flush=True, | ||
| 223 | - ) | ||
| 224 | - | ||
| 225 | - del abs_diff, rel_diff | ||
| 226 | - del matches | ||
| @@ -0,0 +1,212 @@ | |||
| 1 | +import importlib | ||
| 2 | +import os | ||
| 3 | +import sys | ||
| 4 | +from typing import Any, Iterable, Mapping | ||
| 5 | + | ||
| 6 | +import torch | ||
| 7 | +from torch._inductor.compile_fx import clone_preserve_strides | ||
| 8 | + | ||
| 9 | + | ||
| 10 | +def clone_for_accuracy(arg): | ||
| 11 | + if not isinstance(arg, torch.Tensor): | ||
| 12 | + return arg | ||
| 13 | + cloned = clone_preserve_strides(arg) | ||
| 14 | + return cloned.float() if cloned.dtype == torch.bfloat16 else cloned | ||
| 15 | + | ||
| 16 | + | ||
| 17 | +def compare_outputs( | ||
| 18 | + actual_outputs: Iterable[Any], | ||
| 19 | + expected_outputs: Iterable[Any], | ||
| 20 | + kernel_name: str, | ||
| 21 | + tolerances: Mapping[Any, Mapping[str, float]], | ||
| 22 | +): | ||
| 23 | + failed_indices = [] | ||
| 24 | + for idx, (actual, expected) in enumerate(zip(actual_outputs, expected_outputs)): | ||
| 25 | + if not isinstance(actual, torch.Tensor) or not isinstance(expected, torch.Tensor): | ||
| 26 | + continue | ||
| 27 | + if actual.dtype != expected.dtype: | ||
| 28 | + expected = expected.to(actual.dtype) | ||
| 29 | + | ||
| 30 | + tol = tolerances.get(actual.dtype, tolerances["default"]) | ||
| 31 | + rtol, atol = tol["rtol"], tol["atol"] | ||
| 32 | + matches = torch.isclose(actual, expected, rtol=rtol, atol=atol, equal_nan=True) | ||
| 33 | + if not matches.all(): | ||
| 34 | + _report_mismatch(idx, actual, expected, matches, rtol, atol, kernel_name) | ||
| 35 | + failed_indices.append(idx) | ||
| 36 | + del matches | ||
| 37 | + | ||
| 38 | + return not failed_indices | ||
| 39 | + | ||
| 40 | + | ||
| 41 | +def _report_mismatch(idx, actual, expected, matches, rtol, atol, kernel_name): | ||
| 42 | + try: | ||
| 43 | + abs_diff = torch.abs(actual - expected) | ||
| 44 | + except RuntimeError: | ||
| 45 | + abs_diff = torch.abs(actual.to(torch.float32) - expected.to(torch.float32)) | ||
| 46 | + expected_abs = torch.abs(expected) | ||
| 47 | + if not expected_abs.is_floating_point() and not expected_abs.is_complex(): | ||
| 48 | + expected_abs = expected_abs.to(torch.float32) | ||
| 49 | + rel_diff = abs_diff / torch.clamp(expected_abs, min=1e-20) | ||
| 50 | + rel_diff.masked_fill_(matches, 0) | ||
| 51 | + number_of_elements = matches.numel() | ||
| 52 | + total_mismatches = number_of_elements - int(torch.sum(matches)) | ||
| 53 | + msg = ( | ||
| 54 | + "CHECK ACCURACY FAILED! " | ||
| 55 | + f"Kernel: {kernel_name}, Output idx: {idx}, " | ||
| 56 | + f"Mismatched: {total_mismatches}/{number_of_elements} " | ||
| 57 | + f"({total_mismatches / number_of_elements:.1%}), " | ||
| 58 | + f"Greatest Rel Diff: {rel_diff.max().item()}, " | ||
| 59 | + f"Greatest Abs Diff: {abs_diff.max().item()}, " | ||
| 60 | + f"rtol: {rtol}, atol: {atol}" | ||
| 61 | + ) | ||
| 62 | + print(msg, flush=True) | ||
| 63 | + del abs_diff, rel_diff | ||
| 64 | + | ||
| 65 | + | ||
| 66 | +def get_triton_fx_graph_call(inductor_meta, auto_fallback=False): | ||
| 67 | + kernel_name = inductor_meta.get("kernel_name", "triton_") | ||
| 68 | + traced_graph_hash = inductor_meta.get("traced_graph_hash") | ||
| 69 | + dump_dir = inductor_meta.get("traced_graph_dir", "") | ||
| 70 | + dump_path = os.path.join(dump_dir, traced_graph_hash) | ||
| 71 | + if dump_dir == "" or not os.path.exists(dump_path): | ||
| 72 | + return None, None, None, None | ||
| 73 | + sys.path.append(dump_path) | ||
| 74 | + fx_module = importlib.import_module(traced_graph_hash) | ||
| 75 | + sys.path.remove(dump_path) | ||
| 76 | + | ||
| 77 | + model = fx_module.model | ||
| 78 | + num_inputs = fx_module.num_inputs | ||
| 79 | + num_outputs = fx_module.num_outputs | ||
| 80 | + non_contiguous_indices = fx_module.non_contiguous_indices | ||
| 81 | + mismatch_indices_shapes = fx_module.mismatch_indices_shapes | ||
| 82 | + | ||
| 83 | + def fx_graph_call(*fx_args): | ||
| 84 | + fx_inputs = [fx_args[idx].contiguous() if idx in non_contiguous_indices['inputs'] else \ | ||
| 85 | + fx_args[idx] for idx in range(num_inputs)] | ||
| 86 | + if len(mismatch_indices_shapes): | ||
| 87 | + for ind, shape in mismatch_indices_shapes.items(): | ||
| 88 | + if ind >= num_inputs: | ||
| 89 | + break | ||
| 90 | + fx_inputs[ind] = fx_inputs[ind].reshape(shape) | ||
| 91 | + model_outputs = model.forward(*fx_inputs) | ||
| 92 | + for idx, (out1, out2) in enumerate(zip(model_outputs, fx_args[num_inputs:(num_inputs + num_outputs)])): | ||
| 93 | + out1 = out1.reshape(out2.shape) | ||
| 94 | + if idx in non_contiguous_indices['outputs']: | ||
| 95 | + out2.copy_(out1) | ||
| 96 | + else: | ||
| 97 | + out2.data = out1.data | ||
| 98 | + | ||
| 99 | + def fallback_call(*args): | ||
| 100 | + fx_args = [args[idx] for idx in fx_module.call_args_mapping] | ||
| 101 | + return fx_graph_call(*fx_args) | ||
| 102 | + | ||
| 103 | + if auto_fallback: | ||
| 104 | + return fallback_call, kernel_name, None, None | ||
| 105 | + return fx_graph_call, kernel_name, dump_path, fx_module | ||
| 106 | + | ||
| 107 | + | ||
| 108 | +def check_accuracy_triton(*args, launcher, grid, stream, inductor_meta, **kwargs): | ||
| 109 | + import torch_npu._inductor.config as npu_config | ||
| 110 | + fx_graph_call, kernel_name, dump_path, fx_module = get_triton_fx_graph_call(inductor_meta) | ||
| 111 | + if not fx_graph_call: | ||
| 112 | + return None | ||
| 113 | + call_outputs_indices = fx_module.call_args_mapping[fx_module.num_inputs:] | ||
| 114 | + | ||
| 115 | + fx_args = [] | ||
| 116 | + for idx in fx_module.call_args_mapping: | ||
| 117 | + arg = args[idx] | ||
| 118 | + if isinstance(arg, torch.Tensor): | ||
| 119 | + fx_args.append(clone_for_accuracy(arg)) | ||
| 120 | + | ||
| 121 | + fx_graph_call(*fx_args) | ||
| 122 | + | ||
| 123 | + launcher(*args, **kwargs, stream=stream) | ||
| 124 | + | ||
| 125 | + compare_outputs( | ||
| 126 | + [args[i] for i in call_outputs_indices], | ||
| 127 | + fx_args[fx_module.num_inputs:], | ||
| 128 | + kernel_name=kernel_name, | ||
| 129 | + tolerances=npu_config.acc_comp_tol, | ||
| 130 | + ) | ||
| 131 | + | ||
| 132 | + for arg in fx_args: | ||
| 133 | + del arg | ||
| 134 | + return True | ||
| 135 | + | ||
| 136 | + | ||
| 137 | +def check_accuracy_mlir(*args, kernel_name, launchers, num_outputs, dynamic, **kwargs): | ||
| 138 | + from torch_npu._inductor.ascend_npu_ir.ascend_npu_ir import config as anir_config | ||
| 139 | + launcher_fx = launchers[1] | ||
| 140 | + launcher = launchers[0] | ||
| 141 | + | ||
| 142 | + num_inputs = len(args) - num_outputs | ||
| 143 | + fx_outputs = [clone_for_accuracy(arg) for arg in args[num_inputs:]] | ||
| 144 | + fx_inputs = [clone_for_accuracy(arg) for arg in args[:num_inputs]] | ||
| 145 | + fx_args = fx_inputs + fx_outputs | ||
| 146 | + | ||
| 147 | + launcher_fx(*fx_args, **kwargs) | ||
| 148 | + | ||
| 149 | + if dynamic: | ||
| 150 | + args_new = () | ||
| 151 | + for arg in args: | ||
| 152 | + if not torch.is_tensor(arg): | ||
| 153 | + args_new = args_new + (arg,) | ||
| 154 | + continue | ||
| 155 | + args_new = args_new + (arg, arg, 0) + arg.size() + arg.stride() | ||
| 156 | + else: | ||
| 157 | + args_new = args | ||
| 158 | + | ||
| 159 | + output = launcher(*args_new, **kwargs) | ||
| 160 | + result = compare_outputs( | ||
| 161 | + args[num_inputs:], | ||
| 162 | + fx_outputs, | ||
| 163 | + kernel_name=kernel_name, | ||
| 164 | + tolerances=anir_config.acc_comp_tol, | ||
| 165 | + ) | ||
| 166 | + del fx_inputs | ||
| 167 | + return (output, result) | ||
| 168 | + | ||
| 169 | + | ||
| 170 | +def _load_fx_model(acc_meta): | ||
| 171 | + """Load the traced FX GraphModule from disk for accuracy comparison.""" | ||
| 172 | + if acc_meta.get('_fx_model') is not None: | ||
| 173 | + return acc_meta['_fx_model'] | ||
| 174 | + dump_path = os.path.join( | ||
| 175 | + os.getenv("TORCHINDUCTOR_CACHE_DIR"), | ||
| 176 | + acc_meta['traced_graph_cache'], | ||
| 177 | + str(acc_meta['device_index']), | ||
| 178 | + acc_meta['traced_graph_hash'], | ||
| 179 | + ) | ||
| 180 | + sys.path.insert(0, dump_path) | ||
| 181 | + try: | ||
| 182 | + module = importlib.import_module(acc_meta['traced_graph_hash']) | ||
| 183 | + finally: | ||
| 184 | + sys.path.remove(dump_path) | ||
| 185 | + Model = getattr(module, acc_meta['traced_graph_hash']) | ||
| 186 | + model = Model() | ||
| 187 | + acc_meta['_fx_model'] = model | ||
| 188 | + return model | ||
| 189 | + | ||
| 190 | + | ||
| 191 | +def check_accuracy_dvm(kobj, acc_meta, kernel_name, args): | ||
| 192 | + """Run DVM kernel then compare outputs against FX graph reference.""" | ||
| 193 | + from torch_npu._inductor.ascend_npu_ir.ascend_npu_ir import config as anir_config | ||
| 194 | + | ||
| 195 | + fx_model = _load_fx_model(acc_meta) | ||
| 196 | + | ||
| 197 | + num_outputs = acc_meta['num_outputs'] | ||
| 198 | + num_inputs = len(args) - num_outputs | ||
| 199 | + | ||
| 200 | + fx_inputs = [clone_for_accuracy(arg) for arg in args[:num_inputs]] | ||
| 201 | + fx_outputs = fx_model.forward(*fx_inputs) | ||
| 202 | + if not isinstance(fx_outputs, (tuple, list)): | ||
| 203 | + fx_outputs = (fx_outputs,) | ||
| 204 | + | ||
| 205 | + kobj.run(*args) | ||
| 206 | + | ||
| 207 | + compare_outputs( | ||
| 208 | + args[num_inputs:], | ||
| 209 | + fx_outputs, | ||
| 210 | + kernel_name=kernel_name, | ||
| 211 | + tolerances=anir_config.acc_comp_tol, | ||
| 212 | + ) | ||
| @@ -81,6 +81,7 @@ except ImportError: | |||
| 81 | 81 | ||
| 82 | import torch_npu | 82 | import torch_npu |
| 83 | from torch_npu.utils._error_code import ErrCode, pta_error | 83 | from torch_npu.utils._error_code import ErrCode, pta_error |
| 84 | +from torch_npu._inductor.npu_compare import check_accuracy_triton | ||
| 84 | 85 | ||
| 85 | from ..codegen.tile_generator import TileGenerator | 86 | from ..codegen.tile_generator import TileGenerator |
| 86 | from ..codegen.triton_utils import NPUKernelType | 87 | from ..codegen.triton_utils import NPUKernelType |
| @@ -451,40 +452,6 @@ class NPUCachingAutotuner(CachingAutotuner): | |||
| 451 | self.exceptions = [] | 452 | self.exceptions = [] |
| 452 | self.fn_name = None | 453 | self.fn_name = None |
| 453 | 454 | ||
| 454 | - | ||
| 455 | - def api_accuracy_checker(expected, actual, kernel_name, dump_path): | ||
| 456 | - from msprobe.core.common.const import CompareConst | ||
| 457 | - from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import BENCHMARK_COMPARE_SUPPORT_LIST | ||
| 458 | - from msprobe.pytorch.api_accuracy_checker.triton_adapter.get_compare_result import get_compare_result | ||
| 459 | - from msprobe.pytorch.api_accuracy_checker.triton_adapter.precision_compare import precision_compare | ||
| 460 | - from msprobe.pytorch.api_accuracy_checker.triton_adapter.common.compare_utils import \ | ||
| 461 | - convert_compare_column_to_row, print_check_details | ||
| 462 | - from msprobe.pytorch.api_accuracy_checker.triton_adapter.precision_standard.triton_standard_register import \ | ||
| 463 | - exist_in_precision_standard | ||
| 464 | - | ||
| 465 | - dtype = actual.dtype | ||
| 466 | - | ||
| 467 | - # only float use precision standard | ||
| 468 | - if exist_in_precision_standard(kernel_name): | ||
| 469 | - if str(dtype) in BENCHMARK_COMPARE_SUPPORT_LIST: | ||
| 470 | - compare_column = precision_compare(kernel_name, expected, actual, dtype) # calc metrics | ||
| 471 | - compare_row = convert_compare_column_to_row(compare_column, kernel_name) | ||
| 472 | - status = get_compare_result(compare_row, kernel_name) # get compare results | ||
| 473 | - if status == CompareConst.ERROR: | ||
| 474 | - log.warning(f'CHECK ACCURACY FAILED! kernel: {kernel_name}, Dump Path: {dump_path}') | ||
| 475 | - print_check_details(compare_column, kernel_name) | ||
| 476 | - actual.copy_(expected) | ||
| 477 | - checked_by_msprobe = True | ||
| 478 | - else: | ||
| 479 | - log.warning(f'The data type {dtype} is not supported for new precision standard. ' | ||
| 480 | - f'Check accuracy by tolerance method.') | ||
| 481 | - checked_by_msprobe = False | ||
| 482 | - else: | ||
| 483 | - log.warning(f'kernel_name {kernel_name} does not in new precision standard. ' | ||
| 484 | - f'Check accuracy by tolerance method.') | ||
| 485 | - checked_by_msprobe = False | ||
| 486 | - return checked_by_msprobe | ||
| 487 | - | ||
| 488 | def precompile( | 455 | def precompile( |
| 489 | self, | 456 | self, |
| 490 | warm_cache_only=False, | 457 | warm_cache_only=False, |
| @@ -976,47 +943,6 @@ class NPUCachingAutotuner(CachingAutotuner): | |||
| 976 | return None | 943 | return None |
| 977 | return dump_path | 944 | return dump_path |
| 978 | 945 | ||
| 979 | - def get_fx_graph_call(self, auto_fallback=False): | ||
| 980 | - kernel_name = self.inductor_meta.get("kernel_name", "triton_") | ||
| 981 | - traced_graph_hash = self.inductor_meta.get("traced_graph_hash") | ||
| 982 | - dump_dir = self.inductor_meta.get("traced_graph_dir", "") | ||
| 983 | - dump_path = os.path.join(dump_dir, traced_graph_hash) | ||
| 984 | - if dump_dir == "" or not os.path.exists(dump_path): | ||
| 985 | - return None, None, None, None | ||
| 986 | - sys.path.append(dump_path) | ||
| 987 | - fx_module = importlib.import_module(traced_graph_hash) | ||
| 988 | - sys.path.remove(dump_path) | ||
| 989 | - | ||
| 990 | - model = fx_module.model | ||
| 991 | - num_inputs = fx_module.num_inputs | ||
| 992 | - num_outputs = fx_module.num_outputs | ||
| 993 | - non_contiguous_indices = fx_module.non_contiguous_indices | ||
| 994 | - mismatch_indices_shapes = fx_module.mismatch_indices_shapes | ||
| 995 | - | ||
| 996 | - def fx_graph_call(*fx_args): | ||
| 997 | - fx_inputs = [fx_args[idx].contiguous() if idx in non_contiguous_indices['inputs'] else \ | ||
| 998 | - fx_args[idx] for idx in range(num_inputs)] | ||
| 999 | - if len(mismatch_indices_shapes): | ||
| 1000 | - for ind, shape in mismatch_indices_shapes.items(): | ||
| 1001 | - if ind >= num_inputs: | ||
| 1002 | - break | ||
| 1003 | - fx_inputs[ind] = fx_inputs[ind].reshape(shape) | ||
| 1004 | - model_outputs = model.forward(*fx_inputs) | ||
| 1005 | - for idx, (out1, out2) in enumerate(zip(model_outputs, fx_args[num_inputs:(num_inputs + num_outputs)])): | ||
| 1006 | - out1 = out1.reshape(out2.shape) | ||
| 1007 | - if idx in non_contiguous_indices['outputs']: | ||
| 1008 | - out2.copy_(out1) | ||
| 1009 | - else: | ||
| 1010 | - out2.data = out1.data | ||
| 1011 | - | ||
| 1012 | - def fallback_call(*args): | ||
| 1013 | - fx_args = [args[idx] for idx in fx_module.call_args_mapping] | ||
| 1014 | - return fx_graph_call(*fx_args) | ||
| 1015 | - | ||
| 1016 | - if auto_fallback: | ||
| 1017 | - return fallback_call, kernel_name, None, None | ||
| 1018 | - return fx_graph_call, kernel_name, dump_path, fx_module | ||
| 1019 | - | ||
| 1020 | def data_dump(self, *args, dump_path=None): | 946 | def data_dump(self, *args, dump_path=None): |
| 1021 | dump_path = self.get_fx_graph_dump_path() if dump_path is None else dump_path | 947 | dump_path = self.get_fx_graph_dump_path() if dump_path is None else dump_path |
| 1022 | if dump_path is None: | 948 | if dump_path is None: |
| @@ -1037,61 +963,6 @@ class NPUCachingAutotuner(CachingAutotuner): | |||
| 1037 | self.fn_name = self.kernel_name | 963 | self.fn_name = self.kernel_name |
| 1038 | return self.fn_name | 964 | return self.fn_name |
| 1039 | 965 | ||
| 1040 | - def check_accuracy(self, *args, launcher, grid, stream, **kwargs): | ||
| 1041 | - fx_graph_call, kernel_name, dump_path, fx_module = self.get_fx_graph_call() | ||
| 1042 | - if not fx_graph_call: | ||
| 1043 | - return None | ||
| 1044 | - call_outputs_indices = fx_module.call_args_mapping[fx_module.num_inputs:] | ||
| 1045 | - | ||
| 1046 | - fx_args = [] | ||
| 1047 | - for idx in fx_module.call_args_mapping: | ||
| 1048 | - arg = args[idx] | ||
| 1049 | - if isinstance(arg, torch.Tensor): | ||
| 1050 | - fx_arg = clone_preserve_strides(arg).float() if arg.dtype == torch.bfloat16 else clone_preserve_strides( | ||
| 1051 | - arg) | ||
| 1052 | - fx_args.append(fx_arg) | ||
| 1053 | - | ||
| 1054 | - fx_graph_call(*fx_args) | ||
| 1055 | - | ||
| 1056 | - launcher( | ||
| 1057 | - *args, | ||
| 1058 | - **kwargs, | ||
| 1059 | - stream=stream, | ||
| 1060 | - ) | ||
| 1061 | - | ||
| 1062 | - try: | ||
| 1063 | - import msprobe | ||
| 1064 | - has_msprobe = True | ||
| 1065 | - except ImportError: | ||
| 1066 | - has_msprobe = False | ||
| 1067 | - warning_once(log, "msprobe import failed, please check. " | ||
| 1068 | - "It may be due to missing dependencies or other factors. " | ||
| 1069 | - "Check accuracy by tolerance method.") | ||
| 1070 | - for actual, expected in zip([args[i] for i in call_outputs_indices], fx_args[fx_module.num_inputs:]): | ||
| 1071 | - if actual.dtype != expected.dtype: | ||
| 1072 | - expected = expected.to(actual.dtype) | ||
| 1073 | - checked_by_msprobe = False | ||
| 1074 | - if has_msprobe: | ||
| 1075 | - checked_by_msprobe = self.api_accuracy_checker(expected, actual, kernel_name, dump_path) | ||
| 1076 | - if not has_msprobe or not checked_by_msprobe: | ||
| 1077 | - acc_comp_tol = npu_config.acc_comp_tol.get(actual.dtype, npu_config.acc_comp_tol['default']) | ||
| 1078 | - rtol = acc_comp_tol['rtol'] | ||
| 1079 | - atol = acc_comp_tol['atol'] | ||
| 1080 | - | ||
| 1081 | - matches = torch.isclose( | ||
| 1082 | - actual, expected, rtol=rtol, atol=atol, equal_nan=True | ||
| 1083 | - ) | ||
| 1084 | - if not matches.all(): | ||
| 1085 | - abs_diff = torch.abs(actual - expected) | ||
| 1086 | - rel_diff = abs_diff / torch.abs(expected) | ||
| 1087 | - rel_diff.masked_fill_(matches, 0) | ||
| 1088 | - log.warning(f"CHECK ACCURACY FAILED! Greatest Relative Difference: {rel_diff.max().item()}, " | ||
| 1089 | - f"Kernel Name: {kernel_name}, Dump Path: {dump_path}") | ||
| 1090 | - del matches | ||
| 1091 | - for arg in fx_args: | ||
| 1092 | - del arg | ||
| 1093 | - return True | ||
| 1094 | - | ||
| 1095 | 966 | ||
| 1096 | def is_run_debug(self): | 967 | def is_run_debug(self): |
| 1097 | return npu_config.dump_fx_graph or npu_config.check_accuracy | 968 | return npu_config.dump_fx_graph or npu_config.check_accuracy |
| @@ -1108,7 +979,14 @@ class NPUCachingAutotuner(CachingAutotuner): | |||
| 1108 | _ = self.data_dump(*args) | 979 | _ = self.data_dump(*args) |
| 1109 | 980 | ||
| 1110 | if npu_config.check_accuracy: | 981 | if npu_config.check_accuracy: |
| 1111 | - if self.check_accuracy(*args, launcher=launcher, grid=grid_, stream=stream, **kwargs): | 982 | + if check_accuracy_triton( |
| 983 | + *args, | ||
| 984 | + launcher=launcher, | ||
| 985 | + grid=grid_, | ||
| 986 | + stream=stream, | ||
| 987 | + inductor_meta=self.inductor_meta, | ||
| 988 | + **kwargs | ||
| 989 | + ): | ||
| 1112 | return "check_accuracy" | 990 | return "check_accuracy" |
| 1113 | 991 | ||
| 1114 | log.info(f"No debug mode is activated for kernel {kernel_name}.") | 992 | log.info(f"No debug mode is activated for kernel {kernel_name}.") |