已合并
【inductor】bugfix for noinear codegen, sync from pr39111 #39768
stonexxx创建于 7月1日
【inductor】bugfix for noinear codegen, sync from pr39111 #39768
已合并
共 5 个文件变更+34-10
| @@ -32,7 +32,7 @@ import torch_npu | |||
| 32 | 'constants': {}, 'mix_mode': 'aiv'}, | 32 | 'constants': {}, 'mix_mode': 'aiv'}, |
| 33 | inductor_meta={'autotune_hints': set(), 'kernel_name': 'triton_unk_fused_add_0', 'mutated_arg_names': [], | 33 | inductor_meta={'autotune_hints': set(), 'kernel_name': 'triton_unk_fused_add_0', 'mutated_arg_names': [], |
| 34 | 'backend_hash': 'bc71dba4086164e7ac2b0779fa861dbf7467f0265d4a57b8f48cf6dda02b150f', 'split_axis': [0], | 34 | 'backend_hash': 'bc71dba4086164e7ac2b0779fa861dbf7467f0265d4a57b8f48cf6dda02b150f', 'split_axis': [0], |
| 35 | - 'tiling_axis': [0, 1], 'no_loop_axis': [1], 'axis_names': ['y0', 'x1'], 'low_dims': {1}, 'numof_reduction_axis': 0, | 35 | + 'tiling_axis': [0, 1], 'no_loop_axis': [1], 'axis_names': ['y0', 'x1'], 'low_dims': {1}, 'numof_reduction_axis': 0, 'inductor_ascend_linear_mode': 'linear', |
| 36 | 'split_axis_dtype': torch.float16, 'dual_reduction': False, 'traced_graph_hash': 'TRACED_GRAPH_HASH', | 36 | 'split_axis_dtype': torch.float16, 'dual_reduction': False, 'traced_graph_hash': 'TRACED_GRAPH_HASH', |
| 37 | 'traced_graph_dir': 'TRACED_GRAPH_DIR'}, | 37 | 'traced_graph_dir': 'TRACED_GRAPH_DIR'}, |
| 38 | min_elem_per_thread=0 | 38 | min_elem_per_thread=0 |
| @@ -90,9 +90,10 @@ class NPUCombinedScheduling(CUDACombinedScheduling, TritonScheduling): | |||
| 90 | try: | 90 | try: |
| 91 | return self._triton_scheduling.codegen_node(node) | 91 | return self._triton_scheduling.codegen_node(node) |
| 92 | except Exception: | 92 | except Exception: |
| 93 | - log.exception( | 93 | + log.debug( |
| 94 | "linear codegen for node %s raise error, fallback to origin codegen", | 94 | "linear codegen for node %s raise error, fallback to origin codegen", |
| 95 | node, | 95 | node, |
| 96 | + exc_info=True, | ||
| 96 | ) | 97 | ) |
 | |||
| 97 | # regroup snode | 98 | # regroup snode |
| 98 | for snode in node.get_nodes(): | 99 | for snode in node.get_nodes(): |
| @@ -55,9 +55,9 @@ class NPUNoLinearTritonScheduling(TritonScheduling): | |||
| 55 | def __init__(self, input_scheduler): | 55 | def __init__(self, input_scheduler): |
| 56 | super().__init__(input_scheduler) | 56 | super().__init__(input_scheduler) |
| 57 | from ..config import inductor_ascend_linear_mode | 57 | from ..config import inductor_ascend_linear_mode |
| 58 | - self.kernel_type = NPUTritonKernelWithLoop | 58 | + self.kernel_type = NPUTritonKernel |
| 59 | - if inductor_ascend_linear_mode == 'no_linear': | 59 | + if inductor_ascend_linear_mode == 'no_linear_loop': |
| 60 | - self.kernel_type = NPUTritonKernel | 60 | + self.kernel_type = NPUTritonKernelWithLoop |
| 61 | 61 | ||
| 62 | class NPUTritonScheduling(TritonScheduling): | 62 | class NPUTritonScheduling(TritonScheduling): |
| 63 | def __init__(self, input_scheduler): | 63 | def __init__(self, input_scheduler): |
| @@ -832,10 +832,10 @@ class NPUTritonKernel(TritonKernel): | |||
| 832 | from ..config import inductor_ascend_linear_mode | 832 | from ..config import inductor_ascend_linear_mode |
| 833 | if inductor_ascend_linear_mode == "linear": | 833 | if inductor_ascend_linear_mode == "linear": |
| 834 | # Linear fallback to no_linear_loop | 834 | # Linear fallback to no_linear_loop |
| 835 | - inductor_meta["inductor_ascend_linear_mode"] = "no_linear_loop" | 835 | + inductor_meta["inductor_ascend_linear_mode"] = "no_linear" |
| 836 | else: | 836 | else: |
| 837 | inductor_meta["inductor_ascend_linear_mode"] = inductor_ascend_linear_mode | 837 | inductor_meta["inductor_ascend_linear_mode"] = inductor_ascend_linear_mode |
| 838 | - inductor_meta["npu_kernel_type"] = "simt_only" | 838 | + inductor_meta["npu_kernel_type"] = str(NPUKernelType.SIMD_SIMT_MIX) |
| 839 | inductor_meta["split_axis"] = split_axis | 839 | inductor_meta["split_axis"] = split_axis |
| 840 | inductor_meta["tiling_axis"] = tiling_axis | 840 | inductor_meta["tiling_axis"] = tiling_axis |
| 841 | inductor_meta["low_dims"] = [tiling_axis[-1]] | 841 | inductor_meta["low_dims"] = [tiling_axis[-1]] |
| @@ -693,6 +693,27 @@ class NPUCachingAutotuner(CachingAutotuner): | |||
| 693 | items.append({"config": cfg, "ttir": "", "arg_bindings": ""}) | 693 | items.append({"config": cfg, "ttir": "", "arg_bindings": ""}) |
| 694 | return items | 694 | return items |
| 695 | 695 | ||
| 696 | + def _select_ttir_test_config(self): | ||
| 697 | + smallest_config = None | ||
| 698 | + min_sub_product = float("inf") | ||
| 699 | + for cfg in self.configs: | ||
| 700 | + kwargs = getattr(cfg, "kwargs", None) or {} | ||
| 701 | + current_sub_product = 1 | ||
| 702 | + has_sub_tiling = False | ||
| 703 | + for tiling_name, tiling in kwargs.items(): | ||
| 704 | + if not isinstance(tiling_name, str) or not tiling_name.endswith("SUB"): | ||
| 705 | + continue | ||
| 706 | + try: | ||
| 707 | + tiling_value = int(tiling) | ||
| 708 | + except (TypeError, ValueError): | ||
| 709 | + continue | ||
| 710 | + current_sub_product *= tiling_value | ||
| 711 | + has_sub_tiling = True | ||
| 712 | + if has_sub_tiling and current_sub_product < min_sub_product: | ||
| 713 | + min_sub_product = current_sub_product | ||
| 714 | + smallest_config = cfg | ||
| 715 | + return smallest_config if smallest_config is not None else self.configs[0] | ||
| 716 | + | ||
| 696 | def _triton_make_ttir(self): | 717 | def _triton_make_ttir(self): |
| 697 | if not self.configs: | 718 | if not self.configs: |
| 698 | raise NoTritonConfigsError("No triton configs are available") | 719 | raise NoTritonConfigsError("No triton configs are available") |
| @@ -720,7 +741,7 @@ class NPUCachingAutotuner(CachingAutotuner): | |||
| 720 | compile_results = [] | 741 | compile_results = [] |
| 721 | exc = None | 742 | exc = None |
| 722 | exc_stack = "" | 743 | exc_stack = "" |
| 723 | - test_config = self.configs[0] | 744 | + test_config = self._select_ttir_test_config() |
| 724 | try: | 745 | try: |
| 725 | compile_results.append(make_ttir_from_cfg(test_config)) | 746 | compile_results.append(make_ttir_from_cfg(test_config)) |
| 726 | except Exception as e: | 747 | except Exception as e: |
| @@ -912,6 +933,9 @@ class NPUCachingAutotuner(CachingAutotuner): | |||
| 912 | if compile_meta['compile_mode'] == NPUKernelType.SIMT_ONLY.compile_mode(): | 933 | if compile_meta['compile_mode'] == NPUKernelType.SIMT_ONLY.compile_mode(): |
| 913 | options['simt_stack_limit'] = npu_config.simt_default_warp_stacksize | 934 | options['simt_stack_limit'] = npu_config.simt_default_warp_stacksize |
| 914 | 935 | ||
| 936 | + if self.inductor_meta.get("inductor_ascend_linear_mode", "no_linear") == "no_linear": | ||
| 937 | + options['enable_auto_blockify'] = True | ||
| 938 | + | ||
| 915 | compile_kwargs = { | 939 | compile_kwargs = { |
| 916 | "target": target, | 940 | "target": target, |
| 917 | "options": options, | 941 | "options": options, |
| @@ -1541,8 +1565,6 @@ def triton_config_npu_index( | |||
| 1541 | configs.extend(tile_generator.descend_split_tiling()) | 1565 | configs.extend(tile_generator.descend_split_tiling()) |
| 1542 | tile_generator.set_kernel_type(NPUKernelType.SIMT_TEMPLATE) | 1566 | tile_generator.set_kernel_type(NPUKernelType.SIMT_TEMPLATE) |
| 1543 | configs.extend(tile_generator.descend_split_tiling()) | 1567 | configs.extend(tile_generator.descend_split_tiling()) |
| 1544 | - tile_generator.set_kernel_type(NPUKernelType.SIMD) | ||
| 1545 | - configs.extend(tile_generator.descend_split_tiling()) | ||
| 1546 | else: | 1568 | else: |
| 1547 | configs = tile_generator.descend_split_tiling() | 1569 | configs = tile_generator.descend_split_tiling() |
| 1548 | 1570 | ||
| @@ -1575,6 +1597,7 @@ def triton_config_npu_index( | |||
| 1575 | for tiling, tling_value in tiling_kwargs.items(): | 1597 | for tiling, tling_value in tiling_kwargs.items(): |
| 1576 | if isinstance(tiling, str) and tiling.endswith("SUB"): | 1598 | if isinstance(tiling, str) and tiling.endswith("SUB"): |
| 1577 | tiling_cfg.kwargs[tiling.rstrip("_SUB")] = tling_value | 1599 | tiling_cfg.kwargs[tiling.rstrip("_SUB")] = tling_value |
| 1600 | + tiling_cfg.kwargs.pop(tiling) | ||
| 1578 | elif inductor_ascend_linear_mode == "no_linear_loop": | 1601 | elif inductor_ascend_linear_mode == "no_linear_loop": |
| 1579 | for tiling_cfg in configs: | 1602 | for tiling_cfg in configs: |
| 1580 | tiling_kwargs = copy.deepcopy(tiling_cfg.kwargs) | 1603 | tiling_kwargs = copy.deepcopy(tiling_cfg.kwargs) |
🟡 Medium Priority
变更行：npu_combined_scheduling.py 第92-97行。原代码在 linear codegen 失败时使用
log.exception(...)记录异常（ERROR 级别，自动附带 traceback），新代码改为log.debug(..., exc_info=True)（DEBUG 级别）。影响：当用户配置
inductor_ascend_linear_mode = "linear"（也是默认值）且 linear codegen 因任何原因失败时，原代码会在默认日志级别（WARNING）下输出完整的异常信息，开发者可以感知到 fallback 的发生。新代码将日志降级为 DEBUG，在生产环境的默认日志级别下，这些 fallback 完全不可见。如果 linear codegen 存在持续性问题（如某个算子始终无法走 linear 路径），故障排查将变得更加困难——异常被静默吞掉，只留下 fallback 行为生效，开发者可能完全不知道 linear codegen 已经失败了。虽然 fallback 本身是预期行为（注释写 "fallback to origin codegen"），但用 DEBUG 级别记录异常会丢失可观测性。建议至少使用
log.warning或log.info，或者在降级的同时增加一个 counter/metrics 来统计 fallback 次数，便于运维监控。建议：将
log.debug至少提升为log.warning或log.info，确保 linear codegen 的静默 fallback 在默认日志级别下可被观测。或者保持 debug 但额外增加一个 metric/counter 来记录 fallback 次数。