已合并
[inductor][fix]delete the ops from fallback_lsit and remove "config.fallback_random = True"(2) #37867
qinhanmin创建于 6月8日
[inductor][fix]delete the ops from fallback_lsit and remove "config.fallback_random = True"(2) #37867
已合并
共 5 个文件变更+2-120
| @@ -43,4 +43,5 @@ class TestDropoutWithCheckpointRecompute(TestUtils): | |||
| 43 | instantiate_parametrized_tests(TestDropoutWithCheckpointRecompute) | 43 | instantiate_parametrized_tests(TestDropoutWithCheckpointRecompute) |
| 44 | 44 | ||
| 45 | if __name__ == "__main__": | 45 | if __name__ == "__main__": |
| 46 | - run_tests() | 46 | + with torch._inductor.config.patch("fallback_random", True): |
| 47 | + run_tests() | ||
| @@ -93,7 +93,6 @@ def _load_triton_backend(): | |||
| 93 | post_grad_custom_pass_fuc, | 93 | post_grad_custom_pass_fuc, |
| 94 | pre_grad_custom_pass_fuc, | 94 | pre_grad_custom_pass_fuc, |
| 95 | ) | 95 | ) |
| 96 | - from .fx_passes.pattern_match.npu_fusion_attention_graph import register_fa_pass | ||
| 97 | from .fx_passes.joint_graph import patch_constant_fold_uniform_value | 96 | from .fx_passes.joint_graph import patch_constant_fold_uniform_value |
| 98 | from .ir import patch_num_splits | 97 | from .ir import patch_num_splits |
| 99 | from .kernel import ( | 98 | from .kernel import ( |
| @@ -227,7 +226,6 @@ def _load_triton_backend(): | |||
| 227 | if max_precompiled_thread_num > 1: | 226 | if max_precompiled_thread_num > 1: |
| 228 | _replace_precompile() | 227 | _replace_precompile() |
| 229 | 228 | ||
| 230 | - register_fa_pass() | ||
| 231 | patch_get_first_incompatible_cudagraph_node() | 229 | patch_get_first_incompatible_cudagraph_node() |
| 232 | patch_get_optimization_cflags() | 230 | patch_get_optimization_cflags() |
| 233 | patch_extract_read_writes() | 231 | patch_extract_read_writes() |
| @@ -26,8 +26,6 @@ if not enable_inplace_buffers: | |||
| 26 | # inductor debug switch | 26 | # inductor debug switch |
| 27 | config.trace.enabled = True | 27 | config.trace.enabled = True |
| 28 | 28 | ||
| 29 | -config.fallback_random = True | ||
| 30 | - | ||
| 31 | device = torch.npu.current_device() | 29 | device = torch.npu.current_device() |
| 32 | prop = torch.npu.get_device_properties(device) | 30 | prop = torch.npu.get_device_properties(device) |
| 33 | 31 | ||
| @@ -1,12 +1,10 @@ | |||
| 1 | # -*- coding: utf-8 -*- | 1 | # -*- coding: utf-8 -*- |
| 2 | # Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. | 2 | # Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. |
| 3 | -import functools | ||
| 4 | import sympy | 3 | import sympy |
| 5 | import torch | 4 | import torch |
| 6 | import torch.nn.functional as F | 5 | import torch.nn.functional as F |
| 7 | from torch.autograd import Function | 6 | from torch.autograd import Function |
| 8 | from torch.library import Library, impl | 7 | from torch.library import Library, impl |
| 9 | -from torch._inductor.pattern_matcher import init_once_fakemode | ||
| 10 | import torch_npu | 8 | import torch_npu |
| 11 | 9 | ||
| 12 | npu_def = Library("npu_graph", "DEF") | 10 | npu_def = Library("npu_graph", "DEF") |
| @@ -155,102 +153,3 @@ def npu_fusion_attention_graph(query, key, value, head_num, input_layout, pse=No | |||
| 155 | 153 | ||
| 156 | 154 | ||
| 157 | torch_npu.npu_fusion_attention_graph = npu_fusion_attention_graph | 155 | torch_npu.npu_fusion_attention_graph = npu_fusion_attention_graph |
| 158 | - | ||
| 159 | - | ||
| 160 | - | ||
| 161 | -def register_fa_pass(): | ||
| 162 | - TOKEN_MAX = 2147483647 | ||
| 163 | - from torch._inductor.pattern_matcher import register_replacement, fwd_only, joint_fwd_bwd | ||
| 164 | - from torch._inductor.fx_passes.joint_graph import patterns | ||
| 165 | - from torch._dynamo.utils import counters | ||
| 166 | - from torch._inductor.fx_passes.fuse_attention import partialize_and_update_signature | ||
| 167 | - | ||
| 168 | - def _npu_fusion_attention_graph_pattern_1(query, key, value, inv_scale_factor, dropout_p): | ||
| 169 | - q = query.permute(0, 2, 1, 3) | ||
| 170 | - k = key.permute(0, 2, 1, 3) | ||
| 171 | - v = value.permute(0, 2, 1, 3) | ||
| 172 | - return torch.nn.functional.dropout( | ||
| 173 | - torch.matmul(q, k.transpose(-2, -1)).div(inv_scale_factor).softmax(dim=-1), | ||
| 174 | - p=dropout_p, | ||
| 175 | - ).matmul(v) | ||
| 176 | - | ||
| 177 | - def _npu_fusion_attention_graph_replacement_1(query, key, value, inv_scale_factor, dropout_p): | ||
| 178 | - counters["inductor"]["fuse_attention"] += 1 | ||
| 179 | - head_num = query.size(2) | ||
| 180 | - input_layout = "BNSD" | ||
| 181 | - return torch_npu.npu_fusion_attention_graph( | ||
| 182 | - query.transpose(1, 2), | ||
| 183 | - key.transpose(1, 2), | ||
| 184 | - value.transpose(1, 2), | ||
| 185 | - head_num, | ||
| 186 | - input_layout, | ||
| 187 | - None, | ||
| 188 | - atten_mask=None, | ||
| 189 | - scale=inv_scale_factor, | ||
| 190 | - keep_prob=1.0 - dropout_p, | ||
| 191 | - )[0] | ||
| 192 | - | ||
| 193 | - def _get_sfdp_patterns(): | ||
| 194 | - device = 'npu' | ||
| 195 | - g_inp = functools.partial( | ||
| 196 | - torch.empty, (2, 4, 8, 16), device=device, requires_grad=True | ||
| 197 | - ) | ||
| 198 | - c_inp = functools.partial(torch.tensor, 2.0, device=g_inp().device) | ||
| 199 | - d = {"dropout_p": 0.113377} | ||
| 200 | - candidates = [] | ||
| 201 | - for dtype in [torch.float]: | ||
| 202 | - g = functools.partial(g_inp, dtype=dtype) | ||
| 203 | - c = functools.partial(c_inp, dtype=dtype) | ||
| 204 | - candidates.append(( | ||
| 205 | - _npu_fusion_attention_graph_pattern_1, | ||
| 206 | - _npu_fusion_attention_graph_replacement_1, | ||
| 207 | - [g(), g(), g(), c()], | ||
| 208 | - d, | ||
| 209 | - )) | ||
| 210 | - | ||
| 211 | - for pattern, replacement, args, workaround in candidates: | ||
| 212 | - # gets serialized to a python file and does not require tracing at runtime. | ||
| 213 | - if not isinstance(workaround, dict): | ||
| 214 | - raise ValueError("workaround not dict") | ||
| 215 | - name = pattern.__name__ | ||
| 216 | - | ||
| 217 | - if dtype != torch.float: | ||
| 218 | - name += "_half" | ||
| 219 | - | ||
| 220 | - if args[0].size(0) == 1: | ||
| 221 | - name += "_bs1" | ||
| 222 | - | ||
| 223 | - training_name = name + "_training" | ||
| 224 | - yield training_name, { | ||
| 225 | - "search_fn": pattern, | ||
| 226 | - "replace_fn": replacement, | ||
| 227 | - "example_inputs": args, | ||
| 228 | - "trace_fn": joint_fwd_bwd, | ||
| 229 | - "pass_dicts": patterns, | ||
| 230 | - "scalar_workaround": workaround, | ||
| 231 | - } | ||
| 232 | - | ||
| 233 | - if workaround: | ||
| 234 | - if not (len(workaround) == 1 and "dropout_p" in workaround): | ||
| 235 | - raise ValueError("not (len(workaround) == 1 and dropout_p in workaround)") | ||
| 236 | - # functools.partial insufficient because we look at signature downstream | ||
| 237 | - pattern = partialize_and_update_signature(pattern, dropout_p=0.0) | ||
| 238 | - replacement = partialize_and_update_signature( | ||
| 239 | - replacement, dropout_p=0.0 | ||
| 240 | - ) | ||
| 241 | - workaround = {} | ||
| 242 | - | ||
| 243 | - inference_name = name + "_inference" | ||
| 244 | - yield inference_name, { | ||
| 245 | - "search_fn": pattern, | ||
| 246 | - "replace_fn": replacement, | ||
| 247 | - "example_inputs": args, | ||
| 248 | - "trace_fn": fwd_only, | ||
| 249 | - "pass_dicts": patterns, | ||
| 250 | - "scalar_workaround": workaround, | ||
| 251 | - } | ||
| 252 | - | ||
| 253 | - for _, register_replacement_kwargs in _get_sfdp_patterns(): | ||
| 254 | - register_replacement( | ||
| 255 | - **register_replacement_kwargs, | ||
| 256 | - ) | ||
| @@ -112,8 +112,6 @@ NPU_EXTRA_FALLBACK_LIST = [ | |||
| 112 | aten._foobar.out, | 112 | aten._foobar.out, |
| 113 | aten._int_mm.out, | 113 | aten._int_mm.out, |
| 114 | aten._jagged_to_padded_dense_forward.default, | 114 | aten._jagged_to_padded_dense_forward.default, |
| 115 | - aten._local_scalar_dense, | ||
| 116 | - aten._local_scalar_dense.default, | ||
| 117 | aten._neg_view, | 115 | aten._neg_view, |
| 118 | aten._neg_view.default, | 116 | aten._neg_view.default, |
| 119 | aten._padded_dense_to_jagged_forward, | 117 | aten._padded_dense_to_jagged_forward, |
| @@ -577,10 +575,6 @@ NPU_EXTRA_FALLBACK_LIST = [ | |||
| 577 | prims.igamma, | 575 | prims.igamma, |
| 578 | prims.igammac, | 576 | prims.igammac, |
| 579 | prims.inductor_force_stride_order.default, | 577 | prims.inductor_force_stride_order.default, |
| 580 | - prims.inductor_lookup_seed.default, | ||
| 581 | - prims.inductor_randint.default, | ||
| 582 | - prims.inductor_random.default, | ||
| 583 | - prims.inductor_seed.default, | ||
| 584 | prims.le, | 578 | prims.le, |
| 585 | prims.le.default, | 579 | prims.le.default, |
| 586 | prims.lgamma, | 580 | prims.lgamma, |
| @@ -824,14 +818,6 @@ TORCH_NATIVE_FALLBACK_LIST = [ | |||
| 824 | aten.polygamma.out, | 818 | aten.polygamma.out, |
| 825 | aten.rand.default, | 819 | aten.rand.default, |
| 826 | aten.rand.generator, | 820 | aten.rand.generator, |
| 827 | - aten.randint.default, | ||
| 828 | - aten.randint.generator, | ||
| 829 | - aten.randint.generator_out, | ||
| 830 | - aten.randint.low, | ||
| 831 | - aten.randint.low_generator, | ||
| 832 | - aten.randint.low_generator_out, | ||
| 833 | - aten.randint.low_out, | ||
| 834 | - aten.randint.out, | ||
| 835 | aten.randn.default, | 821 | aten.randn.default, |
| 836 | aten.randn.generator, | 822 | aten.randn.generator, |
| 837 | aten.randperm.default, | 823 | aten.randperm.default, |