已合并
修复prof开关写死问题 #3583
chenweiheng创建于 7 天前
修复prof开关写死问题 #3583
已合并
共 4 个文件变更+37-9
| @@ -410,20 +410,17 @@ def codegen_cpp_wrapper(graph: FusedASCGraph): | |||
| 410 | launch_signature.extend([v.signature for v in itertools.chain(inputs, outputs, workspaces)]) | 410 | launch_signature.extend([v.signature for v in itertools.chain(inputs, outputs, workspaces)]) |
| 411 | launch_signature.append(f"{tiling_dtype} *tiling_data") | 411 | launch_signature.append(f"{tiling_dtype} *tiling_data") |
| 412 | 412 | ||
| 413 | - # Generate prof report code | ||
| 414 | - try: | ||
| 415 | - from torch_npu.profiler._profiler_path_creator import ProfPathCreator | ||
| 416 | - prof_enabled_str = "true" if ProfPathCreator().is_prof_inited else "false" | ||
| 417 | - except Exception: | ||
| 418 | - prof_enabled_str = "false" | ||
| 419 | - | ||
| 420 | prof_pop = " if (prof_enabled) { aclprofRangePop(); }" | 413 | prof_pop = " if (prof_enabled) { aclprofRangePop(); }" |
| 421 | 414 | ||
| 422 | prof_header = f'''#include "acl/acl_prof.h" | 415 | prof_header = f'''#include "acl/acl_prof.h" |
| 423 | #include "c10/util/generic_math.h" | 416 | #include "c10/util/generic_math.h" |
| 424 | #include <cmath> | 417 | #include <cmath> |
| 425 | 418 | ||
| 426 | -static const bool prof_enabled = {prof_enabled_str}; | 419 | +static bool prof_enabled = false; |
| 420 | + | ||
| 421 | +extern "C" void set_prof_enabled(uint8_t enabled) {{ | ||
| 422 | + prof_enabled = (enabled != 0); | ||
| 423 | +}} | ||
| 427 | 424 | ||
| 428 | #ifndef ACL_PROF_TENSOR_INFO_DEFINED | 425 | #ifndef ACL_PROF_TENSOR_INFO_DEFINED |
| 429 | #define ACL_PROF_TENSOR_INFO_DEFINED | 426 | #define ACL_PROF_TENSOR_INFO_DEFINED |
| @@ -8,7 +8,7 @@ from typing import List | |||
| 8 | import torch | 8 | import torch |
| 9 | from typing_extensions import override | 9 | from typing_extensions import override |
| 10 | 10 | ||
| 11 | -from ._kernel import NpuInductorKernel, _to_ctypes_arg | 11 | +from ._kernel import NpuInductorKernel, _to_ctypes_arg, _sync_prof_enabled |
| 12 | from ..common import logger | 12 | from ..common import logger |
| 13 | 13 | ||
| 14 | 14 | ||
| @@ -35,11 +35,13 @@ class NpuInductorAutoTuneKernel(NpuInductorKernel): | |||
| 35 | self.dl = cdll.LoadLibrary(self.lib_wrapper) | 35 | self.dl = cdll.LoadLibrary(self.lib_wrapper) |
| 36 | self.kernel = self.dl.wrapper | 36 | self.kernel = self.dl.wrapper |
| 37 | self._set_debug_log_path() | 37 | self._set_debug_log_path() |
| 38 | + _sync_prof_enabled(self.dl, self.lib_wrapper) | ||
| 38 | self.lib_kernel, self.kernel_key = autotune(args, self.dl, self.name, self.kernel_infos) | 39 | self.lib_kernel, self.kernel_key = autotune(args, self.dl, self.name, self.kernel_infos) |
| 39 | 40 | ||
| 40 | 41 | ||
| 41 | def run(self, args): | 42 | def run(self, args): |
| 42 | self._set_debug_log_path() | 43 | self._set_debug_log_path() |
| 44 | + _sync_prof_enabled(self.dl, self.lib_wrapper) | ||
| 43 | return self.kernel(*self.to_ctypes_args(args), self.default_stream, self.kernel_key.encode('utf-8')) | 45 | return self.kernel(*self.to_ctypes_args(args), self.default_stream, self.kernel_key.encode('utf-8')) |
| 44 | 46 | ||
| 45 | 47 | ||
| @@ -30,6 +30,21 @@ def _set_library_debug_log_path(dl, log_path): | |||
| 30 | setter(None if log_path is None else os.fsencode(log_path)) | 30 | setter(None if log_path is None else os.fsencode(log_path)) |
| 31 | 31 | ||
| 32 | 32 | ||
| 33 | +_prof_enabled_cache = {} | ||
| 34 | + | ||
| 35 | + | ||
| 36 | +def _sync_prof_enabled(dl, lib_wrapper): | ||
| 37 | + try: | ||
| 38 | + from torch_npu.profiler._profiler_path_creator import ProfPathCreator | ||
| 39 | + | ||
| 40 | + enabled = 1 if ProfPathCreator().is_prof_inited else 0 | ||
| 41 | + except Exception: | ||
| 42 | + enabled = 0 | ||
| 43 | + if _prof_enabled_cache.get(lib_wrapper) != enabled: | ||
| 44 | + dl.set_prof_enabled(c_uint8(enabled)) | ||
| 45 | + _prof_enabled_cache[lib_wrapper] = enabled | ||
| 46 | + | ||
| 47 | + | ||
| 33 | class NpuInductorKernel: | 48 | class NpuInductorKernel: |
| 34 | default_stream = c_void_p(0) | 49 | default_stream = c_void_p(0) |
| 35 | 50 | ||
| @@ -67,6 +82,7 @@ class NpuInductorKernel: | |||
| 67 | self.dl = cdll.LoadLibrary(self.lib_wrapper) | 82 | self.dl = cdll.LoadLibrary(self.lib_wrapper) |
| 68 | self.kernel = self.dl.wrapper | 83 | self.kernel = self.dl.wrapper |
| 69 | self._set_debug_log_path() | 84 | self._set_debug_log_path() |
| 85 | + _sync_prof_enabled(self.dl, self.lib_wrapper) | ||
| 70 | if ( | 86 | if ( |
| 71 | self.dl.init( | 87 | self.dl.init( |
| 72 | self.lib_kernel.encode('utf-8'), | 88 | self.lib_kernel.encode('utf-8'), |
| @@ -98,6 +114,7 @@ class NpuInductorKernel: | |||
| 98 | # The same cached wrapper.so can be called by different model objects on the same Python thread. | 114 | # The same cached wrapper.so can be called by different model objects on the same Python thread. |
| 99 | # Refresh its thread-local destination for every launch rather than only at load time. | 115 | # Refresh its thread-local destination for every launch rather than only at load time. |
| 100 | self._set_debug_log_path() | 116 | self._set_debug_log_path() |
| 117 | + _sync_prof_enabled(self.dl, self.lib_wrapper) | ||
| 101 | return self.kernel( | 118 | return self.kernel( |
| 102 | *self.to_ctypes_args(args), | 119 | *self.to_ctypes_args(args), |
| 103 | self.default_stream, | 120 | self.default_stream, |
| @@ -82,6 +82,18 @@ class ProfilingCodegenTest(unittest.TestCase): | |||
| 82 | f"{prefix}missing c10/util/generic_math.h (c10::div_floor_integer declaration)", | 82 | f"{prefix}missing c10/util/generic_math.h (c10::div_floor_integer declaration)", |
| 83 | ) | 83 | ) |
| 84 | self.assertIn("<cmath>", src, f"{prefix}missing <cmath> (std::floor declaration)") | 84 | self.assertIn("<cmath>", src, f"{prefix}missing <cmath> (std::floor declaration)") |
| 85 | + self.assertIn( | ||
| 86 | + "set_prof_enabled", | ||
| 87 | + src, | ||
| 88 | + f"{prefix}missing set_prof_enabled export; " | ||
| 89 | + "prof_enabled should be a runtime-mutable global, not a compile-time constant", | ||
| 90 | + ) | ||
| 91 | + self.assertNotIn( | ||
| 92 | + "static const bool prof_enabled", | ||
| 93 | + src, | ||
| 94 | + f"{prefix}compile-time prof_enabled constant found; " | ||
| 95 | + "prof_enabled should be a runtime-mutable global set via set_prof_enabled()", | ||
| 96 | + ) | ||
| 85 | 97 | ||
| 86 | def _assert_no_python_floordiv(self, src, label=""): | 98 | def _assert_no_python_floordiv(self, src, label=""): |
| 87 | """断言 fill_prof_tensors 的 shape 赋值行无 Python 风格 //。 | 99 | """断言 fill_prof_tensors 的 shape 赋值行无 Python 风格 //。 |