Star854
1342
代码
代码
Issues1621
Pull Requests2201
项目讨论
Wiki
分析
Star854
1342
已合并
[feat][v2.10.0]torch_npu profiler analysis ability of display prof level0 && kernel shapes #44928
xieanran创建于 16 天前
[feat][v2.10.0]torch_npu profiler analysis ability of display prof level0 && kernel shapes #44928
已合并
xieanran创建于 16 天前
从SoraAzzz/pytorchl0shape2.10.0合入到Ascend/pytorchv2.10.0
共 3 个文件变更+96-49
Mtest/profiler/analysis/prof_view/test_kernel_view_parser.py+85-47
@@ -1,56 +1,94 @@
1-import unittest1+import tempfile
2-from unittest.mock import patch, MagicMock2+from unittest.mock import patch
3-from torch_npu.profiler.analysis.prof_common_func._constant import Constant3+ 
4+from torch_npu.profiler.analysis.prof_bean._op_summary_bean import OpSummaryBean
5+from torch_npu.profiler.analysis.prof_common_func._csv_headers import CsvHeaders
4from torch_npu.profiler.analysis.prof_view._kernel_view_parser import KernelViewParser6from torch_npu.profiler.analysis.prof_view._kernel_view_parser import KernelViewParser
7+from torch_npu.testing.testcase import TestCase, run_tests
5 8 
6 9 
7-def run_test():10+class TestKernelViewParser(TestCase):
8- loader = unittest.TestLoader()
9- suite = unittest.TestSuite()
10- suite.addTests(loader.loadTestsFromTestCase(TestKernelViewParser))
11 11 
12- runner = unittest.TextTestRunner(verbosity=2)12+ def setUp(self):
13- runner.run(suite)13+ super().setUp()
14- 14+ self.temp_dir = tempfile.TemporaryDirectory()
15- 15+ self.parser_params = {
16-class TestKernelViewParser(unittest.TestCase):16+ "profiler_path": self.temp_dir.name,
17- 17+ "output_path": self.temp_dir.name,
18- def test_profect_map_for_headers_mixed(self):
19- input_headers = ["Op Name", "Unknown Header", "Kernel Duration"]
20- result = KernelViewParser._project_map_for_headers(input_headers)
21- expected = {"op_name", "Unknown Header", "Kernel Duration"}
22- self.assertEqual(result, expected)
23- 
24- @patch('torch_npu.profiler.analysis.prof_view._kernel_view_parser.ProfilerConifg')
25- @patch('torch_npu.profiler.analysis.prof_view._kernel_view_parser.CANNFileParser')
26- @patch('torch_npu.profiler.analysis.prof_view._kernel_view_parser.FileManager')
27- @patch('torch_npu.profiler.analysis.prof_view._kernel_view_parser.FwkCANNRelationParser')
28- def test_run_success(self, mock_relation_parser, mock_file_manager, mock_cann_parser,
29- mock_config):
30- mock_config_instance = mock_config.return_value
31- mock_config_instance.load_info.return_value = None
32- mock_cann_parser_instance = mock_cann_parser.return_value
33- mock_cann_parser_instance.get_file_list_by_type.return_value = ["test_file.csv"]
34- mock_file_manager.read_csv_file.return_value = [MagicMock(row=["test", "data"])]
35- mock_relation_parser_instance = mock_relation_parser.return_value
36- mock_relation_parser_instance.get_step_range.return_value = [{"step_id": 1, "start_ts": 1000, "end_ts": 2000}]
37- 
38- parser = KernelViewParser("test", {})
39- parser._profiler_path = "/test/path"
40- parser._output_path = "/test/output"
41- deps_data = {
42- Constant.TREE_BUILD_PARSER: [MagicMock()],
43- Constant.RELATION_PARSER: {"test": "data"}
44 }18 }
45- result = parser.run(deps_data)19+ OpSummaryBean.headers = []
46- self.assertEqual(result, (Constant.SUCCESS, None))20+ 
21+ def tearDown(self):
22+ OpSummaryBean.headers = []
23+ self.temp_dir.cleanup()
24+ super().tearDown()
25+ 
26+ def test_project_map_for_headers(self):
27+ input_headers = ["Op Name", "Unknown Header", "Task Duration(us)"]
47 28 
48- def test_profect_map_for_headers_matching(self):
49- input_headers = ["Name", "Kernel Time (ns)", "Calls"]
50 result = KernelViewParser._project_map_for_headers(input_headers)29 result = KernelViewParser._project_map_for_headers(input_headers)
51- expected = ["op_name", "kernel_time", "calls"]30+ 
52- self.assertEqual(result, expected)31+ self.assertEqual(["Name", "Unknown Header", "Duration(us)"], result)
32+ 
33+ def test_get_kernel_headers_for_level0_without_shape(self):
34+ all_headers = CsvHeaders.OP_SUMMARY_SHOW_HEADERS + ["Model ID"]
35+ 
36+ result = KernelViewParser._get_kernel_headers(all_headers, False)
37+ 
38+ self.assertEqual(CsvHeaders.OP_SUMMARY_SHOW_HEADERS, result)
39+ 
40+ def test_get_kernel_headers_for_level0_with_shape(self):
41+ all_headers = (CsvHeaders.OP_SUMMARY_SHOW_HEADERS + ["Model ID"]
42+ + CsvHeaders.OP_SUMMARY_SHAPE_HEADERS)
43+ 
44+ result = KernelViewParser._get_kernel_headers(all_headers, False)
45+ 
46+ self.assertEqual(CsvHeaders.OP_SUMMARY_SHOW_HEADERS + CsvHeaders.OP_SUMMARY_SHAPE_HEADERS, result)
47+ 
48+ def test_get_kernel_headers_for_non_level0(self):
49+ all_headers = CsvHeaders.OP_SUMMARY_SHOW_HEADERS + ["Model ID"]
50+ 
51+ result = KernelViewParser._get_kernel_headers(all_headers, True)
52+ 
53+ self.assertEqual(all_headers, result)
54+ 
55+ @patch("torch_npu.profiler.analysis.prof_view._kernel_view_parser.ProfilerConfig")
56+ @patch("torch_npu.profiler.analysis.prof_view._kernel_view_parser.CANNFileParser")
57+ @patch("torch_npu.profiler.analysis.prof_view._kernel_view_parser.FileManager")
58+ def test_generate_level0_view_without_shape(self, mock_file_manager, mock_cann_parser, mock_config):
59+ source_headers = CsvHeaders.OP_SUMMARY_SHOW_HEADERS + ["Model ID"]
60+ source_data = {header: str(index) for index, header in enumerate(source_headers)}
61+ mock_cann_parser.return_value.get_file_list_by_type.return_value = ["op_summary.csv"]
62+ mock_file_manager.read_csv_file.return_value = [OpSummaryBean(source_data)]
63+ mock_config.return_value.is_all_kernel_headers.return_value = False
64+ parser = KernelViewParser("test", self.parser_params)
65+ 
66+ parser.generate_view()
67+ 
68+ expected_row = [[source_data.get(header) for header in CsvHeaders.OP_SUMMARY_SHOW_HEADERS]]
69+ mock_file_manager.create_csv_file.assert_called_once_with(
70+ parser._output_path, expected_row, parser.KERNEL_VIEW, CsvHeaders.OP_SUMMARY_KERNEL_BASE_HEADERS)
71+ 
72+ @patch("torch_npu.profiler.analysis.prof_view._kernel_view_parser.ProfilerConfig")
73+ @patch("torch_npu.profiler.analysis.prof_view._kernel_view_parser.CANNFileParser")
74+ @patch("torch_npu.profiler.analysis.prof_view._kernel_view_parser.FileManager")
75+ def test_generate_level0_view_with_shape(self, mock_file_manager, mock_cann_parser, mock_config):
76+ source_headers = (CsvHeaders.OP_SUMMARY_SHOW_HEADERS + ["Model ID"]
77+ + CsvHeaders.OP_SUMMARY_SHAPE_HEADERS)
78+ source_data = {header: str(index) for index, header in enumerate(source_headers)}
79+ mock_cann_parser.return_value.get_file_list_by_type.return_value = ["op_summary.csv"]
80+ mock_file_manager.read_csv_file.return_value = [OpSummaryBean(source_data)]
81+ mock_config.return_value.is_all_kernel_headers.return_value = False
82+ parser = KernelViewParser("test", self.parser_params)
83+ 
84+ parser.generate_view()
85+ 
86+ expected_source_headers = CsvHeaders.OP_SUMMARY_SHOW_HEADERS + CsvHeaders.OP_SUMMARY_SHAPE_HEADERS
87+ expected_row = [[source_data.get(header) for header in expected_source_headers]]
88+ expected_output_headers = CsvHeaders.OP_SUMMARY_KERNEL_BASE_HEADERS + CsvHeaders.OP_SUMMARY_SHAPE_HEADERS
89+ mock_file_manager.create_csv_file.assert_called_once_with(
90+ parser._output_path, expected_row, parser.KERNEL_VIEW, expected_output_headers)
53 91 
54 92 
55-if __name__ == '__main__':93+if __name__ == "__main__":
56- run_test()94+ run_tests()
Mtorch_npu/profiler/analysis/prof_common_func/_csv_headers.py+2-0
@@ -3,5 +3,7 @@ class CsvHeaders(object):
3 TASK_START_TIME = "Task Start Time(us)"3 TASK_START_TIME = "Task Start Time(us)"
4 OP_SUMMARY_SHOW_HEADERS = ["Device_id", "Op Name", "OP Type", "Task Type", TASK_START_TIME, "Task Duration(us)",4 OP_SUMMARY_SHOW_HEADERS = ["Device_id", "Op Name", "OP Type", "Task Type", TASK_START_TIME, "Task Duration(us)",
5 "Task Wait Time(us)", "Block Num"]5 "Task Wait Time(us)", "Block Num"]
6+ OP_SUMMARY_SHAPE_HEADERS = ["Input Shapes", "Input Data Types", "Input Formats", "Output Shapes",
7+ "Output Data Types", "Output Formats"]
6 OP_SUMMARY_KERNEL_BASE_HEADERS = ["Device_id", "Name", "Type", "Accelerator Core", "Start Time(us)", "Duration(us)",8 OP_SUMMARY_KERNEL_BASE_HEADERS = ["Device_id", "Name", "Type", "Accelerator Core", "Start Time(us)", "Duration(us)",
7 "Wait Time(us)", "Block Num"]9 "Wait Time(us)", "Block Num"]
Mtorch_npu/profiler/analysis/prof_view/_kernel_view_parser.py+9-2
@@ -32,6 +32,13 @@ class KernelViewParser(BaseParser):
32 output_headers.append(header)32 output_headers.append(header)
33 return output_headers33 return output_headers
34 34 
35+ @classmethod
36+ def _get_kernel_headers(cls, all_headers: list, is_all_kernel_headers: bool) -> list:
37+ if is_all_kernel_headers:
38+ return all_headers
39+ shape_headers = [header for header in CsvHeaders.OP_SUMMARY_SHAPE_HEADERS if header in all_headers]
40+ return CsvHeaders.OP_SUMMARY_SHOW_HEADERS + shape_headers
41+ 
35 def run(self, deps_data: dict):42 def run(self, deps_data: dict):
36 ProfilerLogger.init(self._profiler_path, "KernelViewParser")43 ProfilerLogger.init(self._profiler_path, "KernelViewParser")
37 self.logger = ProfilerLogger.get_instance()44 self.logger = ProfilerLogger.get_instance()
@@ -53,8 +60,8 @@ class KernelViewParser(BaseParser):
53 for file_path in op_summary_file_set:60 for file_path in op_summary_file_set:
54 all_data = FileManager.read_csv_file(file_path, OpSummaryBean)61 all_data = FileManager.read_csv_file(file_path, OpSummaryBean)
55 if all_data:62 if all_data:
56- OpSummaryBean.headers = all_data[63+ OpSummaryBean.headers = self._get_kernel_headers(
57- 0].all_headers if ProfilerConfig().is_all_kernel_headers() else CsvHeaders.OP_SUMMARY_SHOW_HEADERS64+ all_data[0].all_headers, ProfilerConfig().is_all_kernel_headers())
58 output_headers = self._project_map_for_headers(OpSummaryBean.headers)65 output_headers = self._project_map_for_headers(OpSummaryBean.headers)
59 if not self.step_range:66 if not self.step_range:
60 summary_data.extend([data.row for data in all_data])67 summary_data.extend([data.row for data in all_data])