Star119
53
代码
代码
Issues17
Pull Requests27
流水线
项目讨论
Wiki
分析
Star119
53
已合并
【重构】win本地执行 ut失败问题处理，部分ut代码重构 #24
wangzixuan创建于 1月9日
【重构】win本地执行 ut失败问题处理，部分ut代码重构 #24
已合并
wangzixuan创建于 1月9日
从Seanesmhxocism/msproflocal合入到Ascend/msprofmaster
共 15 个文件变更+232-282
Manalysis/viewer/biu_perf/biu_perf_chip6_viewer.py+1-1
@@ -65,7 +65,7 @@ class BiuPerfChip6Viewer(BaseViewer, ABC):
65 column_trace_data))65 column_trace_data))
66 return result66 return result
67 67 
68- def get_timeline_data(self: any) -> str:68+ def get_timeline_data(self: any) -> list:
69 """69 """
70 get model list timeline data70 get model list timeline data
71 @return:timeline trace data71 @return:timeline trace data
Manalysis/viewer/cpu_data_report.py+1-4
@@ -111,13 +111,10 @@ def get_cpu_hot_function(project_path: str, db_name: str, table_name: str, heade
111 try:111 try:
112 if not (conn and curs) or not DBManager.judge_table_exist(curs, table_name):112 if not (conn and curs) or not DBManager.judge_table_exist(curs, table_name):
113 return MsvpConstant.MSVP_EMPTY_DATA113 return MsvpConstant.MSVP_EMPTY_DATA
114+ total_cycles = curs.execute("SELECT SUM(r11) FROM {};".format(table_name)).fetchone()[0]
114 except sqlite3.Error:115 except sqlite3.Error:
115 return MsvpConstant.MSVP_EMPTY_DATA116 return MsvpConstant.MSVP_EMPTY_DATA
116 else:117 else:
117- try:
118- total_cycles = curs.execute("SELECT SUM(r11) FROM {};".format(table_name)).fetchone()[0]
119- except sqlite3.Error:
120- return MsvpConstant.MSVP_EMPTY_DATA
121 if not total_cycles:118 if not total_cycles:
122 return MsvpConstant.MSVP_EMPTY_DATA119 return MsvpConstant.MSVP_EMPTY_DATA
123 cpu_hot_func_sql = "SELECT func,module,SUM(r11) AS cycles,CAST(1.0*SUM(r11)*100/? AS decimal(8,{})) " \120 cpu_hot_func_sql = "SELECT func,module,SUM(r11) AS cycles,CAST(1.0*SUM(r11)*100/? AS decimal(8,{})) " \
Mtest/msprof_python/ut/testcase/common_function/test_create_runtime_db.py+0-2
@@ -214,8 +214,6 @@ class TestParsingRuntimeData(unittest.TestCase):
214 26050, 0, 526, 0, 14002, 26675, 224, 12, 58006, 101612566070, 2, 0, 0, 0)214 26050, 0, 526, 0, 14002, 26675, 224, 12, 58006, 101612566070, 2, 0, 0, 0)
215 with mock.patch('os.path.join', return_value='test\\data'), \215 with mock.patch('os.path.join', return_value='test\\data'), \
216 mock.patch(NAMESPACE + '.ParsingRuntimeData.insert_data'), \216 mock.patch(NAMESPACE + '.ParsingRuntimeData.insert_data'), \
217- mock.patch('common_func.file_manager.is_linux', return_value=True), \
218- mock.patch('os.stat', return_value=Mock(st_mode=0o000, st_uid=0)), \
219 mock.patch(NAMESPACE + '.logging.info'):217 mock.patch(NAMESPACE + '.logging.info'):
220 with mock.patch('builtins.open', mock.mock_open(read_data=data_1)), \218 with mock.patch('builtins.open', mock.mock_open(read_data=data_1)), \
221 mock.patch('os.path.getsize', return_value=200), \219 mock.patch('os.path.getsize', return_value=200), \
Mtest/msprof_python/ut/testcase/common_function/test_file_manager.py+13-5
@@ -23,6 +23,7 @@ from collections import namedtuple
23from unittest import mock23from unittest import mock
24from unittest.mock import Mock24from unittest.mock import Mock
25 25 
26+from common_func.common import is_linux
26from common_func.constant import Constant27from common_func.constant import Constant
27from common_func.file_manager import FileManager, is_root_user, check_parent_dir_invalid28from common_func.file_manager import FileManager, is_root_user, check_parent_dir_invalid
28from common_func.file_manager import check_db_path_valid29from common_func.file_manager import check_db_path_valid
@@ -201,6 +202,9 @@ class TestFileManager(unittest.TestCase):
201 def test_check_path_valid_should_return_error_when_is_not_root_and_not_current_user(self):202 def test_check_path_valid_should_return_error_when_is_not_root_and_not_current_user(self):
202 path = '/path/host/host'203 path = '/path/host/host'
203 is_file = False204 is_file = False
205+ # win平台无校验必要
206+ if not is_linux():
207+ return
204 with mock.patch('os.path.exists', return_value=True), \208 with mock.patch('os.path.exists', return_value=True), \
205 mock.patch('os.path.isdir', return_value=True), \209 mock.patch('os.path.isdir', return_value=True), \
206 mock.patch('os.path.islink', return_value=False), \210 mock.patch('os.path.islink', return_value=False), \
@@ -322,14 +326,18 @@ class TestFileManager(unittest.TestCase):
322 self.assertFalse(is_link(path))326 self.assertFalse(is_link(path))
323 327 
324 def test_is_root_user_should_return_true_when_is_root(self):328 def test_is_root_user_should_return_true_when_is_root(self):
325- with mock.patch('os.getuid', return_value=0), \329+ if is_linux():
326- mock.patch(NAMESPACE + '.is_linux', return_value=True):330+ with mock.patch('os.getuid', return_value=0):
331+ self.assertFalse(is_root_user())
332+ else:
327 self.assertTrue(is_root_user())333 self.assertTrue(is_root_user())
328 334 
329 def test_is_root_user_should_return_false_when_is_not_root(self):335 def test_is_root_user_should_return_false_when_is_not_root(self):
330- with mock.patch('os.getuid', return_value=1), \336+ if is_linux():
331- mock.patch(NAMESPACE + '.is_linux', return_value=True):337+ with mock.patch('os.getuid', return_value=1):
332- self.assertFalse(is_root_user())338+ self.assertFalse(is_root_user())
339+ else:
340+ self.assertTrue(is_root_user())
333 341 
334 def test_check_parent_dir_invalid_should_return_true_when_check_path_valid_failed(self):342 def test_check_parent_dir_invalid_should_return_true_when_check_path_valid_failed(self):
335 try:343 try:
Mtest/msprof_python/ut/testcase/model/aic/test_ai_core_sample_model.py+6-6
@@ -98,6 +98,12 @@ class TestAiCoreSampleModel(unittest.TestCase):
98 check = AiCoreSampleModel('test', 'aicore.db', ['EventCount'], 'ai_core_metrics')98 check = AiCoreSampleModel('test', 'aicore.db', ['EventCount'], 'ai_core_metrics')
99 result = check.insert_metric_summary_table(freq, metric_key)99 result = check.insert_metric_summary_table(freq, metric_key)
100 self.assertEqual(result, None)100 self.assertEqual(result, None)
101+ with mock.patch(NAMESPACE + '.ConfigMgr.read_sample_config', return_value={}),\
102+ mock.patch(NAMESPACE + '.AiCoreSampleModel._get_metrics', side_effect=OSError), \
103+ mock.patch(NAMESPACE + '.logging.error'), \
104+ mock.patch(NAMESPACE + '.error'):
105+ check = AiCoreSampleModel('test', 'aicore.db', ['EventCount'], 'ai_core_metrics')
106+ check.insert_metric_summary_table(freq, 'ArithmeticUtilization')
101 with mock.patch(NAMESPACE + '.ConfigMgr.read_sample_config', return_value={}), \107 with mock.patch(NAMESPACE + '.ConfigMgr.read_sample_config', return_value={}), \
102 mock.patch(NAMESPACE + '.logging.info'), \108 mock.patch(NAMESPACE + '.logging.info'), \
103 mock.patch(NAMESPACE + '.AiCoreSampleModel.sql_insert_metric_summary_table',109 mock.patch(NAMESPACE + '.AiCoreSampleModel.sql_insert_metric_summary_table',
@@ -108,12 +114,6 @@ class TestAiCoreSampleModel(unittest.TestCase):
108 check.insert_metric_summary_table(freq, 'ArithmeticUtilization')114 check.insert_metric_summary_table(freq, 'ArithmeticUtilization')
109 res[0].commit()115 res[0].commit()
110 db_manager.destroy(res)116 db_manager.destroy(res)
111- with mock.patch(NAMESPACE + '.ConfigMgr.read_sample_config', return_value={}),\
112- mock.patch(NAMESPACE + '.AiCoreSampleModel._get_metrics', side_effect=OSError), \
113- mock.patch(NAMESPACE + '.logging.error'), \
114- mock.patch(NAMESPACE + '.error'):
115- check = AiCoreSampleModel('test', 'aicore.db', ['EventCount'], 'ai_core_metrics')
116- check.insert_metric_summary_table(freq, 'ArithmeticUtilization')
117 117 
118 def test_insert_metric_value(self):118 def test_insert_metric_value(self):
119 metrics_config = OrderedDict([('total_cycles', 'task_cyc'),119 metrics_config = OrderedDict([('total_cycles', 'task_cyc'),
Mtest/msprof_python/ut/testcase/model/stars/test_block_log_model.py+2-5
@@ -50,13 +50,10 @@ class TestBlockLogModel(TestDirCRBaseModel):
50 [2, 24, 2, task_type_ai_core, 1, 128, 3000, 4000, 1000],50 [2, 24, 2, task_type_ai_core, 1, 128, 3000, 4000, 1000],
51 [2, 25, 3, task_type_ai_core, 1, 128, 5000, 6000, 1000],51 [2, 25, 3, task_type_ai_core, 1, 128, 5000, 6000, 1000],
52 ]52 ]
53- model = BlockLogModel(self.PROF_DEVICE_DIR, DBNameConstant.DB_SOC_LOG, [DBNameConstant.TABLE_BLOCK_LOG])53+ with BlockLogModel(self.PROF_DEVICE_DIR, DBNameConstant.DB_SOC_LOG, [DBNameConstant.TABLE_BLOCK_LOG]) as model:
54- model.init()54+ model.flush(log_data)
55- model.flush(log_data)
56- model.finalize()
57 55 
58 with BlockLogViewModel(self.PROF_DEVICE_DIR, DBNameConstant.DB_SOC_LOG,56 with BlockLogViewModel(self.PROF_DEVICE_DIR, DBNameConstant.DB_SOC_LOG,
59 [DBNameConstant.TABLE_BLOCK_LOG]) as view_model:57 [DBNameConstant.TABLE_BLOCK_LOG]) as view_model:
60- view_model.init()
61 device_tasks = view_model.get_block_log_data()58 device_tasks = view_model.get_block_log_data()
62 self.assertEqual(len(device_tasks), 3)59 self.assertEqual(len(device_tasks), 3)
Mtest/msprof_python/ut/testcase/mscalculate/tiling_block_dim/test_block_dim_calculator.py+13-2
@@ -15,6 +15,7 @@
15# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.15# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
16# See the Mulan PSL v2 for more details.16# See the Mulan PSL v2 for more details.
17# -------------------------------------------------------------------------17# -------------------------------------------------------------------------
18+import gc
18import os19import os
19import unittest20import unittest
20 21 
@@ -28,7 +29,7 @@ from sqlite.db_manager import DBOpen
28NAMESPACE = 'mscalculate.tiling_block_dim.block_dim_calculator'29NAMESPACE = 'mscalculate.tiling_block_dim.block_dim_calculator'
29 30 
30 31 
31-class TestKfcCalculator(unittest.TestCase):32+class TestBlockDimCalculator(unittest.TestCase):
32 DIR_PATH = os.path.join(os.path.dirname(__file__), 'DT_BlockDimCalculator')33 DIR_PATH = os.path.join(os.path.dirname(__file__), 'DT_BlockDimCalculator')
33 DEVICE_SQLITE_DIR = os.path.join(DIR_PATH, 'PROF', 'device_0', 'sqlite')34 DEVICE_SQLITE_DIR = os.path.join(DIR_PATH, 'PROF', 'device_0', 'sqlite')
34 HOST_SQLITE_DIR = os.path.join(DIR_PATH, 'PROF', 'host', 'sqlite')35 HOST_SQLITE_DIR = os.path.join(DIR_PATH, 'PROF', 'host', 'sqlite')
@@ -51,6 +52,7 @@ class TestKfcCalculator(unittest.TestCase):
51 db_open._connect_db()52 db_open._connect_db()
52 db_open.create_table(create_sql)53 db_open.create_table(create_sql)
53 db_open.insert_data(DBNameConstant.TABLE_GE_TASK, data)54 db_open.insert_data(DBNameConstant.TABLE_GE_TASK, data)
55+ db_open._destroy_db_connect()
54 56 
55 def construct_ts_track_db(self):57 def construct_ts_track_db(self):
56 create_block_dim_sql = "CREATE TABLE IF NOT EXISTS " + DBNameConstant.TABLE_BLOCK_DIM + \58 create_block_dim_sql = "CREATE TABLE IF NOT EXISTS " + DBNameConstant.TABLE_BLOCK_DIM + \
@@ -69,13 +71,22 @@ class TestKfcCalculator(unittest.TestCase):
69 db_open.create_table(create_block_dim_sql)71 db_open.create_table(create_block_dim_sql)
70 db_open.insert_data(DBNameConstant.TABLE_BLOCK_DIM, block_dim_data)72 db_open.insert_data(DBNameConstant.TABLE_BLOCK_DIM, block_dim_data)
71 db_open.insert_data(DBNameConstant.TABLE_DEVICE_TASK_FLIP, flip_num_data)73 db_open.insert_data(DBNameConstant.TABLE_DEVICE_TASK_FLIP, flip_num_data)
74+ db_open._destroy_db_connect()
72 75 
73 def setUp(self) -> None:76 def setUp(self) -> None:
77+ if os.path.exists(self.DEVICE_SQLITE_DIR):
78+ clear_dt_project(self.DEVICE_SQLITE_DIR)
74 os.makedirs(self.DEVICE_SQLITE_DIR)79 os.makedirs(self.DEVICE_SQLITE_DIR)
80+ 
81+ if os.path.exists(self.HOST_SQLITE_DIR):
82+ clear_dt_project(self.HOST_SQLITE_DIR)
75 os.makedirs(self.HOST_SQLITE_DIR)83 os.makedirs(self.HOST_SQLITE_DIR)
76 84 
77 def tearDown(self) -> None:85 def tearDown(self) -> None:
78- clear_dt_project(self.DIR_PATH)86+ # 清理BlockDimCalculator中model链接，避免后续数据清理失败
87+ gc.collect()
88+ if os.path.exists(self.DIR_PATH):
89+ clear_dt_project(self.DIR_PATH)
79 90 
80 def test_ms_run_should_return_when_contain_block_dim_data(self: any) -> None:91 def test_ms_run_should_return_when_contain_block_dim_data(self: any) -> None:
81 self.construct_ts_track_db()92 self.construct_ts_track_db()
Mtest/msprof_python/ut/testcase/msprof/common/test_msprof_query_data.py+1-0
@@ -110,6 +110,7 @@ class TestMsprofQueryData(unittest.TestCase):
110 self.assertEqual(result, [[1, 1, "N/A"]])110 self.assertEqual(result, [[1, 1, "N/A"]])
111 _db_manager.clear_table("step_trace_data")111 _db_manager.clear_table("step_trace_data")
112 result = msprof_query_data._update_top_iteration_info([(1, 1), ], {1}, res[1])112 result = msprof_query_data._update_top_iteration_info([(1, 1), ], {1}, res[1])
113+ _db_manager.destroy(res)
113 self.assertEqual(result, [])114 self.assertEqual(result, [])
114 115 
115 def test_assembly_job_info_1(self):116 def test_assembly_job_info_1(self):
Mtest/msprof_python/ut/testcase/msprof/interface/test_msprof_export.py+2-2
@@ -762,8 +762,8 @@ class TestExportCommand(unittest.TestCase):
762 test._start_view(path_table)762 test._start_view(path_table)
763 763 
764 def test__start_view_export_summary_with_so(self) -> None:764 def test__start_view_export_summary_with_so(self) -> None:
765- path_table = {"collection_path": "test", "host": "test/host", "device": ["test/device_0"]}765+ path_table = {"collection_path": "./", "host": "./host", "device": ["./device_0"]}
766- args_dic = {"collection_path": "test", "iteration_id": None, "model_id": None, "iteration_count": None}766+ args_dic = {"collection_path": "./", "iteration_id": None, "model_id": None, "iteration_count": None}
767 args = Namespace(**args_dic)767 args = Namespace(**args_dic)
768 with mock.patch(NAMESPACE + '.ExportCommand._handle_export'), \768 with mock.patch(NAMESPACE + '.ExportCommand._handle_export'), \
769 mock.patch('framework.load_info_manager.LoadInfoManager.load_info'), \769 mock.patch('framework.load_info_manager.LoadInfoManager.load_info'), \
Mtest/msprof_python/ut/testcase/tuning/test_data_manager.py+4-0
@@ -94,16 +94,20 @@ class TestDataLoader(unittest.TestCase):
94 insert_sql = "insert into {0} values ({value})".format("GELoad", value="?," * (len(data[0]) - 1) + "?")94 insert_sql = "insert into {0} values ({value})".format("GELoad", value="?," * (len(data[0]) - 1) + "?")
95 db_manager = DBManager()95 db_manager = DBManager()
96 res = db_manager.create_table("ge_model_info.db", create_sql, insert_sql, data)96 res = db_manager.create_table("ge_model_info.db", create_sql, insert_sql, data)
97+ db_manager.destroy(res)
97 with mock.patch(NAMESPACE + '.OpSummaryTuningDataHandle.is_network', return_value=False):98 with mock.patch(NAMESPACE + '.OpSummaryTuningDataHandle.is_network', return_value=False):
98 result = OpSummaryTuningDataHandle.select_memory_workspace(project, device_id)99 result = OpSummaryTuningDataHandle.select_memory_workspace(project, device_id)
99 self.assertEqual(result, [])100 self.assertEqual(result, [])
101+ 
100 db_manager = DBManager()102 db_manager = DBManager()
101 res = db_manager.create_table("ge_model_info.db", create_sql, insert_sql, data)103 res = db_manager.create_table("ge_model_info.db", create_sql, insert_sql, data)
104+ db_manager.destroy(res)
102 with mock.patch(NAMESPACE + '.OpSummaryTuningDataHandle.is_network', return_value=True), \105 with mock.patch(NAMESPACE + '.OpSummaryTuningDataHandle.is_network', return_value=True), \
103 mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=(res[0], res[1])), \106 mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=(res[0], res[1])), \
104 mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True):107 mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True):
105 result = OpSummaryTuningDataHandle.select_memory_workspace(project, device_id)108 result = OpSummaryTuningDataHandle.select_memory_workspace(project, device_id)
106 self.assertEqual(result, [])109 self.assertEqual(result, [])
110+ 
107 db_manager = DBManager()111 db_manager = DBManager()
108 res = db_manager.create_table("ge_model_info.db", create_sql, insert_sql, data)112 res = db_manager.create_table("ge_model_info.db", create_sql, insert_sql, data)
109 with mock.patch(NAMESPACE + '.DBManager.destroy_db_connect'):113 with mock.patch(NAMESPACE + '.DBManager.destroy_db_connect'):
Mtest/msprof_python/ut/testcase/viewer/lpm/test_voltage_viewer.py+16-18
@@ -59,16 +59,15 @@ class TestVoltageViewer(TestDirCRBaseModel):
59 [950, 4]59 [950, 4]
60 ]60 ]
61 61 
62- aic_voltage_viewer_model = AicVoltageViewerModel(self.params)62+ with AicVoltageViewerModel(self.params) as aic_voltage_viewer_model:
63- aic_voltage_viewer_model.init()63+ aic_voltage_viewer_model.create_table()
64- aic_voltage_viewer_model.create_table()64+ aic_voltage_viewer_model.insert_data_to_db(DBNameConstant.TABLE_AIC_VOLTAGE, data)
65- aic_voltage_viewer_model.insert_data_to_db(DBNameConstant.TABLE_AIC_VOLTAGE, data)65+ result = aic_voltage_viewer_model.get_data()
66- result = aic_voltage_viewer_model.get_data()
67 66 
68- self.assertEqual(len(result), 3)67+ self.assertEqual(len(result), 3)
69- self.assertEqual(result[0], (900, 2))68+ self.assertEqual(result[0], (900, 2))
70- self.assertEqual(result[1], (850, 3))69+ self.assertEqual(result[1], (850, 3))
71- self.assertEqual(result[2], (950, 4))70+ self.assertEqual(result[2], (950, 4))
72 71 
73 def test_bus_voltage_data_viewer_model_get_data_return_ok(self):72 def test_bus_voltage_data_viewer_model_get_data_return_ok(self):
74 data = [73 data = [
@@ -77,16 +76,15 @@ class TestVoltageViewer(TestDirCRBaseModel):
77 [950, 4]76 [950, 4]
78 ]77 ]
79 78 
80- bus_voltage_viewer_model = BusVoltageViewerModel(self.params)79+ with BusVoltageViewerModel(self.params) as bus_voltage_viewer_model:
81- bus_voltage_viewer_model.init()80+ bus_voltage_viewer_model.create_table()
82- bus_voltage_viewer_model.create_table()81+ bus_voltage_viewer_model.insert_data_to_db(DBNameConstant.TABLE_BUS_VOLTAGE, data)
83- bus_voltage_viewer_model.insert_data_to_db(DBNameConstant.TABLE_BUS_VOLTAGE, data)82+ result = bus_voltage_viewer_model.get_data()
84- result = bus_voltage_viewer_model.get_data()
85 83 
86- self.assertEqual(len(result), 3)84+ self.assertEqual(len(result), 3)
87- self.assertEqual(result[0], (900, 2))85+ self.assertEqual(result[0], (900, 2))
88- self.assertEqual(result[1], (850, 3))86+ self.assertEqual(result[1], (850, 3))
89- self.assertEqual(result[2], (950, 4))87+ self.assertEqual(result[2], (950, 4))
90 88 
91 def test_voltage_viewer_get_all_data_return_ok(self):89 def test_voltage_viewer_get_all_data_return_ok(self):
92 """90 """
Mtest/msprof_python/ut/testcase/viewer/test_cpu_data_report.py+67-93
@@ -13,12 +13,13 @@
13# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.13# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14# See the Mulan PSL v2 for more details.14# See the Mulan PSL v2 for more details.
15# -------------------------------------------------------------------------15# -------------------------------------------------------------------------
16+import os.path
16import sqlite317import sqlite3
17import unittest18import unittest
18from unittest import mock19from unittest import mock
19 20 
20from common_func.msvp_constant import MsvpConstant21from common_func.msvp_constant import MsvpConstant
21-from sqlite.db_manager import DBManager22+from sqlite.db_manager import DBOpen
22from viewer.cpu_data_report import get_aictrl_pmu_events, get_ts_pmu_events, get_cpu_hot_function23from viewer.cpu_data_report import get_aictrl_pmu_events, get_ts_pmu_events, get_cpu_hot_function
23 24 
24NAMESPACE = 'viewer.cpu_data_report'25NAMESPACE = 'viewer.cpu_data_report'
@@ -28,124 +29,97 @@ sample_config = {"ts_cpu_profiling_events": "0x0,0x1,0x2"}
28 29 
29class TestCPUDataReport(unittest.TestCase):30class TestCPUDataReport(unittest.TestCase):
30 def test_get_aictrl_pmu_events_1(self):31 def test_get_aictrl_pmu_events_1(self):
31- with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=(True, True)), \32+ db_name = "test_get_aictrl_pmu_events_1_aicpu_0.db"
32- mock.patch(NAMESPACE + '.DBManager.judge_table_exist', side_effect=sqlite3.Error):33+ with DBOpen(db_name) as db_open:
33- res = get_aictrl_pmu_events('', 'aicpu_0.db', 'OriginalData', headers)34+ project_path = os.path.dirname(os.path.dirname(db_open.db_path))
35+ # table not exist
36+ res = get_aictrl_pmu_events(project_path, db_name, 'OriginalData', headers)
34 self.assertEqual(MsvpConstant.MSVP_EMPTY_DATA, res)37 self.assertEqual(MsvpConstant.MSVP_EMPTY_DATA, res)
35- 38+ # sqlite error
36- with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=(True, True)), \39+ with mock.patch(NAMESPACE + '.DBManager.judge_table_exist', side_effect=sqlite3.Error):
37- mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=False):40+ res = get_aictrl_pmu_events(project_path, db_name, 'OriginalData', headers)
38- res = get_aictrl_pmu_events('', 'aicpu_0.db', 'OriginalData', headers)41+ self.assertEqual(MsvpConstant.MSVP_EMPTY_DATA, res)
39- self.assertEqual(res, MsvpConstant.MSVP_EMPTY_DATA)
40 42 
41 def test_get_aictrl_pmu_events_2(self):43 def test_get_aictrl_pmu_events_2(self):
42 create_sql = "CREATE TABLE IF NOT EXISTS OriginalData (common, pid, tid, core, timestamp," \44 create_sql = "CREATE TABLE IF NOT EXISTS OriginalData (common, pid, tid, core, timestamp," \
43 " pmucount, pmuevent, ip, function, offset, module, replayid)"45 " pmucount, pmuevent, ip, function, offset, module, replayid)"
44 data = (("swapper", 0, 0, 7, 1467.43561, 8017770, "r11", "ffff000008087e80", "unknown",46 data = (("swapper", 0, 0, 7, 1467.43561, 8017770, "r11", "ffff000008087e80", "unknown",
45 "unknown", "[unknown]", 0),)47 "unknown", "[unknown]", 0),)
46- insert_sql = "insert into {0} values ({value})".format(48+ db_name = "test_get_aictrl_pmu_events_2_aicpu_0.db"
47- "OriginalData", value="?," * (len(data[0]) - 1) + "?")49+ with DBOpen(db_name) as db_open:
48- db_manager = DBManager()50+ db_open.create_table(create_sql)
49- test_sql = db_manager.create_table("aicpu_0.db", create_sql, insert_sql, data)51+ db_open.insert_data("OriginalData", data)
50- 52+ project_path = os.path.dirname(os.path.dirname(db_open.db_path))
51- with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=test_sql), \53+ res = get_aictrl_pmu_events(project_path, db_name, 'OriginalData', headers)
52- mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True):54+ self.assertEqual(res[2], 1)
53- res = get_aictrl_pmu_events('', 'aicpu_0.db', 'OriginalData', headers)55+ with mock.patch(NAMESPACE + '.read_cpu_cfg', return_value=None):
54- self.assertEqual(res[2], 1)56+ res = get_aictrl_pmu_events(project_path, db_name, 'OriginalData', headers)
55- 57+ self.assertEqual(res, MsvpConstant.MSVP_EMPTY_DATA)
56- test_sql = db_manager.connect_db('aicpu_0.db')
57- with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=test_sql), \
58- mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True), \
59- mock.patch(NAMESPACE + '.read_cpu_cfg', return_value=None):
60- res = get_aictrl_pmu_events('', 'aicpu_0.db', 'OriginalData', headers)
61- self.assertEqual(res, MsvpConstant.MSVP_EMPTY_DATA)
62- test_sql = db_manager.connect_db('aicpu_0.db')
63- (test_sql[1]).execute("drop Table OriginalData")
64- db_manager.destroy(test_sql)
65 58 
66 def test_get_ts_pmu_events_1(self):59 def test_get_ts_pmu_events_1(self):
67- with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=(True, True)), \60+ db_name = "test_get_ts_pmu_events_1_tscpu_0.db"
68- mock.patch("common_func.config_mgr.ConfigMgr.pre_check_sample", return_value=sample_config), \61+ # no db, connect failed
69- mock.patch(NAMESPACE + '.DBManager.judge_table_exist', side_effect=sqlite3.Error):62+ res = get_ts_pmu_events('', db_name, 'OriginalData', headers)
70- res = get_ts_pmu_events('', 'tscpu_0.db', 'OriginalData', headers)
71 self.assertEqual(res, MsvpConstant.MSVP_EMPTY_DATA)63 self.assertEqual(res, MsvpConstant.MSVP_EMPTY_DATA)
72 64 
73- with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=(None, None)), \65+ with DBOpen(db_name) as db_open, \
74 mock.patch("common_func.config_mgr.ConfigMgr.pre_check_sample", return_value=sample_config):66 mock.patch("common_func.config_mgr.ConfigMgr.pre_check_sample", return_value=sample_config):
75- res = get_ts_pmu_events('', 'tscpu_0.db', 'OriginalData', headers)67+ project_path = os.path.dirname(os.path.dirname(db_open.db_path))
76- self.assertEqual(res, MsvpConstant.MSVP_EMPTY_DATA)68+ # table not exist
77- 69+ res = get_ts_pmu_events(project_path, db_name, 'OriginalData', headers)
78- with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=(True, True)), \70+ self.assertEqual(res, MsvpConstant.MSVP_EMPTY_DATA)
79- mock.patch("common_func.config_mgr.ConfigMgr.pre_check_sample", return_value=sample_config), \71+ # sqlite error
80- mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=False):72+ with mock.patch(NAMESPACE + '.DBManager.judge_table_exist', side_effect=sqlite3.Error):
81- res = get_ts_pmu_events('', 'tscpu_0.db', 'OriginalData', headers)73+ res = get_ts_pmu_events(project_path, db_name, 'OriginalData', headers)
82- self.assertEqual(res, MsvpConstant.MSVP_EMPTY_DATA)74+ self.assertEqual(res, MsvpConstant.MSVP_EMPTY_DATA)
83 75 
84 def test_get_ts_pmu_events_2(self):76 def test_get_ts_pmu_events_2(self):
85 create_sql = "CREATE TABLE IF NOT EXISTS OriginalData (count, event)"77 create_sql = "CREATE TABLE IF NOT EXISTS OriginalData (count, event)"
86 data = ((1, "0x0"),)78 data = ((1, "0x0"),)
87- insert_sql = "insert into {0} values ({value})".format(79+ db_name = "test_get_ts_pmu_events_2_tscpu_0.db"
88- "OriginalData", value="?," * (len(data[0]) - 1) + "?")80+ with DBOpen(db_name) as db_open, \
89- db_manager = DBManager()
90- test_sql = db_manager.create_table("tscpu_0.db", create_sql, insert_sql, data)
91- 
92- with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=test_sql), \
93- mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True), \
94 mock.patch("common_func.config_mgr.ConfigMgr.pre_check_sample", return_value=sample_config):81 mock.patch("common_func.config_mgr.ConfigMgr.pre_check_sample", return_value=sample_config):
95- res = get_ts_pmu_events('', 'tscpu_0.db', 'OriginalData', headers)82+ db_open.create_table(create_sql)
96- self.assertEqual(res[2], 1)83+ db_open.insert_data("OriginalData", data)
97- 84+ project_path = os.path.dirname(os.path.dirname(db_open.db_path))
98- test_sql = db_manager.connect_db('tscpu_0.db')85+ # valid read_cpu_cfg
99- with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=test_sql), \86+ res = get_ts_pmu_events(project_path, db_name, 'OriginalData', headers)
100- mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True), \87+ self.assertEqual(res[2], 1)
101- mock.patch("common_func.config_mgr.ConfigMgr.pre_check_sample", return_value=sample_config), \88+ # invalid read_cpu_cfg
102- mock.patch(NAMESPACE + '.read_cpu_cfg', return_value=None):89+ with mock.patch(NAMESPACE + '.read_cpu_cfg', return_value=None):
103- res = get_ts_pmu_events('', 'tscpu_0.db', 'OriginalData', headers)90+ res = get_ts_pmu_events(project_path, db_name, 'OriginalData', headers)
104- self.assertEqual(res[2], 1)91+ self.assertEqual(res[2], 1)
105- test_sql = db_manager.connect_db('tscpu_0.db')
106- (test_sql[1]).execute("drop Table OriginalData")
107- db_manager.destroy(test_sql)
108 92 
109 def test_get_cpu_hot_function_1(self):93 def test_get_cpu_hot_function_1(self):
110- with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=(True, True)), \94+ db_name = "test_get_cpu_hot_function_1_aicpu_0.db"
111- mock.patch(NAMESPACE + '.DBManager.judge_table_exist', side_effect=sqlite3.Error):95+ with DBOpen(db_name) as db_open:
112- res = get_cpu_hot_function('', 'aicpu_0.db', 'OriginalData', headers)96+ project_path = os.path.dirname(os.path.dirname(db_open.db_path))
113- self.assertEqual(res, MsvpConstant.MSVP_EMPTY_DATA)97+ # table not exist
114- 98+ res = get_aictrl_pmu_events(project_path, db_name, 'OriginalData', headers)
115- with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=(True, True)), \99+ self.assertEqual(MsvpConstant.MSVP_EMPTY_DATA, res)
116- mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=False):100+ # sqlite error
117- res = get_cpu_hot_function('', 'aicpu_0.db', 'OriginalData', headers)101+ with mock.patch(NAMESPACE + '.DBManager.judge_table_exist', side_effect=sqlite3.Error):
118- self.assertEqual(res, MsvpConstant.MSVP_EMPTY_DATA)102+ res = get_aictrl_pmu_events(project_path, db_name, 'OriginalData', headers)
103+ self.assertEqual(MsvpConstant.MSVP_EMPTY_DATA, res)
119 104 
120 def test_get_cpu_hot_function_2(self):105 def test_get_cpu_hot_function_2(self):
121- db_manager = DBManager()106+ db_name = "test_get_cpu_hot_function_2_aicpu_0.db"
122 create_sql = "CREATE TABLE IF NOT EXISTS OriginalData (common, pid, tid, core, timestamp," \107 create_sql = "CREATE TABLE IF NOT EXISTS OriginalData (common, pid, tid, core, timestamp," \
123 " pmucount, cycles, ip, func, offset, module, r11)"108 " pmucount, cycles, ip, func, offset, module, r11)"
124- create_accu_sql = "CREATE TABLE IF NOT EXISTS OriginalData (common, pid, tid, core, timestamp," \
125- " pmucount, cycles, ip, func, offset, module, r11)"
126 data = (("swapper", 0, 0, 7, 1467.43561, 8017770, "r11", "ffff000008087e80", "unknown",109 data = (("swapper", 0, 0, 7, 1467.43561, 8017770, "r11", "ffff000008087e80", "unknown",
127 "unknown", "[unknown]", 3),)110 "unknown", "[unknown]", 3),)
128- insert_sql = "insert into {0} values ({value})".format(111+ with DBOpen(db_name) as db_open:
129- "OriginalData", value="?," * (len(data[0]) - 1) + "?")112+ db_open.create_table(create_sql)
130- insert_accu_sql = "insert into {0} values ({value})".format(113+ project_path = os.path.dirname(os.path.dirname(db_open.db_path))
131- "OriginalData", value="?," * (len(data[0]) - 1) + "?")114+ # no data
132- db_manager.create_table("aicpu_0.db", create_accu_sql, insert_accu_sql, data)115+ res = get_cpu_hot_function(project_path, db_name, 'OriginalData', headers)
133- test_sql = db_manager.create_table("aicpu_0.db", create_sql, insert_sql, data)116+ self.assertEqual(res[2], 0)
134 117 
135- with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=test_sql), \118+ # 原来就是插了2条一样的，意义不明
136- mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True):119+ db_open.insert_data("OriginalData", data)
137- res = get_cpu_hot_function('', 'aicpu_0.db', 'OriginalData', headers)120+ db_open.insert_data("OriginalData", data)
138- self.assertEqual(res[2], 1)121+ res = get_cpu_hot_function(project_path, db_name, 'OriginalData', headers)
139- 122+ self.assertEqual(res[2], 1)
140- test_sql = db_manager.connect_db('aicpu_0.db')
141- (test_sql[1]).execute("delete from OriginalData")
142- with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=test_sql), \
143- mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True):
144- res = get_cpu_hot_function('', 'aicpu_0.db', 'OriginalData', headers)
145- self.assertEqual(res[2], 0)
146- test_sql = db_manager.connect_db('aicpu_0.db')
147- (test_sql[1]).execute("drop Table OriginalData")
148- db_manager.destroy(test_sql)
149 123 
150 124 
151if __name__ == '__main__':125if __name__ == '__main__':
Mtest/msprof_python/ut/testcase/viewer/test_get_msvp_llc_timeline_training.py+0-5
@@ -279,7 +279,6 @@ class TestLLCTimelineTrain(unittest.TestCase):
279 db_manager = DBManager()279 db_manager = DBManager()
280 test_sql = db_manager.create_table("hbm.db")280 test_sql = db_manager.create_table("hbm.db")
281 db_manager.destroy(test_sql)281 db_manager.destroy(test_sql)
282- test_sql = db_manager.create_table("hbm.db")
283 with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=test_sql), \282 with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=test_sql), \
284 mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True), \283 mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True), \
285 mock.patch(NAMESPACE + '._reformat_hbm_data', side_effect=OSError):284 mock.patch(NAMESPACE + '._reformat_hbm_data', side_effect=OSError):
@@ -288,7 +287,6 @@ class TestLLCTimelineTrain(unittest.TestCase):
288 287 
289 test_sql = db_manager.create_table("hbm.db")288 test_sql = db_manager.create_table("hbm.db")
290 db_manager.destroy(test_sql)289 db_manager.destroy(test_sql)
291- test_sql = db_manager.create_table("hbm.db")
292 with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=(None, None)), \290 with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=(None, None)), \
293 mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True):291 mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True):
294 res = get_hbm_timeline(param)292 res = get_hbm_timeline(param)
@@ -296,7 +294,6 @@ class TestLLCTimelineTrain(unittest.TestCase):
296 294 
297 test_sql = db_manager.create_table("hbm.db")295 test_sql = db_manager.create_table("hbm.db")
298 db_manager.destroy(test_sql)296 db_manager.destroy(test_sql)
299- test_sql = db_manager.create_table("hbm.db")
300 with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=test_sql), \297 with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=test_sql), \
301 mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True), \298 mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True), \
302 mock.patch(NAMESPACE + '._reformat_hbm_data', return_value=[]):299 mock.patch(NAMESPACE + '._reformat_hbm_data', return_value=[]):
@@ -306,12 +303,10 @@ class TestLLCTimelineTrain(unittest.TestCase):
306 303 
307 test_sql = db_manager.create_table("hbm.db")304 test_sql = db_manager.create_table("hbm.db")
308 db_manager.destroy(test_sql)305 db_manager.destroy(test_sql)
309- test_sql = db_manager.create_table("hbm.db")
310 with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=test_sql), \306 with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=test_sql), \
311 mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=False):307 mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=False):
312 res = get_hbm_timeline(param)308 res = get_hbm_timeline(param)
313 self.assertEqual(len(res), 0)309 self.assertEqual(len(res), 0)
314- db_manager.destroy(test_sql)
315 310 
316 def test_get_hbm_timeline_3(self):311 def test_get_hbm_timeline_3(self):
317 param['end_time'] = 0312 param['end_time'] = 0
Mtest/msprof_python/ut/testcase/viewer/test_runtime_report.py+47-60
@@ -23,7 +23,7 @@ from common_func.db_name_constant import DBNameConstant
23from common_func.memcpy_constant import MemoryCopyConstant23from common_func.memcpy_constant import MemoryCopyConstant
24from common_func.ms_constant.str_constant import StrConstant24from common_func.ms_constant.str_constant import StrConstant
25from common_func.msvp_constant import MsvpConstant25from common_func.msvp_constant import MsvpConstant
26-from sqlite.db_manager import DBManager26+from sqlite.db_manager import DBManager, DBOpen
27from viewer.runtime_report import _get_output_event_counter27from viewer.runtime_report import _get_output_event_counter
28from viewer.runtime_report import add_memcpy_data28from viewer.runtime_report import add_memcpy_data
29from viewer.runtime_report import add_op_total29from viewer.runtime_report import add_op_total
@@ -50,36 +50,29 @@ class TestRuntimeReport(unittest.TestCase):
50 def test_get_task_scheduler_data_1(self):50 def test_get_task_scheduler_data_1(self):
51 db_manager = DBManager()51 db_manager = DBManager()
52 test_sql = db_manager.create_table("runtime.db")52 test_sql = db_manager.create_table("runtime.db")
53- configs = {"headers": "Time(%),Time(us),Count,Avg(us),Min(us),Max(us),"53+ tmp_config = {"headers": "Time(%),Time(us),Count,Avg(us),Min(us),Max(us),"
54- "Waiting(us),Running(us),Pending(us),Type,API,Task ID,Op Name,Stream ID"}54+ "Waiting(us),Running(us),Pending(us),Type,API,Task ID,Op Name,Stream ID"}
55 with mock.patch(NAMESPACE + '.DBManager.check_connect_db_path', return_value=(None, None)), \55 with mock.patch(NAMESPACE + '.DBManager.check_connect_db_path', return_value=(None, None)), \
56 mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=False):56 mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=False):
57- res = get_task_scheduler_data('', "ReportTask", configs, params)57+ res = get_task_scheduler_data('', "ReportTask", tmp_config, params)
58 self.assertEqual(res, ("Time(%),Time(us),Count,Avg(us),Min(us),Max(us),"58 self.assertEqual(res, ("Time(%),Time(us),Count,Avg(us),Min(us),Max(us),"
59- "Waiting(us),Running(us),Pending(us),Type,API,Task ID,Op Name,Stream ID",59+ "Waiting(us),Running(us),Pending(us),Type,API,Task ID,Op Name,Stream ID", [], 0))
60- [],60+ db_manager.destroy(test_sql)
61- 0))
62 61 
63 def test_get_task_scheduler_data_2(self):62 def test_get_task_scheduler_data_2(self):
64 create_sql = "CREATE TABLE IF NOT EXISTS ReportTask" \63 create_sql = "CREATE TABLE IF NOT EXISTS ReportTask" \
65 " (timeratio REAL,time REAL,count INTEGER,avg REAL,min REAL,max REAL,waiting REAL,running REAL," \64 " (timeratio REAL,time REAL,count INTEGER,avg REAL,min REAL,max REAL,waiting REAL,running REAL," \
66 "pending REAL,type TEXT,api TEXT,task_id INTEGER,stream_id INTEGER,device_id)"65 "pending REAL,type TEXT,api TEXT,task_id INTEGER,stream_id INTEGER,device_id)"
67 data = ((0.1, 1, 2, 1, 1, 5, 3, 1, 0.5, "aicore", "a", 0, 0, 0),)66 data = ((0.1, 1, 2, 1, 1, 5, 3, 1, 0.5, "aicore", "a", 0, 0, 0),)
68- db_manager = DBManager()67+ tmp_config = {"headers": "Time(%),Time(us),Count,Avg(us),Min(us),Max(us),"
69- insert_sql = db_manager.insert_sql("ReportTask", data)68+ "Waiting(us),Running(us),Pending(us),Type,API,Task ID,Op Name,Stream ID"}
70- 69+ db_name = "test_get_task_scheduler_data_2_runtime.db"
71- configs = {"headers": "Time(%),Time(us),Count,Avg(us),Min(us),Max(us),"70+ with DBOpen(db_name) as db_open, \
72- "Waiting(us),Running(us),Pending(us),Type,API,Task ID,Op Name,Stream ID"}
73- 
74- test_sql = db_manager.create_table("runtime.db", create_sql, insert_sql, data)
75- with mock.patch(NAMESPACE + '.DBManager.check_connect_db_path', return_value=test_sql), \
76- mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True), \
77 mock.patch(NAMESPACE + '.add_memcpy_data', return_value=data):71 mock.patch(NAMESPACE + '.add_memcpy_data', return_value=data):
78- res = get_task_scheduler_data('', "ReportTask", configs, params)72+ db_open.create_table(create_sql)
79- test_sql = db_manager.connect_db("runtime.db")73+ db_open.insert_data("ReportTask", data)
80- (test_sql[1]).execute("drop Table ReportTask")74+ res = get_task_scheduler_data(db_open.db_path, "ReportTask", tmp_config, params)
81- db_manager.destroy(test_sql)75+ self.assertEqual(res[2], 1)
82- self.assertEqual(res[2], 1)
83 76 
84 def test_add_memcpy_data(self):77 def test_add_memcpy_data(self):
85 data = [(50, 10000, 1, 10000, 10000,78 data = [(50, 10000, 1, 10000, 10000,
@@ -129,20 +122,17 @@ class TestRuntimeReport(unittest.TestCase):
129 "icache_miss_rate, device_id, task_id, stream_id, index_id, model_id)"122 "icache_miss_rate, device_id, task_id, stream_id, index_id, model_id)"
130 data = ((0.0426514705882353, 58006, 0.019154, 0.449091473295866, 0, 0, 0.000387, 0.00906802744543668, 0, 0,123 data = ((0.0426514705882353, 58006, 0.019154, 0.449091473295866, 0, 0, 0.000387, 0.00906802744543668, 0, 0,
131 0.010296, 0.241388821846016, 0.019614, 0.459866220735786, 0.0535714285714286, 0, 3, 5, 1, 1),)124 0.010296, 0.241388821846016, 0.019614, 0.459866220735786, 0.0535714285714286, 0, 3, 5, 1, 1),)
132- db_manager = DBManager()125+ db_name = "test_get_task_based_core_data_2_" + DBNameConstant.DB_RUNTIME
133- insert_sql = db_manager.insert_sql(DBNameConstant.TABLE_METRIC_SUMMARY, data)126+ with DBOpen(db_name) as db_open, \
134- 
135- test_sql = db_manager.create_table(DBNameConstant.DB_RUNTIME, create_sql, insert_sql, data)
136- with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=test_sql), \
137 mock.patch("common_func.config_mgr.ConfigMgr.pre_check_sample", return_value=configs), \127 mock.patch("common_func.config_mgr.ConfigMgr.pre_check_sample", return_value=configs), \
138- mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True), \
139 mock.patch(NAMESPACE + '.add_op_total', return_value=[]), \128 mock.patch(NAMESPACE + '.add_op_total', return_value=[]), \
140- mock.patch(NAMESPACE + '.cal_metrics', return_value=[1, 2]):129+ mock.patch(NAMESPACE + '.cal_metrics', return_value=[1, 2]), \
141- res = get_task_based_core_data('', DBNameConstant.DB_RUNTIME, params)130+ mock.patch('common_func.path_manager.PathManager.get_db_path', return_value=db_open.db_path):
142- test_sql = db_manager.connect_db(DBNameConstant.DB_RUNTIME)131+ db_open.create_table(create_sql)
143- (test_sql[1]).execute("drop Table {}".format(DBNameConstant.TABLE_METRIC_SUMMARY))132+ db_open.insert_data(DBNameConstant.TABLE_METRIC_SUMMARY, data)
144- db_manager.destroy(test_sql)133+ project_path = os.path.dirname(os.path.dirname(db_open.db_path))
145- self.assertEqual(res[0], 1)134+ res = get_task_based_core_data(project_path, DBNameConstant.DB_RUNTIME, params)
135+ self.assertEqual(res[0], 1)
146 136 
147 def test_get_task_based_core_data_3(self):137 def test_get_task_based_core_data_3(self):
148 params["data_type"] = StrConstant.AI_VECTOR_CORE_PMU_EVENTS138 params["data_type"] = StrConstant.AI_VECTOR_CORE_PMU_EVENTS
@@ -152,19 +142,18 @@ class TestRuntimeReport(unittest.TestCase):
152 "icache_miss_rate, device_id, task_id, stream_id, index_id, model_id)"142 "icache_miss_rate, device_id, task_id, stream_id, index_id, model_id)"
153 data = ((0.0426514705882353, 58001, 0.019154, 0.449091473295866, 0, 0, 0.000387, 0.00906802744543668, 0, 0,143 data = ((0.0426514705882353, 58001, 0.019154, 0.449091473295866, 0, 0, 0.000387, 0.00906802744543668, 0, 0,
154 0.010296, 0.241388821846016, 0.019614, 0.459866220735786, 0.0535714285714286, 0, 3, 5, 1, 1),)144 0.010296, 0.241388821846016, 0.019614, 0.459866220735786, 0.0535714285714286, 0, 3, 5, 1, 1),)
155- db_manager = DBManager()145+ db_name = "test_get_task_based_core_data_3_" + DBNameConstant.DB_RUNTIME
156- insert_sql = db_manager.insert_sql(DBNameConstant.TABLE_AIV_METRIC_SUMMARY, data)146+ with DBOpen(db_name) as db_open, \
147+ mock.patch("common_func.config_mgr.ConfigMgr.pre_check_sample", return_value=configs), \
148+ mock.patch(NAMESPACE + '._get_output_event_counter', return_value=[1, 2]), \
149+ mock.patch('common_func.path_manager.PathManager.get_db_path', return_value=db_open.db_path):
150+ db_open.create_table(create_sql)
151+ db_open.insert_data(DBNameConstant.TABLE_AIV_METRIC_SUMMARY, data)
152+ project_path = os.path.dirname(os.path.dirname(db_open.db_path))
153+ res = get_task_based_core_data(project_path, DBNameConstant.DB_RUNTIME, params)
154+ self.assertEqual(res[0], 1)
157 155 
158- test_sql = db_manager.create_table(DBNameConstant.DB_RUNTIME, create_sql, insert_sql, data)156+ def test_get_output_task_type(self):
159- with mock.patch(NAMESPACE + '.DBManager.check_connect_db', return_value=test_sql), \
160- mock.patch(NAMESPACE + '._get_output_event_counter', return_value=[1, 2]):
161- res = get_task_based_core_data('', DBNameConstant.DB_RUNTIME, params)
162- test_sql = db_manager.connect_db(DBNameConstant.DB_RUNTIME)
163- (test_sql[1]).execute("drop Table {}".format(DBNameConstant.TABLE_AIV_METRIC_SUMMARY))
164- db_manager.destroy(test_sql)
165- self.assertEqual(res[0], 1)
166- 
167- def test_get_output_tasktype(self):
168 db_manager = DBManager()157 db_manager = DBManager()
169 test_sql = db_manager.create_table(DBNameConstant.DB_RUNTIME)158 test_sql = db_manager.create_table(DBNameConstant.DB_RUNTIME)
170 159 
@@ -200,20 +189,20 @@ class TestRuntimeReport(unittest.TestCase):
200 189 
201 def test_get_opname_2(self):190 def test_get_opname_2(self):
202 create_ge_sql = "CREATE TABLE IF NOT EXISTS " + DBNameConstant.TABLE_GE_TASK + \191 create_ge_sql = "CREATE TABLE IF NOT EXISTS " + DBNameConstant.TABLE_GE_TASK + \
203- " (device_id, model_name, model_id, op_name, stream_id, task_id, block_dim, op_state, " \192+ " (device_id, model_name, model_id, op_name, stream_id, task_id, batch_id, block_dim, " \
204- "task_type, op_type, iter_id, input_count, input_formats, input_data_types, input_shapes, " \193+ "op_state, task_type, op_type, iter_id, input_count, input_formats, input_data_types, " \
205- "output_count, output_formats, output_data_types, output_shapes)"194+ "input_shapes, output_count, output_formats, output_data_types, output_shapes)"
206- data = ((0, "resnet50", 1, "trans_TransData_0", 5, 3, 1, "static", "AI_CORE", "TransData", 0, 1, "NCHW",195+ data = ((0, "resnet50", 1, "trans_TransData_0", 5, 3, 0, 1, "static", "AI_CORE", "TransData", 0, 1, "NCHW",
207 "DT_FLOAT16", "1,3,224,224", 1, "NC1HWC0", "DT_FLOAT16", "1,1,224,224,16"),)196 "DT_FLOAT16", "1,3,224,224", 1, "NC1HWC0", "DT_FLOAT16", "1,1,224,224,16"),)
208- db_manager = DBManager()197+ db_name = "test_get_opname_2_" + DBNameConstant.DB_GE_INFO
209- insert_ge_sql = db_manager.insert_sql(DBNameConstant.TABLE_GE_TASK, data)198+ with DBOpen(db_name) as db_open, \
210- res = db_manager.create_table(DBNameConstant.DB_GE_INFO, create_ge_sql, insert_ge_sql, data)199+ mock.patch('common_func.path_manager.PathManager.get_db_path', return_value=db_open.db_path):
211- 200+ db_open.create_table(create_ge_sql)
212- res_dir = os.path.realpath(os.path.join(db_manager.db_path, ".."))201+ db_open.insert_data(DBNameConstant.TABLE_GE_TASK, data)
213- res_ge = get_opname([3, 5, -1], res_dir, res[1])202+ res_dir = os.path.realpath(os.path.join(db_open.db_path, ".."))
214- (res[1]).execute("drop Table {}".format(DBNameConstant.TABLE_GE_TASK))203+ # [-1, 3, 5, 0]: unknown, task_id, stream_id, batch_id
215- db_manager.destroy(res)204+ res_ge = get_opname([-1, 3, 5, 0], res_dir, db_open.db_curs)
216- # self.assertEqual(res_ge, 'trans_TransData_0')205+ self.assertEqual(res_ge, 'trans_TransData_0')
217 206 
218 def test_get_opname_3(self):207 def test_get_opname_3(self):
219 with mock.patch('os.path.exists', return_value=False):208 with mock.patch('os.path.exists', return_value=False):
@@ -239,8 +228,6 @@ class TestRuntimeReport(unittest.TestCase):
239 with mock.patch(NAMESPACE + ".get_opname", return_value=[1]):228 with mock.patch(NAMESPACE + ".get_opname", return_value=[1]):
240 res = add_op_total([[3, 4, -1]], res_dir)229 res = add_op_total([[3, 4, -1]], res_dir)
241 res_ge = add_op_total([[3, 4, -1]], res_dir)230 res_ge = add_op_total([[3, 4, -1]], res_dir)
242- # test_ge_sql = db_manager_ge.connect_db(DB_GE_INFO)
243- # test_sql = db_manager.connect_db(DB_RTS_TRACK)
244 db_manager_ge.destroy(test_ge_sql)231 db_manager_ge.destroy(test_ge_sql)
245 db_manager.destroy(test_sql)232 db_manager.destroy(test_sql)
246 self.assertEqual(len(res), 1)233 self.assertEqual(len(res), 1)
Mtest/msprof_python/ut/testcase/viewer/training/test_step_trace_viewer.py+59-79
@@ -27,7 +27,7 @@ from constant.constant import ITER_RANGE
27from constant.info_json_construct import DeviceInfo27from constant.info_json_construct import DeviceInfo
28from constant.info_json_construct import InfoJson28from constant.info_json_construct import InfoJson
29from constant.info_json_construct import InfoJsonReaderManager29from constant.info_json_construct import InfoJsonReaderManager
30-from sqlite.db_manager import DBManager30+from sqlite.db_manager import DBManager, DBOpen
31from viewer.training.step_trace_viewer import StepTraceViewer31from viewer.training.step_trace_viewer import StepTraceViewer
32 32 
33NAMESPACE = 'viewer.training.step_trace_viewer'33NAMESPACE = 'viewer.training.step_trace_viewer'
@@ -39,35 +39,21 @@ DB_TRACE = DBNameConstant.DB_TRACE
39TABLE_TRAINING_TRACE = DBNameConstant.TABLE_TRAINING_TRACE39TABLE_TRAINING_TRACE = DBNameConstant.TABLE_TRAINING_TRACE
40TABLE_ALL_REDUCE = DBNameConstant.TABLE_ALL_REDUCE40TABLE_ALL_REDUCE = DBNameConstant.TABLE_ALL_REDUCE
41 41 
42+DATA1 = (('127.0.0.1', 4, 1, 65, 144, 1013979616375, 65, 145, 1013981208724,
43+ 71, 189, 1013981571636, 64, 5, 2955261, 1592349, 362912, 0, 0),)
42 44 
43-def create_trace_db():45+DATA2 = (('127.0.0.1', 4, 1013981571636, 1013980654091, 70, 4, 1013981072280, 70, 103, 0),
44- db_manager = DBManager()46+ ('127.0.0.1', 4, 1013981571636, 1013981251417, 72, 6, 1013981476623, 72, 105, 0),)
45- if os.path.exists(os.path.join(db_manager.db_path, DB_TRACE)):
46- os.remove(os.path.join(db_manager.db_path, DB_TRACE))
47 47 
48- data1 = (('127.0.0.1', 4, 1, 65, 144, 1013979616375, 65, 145, 1013981208724,48+TRAINING_TRACE_SQL = "create table if not exists {0} (host_id int, device_id int, iteration_id int, " \
49- 71, 189, 1013981571636, 64, 5, 1955261, 1592349, 362912, 0, 0),)49+ "job_stream int, job_task int, FP_start int, FP_stream int, FP_task int, " \
50+ "BP_end int, BP_stream int, BP_task int, iteration_end int, iter_stream int," \
51+ "iter_task int, iteration_time int, fp_bp_time int, grad_refresh_bound int, " \
52+ "data_aug_bound int default 0, model_id int default 0)".format(TABLE_TRAINING_TRACE)
50 53 
51- data2 = (('127.0.0.1', 4, 1013981571636, 1013980654091, 70, 4, 1013981072280, 70, 103, 0),54+ALL_REDUCE_SQL = "create table if not exists {0}(host_id int, device_id int," \
52- ('127.0.0.1', 4, 1013981571636, 1013981251417, 72, 6, 1013981476623, 72, 105, 0),)55+ "iteration_end int, start int, start_stream int, start_task int," \
53- 56+ "end int, end_stream int, end_task int, model_id int default 0)".format(TABLE_ALL_REDUCE)
54- sql = "create table if not exists {0} (host_id int, device_id int, iteration_id int, " \
55- "job_stream int, job_task int, FP_start int, FP_stream int, FP_task int, " \
56- "BP_end int, BP_stream int, BP_task int, iteration_end int, iter_stream int," \
57- "iter_task int, iteration_time int, fp_bp_time int, grad_refresh_bound int, " \
58- "data_aug_bound int default 0, model_id int default 0)".format(TABLE_TRAINING_TRACE)
59- insert_sql = "insert into {0} values ({value})".format(TABLE_TRAINING_TRACE, value="?," * (len(data1[0]) - 1) + "?")
60- conn, cur = db_manager.create_table(DB_TRACE, sql, insert_sql, data1)
61- conn.close()
62- 
63- sql = "create table if not exists {0}(host_id int, device_id int," \
64- "iteration_end int, start int, start_stream int, start_task int," \
65- "end int, end_stream int, end_task int, model_id int default 0)".format(
66- TABLE_ALL_REDUCE)
67- insert_sql = "insert into {0} values ({value})".format(TABLE_ALL_REDUCE, value="?," * (len(data2[0]) - 1) + "?")
68- conn, cur = db_manager.create_table(DB_TRACE, sql, insert_sql, data2)
69- 
70- return db_manager, conn, cur
71 57 
72 58 
73class TestStepTraceViewer(unittest.TestCase):59class TestStepTraceViewer(unittest.TestCase):
@@ -75,33 +61,27 @@ class TestStepTraceViewer(unittest.TestCase):
75 def test_add_reduce_headers_should_return_headers_with_length_is_1(self):61 def test_add_reduce_headers_should_return_headers_with_length_is_1(self):
76 test_message = {'job_id': 'job_default', 'device_id': '4'}62 test_message = {'job_id': 'job_default', 'device_id': '4'}
77 headers = ["Model ID"]63 headers = ["Model ID"]
78- db_manager = DBManager()
79- if os.path.exists(os.path.join(db_manager.db_path, DB_TRACE)):
80- os.remove(os.path.join(db_manager.db_path, DB_TRACE))
81 sql = "create table if not exists {0}(host_id int, device_id int," \64 sql = "create table if not exists {0}(host_id int, device_id int," \
82 "iteration_end int, start int, start_stream int, start_task int," \65 "iteration_end int, start int, start_stream int, start_task int," \
83 "end int, end_stream int, end_task int, model_id int default 0)".format(66 "end int, end_stream int, end_task int, model_id int default 0)".format(
84 TABLE_ALL_REDUCE)67 TABLE_ALL_REDUCE)
85- conn, cur = db_manager.create_table(DB_TRACE, sql)68+ db_name = "test_add_reduce_headers_should_return_headers_with_length_is_1_" + DB_TRACE
86- 69+ with DBOpen(db_name) as db_open:
87- StepTraceViewer.add_reduce_headers(conn, headers, test_message)70+ db_open.create_table(sql)
88- conn.close()71+ StepTraceViewer.add_reduce_headers(db_open.db_conn, headers, test_message)
89- 72+ self.assertEqual(len(headers), 1)
90- if os.path.exists(db_manager.db_name):
91- os.remove(db_manager.db_name)
92- self.assertEqual(len(headers), 1)
93 73 
94 def test_add_reduce_headers_should_return_headers_with_length_is_5(self):74 def test_add_reduce_headers_should_return_headers_with_length_is_5(self):
95 test_message = {'job_id': 'job_default', 'device_id': '4'}75 test_message = {'job_id': 'job_default', 'device_id': '4'}
96 headers = ["Model ID"]76 headers = ["Model ID"]
97- db_manager, conn, curs = create_trace_db()77+ db_name = "test_add_reduce_headers_should_return_headers_with_length_is_5_" + DB_TRACE
98- 78+ with DBOpen(db_name) as db_open:
99- StepTraceViewer.add_reduce_headers(conn, headers, test_message)79+ db_open.create_table(TRAINING_TRACE_SQL)
100- conn.close()80+ db_open.create_table(ALL_REDUCE_SQL)
101- 81+ db_open.insert_data(TABLE_TRAINING_TRACE, DATA1)
102- if os.path.exists(db_manager.db_name):82+ db_open.insert_data(TABLE_ALL_REDUCE, DATA2)
103- os.remove(db_manager.db_name)83+ StepTraceViewer.add_reduce_headers(db_open.db_conn, headers, test_message)
104- self.assertEqual(len(headers), 5)84+ self.assertEqual(len(headers), 5)
105 85 
106 def test_add_reduce_headers_should_return_headers_with_length_is_11(self):86 def test_add_reduce_headers_should_return_headers_with_length_is_11(self):
107 data = (87 data = (
@@ -111,17 +91,15 @@ class TestStepTraceViewer(unittest.TestCase):
111 )91 )
112 test_message = {'job_id': 'job_default', 'device_id': '4'}92 test_message = {'job_id': 'job_default', 'device_id': '4'}
113 headers = ["Model ID"]93 headers = ["Model ID"]
114- db_manager, conn, curs = create_trace_db()94+ db_name = "test_add_reduce_headers_should_return_headers_with_length_is_11_" + DB_TRACE
115- insert_sql = "insert into {0} values ({value})".format(95+ with DBOpen(db_name) as db_open:
116- TABLE_ALL_REDUCE, value="?," * (len(data[0]) - 1) + "?")96+ db_open.create_table(TRAINING_TRACE_SQL)
117- db_manager._insert_data(insert_sql, data)97+ db_open.create_table(ALL_REDUCE_SQL)
118- 98+ db_open.insert_data(TABLE_TRAINING_TRACE, DATA1)
119- StepTraceViewer.add_reduce_headers(conn, headers, test_message)99+ db_open.insert_data(TABLE_ALL_REDUCE, DATA2)
120- conn.close()100+ db_open.insert_data(TABLE_ALL_REDUCE, data)
121- 101+ StepTraceViewer.add_reduce_headers(db_open.db_conn, headers, test_message)
122- if os.path.exists(db_manager.db_name):102+ self.assertEqual(len(headers), 11)
123- os.remove(db_manager.db_name)
124- self.assertEqual(len(headers), 11)
125 103 
126 def test_get_step_trace_summary(self):104 def test_get_step_trace_summary(self):
127 message1 = {'job_id': 'job_default', 'device_id': '4'}105 message1 = {'job_id': 'job_default', 'device_id': '4'}
@@ -133,42 +111,44 @@ class TestStepTraceViewer(unittest.TestCase):
133 self.assertEqual(res, ([], [], 0))111 self.assertEqual(res, ([], [], 0))
134 112 
135 message1 = {'job_id': 'job_default', 'device_id': '4', 'project_path': ''}113 message1 = {'job_id': 'job_default', 'device_id': '4', 'project_path': ''}
136- db_manager, conn, curs = create_trace_db()114+ db_name = "test_get_step_trace_summary_" + DB_TRACE
137- conn.close()115+ with DBOpen(db_name) as db_open, \
138- 116+ mock.patch(NAMESPACE + '.PathManager.get_sql_dir', return_value=db_open.db_path):
139- with mock.patch(NAMESPACE + '.PathManager.get_sql_dir', return_value=db_manager.db_path):117+ db_open.create_table(TRAINING_TRACE_SQL)
118+ db_open.create_table(ALL_REDUCE_SQL)
119+ db_open.insert_data(TABLE_TRAINING_TRACE, DATA1)
120+ db_open.insert_data(TABLE_ALL_REDUCE, DATA2)
140 res = StepTraceViewer.get_step_trace_summary(message1)121 res = StepTraceViewer.get_step_trace_summary(message1)
141- 122+ self.assertEqual(res[2], 0)
142- if os.path.exists(db_manager.db_name):
143- os.remove(db_manager.db_name)
144- self.assertEqual(res[2], 0)
145 123 
146 def test_step_trace_timeline(self):124 def test_step_trace_timeline(self):
147 message1 = {'job_id': 'job_default', 'device_id': '4', 'project_path': ''}125 message1 = {'job_id': 'job_default', 'device_id': '4', 'project_path': ''}
148- db_manager, conn, curs = create_trace_db()
149- conn.close()
150 InfoJsonReaderManager(info_json=InfoJson(devices='0', DeviceInfo=[126 InfoJsonReaderManager(info_json=InfoJson(devices='0', DeviceInfo=[
151 DeviceInfo(hwts_frequency='100').device_info])).process()127 DeviceInfo(hwts_frequency='100').device_info])).process()
152- with mock.patch(NAMESPACE + '.PathManager.get_sql_dir', return_value=db_manager.db_path):128+ db_name = "test_step_trace_timeline_" + DB_TRACE
129+ with DBOpen(db_name) as db_open, \
130+ mock.patch(NAMESPACE + '.PathManager.get_sql_dir', return_value=db_open.db_path):
131+ db_open.create_table(TRAINING_TRACE_SQL)
132+ db_open.create_table(ALL_REDUCE_SQL)
133+ db_open.insert_data(TABLE_TRAINING_TRACE, DATA1)
134+ db_open.insert_data(TABLE_ALL_REDUCE, DATA2)
153 res = StepTraceViewer.get_step_trace_timeline(message1)135 res = StepTraceViewer.get_step_trace_timeline(message1)
154- 136+ self.assertEqual(len(res), 0)
155- if os.path.exists(db_manager.db_name):
156- os.remove(db_manager.db_name)
157- self.assertEqual(len(res), 0)
158 137 
159 def test_get_one_iter_timeline_data(self):138 def test_get_one_iter_timeline_data(self):
160- db_manager, conn, curs = create_trace_db()
161- conn.close()
162 InfoJsonReaderManager(info_json=InfoJson(devices='0', DeviceInfo=[139 InfoJsonReaderManager(info_json=InfoJson(devices='0', DeviceInfo=[
163 DeviceInfo(hwts_frequency='100').device_info])).process()140 DeviceInfo(hwts_frequency='100').device_info])).process()
164 ProfilingScene().init("")141 ProfilingScene().init("")
165- with mock.patch(NAMESPACE + '.PathManager.get_sql_dir', return_value=db_manager.db_path), \142+ db_name = "test_get_one_iter_timeline_data_" + DB_TRACE
143+ with DBOpen(db_name) as db_open, \
144+ mock.patch(NAMESPACE + '.PathManager.get_sql_dir', return_value=db_open.db_path), \
166 mock.patch('common_func.utils.Utils.get_scene', return_value=Constant.STEP_INFO):145 mock.patch('common_func.utils.Utils.get_scene', return_value=Constant.STEP_INFO):
146+ db_open.create_table(TRAINING_TRACE_SQL)
147+ db_open.create_table(ALL_REDUCE_SQL)
148+ db_open.insert_data(TABLE_TRAINING_TRACE, DATA1)
149+ db_open.insert_data(TABLE_ALL_REDUCE, DATA2)
167 res = StepTraceViewer.get_one_iter_timeline_data("", ITER_RANGE)150 res = StepTraceViewer.get_one_iter_timeline_data("", ITER_RANGE)
168- 151+ self.assertEqual(len(res), 0)
169- db_manager.conn.close()
170- 
171- self.assertEqual(len(res), 0)
172 152 
173 def test_reformat_step_trace_data(self):153 def test_reformat_step_trace_data(self):
174 data_list = [154 data_list = [