已合并
fix mstx data analysis err in acl graph scene #261
mei-feiyao创建于 5月30日
fix mstx data analysis err in acl graph scene #261
已合并
共 12 个文件变更+951-319
| @@ -19,6 +19,7 @@ import decimal | |||
| 19 | import json | 19 | import json |
| 20 | import logging | 20 | import logging |
| 21 | import os | 21 | import os |
| 22 | +import re | ||
| 22 | from decimal import Decimal | 23 | from decimal import Decimal |
| 23 | from typing import Dict | 24 | from typing import Dict |
| 24 | 25 | ||
| @@ -497,6 +498,20 @@ class InfoConfReader: | |||
| 497 | prof_level = self._get_prof_level() | 498 | prof_level = self._get_prof_level() |
| 498 | return prof_level in (StrConstant.PROF_LEVEL_0, StrConstant.PROF_LEVEL_0_HISI) | 499 | return prof_level in (StrConstant.PROF_LEVEL_0, StrConstant.PROF_LEVEL_0_HISI) |
| 499 | 500 | ||
| 501 | + def get_cann_version(self) -> tuple: | ||
| 502 | + """ | ||
| 503 | + get cann version info(format: 9.0.0 or 9.0.0-beta.0 or 9.0.T106) from info.json | ||
| 504 | + return: major, minor, micro | ||
| 505 | + """ | ||
| 506 | + cann_version = self._info_json.get("cannVersion", "") | ||
| 507 | + # Extract major and minor version numbers | ||
| 508 | + version_match = re.match(r"^(\d+)\.(\d+)", cann_version) | ||
| 509 | + if version_match: | ||
| 510 | + major, minor = version_match.groups() | ||
| 511 | + return int(major), int(minor) | ||
| 512 | + logging.warning("Cann version format is invalid or missing in info.json, version string: %s", cann_version) | ||
| 513 | + return 0, 0 | ||
| 514 | + | ||
| 500 | def _get_prof_level(self: any) -> str: | 515 | def _get_prof_level(self: any) -> str: |
| 501 | """ | 516 | """ |
| 502 | get prof level from sample.json | 517 | get prof level from sample.json |
| @@ -15,10 +15,13 @@ | |||
| 15 | * -------------------------------------------------------------------------*/ | 15 | * -------------------------------------------------------------------------*/ |
| 16 | 16 | ||
| 17 | 17 | ||
| 18 | + | ||
| 18 | 19 | ||
| 19 | 20 | ||
| 20 | -namespace Analysis { | 21 | +namespace Analysis |
| 21 | -namespace Domain { | 22 | +{ |
| 23 | +namespace Domain | ||
| 24 | +{ | ||
| 22 | using namespace Analysis::Domain::Environment; | 25 | using namespace Analysis::Domain::Environment; |
| 23 | using namespace Analysis::Utils; | 26 | using namespace Analysis::Utils; |
| 24 | MsprofTxDeviceProcessor::MsprofTxDeviceProcessor(const std::string &profPath) : DataProcessor(profPath) {} | 27 | MsprofTxDeviceProcessor::MsprofTxDeviceProcessor(const std::string &profPath) : DataProcessor(profPath) {} |
| @@ -26,39 +29,47 @@ MsprofTxDeviceProcessor::MsprofTxDeviceProcessor(const std::string &profPath) : | |||
| 26 | bool MsprofTxDeviceProcessor::ProcessOneDevice(std::vector<MsprofTxDeviceData> &res, const std::string &devPath) | 29 | bool MsprofTxDeviceProcessor::ProcessOneDevice(std::vector<MsprofTxDeviceData> &res, const std::string &devPath) |
| 27 | { | 30 | { |
| 28 | uint16_t deviceId = GetDeviceIdByDevicePath(devPath); | 31 | uint16_t deviceId = GetDeviceIdByDevicePath(devPath); |
| 29 | - if (deviceId == INVALID_DEVICE_ID) { | 32 | + if (deviceId == INVALID_DEVICE_ID) |
| 33 | + { | ||
| 30 | ERROR("the invalid deviceId cannot to be identified."); | 34 | ERROR("the invalid deviceId cannot to be identified."); |
| 31 | return false; | 35 | return false; |
| 32 | } | 36 | } |
| 33 | ProfTimeRecord record; | 37 | ProfTimeRecord record; |
| 34 | - if (!Context::GetInstance().GetProfTimeRecordInfo(record, profPath_, deviceId)) { | 38 | + if (!Context::GetInstance().GetProfTimeRecordInfo(record, profPath_, deviceId)) |
| 39 | + { | ||
| 35 | ERROR("GetProfTimeRecordInfo failed, profPath is %, device id is %.", profPath_, deviceId); | 40 | ERROR("GetProfTimeRecordInfo failed, profPath is %, device id is %.", profPath_, deviceId); |
| 36 | return false; | 41 | return false; |
| 37 | } | 42 | } |
| 38 | Utils::SyscntConversionParams params; | 43 | Utils::SyscntConversionParams params; |
| 39 | - if (!Context::GetInstance().GetSyscntConversionParams(params, deviceId, profPath_)) { | 44 | + if (!Context::GetInstance().GetSyscntConversionParams(params, deviceId, profPath_)) |
| 45 | + { | ||
| 40 | ERROR("GetSyscntConversionParams failed, profPath is %.", profPath_); | 46 | ERROR("GetSyscntConversionParams failed, profPath is %.", profPath_); |
| 41 | return false; | 47 | return false; |
| 42 | } | 48 | } |
| 43 | DBInfo stepTraceDB("step_trace.db", "StepTrace"); | 49 | DBInfo stepTraceDB("step_trace.db", "StepTrace"); |
| 44 | std::string dbPath = Utils::File::PathJoin({devPath, SQLITE, stepTraceDB.dbName}); | 50 | std::string dbPath = Utils::File::PathJoin({devPath, SQLITE, stepTraceDB.dbName}); |
| 45 | - if (!stepTraceDB.ConstructDBRunner(dbPath)) { | 51 | + if (!stepTraceDB.ConstructDBRunner(dbPath)) |
| 52 | + { | ||
| 46 | return false; | 53 | return false; |
| 47 | } | 54 | } |
| 48 | auto status = CheckPathAndTable(dbPath, stepTraceDB, false); | 55 | auto status = CheckPathAndTable(dbPath, stepTraceDB, false); |
| 49 | - if (status != CHECK_SUCCESS) { | 56 | + if (status != CHECK_SUCCESS) |
| 50 | - if (status == CHECK_FAILED) { | 57 | + { |
| 58 | + if (status == CHECK_FAILED) | ||
| 59 | + { | ||
| 51 | return false; | 60 | return false; |
| 52 | } | 61 | } |
| 53 | return true; | 62 | return true; |
| 54 | } | 63 | } |
| 55 | auto oriData = LoadData(stepTraceDB, dbPath); | 64 | auto oriData = LoadData(stepTraceDB, dbPath); |
| 56 | - if (oriData.empty()) { | 65 | + if (oriData.empty()) |
| 66 | + { | ||
| 57 | WARN("StepTrace for msprofTx original data is empty. DBPath is %", dbPath); | 67 | WARN("StepTrace for msprofTx original data is empty. DBPath is %", dbPath); |
| 58 | return true; | 68 | return true; |
| 59 | } | 69 | } |
| 60 | auto formatData = FormatData(oriData, record, deviceId, params); | 70 | auto formatData = FormatData(oriData, record, deviceId, params); |
| 61 | - if (formatData.empty()) { | 71 | + if (formatData.empty()) |
| 72 | + { | ||
| 62 | ERROR("StepTrace for msprofTx data format failed, DBPath is %", dbPath); | 73 | ERROR("StepTrace for msprofTx data format failed, DBPath is %", dbPath); |
| 63 | return false; | 74 | return false; |
| 64 | } | 75 | } |
| @@ -72,10 +83,12 @@ bool MsprofTxDeviceProcessor::Process(DataInventory &dataInventory) | |||
| 72 | bool flag = true; | 83 | bool flag = true; |
| 73 | auto deviceList = Utils::File::GetFilesWithPrefix(profPath_, DEVICE_PREFIX); | 84 | auto deviceList = Utils::File::GetFilesWithPrefix(profPath_, DEVICE_PREFIX); |
| 74 | std::vector<MsprofTxDeviceData> res; | 85 | std::vector<MsprofTxDeviceData> res; |
| 75 | - for (const auto& devicePath : deviceList) { | 86 | + for (const auto &devicePath : deviceList) |
| 87 | + { | ||
| 76 | flag = ProcessOneDevice(res, devicePath) && flag; | 88 | flag = ProcessOneDevice(res, devicePath) && flag; |
| 77 | } | 89 | } |
| 78 | - if (!SaveToDataInventory<MsprofTxDeviceData>(std::move(res), dataInventory, PROCESSOR_NAME_TASK)) { | 90 | + if (!SaveToDataInventory<MsprofTxDeviceData>(std::move(res), dataInventory, PROCESSOR_NAME_TASK)) |
| 91 | + { | ||
| 79 | ERROR("Save data failed, %.", PROCESSOR_NAME_TASK); | 92 | ERROR("Save data failed, %.", PROCESSOR_NAME_TASK); |
| 80 | flag = false; | 93 | flag = false; |
| 81 | } | 94 | } |
| @@ -85,49 +98,168 @@ bool MsprofTxDeviceProcessor::Process(DataInventory &dataInventory) | |||
| 85 | OriMsprofTxDeviceData MsprofTxDeviceProcessor::LoadData(const DBInfo &stepTraceDB, const std::string &dbPath) | 98 | OriMsprofTxDeviceData MsprofTxDeviceProcessor::LoadData(const DBInfo &stepTraceDB, const std::string &dbPath) |
| 86 | { | 99 | { |
| 87 | OriMsprofTxDeviceData oriData; | 100 | OriMsprofTxDeviceData oriData; |
| 88 | - if (stepTraceDB.dbRunner == nullptr) { | 101 | + if (stepTraceDB.dbRunner == nullptr) |
| 102 | + { | ||
| 89 | ERROR("Create % connection failed.", dbPath); | 103 | ERROR("Create % connection failed.", dbPath); |
| 90 | return oriData; | 104 | return oriData; |
| 91 | } | 105 | } |
| 92 | - std::string sql{"SELECT model_id, index_id, stream_id, task_id, timestamp FROM " + stepTraceDB.tableName + | 106 | + std::string sql{"SELECT model_id, index_id, stream_id, task_id, timestamp, tag_id FROM " + stepTraceDB.tableName + |
| 93 | - " WHERE tag_id = 11"}; | 107 | + " WHERE tag_id = 11 or tag_id = 12"}; // 11: mark data; 12: range data |
| 94 | - if (!stepTraceDB.dbRunner->QueryData(sql, oriData)) { | 108 | + if (!stepTraceDB.dbRunner->QueryData(sql, oriData)) |
| 109 | + { | ||
| 95 | ERROR("Failed to obtain data from the % table.", stepTraceDB.tableName); | 110 | ERROR("Failed to obtain data from the % table.", stepTraceDB.tableName); |
| 96 | } | 111 | } |
| 97 | return oriData; | 112 | return oriData; |
| 98 | } | 113 | } |
| 99 | 114 | ||
| 100 | -std::vector<MsprofTxDeviceData> MsprofTxDeviceProcessor::FormatData( | 115 | +std::vector<MsprofTxDeviceData> MsprofTxDeviceProcessor::FormatData(OriMsprofTxDeviceData &oriData, |
| 101 | - OriMsprofTxDeviceData &oriData, const ProfTimeRecord &record, const uint16_t deviceId, | 116 | + const ProfTimeRecord &record, |
| 102 | - const SyscntConversionParams ¶ms) | 117 | + const uint16_t deviceId, |
| 118 | + const SyscntConversionParams ¶ms) | ||
| 103 | { | 119 | { |
| 104 | std::vector<MsprofTxDeviceData> processedData; | 120 | std::vector<MsprofTxDeviceData> processedData; |
| 105 | - if (!Utils::Reserve(processedData, oriData.size())) { | 121 | + if (!Utils::Reserve(processedData, oriData.size())) |
| 122 | + { | ||
| 106 | ERROR("Reserve for AscendTask data failed."); | 123 | ERROR("Reserve for AscendTask data failed."); |
| 107 | return processedData; | 124 | return processedData; |
| 108 | } | 125 | } |
| 126 | + auto cannVersion = Context::GetInstance().GetCannVersion(deviceId, profPath_); | ||
| 127 | + bool isLegacy = cannVersion.empty() || (cannVersion[0] < 9) || (cannVersion[0] == 9 && cannVersion[1] < 1); | ||
| 128 | + if (isLegacy) | ||
| 129 | + { | ||
| 130 | + FormatDataByLegacyRule(processedData, oriData, record, deviceId, params); | ||
| 131 | + } | ||
| 132 | + else | ||
| 133 | + { | ||
| 134 | + OriMsprofTxDeviceData oriMarkData; | ||
| 135 | + OriMsprofTxDeviceData oriRangeData; | ||
| 136 | + for (const auto &data : oriData) | ||
| 137 | + { | ||
| 138 | + uint32_t tagId = std::get<5>(data); | ||
| 139 | + if (tagId == 11) | ||
| 140 | + { | ||
| 141 | + oriMarkData.push_back(data); | ||
| 142 | + } | ||
| 143 | + else if (tagId == 12) | ||
| 144 | + { | ||
| 145 | + oriRangeData.push_back(data); | ||
| 146 | + } | ||
| 147 | + else | ||
| 148 | + { | ||
| 149 | + ERROR("Unexpected tag_id % in msprofTx device data, profPath is %.", tagId, profPath_); | ||
| 150 | + } | ||
| 151 | + } | ||
| 152 | + FormatDataByMarkData(processedData, oriMarkData, record, deviceId, params); | ||
| 153 | + FormatDataByRangeData(processedData, oriRangeData, record, deviceId, params); | ||
| 154 | + } | ||
| 155 | + | ||
| 156 | + return processedData; | ||
| 157 | +} | ||
| 158 | + | ||
| 159 | +void MsprofTxDeviceProcessor::FormatDataByLegacyRule(std::vector<MsprofTxDeviceData> &processedData, | ||
| 160 | + OriMsprofTxDeviceData &oriData, const ProfTimeRecord &record, | ||
| 161 | + const uint16_t deviceId, const SyscntConversionParams ¶ms) | ||
| 162 | +{ | ||
| 109 | MsprofTxDeviceData data; | 163 | MsprofTxDeviceData data; |
| 110 | uint64_t start; | 164 | uint64_t start; |
| 111 | data.deviceId = deviceId; | 165 | data.deviceId = deviceId; |
| 112 | - std::sort(oriData.begin(), oriData.end(), [](TxDeviceData &lData, TxDeviceData rData) { | 166 | + std::sort(oriData.begin(), oriData.end(), |
| 113 | - if (std::get<1>(lData) != std::get<1>(rData)) { // 按照index_id、timestamp排序，即第1、4位 | 167 | + [](TxDeviceData &lData, TxDeviceData rData) |
| 114 | - return std::get<1>(lData) < std::get<1>(rData); // 第1为为index_id | 168 | + { |
| 115 | - } else { | 169 | + if (std::get<1>(lData) != std::get<1>(rData)) |
| 116 | - return std::get<4>(lData) < std::get<4>(rData); // 第4位为timestamp | 170 | + { // 按照index_id、timestamp排序，即第1、4位 |
| 117 | - } | 171 | + return std::get<1>(lData) < std::get<1>(rData); // 第1为为index_id |
| 118 | - }); | 172 | + } |
| 119 | - for (const auto& row : oriData) { | 173 | + else |
| 120 | - std::tie(data.modelId, data.indexId, data.streamId, data.taskId, start) = row; | 174 | + { |
| 175 | + return std::get<4>(lData) < std::get<4>(rData); // 第4位为timestamp | ||
| 176 | + } | ||
| 177 | + }); | ||
| 178 | + for (const auto &row : oriData) | ||
| 179 | + { | ||
| 180 | + std::tie(data.modelId, data.indexId, data.streamId, data.taskId, start, std::ignore) = row; | ||
| 121 | data.connectionId = data.indexId + START_CONNECTION_ID_MSTX; | 181 | data.connectionId = data.indexId + START_CONNECTION_ID_MSTX; |
| 122 | HPFloat startTimestamp = Utils::GetTimeFromSyscnt(start, params); | 182 | HPFloat startTimestamp = Utils::GetTimeFromSyscnt(start, params); |
| 123 | data.timestamp = GetLocalTime(startTimestamp, record).Uint64(); | 183 | data.timestamp = GetLocalTime(startTimestamp, record).Uint64(); |
| 124 | - if (!processedData.empty() && data.indexId == processedData.back().indexId) { | 184 | + if (!processedData.empty() && data.indexId == processedData.back().indexId) |
| 185 | + { | ||
| 125 | processedData.back().duration = static_cast<double>(data.timestamp - processedData.back().timestamp); | 186 | processedData.back().duration = static_cast<double>(data.timestamp - processedData.back().timestamp); |
| 126 | - } else { | 187 | + } |
| 188 | + else | ||
| 189 | + { | ||
| 127 | processedData.push_back(data); | 190 | processedData.push_back(data); |
| 128 | } | 191 | } |
| 129 | } | 192 | } |
| 130 | - return processedData; | ||
| 131 | } | 193 | } |
| 194 | + | ||
| 195 | +void MsprofTxDeviceProcessor::FormatDataByMarkData(std::vector<MsprofTxDeviceData> &processedData, | ||
 | |||
| 196 | + OriMsprofTxDeviceData &oriData, const ProfTimeRecord &record, | ||
| 197 | + const uint16_t deviceId, const SyscntConversionParams ¶ms) | ||
| 198 | +{ | ||
| 199 | + MsprofTxDeviceData data; | ||
| 200 | + data.deviceId = deviceId; | ||
| 201 | + uint64_t start = 0; | ||
| 202 | + for (const auto &row : oriData) | ||
| 203 | + { | ||
| 204 | + std::tie(data.modelId, data.indexId, data.streamId, data.taskId, start, std::ignore) = row; | ||
| 205 | + data.connectionId = data.indexId + START_CONNECTION_ID_MSTX; | ||
| 206 | + HPFloat startTimestamp = Utils::GetTimeFromSyscnt(start, params); | ||
| 207 | + data.timestamp = GetLocalTime(startTimestamp, record).Uint64(); | ||
| 208 | + processedData.push_back(data); | ||
| 209 | + } | ||
| 132 | } | 210 | } |
| 211 | + | ||
| 212 | +void MsprofTxDeviceProcessor::FormatDataByRangeData(std::vector<MsprofTxDeviceData> &processedData, | ||
[review] 问题：排序只按 index_id 和 timestamp 排序，没有同时考虑 model_id、stream_id、task_id 等字段。 影响：不同 task/stream 但 index_id 相同的数据可能被排序到一起，后续按相邻两条配对时可能错误配对。 建议：排序和配对时建议同时纳入 model_id、index_id、stream_id、task_id 等关键字段，确保同一任务的 start/end 才会被配对。   | |||
| 213 | + OriMsprofTxDeviceData &oriData, const ProfTimeRecord &record, | ||
| 214 | + const uint16_t deviceId, const SyscntConversionParams ¶ms) | ||
| 215 | +{ | ||
| 216 | + // sort range data by index_id and timestamp | ||
| 217 | + std::sort(oriData.begin(), oriData.end(), | ||
| 218 | + [](const TxDeviceData &l, const TxDeviceData &r) | ||
| 219 | + { | ||
| 220 | + if (std::get<1>(l) != std::get<1>(r)) | ||
| 221 | + { | ||
| 222 | + return std::get<1>(l) < std::get<1>(r); // 1: index_id | ||
| 223 | + } | ||
| 224 | + return std::get<4>(l) < std::get<4>(r); // 4: timestamp | ||
| 225 | + }); | ||
| 226 | + | ||
| 227 | + // Traverse the data and pair them two by two (grouping adjacent entries with the same indexId) | ||
| 228 | + size_t idx = 0; | ||
| 229 | + const size_t total = oriData.size(); | ||
| 230 | + while (idx < total) | ||
| 231 | + { | ||
| 232 | + const auto &rowFirst = oriData[idx]; | ||
| 233 | + uint32_t currIndexId = std::get<1>(rowFirst); | ||
| 234 | + if (idx + 1 >= total || std::get<1>(oriData[idx + 1]) != currIndexId) | ||
| 235 | + { | ||
| 236 | + idx++; | ||
| 237 | + WARN("Unpaired range data with index_id % for device id %.", currIndexId, deviceId); | ||
[review] 问题：WARN 日志格式字符串为 "Unpaired range data with index_id % for device id %."，占位符不完整。 影响：日志可能无法正确打印 currIndexId 和 deviceId，甚至在部分日志宏实现下出现格式化异常。 建议：根据项目日志宏格式修正占位符，例如 printf 风格应写为： WARN("Unpaired range data with index_id %u for device id %u.", currIndexId, deviceId); 如果项目使用 fmt 风格，则写为： WARN("Unpaired range data with index_id {} for device id {}.", currIndexId, deviceId); | |||
| 238 | + continue; | ||
| 239 | + } | ||
| 240 | + const auto &rowSecond = oriData[idx + 1]; | ||
| 241 | + MsprofTxDeviceData data; | ||
| 242 | + uint64_t startFirst = 0; | ||
| 243 | + std::tie(data.modelId, data.indexId, data.streamId, data.taskId, startFirst, std::ignore) = rowFirst; | ||
| 244 | + data.deviceId = deviceId; | ||
| 245 | + data.connectionId = data.indexId + START_CONNECTION_ID_MSTX; | ||
| 246 | + | ||
| 247 | + // range start time | ||
| 248 | + HPFloat tsFirst = Utils::GetTimeFromSyscnt(startFirst, params); | ||
| 249 | + uint64_t timeFirst = GetLocalTime(tsFirst, record).Uint64(); | ||
| 250 | + | ||
| 251 | + // range end time | ||
| 252 | + uint64_t startSecond = std::get<4>(rowSecond); | ||
| 253 | + HPFloat tsSecond = Utils::GetTimeFromSyscnt(startSecond, params); | ||
| 254 | + uint64_t timeSecond = GetLocalTime(tsSecond, record).Uint64(); | ||
[review] 问题：duration 使用 timeSecond - timeFirst 计算，但没有校验 timeSecond >= timeFirst。 影响：如果时间换算异常、数据乱序或 end 早于 start，uint64_t 相减会发生下溢，得到一个非常大的 duration，影响后续展示和分析。 建议：计算前增加判断： if (timeSecond < timeFirst) { WARN(...); idx += 2; continue; } 或者将该条数据标记为异常。 | |||
| 255 | + | ||
| 256 | + data.timestamp = timeFirst; | ||
| 257 | + data.duration = static_cast<double>(timeSecond - timeFirst); | ||
| 258 | + | ||
| 259 | + processedData.push_back(data); | ||
| 260 | + | ||
| 261 | + idx += 2; // move to the next pair | ||
| 262 | + } | ||
| 133 | } | 263 | } |
| 264 | +} // namespace Domain | ||
| 265 | +} // namespace Analysis | ||
| @@ -20,26 +20,37 @@ | |||
| 20 | 20 | ||
| 21 | 21 | ||
| 22 | 22 | ||
| 23 | -namespace Analysis { | 23 | +namespace Analysis |
| 24 | -namespace Domain { | 24 | +{ |
| 25 | -// model_id, index_id, stream_id, task_id, timestamp | 25 | +namespace Domain |
| 26 | -using TxDeviceData = std::tuple<uint32_t, uint32_t, uint32_t, uint32_t, uint64_t>; | 26 | +{ |
| 27 | +// model_id, index_id, stream_id, task_id, timestamp, tag_id | ||
| 28 | +using TxDeviceData = std::tuple<uint32_t, uint32_t, uint32_t, uint32_t, uint64_t, uint32_t>; | ||
| 27 | using OriMsprofTxDeviceData = std::vector<TxDeviceData>; | 29 | using OriMsprofTxDeviceData = std::vector<TxDeviceData>; |
| 28 | 30 | ||
| 29 | -class MsprofTxDeviceProcessor : public DataProcessor { | 31 | +class MsprofTxDeviceProcessor : public DataProcessor |
| 30 | -public: | 32 | +{ |
| 33 | + public: | ||
| 31 | MsprofTxDeviceProcessor() = default; | 34 | MsprofTxDeviceProcessor() = default; |
| 32 | - explicit MsprofTxDeviceProcessor(const std::string &profPath); | 35 | + explicit MsprofTxDeviceProcessor(const std::string &profPath); |
| 33 | -private: | 36 | + |
| 34 | - bool Process(DataInventory& dataInventory) override; | 37 | + private: |
| 38 | + bool Process(DataInventory &dataInventory) override; | ||
| 35 | OriMsprofTxDeviceData LoadData(const DBInfo &stepTraceDB, const std::string &dbPath); | 39 | OriMsprofTxDeviceData LoadData(const DBInfo &stepTraceDB, const std::string &dbPath); |
| 36 | bool ProcessOneDevice(std::vector<MsprofTxDeviceData> &res, const std::string &devPath); | 40 | bool ProcessOneDevice(std::vector<MsprofTxDeviceData> &res, const std::string &devPath); |
| 37 | - std::vector<MsprofTxDeviceData> FormatData(OriMsprofTxDeviceData &oriData, | 41 | + std::vector<MsprofTxDeviceData> FormatData(OriMsprofTxDeviceData &oriData, const Utils::ProfTimeRecord &record, |
| 38 | - const Utils::ProfTimeRecord &record, | 42 | + const uint16_t deviceId, const Utils::SyscntConversionParams ¶ms); |
| 39 | - const uint16_t deviceId, | 43 | + void FormatDataByLegacyRule(std::vector<MsprofTxDeviceData> &processedData, OriMsprofTxDeviceData &oriData, |
| 40 | - const Utils::SyscntConversionParams ¶ms); | 44 | + const Utils::ProfTimeRecord &record, const uint16_t deviceId, |
| 45 | + const Utils::SyscntConversionParams ¶ms); | ||
| 46 | + void FormatDataByMarkData(std::vector<MsprofTxDeviceData> &processedData, OriMsprofTxDeviceData &oriData, | ||
| 47 | + const Utils::ProfTimeRecord &record, const uint16_t deviceId, | ||
| 48 | + const Utils::SyscntConversionParams ¶ms); | ||
| 49 | + void FormatDataByRangeData(std::vector<MsprofTxDeviceData> &processedData, OriMsprofTxDeviceData &oriData, | ||
| 50 | + const Utils::ProfTimeRecord &record, const uint16_t deviceId, | ||
| 51 | + const Utils::SyscntConversionParams ¶ms); | ||
| 41 | }; | 52 | }; |
| 42 | -} | 53 | +} // namespace Domain |
| 43 | -} | 54 | +} // namespace Analysis |
| 44 | 55 | ||
| 45 | -#endif // ANALYSIS_DOMAIN_MSPROFTX_PROCESSOR_H | 56 | +#endif // ANALYSIS_DOMAIN_MSPROFTX_PROCESSOR_H |
| @@ -16,20 +16,25 @@ | |||
| 16 | 16 | ||
| 17 | 17 | ||
| 18 | 18 | ||
| 19 | + | ||
| 19 | 20 | ||
| 20 | 21 | ||
| 22 | + | ||
| 21 | 23 | ||
| 22 | 24 | ||
| 23 | - | ||
| 24 | 25 | ||
| 25 | -namespace Analysis { | 26 | +namespace Analysis |
| 26 | -namespace Domain { | 27 | +{ |
| 27 | -namespace Environment { | 28 | +namespace Domain |
| 29 | +{ | ||
| 30 | +namespace Environment | ||
| 31 | +{ | ||
| 28 | using namespace Analysis; | 32 | using namespace Analysis; |
| 29 | using namespace Analysis::Utils; | 33 | using namespace Analysis::Utils; |
| 30 | using namespace Viewer::Database; | 34 | using namespace Viewer::Database; |
| 31 | 35 | ||
| 32 | -namespace { | 36 | +namespace |
| 37 | +{ | ||
| 33 | const uint32_t ALL_EXPORT_VERSION = 0x072211; // 2023年10月30号之后支持全导的驱动版本号 0x072211 = 467473 | 38 | const uint32_t ALL_EXPORT_VERSION = 0x072211; // 2023年10月30号之后支持全导的驱动版本号 0x072211 = 467473 |
| 34 | const uint64_t DEFAULT_DURATION_TIME_US = DEFAULT_DURATION_TIME_NS / MILLI_SECOND; | 39 | const uint64_t DEFAULT_DURATION_TIME_US = DEFAULT_DURATION_TIME_NS / MILLI_SECOND; |
| 35 | const std::string DEFAULT_HOST_UID = "0"; | 40 | const std::string DEFAULT_HOST_UID = "0"; |
| @@ -42,30 +47,42 @@ const std::string HOST_START_LOG = "host_start.log"; | |||
| 42 | const std::string DEVICE_START_LOG = "dev_start.log"; | 47 | const std::string DEVICE_START_LOG = "dev_start.log"; |
| 43 | // CHECK_VALUES存放一定存在的字段,字段不存在Load失败。 | 48 | // CHECK_VALUES存放一定存在的字段,字段不存在Load失败。 |
| 44 | // 已经校验过的字段,可直接使用.at();未被校验过的字段则使用.value(),来设置默认值。确保读取正常 | 49 | // 已经校验过的字段,可直接使用.at();未被校验过的字段则使用.value(),来设置默认值。确保读取正常 |
| 45 | -const std::set<std::string> CHECK_VALUES = { | 50 | +const std::set<std::string> CHECK_VALUES = {"platform_version", |
| 46 | - "platform_version", "startCollectionTimeBegin", | 51 | + "startCollectionTimeBegin", |
| 47 | - "startClockMonotonicRaw", "pid", "CPU", "DeviceInfo", "llc_profiling", "ai_core_profiling_mode", "hostname", | 52 | + "startClockMonotonicRaw", |
| 48 | - "memoryTotal", "netCard" | 53 | + "pid", |
| 49 | -}; | 54 | + "CPU", |
| 50 | -} | 55 | + "DeviceInfo", |
| 56 | + "llc_profiling", | ||
| 57 | + "ai_core_profiling_mode", | ||
| 58 | + "hostname", | ||
| 59 | + "memoryTotal", | ||
| 60 | + "netCard"}; | ||
| 61 | +} // namespace | ||
| 51 | 62 | ||
| 52 | bool Context::Load(const std::set<std::string> &profPaths) | 63 | bool Context::Load(const std::set<std::string> &profPaths) |
| 53 | { | 64 | { |
| 54 | - for (const auto &profPath : profPaths) { | 65 | + for (const auto &profPath : profPaths) |
| 66 | + { | ||
| 55 | std::vector<std::string> deviceDirs = File::GetOriginData(profPath, {HOST, DEVICE_PREFIX}, {}); | 67 | std::vector<std::string> deviceDirs = File::GetOriginData(profPath, {HOST, DEVICE_PREFIX}, {}); |
| 56 | - for (const auto &deviceDir: deviceDirs) { | 68 | + for (const auto &deviceDir : deviceDirs) |
| 69 | + { | ||
| 57 | uint16_t deviceId = Utils::GetDeviceIdByDevicePath(deviceDir); | 70 | uint16_t deviceId = Utils::GetDeviceIdByDevicePath(deviceDir); |
| 58 | - if (deviceId == INVALID_DEVICE_ID) { | 71 | + if (deviceId == INVALID_DEVICE_ID) |
| 72 | + { | ||
| 59 | ERROR("The prof path's deviceId is invalid."); | 73 | ERROR("The prof path's deviceId is invalid."); |
| 60 | return false; | 74 | return false; |
| 61 | } | 75 | } |
| 62 | - if (!LoadJsonData(profPath, deviceDir, deviceId)) { | 76 | + if (!LoadJsonData(profPath, deviceDir, deviceId)) |
| 77 | + { | ||
| 63 | return false; | 78 | return false; |
| 64 | } | 79 | } |
| 65 | - if (!LoadLogData(profPath, deviceDir, deviceId)) { | 80 | + if (!LoadLogData(profPath, deviceDir, deviceId)) |
| 81 | + { | ||
| 66 | return false; | 82 | return false; |
| 67 | } | 83 | } |
| 68 | - if (!CheckInfoValueIsValid(profPath, deviceId)) { | 84 | + if (!CheckInfoValueIsValid(profPath, deviceId)) |
| 85 | + { | ||
| 69 | return false; | 86 | return false; |
| 70 | } | 87 | } |
| 71 | } | 88 | } |
| @@ -75,28 +92,33 @@ bool Context::Load(const std::set<std::string> &profPaths) | |||
| 75 | 92 | ||
| 76 | bool Context::LoadJsonData(const std::string &profPath, const std::string &deviceDir, uint16_t deviceId) | 93 | bool Context::LoadJsonData(const std::string &profPath, const std::string &deviceDir, uint16_t deviceId) |
| 77 | { | 94 | { |
| 78 | - for (const auto &fileName: {INFO_JSON, SAMPLE_JSON, START_INFO, END_INFO}) { | 95 | + for (const auto &fileName : {INFO_JSON, SAMPLE_JSON, START_INFO, END_INFO}) |
| 96 | + { | ||
| 79 | std::vector<std::string> files = File::GetOriginData(deviceDir, {fileName}, {"done"}); | 97 | std::vector<std::string> files = File::GetOriginData(deviceDir, {fileName}, {"done"}); |
| 80 | - if (files.size() != 1) { | 98 | + if (files.size() != 1) |
| 99 | + { | ||
| 81 | ERROR("The number of % in % is invalid, file num is: %.", fileName, deviceDir, files.size()); | 100 | ERROR("The number of % in % is invalid, file num is: %.", fileName, deviceDir, files.size()); |
| 82 | - if (fileName == END_INFO) { | 101 | + if (fileName == END_INFO) |
| 102 | + { | ||
| 83 | continue; | 103 | continue; |
| 84 | } | 104 | } |
| 85 | return false; | 105 | return false; |
| 86 | } | 106 | } |
| 87 | FileReader fd(files.back()); | 107 | FileReader fd(files.back()); |
| 88 | nlohmann::json content; | 108 | nlohmann::json content; |
| 89 | - if (fd.ReadJson(content) != ANALYSIS_OK) { | 109 | + if (fd.ReadJson(content) != ANALYSIS_OK) |
| 110 | + { | ||
| 90 | ERROR("Load json context failed: '%'.", files.back()); | 111 | ERROR("Load json context failed: '%'.", files.back()); |
| 91 | return false; | 112 | return false; |
| 92 | } | 113 | } |
| 93 | 114 | ||
| 94 | - if (fileName == START_INFO && content.contains("clockMonotonicRaw") | 115 | + if (fileName == START_INFO && content.contains("clockMonotonicRaw") && content.contains("collectionTimeBegin")) |
| 95 | - && content.contains("collectionTimeBegin")) { | 116 | + { |
| 96 | content["startClockMonotonicRaw"] = content["clockMonotonicRaw"]; | 117 | content["startClockMonotonicRaw"] = content["clockMonotonicRaw"]; |
| 97 | content["startCollectionTimeBegin"] = content["collectionTimeBegin"]; | 118 | content["startCollectionTimeBegin"] = content["collectionTimeBegin"]; |
| 98 | - } else if (fileName == END_INFO && content.contains("clockMonotonicRaw") | 119 | + } |
| 99 | - && content.contains("collectionTimeEnd")) { | 120 | + else if (fileName == END_INFO && content.contains("clockMonotonicRaw") && content.contains("collectionTimeEnd")) |
| 121 | + { | ||
| 100 | content["endClockMonotonicRaw"] = content["clockMonotonicRaw"]; | 122 | content["endClockMonotonicRaw"] = content["clockMonotonicRaw"]; |
| 101 | content["endCollectionTimeEnd"] = content["collectionTimeEnd"]; | 123 | content["endCollectionTimeEnd"] = content["collectionTimeEnd"]; |
| 102 | } | 124 | } |
| @@ -108,47 +130,60 @@ bool Context::LoadJsonData(const std::string &profPath, const std::string &devic | |||
| 108 | 130 | ||
| 109 | bool Context::LoadLogData(const std::string &profPath, const std::string &deviceDir, uint16_t deviceId) | 131 | bool Context::LoadLogData(const std::string &profPath, const std::string &deviceDir, uint16_t deviceId) |
| 110 | { | 132 | { |
| 111 | - const int expectTokenSize = 2; // 2代表：前后的key和value | 133 | + const int expectTokenSize = 2; // 2代表：前后的key和value |
| 112 | // host就用host_start_log，device就用device_start_log。 | 134 | // host就用host_start_log，device就用device_start_log。 |
| 113 | std::vector<std::string> fileNameList{HOST_START_LOG}; | 135 | std::vector<std::string> fileNameList{HOST_START_LOG}; |
| 114 | - if (deviceId != HOST_ID) { | 136 | + if (deviceId != HOST_ID) |
| 137 | + { | ||
| 115 | fileNameList.emplace_back(DEVICE_START_LOG); | 138 | fileNameList.emplace_back(DEVICE_START_LOG); |
| 116 | } | 139 | } |
| 117 | - for (const std::string& fileName : fileNameList) { | 140 | + for (const std::string &fileName : fileNameList) |
| 141 | + { | ||
| 118 | std::vector<std::string> files = File::GetOriginData(deviceDir, {fileName}, {"done"}); | 142 | std::vector<std::string> files = File::GetOriginData(deviceDir, {fileName}, {"done"}); |
| 119 | // host、device底下只有1份log | 143 | // host、device底下只有1份log |
| 120 | - if (files.size() != 1) { | 144 | + if (files.size() != 1) |
| 145 | + { | ||
| 121 | ERROR("The number of % in % is invalid, file num is: %.", fileName, deviceDir, files.size()); | 146 | ERROR("The number of % in % is invalid, file num is: %.", fileName, deviceDir, files.size()); |
| 122 | return false; | 147 | return false; |
| 123 | } | 148 | } |
| 124 | FileReader fd(files.back()); | 149 | FileReader fd(files.back()); |
| 125 | std::vector<std::string> text; | 150 | std::vector<std::string> text; |
| 126 | - if (fd.ReadText(text) != ANALYSIS_OK) { | 151 | + if (fd.ReadText(text) != ANALYSIS_OK) |
| 152 | + { | ||
| 127 | ERROR("Load log text failed: '%'.", files.back()); | 153 | ERROR("Load log text failed: '%'.", files.back()); |
| 128 | return false; | 154 | return false; |
| 129 | } | 155 | } |
| 130 | - for (const auto &line : text) { | 156 | + for (const auto &line : text) |
| 157 | + { | ||
| 131 | auto tokens = Utils::Split(line, ":"); | 158 | auto tokens = Utils::Split(line, ":"); |
| 132 | - if (tokens.size() != expectTokenSize) { | 159 | + if (tokens.size() != expectTokenSize) |
| 160 | + { | ||
| 133 | continue; | 161 | continue; |
| 134 | } | 162 | } |
| 135 | context_[profPath][deviceId][tokens[0]] = tokens[1]; | 163 | context_[profPath][deviceId][tokens[0]] = tokens[1]; |
| 136 | } | 164 | } |
| 137 | 165 | ||
| 138 | - if (!context_[profPath][deviceId].contains("clock_monotonic_raw") | 166 | + if (!context_[profPath][deviceId].contains("clock_monotonic_raw") || |
| 139 | - || !context_[profPath][deviceId].contains("cntvct")) { | 167 | + !context_[profPath][deviceId].contains("cntvct")) |
| 168 | + { | ||
| 140 | return false; | 169 | return false; |
| 141 | } | 170 | } |
| 142 | 171 | ||
| 143 | - if (fileName == DEVICE_START_LOG) { | 172 | + if (fileName == DEVICE_START_LOG) |
| 173 | + { | ||
| 144 | context_[profPath][deviceId]["devMonotonic"] = context_[profPath][deviceId]["clock_monotonic_raw"]; | 174 | context_[profPath][deviceId]["devMonotonic"] = context_[profPath][deviceId]["clock_monotonic_raw"]; |
| 145 | context_[profPath][deviceId]["devCntvct"] = context_[profPath][deviceId]["cntvct"]; | 175 | context_[profPath][deviceId]["devCntvct"] = context_[profPath][deviceId]["cntvct"]; |
| 146 | - } else if (fileName == HOST_START_LOG) { | 176 | + } |
| 177 | + else if (fileName == HOST_START_LOG) | ||
| 178 | + { | ||
| 147 | context_[profPath][deviceId]["hostMonotonic"] = context_[profPath][deviceId]["clock_monotonic_raw"]; | 179 | context_[profPath][deviceId]["hostMonotonic"] = context_[profPath][deviceId]["clock_monotonic_raw"]; |
| 148 | context_[profPath][deviceId]["hostCntvct"] = context_[profPath][deviceId]["cntvct"]; | 180 | context_[profPath][deviceId]["hostCntvct"] = context_[profPath][deviceId]["cntvct"]; |
| 149 | - if (context_[profPath][deviceId].contains("cntvct_diff")) { | 181 | + if (context_[profPath][deviceId].contains("cntvct_diff")) |
| 182 | + { | ||
| 150 | context_[profPath][deviceId]["hostCntvctDiff"] = context_[profPath][deviceId]["cntvct_diff"]; | 183 | context_[profPath][deviceId]["hostCntvctDiff"] = context_[profPath][deviceId]["cntvct_diff"]; |
| 151 | - } else { | 184 | + } |
| 185 | + else | ||
| 186 | + { | ||
| 152 | context_[profPath][deviceId]["hostCntvctDiff"] = "0"; | 187 | context_[profPath][deviceId]["hostCntvctDiff"] = "0"; |
| 153 | } | 188 | } |
| 154 | } | 189 | } |
| @@ -159,39 +194,61 @@ bool Context::LoadLogData(const std::string &profPath, const std::string &device | |||
| 159 | bool Context::CheckInfoValueIsValid(const std::string &profPath, uint16_t deviceId) | 194 | bool Context::CheckInfoValueIsValid(const std::string &profPath, uint16_t deviceId) |
| 160 | { | 195 | { |
| 161 | const auto &info = GetInfoByDeviceId(deviceId, profPath); | 196 | const auto &info = GetInfoByDeviceId(deviceId, profPath); |
| 162 | - for (auto &valueName : CHECK_VALUES) { | 197 | + for (auto &valueName : CHECK_VALUES) |
| 163 | - if (!info.contains(valueName)) { | 198 | + { |
| 164 | - ERROR("The key called %, not in the context info, " | 199 | + if (!info.contains(valueName)) |
| 165 | - "the ProfPath is %, DeviceId is %.", valueName, profPath, deviceId); | 200 | + { |
| 201 | + ERROR( | ||
| 202 | + "The key called %, not in the context info, " | ||
| 203 | + "the ProfPath is %, DeviceId is %.", | ||
| 204 | + valueName, profPath, deviceId); | ||
| 166 | return false; | 205 | return false; |
| 167 | } | 206 | } |
| 168 | } | 207 | } |
| 169 | - if (deviceId == HOST_ID) { | 208 | + if (deviceId == HOST_ID) |
| 170 | - if (!info.at("CPU").is_array()) { | 209 | + { |
| 171 | - ERROR("CPU's value is invalid, " | 210 | + if (!info.at("CPU").is_array()) |
| 172 | - "the ProfPath is %, DeviceId is %.", profPath, deviceId); | 211 | + { |
| 212 | + ERROR( | ||
| 213 | + "CPU's value is invalid, " | ||
| 214 | + "the ProfPath is %, DeviceId is %.", | ||
| 215 | + profPath, deviceId); | ||
| 173 | return false; | 216 | return false; |
| 174 | } | 217 | } |
| 175 | - if (!info.at("CPU").back().contains("Frequency")) { | 218 | + if (!info.at("CPU").back().contains("Frequency")) |
| 176 | - ERROR("There is no Frequency in context info, " | 219 | + { |
| 177 | - "the ProfPath is %, DeviceId is %.", profPath, deviceId); | 220 | + ERROR( |
| 221 | + "There is no Frequency in context info, " | ||
| 222 | + "the ProfPath is %, DeviceId is %.", | ||
| 223 | + profPath, deviceId); | ||
| 178 | return false; | 224 | return false; |
| 179 | } | 225 | } |
| 180 | - } else { | 226 | + } |
| 181 | - if (!info.at("DeviceInfo").is_array() || (info.at("DeviceInfo").size() != 1)) { | 227 | + else |
| 182 | - ERROR("DeviceInfo's value is invalid, " | 228 | + { |
| 183 | - "the ProfPath is %, DeviceId is %.", profPath, deviceId); | 229 | + if (!info.at("DeviceInfo").is_array() || (info.at("DeviceInfo").size() != 1)) |
| 230 | + { | ||
| 231 | + ERROR( | ||
| 232 | + "DeviceInfo's value is invalid, " | ||
| 233 | + "the ProfPath is %, DeviceId is %.", | ||
| 234 | + profPath, deviceId); | ||
| 184 | return false; | 235 | return false; |
| 185 | } | 236 | } |
| 186 | auto freqArr = info.at("DeviceInfo").back(); | 237 | auto freqArr = info.at("DeviceInfo").back(); |
| 187 | - if (!freqArr.contains("hwts_frequency") || freqArr.at("hwts_frequency").empty()) { | 238 | + if (!freqArr.contains("hwts_frequency") || freqArr.at("hwts_frequency").empty()) |
| 188 | - ERROR("There is no hwts_frequency in context info, " | 239 | + { |
| 189 | - "the ProfPath is %, DeviceId is %.", profPath, deviceId); | 240 | + ERROR( |
| 241 | + "There is no hwts_frequency in context info, " | ||
| 242 | + "the ProfPath is %, DeviceId is %.", | ||
| 243 | + profPath, deviceId); | ||
| 190 | return false; | 244 | return false; |
| 191 | } | 245 | } |
| 192 | - if (!freqArr.contains("aic_frequency") || freqArr.at("aic_frequency").empty()) { | 246 | + if (!freqArr.contains("aic_frequency") || freqArr.at("aic_frequency").empty()) |
| 193 | - ERROR("There is no aic_frequency in context info, " | 247 | + { |
| 194 | - "the ProfPath is %, DeviceId is %.", profPath, deviceId); | 248 | + ERROR( |
| 249 | + "There is no aic_frequency in context info, " | ||
| 250 | + "the ProfPath is %, DeviceId is %.", | ||
| 251 | + profPath, deviceId); | ||
| 195 | return false; | 252 | return false; |
| 196 | } | 253 | } |
| 197 | } | 254 | } |
| @@ -201,24 +258,27 @@ bool Context::CheckInfoValueIsValid(const std::string &profPath, uint16_t device | |||
| 201 | bool Context::IsAllExport() | 258 | bool Context::IsAllExport() |
| 202 | { | 259 | { |
| 203 | const auto &info = GetInfoByDeviceId(); | 260 | const auto &info = GetInfoByDeviceId(); |
| 204 | - if (info.empty()) { | 261 | + if (info.empty()) |
| 262 | + { | ||
| 205 | ERROR("IsAllExport device info is empty."); | 263 | ERROR("IsAllExport device info is empty."); |
| 206 | return false; | 264 | return false; |
| 207 | } | 265 | } |
| 208 | // 低版本的驱动中不存在drvVersion字段，该字段使用时需要默认值 | 266 | // 低版本的驱动中不存在drvVersion字段，该字段使用时需要默认值 |
| 209 | auto drvVersion = info.value("drvVersion", 0u); | 267 | auto drvVersion = info.value("drvVersion", 0u); |
| 210 | - if (drvVersion < ALL_EXPORT_VERSION) { | 268 | + if (drvVersion < ALL_EXPORT_VERSION) |
| 269 | + { | ||
| 211 | WARN("DrvVersion not support all export, ALL_EXPORT_VERSION is %", ALL_EXPORT_VERSION); | 270 | WARN("DrvVersion not support all export, ALL_EXPORT_VERSION is %", ALL_EXPORT_VERSION); |
| 212 | return false; | 271 | return false; |
| 213 | } | 272 | } |
| 214 | uint16_t chip; | 273 | uint16_t chip; |
| 215 | - if (StrToU16(chip, info.at("platform_version")) != ANALYSIS_OK) { | 274 | + if (StrToU16(chip, info.at("platform_version")) != ANALYSIS_OK) |
| 275 | + { | ||
| 216 | ERROR("str to uint16_t failed."); | 276 | ERROR("str to uint16_t failed."); |
| 217 | return false; | 277 | return false; |
| 218 | } | 278 | } |
| 219 | - if (chip == static_cast<uint16_t>(Chip::CHIP_V1_1_0) || | 279 | + if (chip == static_cast<uint16_t>(Chip::CHIP_V1_1_0) || chip == static_cast<uint16_t>(Chip::CHIP_V3_1_0) || |
| 220 | - chip == static_cast<uint16_t>(Chip::CHIP_V3_1_0) || | 280 | + chip == static_cast<uint16_t>(Chip::CHIP_V1_1_3)) |
| 221 | - chip == static_cast<uint16_t>(Chip::CHIP_V1_1_3)) { | 281 | + { |
| 222 | WARN("Platform_version not support all export."); | 282 | WARN("Platform_version not support all export."); |
| 223 | return false; | 283 | return false; |
| 224 | } | 284 | } |
| @@ -229,17 +289,20 @@ nlohmann::json Context::GetInfoByDeviceId(uint16_t deviceId, const std::string & | |||
| 229 | { | 289 | { |
| 230 | nlohmann::json emptyJson; | 290 | nlohmann::json emptyJson; |
| 231 | auto profInfo = profPath.empty() ? context_.begin() : context_.find(profPath); | 291 | auto profInfo = profPath.empty() ? context_.begin() : context_.find(profPath); |
| 232 | - if (profInfo == context_.end()) { | 292 | + if (profInfo == context_.end()) |
| 293 | + { | ||
| 233 | ERROR("Can't find PROF file. input path %.", profPath); | 294 | ERROR("Can't find PROF file. input path %.", profPath); |
| 234 | return emptyJson; | 295 | return emptyJson; |
| 235 | } | 296 | } |
| 236 | auto deviceInfo = profInfo->second; | 297 | auto deviceInfo = profInfo->second; |
| 237 | - if (deviceInfo.begin() == deviceInfo.end()) { | 298 | + if (deviceInfo.begin() == deviceInfo.end()) |
| 299 | + { | ||
| 238 | ERROR("Can't find host or device file. input path %.", profPath); | 300 | ERROR("Can't find host or device file. input path %.", profPath); |
| 239 | return emptyJson; | 301 | return emptyJson; |
| 240 | } | 302 | } |
| 241 | // 若不设置deviceId(使用默认id),则直接返回第一条info | 303 | // 若不设置deviceId(使用默认id),则直接返回第一条info |
| 242 | - if (deviceId == DEFAULT_DEVICE_ID || deviceInfo.find(deviceId) == deviceInfo.end()) { | 304 | + if (deviceId == DEFAULT_DEVICE_ID || deviceInfo.find(deviceId) == deviceInfo.end()) |
| 305 | + { | ||
| 243 | return deviceInfo.begin()->second; | 306 | return deviceInfo.begin()->second; |
| 244 | } | 307 | } |
| 245 | return deviceInfo[deviceId]; | 308 | return deviceInfo[deviceId]; |
| @@ -249,11 +312,13 @@ uint16_t Context::GetPlatformVersion(uint16_t deviceId, const std::string &profP | |||
| 249 | { | 312 | { |
| 250 | const auto &info = GetInfoByDeviceId(deviceId, profPath); | 313 | const auto &info = GetInfoByDeviceId(deviceId, profPath); |
| 251 | uint16_t platformVersion = UINT16_MAX; | 314 | uint16_t platformVersion = UINT16_MAX; |
| 252 | - if (info.empty()) { | 315 | + if (info.empty()) |
| 316 | + { | ||
| 253 | ERROR("GetPlatformVersion device info is empty, input path %, deviceId %", profPath, deviceId); | 317 | ERROR("GetPlatformVersion device info is empty, input path %, deviceId %", profPath, deviceId); |
| 254 | return platformVersion; | 318 | return platformVersion; |
| 255 | } | 319 | } |
| 256 | - if (StrToU16(platformVersion, info.at("platform_version")) != ANALYSIS_OK) { | 320 | + if (StrToU16(platformVersion, info.at("platform_version")) != ANALYSIS_OK) |
| 321 | + { | ||
| 257 | ERROR("PlatformVersion to uint16_t failed, input path %, deviceId %", profPath, deviceId); | 322 | ERROR("PlatformVersion to uint16_t failed, input path %, deviceId %", profPath, deviceId); |
| 258 | } | 323 | } |
| 259 | return platformVersion; | 324 | return platformVersion; |
| @@ -262,31 +327,37 @@ uint16_t Context::GetPlatformVersion(uint16_t deviceId, const std::string &profP | |||
| 262 | bool Context::GetProfTimeRecordInfo(Utils::ProfTimeRecord &record, const std::string &profPath, uint16_t deviceId) | 327 | bool Context::GetProfTimeRecordInfo(Utils::ProfTimeRecord &record, const std::string &profPath, uint16_t deviceId) |
| 263 | { | 328 | { |
| 264 | auto info = GetInfoByDeviceId(deviceId, profPath); | 329 | auto info = GetInfoByDeviceId(deviceId, profPath); |
| 265 | - if (info.empty()) { | 330 | + if (info.empty()) |
| 331 | + { | ||
| 266 | WARN("There is no host time log in %, it will use device time log!", profPath); | 332 | WARN("There is no host time log in %, it will use device time log!", profPath); |
| 267 | info = GetInfoByDeviceId(DEFAULT_DEVICE_ID, profPath); | 333 | info = GetInfoByDeviceId(DEFAULT_DEVICE_ID, profPath); |
| 268 | - if (info.empty()) { | 334 | + if (info.empty()) |
| 335 | + { | ||
| 269 | ERROR("No device time log in %, device id is %.", profPath, deviceId); | 336 | ERROR("No device time log in %, device id is %.", profPath, deviceId); |
| 270 | return false; | 337 | return false; |
| 271 | } | 338 | } |
| 272 | } | 339 | } |
| 273 | uint64_t startTimeUs = UINT64_MAX; | 340 | uint64_t startTimeUs = UINT64_MAX; |
| 274 | - if (StrToU64(startTimeUs, info.at("startCollectionTimeBegin")) != ANALYSIS_OK) { | 341 | + if (StrToU64(startTimeUs, info.at("startCollectionTimeBegin")) != ANALYSIS_OK) |
| 342 | + { | ||
| 275 | ERROR("StartTime to uint64_t failed. Prof path is %, device id is %.", profPath, deviceId); | 343 | ERROR("StartTime to uint64_t failed. Prof path is %, device id is %.", profPath, deviceId); |
| 276 | return false; | 344 | return false; |
| 277 | } | 345 | } |
| 278 | uint64_t endTimeUs = DEFAULT_DURATION_TIME_US + startTimeUs; | 346 | uint64_t endTimeUs = DEFAULT_DURATION_TIME_US + startTimeUs; |
| 279 | - if (StrToU64(endTimeUs, info.value("endCollectionTimeEnd", std::to_string(endTimeUs))) != ANALYSIS_OK) { | 347 | + if (StrToU64(endTimeUs, info.value("endCollectionTimeEnd", std::to_string(endTimeUs))) != ANALYSIS_OK) |
| 348 | + { | ||
| 280 | ERROR("EndTime to uint64_t failed. Prof path is %, device id is %.", profPath, deviceId); | 349 | ERROR("EndTime to uint64_t failed. Prof path is %, device id is %.", profPath, deviceId); |
| 281 | return false; | 350 | return false; |
| 282 | } | 351 | } |
| 283 | uint64_t baseTimeNs = UINT64_MAX; | 352 | uint64_t baseTimeNs = UINT64_MAX; |
| 284 | - if (StrToU64(baseTimeNs, info.at("startClockMonotonicRaw")) != ANALYSIS_OK) { | 353 | + if (StrToU64(baseTimeNs, info.at("startClockMonotonicRaw")) != ANALYSIS_OK) |
| 354 | + { | ||
| 285 | ERROR("BaseTime to uint64_t failed. Prof path is %, device id is %.", profPath, deviceId); | 355 | ERROR("BaseTime to uint64_t failed. Prof path is %, device id is %.", profPath, deviceId); |
| 286 | return false; | 356 | return false; |
| 287 | } | 357 | } |
| 288 | // 先判断时间之间的大小关系，确保后续计算时整数不回绕 | 358 | // 先判断时间之间的大小关系，确保后续计算时整数不回绕 |
| 289 | - if ((startTimeUs * MILLI_SECOND < baseTimeNs)) { | 359 | + if ((startTimeUs * MILLI_SECOND < baseTimeNs)) |
| 360 | + { | ||
| 290 | ERROR("The value of startTimeUs and baseTimeNs is invalid. Path is %, device id is %.", profPath, deviceId); | 361 | ERROR("The value of startTimeUs and baseTimeNs is invalid. Path is %, device id is %.", profPath, deviceId); |
| 291 | return false; | 362 | return false; |
| 292 | } | 363 | } |
| @@ -301,25 +372,29 @@ uint32_t Context::GetPidFromInfoJson(uint16_t deviceId, const std::string &profP | |||
| 301 | { | 372 | { |
| 302 | const auto &info = GetInfoByDeviceId(deviceId, profPath); | 373 | const auto &info = GetInfoByDeviceId(deviceId, profPath); |
| 303 | uint32_t pid = 0; | 374 | uint32_t pid = 0; |
| 304 | - if (info.empty()) { | 375 | + if (info.empty()) |
| 376 | + { | ||
| 305 | ERROR("GetPidFromInfoJson device info is empty, input path %, deviceId %", profPath, deviceId); | 377 | ERROR("GetPidFromInfoJson device info is empty, input path %, deviceId %", profPath, deviceId); |
| 306 | return pid; | 378 | return pid; |
| 307 | } | 379 | } |
| 308 | const std::string strPid = info.at("pid"); | 380 | const std::string strPid = info.at("pid"); |
| 309 | - if (strPid == "NA") { | 381 | + if (strPid == "NA") |
| 382 | + { | ||
| 310 | WARN("pid is NA and will be set 0."); | 383 | WARN("pid is NA and will be set 0."); |
| 311 | return pid; | 384 | return pid; |
| 312 | } | 385 | } |
| 313 | - if (StrToU32(pid, strPid) != ANALYSIS_OK) { | 386 | + if (StrToU32(pid, strPid) != ANALYSIS_OK) |
| 387 | + { | ||
| 314 | ERROR("Pid to uint32_t failed, input path %, deviceId %", profPath, deviceId); | 388 | ERROR("Pid to uint32_t failed, input path %, deviceId %", profPath, deviceId); |
| 315 | } | 389 | } |
| 316 | return pid; | 390 | return pid; |
| 317 | } | 391 | } |
| 318 | 392 | ||
| 319 | -std::string Context::GetPidNameFromInfoJson(uint16_t deviceId, const std::string& profPath) | 393 | +std::string Context::GetPidNameFromInfoJson(uint16_t deviceId, const std::string &profPath) |
| 320 | { | 394 | { |
| 321 | const auto &info = GetInfoByDeviceId(deviceId, profPath); | 395 | const auto &info = GetInfoByDeviceId(deviceId, profPath); |
| 322 | - if (info.empty()) { | 396 | + if (info.empty()) |
| 397 | + { | ||
| 323 | ERROR("GetPidNameFromInfoJson device info is empty."); | 398 | ERROR("GetPidNameFromInfoJson device info is empty."); |
| 324 | return ""; | 399 | return ""; |
| 325 | } | 400 | } |
| @@ -329,7 +404,8 @@ std::string Context::GetPidNameFromInfoJson(uint16_t deviceId, const std::string | |||
| 329 | int64_t Context::GetMsBinPid(const std::string &profPath) | 404 | int64_t Context::GetMsBinPid(const std::string &profPath) |
| 330 | { | 405 | { |
| 331 | const auto &info = GetInfoByDeviceId(DEFAULT_DEVICE_ID, profPath); | 406 | const auto &info = GetInfoByDeviceId(DEFAULT_DEVICE_ID, profPath); |
| 332 | - if (info.empty()) { | 407 | + if (info.empty()) |
| 408 | + { | ||
| 333 | PRINT_ERROR("Samplejson is empty, path %.", profPath); | 409 | PRINT_ERROR("Samplejson is empty, path %.", profPath); |
| 334 | return analysis::dvvp::common::config::MSVP_MMPROCESS; | 410 | return analysis::dvvp::common::config::MSVP_MMPROCESS; |
| 335 | } | 411 | } |
| @@ -340,62 +416,76 @@ int64_t Context::GetMsBinPid(const std::string &profPath) | |||
| 340 | std::string Context::GetLLCProfiling(uint16_t deviceId, const std::string &profPath) | 416 | std::string Context::GetLLCProfiling(uint16_t deviceId, const std::string &profPath) |
| 341 | { | 417 | { |
| 342 | const auto &info = GetInfoByDeviceId(deviceId, profPath); | 418 | const auto &info = GetInfoByDeviceId(deviceId, profPath); |
| 343 | - if (info.empty()) { | 419 | + if (info.empty()) |
| 420 | + { | ||
| 344 | ERROR("Samplejson is empty, path %.", profPath); | 421 | ERROR("Samplejson is empty, path %.", profPath); |
| 345 | return ""; | 422 | return ""; |
| 346 | } | 423 | } |
| 347 | return info.value("llc_profiling", ""); | 424 | return info.value("llc_profiling", ""); |
| 348 | } | 425 | } |
| 349 | 426 | ||
| 350 | -bool Context::GetSyscntConversionParams(Utils::SyscntConversionParams ¶ms, | 427 | +bool Context::GetSyscntConversionParams(Utils::SyscntConversionParams ¶ms, uint16_t deviceId, |
| 351 | - uint16_t deviceId, const std::string &profPath) | 428 | + const std::string &profPath) |
| 352 | { | 429 | { |
| 353 | auto info = GetInfoByDeviceId(deviceId, profPath); | 430 | auto info = GetInfoByDeviceId(deviceId, profPath); |
| 354 | // host freq可用作host cnt计算，也可用于host diff计算 | 431 | // host freq可用作host cnt计算，也可用于host diff计算 |
| 355 | - if (info.empty()) { | 432 | + if (info.empty()) |
| 433 | + { | ||
| 356 | ERROR("GetSyscntConversionParams device info is empty, input path %, deviceId %", profPath, deviceId); | 434 | ERROR("GetSyscntConversionParams device info is empty, input path %, deviceId %", profPath, deviceId); |
| 357 | return false; | 435 | return false; |
| 358 | } | 436 | } |
| 359 | std::string hostFreqStr = info.at("CPU").back().at("Frequency"); | 437 | std::string hostFreqStr = info.at("CPU").back().at("Frequency"); |
| 360 | double hostFreq = DEFAULT_FREQ; | 438 | double hostFreq = DEFAULT_FREQ; |
| 361 | - if (hostFreqStr.empty()) { | 439 | + if (hostFreqStr.empty()) |
| 440 | + { | ||
| 362 | INFO("HostFreq is empty, it will be set 1000.0 ."); | 441 | INFO("HostFreq is empty, it will be set 1000.0 ."); |
| 363 | - } else if (StrToDouble(hostFreq, hostFreqStr) != ANALYSIS_OK) { | 442 | + } |
| 443 | + else if (StrToDouble(hostFreq, hostFreqStr) != ANALYSIS_OK) | ||
| 444 | + { | ||
| 364 | ERROR("HostFreq to double failed, input path %, deviceId %", profPath, deviceId); | 445 | ERROR("HostFreq to double failed, input path %, deviceId %", profPath, deviceId); |
| 365 | return false; | 446 | return false; |
| 366 | } | 447 | } |
| 367 | - if (deviceId == HOST_ID) { | 448 | + if (deviceId == HOST_ID) |
| 449 | + { | ||
| 368 | params.freq = hostFreq; | 450 | params.freq = hostFreq; |
| 369 | - } else { | 451 | + } |
| 452 | + else | ||
| 453 | + { | ||
| 370 | // freq 来自info.json | 454 | // freq 来自info.json |
| 371 | - if (StrToDouble(params.freq, info.at("DeviceInfo").back().at("hwts_frequency")) != ANALYSIS_OK) { | 455 | + if (StrToDouble(params.freq, info.at("DeviceInfo").back().at("hwts_frequency")) != ANALYSIS_OK) |
| 456 | + { | ||
| 372 | ERROR("DeviceFreq to double failed, input path %, deviceId %", profPath, deviceId); | 457 | ERROR("DeviceFreq to double failed, input path %, deviceId %", profPath, deviceId); |
| 373 | return false; | 458 | return false; |
| 374 | } | 459 | } |
| 375 | } | 460 | } |
| 376 | - if (IsDoubleEqual(params.freq, 0) || IsDoubleEqual(hostFreq, 0)) { | 461 | + if (IsDoubleEqual(params.freq, 0) || IsDoubleEqual(hostFreq, 0)) |
| 462 | + { | ||
| 377 | ERROR("Freq is 0, can't be used to calculate, input path %, deviceId %", profPath, deviceId); | 463 | ERROR("Freq is 0, can't be used to calculate, input path %, deviceId %", profPath, deviceId); |
| 378 | return false; | 464 | return false; |
| 379 | } | 465 | } |
| 380 | params.hostFreq = hostFreq; | 466 | params.hostFreq = hostFreq; |
| 381 | // host取host的cnt， device取device的cnt | 467 | // host取host的cnt， device取device的cnt |
| 382 | std::string cntName = (deviceId == HOST_ID) ? "hostCntvct" : "devCntvct"; | 468 | std::string cntName = (deviceId == HOST_ID) ? "hostCntvct" : "devCntvct"; |
| 383 | - if (StrToU64(params.sysCnt, info.value(cntName, "0")) != ANALYSIS_OK) { | 469 | + if (StrToU64(params.sysCnt, info.value(cntName, "0")) != ANALYSIS_OK) |
| 470 | + { | ||
| 384 | ERROR("SysCnt to uint64_t failed, input path %, deviceId %", profPath, deviceId); | 471 | ERROR("SysCnt to uint64_t failed, input path %, deviceId %", profPath, deviceId); |
| 385 | return false; | 472 | return false; |
| 386 | } | 473 | } |
| 387 | // hostCnt fetch from the host_start_log file | 474 | // hostCnt fetch from the host_start_log file |
| 388 | - if (StrToU64(params.hostCnt, info.value("hostCntvct", "0")) != ANALYSIS_OK) { | 475 | + if (StrToU64(params.hostCnt, info.value("hostCntvct", "0")) != ANALYSIS_OK) |
| 476 | + { | ||
| 389 | ERROR("hostCnt to uint64_t failed, input path %, deviceId %", profPath, deviceId); | 477 | ERROR("hostCnt to uint64_t failed, input path %, deviceId %", profPath, deviceId); |
| 390 | return false; | 478 | return false; |
| 391 | } | 479 | } |
| 392 | // hostMonotonic 来自 host_start_log 的 clock_monotonic_raw | 480 | // hostMonotonic 来自 host_start_log 的 clock_monotonic_raw |
| 393 | - if (StrToU64(params.hostMonotonic, info.value("hostMonotonic", "0")) != ANALYSIS_OK) { | 481 | + if (StrToU64(params.hostMonotonic, info.value("hostMonotonic", "0")) != ANALYSIS_OK) |
| 482 | + { | ||
| 394 | ERROR("HostMonotonic to uint64_t failed, input path %, deviceId %", profPath, deviceId); | 483 | ERROR("HostMonotonic to uint64_t failed, input path %, deviceId %", profPath, deviceId); |
| 395 | return false; | 484 | return false; |
| 396 | } | 485 | } |
| 397 | uint64_t diff = 0; | 486 | uint64_t diff = 0; |
| 398 | - if (StrToU64(diff, info.value("hostCntvctDiff", "0")) != ANALYSIS_OK) { | 487 | + if (StrToU64(diff, info.value("hostCntvctDiff", "0")) != ANALYSIS_OK) |
| 488 | + { | ||
| 399 | WARN("HostCntvctDiff to uint64_t failed, input path %, deviceId %", profPath, deviceId); | 489 | WARN("HostCntvctDiff to uint64_t failed, input path %, deviceId %", profPath, deviceId); |
| 400 | // diff 异常不影响数据解析，部分时间存在误差 | 490 | // diff 异常不影响数据解析，部分时间存在误差 |
| 401 | return true; | 491 | return true; |
| @@ -407,17 +497,20 @@ bool Context::GetSyscntConversionParams(Utils::SyscntConversionParams ¶ms, | |||
| 407 | 497 | ||
| 408 | bool Context::GetPmuFreq(double &freq, uint16_t deviceId, const std::string &profPath) | 498 | bool Context::GetPmuFreq(double &freq, uint16_t deviceId, const std::string &profPath) |
| 409 | { | 499 | { |
| 410 | - if (deviceId == HOST_ID) { | 500 | + if (deviceId == HOST_ID) |
| 501 | + { | ||
| 411 | ERROR("Host do not have aic or aiv frequency!"); | 502 | ERROR("Host do not have aic or aiv frequency!"); |
| 412 | return false; | 503 | return false; |
| 413 | } | 504 | } |
| 414 | auto info = GetInfoByDeviceId(deviceId, profPath); | 505 | auto info = GetInfoByDeviceId(deviceId, profPath); |
| 415 | - if (info.empty()) { | 506 | + if (info.empty()) |
| 507 | + { | ||
| 416 | ERROR("GetPmuFreq device info is empty, input path %, deviceId %", profPath, deviceId); | 508 | ERROR("GetPmuFreq device info is empty, input path %, deviceId %", profPath, deviceId); |
| 417 | return false; | 509 | return false; |
| 418 | } | 510 | } |
| 419 | // freq 来自info.json | 511 | // freq 来自info.json |
| 420 | - if (StrToDouble(freq, info.at("DeviceInfo").back().at("aic_frequency")) != ANALYSIS_OK) { | 512 | + if (StrToDouble(freq, info.at("DeviceInfo").back().at("aic_frequency")) != ANALYSIS_OK) |
| 513 | + { | ||
| 421 | ERROR("DeviceFreq to double failed, input path %, deviceId %", profPath, deviceId); | 514 | ERROR("DeviceFreq to double failed, input path %, deviceId %", profPath, deviceId); |
| 422 | return false; | 515 | return false; |
| 423 | } | 516 | } |
| @@ -427,7 +520,8 @@ bool Context::GetPmuFreq(double &freq, uint16_t deviceId, const std::string &pro | |||
| 427 | bool Context::GetMetricMode(std::string &metricMode, const std::string &profPath) | 520 | bool Context::GetMetricMode(std::string &metricMode, const std::string &profPath) |
| 428 | { | 521 | { |
| 429 | auto info = GetInfoByDeviceId(DEFAULT_DEVICE_ID, profPath); | 522 | auto info = GetInfoByDeviceId(DEFAULT_DEVICE_ID, profPath); |
| 430 | - if (info.empty()) { | 523 | + if (info.empty()) |
| 524 | + { | ||
| 431 | ERROR("GetMetricMode device info is empty."); | 525 | ERROR("GetMetricMode device info is empty."); |
| 432 | return false; | 526 | return false; |
| 433 | } | 527 | } |
| @@ -437,38 +531,47 @@ bool Context::GetMetricMode(std::string &metricMode, const std::string &profPath | |||
| 437 | 531 | ||
| 438 | bool Context::GetClockMonotonicRaw(uint64_t &monotonicRaw, bool isHost, uint16_t deviceId, const std::string &profPath) | 532 | bool Context::GetClockMonotonicRaw(uint64_t &monotonicRaw, bool isHost, uint16_t deviceId, const std::string &profPath) |
| 439 | { | 533 | { |
| 440 | - if (!isHost && (deviceId == HOST_ID)) { | 534 | + if (!isHost && (deviceId == HOST_ID)) |
| 535 | + { | ||
| 441 | ERROR("GetClockMonotonicRaw host do not have device monotonic!"); | 536 | ERROR("GetClockMonotonicRaw host do not have device monotonic!"); |
| 442 | return false; | 537 | return false; |
| 443 | } | 538 | } |
| 444 | auto info = GetInfoByDeviceId(deviceId, profPath); | 539 | auto info = GetInfoByDeviceId(deviceId, profPath); |
| 445 | - if (info.empty()) { | 540 | + if (info.empty()) |
| 541 | + { | ||
| 446 | ERROR("GetClockMonotonicRaw device info is empty."); | 542 | ERROR("GetClockMonotonicRaw device info is empty."); |
| 447 | return false; | 543 | return false; |
| 448 | } | 544 | } |
| 449 | // host取host monotonic， device取device的monotonic | 545 | // host取host monotonic， device取device的monotonic |
| 450 | std::string monotonic = isHost ? "hostMonotonic" : "devMonotonic"; | 546 | std::string monotonic = isHost ? "hostMonotonic" : "devMonotonic"; |
| 451 | - if (StrToU64(monotonicRaw, info.value(monotonic, "0")) != ANALYSIS_OK) { | 547 | + if (StrToU64(monotonicRaw, info.value(monotonic, "0")) != ANALYSIS_OK) |
| 548 | + { | ||
| 452 | ERROR("Monotonic to uint64_t failed, input path %, deviceId %", profPath, deviceId); | 549 | ERROR("Monotonic to uint64_t failed, input path %, deviceId %", profPath, deviceId); |
| 453 | return false; | 550 | return false; |
| 454 | } | 551 | } |
| 455 | - if (!isHost) { | 552 | + if (!isHost) |
| 553 | + { | ||
| 456 | return true; | 554 | return true; |
| 457 | } | 555 | } |
| 458 | std::string hostFreqStr = info.at("CPU").back().at("Frequency"); | 556 | std::string hostFreqStr = info.at("CPU").back().at("Frequency"); |
| 459 | double hostFreq = DEFAULT_FREQ; | 557 | double hostFreq = DEFAULT_FREQ; |
| 460 | - if (hostFreqStr.empty()) { | 558 | + if (hostFreqStr.empty()) |
| 559 | + { | ||
| 461 | INFO("HostFreq is empty, it will be set 1000.0 ."); | 560 | INFO("HostFreq is empty, it will be set 1000.0 ."); |
| 462 | - } else if (StrToDouble(hostFreq, hostFreqStr) != ANALYSIS_OK) { | 561 | + } |
| 562 | + else if (StrToDouble(hostFreq, hostFreqStr) != ANALYSIS_OK) | ||
| 563 | + { | ||
| 463 | ERROR("HostFreq to double failed, input path %, deviceId %", profPath, deviceId); | 564 | ERROR("HostFreq to double failed, input path %, deviceId %", profPath, deviceId); |
| 464 | return false; | 565 | return false; |
| 465 | } | 566 | } |
| 466 | - if (IsDoubleEqual(hostFreq, 0)) { | 567 | + if (IsDoubleEqual(hostFreq, 0)) |
| 568 | + { | ||
| 467 | ERROR("Freq is 0, can't be used to calculate."); | 569 | ERROR("Freq is 0, can't be used to calculate."); |
| 468 | return false; | 570 | return false; |
| 469 | } | 571 | } |
| 470 | uint64_t diff = 0; | 572 | uint64_t diff = 0; |
| 471 | - if (StrToU64(diff, info.value("hostCntvctDiff", "0")) != ANALYSIS_OK) { | 573 | + if (StrToU64(diff, info.value("hostCntvctDiff", "0")) != ANALYSIS_OK) |
| 574 | + { | ||
| 472 | WARN("HostCntvctDiff to uint64_t failed, input path %, deviceId %", profPath, deviceId); | 575 | WARN("HostCntvctDiff to uint64_t failed, input path %, deviceId %", profPath, deviceId); |
| 473 | // diff 异常不影响数据解析，部分时间存在误差 | 576 | // diff 异常不影响数据解析，部分时间存在误差 |
| 474 | return true; | 577 | return true; |
| @@ -481,7 +584,8 @@ bool Context::GetClockMonotonicRaw(uint64_t &monotonicRaw, bool isHost, uint16_t | |||
| 481 | std::string Context::GetHostUid(uint16_t deviceId, const std::string &profPath) | 584 | std::string Context::GetHostUid(uint16_t deviceId, const std::string &profPath) |
| 482 | { | 585 | { |
| 483 | const auto &info = GetInfoByDeviceId(deviceId, profPath); | 586 | const auto &info = GetInfoByDeviceId(deviceId, profPath); |
| 484 | - if (info.empty()) { | 587 | + if (info.empty()) |
| 588 | + { | ||
| 485 | ERROR("GetHostUid InfoJson info is empty, input path %, deviceId %", profPath, deviceId); | 589 | ERROR("GetHostUid InfoJson info is empty, input path %, deviceId %", profPath, deviceId); |
| 486 | return DEFAULT_HOST_UID; | 590 | return DEFAULT_HOST_UID; |
| 487 | } | 591 | } |
| @@ -491,7 +595,8 @@ std::string Context::GetHostUid(uint16_t deviceId, const std::string &profPath) | |||
| 491 | std::string Context::GetHostName(uint16_t deviceId, const std::string &profPath) | 595 | std::string Context::GetHostName(uint16_t deviceId, const std::string &profPath) |
| 492 | { | 596 | { |
| 493 | const auto &info = GetInfoByDeviceId(deviceId, profPath); | 597 | const auto &info = GetInfoByDeviceId(deviceId, profPath); |
| 494 | - if (info.empty()) { | 598 | + if (info.empty()) |
| 599 | + { | ||
| 495 | ERROR("GetHostName InfoJson info is empty."); | 600 | ERROR("GetHostName InfoJson info is empty."); |
| 496 | return ""; | 601 | return ""; |
| 497 | } | 602 | } |
| @@ -500,28 +605,30 @@ std::string Context::GetHostName(uint16_t deviceId, const std::string &profPath) | |||
| 500 | 605 | ||
| 501 | std::vector<std::string> Context::GetQosEvents(uint16_t deviceId, const std::string &profPath) | 606 | std::vector<std::string> Context::GetQosEvents(uint16_t deviceId, const std::string &profPath) |
| 502 | { | 607 | { |
| 503 | - if (deviceId == HOST_ID) { | 608 | + if (deviceId == HOST_ID) |
| 609 | + { | ||
| 504 | ERROR("Host do not have qosEvents!"); | 610 | ERROR("Host do not have qosEvents!"); |
| 505 | return {}; | 611 | return {}; |
| 506 | } | 612 | } |
| 507 | auto info = GetInfoByDeviceId(deviceId, profPath); | 613 | auto info = GetInfoByDeviceId(deviceId, profPath); |
| 508 | - if (info.empty()) { | 614 | + if (info.empty()) |
| 615 | + { | ||
| 509 | ERROR("GetQosEvents device info is empty."); | 616 | ERROR("GetQosEvents device info is empty."); |
| 510 | return {}; | 617 | return {}; |
| 511 | } | 618 | } |
| 512 | std::string qosEvents = info.value("qosEvents", ""); | 619 | std::string qosEvents = info.value("qosEvents", ""); |
| 513 | - if (qosEvents.empty()) { | 620 | + if (qosEvents.empty()) |
| 621 | + { | ||
| 514 | INFO("Check qosProfiling is on or off, if it is on, maybe some mistakes have happened"); | 622 | INFO("Check qosProfiling is on or off, if it is on, maybe some mistakes have happened"); |
| 515 | return {}; | 623 | return {}; |
| 516 | - } else { | 624 | + } |
| 625 | + else | ||
| 626 | + { | ||
| 517 | return Split(qosEvents, ","); | 627 | return Split(qosEvents, ","); |
| 518 | } | 628 | } |
| 519 | } | 629 | } |
| 520 | 630 | ||
| 521 | -void Context::Clear() | 631 | +void Context::Clear() { context_.clear(); } |
| 522 | -{ | ||
| 523 | - context_.clear(); | ||
| 524 | -} | ||
| 525 | 632 | ||
| 526 | bool Context::IsStarsChip(uint16_t platformVersion) | 633 | bool Context::IsStarsChip(uint16_t platformVersion) |
| 527 | { | 634 | { |
| @@ -537,10 +644,7 @@ bool Context::IsChipV1(uint16_t platformVersion) | |||
| 537 | return static_cast<bool>(checkList.count(platformVersion)); | 644 | return static_cast<bool>(checkList.count(platformVersion)); |
| 538 | } | 645 | } |
| 539 | 646 | ||
| 540 | -bool Context::IsChipV4(uint16_t platformVersion) | 647 | +bool Context::IsChipV4(uint16_t platformVersion) { return platformVersion == static_cast<int>(Chip::CHIP_V4_1_0); } |
| 541 | -{ | ||
| 542 | - return platformVersion == static_cast<int>(Chip::CHIP_V4_1_0); | ||
| 543 | -} | ||
| 544 | 648 | ||
| 545 | bool Context::IsChipV6(uint16_t platformVersion) | 649 | bool Context::IsChipV6(uint16_t platformVersion) |
| 546 | { | 650 | { |
| @@ -550,19 +654,18 @@ bool Context::IsChipV6(uint16_t platformVersion) | |||
| 550 | return static_cast<bool>(checkList.count(platformVersion)); | 654 | return static_cast<bool>(checkList.count(platformVersion)); |
| 551 | } | 655 | } |
| 552 | 656 | ||
| 553 | -bool Context::IsFirstChipV1(uint16_t platformVersion) | 657 | +bool Context::IsFirstChipV1(uint16_t platformVersion) { return platformVersion == static_cast<int>(Chip::CHIP_V1_1_0); } |
| 554 | -{ | ||
| 555 | - return platformVersion == static_cast<int>(Chip::CHIP_V1_1_0); | ||
| 556 | -} | ||
| 557 | 658 | ||
| 558 | uint16_t Context::GetAiCoreNum(uint16_t deviceId, const std::string &profPath) | 659 | uint16_t Context::GetAiCoreNum(uint16_t deviceId, const std::string &profPath) |
| 559 | { | 660 | { |
| 560 | - if (deviceId == HOST_ID) { | 661 | + if (deviceId == HOST_ID) |
| 662 | + { | ||
| 561 | ERROR("Host do not have ai core num!"); | 663 | ERROR("Host do not have ai core num!"); |
| 562 | return 0; | 664 | return 0; |
| 563 | } | 665 | } |
| 564 | auto info = GetInfoByDeviceId(deviceId, profPath); | 666 | auto info = GetInfoByDeviceId(deviceId, profPath); |
| 565 | - if (info.empty()) { | 667 | + if (info.empty()) |
| 668 | + { | ||
| 566 | ERROR("GetAiCoreNum device info is empty, input path %, deviceId %", profPath, deviceId); | 669 | ERROR("GetAiCoreNum device info is empty, input path %, deviceId %", profPath, deviceId); |
| 567 | return 0; | 670 | return 0; |
| 568 | } | 671 | } |
| @@ -573,7 +676,8 @@ uint64_t Context::GetTotalMem(uint16_t deviceId, const std::string &profPath) | |||
| 573 | { | 676 | { |
| 574 | auto info = GetInfoByDeviceId(deviceId, profPath); | 677 | auto info = GetInfoByDeviceId(deviceId, profPath); |
| 575 | // 这里如果没有host目录 会去取到device的数据 本质上依赖外层路径校验 | 678 | // 这里如果没有host目录 会去取到device的数据 本质上依赖外层路径校验 |
| 576 | - if (info.empty()) { | 679 | + if (info.empty()) |
| 680 | + { | ||
| 577 | ERROR("info is empty, input path %, deviceId %", profPath, deviceId); | 681 | ERROR("info is empty, input path %, deviceId %", profPath, deviceId); |
| 578 | return 0; | 682 | return 0; |
| 579 | } | 683 | } |
| @@ -584,13 +688,15 @@ uint64_t Context::GetNetCardTotalSpeed(uint16_t deviceId, const std::string &pro | |||
| 584 | { | 688 | { |
| 585 | auto info = GetInfoByDeviceId(deviceId, profPath); | 689 | auto info = GetInfoByDeviceId(deviceId, profPath); |
| 586 | // 这里如果没有host目录 会去取到device的数据 本质上依赖外层路径校验 | 690 | // 这里如果没有host目录 会去取到device的数据 本质上依赖外层路径校验 |
| 587 | - if (info.empty()) { | 691 | + if (info.empty()) |
| 692 | + { | ||
| 588 | ERROR("info is empty, input path %, deviceId %", profPath, deviceId); | 693 | ERROR("info is empty, input path %, deviceId %", profPath, deviceId); |
| 589 | return 0; | 694 | return 0; |
| 590 | } | 695 | } |
| 591 | uint64_t totalSpeed = 0; | 696 | uint64_t totalSpeed = 0; |
| 592 | // 负数不计数 | 697 | // 负数不计数 |
| 593 | - for (const auto netCard : info.at("netCard")) { | 698 | + for (const auto netCard : info.at("netCard")) |
| 699 | + { | ||
| 594 | auto speed = netCard.value("speed", 0); | 700 | auto speed = netCard.value("speed", 0); |
| 595 | totalSpeed += (speed < 0) ? 0 : static_cast<uint64_t>(speed); | 701 | totalSpeed += (speed < 0) ? 0 : static_cast<uint64_t>(speed); |
| 596 | } | 702 | } |
| @@ -601,7 +707,8 @@ bool Context::IsLevel0(const std::string &profPath) | |||
| 601 | { | 707 | { |
| 602 | auto info = GetInfoByDeviceId(DEFAULT_DEVICE_ID, profPath); | 708 | auto info = GetInfoByDeviceId(DEFAULT_DEVICE_ID, profPath); |
| 603 | // 这里如果没有host目录 会去取到device的数据 本质上依赖外层路径校验 | 709 | // 这里如果没有host目录 会去取到device的数据 本质上依赖外层路径校验 |
| 604 | - if (info.empty()) { | 710 | + if (info.empty()) |
| 711 | + { | ||
| 605 | ERROR("info is empty, input path %", profPath); | 712 | ERROR("info is empty, input path %", profPath); |
| 606 | return true; | 713 | return true; |
| 607 | } | 714 | } |
| @@ -611,6 +718,63 @@ bool Context::IsLevel0(const std::string &profPath) | |||
| 611 | return (level0Set.find(profLevel) != level0Set.end()); | 718 | return (level0Set.find(profLevel) != level0Set.end()); |
| 612 | } | 719 | } |
| 613 | 720 | ||
| 721 | +std::vector<uint16_t> Context::GetCannVersion(uint16_t deviceId, const std::string &profPath) | ||
| 722 | +{ | ||
| 723 | + std::vector<uint16_t> emptyVersion; | ||
| 724 | + std::vector<uint16_t> retVersion; | ||
| 725 | + constexpr size_t CANN_VERSION_CNT = 2; // 2: major version and minor version | ||
| 726 | + auto info = GetInfoByDeviceId(deviceId, profPath); | ||
| 727 | + if (info.empty()) | ||
| 728 | + { | ||
| 729 | + ERROR("GetCannVersion device info is empty, input path %, deviceId %", profPath, deviceId); | ||
| 730 | + return emptyVersion; | ||
| 731 | + } | ||
| 732 | + if (info.contains("cannVersion") && info.at("cannVersion").is_string()) | ||
| 733 | + { | ||
| 734 | + // cann version format: major.minor.patch(-beta.x), like 9.1.0 or 9.1.0-beta.0 or 9.1.T100 | ||
| 735 | + std::string verStr = info.at("cannVersion").get<std::string>(); | ||
| 736 | + uint16_t verVal = 0; | ||
| 737 | + for (size_t i = 0; i < CANN_VERSION_CNT; ++i) | ||
| 738 | + { | ||
| 739 | + size_t pos = verStr.find("."); | ||
| 740 | + std::string seg; | ||
| 741 | + if (pos == std::string::npos) | ||
| 742 | + { | ||
| 743 | + seg = verStr; | ||
| 744 | + } | ||
| 745 | + else | ||
| 746 | + { | ||
| 747 | + seg = verStr.substr(0, pos); | ||
| 748 | + verStr = verStr.substr(pos + 1); | ||
| 749 | + } | ||
| 750 | + if (seg.empty()) | ||
| 751 | + { | ||
| 752 | + return emptyVersion; | ||
| 753 | + } | ||
| 754 | + for (char c : seg) | ||
| 755 | + { | ||
| 756 | + if (!std::isdigit(c)) | ||
| 757 | + { | ||
| 758 | + return emptyVersion; | ||
| 759 | + } | ||
| 760 | + } | ||
| 761 | + if (StrToU16(verVal, seg) != ANALYSIS_OK) | ||
| 762 | + { | ||
| 763 | + ERROR("CannVersion segment to uint16_t failed, input path %, deviceId %", profPath, deviceId); | ||
| 764 | + return emptyVersion; | ||
| 765 | + } | ||
| 766 | + retVersion.push_back(verVal); | ||
| 767 | + | ||
| 768 | + if (verStr.empty()) | ||
| 769 | + { | ||
| 770 | + break; | ||
| 771 | + } | ||
| 772 | + } | ||
| 773 | + return retVersion.size() == CANN_VERSION_CNT ? retVersion : emptyVersion; | ||
| 774 | + } | ||
| 775 | + return emptyVersion; | ||
| 776 | +} | ||
| 777 | + | ||
| 614 | } // namespace Environment | 778 | } // namespace Environment |
| 615 | -} // namespace Parser | 779 | +} // namespace Domain |
| 616 | } // namespace Analysis | 780 | } // namespace Analysis |
| @@ -18,23 +18,26 @@ | |||
| 18 | 18 | ||
| 19 | 19 | ||
| 20 | 20 | ||
| 21 | - | ||
| 22 | 21 | ||
| 22 | + | ||
| 23 | 23 | ||
| 24 | 24 | ||
| 25 | - | ||
| 26 | - | ||
| 27 | 25 | ||
| 28 | 26 | ||
| 27 | + | ||
| 29 | 28 | ||
| 30 | -namespace Analysis { | 29 | +namespace Analysis |
| 31 | -namespace Domain { | 30 | +{ |
| 32 | -namespace Environment { | 31 | +namespace Domain |
| 32 | +{ | ||
| 33 | +namespace Environment | ||
| 34 | +{ | ||
| 33 | const uint16_t HOST_ID = 64; | 35 | const uint16_t HOST_ID = 64; |
| 34 | // UINT16_MAX为非法device id, HOST_ID + 1 为默认device id | 36 | // UINT16_MAX为非法device id, HOST_ID + 1 为默认device id |
| 35 | const uint16_t INVALID_DEVICE_ID = UINT16_MAX; | 37 | const uint16_t INVALID_DEVICE_ID = UINT16_MAX; |
| 36 | const uint16_t DEFAULT_DEVICE_ID = HOST_ID + 1; | 38 | const uint16_t DEFAULT_DEVICE_ID = HOST_ID + 1; |
| 37 | -enum class Chip : uint16_t { | 39 | +enum class Chip : uint16_t |
| 40 | +{ | ||
| 38 | CHIP_V1_1_0 = 0, | 41 | CHIP_V1_1_0 = 0, |
| 39 | CHIP_V2_1_0 = 1, | 42 | CHIP_V2_1_0 = 1, |
| 40 | CHIP_V3_1_0 = 2, | 43 | CHIP_V3_1_0 = 2, |
| @@ -52,12 +55,14 @@ enum class Chip : uint16_t { | |||
| 52 | // 通过 std::unordered_map<std::string, std::unordered_map<uint16_t, nlohmann::json>> 结构的成员变量context_ | 55 | // 通过 std::unordered_map<std::string, std::unordered_map<uint16_t, nlohmann::json>> 结构的成员变量context_ |
| 53 | // 以prof, deviceId两层进行数据路径分割，将该device目录下的对应json和log进行key值合并，统一整合为一份json对象 | 56 | // 以prof, deviceId两层进行数据路径分割，将该device目录下的对应json和log进行key值合并，统一整合为一份json对象 |
| 54 | // 数据查询以prof（无prof则默认为begin()）和deviceId（必选）进行查找 | 57 | // 数据查询以prof（无prof则默认为begin()）和deviceId（必选）进行查找 |
| 55 | -class Context : public Utils::Singleton<Context> { | 58 | +class Context : public Utils::Singleton<Context> |
| 56 | -public: | 59 | +{ |
| 60 | + public: | ||
| 57 | bool Load(const std::set<std::string> &profPaths); | 61 | bool Load(const std::set<std::string> &profPaths); |
| 58 | bool IsAllExport(); | 62 | bool IsAllExport(); |
| 59 | void Clear(); | 63 | void Clear(); |
| 60 | -public: | 64 | + |
| 65 | + public: | ||
| 61 | // 获取start_info end_info中的时间 | 66 | // 获取start_info end_info中的时间 |
| 62 | bool GetProfTimeRecordInfo(Utils::ProfTimeRecord &record, const std::string &profPath = "", | 67 | bool GetProfTimeRecordInfo(Utils::ProfTimeRecord &record, const std::string &profPath = "", |
| 63 | uint16_t deviceId = HOST_ID); | 68 | uint16_t deviceId = HOST_ID); |
| @@ -90,7 +95,10 @@ public: | |||
| 90 | uint64_t GetTotalMem(uint16_t deviceId, const std::string &profPath); | 95 | uint64_t GetTotalMem(uint16_t deviceId, const std::string &profPath); |
| 91 | uint64_t GetNetCardTotalSpeed(uint16_t deviceId, const std::string &profPath); | 96 | uint64_t GetNetCardTotalSpeed(uint16_t deviceId, const std::string &profPath); |
| 92 | bool IsLevel0(const std::string &profPath); | 97 | bool IsLevel0(const std::string &profPath); |
| 93 | -public: | 98 | + // 获取cann version的major version和minor version |
| 99 | + std::vector<uint16_t> GetCannVersion(uint16_t deviceId, const std::string &profPath); | ||
| 100 | + | ||
| 101 | + public: | ||
| 94 | // 获取对应device的芯片型号 | 102 | // 获取对应device的芯片型号 |
| 95 | uint16_t GetPlatformVersion(uint16_t deviceId = DEFAULT_DEVICE_ID, const std::string &profPath = ""); | 103 | uint16_t GetPlatformVersion(uint16_t deviceId = DEFAULT_DEVICE_ID, const std::string &profPath = ""); |
| 96 | // 判断芯片类型 | 104 | // 判断芯片类型 |
| @@ -102,7 +110,7 @@ public: | |||
| 102 | // 校验是否为CHIP_V1_1_0 | 110 | // 校验是否为CHIP_V1_1_0 |
| 103 | static bool IsFirstChipV1(uint16_t platformVersion); | 111 | static bool IsFirstChipV1(uint16_t platformVersion); |
| 104 | 112 | ||
| 105 | -private: | 113 | + private: |
| 106 | nlohmann::json GetInfoByDeviceId(uint16_t deviceId = DEFAULT_DEVICE_ID, const std::string &profPath = ""); | 114 | nlohmann::json GetInfoByDeviceId(uint16_t deviceId = DEFAULT_DEVICE_ID, const std::string &profPath = ""); |
| 107 | bool LoadJsonData(const std::string &profPath, const std::string &deviceDir, uint16_t deviceId); | 115 | bool LoadJsonData(const std::string &profPath, const std::string &deviceDir, uint16_t deviceId); |
| 108 | bool LoadLogData(const std::string &profPath, const std::string &deviceDir, uint16_t deviceId); | 116 | bool LoadLogData(const std::string &profPath, const std::string &deviceDir, uint16_t deviceId); |
| @@ -110,6 +118,6 @@ private: | |||
| 110 | std::unordered_map<std::string, std::map<uint16_t, nlohmann::json>> context_; | 118 | std::unordered_map<std::string, std::map<uint16_t, nlohmann::json>> context_; |
| 111 | }; // class Context | 119 | }; // class Context |
| 112 | } // namespace Environment | 120 | } // namespace Environment |
| 113 | -} // namespace Parser | 121 | +} // namespace Domain |
| 114 | } // namespace Analysis | 122 | } // namespace Analysis |
| 115 | -#endif // ANALYSIS_PARSER_ENVIRONMENT_CONTEXT_H | 123 | +#endif // ANALYSIS_PARSER_ENVIRONMENT_CONTEXT_H |
| @@ -42,22 +42,18 @@ import multiprocessing | |||
| 42 | import os | 42 | import os |
| 43 | import re | 43 | import re |
| 44 | import shutil | 44 | import shutil |
| 45 | -import csv | ||
| 46 | 45 | ||
| 47 | from common_func.common import print_info | 46 | from common_func.common import print_info |
| 48 | -from common_func.common import warn | ||
| 49 | from common_func.constant import Constant | 47 | from common_func.constant import Constant |
| 50 | from common_func.data_check_manager import DataCheckManager | 48 | from common_func.data_check_manager import DataCheckManager |
| 51 | from common_func.file_manager import FdOpen | 49 | from common_func.file_manager import FdOpen |
| 52 | from common_func.file_manager import FileOpen | 50 | from common_func.file_manager import FileOpen |
| 53 | -from common_func.file_manager import check_path_valid | ||
| 54 | from common_func.file_slice_helper import FileSliceHelper | 51 | from common_func.file_slice_helper import FileSliceHelper |
| 55 | from common_func.ms_constant.str_constant import StrConstant | 52 | from common_func.ms_constant.str_constant import StrConstant |
| 56 | from common_func.msprof_common import MsProfCommonConstant | 53 | from common_func.msprof_common import MsProfCommonConstant |
| 57 | from common_func.msprof_common import get_path_dir | 54 | from common_func.msprof_common import get_path_dir |
| 58 | from common_func.msprof_common import get_valid_sub_path | 55 | from common_func.msprof_common import get_valid_sub_path |
| 59 | from common_func.msprof_exception import ProfException | 56 | from common_func.msprof_exception import ProfException |
| 60 | -from common_func.msvp_common import check_dir_writable | ||
| 61 | from common_func.msvp_common import is_number | 57 | from common_func.msvp_common import is_number |
| 62 | from common_func.path_manager import PathManager | 58 | from common_func.path_manager import PathManager |
| 63 | from common_func.utils import Utils | 59 | from common_func.utils import Utils |
| @@ -68,6 +64,7 @@ class MsprofOutputSummary: | |||
| 68 | """ | 64 | """ |
| 69 | class used to export all job data. | 65 | class used to export all job data. |
| 70 | """ | 66 | """ |
| 67 | + | ||
| 71 | DEVICE_PREFIX = "device_" | 68 | DEVICE_PREFIX = "device_" |
| 72 | INVALID_SUFFIX = "invalid" | 69 | INVALID_SUFFIX = "invalid" |
| 73 | MSPROF_HOST = "host" | 70 | MSPROF_HOST = "host" |
| @@ -76,9 +73,7 @@ class MsprofOutputSummary: | |||
| 76 | SLICE_LEN = 7 | 73 | SLICE_LEN = 7 |
| 77 | README = "README.txt" | 74 | README = "README.txt" |
| 78 | FILE_MAX_SIZE = 1024 * 1024 * 1024 | 75 | FILE_MAX_SIZE = 1024 * 1024 * 1024 |
| 79 | - JSON_LIST = [ | 76 | + JSON_LIST = ["msprof", "step_trace", "msprof_tx"] |
| 80 | - "msprof", "step_trace", "msprof_tx" | ||
| 81 | - ] | ||
| 82 | MSPROF_TX = "msprof_tx" | 77 | MSPROF_TX = "msprof_tx" |
| 83 | MSPROFTX_DEVICE_START_TIME_HEADER = "Device Start_time(us)" | 78 | MSPROFTX_DEVICE_START_TIME_HEADER = "Device Start_time(us)" |
| 84 | MSPROFTX_DEVICE_END_TIME_HEADER = "Device End_time(us)" | 79 | MSPROFTX_DEVICE_END_TIME_HEADER = "Device End_time(us)" |
| @@ -108,7 +103,7 @@ class MsprofOutputSummary: | |||
| 108 | """ | 103 | """ |
| 109 | match = re.search(r'(_\d)?(_slice_\d+)?_\d+', file_name) | 104 | match = re.search(r'(_\d)?(_slice_\d+)?_\d+', file_name) |
| 110 | if match and match.start() > 0: | 105 | if match and match.start() > 0: |
| 111 | - return file_name[:match.start()] | 106 | + return file_name[: match.start()] |
| 112 | logging.warning("The file name %s is invalid!", file_name) | 107 | logging.warning("The file name %s is invalid!", file_name) |
| 113 | return "invalid" | 108 | return "invalid" |
| 114 | 109 | ||
| @@ -118,8 +113,11 @@ class MsprofOutputSummary: | |||
| 118 | for index, filename in enumerate(file_set): | 113 | for index, filename in enumerate(file_set): |
| 119 | desc = file_dict.get(filename) | 114 | desc = file_dict.get(filename) |
| 120 | if not desc: | 115 | if not desc: |
| 121 | - desc = "Here is no description about this file: " + filename + \ | 116 | + desc = ( |
| 122 | - ", please check in 'Profiling Instructions'!\n" | 117 | + "Here is no description about this file: " |
| 118 | + + filename | ||
| 119 | + + ", please check in 'Profiling Instructions'!\n" | ||
| 120 | + ) | ||
| 123 | context += f"{str(index + 1)}.{filename}{suffix}:{desc}" | 121 | context += f"{str(index + 1)}.{filename}{suffix}:{desc}" |
| 124 | context += "\n" | 122 | context += "\n" |
| 125 | return context | 123 | return context |
| @@ -137,12 +135,12 @@ class MsprofOutputSummary: | |||
| 137 | if not MsprofOutputSummary._valid_pos(underscore_pos, point_pos, filename): | 135 | if not MsprofOutputSummary._valid_pos(underscore_pos, point_pos, filename): |
| 138 | logging.warning("The file name %s is invalid!", filename) | 136 | logging.warning("The file name %s is invalid!", filename) |
| 139 | continue | 137 | continue |
| 140 | - time_str = filename[underscore_pos + 1: point_pos] | 138 | + time_str = filename[underscore_pos + 1 : point_pos] |
| 141 | if not is_number(time_str): | 139 | if not is_number(time_str): |
| 142 | logging.warning("The file name %s is invalid!", filename) | 140 | logging.warning("The file name %s is invalid!", filename) |
| 143 | continue | 141 | continue |
| 144 | time = int(time_str) | 142 | time = int(time_str) |
| 145 | - key = filename[:underscore_pos + 1] | 143 | + key = filename[: underscore_pos + 1] |
| 146 | value = file_dict.get(key, 0) | 144 | value = file_dict.get(key, 0) |
| 147 | if not value or value < time: | 145 | if not value or value < time: |
| 148 | file_dict.update({key: time}) | 146 | file_dict.update({key: time}) |
| @@ -173,8 +171,10 @@ class MsprofOutputSummary: | |||
| 173 | self._export_msprof_timeline() | 171 | self._export_msprof_timeline() |
| 174 | self._export_readme_file() | 172 | self._export_readme_file() |
| 175 | output_path = os.path.join(self._output, PathManager.MINDSTUDIO_PROFILER_OUTPUT) | 173 | output_path = os.path.join(self._output, PathManager.MINDSTUDIO_PROFILER_OUTPUT) |
| 176 | - print_info(MsProfCommonConstant.COMMON_FILE_NAME, f"End exporting {command_type} output_file." \ | 174 | + print_info( |
| 177 | - f"The file is stored in the {output_path} path.") | 175 | + MsProfCommonConstant.COMMON_FILE_NAME, |
| 176 | + f"End exporting {command_type} output_file.The file is stored in the {output_path} path.", | ||
| 177 | + ) | ||
| 178 | 178 | ||
| 179 | def _is_in_prof_file(self): | 179 | def _is_in_prof_file(self): |
| 180 | """ | 180 | """ |
| @@ -183,8 +183,7 @@ class MsprofOutputSummary: | |||
| 183 | """ | 183 | """ |
| 184 | file_list = os.listdir(self._output) | 184 | file_list = os.listdir(self._output) |
| 185 | for file_name in file_list: | 185 | for file_name in file_list: |
| 186 | - if file_name == self.MSPROF_HOST or \ | 186 | + if file_name == self.MSPROF_HOST or file_name.startswith(self.DEVICE_PREFIX): |
| 187 | - file_name.startswith(self.DEVICE_PREFIX): | ||
| 188 | return True | 187 | return True |
| 189 | return False | 188 | return False |
| 190 | 189 | ||
| @@ -226,8 +225,7 @@ class MsprofOutputSummary: | |||
| 226 | """ | 225 | """ |
| 227 | if not (folder_name == self.MSPROF_HOST or folder_name.startswith(self.DEVICE_PREFIX)): | 226 | if not (folder_name == self.MSPROF_HOST or folder_name.startswith(self.DEVICE_PREFIX)): |
| 228 | return | 227 | return |
| 229 | - summary_path = os.path.realpath( | 228 | + summary_path = os.path.realpath(os.path.join(self._output, folder_name, MsProfCommonConstant.SUMMARY)) |
| 230 | - os.path.join(self._output, folder_name, MsProfCommonConstant.SUMMARY)) | ||
| 231 | if not os.path.exists(summary_path): | 229 | if not os.path.exists(summary_path): |
| 232 | return | 230 | return |
| 233 | file_list = self.get_newest_file_list(os.listdir(summary_path), file_suffix) | 231 | file_list = self.get_newest_file_list(os.listdir(summary_path), file_suffix) |
| @@ -239,7 +237,7 @@ class MsprofOutputSummary: | |||
| 239 | StrConstant.PARAM_DATA_TYPE: self._get_file_name(file_name), | 237 | StrConstant.PARAM_DATA_TYPE: self._get_file_name(file_name), |
| 240 | StrConstant.PARAM_EXPORT_TYPE: MsProfCommonConstant.SUMMARY, | 238 | StrConstant.PARAM_EXPORT_TYPE: MsProfCommonConstant.SUMMARY, |
| 241 | StrConstant.PARAM_EXPORT_FORMAT: StrConstant.EXPORT_JSON, | 239 | StrConstant.PARAM_EXPORT_FORMAT: StrConstant.EXPORT_JSON, |
| 242 | - StrConstant.PARAM_EXPORT_DUMP_FOLDER: PathManager.MINDSTUDIO_PROFILER_OUTPUT | 240 | + StrConstant.PARAM_EXPORT_DUMP_FOLDER: PathManager.MINDSTUDIO_PROFILER_OUTPUT, |
| 243 | } | 241 | } |
| 244 | shutil.copy(os.path.join(summary_path, file_name), FileSliceHelper.make_export_file_name(params)) | 242 | shutil.copy(os.path.join(summary_path, file_name), FileSliceHelper.make_export_file_name(params)) |
| 245 | 243 | ||
| @@ -258,10 +256,13 @@ class MsprofOutputSummary: | |||
| 258 | self._copy_summary_data(sub_dir, StrConstant.FILE_SUFFIX_JSON, True) | 256 | self._copy_summary_data(sub_dir, StrConstant.FILE_SUFFIX_JSON, True) |
| 259 | 257 | ||
| 260 | error = "Output: An exception occurs when multiple processes process the summary file. The error is %s" | 258 | error = "Output: An exception occurs when multiple processes process the summary file. The error is %s" |
| 261 | - pool = multiprocessing.Pool(processes=4) | 259 | + pool = multiprocessing.Pool(processes=4) # pylint: disable=R1732 |
| 262 | for summary_file in summary_file_set: | 260 | for summary_file in summary_file_set: |
| 263 | - pool.apply_async(func=self._save_summary_data, args=(summary_file, sub_dirs), | 261 | + pool.apply_async( |
| 264 | - error_callback=lambda error_info: logging.error(error, error_info)) | 262 | + func=self._save_summary_data, |
| 263 | + args=(summary_file, sub_dirs), | ||
| 264 | + error_callback=lambda error_info: logging.error(error, error_info), | ||
| 265 | + ) | ||
| 265 | pool.close() | 266 | pool.close() |
| 266 | pool.join() | 267 | pool.join() |
| 267 | 268 | ||
| @@ -270,8 +271,7 @@ class MsprofOutputSummary: | |||
| 270 | get target summary file in summary dir | 271 | get target summary file in summary dir |
| 271 | """ | 272 | """ |
| 272 | device_summary_set = set() | 273 | device_summary_set = set() |
| 273 | - summary_path = os.path.realpath( | 274 | + summary_path = os.path.realpath(os.path.join(device_path, MsProfCommonConstant.SUMMARY)) |
| 274 | - os.path.join(device_path, MsProfCommonConstant.SUMMARY)) | ||
| 275 | if not os.path.exists(summary_path): | 275 | if not os.path.exists(summary_path): |
| 276 | return device_summary_set | 276 | return device_summary_set |
| 277 | file_list = os.listdir(summary_path) | 277 | file_list = os.listdir(summary_path) |
| @@ -288,22 +288,21 @@ class MsprofOutputSummary: | |||
| 288 | StrConstant.PARAM_EXPORT_TYPE: MsProfCommonConstant.SUMMARY, | 288 | StrConstant.PARAM_EXPORT_TYPE: MsProfCommonConstant.SUMMARY, |
| 289 | StrConstant.PARAM_EXPORT_FORMAT: self._export_format, | 289 | StrConstant.PARAM_EXPORT_FORMAT: self._export_format, |
| 290 | StrConstant.PARAM_RESULT_DIR: self._output, | 290 | StrConstant.PARAM_RESULT_DIR: self._output, |
| 291 | - StrConstant.PARAM_EXPORT_DUMP_FOLDER: PathManager.MINDSTUDIO_PROFILER_OUTPUT | 291 | + StrConstant.PARAM_EXPORT_DUMP_FOLDER: PathManager.MINDSTUDIO_PROFILER_OUTPUT, |
| 292 | } | 292 | } |
| 293 | helper = FileSliceHelper(params, [], []) | 293 | helper = FileSliceHelper(params, [], []) |
| 294 | for sub_dir in sorted(sub_dirs): | 294 | for sub_dir in sorted(sub_dirs): |
| 295 | sub_path = get_valid_sub_path(self._output, sub_dir, False) | 295 | sub_path = get_valid_sub_path(self._output, sub_dir, False) |
| 296 | if not DataCheckManager.contain_info_json_data(sub_path): | 296 | if not DataCheckManager.contain_info_json_data(sub_path): |
| 297 | continue | 297 | continue |
| 298 | - summary_path = os.path.realpath( | 298 | + summary_path = os.path.realpath(os.path.join(sub_path, MsProfCommonConstant.SUMMARY)) |
| 299 | - os.path.join(sub_path, MsProfCommonConstant.SUMMARY)) | ||
| 300 | if not os.path.exists(summary_path): | 299 | if not os.path.exists(summary_path): |
| 301 | continue | 300 | continue |
| 302 | file_list = os.listdir(summary_path) | 301 | file_list = os.listdir(summary_path) |
| 303 | if sub_dir == self.MSPROF_HOST: | 302 | if sub_dir == self.MSPROF_HOST: |
| 304 | device_id = "host" | 303 | device_id = "host" |
| 305 | else: | 304 | else: |
| 306 | - device_id = os.path.basename(sub_path)[self.DEVICE_ID_PREFIX_LEN:] | 305 | + device_id = os.path.basename(sub_path)[self.DEVICE_ID_PREFIX_LEN :] |
| 307 | for file_name in self.get_newest_file_list(file_list, StrConstant.FILE_SUFFIX_CSV): | 306 | for file_name in self.get_newest_file_list(file_list, StrConstant.FILE_SUFFIX_CSV): |
| 308 | if not file_name.startswith(targe_name) or (targe_name == "aicpu" and file_name.startswith("aicpu_mi")): | 307 | if not file_name.startswith(targe_name) or (targe_name == "aicpu" and file_name.startswith("aicpu_mi")): |
| 309 | continue | 308 | continue |
| @@ -323,57 +322,84 @@ class MsprofOutputSummary: | |||
| 323 | def _update_msproftx_device_data(self, file_name_path: str, device_id: str): | 322 | def _update_msproftx_device_data(self, file_name_path: str, device_id: str): |
| 324 | cnt = 0 | 323 | cnt = 0 |
| 325 | with FileOpen(file_name_path, mode='r', max_size=self.FILE_MAX_SIZE) as _csv_file: | 324 | with FileOpen(file_name_path, mode='r', max_size=self.FILE_MAX_SIZE) as _csv_file: |
| 326 | - header = _csv_file.file_reader.readline() | 325 | + _csv_file.file_reader.readline() |
| 327 | line_num = 0 | 326 | line_num = 0 |
| 328 | - all_data = [''] * FileSliceHelper.CSV_LIMIT | ||
| 329 | for index, row in enumerate(self.read_file(_csv_file.file_reader)): | 327 | for index, row in enumerate(self.read_file(_csv_file.file_reader)): |
| 330 | line_num = index + 1 | 328 | line_num = index + 1 |
| 331 | if line_num > FileSliceHelper.CSV_LIMIT: | 329 | if line_num > FileSliceHelper.CSV_LIMIT: |
| 332 | - logging.error("The CSV file size limit is %d rows, and the size of the %s file " | 330 | + logging.error( |
| 333 | - "has exceeded the limit. ", FileSliceHelper.CSV_LIMIT, file_name_path) | 331 | + "The CSV file size limit is %d rows, and the size of the %s file has exceeded the limit. ", |
| 332 | + FileSliceHelper.CSV_LIMIT, | ||
| 333 | + file_name_path, | ||
| 334 | + ) | ||
| 334 | self._msproftx_device_data_dict = {} | 335 | self._msproftx_device_data_dict = {} |
| 335 | return | 336 | return |
| 336 | data = row.split(',') | 337 | data = row.split(',') |
| 337 | if len(data) < 3: | 338 | if len(data) < 3: |
| 338 | cnt += 1 | 339 | cnt += 1 |
| 339 | continue | 340 | continue |
| 340 | - self._msproftx_device_data_dict[data[0]] = {"start_time": data[1], "end_time": data[2], | 341 | + self._msproftx_device_data_dict.setdefault(data[0], []).append( |
| 341 | - "device_id": device_id} | 342 | + {"start_time": data[1], "end_time": data[2], "device_id": device_id} |
| 343 | + ) | ||
| 342 | if cnt != 0: | 344 | if cnt != 0: |
| 343 | - logging.error("The MSPROF_TX_DEVICE_CSV file contains %d lines whose length is " | 345 | + logging.error("The MSPROF_TX_DEVICE_CSV file contains %d lines whose length is less than 3. ", cnt) |
| 344 | - "less than 3. ", cnt) | ||
| 345 | 346 | ||
| 346 | def _update_msproftx_host_data(self, file_name_path: str, helper: FileSliceHelper): | 347 | def _update_msproftx_host_data(self, file_name_path: str, helper: FileSliceHelper): |
| 348 | + max_limit = FileSliceHelper.CSV_LIMIT | ||
| 349 | + data_map = self._msproftx_device_data_dict | ||
| 350 | + all_data = [] | ||
| 351 | + | ||
| 347 | with FileOpen(file_name_path, mode='r', max_size=self.FILE_MAX_SIZE) as _csv_file: | 352 | with FileOpen(file_name_path, mode='r', max_size=self.FILE_MAX_SIZE) as _csv_file: |
| 353 | + # handle csv header | ||
| 348 | header = _csv_file.file_reader.readline() | 354 | header = _csv_file.file_reader.readline() |
| 349 | if header and helper.check_header_is_empty(): | 355 | if header and helper.check_header_is_empty(): |
| 350 | - csv_header = [self.DEVICE_ID, *list(header.strip().split(','))[:-1], | 356 | + header_parts = header.strip().split(',')[:-1] |
| 351 | - self.MSPROFTX_DEVICE_START_TIME_HEADER, self.MSPROFTX_DEVICE_END_TIME_HEADER] | 357 | + csv_header = [ |
| 358 | + self.DEVICE_ID, | ||
| 359 | + *header_parts, | ||
| 360 | + self.MSPROFTX_DEVICE_START_TIME_HEADER, | ||
| 361 | + self.MSPROFTX_DEVICE_END_TIME_HEADER, | ||
| 362 | + ] | ||
| 352 | csv_header[-1] += '\n' | 363 | csv_header[-1] += '\n' |
| 353 | helper.set_header(csv_header) | 364 | helper.set_header(csv_header) |
| 354 | 365 | ||
| 355 | - line_num = 0 | 366 | + for row in self.read_file(_csv_file.file_reader): |
| 356 | - all_data = [''] * FileSliceHelper.CSV_LIMIT | 367 | + if len(all_data) >= max_limit: |
| 357 | - for index, row in enumerate(self.read_file(_csv_file.file_reader)): | 368 | + logging.error( |
| 358 | - line_num = index + 1 | 369 | + "The CSV file size limit is %d rows, and the size of the %s file has exceeded the limit. ", |
| 359 | - if line_num > FileSliceHelper.CSV_LIMIT: | 370 | + FileSliceHelper.CSV_LIMIT, |
| 360 | - logging.error("The CSV file size limit is %d rows, and the size of the %s file " | 371 | + file_name_path, |
| 361 | - "has exceeded the limit. ", FileSliceHelper.CSV_LIMIT, file_name_path) | 372 | + ) |
| 362 | return | 373 | return |
| 363 | - mark_id = row.strip().split(',')[-1] | 374 | + stripped_row = row.strip() |
| 364 | - row = row.rsplit(',', 1)[0] | 375 | + mark_id = stripped_row.split(',')[-1] |
| 365 | - if self._msproftx_device_data_dict.get(mark_id): | 376 | + origin_row = row.rsplit(',', 1)[0] |
| 366 | - row = ','.join([row, self._msproftx_device_data_dict[mark_id]["start_time"], | ||
| 367 | - self._msproftx_device_data_dict[mark_id]["end_time"]]) | ||
| 368 | - device_id = self._msproftx_device_data_dict[mark_id]["device_id"] | ||
| 369 | - else: | ||
| 370 | - row = ','.join([row, Constant.NA, Constant.NA + '\n']) | ||
| 371 | - device_id = "host" | ||
| 372 | - all_data[index] = f'{device_id},{row}' | ||
| 373 | - helper.insert_data(all_data[:line_num]) | ||
| 374 | 377 | ||
| 375 | - def _insert_summary_data(self, file_name_path: str, device_id: str, | 378 | + # match multi tx data with same mark id |
| 376 | - helper: FileSliceHelper): | 379 | + item_list = data_map.get(mark_id) |
| 380 | + if item_list: | ||
| 381 | + for item in item_list: | ||
| 382 | + start_t = item["start_time"] | ||
| 383 | + end_t = item["end_time"] | ||
| 384 | + dev_id = item["device_id"] | ||
| 385 | + new_row = ','.join([origin_row, start_t, end_t]) | ||
| 386 | + all_data.append(f"{dev_id},{new_row}") | ||
| 387 | + | ||
| 388 | + if len(all_data) >= max_limit: | ||
| 389 | + logging.error( | ||
| 390 | + "The CSV file size limit is %d rows, and the size of the %s file has exceeded the limit. ", | ||
| 391 | + FileSliceHelper.CSV_LIMIT, | ||
| 392 | + file_name_path, | ||
| 393 | + ) | ||
| 394 | + return | ||
| 395 | + | ||
| 396 | + else: | ||
| 397 | + new_row = ','.join([origin_row, Constant.NA, Constant.NA]) + '\n' | ||
| 398 | + all_data.append(f"host,{new_row}") | ||
| 399 | + | ||
| 400 | + helper.insert_data(all_data) | ||
| 401 | + | ||
| 402 | + def _insert_summary_data(self, file_name_path: str, device_id: str, helper: FileSliceHelper): | ||
| 377 | with FileOpen(file_name_path, mode='r', max_size=self.FILE_MAX_SIZE) as _csv_file: | 403 | with FileOpen(file_name_path, mode='r', max_size=self.FILE_MAX_SIZE) as _csv_file: |
| 378 | header = _csv_file.file_reader.readline() | 404 | header = _csv_file.file_reader.readline() |
| 379 | if header and helper.check_header_is_empty(): | 405 | if header and helper.check_header_is_empty(): |
| @@ -385,8 +411,11 @@ class MsprofOutputSummary: | |||
| 385 | for index, row in enumerate(self.read_file(_csv_file.file_reader)): | 411 | for index, row in enumerate(self.read_file(_csv_file.file_reader)): |
| 386 | line_num = index + 1 | 412 | line_num = index + 1 |
| 387 | if line_num > FileSliceHelper.CSV_LIMIT: | 413 | if line_num > FileSliceHelper.CSV_LIMIT: |
| 388 | - logging.error("The CSV file size limit is %d rows, and the size of the %s file " | 414 | + logging.error( |
| 389 | - "has exceeded the limit. ", FileSliceHelper.CSV_LIMIT, file_name_path) | 415 | + "The CSV file size limit is %d rows, and the size of the %s file has exceeded the limit. ", |
| 416 | + FileSliceHelper.CSV_LIMIT, | ||
| 417 | + file_name_path, | ||
| 418 | + ) | ||
| 390 | return | 419 | return |
| 391 | all_data[index] = f'{device_id},{row}' | 420 | all_data[index] = f'{device_id},{row}' |
| 392 | helper.insert_data(all_data[:line_num]) | 421 | helper.insert_data(all_data[:line_num]) |
| @@ -402,13 +431,14 @@ class MsprofOutputSummary: | |||
| 402 | processes = [] | 431 | processes = [] |
| 403 | for json_file in self.JSON_LIST: | 432 | for json_file in self.JSON_LIST: |
| 404 | try: | 433 | try: |
| 405 | - process = multiprocessing.Process(target=self._save_timeline_data, | 434 | + process = multiprocessing.Process(target=self._save_timeline_data, args=(json_file, sub_dirs)) |
| 406 | - args=(json_file, sub_dirs)) | ||
| 407 | process.start() | 435 | process.start() |
| 408 | processes.append(process) | 436 | processes.append(process) |
| 409 | except ProfException as err: | 437 | except ProfException as err: |
| 410 | - logging.error("Output: An exception occurs when multiple processes process the timeline file. " | 438 | + logging.error( |
| 411 | - "The error is %s", err) | 439 | + "Output: An exception occurs when multiple processes process the timeline file. The error is %s", |
| 440 | + err, | ||
| 441 | + ) | ||
| 412 | return | 442 | return |
| 413 | for process in processes: | 443 | for process in processes: |
| 414 | process.join() | 444 | process.join() |
| @@ -420,8 +450,9 @@ class MsprofOutputSummary: | |||
| 420 | sub_path = get_valid_sub_path(self._output, sub_dir, False) | 450 | sub_path = get_valid_sub_path(self._output, sub_dir, False) |
| 421 | if not DataCheckManager.contain_info_json_data(sub_path): | 451 | if not DataCheckManager.contain_info_json_data(sub_path): |
| 422 | continue | 452 | continue |
| 423 | - timeline_file_dict, slice_count = \ | 453 | + timeline_file_dict, slice_count = self._get_timeline_file_with_slice( |
| 424 | - self._get_timeline_file_with_slice(targe_name, sub_path, timeline_file_dict) | 454 | + targe_name, sub_path, timeline_file_dict |
| 455 | + ) | ||
| 425 | slice_max_count = max(slice_count, slice_max_count) | 456 | slice_max_count = max(slice_count, slice_max_count) |
| 426 | 457 | ||
| 427 | params = { | 458 | params = { |
| @@ -429,22 +460,23 @@ class MsprofOutputSummary: | |||
| 429 | StrConstant.PARAM_EXPORT_TYPE: MsProfCommonConstant.TIMELINE, | 460 | StrConstant.PARAM_EXPORT_TYPE: MsProfCommonConstant.TIMELINE, |
| 430 | StrConstant.PARAM_EXPORT_FORMAT: self._export_format, | 461 | StrConstant.PARAM_EXPORT_FORMAT: self._export_format, |
| 431 | StrConstant.PARAM_RESULT_DIR: self._output, | 462 | StrConstant.PARAM_RESULT_DIR: self._output, |
| 432 | - StrConstant.PARAM_EXPORT_DUMP_FOLDER: PathManager.MINDSTUDIO_PROFILER_OUTPUT | 463 | + StrConstant.PARAM_EXPORT_DUMP_FOLDER: PathManager.MINDSTUDIO_PROFILER_OUTPUT, |
| 433 | } | 464 | } |
| 434 | 465 | ||
| 435 | error = "Output: An exception occurs when multiple processes process the timeline file. The error is %s" | 466 | error = "Output: An exception occurs when multiple processes process the timeline file. The error is %s" |
| 436 | - pool = multiprocessing.Pool(processes=4) | 467 | + pool = multiprocessing.Pool(processes=4) # pylint: disable=R1732 |
| 437 | - is_need_slice = True if slice_max_count else False | 468 | + is_need_slice = bool(slice_max_count) |
| 438 | for index in range(slice_max_count + 1): | 469 | for index in range(slice_max_count + 1): |
| 439 | helper = FileSliceHelper(params, [], []) | 470 | helper = FileSliceHelper(params, [], []) |
| 440 | - pool.apply_async(func=self._insert_json_data, | 471 | + pool.apply_async( |
| 441 | - args=(timeline_file_dict.get(index, []), helper, is_need_slice, index), | 472 | + func=self._insert_json_data, |
| 442 | - error_callback=lambda error_info: logging.error(error, error_info)) | 473 | + args=(timeline_file_dict.get(index, []), helper, is_need_slice, index), |
| 474 | + error_callback=lambda error_info: logging.error(error, error_info), | ||
| 475 | + ) | ||
| 443 | pool.close() | 476 | pool.close() |
| 444 | pool.join() | 477 | pool.join() |
| 445 | 478 | ||
| 446 | - def _insert_json_data(self, file_list: list, helper: FileSliceHelper, | 479 | + def _insert_json_data(self, file_list: list, helper: FileSliceHelper, is_need_slice: bool, slice_index: int): |
| 447 | - is_need_slice: bool, slice_index: int): | ||
| 448 | """ | 480 | """ |
| 449 | 1 one device only have one "slice_0" | 481 | 1 one device only have one "slice_0" |
| 450 | 2 if only one device, then no need to merge file, copy will be better. | 482 | 2 if only one device, then no need to merge file, copy will be better. |
| @@ -455,10 +487,11 @@ class MsprofOutputSummary: | |||
| 455 | StrConstant.PARAM_DATA_TYPE: self._get_file_name(os.path.basename(file_list[0])), | 487 | StrConstant.PARAM_DATA_TYPE: self._get_file_name(os.path.basename(file_list[0])), |
| 456 | StrConstant.PARAM_EXPORT_TYPE: MsProfCommonConstant.TIMELINE, | 488 | StrConstant.PARAM_EXPORT_TYPE: MsProfCommonConstant.TIMELINE, |
| 457 | StrConstant.PARAM_EXPORT_FORMAT: self._export_format, | 489 | StrConstant.PARAM_EXPORT_FORMAT: self._export_format, |
| 458 | - StrConstant.PARAM_EXPORT_DUMP_FOLDER: PathManager.MINDSTUDIO_PROFILER_OUTPUT | 490 | + StrConstant.PARAM_EXPORT_DUMP_FOLDER: PathManager.MINDSTUDIO_PROFILER_OUTPUT, |
| 459 | } | 491 | } |
| 460 | - shutil.copy(file_list[0], | 492 | + shutil.copy( |
| 461 | - FileSliceHelper.make_export_file_name(params, slice_index, slice_switch=is_need_slice)) | 493 | + file_list[0], FileSliceHelper.make_export_file_name(params, slice_index, slice_switch=is_need_slice) |
| 494 | + ) | ||
| 462 | return | 495 | return |
| 463 | for _file_name in file_list: | 496 | for _file_name in file_list: |
| 464 | helper.insert_data(Utils.get_json_data(_file_name)) | 497 | helper.insert_data(Utils.get_json_data(_file_name)) |
| @@ -472,20 +505,20 @@ class MsprofOutputSummary: | |||
| 472 | 1:xxx_slice_1.json | 505 | 1:xxx_slice_1.json |
| 473 | """ | 506 | """ |
| 474 | slice_max_count = 0 | 507 | slice_max_count = 0 |
| 475 | - timeline_path = os.path.realpath( | 508 | + timeline_path = os.path.realpath(os.path.join(dir_path, MsProfCommonConstant.TIMELINE)) |
| 476 | - os.path.join(dir_path, MsProfCommonConstant.TIMELINE)) | ||
| 477 | if not os.path.exists(timeline_path): | 509 | if not os.path.exists(timeline_path): |
| 478 | return timeline_file_dict, slice_max_count | 510 | return timeline_file_dict, slice_max_count |
| 479 | file_list = os.listdir(timeline_path) | 511 | file_list = os.listdir(timeline_path) |
| 480 | for _file_name in self.get_newest_file_list(file_list, StrConstant.FILE_SUFFIX_JSON): | 512 | for _file_name in self.get_newest_file_list(file_list, StrConstant.FILE_SUFFIX_JSON): |
| 481 | - if (not _file_name.startswith(target_name) or | 513 | + if not _file_name.startswith(target_name) or ( |
| 482 | - (target_name == "msprof" and _file_name.startswith("msprof_tx"))): | 514 | + target_name == "msprof" and _file_name.startswith("msprof_tx") |
| 515 | + ): | ||
| 483 | continue | 516 | continue |
| 484 | match = re.search(r'_slice_\d+', _file_name) | 517 | match = re.search(r'_slice_\d+', _file_name) |
| 485 | file_name = os.path.join(timeline_path, _file_name) | 518 | file_name = os.path.join(timeline_path, _file_name) |
| 486 | slice_count = 0 | 519 | slice_count = 0 |
| 487 | if match and match.start() > 0: | 520 | if match and match.start() > 0: |
| 488 | - slice_count = _file_name[match.start() + self.SLICE_LEN: match.end()] | 521 | + slice_count = _file_name[match.start() + self.SLICE_LEN : match.end()] |
| 489 | if not is_number(slice_count): | 522 | if not is_number(slice_count): |
| 490 | logging.warning("This file name is invalid: %s", file_name) | 523 | logging.warning("This file name is invalid: %s", file_name) |
| 491 | continue | 524 | continue |
| @@ -507,11 +540,8 @@ class MsprofOutputSummary: | |||
| 507 | summary_set.add(self._get_file_name(ori_filename)) | 540 | summary_set.add(self._get_file_name(ori_filename)) |
| 508 | elif ori_filename.endswith(StrConstant.FILE_SUFFIX_JSON): | 541 | elif ori_filename.endswith(StrConstant.FILE_SUFFIX_JSON): |
| 509 | timeline_set.add(self._get_file_name(ori_filename)) | 542 | timeline_set.add(self._get_file_name(ori_filename)) |
| 510 | - | ||
| 511 | file_path = os.path.join(self._output_dir, self.README) | 543 | file_path = os.path.join(self._output_dir, self.README) |
| 512 | with FdOpen(file_path) as readme: | 544 | with FdOpen(file_path) as readme: |
| 513 | - context = self._get_readme_info(timeline_set, timeline_dict, | 545 | + context = self._get_readme_info(timeline_set, timeline_dict, StrConstant.FILE_SUFFIX_JSON) |
| 514 | - StrConstant.FILE_SUFFIX_JSON) | 546 | + context += self._get_readme_info(summary_set, summary_dict, StrConstant.FILE_SUFFIX_CSV) |
| 515 | - context += self._get_readme_info(summary_set, summary_dict, | ||
| 516 | - StrConstant.FILE_SUFFIX_CSV) | ||
| 517 | readme.write(context) | 547 | readme.write(context) |
| @@ -14,8 +14,13 @@ | |||
| 14 | # See the Mulan PSL v2 for more details. | 14 | # See the Mulan PSL v2 for more details. |
| 15 | # ------------------------------------------------------------------------- | 15 | # ------------------------------------------------------------------------- |
| 16 | 16 | ||
| 17 | +import logging | ||
| 18 | + | ||
| 19 | +from itertools import groupby | ||
| 20 | +from operator import attrgetter | ||
| 17 | from common_func.db_manager import DBManager | 21 | from common_func.db_manager import DBManager |
| 18 | from common_func.db_name_constant import DBNameConstant | 22 | from common_func.db_name_constant import DBNameConstant |
| 23 | +from common_func.info_conf_reader import InfoConfReader | ||
| 19 | from msconfig.config_manager import ConfigManager | 24 | from msconfig.config_manager import ConfigManager |
| 20 | from msmodel.interface.parser_model import ParserModel | 25 | from msmodel.interface.parser_model import ParserModel |
| 21 | from profiling_bean.db_dto.msproftx_dto import MsprofTxDto, MsprofTxExDto | 26 | from profiling_bean.db_dto.msproftx_dto import MsprofTxDto, MsprofTxExDto |
| @@ -26,9 +31,10 @@ class MsprofTxModel(ParserModel): | |||
| 26 | """ | 31 | """ |
| 27 | db operator for msproftx parser | 32 | db operator for msproftx parser |
| 28 | """ | 33 | """ |
| 34 | + | ||
| 29 | TABLES_PATH = ConfigManager.TABLES | 35 | TABLES_PATH = ConfigManager.TABLES |
| 30 | 36 | ||
| 31 | - def __init__(self: any, result_dir: str, db_name: str, table_list: list) -> None: | 37 | + def __init__(self: any, result_dir: str, db_name: str, table_list: list) -> None: # pylint: disable=W0246 |
| 32 | super().__init__(result_dir, db_name, table_list) | 38 | super().__init__(result_dir, db_name, table_list) |
| 33 | 39 | ||
| 34 | def flush(self: any, data_list: list) -> None: | 40 | def flush(self: any, data_list: list) -> None: |
| @@ -45,16 +51,19 @@ class MsprofTxModel(ParserModel): | |||
| 45 | """ | 51 | """ |
| 46 | if not DBManager.judge_table_exist(self.cur, DBNameConstant.TABLE_MSPROFTX): | 52 | if not DBManager.judge_table_exist(self.cur, DBNameConstant.TABLE_MSPROFTX): |
| 47 | return [] | 53 | return [] |
| 48 | - all_data_sql = f"select category, pid, tid, start_time, (end_time-start_time) as dur_time, payload_type, " \ | 54 | + all_data_sql = ( |
| 49 | - f"payload_value, message_type, message, event_type " \ | 55 | + "select category, pid, tid, start_time, (end_time-start_time) as dur_time, payload_type, " |
| 50 | - f"from {DBNameConstant.TABLE_MSPROFTX}" | 56 | + "payload_value, message_type, message, event_type from `{table}`" |
| 57 | + ).format(table=DBNameConstant.TABLE_MSPROFTX) # nosec B608 | ||
| 51 | return DBManager.fetch_all_data(self.cur, all_data_sql, dto_class=MsprofTxDto) | 58 | return DBManager.fetch_all_data(self.cur, all_data_sql, dto_class=MsprofTxDto) |
| 52 | 59 | ||
| 53 | def get_summary_data(self: any) -> list: | 60 | def get_summary_data(self: any) -> list: |
| 54 | if not DBManager.judge_table_exist(self.cur, DBNameConstant.TABLE_MSPROFTX): | 61 | if not DBManager.judge_table_exist(self.cur, DBNameConstant.TABLE_MSPROFTX): |
| 55 | return [] | 62 | return [] |
| 56 | - all_data_sql = f"select pid, tid, category, event_type, payload_type, payload_value, start_time, " \ | 63 | + all_data_sql = ( |
| 57 | - f"end_time, message_type, message from {DBNameConstant.TABLE_MSPROFTX}" | 64 | + "select pid, tid, category, event_type, payload_type, payload_value, start_time, " |
| 65 | + "end_time, message_type, message from `{table}`" | ||
| 66 | + ).format(table=DBNameConstant.TABLE_MSPROFTX) # nosec B608 | ||
| 58 | return DBManager.fetch_all_data(self.cur, all_data_sql) | 67 | return DBManager.fetch_all_data(self.cur, all_data_sql) |
| 59 | 68 | ||
| 60 | 69 | ||
| @@ -62,6 +71,7 @@ class MsprofTxExModel(ParserModel): | |||
| 62 | """ | 71 | """ |
| 63 | db operator for msproftx ex parser | 72 | db operator for msproftx ex parser |
| 64 | """ | 73 | """ |
| 74 | + | ||
| 65 | def __init__(self: any, result_dir: str, db_name: str, table_list: list): | 75 | def __init__(self: any, result_dir: str, db_name: str, table_list: list): |
| 66 | super().__init__(result_dir, db_name, table_list) | 76 | super().__init__(result_dir, db_name, table_list) |
| 67 | self.default_task_duration = 0 | 77 | self.default_task_duration = 0 |
| @@ -80,8 +90,10 @@ class MsprofTxExModel(ParserModel): | |||
| 80 | """ | 90 | """ |
| 81 | if not DBManager.judge_table_exist(self.cur, DBNameConstant.TABLE_MSPROFTX_EX): | 91 | if not DBManager.judge_table_exist(self.cur, DBNameConstant.TABLE_MSPROFTX_EX): |
| 82 | return [] | 92 | return [] |
| 83 | - all_data_sql = f"select pid, tid, event_type, start_time, (end_time-start_time) as dur_time, " \ | 93 | + all_data_sql = ( |
| 84 | - f"mark_id, message, domain from {DBNameConstant.TABLE_MSPROFTX_EX}" | 94 | + "select pid, tid, event_type, start_time, (end_time-start_time) as dur_time, " |
| 95 | + "mark_id, message, domain from `{table}`" | ||
| 96 | + ).format(table=DBNameConstant.TABLE_MSPROFTX_EX) # nosec B608 | ||
| 85 | return DBManager.fetch_all_data(self.cur, all_data_sql, dto_class=MsprofTxExDto) | 97 | return DBManager.fetch_all_data(self.cur, all_data_sql, dto_class=MsprofTxExDto) |
| 86 | 98 | ||
| 87 | def get_summary_data(self) -> list: | 99 | def get_summary_data(self) -> list: |
| @@ -90,29 +102,66 @@ class MsprofTxExModel(ParserModel): | |||
| 90 | """ | 102 | """ |
| 91 | if not DBManager.judge_table_exist(self.cur, DBNameConstant.TABLE_MSPROFTX_EX): | 103 | if not DBManager.judge_table_exist(self.cur, DBNameConstant.TABLE_MSPROFTX_EX): |
| 92 | return [] | 104 | return [] |
| 93 | - all_data_sql = f"select pid, tid, event_type, start_time, end_time, " \ | 105 | + all_data_sql = ( |
| 94 | - f"message, domain, mark_id from {DBNameConstant.TABLE_MSPROFTX_EX}" | 106 | + "select pid, tid, event_type, start_time, end_time, message, domain, mark_id from `{table}`" |
| 107 | + ).format(table=DBNameConstant.TABLE_MSPROFTX_EX) # nosec B608 | ||
| 95 | return DBManager.fetch_all_data(self.cur, all_data_sql) | 108 | return DBManager.fetch_all_data(self.cur, all_data_sql) |
| 96 | 109 | ||
| 97 | def get_device_data(self) -> list: | 110 | def get_device_data(self) -> list: |
| 98 | - | ||
| 99 | if not DBManager.judge_table_exist(self.cur, DBNameConstant.TABLE_STEP_TRACE): | 111 | if not DBManager.judge_table_exist(self.cur, DBNameConstant.TABLE_STEP_TRACE): |
| 100 | return [] | 112 | return [] |
| 101 | - all_data_sql = 'select index_id, timestamp, stream_id, task_id from {} ' \ | 113 | + all_data_sql = ( |
| 102 | - 'where tag_id = 11'.format(DBNameConstant.TABLE_STEP_TRACE) | 114 | + 'select index_id, timestamp, stream_id, task_id, tag_id from `{table}` where tag_id = 11 or tag_id = 12' |
| 115 | + ).format(table=DBNameConstant.TABLE_STEP_TRACE) # nosec B608 | ||
| 103 | task_list = DBManager.fetch_all_data(self.cur, all_data_sql, dto_class=MsproftxMarkDto) | 116 | task_list = DBManager.fetch_all_data(self.cur, all_data_sql, dto_class=MsproftxMarkDto) |
| 104 | if not task_list: | 117 | if not task_list: |
| 105 | return [] | 118 | return [] |
| 106 | res_task_data = [] | 119 | res_task_data = [] |
| 107 | - task_list.sort(key=lambda x: (x.index_id, x.timestamp)) | 120 | + |
| 108 | - res_task_data.append([task_list[0].index_id, task_list[0].timestamp, | 121 | + major_version, minor_version = InfoConfReader().get_cann_version() |
| 109 | - task_list[0].stream_id, task_list[0].task_id, self.default_task_duration]) | 122 | + # for cann version >= 9.1.0, tag id 11 is used for mark data, and tag id 12 is used for range data |
| 110 | - for i in range(1, len(task_list)): | 123 | + # so we handle these data differently according to cann version |
| 111 | - if task_list[i].index_id == task_list[i - 1].index_id: | 124 | + if (major_version, minor_version) >= (9, 1): |
| 112 | - # set range data duration | 125 | + range_data_list = [] |
| 113 | - res_task_data[-1][4] = task_list[i].timestamp - res_task_data[-1][1] | 126 | + for task in task_list: |
| 114 | - else: | 127 | + if task.tag_id == 11: |
| 115 | - res_task_data.append([task_list[i].index_id, task_list[i].timestamp, | 128 | + res_task_data.append([task.index_id, task.timestamp, task.stream_id, task.task_id, 0]) |
| 116 | - task_list[i].stream_id, task_list[i].task_id, self.default_task_duration]) | 129 | + else: |
| 130 | + range_data_list.append(task) | ||
| 131 | + range_data_list.sort(key=lambda x: (x.index_id, x.timestamp)) | ||
| 132 | + for index_id, group in groupby(range_data_list, key=attrgetter('index_id')): | ||
| 133 | + iterms = list(group) | ||
| 134 | + for i in range(0, len(iterms) - 1, 2): | ||
| 135 | + start_time = iterms[i].timestamp | ||
| 136 | + end_time = iterms[i + 1].timestamp | ||
| 137 | + duration = end_time - start_time | ||
| 138 | + res_task_data.append([index_id, start_time, iterms[i].stream_id, iterms[i].task_id, duration]) | ||
| 139 | + if len(iterms) % 2 != 0: | ||
| 140 | + logging.warning("Unpaired range data with index_id %d, odd count %d.", index_id, len(iterms)) | ||
| 141 | + else: | ||
| 142 | + task_list.sort(key=lambda x: (x.index_id, x.timestamp)) | ||
| 143 | + res_task_data.append( | ||
| 144 | + [ | ||
| 145 | + task_list[0].index_id, | ||
| 146 | + task_list[0].timestamp, | ||
| 147 | + task_list[0].stream_id, | ||
| 148 | + task_list[0].task_id, | ||
| 149 | + self.default_task_duration, | ||
| 150 | + ] | ||
| 151 | + ) | ||
| 152 | + for i in range(1, len(task_list)): | ||
| 153 | + if task_list[i].index_id == task_list[i - 1].index_id: | ||
| 154 | + # set range data duration | ||
| 155 | + res_task_data[-1][4] = task_list[i].timestamp - res_task_data[-1][1] | ||
| 156 | + else: | ||
| 157 | + res_task_data.append( | ||
| 158 | + [ | ||
| 159 | + task_list[i].index_id, | ||
| 160 | + task_list[i].timestamp, | ||
| 161 | + task_list[i].stream_id, | ||
| 162 | + task_list[i].task_id, | ||
| 163 | + self.default_task_duration, | ||
| 164 | + ] | ||
| 165 | + ) | ||
| 117 | 166 | ||
| 118 | return res_task_data | 167 | return res_task_data |
| @@ -26,6 +26,7 @@ class StepTraceOriginDto(metaclass=InstanceCheckMeta): | |||
| 26 | """ | 26 | """ |
| 27 | step trace origin DATA dto | 27 | step trace origin DATA dto |
| 28 | """ | 28 | """ |
| 29 | + | ||
| 29 | index_id: int = None | 30 | index_id: int = None |
| 30 | model_id: int = None | 31 | model_id: int = None |
| 31 | stream_id: int = None | 32 | stream_id: int = None |
| @@ -39,6 +40,7 @@ class StepTraceDto(metaclass=InstanceCheckMeta): | |||
| 39 | """ | 40 | """ |
| 40 | step trace dto | 41 | step trace dto |
| 41 | """ | 42 | """ |
| 43 | + | ||
| 42 | index_id: int = None | 44 | index_id: int = None |
| 43 | iter_id: int = None | 45 | iter_id: int = None |
| 44 | model_id: int = None | 46 | model_id: int = None |
| @@ -51,6 +53,7 @@ class TrainingTraceDto(metaclass=InstanceCheckMeta): | |||
| 51 | """ | 53 | """ |
| 52 | Training trace dto | 54 | Training trace dto |
| 53 | """ | 55 | """ |
| 56 | + | ||
| 54 | bp_end: float = None | 57 | bp_end: float = None |
| 55 | data_aug_bound: str = None | 58 | data_aug_bound: str = None |
| 56 | device_id: int = None | 59 | device_id: int = None |
| @@ -68,10 +71,12 @@ class MsproftxMarkDto(metaclass=InstanceCheckMeta): | |||
| 68 | """ | 71 | """ |
| 69 | msprofts ex mark dto | 72 | msprofts ex mark dto |
| 70 | """ | 73 | """ |
| 74 | + | ||
| 71 | index_id: int = 0 | 75 | index_id: int = 0 |
| 72 | timestamp: int = 0 | 76 | timestamp: int = 0 |
| 73 | stream_id: int = 0 | 77 | stream_id: int = 0 |
| 74 | task_id: int = 0 | 78 | task_id: int = 0 |
| 79 | + tag_id: int = 0 | ||
| 75 | 80 | ||
| 76 | 81 | ||
| 77 | Iteration = namedtuple("Iteration", ["model_id", "iteration_id", "iteration_count"]) | 82 | Iteration = namedtuple("Iteration", ["model_id", "iteration_id", "iteration_count"]) |
| @@ -81,6 +86,7 @@ class IterationRange(Iteration): | |||
| 81 | """ | 86 | """ |
| 82 | iteration range for model execute. | 87 | iteration range for model execute. |
| 83 | """ | 88 | """ |
| 89 | + | ||
| 84 | MAX_ITERATION_COUNT = 5 | 90 | MAX_ITERATION_COUNT = 5 |
| 85 | 91 | ||
| 86 | def __repr__(self): | 92 | def __repr__(self): |
| @@ -36,8 +36,13 @@ const std::string TABLE_NAME = "StepTrace"; | |||
| 36 | using DbDataType = std::vector<std::tuple<uint32_t, uint32_t, uint64_t, uint32_t, uint32_t, uint32_t>>; | 36 | using DbDataType = std::vector<std::tuple<uint32_t, uint32_t, uint64_t, uint32_t, uint32_t, uint32_t>>; |
| 37 | 37 | ||
| 38 | DbDataType DATA_A{{0, 4294967295, 26248923229230, 2, 10, 11}, | 38 | DbDataType DATA_A{{0, 4294967295, 26248923229230, 2, 10, 11}, |
| 39 | - {0, 4294967295, 26248923229240, 2, 10, 11}, | 39 | + {0, 4294967295, 26248923229240, 2, 10, 11}, // mark data in aclgraph replay scene with index id 0 |
| 40 | - {1, 4294967295, 26248923229340, 2, 14, 11}}; | 40 | + {1, 4294967295, 26248923229340, 2, 14, 12}, |
| 41 | + {1, 4294967295, 26248923229440, 2, 15, 12}, | ||
| 42 | + {2, 4294967295, 26248923229540, 2, 16, 12}, | ||
| 43 | + {2, 4294967295, 26248923229640, 2, 17, 12}, | ||
| 44 | + {1, 4294967295, 26248923229740, 2, 14, 12}, // range data in aclgraph replay scene with index id 1 | ||
| 45 | + {1, 4294967295, 26248923229840, 2, 15, 12}}; | ||
| 41 | } | 46 | } |
| 42 | 47 | ||
| 43 | class MsprofTxDeviceProcessorUTest : public testing::Test { | 48 | class MsprofTxDeviceProcessorUTest : public testing::Test { |
| @@ -73,7 +78,7 @@ protected: | |||
| 73 | } | 78 | } |
| 74 | }; | 79 | }; |
| 75 | 80 | ||
| 76 | -TEST_F(MsprofTxDeviceProcessorUTest, ShouldReturnTrueWhenProcessorRunSuccess) | 81 | +TEST_F(MsprofTxDeviceProcessorUTest, ShouldReturnTrueWhenProcessorRunSuccessWhenCannVersionIsNewerThan900) |
| 77 | { | 82 | { |
| 78 | DataInventory dataInventory; | 83 | DataInventory dataInventory; |
| 79 | auto processor = MsprofTxDeviceProcessor(PROF_PATH_A); | 84 | auto processor = MsprofTxDeviceProcessor(PROF_PATH_A); |
| @@ -86,11 +91,39 @@ TEST_F(MsprofTxDeviceProcessorUTest, ShouldReturnTrueWhenProcessorRunSuccess) | |||
| 86 | {"CPU", {{{"Frequency", "100.000000"}}}}, | 91 | {"CPU", {{{"Frequency", "100.000000"}}}}, |
| 87 | {"hostMonotonic", "651599377155020"}, | 92 | {"hostMonotonic", "651599377155020"}, |
| 88 | }; | 93 | }; |
| 89 | - MOCKER_CPP(&Analysis::Domain::Environment::Context::GetInfoByDeviceId).stubs().will(returnValue(record)); | 94 | + MOCKER_CPP(&Context::GetInfoByDeviceId).stubs().will(returnValue(record)); |
| 90 | MOCKER_CPP(&Context::GetSyscntConversionParams).stubs().will(returnValue(true)); | 95 | MOCKER_CPP(&Context::GetSyscntConversionParams).stubs().will(returnValue(true)); |
| 96 | + MOCKER_CPP(&Context::GetCannVersion).stubs().will(returnValue(std::vector<uint16_t>{9, 1, 0})); | ||
| 91 | EXPECT_TRUE(processor.Run(dataInventory, PROCESSOR_NAME_TASK)); | 97 | EXPECT_TRUE(processor.Run(dataInventory, PROCESSOR_NAME_TASK)); |
| 92 | auto res = dataInventory.GetPtr<std::vector<MsprofTxDeviceData>>(); | 98 | auto res = dataInventory.GetPtr<std::vector<MsprofTxDeviceData>>(); |
| 93 | - EXPECT_EQ(2ul, res->size()); | 99 | + EXPECT_EQ(5ul, res->size()); |
| 100 | + MOCKER_CPP(&Context::GetInfoByDeviceId).reset(); | ||
| 101 | + MOCKER_CPP(&Context::GetSyscntConversionParams).reset(); | ||
| 102 | + MOCKER_CPP(&Context::GetCannVersion).reset(); | ||
| 103 | +} | ||
| 104 | + | ||
| 105 | +TEST_F(MsprofTxDeviceProcessorUTest, ShouldReturnTrueWhenProcessorRunSuccessWhenCannVersionIsOlderThan900) | ||
| 106 | +{ | ||
| 107 | + DataInventory dataInventory; | ||
| 108 | + auto processor = MsprofTxDeviceProcessor(PROF_PATH_A); | ||
| 109 | + nlohmann::json record = { | ||
| 110 | + {"startCollectionTimeBegin", "1701069324370978"}, | ||
| 111 | + {"endCollectionTimeEnd", "1701069338159976"}, | ||
| 112 | + {"startClockMonotonicRaw", "10071129942580"}, | ||
| 113 | + {"pid", "10"}, | ||
| 114 | + {"hostCntvct", "65177261204177"}, | ||
| 115 | + {"CPU", {{{"Frequency", "100.000000"}}}}, | ||
| 116 | + {"hostMonotonic", "651599377155020"}, | ||
| 117 | + }; | ||
| 118 | + MOCKER_CPP(&Context::GetInfoByDeviceId).stubs().will(returnValue(record)); | ||
| 119 | + MOCKER_CPP(&Context::GetSyscntConversionParams).stubs().will(returnValue(true)); | ||
| 120 | + MOCKER_CPP(&Context::GetCannVersion).stubs().will(returnValue(std::vector<uint16_t>{})); | ||
| 121 | + EXPECT_TRUE(processor.Run(dataInventory, PROCESSOR_NAME_TASK)); | ||
| 122 | + auto res = dataInventory.GetPtr<std::vector<MsprofTxDeviceData>>(); | ||
| 123 | + EXPECT_EQ(3ul, res->size()); | ||
| 124 | + MOCKER_CPP(&Context::GetInfoByDeviceId).reset(); | ||
| 125 | + MOCKER_CPP(&Context::GetSyscntConversionParams).reset(); | ||
| 126 | + MOCKER_CPP(&Context::GetCannVersion).reset(); | ||
| 94 | } | 127 | } |
| 95 | 128 | ||
| 96 | TEST_F(MsprofTxDeviceProcessorUTest, ShouldReturnFalseWhenCheckFailed) | 129 | TEST_F(MsprofTxDeviceProcessorUTest, ShouldReturnFalseWhenCheckFailed) |
| @@ -1126,3 +1126,134 @@ TEST_F(ContextUTest, TestIsLevel0ShouldCheckProfLevel) | |||
| 1126 | EXPECT_TRUE(Context::GetInstance().IsLevel0(File::PathJoin({CONTEXT_DIR, TEST_DIR}))); | 1126 | EXPECT_TRUE(Context::GetInstance().IsLevel0(File::PathJoin({CONTEXT_DIR, TEST_DIR}))); |
| 1127 | MOCKER_CPP(&Context::GetInfoByDeviceId).reset(); | 1127 | MOCKER_CPP(&Context::GetInfoByDeviceId).reset(); |
| 1128 | } | 1128 | } |
| 1129 | + | ||
| 1130 | +TEST_F(ContextUTest, TestGetCannVersionShouldReturnEmptyWhenInfoIsEmpty) | ||
| 1131 | +{ | ||
| 1132 | + EXPECT_TRUE(File::DeleteFile(File::PathJoin({CONTEXT_DIR, TEST_DIR, HOST, INFO_JSON}))); | ||
| 1133 | + auto cannVersion = Context::GetInstance().GetCannVersion(HOST_ID, File::PathJoin({CONTEXT_DIR, TEST_DIR})); | ||
| 1134 | + EXPECT_EQ(cannVersion.size(), 0); | ||
| 1135 | +} | ||
| 1136 | + | ||
| 1137 | +TEST_F(ContextUTest, TestGetCannVersionShouldReturnRightValueWhenCannVersionInfoIsReleaseVersionFormat) | ||
| 1138 | +{ | ||
| 1139 | + EXPECT_TRUE(File::DeleteFile(File::PathJoin({CONTEXT_DIR, TEST_DIR, HOST, INFO_JSON}))); | ||
| 1140 | + MOCKER_CPP(&Context::CheckInfoValueIsValid).stubs().will(returnValue(true)); | ||
| 1141 | + // info.json | ||
| 1142 | + nlohmann::json info = { | ||
| 1143 | + {"cannVersion", "9.1.0"} | ||
| 1144 | + }; | ||
| 1145 | + FileWriter infoWriter(File::PathJoin({CONTEXT_DIR, TEST_DIR, HOST, INFO_JSON})); | ||
| 1146 | + infoWriter.WriteText(info.dump()); | ||
| 1147 | + | ||
| 1148 | + EXPECT_TRUE(Context::GetInstance().Load({File::PathJoin({CONTEXT_DIR, TEST_DIR})})); | ||
| 1149 | + auto cannVersion = Context::GetInstance().GetCannVersion(HOST_ID, File::PathJoin({CONTEXT_DIR, TEST_DIR})); | ||
| 1150 | + EXPECT_EQ(cannVersion[0], 9); | ||
| 1151 | + EXPECT_EQ(cannVersion[1], 1); | ||
| 1152 | + MOCKER_CPP(&Context::CheckInfoValueIsValid).reset(); | ||
| 1153 | +} | ||
| 1154 | + | ||
| 1155 | +TEST_F(ContextUTest, TestGetCannVersionShouldReturnRightValueWhenCannVersionInfoIsTestVersionFormat) | ||
| 1156 | +{ | ||
| 1157 | + EXPECT_TRUE(File::DeleteFile(File::PathJoin({CONTEXT_DIR, TEST_DIR, HOST, INFO_JSON}))); | ||
| 1158 | + MOCKER_CPP(&Context::CheckInfoValueIsValid).stubs().will(returnValue(true)); | ||
| 1159 | + // info.json | ||
| 1160 | + nlohmann::json info = { | ||
| 1161 | + {"cannVersion", "9.1.T100"} | ||
| 1162 | + }; | ||
| 1163 | + FileWriter infoWriter(File::PathJoin({CONTEXT_DIR, TEST_DIR, HOST, INFO_JSON})); | ||
| 1164 | + infoWriter.WriteText(info.dump()); | ||
| 1165 | + | ||
| 1166 | + EXPECT_TRUE(Context::GetInstance().Load({File::PathJoin({CONTEXT_DIR, TEST_DIR})})); | ||
| 1167 | + auto cannVersion = Context::GetInstance().GetCannVersion(HOST_ID, File::PathJoin({CONTEXT_DIR, TEST_DIR})); | ||
| 1168 | + EXPECT_EQ(cannVersion[0], 9); | ||
| 1169 | + EXPECT_EQ(cannVersion[1], 1); | ||
| 1170 | + MOCKER_CPP(&Context::CheckInfoValueIsValid).reset(); | ||
| 1171 | +} | ||
| 1172 | + | ||
| 1173 | +TEST_F(ContextUTest, TestGetCannVersionShouldReturnRightValueWhenCannVersionInfoIsBetaVersionFormat) | ||
| 1174 | +{ | ||
| 1175 | + EXPECT_TRUE(File::DeleteFile(File::PathJoin({CONTEXT_DIR, TEST_DIR, HOST, INFO_JSON}))); | ||
| 1176 | + MOCKER_CPP(&Context::CheckInfoValueIsValid).stubs().will(returnValue(true)); | ||
| 1177 | + // info.json | ||
| 1178 | + nlohmann::json info = { | ||
| 1179 | + {"cannVersion", "9.1.0-beta.0"} | ||
| 1180 | + }; | ||
| 1181 | + FileWriter infoWriter(File::PathJoin({CONTEXT_DIR, TEST_DIR, HOST, INFO_JSON})); | ||
| 1182 | + infoWriter.WriteText(info.dump()); | ||
| 1183 | + | ||
| 1184 | + EXPECT_TRUE(Context::GetInstance().Load({File::PathJoin({CONTEXT_DIR, TEST_DIR})})); | ||
| 1185 | + auto cannVersion = Context::GetInstance().GetCannVersion(HOST_ID, File::PathJoin({CONTEXT_DIR, TEST_DIR})); | ||
| 1186 | + EXPECT_EQ(cannVersion[0], 9); | ||
| 1187 | + EXPECT_EQ(cannVersion[1], 1); | ||
| 1188 | + MOCKER_CPP(&Context::CheckInfoValueIsValid).reset(); | ||
| 1189 | +} | ||
| 1190 | + | ||
| 1191 | +TEST_F(ContextUTest, TestGetCannVersionShouldReturnEmptyWhenCannVersionInfoNotExists) | ||
| 1192 | +{ | ||
| 1193 | + EXPECT_TRUE(File::DeleteFile(File::PathJoin({CONTEXT_DIR, TEST_DIR, HOST, INFO_JSON}))); | ||
| 1194 | + MOCKER_CPP(&Context::CheckInfoValueIsValid).stubs().will(returnValue(true)); | ||
| 1195 | + // info.json | ||
| 1196 | + nlohmann::json info = { | ||
| 1197 | + {"ai_core_profiling_mode", "task-based"}, | ||
| 1198 | + {"llc_profiling", "read"}, | ||
| 1199 | + {"profLevel", "l1"}, | ||
| 1200 | + }; | ||
| 1201 | + FileWriter infoWriter(File::PathJoin({CONTEXT_DIR, TEST_DIR, HOST, INFO_JSON})); | ||
| 1202 | + infoWriter.WriteText(info.dump()); | ||
| 1203 | + | ||
| 1204 | + EXPECT_TRUE(Context::GetInstance().Load({File::PathJoin({CONTEXT_DIR, TEST_DIR})})); | ||
| 1205 | + auto cannVersion = Context::GetInstance().GetCannVersion(HOST_ID, File::PathJoin({CONTEXT_DIR, TEST_DIR})); | ||
| 1206 | + EXPECT_EQ(cannVersion.size(), 0); | ||
| 1207 | + MOCKER_CPP(&Context::CheckInfoValueIsValid).reset(); | ||
| 1208 | +} | ||
| 1209 | + | ||
| 1210 | +TEST_F(ContextUTest, TestGetCannVersionShouldReturnEmptyWhenCannVersionInfoIsInvalidFormatOne) | ||
| 1211 | +{ | ||
| 1212 | + EXPECT_TRUE(File::DeleteFile(File::PathJoin({CONTEXT_DIR, TEST_DIR, HOST, INFO_JSON}))); | ||
| 1213 | + MOCKER_CPP(&Context::CheckInfoValueIsValid).stubs().will(returnValue(true)); | ||
| 1214 | + // info.json | ||
| 1215 | + nlohmann::json info = { | ||
| 1216 | + {"cannVersion", "invalid_version_format"} | ||
| 1217 | + }; | ||
| 1218 | + FileWriter infoWriter(File::PathJoin({CONTEXT_DIR, TEST_DIR, HOST, INFO_JSON})); | ||
| 1219 | + infoWriter.WriteText(info.dump()); | ||
| 1220 | + | ||
| 1221 | + EXPECT_TRUE(Context::GetInstance().Load({File::PathJoin({CONTEXT_DIR, TEST_DIR})})); | ||
| 1222 | + auto cannVersion = Context::GetInstance().GetCannVersion(HOST_ID, File::PathJoin({CONTEXT_DIR, TEST_DIR})); | ||
| 1223 | + EXPECT_EQ(cannVersion.size(), 0); | ||
| 1224 | + MOCKER_CPP(&Context::CheckInfoValueIsValid).reset(); | ||
| 1225 | +} | ||
| 1226 | + | ||
| 1227 | +TEST_F(ContextUTest, TestGetCannVersionShouldReturnEmptyWhenCannVersionInfoIsInvalidFormatTwo) | ||
| 1228 | +{ | ||
| 1229 | + EXPECT_TRUE(File::DeleteFile(File::PathJoin({CONTEXT_DIR, TEST_DIR, HOST, INFO_JSON}))); | ||
| 1230 | + MOCKER_CPP(&Context::CheckInfoValueIsValid).stubs().will(returnValue(true)); | ||
| 1231 | + // info.json | ||
| 1232 | + nlohmann::json info = { | ||
| 1233 | + {"cannVersion", "9.version0"} | ||
| 1234 | + }; | ||
| 1235 | + FileWriter infoWriter(File::PathJoin({CONTEXT_DIR, TEST_DIR, HOST, INFO_JSON})); | ||
| 1236 | + infoWriter.WriteText(info.dump()); | ||
| 1237 | + | ||
| 1238 | + EXPECT_TRUE(Context::GetInstance().Load({File::PathJoin({CONTEXT_DIR, TEST_DIR})})); | ||
| 1239 | + auto cannVersion = Context::GetInstance().GetCannVersion(HOST_ID, File::PathJoin({CONTEXT_DIR, TEST_DIR})); | ||
| 1240 | + EXPECT_EQ(cannVersion.size(), 0); | ||
| 1241 | + MOCKER_CPP(&Context::CheckInfoValueIsValid).reset(); | ||
| 1242 | +} | ||
| 1243 | + | ||
| 1244 | +TEST_F(ContextUTest, TestGetCannVersionShouldReturnEmptyWhenCannVersionInfoIsInvalidFormatThree) | ||
| 1245 | +{ | ||
| 1246 | + EXPECT_TRUE(File::DeleteFile(File::PathJoin({CONTEXT_DIR, TEST_DIR, HOST, INFO_JSON}))); | ||
| 1247 | + MOCKER_CPP(&Context::CheckInfoValueIsValid).stubs().will(returnValue(true)); | ||
| 1248 | + // info.json | ||
| 1249 | + nlohmann::json info = { | ||
| 1250 | + {"cannVersion", "9."} | ||
| 1251 | + }; | ||
| 1252 | + FileWriter infoWriter(File::PathJoin({CONTEXT_DIR, TEST_DIR, HOST, INFO_JSON})); | ||
| 1253 | + infoWriter.WriteText(info.dump()); | ||
| 1254 | + | ||
| 1255 | + EXPECT_TRUE(Context::GetInstance().Load({File::PathJoin({CONTEXT_DIR, TEST_DIR})})); | ||
| 1256 | + auto cannVersion = Context::GetInstance().GetCannVersion(HOST_ID, File::PathJoin({CONTEXT_DIR, TEST_DIR})); | ||
| 1257 | + EXPECT_EQ(cannVersion.size(), 0); | ||
| 1258 | + MOCKER_CPP(&Context::CheckInfoValueIsValid).reset(); | ||
| 1259 | +} | ||
| @@ -147,6 +147,21 @@ class TestInfoConfReader(unittest.TestCase): | |||
| 147 | InfoConfReader()._info_json = {} | 147 | InfoConfReader()._info_json = {} |
| 148 | self.assertEqual("", InfoConfReader().get_data_under_device('test_type')) | 148 | self.assertEqual("", InfoConfReader().get_data_under_device('test_type')) |
| 149 | 149 | ||
| 150 | + def test_get_cann_version_should_return_cann_version_when_cann_version_is_valid(self): | ||
| 151 | + InfoConfReader()._info_json = {"cannVersion": "9.1.0"} | ||
| 152 | + self.assertEqual((9, 1), InfoConfReader().get_cann_version()) | ||
| 153 | + InfoConfReader()._info_json = {"cannVersion": "9.1.0-beta.0"} | ||
| 154 | + self.assertEqual((9, 1), InfoConfReader().get_cann_version()) | ||
| 155 | + InfoConfReader()._info_json = {"cannVersion": "9.1.T106"} | ||
| 156 | + self.assertEqual((9, 1), InfoConfReader().get_cann_version()) | ||
| 157 | + InfoConfReader()._info_json = {} | ||
| 158 | + | ||
| 159 | + def test_get_cann_version_should_return_0_when_cann_version_is_invalid(self): | ||
| 160 | + InfoConfReader()._info_json = {"cannVersion": "invalid_version"} | ||
| 161 | + self.assertEqual((0, 0), InfoConfReader().get_cann_version()) | ||
| 162 | + InfoConfReader()._info_json = {} | ||
| 163 | + self.assertEqual((0, 0), InfoConfReader().get_cann_version()) | ||
| 164 | + | ||
| 150 | 165 | ||
| 151 | if __name__ == '__main__': | 166 | if __name__ == '__main__': |
| 152 | unittest.main() | 167 | unittest.main() |
| @@ -16,6 +16,7 @@ | |||
| 16 | import unittest | 16 | import unittest |
| 17 | from unittest import mock | 17 | from unittest import mock |
| 18 | 18 | ||
| 19 | +from common_func.info_conf_reader import InfoConfReader | ||
| 19 | from msmodel.msproftx.msproftx_model import MsprofTxModel, MsprofTxExModel | 20 | from msmodel.msproftx.msproftx_model import MsprofTxModel, MsprofTxExModel |
| 20 | from profiling_bean.db_dto.step_trace_dto import MsproftxMarkDto | 21 | from profiling_bean.db_dto.step_trace_dto import MsproftxMarkDto |
| 21 | 22 | ||
| @@ -80,22 +81,59 @@ class TestMsprofTxExModel(unittest.TestCase): | |||
| 80 | res = check.get_device_data() | 81 | res = check.get_device_data() |
| 81 | self.assertEqual(res, []) | 82 | self.assertEqual(res, []) |
| 82 | 83 | ||
| 83 | - def test_get_device_summary_data_should_return_true_when_last_data_is_range(self): | 84 | + def test_get_device_summary_data_should_return_true_when_last_data_is_range_and_cann_version_before_910(self): |
| 84 | with mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True), \ | 85 | with mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True), \ |
| 85 | mock.patch(NAMESPACE + '.DBManager.fetch_all_data', return_value=[ | 86 | mock.patch(NAMESPACE + '.DBManager.fetch_all_data', return_value=[ |
| 86 | - MsproftxMarkDto(0, 10, 0, 0), | 87 | + MsproftxMarkDto(index_id=0, timestamp=10, stream_id=0, task_id=0, tag_id=11), |
| 87 | - MsproftxMarkDto(1, 11, 0, 1), | 88 | + MsproftxMarkDto(index_id=1, timestamp=11, stream_id=0, task_id=1, tag_id=11), |
| 88 | - MsproftxMarkDto(1, 12, 0, 2)]): | 89 | + MsproftxMarkDto(index_id=1, timestamp=12, stream_id=0, task_id=2, tag_id=11)]): |
| 90 | + InfoConfReader()._info_json = {'cannVersion': '9.0.0'} | ||
| 89 | check = MsprofTxExModel('test6', 'step_trace.db', ['StepTrace']) | 91 | check = MsprofTxExModel('test6', 'step_trace.db', ['StepTrace']) |
| 90 | res = check.get_device_data() | 92 | res = check.get_device_data() |
| 91 | self.assertEqual(res, [[0, 10, 0, 0, 0], [1, 11, 0, 1, 1]]) | 93 | self.assertEqual(res, [[0, 10, 0, 0, 0], [1, 11, 0, 1, 1]]) |
| 92 | 94 | ||
| 93 | - def test_get_device_summary_data_should_return_true_when_last_data_is_mark(self): | 95 | + def test_get_device_summary_data_should_return_true_when_last_data_is_mark_and_cann_version_before_910(self): |
| 94 | with mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True), \ | 96 | with mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True), \ |
| 95 | mock.patch(NAMESPACE + '.DBManager.fetch_all_data', return_value=[ | 97 | mock.patch(NAMESPACE + '.DBManager.fetch_all_data', return_value=[ |
| 96 | - MsproftxMarkDto(0, 10, 0, 0), | 98 | + MsproftxMarkDto(index_id=0, timestamp=10, stream_id=0, task_id=0, tag_id=11), |
| 97 | - MsproftxMarkDto(0, 11, 0, 1), | 99 | + MsproftxMarkDto(index_id=0, timestamp=11, stream_id=0, task_id=1, tag_id=11), |
| 98 | - MsproftxMarkDto(1, 12, 0, 2)]): | 100 | + MsproftxMarkDto(index_id=1, timestamp=12, stream_id=0, task_id=2, tag_id=11)]): |
| 101 | + InfoConfReader()._info_json = {'cannVersion': '9.0.0'} | ||
| 99 | check = MsprofTxExModel('test7', 'step_trace.db', ['StepTrace']) | 102 | check = MsprofTxExModel('test7', 'step_trace.db', ['StepTrace']) |
| 100 | res = check.get_device_data() | 103 | res = check.get_device_data() |
| 101 | - self.assertEqual(res, [[0, 10, 0, 0, 1], [1, 12, 0, 2, 0]]) | 104 | + self.assertEqual(res, [[0, 10, 0, 0, 1], [1, 12, 0, 2, 0]]) |
| 105 | + | ||
| 106 | + def test_get_device_summary_data_should_return_true_when_last_data_is_range_and_cann_version_after_910(self): | ||
| 107 | + with mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True), \ | ||
| 108 | + mock.patch(NAMESPACE + '.DBManager.fetch_all_data', return_value=[ | ||
| 109 | + MsproftxMarkDto(index_id=0, timestamp=10, stream_id=0, task_id=0, tag_id=11), | ||
| 110 | + MsproftxMarkDto(index_id=1, timestamp=11, stream_id=0, task_id=1, tag_id=12), | ||
| 111 | + MsproftxMarkDto(index_id=1, timestamp=12, stream_id=0, task_id=2, tag_id=12)]): | ||
| 112 | + InfoConfReader()._info_json = {'cannVersion': '9.1.0'} | ||
| 113 | + check = MsprofTxExModel('test8', 'step_trace.db', ['StepTrace']) | ||
| 114 | + res = check.get_device_data() | ||
| 115 | + self.assertEqual(res, [[0, 10, 0, 0, 0], [1, 11, 0, 1, 1]]) | ||
| 116 | + | ||
| 117 | + def test_get_device_summary_data_should_return_true_when_last_data_is_mark_and_cann_version_after_910(self): | ||
| 118 | + with mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True), \ | ||
| 119 | + mock.patch(NAMESPACE + '.DBManager.fetch_all_data', return_value=[ | ||
| 120 | + MsproftxMarkDto(index_id=0, timestamp=10, stream_id=0, task_id=0, tag_id=12), | ||
| 121 | + MsproftxMarkDto(index_id=0, timestamp=11, stream_id=0, task_id=1, tag_id=12), | ||
| 122 | + MsproftxMarkDto(index_id=1, timestamp=12, stream_id=0, task_id=2, tag_id=11)]): | ||
| 123 | + InfoConfReader()._info_json = {'cannVersion': '9.1.0'} | ||
| 124 | + check = MsprofTxExModel('test9', 'step_trace.db', ['StepTrace']) | ||
| 125 | + res = check.get_device_data() | ||
| 126 | + self.assertEqual(res, [[1, 12, 0, 2, 0], [0, 10, 0, 0, 1]]) | ||
| 127 | + | ||
| 128 | + def test_get_device_summary_data_should_return_true_when_multiple_data_in_same_id_and_cann_version_after_910(self): | ||
| 129 | + with mock.patch(NAMESPACE + '.DBManager.judge_table_exist', return_value=True), \ | ||
| 130 | + mock.patch(NAMESPACE + '.DBManager.fetch_all_data', return_value=[ | ||
| 131 | + MsproftxMarkDto(index_id=0, timestamp=10, stream_id=0, task_id=0, tag_id=12), | ||
| 132 | + MsproftxMarkDto(index_id=0, timestamp=11, stream_id=0, task_id=1, tag_id=12), | ||
| 133 | + MsproftxMarkDto(index_id=0, timestamp=12, stream_id=0, task_id=2, tag_id=12), | ||
| 134 | + MsproftxMarkDto(index_id=0, timestamp=13, stream_id=0, task_id=3, tag_id=12), | ||
| 135 | + MsproftxMarkDto(index_id=1, timestamp=14, stream_id=0, task_id=2, tag_id=11)]): | ||
| 136 | + InfoConfReader()._info_json = {'cannVersion': '9.1.0'} | ||
| 137 | + check = MsprofTxExModel('test10', 'step_trace.db', ['StepTrace']) | ||
| 138 | + res = check.get_device_data() | ||
| 139 | + self.assertEqual(res, [[1, 14, 0, 2, 0], [0, 10, 0, 0, 1], [0, 12, 0, 2, 1]]) | ||
[review] 问题：oriData 参数类型为 OriMsprofTxDeviceData &，但该函数只遍历数据，不修改 oriData。 影响：接口语义不清晰，调用方无法判断函数是否会修改原始数据。 建议：将参数改为 const OriMsprofTxDeviceData &oriData。