已合并
bugfix: aicpu交付件导出增加deviceId，保持和python解析结果一致 #497
wangzixuan创建于 22 天前
bugfix: aicpu交付件导出增加deviceId，保持和python解析结果一致 #497
已合并
共 14 个文件变更+83-46
| @@ -67,15 +67,16 @@ bool AicpuAssembler::WriteAicpuCsv(const std::vector<AicpuSummaryData> &data) | |||
| 67 | { | 67 | { |
| 68 | return false; | 68 | return false; |
| 69 | } | 69 | } |
| 70 | - headers_ = {"Timestamp(us)", "Node", "Compute_time(us)", "Memcpy_time(us)", "Task_time(us)", | 70 | + headers_ = {"Device_id", "Timestamp(us)", "Node", "Compute_time(us)", "Memcpy_time(us)", |
| 71 | - "Dispatch_time(us)", "Total_time(us)", "Stream ID", "Task ID"}; | 71 | + "Task_time(us)", "Dispatch_time(us)", "Total_time(us)", "Stream ID", "Task ID"}; |
| 72 | res_.clear(); | 72 | res_.clear(); |
| 73 | for (const auto &item : data) | 73 | for (const auto &item : data) |
| 74 | { | 74 | { |
| 75 | res_.emplace_back(std::vector<std::string>{ | 75 | res_.emplace_back(std::vector<std::string>{ |
| 76 | - DivideByPowersOfTenWithPrecision(item.timestampNs), item.nodeName, DoubleToStr(item.computeTimeUs), | 76 | + std::to_string(item.deviceId), DivideByPowersOfTenWithPrecision(item.timestampNs, true), item.nodeName, |
| 77 | - DoubleToStr(item.memcpyTimeUs), DoubleToStr(item.taskTimeUs), DoubleToStr(item.dispatchTimeUs), | 77 | + DoubleToStr(item.computeTimeUs), DoubleToStr(item.memcpyTimeUs), DoubleToStr(item.taskTimeUs), |
| 78 | - DoubleToStr(item.totalTimeUs), std::to_string(item.streamId), std::to_string(item.taskId)}); | 78 | + DoubleToStr(item.dispatchTimeUs), DoubleToStr(item.totalTimeUs), std::to_string(item.streamId), |
| 79 | + std::to_string(item.taskId)}); | ||
| 79 | } | 80 | } |
| 80 | WriteToFile(File::PathJoin({profPath_, Analysis::Common::OUTPUT_PATH, AICPU_NAME}), {}); | 81 | WriteToFile(File::PathJoin({profPath_, Analysis::Common::OUTPUT_PATH, AICPU_NAME}), {}); |
| 81 | return true; | 82 | return true; |
| @@ -87,11 +88,12 @@ bool AicpuAssembler::WriteDpCsv(const std::vector<AicpuDpData> &data) | |||
| 87 | { | 88 | { |
| 88 | return false; | 89 | return false; |
| 89 | } | 90 | } |
| 90 | - headers_ = {"Timestamp(us)", "Action", "Source", "Cached Buffer Size"}; | 91 | + headers_ = {"Device_id", "Timestamp(us)", "Action", "Source", "Cached Buffer Size"}; |
L | |||
| 91 | res_.clear(); | 92 | res_.clear(); |
| 92 | for (const auto &item : data) | 93 | for (const auto &item : data) |
| 93 | { | 94 | { |
| 94 | - res_.emplace_back(std::vector<std::string>{DivideByPowersOfTenWithPrecision(item.timestamp), item.action, | 95 | + res_.emplace_back(std::vector<std::string>{std::to_string(item.deviceId), |
| 96 | + DivideByPowersOfTenWithPrecision(item.timestamp), item.action, | ||
| 95 | item.source, std::to_string(item.bufferSize)}); | 97 | item.source, std::to_string(item.bufferSize)}); |
| 96 | } | 98 | } |
| 97 | WriteToFile(File::PathJoin({profPath_, Analysis::Common::OUTPUT_PATH, AICPU_DP_NAME}), {}); | 99 | WriteToFile(File::PathJoin({profPath_, Analysis::Common::OUTPUT_PATH, AICPU_DP_NAME}), {}); |
| @@ -104,12 +106,13 @@ bool AicpuAssembler::WriteMiCsv(const std::vector<AicpuMiData> &data) | |||
| 104 | { | 106 | { |
| 105 | return false; | 107 | return false; |
| 106 | } | 108 | } |
| 107 | - headers_ = {"Node Name", "Start Time(us)", "End Time(us)", "Queue Size"}; | 109 | + headers_ = {"Device_id", "Node Name", "Start Time(us)", "End Time(us)", "Queue Size"}; |
| 108 | res_.clear(); | 110 | res_.clear(); |
| 109 | for (const auto &item : data) | 111 | for (const auto &item : data) |
| 110 | { | 112 | { |
| 111 | - res_.emplace_back(std::vector<std::string>{item.nodeName, std::to_string(item.startTime), | 113 | + res_.emplace_back(std::vector<std::string>{std::to_string(item.deviceId), item.nodeName, |
| 112 | - std::to_string(item.endTime), std::to_string(item.queueSize)}); | 114 | + std::to_string(item.startTime), std::to_string(item.endTime), |
| 115 | + std::to_string(item.queueSize)}); | ||
| 113 | } | 116 | } |
| 114 | WriteToFile(File::PathJoin({profPath_, Analysis::Common::OUTPUT_PATH, AICPU_MI_NAME}), {}); | 117 | WriteToFile(File::PathJoin({profPath_, Analysis::Common::OUTPUT_PATH, AICPU_MI_NAME}), {}); |
| 115 | return true; | 118 | return true; |
| @@ -49,7 +49,7 @@ uint8_t NpuMemoryAssembler::AssembleData(Analysis::Infra::DataInventory &dataInv | |||
| 49 | std::to_string(item.ddr / Analysis::Common::BYTE_SIZE), | 49 | std::to_string(item.ddr / Analysis::Common::BYTE_SIZE), |
| 50 | std::to_string(item.hbm / Analysis::Common::BYTE_SIZE), | 50 | std::to_string(item.hbm / Analysis::Common::BYTE_SIZE), |
| 51 | std::to_string(item.memory / Analysis::Common::BYTE_SIZE), | 51 | std::to_string(item.memory / Analysis::Common::BYTE_SIZE), |
| 52 | - DivideByPowersOfTenWithPrecision(item.timestamp) + "\t"}); | 52 | + DivideByPowersOfTenWithPrecision(item.timestamp, true)}); |
| 53 | } | 53 | } |
| 54 | if (res_.empty()) | 54 | if (res_.empty()) |
| 55 | { | 55 | { |
| @@ -67,7 +67,7 @@ uint8_t NpuModuleMemAssembler::AssembleData(DataInventory &dataInventory) | |||
| 67 | res_.emplace_back(std::vector<std::string>{ | 67 | res_.emplace_back(std::vector<std::string>{ |
| 68 | std::to_string(item.deviceId), | 68 | std::to_string(item.deviceId), |
| 69 | moduleMap_.find(item.moduleId) != moduleMap_.end() ? moduleMap_.at(item.moduleId) : UNKNOWN, | 69 | moduleMap_.find(item.moduleId) != moduleMap_.end() ? moduleMap_.at(item.moduleId) : UNKNOWN, |
| 70 | - DivideByPowersOfTenWithPrecision(item.timestamp) + "\t", | 70 | + DivideByPowersOfTenWithPrecision(item.timestamp, true), |
| 71 | std::to_string(item.totalReserved / Analysis::Common::BYTE_SIZE), item.deviceType}); | 71 | std::to_string(item.totalReserved / Analysis::Common::BYTE_SIZE), item.deviceType}); |
| 72 | } | 72 | } |
| 73 | if (res_.empty()) | 73 | if (res_.empty()) |
| @@ -287,11 +287,13 @@ void OpSummaryAssembler::CalculateWaitTime() | |||
| 287 | void OpSummaryAssembler::WriteToFile(const std::string &fileName, const std::set<int> &maskCols) | 287 | void OpSummaryAssembler::WriteToFile(const std::string &fileName, const std::set<int> &maskCols) |
| 288 | { | 288 | { |
| 289 | auto timeIndex = GetIndexForVec(headers_, TASK_START_TIME); | 289 | auto timeIndex = GetIndexForVec(headers_, TASK_START_TIME); |
| 290 | + // TASK_START_TIME 是绝对时刻(us)，追加\t强制Excel按文本处理避免精度丢失； | ||
| 291 | + // 必须在CalculateWaitTime完成数字解析/排序后再打标，否则尾部\t会导致StrToDouble失败 | ||
| 290 | for (auto &row : res_) | 292 | for (auto &row : res_) |
| 291 | { | 293 | { |
| 292 | if (timeIndex < static_cast<int>(row.size())) | 294 | if (timeIndex < static_cast<int>(row.size())) |
| 293 | { | 295 | { |
| 294 | - row[timeIndex].append("\t"); | 296 | + row[timeIndex] = FormatHighPrecisionForCsv(row[timeIndex]); |
L 【review】 【问题描述】WriteToFile 中 timeIndex 未排除 -1，存在 row[-1] 越界风险（UB） op_summary_assembler.cpp 中 GetIndexForVec 未命中返回 INVALID_VALUE = -1 （ common_constant.h ），此时 if (timeIndex < row.size()) 对 -1 恒为真，执行 row[timeIndex] = ... 即 row[-1] 越界访问。当前 TASK_START_TIME 恒在 BASE_HEADER 中（索引恒为 8）故不可达，但 guard 逻辑无法防御 -1 ，属潜伏性缺陷；本次提交修改该行时沿用了不安全写法。 【问题分类】健壮性（潜在内存安全） 【修改建议】guard 增加下界判断： if (timeIndex >= 0 && timeIndex < static_cast   wangzixuan 21 天前 评论： 21 天前 评论： | |||
| 295 | } | 297 | } |
| 296 | } | 298 | } |
| 297 | 299 | ||
| @@ -131,15 +131,14 @@ void SummaryStepTraceAssembler::AssembleStepTraceData(const std::vector<TrainTra | |||
| 131 | return; | 131 | return; |
| 132 | } | 132 | } |
| 133 | 133 | ||
| 134 | - const std::string DIVIDE_CHAR = "\t"; | ||
| 135 | for (auto &trainTraceDatum : trainTraceData) | 134 | for (auto &trainTraceDatum : trainTraceData) |
| 136 | { | 135 | { |
| 137 | TraceId traceId = {trainTraceDatum.modelId, trainTraceDatum.iterEnd}; | 136 | TraceId traceId = {trainTraceDatum.modelId, trainTraceDatum.iterEnd}; |
| 138 | std::vector<std::string> row = {std::to_string(trainTraceDatum.deviceId), | 137 | std::vector<std::string> row = {std::to_string(trainTraceDatum.deviceId), |
| 139 | std::to_string(trainTraceDatum.indexId), | 138 | std::to_string(trainTraceDatum.indexId), |
| 140 | - DivideByPowersOfTenWithPrecision(trainTraceDatum.fpStart) + DIVIDE_CHAR, | 139 | + DivideByPowersOfTenWithPrecision(trainTraceDatum.fpStart, true), |
| 141 | - DivideByPowersOfTenWithPrecision(trainTraceDatum.bpEnd) + DIVIDE_CHAR, | 140 | + DivideByPowersOfTenWithPrecision(trainTraceDatum.bpEnd, true), |
| 142 | - DivideByPowersOfTenWithPrecision(trainTraceDatum.iterEnd) + DIVIDE_CHAR, | 141 | + DivideByPowersOfTenWithPrecision(trainTraceDatum.iterEnd, true), |
| 143 | DivideByPowersOfTenWithPrecision(trainTraceDatum.iterTime), | 142 | DivideByPowersOfTenWithPrecision(trainTraceDatum.iterTime), |
| 144 | DivideByPowersOfTenWithPrecision(trainTraceDatum.fpBpTime), | 143 | DivideByPowersOfTenWithPrecision(trainTraceDatum.fpBpTime), |
| 145 | DivideByPowersOfTenWithPrecision(trainTraceDatum.gradRefreshBound), | 144 | DivideByPowersOfTenWithPrecision(trainTraceDatum.gradRefreshBound), |
| @@ -151,7 +150,7 @@ void SummaryStepTraceAssembler::AssembleStepTraceData(const std::vector<TrainTra | |||
| 151 | int count = 0; | 150 | int count = 0; |
| 152 | for (auto &allReduceData : it->second) | 151 | for (auto &allReduceData : it->second) |
| 153 | { | 152 | { |
| 154 | - row.emplace_back(allReduceData.first + DIVIDE_CHAR); | 153 | + row.emplace_back(FormatHighPrecisionForCsv(allReduceData.first)); |
| 155 | row.emplace_back(allReduceData.second); | 154 | row.emplace_back(allReduceData.second); |
| 156 | ++count; | 155 | ++count; |
| 157 | } | 156 | } |
| @@ -193,7 +193,6 @@ void TaskTimeAssembler::AssembleTaskTime(const std::vector<AscendTaskData>& asce | |||
| 193 | return; | 193 | return; |
| 194 | } | 194 | } |
| 195 | 195 | ||
| 196 | - const std::string DIVIDE_CHAR = "\t"; | ||
| 197 | for (auto& ascendTaskDatum : FilterAscendTaskData(ascendTaskData)) | 196 | for (auto& ascendTaskDatum : FilterAscendTaskData(ascendTaskData)) |
| 198 | { | 197 | { |
| 199 | TaskId taskId{ascendTaskDatum.streamId, ascendTaskDatum.batchId, ascendTaskDatum.taskId, | 198 | TaskId taskId{ascendTaskDatum.streamId, ascendTaskDatum.batchId, ascendTaskDatum.taskId, |
| @@ -222,8 +221,8 @@ void TaskTimeAssembler::AssembleTaskTime(const std::vector<AscendTaskData>& asce | |||
| 222 | std::to_string(taskId.streamId), | 221 | std::to_string(taskId.streamId), |
| 223 | std::to_string(taskId.taskId), | 222 | std::to_string(taskId.taskId), |
| 224 | DivideByPowersOfTenWithPrecision(static_cast<uint64_t>(ascendTaskDatum.duration)), | 223 | DivideByPowersOfTenWithPrecision(static_cast<uint64_t>(ascendTaskDatum.duration)), |
| 225 | - DivideByPowersOfTenWithPrecision(ascendTaskDatum.timestamp) + DIVIDE_CHAR, | 224 | + DivideByPowersOfTenWithPrecision(ascendTaskDatum.timestamp, true), |
| 226 | - DivideByPowersOfTenWithPrecision(ascendTaskDatum.end) + DIVIDE_CHAR}; | 225 | + DivideByPowersOfTenWithPrecision(ascendTaskDatum.end, true)}; |
| 227 | res_.emplace_back(row); | 226 | res_.emplace_back(row); |
| 228 | } | 227 | } |
| 229 | } | 228 | } |
| @@ -95,8 +95,8 @@ bool AicpuProcessor::ProcessSingleDevice(const std::string &devicePath, std::vec | |||
| 95 | return false; | 95 | return false; |
| 96 | } | 96 | } |
| 97 | bool flag = LoadAiCpuData(devicePath, deviceId, timeRecord, summaryData); | 97 | bool flag = LoadAiCpuData(devicePath, deviceId, timeRecord, summaryData); |
| 98 | - flag = LoadDpData(devicePath, timeRecord, dpData) && flag; | 98 | + flag = LoadDpData(devicePath, deviceId, timeRecord, dpData) && flag; |
| 99 | - flag = LoadMiData(devicePath, miData) && flag; | 99 | + flag = LoadMiData(devicePath, deviceId, miData) && flag; |
| 100 | return flag; | 100 | return flag; |
| 101 | } | 101 | } |
| 102 | 102 | ||
| @@ -157,7 +157,7 @@ bool AicpuProcessor::LoadAiCpuData(const std::string &devicePath, uint16_t devic | |||
| 157 | return true; | 157 | return true; |
| 158 | } | 158 | } |
| 159 | 159 | ||
| 160 | -bool AicpuProcessor::LoadDpData(const std::string &devicePath, const ProfTimeRecord &timeRecord, | 160 | +bool AicpuProcessor::LoadDpData(const std::string &devicePath, uint16_t deviceId, const ProfTimeRecord &timeRecord, |
| 161 | std::vector<AicpuDpData> &dpData) | 161 | std::vector<AicpuDpData> &dpData) |
| 162 | { | 162 | { |
| 163 | DBInfo dpDB(DB_NAME_AI_CPU, TABLE_NAME_AI_CPU_DP); | 163 | DBInfo dpDB(DB_NAME_AI_CPU, TABLE_NAME_AI_CPU_DP); |
| @@ -184,6 +184,7 @@ bool AicpuProcessor::LoadDpData(const std::string &devicePath, const ProfTimeRec | |||
| 184 | AicpuDpData data; | 184 | AicpuDpData data; |
| 185 | double rawTimestamp = 0; | 185 | double rawTimestamp = 0; |
| 186 | std::tie(rawTimestamp, data.action, data.source, data.bufferSize) = row; | 186 | std::tie(rawTimestamp, data.action, data.source, data.bufferSize) = row; |
| 187 | + data.deviceId = deviceId; | ||
| 187 | HPFloat timestamp{rawTimestamp}; | 188 | HPFloat timestamp{rawTimestamp}; |
| 188 | data.timestamp = GetLocalTime(timestamp, timeRecord).Uint64(); | 189 | data.timestamp = GetLocalTime(timestamp, timeRecord).Uint64(); |
| 189 | dpData.push_back(data); | 190 | dpData.push_back(data); |
| @@ -191,7 +192,7 @@ bool AicpuProcessor::LoadDpData(const std::string &devicePath, const ProfTimeRec | |||
| 191 | return true; | 192 | return true; |
| 192 | } | 193 | } |
| 193 | 194 | ||
| 194 | -bool AicpuProcessor::LoadMiData(const std::string &devicePath, std::vector<AicpuMiData> &miData) | 195 | +bool AicpuProcessor::LoadMiData(const std::string &devicePath, uint16_t deviceId, std::vector<AicpuMiData> &miData) |
| 195 | { | 196 | { |
| 196 | DBInfo miDB(DB_NAME_DATA_PREPROCESS, TABLE_NAME_DATA_QUEUE); | 197 | DBInfo miDB(DB_NAME_DATA_PREPROCESS, TABLE_NAME_DATA_QUEUE); |
| 197 | std::string dbPath = Utils::File::PathJoin({devicePath, SQLITE, miDB.dbName}); | 198 | std::string dbPath = Utils::File::PathJoin({devicePath, SQLITE, miDB.dbName}); |
| @@ -219,6 +220,7 @@ bool AicpuProcessor::LoadMiData(const std::string &devicePath, std::vector<Aicpu | |||
| 219 | double startTime = 0; | 220 | double startTime = 0; |
| 220 | double endTime = 0; | 221 | double endTime = 0; |
| 221 | std::tie(data.nodeName, startTime, endTime, data.queueSize) = row; | 222 | std::tie(data.nodeName, startTime, endTime, data.queueSize) = row; |
| 223 | + data.deviceId = deviceId; | ||
| 222 | data.startTime = static_cast<uint64_t>(startTime); | 224 | data.startTime = static_cast<uint64_t>(startTime); |
| 223 | data.endTime = static_cast<uint64_t>(endTime); | 225 | data.endTime = static_cast<uint64_t>(endTime); |
| 224 | miData.push_back(data); | 226 | miData.push_back(data); |
| @@ -49,9 +49,9 @@ class AicpuProcessor : public DataProcessor | |||
| 49 | std::vector<AicpuDpData> &dpData, std::vector<AicpuMiData> &miData); | 49 | std::vector<AicpuDpData> &dpData, std::vector<AicpuMiData> &miData); |
| 50 | bool LoadAiCpuData(const std::string &devicePath, uint16_t deviceId, const Utils::ProfTimeRecord &timeRecord, | 50 | bool LoadAiCpuData(const std::string &devicePath, uint16_t deviceId, const Utils::ProfTimeRecord &timeRecord, |
| 51 | std::vector<AicpuSummaryData> &summaryData); | 51 | std::vector<AicpuSummaryData> &summaryData); |
| 52 | - bool LoadDpData(const std::string &devicePath, const Utils::ProfTimeRecord &timeRecord, | 52 | + bool LoadDpData(const std::string &devicePath, uint16_t deviceId, const Utils::ProfTimeRecord &timeRecord, |
| 53 | std::vector<AicpuDpData> &dpData); | 53 | std::vector<AicpuDpData> &dpData); |
| 54 | - bool LoadMiData(const std::string &devicePath, std::vector<AicpuMiData> &miData); | 54 | + bool LoadMiData(const std::string &devicePath, uint16_t deviceId, std::vector<AicpuMiData> &miData); |
| 55 | void MatchBatchId(std::vector<AicpuSummaryData> &summaryData, const std::vector<AscendTaskData> &ascendTasks); | 55 | void MatchBatchId(std::vector<AicpuSummaryData> &summaryData, const std::vector<AscendTaskData> &ascendTasks); |
| 56 | void MatchNodeName(std::vector<AicpuSummaryData> &summaryData, const std::vector<TaskInfoData> &taskInfos, | 56 | void MatchNodeName(std::vector<AicpuSummaryData> &summaryData, const std::vector<TaskInfoData> &taskInfos, |
| 57 | bool isChipV6); | 57 | bool isChipV6); |
| @@ -42,6 +42,7 @@ struct AicpuSummaryData | |||
| 42 | 42 | ||
| 43 | struct AicpuDpData | 43 | struct AicpuDpData |
| 44 | { | 44 | { |
| 45 | + uint16_t deviceId = UINT16_MAX; | ||
| 45 | uint64_t timestamp = 0; | 46 | uint64_t timestamp = 0; |
| 46 | std::string action; | 47 | std::string action; |
| 47 | std::string source; | 48 | std::string source; |
| @@ -50,6 +51,7 @@ struct AicpuDpData | |||
| 50 | 51 | ||
| 51 | struct AicpuMiData | 52 | struct AicpuMiData |
| 52 | { | 53 | { |
| 54 | + uint16_t deviceId = UINT16_MAX; | ||
| 53 | std::string nodeName; | 55 | std::string nodeName; |
| 54 | uint64_t startTime = 0; | 56 | uint64_t startTime = 0; |
| 55 | uint64_t endTime = 0; | 57 | uint64_t endTime = 0; |
| @@ -187,7 +187,13 @@ bool IsDoubleEqual(double checkDouble, double standard) | |||
| 187 | 187 | ||
| 188 | std::string AddQuotation(std::string str) { return Join({"\"", str, "\""}, ""); } | 188 | std::string AddQuotation(std::string str) { return Join({"\"", str, "\""}, ""); } |
| 189 | 189 | ||
| 190 | -std::string DivideByPowersOfTenWithPrecision(uint64_t value, int accuracy, int scale) | 190 | +std::string FormatHighPrecisionForCsv(const std::string &value) |
| 191 | +{ | ||
| 192 | + // Excel 打开超过15位精度的数值时会丢精度，追加\t让 Excel 按文本处理该单元格 | ||
| 193 | + return value + "\t"; | ||
| 194 | +} | ||
| 195 | + | ||
| 196 | +std::string DivideByPowersOfTenWithPrecision(uint64_t value, bool isHighPrecision, int accuracy, int scale) | ||
| 191 | { | 197 | { |
| 192 | // scale代表除以10的多少次幂，比如3就是除以10^3，accuracy代表保留位数 | 198 | // scale代表除以10的多少次幂，比如3就是除以10^3，accuracy代表保留位数 |
| 193 | std::string numStr = std::to_string(value); | 199 | std::string numStr = std::to_string(value); |
| @@ -199,16 +205,17 @@ std::string DivideByPowersOfTenWithPrecision(uint64_t value, int accuracy, int s | |||
| 199 | numStr.insert(numStr.size() - scale, "."); | 205 | numStr.insert(numStr.size() - scale, "."); |
| 200 | if (scale == accuracy) | 206 | if (scale == accuracy) |
| 201 | { // 精度与移位数相等，直接返回即可 | 207 | { // 精度与移位数相等，直接返回即可 |
| 202 | - return numStr; | 208 | + return isHighPrecision ? FormatHighPrecisionForCsv(numStr) : numStr; |
| 203 | } | 209 | } |
| 204 | else if (accuracy > scale) | 210 | else if (accuracy > scale) |
| 205 | { // 精度比移位数大，需要末尾补0 | 211 | { // 精度比移位数大，需要末尾补0 |
| 206 | numStr.insert(numStr.end(), accuracy - scale, '0'); | 212 | numStr.insert(numStr.end(), accuracy - scale, '0'); |
| 207 | - return numStr; | 213 | + return isHighPrecision ? FormatHighPrecisionForCsv(numStr) : numStr; |
| 208 | } | 214 | } |
| 209 | else | 215 | else |
| 210 | { // 精度比移位数小，需要截取numStr.size() - (scale - accuracy)长个字符串 | 216 | { // 精度比移位数小，需要截取numStr.size() - (scale - accuracy)长个字符串 |
| 211 | - return numStr.substr(0, numStr.size() + accuracy - scale); | 217 | + std::string res = numStr.substr(0, numStr.size() + accuracy - scale); |
| 218 | + return isHighPrecision ? FormatHighPrecisionForCsv(res) : res; | ||
| 212 | } | 219 | } |
| 213 | } | 220 | } |
| 214 | 221 | ||
| @@ -52,7 +52,13 @@ bool IsNumber(const std::string &s); | |||
| 52 | uint64_t Contact(uint32_t high, uint32_t low); | 52 | uint64_t Contact(uint32_t high, uint32_t low); |
| 53 | bool IsDoubleEqual(double checkDouble, double standard); | 53 | bool IsDoubleEqual(double checkDouble, double standard); |
| 54 | std::string AddQuotation(std::string str); | 54 | std::string AddQuotation(std::string str); |
| 55 | -std::string DivideByPowersOfTenWithPrecision(uint64_t value, int scale = ACCURACY_THREE, int accuracy = ACCURACY_THREE); | 55 | +// Excel 对超过15位的数字会丢精度，绝对时刻(us)转字符串后追加\t可强制按文本处理，与python format_high_precision_for_csv |
| 56 | +// 对齐 | ||
| 57 | +std::string FormatHighPrecisionForCsv(const std::string &value); | ||
| 58 | +// scale代表除以10的多少次幂（如3即除以10^3），accuracy代表保留位数； | ||
| 59 | +// isHighPrecision为true时结果追加\t，用于绝对时刻列 | ||
| 60 | +std::string DivideByPowersOfTenWithPrecision(uint64_t value, bool isHighPrecision = false, | ||
| 61 | + int accuracy = ACCURACY_THREE, int scale = ACCURACY_THREE); | ||
| 56 | bool EndsWith(const std::string &str, const std::string &suffix); | 62 | bool EndsWith(const std::string &str, const std::string &suffix); |
| 57 | std::string DoubleToStr(const double &value, const uint16_t &scale = ACCURACY_THREE); | 63 | std::string DoubleToStr(const double &value, const uint16_t &scale = ACCURACY_THREE); |
| 58 | double RoundToDecimalPlaces(const double num, int decimalPlaces = ACCURACY_THREE); | 64 | double RoundToDecimalPlaces(const double num, int decimalPlaces = ACCURACY_THREE); |
| @@ -43,10 +43,10 @@ const std::string PROF_PATH = File::PathJoin({BASE_PATH, "PROF_0"}); | |||
| 43 | const std::string RESULT_PATH = File::PathJoin({PROF_PATH, Analysis::Common::OUTPUT_PATH}); | 43 | const std::string RESULT_PATH = File::PathJoin({PROF_PATH, Analysis::Common::OUTPUT_PATH}); |
| 44 | 44 | ||
| 45 | const std::string AICPU_HEADER = | 45 | const std::string AICPU_HEADER = |
| 46 | - "Timestamp(us),Node,Compute_time(us),Memcpy_time(us),Task_time(us),Dispatch_time(us),Total_time(us),Stream ID,Task " | 46 | + "Device_id,Timestamp(us),Node,Compute_time(us),Memcpy_time(us),Task_time(us),Dispatch_time(us),Total_time(us)," |
| 47 | - "ID"; | 47 | + "Stream ID,Task ID"; |
| 48 | -const std::string DP_HEADER = "Timestamp(us),Action,Source,Cached Buffer Size"; | 48 | +const std::string DP_HEADER = "Device_id,Timestamp(us),Action,Source,Cached Buffer Size"; |
| 49 | -const std::string MI_HEADER = "Node Name,Start Time(us),End Time(us),Queue Size"; | 49 | +const std::string MI_HEADER = "Device_id,Node Name,Start Time(us),End Time(us),Queue Size"; |
| 50 | 50 | ||
| 51 | std::string FindCsvByName(const std::string &name) | 51 | std::string FindCsvByName(const std::string &name) |
| 52 | { | 52 | { |
| @@ -114,6 +114,7 @@ std::vector<AicpuDpData> GenerateDpData() | |||
| 114 | { | 114 | { |
| 115 | std::vector<AicpuDpData> res; | 115 | std::vector<AicpuDpData> res; |
| 116 | AicpuDpData first; | 116 | AicpuDpData first; |
| 117 | + first.deviceId = 0; | ||
| 117 | first.timestamp = 1000000; | 118 | first.timestamp = 1000000; |
| 118 | first.action = "enqueue"; | 119 | first.action = "enqueue"; |
| 119 | first.source = "src0"; | 120 | first.source = "src0"; |
| @@ -121,6 +122,7 @@ std::vector<AicpuDpData> GenerateDpData() | |||
| 121 | res.push_back(first); | 122 | res.push_back(first); |
| 122 | 123 | ||
| 123 | AicpuDpData second; | 124 | AicpuDpData second; |
| 125 | + second.deviceId = 0; | ||
| 124 | second.timestamp = 2500000; | 126 | second.timestamp = 2500000; |
| 125 | second.action = "dequeue"; | 127 | second.action = "dequeue"; |
| 126 | second.source = "src1"; | 128 | second.source = "src1"; |
| @@ -133,6 +135,7 @@ std::vector<AicpuMiData> GenerateMiData() | |||
| 133 | { | 135 | { |
| 134 | std::vector<AicpuMiData> res; | 136 | std::vector<AicpuMiData> res; |
| 135 | AicpuMiData first; | 137 | AicpuMiData first; |
| 138 | + first.deviceId = 0; | ||
| 136 | first.nodeName = "QueueA"; | 139 | first.nodeName = "QueueA"; |
| 137 | first.startTime = 100; | 140 | first.startTime = 100; |
| 138 | first.endTime = 200; | 141 | first.endTime = 200; |
| @@ -140,6 +143,7 @@ std::vector<AicpuMiData> GenerateMiData() | |||
| 140 | res.push_back(first); | 143 | res.push_back(first); |
| 141 | 144 | ||
| 142 | AicpuMiData second; | 145 | AicpuMiData second; |
| 146 | + second.deviceId = 0; | ||
| 143 | second.nodeName = "QueueB"; | 147 | second.nodeName = "QueueB"; |
| 144 | second.startTime = 300; | 148 | second.startTime = 300; |
| 145 | second.endTime = 400; | 149 | second.endTime = 400; |
| @@ -212,20 +216,21 @@ TEST_F(AicpuAssemblerUTest, ShouldWriteThreeCsvWhenAllDataExist) | |||
| 212 | std::vector<std::string> aicpuLines = ReadCsvLines(aicpuFile); | 216 | std::vector<std::string> aicpuLines = ReadCsvLines(aicpuFile); |
| 213 | ASSERT_EQ(3ul, aicpuLines.size()); | 217 | ASSERT_EQ(3ul, aicpuLines.size()); |
| 214 | EXPECT_EQ(AICPU_HEADER, aicpuLines[0]); | 218 | EXPECT_EQ(AICPU_HEADER, aicpuLines[0]); |
| 215 | - EXPECT_EQ("1000.000,Conv2D,1.5,2.5,500,0.5,10.5,10,20", aicpuLines[1]); | 219 | + // 主 aicpu 的 Timestamp(us) 列对齐 python 打\t，dp/aicpu_mi 不打 |
| 216 | - EXPECT_EQ("2000.000,N/A,3,4,8,1,20,11,30", aicpuLines[2]); | 220 | + EXPECT_EQ("0,1000.000\t,Conv2D,1.5,2.5,500,0.5,10.5,10,20", aicpuLines[1]); |
| 221 | + EXPECT_EQ("0,2000.000\t,N/A,3,4,8,1,20,11,30", aicpuLines[2]); | ||
| 217 | 222 | ||
| 218 | std::vector<std::string> dpLines = ReadCsvLines(dpFile); | 223 | std::vector<std::string> dpLines = ReadCsvLines(dpFile); |
| 219 | ASSERT_EQ(3ul, dpLines.size()); | 224 | ASSERT_EQ(3ul, dpLines.size()); |
| 220 | EXPECT_EQ(DP_HEADER, dpLines[0]); | 225 | EXPECT_EQ(DP_HEADER, dpLines[0]); |
| 221 | - EXPECT_EQ("1000.000,enqueue,src0,128", dpLines[1]); | 226 | + EXPECT_EQ("0,1000.000,enqueue,src0,128", dpLines[1]); |
| 222 | - EXPECT_EQ("2500.000,dequeue,src1,256", dpLines[2]); | 227 | + EXPECT_EQ("0,2500.000,dequeue,src1,256", dpLines[2]); |
| 223 | 228 | ||
| 224 | std::vector<std::string> miLines = ReadCsvLines(miFile); | 229 | std::vector<std::string> miLines = ReadCsvLines(miFile); |
| 225 | ASSERT_EQ(3ul, miLines.size()); | 230 | ASSERT_EQ(3ul, miLines.size()); |
| 226 | EXPECT_EQ(MI_HEADER, miLines[0]); | 231 | EXPECT_EQ(MI_HEADER, miLines[0]); |
| 227 | - EXPECT_EQ("QueueA,100,200,8", miLines[1]); | 232 | + EXPECT_EQ("0,QueueA,100,200,8", miLines[1]); |
| 228 | - EXPECT_EQ("QueueB,300,400,16", miLines[2]); | 233 | + EXPECT_EQ("0,QueueB,300,400,16", miLines[2]); |
| 229 | } | 234 | } |
| 230 | 235 | ||
| 231 | TEST_F(AicpuAssemblerUTest, ShouldWriteAicpuOnly) | 236 | TEST_F(AicpuAssemblerUTest, ShouldWriteAicpuOnly) |
| @@ -252,6 +252,7 @@ TEST_F(AicpuProcessorUTest, ShouldLoadAllThreeTypesAndFillDerivedFields) | |||
| 252 | auto dp = dataInventory.GetPtr<std::vector<AicpuDpData>>(); | 252 | auto dp = dataInventory.GetPtr<std::vector<AicpuDpData>>(); |
| 253 | ASSERT_NE(nullptr, dp); | 253 | ASSERT_NE(nullptr, dp); |
| 254 | ASSERT_EQ(2ul, dp->size()); | 254 | ASSERT_EQ(2ul, dp->size()); |
| 255 | + EXPECT_EQ(0u, dp->at(0).deviceId); | ||
| 255 | EXPECT_EQ(1000000ull, dp->at(0).timestamp); | 256 | EXPECT_EQ(1000000ull, dp->at(0).timestamp); |
| 256 | EXPECT_EQ("enqueue", dp->at(0).action); | 257 | EXPECT_EQ("enqueue", dp->at(0).action); |
| 257 | EXPECT_EQ("src0", dp->at(0).source); | 258 | EXPECT_EQ("src0", dp->at(0).source); |
| @@ -260,6 +261,7 @@ TEST_F(AicpuProcessorUTest, ShouldLoadAllThreeTypesAndFillDerivedFields) | |||
| 260 | auto mi = dataInventory.GetPtr<std::vector<AicpuMiData>>(); | 261 | auto mi = dataInventory.GetPtr<std::vector<AicpuMiData>>(); |
| 261 | ASSERT_NE(nullptr, mi); | 262 | ASSERT_NE(nullptr, mi); |
| 262 | ASSERT_EQ(2ul, mi->size()); | 263 | ASSERT_EQ(2ul, mi->size()); |
| 264 | + EXPECT_EQ(0u, mi->at(0).deviceId); | ||
| 263 | EXPECT_EQ("QueueA", mi->at(0).nodeName); | 265 | EXPECT_EQ("QueueA", mi->at(0).nodeName); |
| 264 | EXPECT_EQ(100ull, mi->at(0).startTime); | 266 | EXPECT_EQ(100ull, mi->at(0).startTime); |
| 265 | EXPECT_EQ(200ull, mi->at(0).endTime); | 267 | EXPECT_EQ(200ull, mi->at(0).endTime); |
| @@ -514,6 +516,7 @@ TEST_F(AicpuProcessorUTest, ShouldSucceedWhenDpOnly) | |||
| 514 | EXPECT_EQ(nullptr, dataInventory.GetPtr<std::vector<AicpuSummaryData>>()); | 516 | EXPECT_EQ(nullptr, dataInventory.GetPtr<std::vector<AicpuSummaryData>>()); |
| 515 | ASSERT_NE(nullptr, dataInventory.GetPtr<std::vector<AicpuDpData>>()); | 517 | ASSERT_NE(nullptr, dataInventory.GetPtr<std::vector<AicpuDpData>>()); |
| 516 | EXPECT_EQ(1ul, dataInventory.GetPtr<std::vector<AicpuDpData>>()->size()); | 518 | EXPECT_EQ(1ul, dataInventory.GetPtr<std::vector<AicpuDpData>>()->size()); |
| 519 | + EXPECT_EQ(0u, dataInventory.GetPtr<std::vector<AicpuDpData>>()->at(0).deviceId); | ||
| 517 | EXPECT_EQ(nullptr, dataInventory.GetPtr<std::vector<AicpuMiData>>()); | 520 | EXPECT_EQ(nullptr, dataInventory.GetPtr<std::vector<AicpuMiData>>()); |
| 518 | } | 521 | } |
| 519 | 522 | ||
| @@ -531,6 +534,7 @@ TEST_F(AicpuProcessorUTest, ShouldSucceedWhenMiOnly) | |||
| 531 | EXPECT_EQ(nullptr, dataInventory.GetPtr<std::vector<AicpuSummaryData>>()); | 534 | EXPECT_EQ(nullptr, dataInventory.GetPtr<std::vector<AicpuSummaryData>>()); |
| 532 | EXPECT_EQ(nullptr, dataInventory.GetPtr<std::vector<AicpuDpData>>()); | 535 | EXPECT_EQ(nullptr, dataInventory.GetPtr<std::vector<AicpuDpData>>()); |
| 533 | ASSERT_NE(nullptr, dataInventory.GetPtr<std::vector<AicpuMiData>>()); | 536 | ASSERT_NE(nullptr, dataInventory.GetPtr<std::vector<AicpuMiData>>()); |
| 537 | + EXPECT_EQ(0u, dataInventory.GetPtr<std::vector<AicpuMiData>>()->at(0).deviceId); | ||
| 534 | EXPECT_EQ("OnlyMi", dataInventory.GetPtr<std::vector<AicpuMiData>>()->at(0).nodeName); | 538 | EXPECT_EQ("OnlyMi", dataInventory.GetPtr<std::vector<AicpuMiData>>()->at(0).nodeName); |
| 535 | } | 539 | } |
| 536 | 540 | ||
| @@ -249,16 +249,24 @@ TEST_F(UtilsUTest, TestDivideByPowersOfTenWithPrecisionShouldReturnTrueValue) | |||
| 249 | EXPECT_EQ("0.012", DivideByPowersOfTenWithPrecision(value)); | 249 | EXPECT_EQ("0.012", DivideByPowersOfTenWithPrecision(value)); |
| 250 | 250 | ||
| 251 | value = 23456; // 入参23456 | 251 | value = 23456; // 入参23456 |
| 252 | - EXPECT_EQ("23.4560", DivideByPowersOfTenWithPrecision(value, 4, 3)); // 长度高于3位，移动3位，精度4位 | 252 | + EXPECT_EQ("23.4560", DivideByPowersOfTenWithPrecision(value, false, 4, 3)); // 长度高于3位，移动3位，精度4位 |
| 253 | 253 | ||
| 254 | value = 58; // 入参58 | 254 | value = 58; // 入参58 |
| 255 | - EXPECT_EQ("0.0580", DivideByPowersOfTenWithPrecision(value, 4, 3)); // 长度低于3位，移动3位，精度4位 | 255 | + EXPECT_EQ("0.0580", DivideByPowersOfTenWithPrecision(value, false, 4, 3)); // 长度低于3位，移动3位，精度4位 |
| 256 | 256 | ||
| 257 | value = 1234567; // 入参1234567 | 257 | value = 1234567; // 入参1234567 |
| 258 | - EXPECT_EQ("1234.56", DivideByPowersOfTenWithPrecision(value, 2, 3)); // 长度高于3位，移动3位，精度2位 | 258 | + EXPECT_EQ("1234.56", DivideByPowersOfTenWithPrecision(value, false, 2, 3)); // 长度高于3位，移动3位，精度2位 |
| 259 | 259 | ||
| 260 | value = 78; // 入参78 | 260 | value = 78; // 入参78 |
| 261 | - EXPECT_EQ("0.07", DivideByPowersOfTenWithPrecision(value, 2, 3)); // 长度小于3位，移动3位，精度2位 | 261 | + EXPECT_EQ("0.07", DivideByPowersOfTenWithPrecision(value, false, 2, 3)); // 长度小于3位，移动3位，精度2位 |
| 262 | +} | ||
| 263 | + | ||
| 264 | +TEST_F(UtilsUTest, TestDivideByPowersOfTenWithPrecisionShouldAppendTabWhenHighPrecision) | ||
| 265 | +{ | ||
| 266 | + // isHighPrecision 为 true 时，结果追加\t，Excel 按文本处理绝对时刻 | ||
| 267 | + EXPECT_EQ("123.456\t", DivideByPowersOfTenWithPrecision(123456, true)); | ||
| 268 | + // 精度/位数与默认值不同时同样追加\t | ||
| 269 | + EXPECT_EQ("1234.56\t", DivideByPowersOfTenWithPrecision(1234567, true, 2, 3)); | ||
| 262 | } | 270 | } |
| 263 | 271 | ||
| 264 | // ========================================================================= | 272 | // ========================================================================= |
【review】 【问题描述】aicpu/aicpu_dp/aicpu_mi CSV 新增 Device_id 列，与 python 侧列定义不一致 aicpu_assembler.cpp 、L91、L109 的 headers 均以 Device_id 开头并输出 item.deviceId （10/5/5 列），但 python 侧：
【问题分类】功能实现
【修改建议】确认对齐目标：若以 python 列为准，则应同步更新 msprof_export_data_config.py 与 aicpu_viewer.py 的输出列（补充 Device_id），或明确 C++ CSV 与 python 导出为独立产物并在文档/配置中保持列定义一致。