已合并
[msprof]补充Capture Stream与MC2通信信息Parser C化模块 #398
gcw_648Mce95创建于 8月4日
[msprof]补充Capture Stream与MC2通信信息Parser C化模块 #398
已合并
共 34 个文件变更+1654-78
| @@ -22,7 +22,6 @@ | |||
| 22 | 22 | ||
| 23 | 23 | ||
| 24 | 24 | ||
| 25 | - | ||
| 26 | 25 | ||
| 27 | 26 | ||
| 28 | 27 | ||
| @@ -123,9 +122,8 @@ REGISTER_DB_SAVER_WITH_DATA(PROCESSOR_NAME_TASK, | |||
| 123 | TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_DEVICE_TX)), | 122 | TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_DEVICE_TX)), |
| 124 | std::vector<AscendTaskData>, std::vector<MsprofTxDeviceData>); | 123 | std::vector<AscendTaskData>, std::vector<MsprofTxDeviceData>); |
| 125 | REGISTER_DB_SAVER_WITH_DATA(PROCESSOR_NAME_COMPUTE_TASK_INFO, | 124 | REGISTER_DB_SAVER_WITH_DATA(PROCESSOR_NAME_COMPUTE_TASK_INFO, |
| 126 | - TOPO_DEPS(TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_COMPUTE_TASK_INFO), | 125 | + TOPO_DEPS(TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_COMPUTE_TASK_INFO)), |
| 127 | - TOPO_NODE(DATA_PROCESSING, PROCESSOR_MC2_COMM_INFO)), | 126 | + std::vector<TaskInfoData>); |
| 128 | - std::vector<TaskInfoData>, std::vector<MC2CommInfoData>); | ||
| 129 | REGISTER_DB_SAVER_WITH_DATA(PROCESSOR_NAME_MEMCPY_INFO, | 127 | REGISTER_DB_SAVER_WITH_DATA(PROCESSOR_NAME_MEMCPY_INFO, |
| 130 | TOPO_DEPS(TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_MEMCPY_INFO)), | 128 | TOPO_DEPS(TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_MEMCPY_INFO)), |
| 131 | std::vector<MemcpyInfoData>); | 129 | std::vector<MemcpyInfoData>); |
| @@ -35,7 +35,7 @@ namespace | |||
| 35 | const std::set<std::string> FILTER_TYPE = {"KERNEL_AICORE", "KERNEL_AIVEC", "FFTS_PLUS", "KERNEL_MIX_AIC", | 35 | const std::set<std::string> FILTER_TYPE = {"KERNEL_AICORE", "KERNEL_AIVEC", "FFTS_PLUS", "KERNEL_MIX_AIC", |
| 36 | "KERNEL_MIX_AIV", "PROFILING_ENABLE", "PROFILING_DISABLE"}; | 36 | "KERNEL_MIX_AIV", "PROFILING_ENABLE", "PROFILING_DISABLE"}; |
| 37 | 37 | ||
| 38 | -void SepOneTask(std::vector<TimeDuration> ×, std::set<uint32_t> &mc2StreamsTable, TaskInfoData &task, | 38 | +void SepOneTask(std::vector<TimeDuration> ×, const std::set<uint32_t> &mc2StreamsTable, TaskInfoData &task, |
| 39 | std::unordered_map<uint16_t, std::vector<TimeDuration>> &compSections) | 39 | std::unordered_map<uint16_t, std::vector<TimeDuration>> &compSections) |
| 40 | { | 40 | { |
| 41 | if (mc2StreamsTable.find(task.streamId) != mc2StreamsTable.end() || EndsWith(task.opName, AICPU_KERNEL) || | 41 | if (mc2StreamsTable.find(task.streamId) != mc2StreamsTable.end() || EndsWith(task.opName, AICPU_KERNEL) || |
| @@ -27,7 +27,6 @@ | |||
| 27 | 27 | ||
| 28 | 28 | ||
| 29 | 29 | ||
| 30 | - | ||
| 31 | 30 | ||
| 32 | 31 | ||
| 33 | 32 | ||
| @@ -120,7 +119,6 @@ REGISTER_PROCESSOR(RoCETimelineProcessor, PROCESSOR_NAME_ROCE_TIMELINE, TOPO_DEP | |||
| 120 | REGISTER_PROCESSOR(NicProcessor, PROCESSOR_NAME_NIC, TOPO_DEPS()); | 119 | REGISTER_PROCESSOR(NicProcessor, PROCESSOR_NAME_NIC, TOPO_DEPS()); |
| 121 | REGISTER_PROCESSOR(RoCEProcessor, PROCESSOR_NAME_ROCE, TOPO_DEPS()); | 120 | REGISTER_PROCESSOR(RoCEProcessor, PROCESSOR_NAME_ROCE, TOPO_DEPS()); |
| 122 | REGISTER_PROCESSOR(QosProcessor, PROCESSOR_NAME_QOS, TOPO_DEPS()); | 121 | REGISTER_PROCESSOR(QosProcessor, PROCESSOR_NAME_QOS, TOPO_DEPS()); |
| 123 | -REGISTER_PROCESSOR(Mc2CommInfoProcessor, PROCESSOR_MC2_COMM_INFO, TOPO_DEPS()); | ||
| 124 | REGISTER_PROCESSOR(MetricProcessor, PROCESSOR_PMU, TOPO_DEPS()); | 122 | REGISTER_PROCESSOR(MetricProcessor, PROCESSOR_PMU, TOPO_DEPS()); |
| 125 | REGISTER_PROCESSOR(MemcpyInfoProcessor, PROCESSOR_NAME_MEMCPY_INFO, TOPO_DEPS()); | 123 | REGISTER_PROCESSOR(MemcpyInfoProcessor, PROCESSOR_NAME_MEMCPY_INFO, TOPO_DEPS()); |
| 126 | REGISTER_PROCESSOR_WITH_DATA(NpuOpMemProcessor, PROCESSOR_NAME_NPU_OP_MEM, | 124 | REGISTER_PROCESSOR_WITH_DATA(NpuOpMemProcessor, PROCESSOR_NAME_NPU_OP_MEM, |
| @@ -136,10 +134,8 @@ REGISTER_PROCESSOR(HostTaskProcessor, PROCESSOR_HOST_TASK, TOPO_DEPS()); | |||
| 136 | REGISTER_PROCESSOR_WITH_DATA(OverlapAnalysisProcessor, PROCESSOR_NAME_OVERLAP_ANALYSIS, | 134 | REGISTER_PROCESSOR_WITH_DATA(OverlapAnalysisProcessor, PROCESSOR_NAME_OVERLAP_ANALYSIS, |
| 137 | TOPO_DEPS(TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_TASK), | 135 | TOPO_DEPS(TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_TASK), |
| 138 | TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_COMPUTE_TASK_INFO), | 136 | TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_COMPUTE_TASK_INFO), |
| 139 | - TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_COMMUNICATION), | 137 | + TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_COMMUNICATION)), |
| 140 | - TOPO_NODE(DATA_PROCESSING, PROCESSOR_MC2_COMM_INFO)), | 138 | + std::vector<AscendTaskData>, std::vector<TaskInfoData>, std::vector<CommunicationOpData>); |
| 141 | - std::vector<AscendTaskData>, std::vector<TaskInfoData>, std::vector<CommunicationOpData>, | ||
| 142 | - std::vector<MC2CommInfoData>); | ||
| 143 | REGISTER_PROCESSOR(LowPowerProcessor, PROCESSOR_NAME_LOW_POWER, TOPO_DEPS()); | 139 | REGISTER_PROCESSOR(LowPowerProcessor, PROCESSOR_NAME_LOW_POWER, TOPO_DEPS()); |
| 144 | REGISTER_PROCESSOR(BiuPerfProcessor, PROCESSOR_NAME_BIU_PERF, TOPO_DEPS()); | 140 | REGISTER_PROCESSOR(BiuPerfProcessor, PROCESSOR_NAME_BIU_PERF, TOPO_DEPS()); |
| 145 | REGISTER_PROCESSOR(UbProcessor, PROCESSOR_NAME_UB, TOPO_DEPS()); | 141 | REGISTER_PROCESSOR(UbProcessor, PROCESSOR_NAME_UB, TOPO_DEPS()); |
| @@ -264,6 +264,13 @@ bool ParserAdditionalInfoAdapter::AdapterAdditionalInfo(MsprofAdditionalInfo* ad | |||
| 264 | case AdditionalInfoFormat::TASK_MEMORY_INFO_TYPE: | 264 | case AdditionalInfoFormat::TASK_MEMORY_INFO_TYPE: |
| 265 | AdapterMemoryInfo(addition, parsed); | 265 | AdapterMemoryInfo(addition, parsed); |
| 266 | return true; | 266 | return true; |
| 267 | + case AdditionalInfoFormat::MC2_COMM_INFO_TYPE: | ||
| 268 | + if (memcpy_s(parsed->data, sizeof(parsed->data), addition->data, sizeof(addition->data)) != EOK) | ||
| 269 | + { | ||
| 270 | + ERROR("adapter mc2 comm info data failed."); | ||
| 271 | + return false; | ||
| 272 | + } | ||
| 273 | + return true; | ||
| 267 | default: | 274 | default: |
| 268 | ERROR("Unsupported Additional Info: %.", static_cast<uint32_t>(parserType)); | 275 | ERROR("Unsupported Additional Info: %.", static_cast<uint32_t>(parserType)); |
| 269 | return false; | 276 | return false; |
| @@ -19,11 +19,14 @@ | |||
| 19 | 19 | ||
| 20 | 20 | ||
| 21 | 21 | ||
| 22 | + | ||
| 22 | 23 | ||
| 23 | 24 | ||
| 24 | 25 | ||
| 26 | + | ||
| 25 | 27 | ||
| 26 | 28 | ||
| 29 | + | ||
| 27 | 30 | ||
| 28 | 31 | ||
| 29 | 32 | ||
| @@ -42,7 +45,19 @@ bool HostTraceWorker::Run() | |||
| 42 | auto hostDataPath = Utils::File::PathJoin({hostPath_, "data"}); | 45 | auto hostDataPath = Utils::File::PathJoin({hostPath_, "data"}); |
| 43 | std::shared_ptr<EventGrouper> grouper; | 46 | std::shared_ptr<EventGrouper> grouper; |
| 44 | MAKE_SHARED_RETURN_VALUE(grouper, EventGrouper, false, hostDataPath); | 47 | MAKE_SHARED_RETURN_VALUE(grouper, EventGrouper, false, hostDataPath); |
| 45 | - grouper->Group(); | 48 | + bool result = grouper->Group(); |
| 49 | + // CaptureStreamInfoData formattedCaptureData; | ||
| 50 | + // if (Host::Cann::kEnableCaptureStreamMc2CppParser) | ||
| 51 | + // { | ||
| 52 | + // if (!DumpCaptureStreamInfo(grouper, formattedCaptureData)) | ||
| 53 | + // { | ||
| 54 | + // result = false; | ||
| 55 | + // } | ||
| 56 | + // if (!DumpMc2CommInfo(grouper, formattedCaptureData)) | ||
| 57 | + // { | ||
| 58 | + // result = false; | ||
| 59 | + // } | ||
| 60 | + // } | ||
| 46 | auto sqlitePath = Utils::File::PathJoin({hostPath_, "sqlite"}); | 61 | auto sqlitePath = Utils::File::PathJoin({hostPath_, "sqlite"}); |
| 47 | RTAddInfoCenter::GetInstance().Load(sqlitePath); | 62 | RTAddInfoCenter::GetInstance().Load(sqlitePath); |
| 48 | DumpRuntimeOpInfo(); | 63 | DumpRuntimeOpInfo(); |
| @@ -75,6 +90,66 @@ bool HostTraceWorker::Run() | |||
| 75 | apiDumpPool.WaitAllTasks(); | 90 | apiDumpPool.WaitAllTasks(); |
| 76 | apiDumpPool.Stop(); | 91 | apiDumpPool.Stop(); |
| 77 | DumpMemcpyInfo(hostDataPath); // 依赖runtime.db中的HostTask, 不能放在pool中 | 92 | DumpMemcpyInfo(hostDataPath); // 依赖runtime.db中的HostTask, 不能放在pool中 |
| 93 | + return result; | ||
| 94 | +} | ||
| 95 | + | ||
| 96 | +bool HostTraceWorker::DumpCaptureStreamInfo(const std::shared_ptr<EventGrouper> &grouper, | ||
| 97 | + CaptureStreamInfoData &formattedCaptureData) | ||
| 98 | +{ | ||
| 99 | + bool result = true; | ||
| 100 | + RTAddInfoCenter::GetInstance().SetCaptureStreamInfoData({}); | ||
| 101 | + const auto &captureData = grouper->GetCaptureStreamInfoData(); | ||
| 102 | + if (captureData.empty()) | ||
| 103 | + { | ||
| 104 | + return true; | ||
| 105 | + } | ||
| 106 | + | ||
| 107 | + CaptureStreamInfoDumper captureDumper(hostPath_); | ||
| 108 | + formattedCaptureData = captureDumper.FormatData(captureData); | ||
| 109 | + if (formattedCaptureData.empty()) | ||
| 110 | + { | ||
| 111 | + ERROR("Format capture stream info failed."); | ||
| 112 | + return false; | ||
| 113 | + } | ||
| 114 | + | ||
| 115 | + std::vector<Analysis::Domain::CaptureStreamInfo> centerData; | ||
| 116 | + if (!Utils::Reserve(centerData, formattedCaptureData.size())) | ||
| 117 | + { | ||
| 118 | + ERROR("Reserve capture stream info center data failed."); | ||
| 119 | + result = false; | ||
| 120 | + } | ||
| 121 | + else | ||
| 122 | + { | ||
| 123 | + for (const auto &item : formattedCaptureData) | ||
| 124 | + { | ||
| 125 | + centerData.emplace_back(item.modelId, item.timeStamp, item.streamId, item.originalStreamId, item.deviceId, | ||
| 126 | + static_cast<uint16_t>(item.batchId), item.captureStatus); | ||
| 127 | + } | ||
| 128 | + RTAddInfoCenter::GetInstance().SetCaptureStreamInfoData(centerData); | ||
| 129 | + } | ||
| 130 | + if (!captureDumper.DumpData(formattedCaptureData)) | ||
| 131 | + { | ||
| 132 | + ERROR("Dump capture stream info failed."); | ||
| 133 | + result = false; | ||
| 134 | + } | ||
| 135 | + return result; | ||
| 136 | +} | ||
| 137 | + | ||
| 138 | +bool HostTraceWorker::DumpMc2CommInfo(const std::shared_ptr<EventGrouper> &grouper, | ||
| 139 | + const CaptureStreamInfoData &formattedCaptureData) | ||
| 140 | +{ | ||
| 141 | + const auto &mc2Data = grouper->GetMc2CommInfoData(); | ||
| 142 | + if (mc2Data.empty()) | ||
| 143 | + { | ||
| 144 | + return true; | ||
| 145 | + } | ||
| 146 | + Mc2CommInfoDumper mc2Dumper(hostPath_); | ||
| 147 | + Mc2CommInfoInput input(mc2Data, formattedCaptureData); | ||
| 148 | + if (!mc2Dumper.DumpData(input)) | ||
| 149 | + { | ||
| 150 | + ERROR("Dump mc2 comm info failed."); | ||
| 151 | + return false; | ||
| 152 | + } | ||
| 78 | return true; | 153 | return true; |
| 79 | } | 154 | } |
| 80 | 155 | ||
| @@ -16,6 +16,7 @@ | |||
| 16 | 16 | ||
| 17 | 17 | ||
| 18 | 18 | ||
| 19 | + | ||
| 19 | 20 | ||
| 20 | 21 | ||
| 21 | 22 | ||
| @@ -23,6 +24,7 @@ | |||
| 23 | 24 | ||
| 24 | 25 | ||
| 25 | 26 | ||
| 27 | + | ||
| 26 | 28 | ||
| 27 | 29 | ||
| 28 | 30 | ||
| @@ -57,6 +59,10 @@ class HostTraceWorker | |||
| 57 | void DumpModelName(ThreadPool &pool, const std::string &hostDataPath); | 59 | void DumpModelName(ThreadPool &pool, const std::string &hostDataPath); |
| 58 | void DumpHostSystemProfileData(ThreadPool &pool); | 60 | void DumpHostSystemProfileData(ThreadPool &pool); |
| 59 | void DumpMemcpyInfo(const std::string &hostDataPath); | 61 | void DumpMemcpyInfo(const std::string &hostDataPath); |
| 62 | + bool DumpCaptureStreamInfo(const std::shared_ptr<EventGrouper> &grouper, | ||
| 63 | + CaptureStreamInfoData &formattedCaptureData); | ||
| 64 | + bool DumpMc2CommInfo(const std::shared_ptr<EventGrouper> &grouper, | ||
| 65 | + const CaptureStreamInfoData &formattedCaptureData); | ||
| 60 | 66 | ||
| 61 | private: | 67 | private: |
| 62 | const uint32_t poolSize_ = 10; | 68 | const uint32_t poolSize_ = 10; |
| @@ -32,6 +32,24 @@ namespace Host | |||
| 32 | { | 32 | { |
| 33 | namespace Cann | 33 | namespace Cann |
| 34 | { | 34 | { |
| 35 | + | ||
| 36 | +constexpr uint32_t MC2_COMM_STREAM_MAX_NUM = 8; | ||
| 37 | +constexpr uint32_t MC2_COMM_RESERVED_UINT32_NUM = 43; | ||
| 38 | + | ||
| 39 | +struct MsprofMc2CommInfo | ||
| 40 | +{ | ||
| 41 | + uint64_t groupName; | ||
| 42 | + uint32_t rankSize; | ||
| 43 | + uint32_t rankId; | ||
| 44 | + uint32_t usrRankId; | ||
| 45 | + uint32_t streamId; | ||
| 46 | + uint32_t streamSize; | ||
| 47 | + uint32_t commStreamIds[MC2_COMM_STREAM_MAX_NUM]; | ||
| 48 | + uint32_t reserved[MC2_COMM_RESERVED_UINT32_NUM]; | ||
| 49 | +}; | ||
| 50 | + | ||
| 51 | +static_assert(sizeof(MsprofMc2CommInfo) == 232, "MC2 payload must match the 256-byte additional record"); | ||
| 52 | + | ||
| 35 | // 该类的作用是Addition数据的解析 | 53 | // 该类的作用是Addition数据的解析 |
| 36 | class AdditionInfoParser : public BaseParser<AdditionInfoParser> | 54 | class AdditionInfoParser : public BaseParser<AdditionInfoParser> |
| 37 | { | 55 | { |
| @@ -53,7 +71,25 @@ class AdditionInfoParser : public BaseParser<AdditionInfoParser> | |||
| 53 | std::vector<std::shared_ptr<ParserConcatTensorInfo>> concatTensorData_; // not owned | 71 | std::vector<std::shared_ptr<ParserConcatTensorInfo>> concatTensorData_; // not owned |
| 54 | }; // class AdditionInfoParser | 72 | }; // class AdditionInfoParser |
| 55 | 73 | ||
| 74 | +// mc2通信数据的解析,复用Additional数据读取流程 | ||
| 75 | +class Mc2CommInfoParser final : public AdditionInfoParser | ||
| 76 | +{ | ||
| 77 | + public: | ||
| 78 | + explicit Mc2CommInfoParser(const std::string &path) : AdditionInfoParser(path, "Mc2CommInfoParser") | ||
| 79 | + { | ||
| 80 | + parserType_ = AdditionalInfoFormat::MC2_COMM_INFO_TYPE; | ||
| 81 | + Init(filePrefix_); | ||
| 82 | + } | ||
| 83 | + | ||
| 84 | + private: | ||
| 85 | + std::vector<std::string> filePrefix_ = { | ||
| 86 | + // "unaging.additional.mc2_comm_info.slice", | ||
| 87 | + // "aging.additional.mc2_comm_info.slice", | ||
| 88 | + }; | ||
| 89 | +}; // class Mc2CommInfoParser | ||
| 90 | + | ||
| 56 | // 该类的作用是CtxId数据的解析 | 91 | // 该类的作用是CtxId数据的解析 |
| 92 | + | ||
| 57 | class CtxIdParser final : public AdditionInfoParser | 93 | class CtxIdParser final : public AdditionInfoParser |
| 58 | { | 94 | { |
| 59 | public: | 95 | public: |
| @@ -24,15 +24,20 @@ | |||
| 24 | 24 | ||
| 25 | 25 | ||
| 26 | 26 | ||
| 27 | -namespace Analysis { | 27 | +namespace Analysis |
| 28 | -namespace Domain { | 28 | +{ |
| 29 | -namespace Host { | 29 | +namespace Domain |
| 30 | -namespace Cann { | 30 | +{ |
| 31 | +namespace Host | ||
| 32 | +{ | ||
| 33 | +namespace Cann | ||
| 34 | +{ | ||
| 31 | 35 | ||
| 32 | using EventQueue = Analysis::Domain::EventQueue; | 36 | using EventQueue = Analysis::Domain::EventQueue; |
| 33 | 37 | ||
| 34 | // CANN数据仓,用于存储各个Type的Events | 38 | // CANN数据仓,用于存储各个Type的Events |
| 35 | -struct CANNWarehouse { | 39 | +struct CANNWarehouse |
| 40 | +{ | ||
| 36 | // API Type只在Model、Node、HCCL Level的Event | 41 | // API Type只在Model、Node、HCCL Level的Event |
| 37 | std::shared_ptr<EventQueue> kernelEvents = nullptr; | 42 | std::shared_ptr<EventQueue> kernelEvents = nullptr; |
| 38 | std::shared_ptr<EventQueue> graphIdMapEvents = nullptr; | 43 | std::shared_ptr<EventQueue> graphIdMapEvents = nullptr; |
| @@ -46,8 +51,8 @@ struct CANNWarehouse { | |||
| 46 | std::shared_ptr<EventQueue> hcclOpInfoEvents = nullptr; | 51 | std::shared_ptr<EventQueue> hcclOpInfoEvents = nullptr; |
| 47 | }; | 52 | }; |
| 48 | 53 | ||
| 49 | -} // namespace Cann | 54 | +} // namespace Cann |
| 50 | -} // namespace Host | 55 | +} // namespace Host |
| 51 | -} // namespace Parser | 56 | +} // namespace Domain |
| 52 | -} // namespace Analysis | 57 | +} // namespace Analysis |
| 53 | -#endif // ANALYSIS_PARSER_HOST_CANN_CANN_WAREHOUSE_H | 58 | +#endif // ANALYSIS_PARSER_HOST_CANN_CANN_WAREHOUSE_H |
| @@ -0,0 +1,36 @@ | |||
| 1 | +/* ------------------------------------------------------------------------- | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This file is part of the MindStudio project. | ||
| 4 | + * | ||
| 5 | + * MindStudio is licensed under Mulan PSL v2. | ||
| 6 | + * You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 7 | + * You may obtain a copy of Mulan PSL v2 at: | ||
| 8 | + * | ||
| 9 | + * http://license.coscl.org.cn/MulanPSL2 | ||
| 10 | + * | ||
| 11 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 12 | + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 13 | + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 14 | + * See the Mulan PSL v2 for more details. | ||
| 15 | + * -------------------------------------------------------------------------*/ | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | +namespace Analysis | ||
| 23 | +{ | ||
| 24 | +namespace Domain | ||
| 25 | +{ | ||
| 26 | +namespace Host | ||
| 27 | +{ | ||
| 28 | +namespace Cann | ||
| 29 | +{ | ||
| 30 | +constexpr bool kEnableCaptureStreamMc2CppParser = false; | ||
| 31 | +} // namespace Cann | ||
| 32 | +} // namespace Host | ||
| 33 | +} // namespace Domain | ||
| 34 | +} // namespace Analysis | ||
| 35 | + | ||
| 36 | + | ||
| @@ -42,6 +42,20 @@ enum class DevType : uint16_t | |||
| 42 | DPU = 1, | 42 | DPU = 1, |
| 43 | }; | 43 | }; |
| 44 | 44 | ||
| 45 | +const std::vector<std::string> CAPTURE_V1_PREFIXES = { | ||
| 46 | + // "unaging.compact.capture_stream_info.slice", | ||
| 47 | + // "aging.compact.capture_stream_info.slice", | ||
| 48 | +}; | ||
| 49 | +const std::vector<std::string> CAPTURE_V2_PREFIXES = { | ||
| 50 | + // "unaging.compact.capture_stream_info_v2.slice", | ||
| 51 | + // "aging.compact.capture_stream_info_v2.slice", | ||
| 52 | +}; | ||
| 53 | + | ||
| 54 | +bool HasCaptureData(const std::string &path, const std::vector<std::string> &prefixes) | ||
| 55 | +{ | ||
| 56 | + return !File::GetOriginData(path, prefixes, {"done", "complete"}).empty(); | ||
| 57 | +} | ||
| 58 | + | ||
| 45 | constexpr uint16_t kDevTypeShift = 12; | 59 | constexpr uint16_t kDevTypeShift = 12; |
| 46 | constexpr uint16_t kDevTypeMask = 0xF; | 60 | constexpr uint16_t kDevTypeMask = 0xF; |
| 47 | 61 | ||
| @@ -107,6 +121,50 @@ int CompactInfoParser::ProduceData() | |||
| 107 | return ANALYSIS_OK; | 121 | return ANALYSIS_OK; |
| 108 | } | 122 | } |
| 109 | 123 | ||
| 124 | +CaptureStreamInfoParser::CaptureStreamInfoParser(const std::string &path) | ||
| 125 | + : CompactInfoParser(path, "CaptureStreamInfoParser") | ||
| 126 | +{ | ||
| 127 | + isV2_ = !HasCaptureData(path, CAPTURE_V1_PREFIXES) && HasCaptureData(path, CAPTURE_V2_PREFIXES); | ||
| 128 | + Init(isV2_ ? CAPTURE_V2_PREFIXES : CAPTURE_V1_PREFIXES); | ||
| 129 | +} | ||
| 130 | + | ||
| 131 | +int CaptureStreamInfoParser::ProduceData() | ||
| 132 | +{ | ||
| 133 | + if (chunkProducer_->Empty()) | ||
| 134 | + { | ||
| 135 | + return ANALYSIS_OK; | ||
| 136 | + } | ||
| 137 | + if (!Reserve(compactData_, chunkProducer_->Size())) | ||
| 138 | + { | ||
| 139 | + ERROR("%: Reserve data failed", parserName_); | ||
| 140 | + return ANALYSIS_ERROR; | ||
| 141 | + } | ||
| 142 | + const auto format = isV2_ ? CaptureStreamFormat::V2 : CaptureStreamFormat::V1; | ||
| 143 | + while (!chunkProducer_->Empty()) | ||
| 144 | + { | ||
| 145 | + std::unique_ptr<char[]> chunk(chunkProducer_->Pop()); | ||
| 146 | + auto compactInfo = ReinterpretConvert<MsprofCompactInfo *>(chunk.get()); | ||
| 147 | + if (!compactInfo) | ||
| 148 | + { | ||
| 149 | + ERROR("%: Pop chunk failed.", parserName_); | ||
| 150 | + return ANALYSIS_ERROR; | ||
| 151 | + } | ||
| 152 | + if (compactInfo->magicNumber != MSPROF_DATA_HEAD_MAGIC_NUM) | ||
| 153 | + { | ||
| 154 | + ERROR("%: The last %th data check failed.", parserName_, chunkProducer_->Size()); | ||
| 155 | + continue; | ||
| 156 | + } | ||
| 157 | + auto parserCompactInfo = std::make_shared<ParserCompactInfo>(); | ||
| 158 | + if (!ParserCompactInfoAdapter::AdapterCaptureStreamInfo(compactInfo, format, parserCompactInfo.get())) | ||
| 159 | + { | ||
| 160 | + ERROR("%: copy captureStreamInfo data failed.", parserName_); | ||
| 161 | + return ANALYSIS_ERROR; | ||
| 162 | + } | ||
| 163 | + compactData_.emplace_back(std::move(parserCompactInfo)); | ||
| 164 | + } | ||
| 165 | + return ANALYSIS_OK; | ||
| 166 | +} | ||
| 167 | + | ||
| 110 | int NodeBasicInfoParser::ProduceData() | 168 | int NodeBasicInfoParser::ProduceData() |
| 111 | { | 169 | { |
| 112 | std::shared_ptr<ChunkGenerator> staticChunkProducer; | 170 | std::shared_ptr<ChunkGenerator> staticChunkProducer; |
| @@ -24,6 +24,7 @@ | |||
| 24 | 24 | ||
| 25 | 25 | ||
| 26 | 26 | ||
| 27 | + | ||
| 27 | 28 | ||
| 28 | 29 | ||
| 29 | namespace Analysis | 30 | namespace Analysis |
| @@ -53,6 +54,19 @@ class CompactInfoParser : public BaseParser<CompactInfoParser> | |||
| 53 | std::vector<std::shared_ptr<Adapter::FlipTask>> flipTaskData_; // not owned | 54 | std::vector<std::shared_ptr<Adapter::FlipTask>> flipTaskData_; // not owned |
| 54 | }; // class CompactInfoParser | 55 | }; // class CompactInfoParser |
| 55 | 56 | ||
| 57 | +// capture stream info数据的解析,复用Compact数据读取流程 | ||
| 58 | +class CaptureStreamInfoParser final : public CompactInfoParser | ||
| 59 | +{ | ||
| 60 | + public: | ||
| 61 | + explicit CaptureStreamInfoParser(const std::string &path); | ||
| 62 | + | ||
| 63 | + private: | ||
| 64 | + int ProduceData() override; | ||
| 65 | + | ||
| 66 | + private: | ||
| 67 | + bool isV2_ = false; | ||
| 68 | +}; // class CaptureStreamInfoParser | ||
| 69 | + | ||
| 56 | // 该类的作用是node basic info数据的解析 | 70 | // 该类的作用是node basic info数据的解析 |
| 57 | class NodeBasicInfoParser final : public CompactInfoParser | 71 | class NodeBasicInfoParser final : public CompactInfoParser |
| 58 | { | 72 | { |
| @@ -17,6 +17,7 @@ | |||
| 17 | 17 | ||
| 18 | 18 | ||
| 19 | 19 | ||
| 20 | + | ||
| 20 | 21 | ||
| 21 | 22 | ||
| 22 | 23 | ||
| @@ -66,10 +67,47 @@ std::vector<std::shared_ptr<Adapter::FlipTask>> &EventGrouper::GetFlipTasks() { | |||
| 66 | std::vector<std::shared_ptr<ParserCompactInfo>> &EventGrouper::GetDpuTrackData() { return dpuTrackData_; } | 67 | std::vector<std::shared_ptr<ParserCompactInfo>> &EventGrouper::GetDpuTrackData() { return dpuTrackData_; } |
| 67 | std::unordered_map<uint64_t, uint64_t> &EventGrouper::GetDpuKernelNameMap() { return dpuKernelNameMap_; } | 68 | std::unordered_map<uint64_t, uint64_t> &EventGrouper::GetDpuKernelNameMap() { return dpuKernelNameMap_; } |
| 68 | 69 | ||
| 70 | +const std::vector<std::shared_ptr<ParserCompactInfo>> &EventGrouper::GetCaptureStreamInfoData() const | ||
| 71 | +{ | ||
| 72 | + return captureStreamInfoData_; | ||
| 73 | +} | ||
| 74 | + | ||
| 75 | +const std::vector<std::shared_ptr<ParserAdditionalInfo>> &EventGrouper::GetMc2CommInfoData() const | ||
| 76 | +{ | ||
| 77 | + return mc2CommInfoData_; | ||
| 78 | +} | ||
| 79 | + | ||
| 80 | +void EventGrouper::ParseCaptureStreamInfo() | ||
| 81 | +{ | ||
| 82 | + Utils::TimeLogger t{"Parse CaptureStreamInfo"}; | ||
| 83 | + CaptureStreamInfoParser parser(hostPath_); | ||
| 84 | + captureStreamInfoData_ = parser.ParseData<ParserCompactInfo>(); | ||
| 85 | + if (parser.GetStatus() == ParserStatus::ERROR) | ||
| 86 | + { | ||
| 87 | + ERROR("Parse capture stream info failed."); | ||
| 88 | + result_ = false; | ||
| 89 | + } | ||
| 90 | +} | ||
| 91 | + | ||
| 92 | +void EventGrouper::ParseMc2CommInfo() | ||
| 93 | +{ | ||
| 94 | + Utils::TimeLogger t{"Parse Mc2CommInfo"}; | ||
| 95 | + Mc2CommInfoParser parser(hostPath_); | ||
| 96 | + mc2CommInfoData_ = parser.ParseData<ParserAdditionalInfo>(); | ||
| 97 | + if (parser.GetStatus() == ParserStatus::ERROR) | ||
| 98 | + { | ||
| 99 | + ERROR("Parse mc2 comm info failed."); | ||
| 100 | + result_ = false; | ||
| 101 | + } | ||
| 102 | +} | ||
| 103 | + | ||
| 69 | bool EventGrouper::Group() | 104 | bool EventGrouper::Group() |
| 70 | { | 105 | { |
| 71 | Utils::TimeLogger t{"Group all events"}; | 106 | Utils::TimeLogger t{"Group all events"}; |
| 72 | - const uint32_t poolSize = 8; | 107 | + result_ = true; |
| 108 | + captureStreamInfoData_.clear(); | ||
| 109 | + mc2CommInfoData_.clear(); | ||
| 110 | + const uint32_t poolSize = 10; | ||
| 73 | ThreadPool pool(poolSize); | 111 | ThreadPool pool(poolSize); |
| 74 | pool.Start(); | 112 | pool.Start(); |
| 75 | GroupTreeEvent(pool); | 113 | GroupTreeEvent(pool); |
| @@ -78,7 +116,7 @@ bool EventGrouper::Group() | |||
| 78 | pool.Stop(); | 116 | pool.Stop(); |
| 79 | SetApiEventKeys(); | 117 | SetApiEventKeys(); |
| 80 | RecordCANNWareHouses(); | 118 | RecordCANNWareHouses(); |
| 81 | - return true; | 119 | + return result_; |
| 82 | } | 120 | } |
| 83 | 121 | ||
| 84 | void EventGrouper::GroupTreeEvent(ThreadPool &pool) | 122 | void EventGrouper::GroupTreeEvent(ThreadPool &pool) |
| @@ -110,6 +148,11 @@ void EventGrouper::GroupTreeEvent(ThreadPool &pool) | |||
| 110 | GroupEvents<NodeAttrInfoParser, ParserCompactInfo, &CANNWarehouse::nodeAttrInfoEvents>( | 148 | GroupEvents<NodeAttrInfoParser, ParserCompactInfo, &CANNWarehouse::nodeAttrInfoEvents>( |
| 111 | "NodeAttrInfo", EventType::EVENT_TYPE_NODE_ATTR_INFO); | 149 | "NodeAttrInfo", EventType::EVENT_TYPE_NODE_ATTR_INFO); |
| 112 | }); | 150 | }); |
| 151 | + // if (kEnableCaptureStreamMc2CppParser) | ||
| 152 | + // { | ||
| 153 | + // pool.AddTask([this]() { ParseCaptureStreamInfo(); }); | ||
| 154 | + // pool.AddTask([this]() { ParseMc2CommInfo(); }); | ||
| 155 | + // } | ||
| 113 | pool.AddTask( | 156 | pool.AddTask( |
| 114 | [this]() | 157 | [this]() |
| 115 | { | 158 | { |
| @@ -18,6 +18,7 @@ | |||
| 18 | 18 | ||
| 19 | 19 | ||
| 20 | 20 | ||
| 21 | + | ||
| 21 | 22 | ||
| 22 | 23 | ||
| 23 | 24 | ||
| @@ -75,6 +76,8 @@ class EventGrouper | |||
| 75 | std::vector<std::shared_ptr<ParserCompactInfo>> &GetDpuTrackData(); | 76 | std::vector<std::shared_ptr<ParserCompactInfo>> &GetDpuTrackData(); |
| 76 | // 获取DpuKernelNameMap | 77 | // 获取DpuKernelNameMap |
| 77 | std::unordered_map<uint64_t, uint64_t> &GetDpuKernelNameMap(); | 78 | std::unordered_map<uint64_t, uint64_t> &GetDpuKernelNameMap(); |
| 79 | + const std::vector<std::shared_ptr<ParserCompactInfo>> &GetCaptureStreamInfoData() const; | ||
| 80 | + const std::vector<std::shared_ptr<ParserAdditionalInfo>> &GetMc2CommInfoData() const; | ||
| 78 | // ACL层建树白名单 | 81 | // ACL层建树白名单 |
| 79 | bool IsBuildTreeWithAcl(const std::shared_ptr<ParserApi> &trace); | 82 | bool IsBuildTreeWithAcl(const std::shared_ptr<ParserApi> &trace); |
| 80 | 83 | ||
| @@ -82,6 +85,8 @@ class EventGrouper | |||
| 82 | bool isKernelApiEvent(const std::shared_ptr<ParserApi> &trace); | 85 | bool isKernelApiEvent(const std::shared_ptr<ParserApi> &trace); |
| 83 | void InitLastKernelTimes(const std::set<uint32_t> &threadIds); | 86 | void InitLastKernelTimes(const std::set<uint32_t> &threadIds); |
| 84 | void RecordCANNWareHouses(); | 87 | void RecordCANNWareHouses(); |
| 88 | + void ParseCaptureStreamInfo(); | ||
| 89 | + void ParseMc2CommInfo(); | ||
| 85 | void SetApiEventKeys(); | 90 | void SetApiEventKeys(); |
| 86 | 91 | ||
| 87 | void GroupTreeEvent(ThreadPool &pool); | 92 | void GroupTreeEvent(ThreadPool &pool); |
| @@ -156,6 +161,9 @@ class EventGrouper | |||
| 156 | std::set<uint32_t> threadIds_; | 161 | std::set<uint32_t> threadIds_; |
| 157 | std::string hostPath_; | 162 | std::string hostPath_; |
| 158 | CANNWarehouses cannWarehouses_; // 所有threadId的数据 | 163 | CANNWarehouses cannWarehouses_; // 所有threadId的数据 |
| 164 | + std::vector<std::shared_ptr<ParserCompactInfo>> captureStreamInfoData_; | ||
| 165 | + std::vector<std::shared_ptr<ParserAdditionalInfo>> mc2CommInfoData_; | ||
| 166 | + std::atomic<bool> result_{true}; | ||
| 159 | // 记录已经处理好的kernelEvents的最晚时间(threadId, level, time) | 167 | // 记录已经处理好的kernelEvents的最晚时间(threadId, level, time) |
| 160 | std::unordered_map<uint32_t, std::unordered_map<uint16_t, std::pair<uint64_t, uint64_t>>> lastKernelTimes_; | 168 | std::unordered_map<uint32_t, std::unordered_map<uint16_t, std::pair<uint64_t, uint64_t>>> lastKernelTimes_; |
| 161 | }; | 169 | }; |
| @@ -40,13 +40,17 @@ void Cann::RTAddInfoCenter::Load(const std::string& path) | |||
| 40 | { | 40 | { |
| 41 | LoadDB(path); | 41 | LoadDB(path); |
| 42 | } | 42 | } |
| 43 | - LoadCaptureInfoDB(path); | 43 | +} |
| 44 | + | ||
| 45 | +void RTAddInfoCenter::SetCaptureStreamInfoData(const std::vector<CaptureStreamInfo>& data) | ||
| 46 | +{ | ||
| 47 | + captureStreamInfoData_ = data; | ||
| 48 | + captureInfoTimeRangeDict_.clear(); | ||
| 44 | BuildCaptureInfoTimeRange(); | 49 | BuildCaptureInfoTimeRange(); |
| 45 | } | 50 | } |
| 46 | 51 | ||
| 47 | void RTAddInfoCenter::Add(const RuntimeOpInfo& info) | 52 | void RTAddInfoCenter::Add(const RuntimeOpInfo& info) |
| 48 | { | 53 | { |
| 49 | - // 仅在 EventGrouper::Group() 内单线程调用(见头文件使用场景注释),与后续读取互不重叠 | ||
| 50 | std::string key = Utils::Join("_", info.deviceId, info.streamId, info.taskId); | 54 | std::string key = Utils::Join("_", info.deviceId, info.streamId, info.taskId); |
| 51 | runtimeOpInfoData_[key] = info; | 55 | runtimeOpInfoData_[key] = info; |
| 52 | dumpList_.push_back(info); | 56 | dumpList_.push_back(info); |
| @@ -116,41 +120,6 @@ void RTAddInfoCenter::BuildCaptureInfoTimeRange() | |||
| 116 | } | 120 | } |
| 117 | } | 121 | } |
| 118 | 122 | ||
| 119 | -void RTAddInfoCenter::LoadCaptureInfoDB(const std::string& path) | ||
| 120 | -{ | ||
| 121 | - StreamInfoDB streamInfoDB; | ||
| 122 | - std::string hostDbDirectory = Utils::File::PathJoin({path, streamInfoDB.GetDBName()}); | ||
| 123 | - DBRunner dbRunner(hostDbDirectory); | ||
| 124 | - if (!File::Exist(hostDbDirectory) || !dbRunner.CheckTableExists("CaptureStreamInfo")) | ||
| 125 | - { | ||
| 126 | - return; | ||
| 127 | - } | ||
| 128 | - std::string sql{ | ||
| 129 | - "SELECT model_id, timestamp, stream_id, original_stream_id, device_id, batch_id, capture_status " | ||
| 130 | - "FROM CaptureStreamInfo"}; | ||
| 131 | - using DataFormat = std::vector<std::tuple<uint64_t, uint64_t, uint32_t, uint32_t, uint16_t, uint32_t, uint16_t>>; | ||
| 132 | - DataFormat result; | ||
| 133 | - if (!dbRunner.QueryData(sql, result)) | ||
| 134 | - { | ||
| 135 | - ERROR("Query capture stream info data failed, db path is %.", hostDbDirectory); | ||
| 136 | - return; | ||
| 137 | - } | ||
| 138 | - | ||
| 139 | - for (const auto& row : result) | ||
| 140 | - { | ||
| 141 | - uint16_t deviceId; | ||
| 142 | - uint32_t originalStreamId; | ||
| 143 | - uint32_t batchId; | ||
| 144 | - uint16_t captureStatus; | ||
| 145 | - uint32_t streamId; | ||
| 146 | - uint64_t modelId; | ||
| 147 | - uint64_t timeStamp; | ||
| 148 | - std::tie(modelId, timeStamp, streamId, originalStreamId, deviceId, batchId, captureStatus) = row; | ||
| 149 | - CaptureStreamInfo info{modelId, timeStamp, streamId, originalStreamId, deviceId, batchId, captureStatus}; | ||
| 150 | - captureStreamInfoData_.push_back(info); | ||
| 151 | - } | ||
| 152 | -} | ||
| 153 | - | ||
| 154 | void RTAddInfoCenter::LoadDB(const std::string& path) | 123 | void RTAddInfoCenter::LoadDB(const std::string& path) |
| 155 | { | 124 | { |
| 156 | RtsTrackDB rtsTrackDb; | 125 | RtsTrackDB rtsTrackDb; |
| @@ -29,6 +29,8 @@ | |||
| 29 | 29 | ||
| 30 | 30 | ||
| 31 | 31 | ||
| 32 | +class RTAddInfoCenterUTest; | ||
| 33 | + | ||
| 32 | namespace Analysis | 34 | namespace Analysis |
| 33 | { | 35 | { |
| 34 | namespace Domain | 36 | namespace Domain |
| @@ -48,16 +50,14 @@ static constexpr uint16_t CAPTURE_STATUS_START = 0; | |||
| 48 | 50 | ||
| 49 | static constexpr uint16_t CAPTURE_STATUS_END = 1; | 51 | static constexpr uint16_t CAPTURE_STATUS_END = 1; |
| 50 | 52 | ||
| 51 | -// 该类是runtime 算子补充信息数据单例类,支持二进制解析写入与 db 兜底。 | 53 | +// 该类是runtime算子补充信息数据单例类 |
| 52 | -// 使用场景(单 PROF 目录、单线程、无并发): | ||
| 53 | -// 1. 一次解析生命周期内先由 EventGrouper::Group() 调 Add 写入二进制解析结果; | ||
| 54 | -// 2. Group 返回后再调 Load 做 db 兜底,随后只读(Get/GetAll/GetDumpList/...)。 | ||
| 55 | -// Add 与读取有严格先后、互不重叠,故不加锁;若未来出现多线程或跨 PROF 目录复用, | ||
| 56 | -// 需先补并发保护或按目录清空单例状态。 | ||
| 57 | class RTAddInfoCenter : public Utils::Singleton<RTAddInfoCenter> | 54 | class RTAddInfoCenter : public Utils::Singleton<RTAddInfoCenter> |
| 58 | { | 55 | { |
| 56 | + friend class ::RTAddInfoCenterUTest; | ||
| 57 | + | ||
| 59 | public: | 58 | public: |
| 60 | void Load(const std::string &path); | 59 | void Load(const std::string &path); |
| 60 | + void SetCaptureStreamInfoData(const std::vector<CaptureStreamInfo> &data); | ||
| 61 | void Add(const RuntimeOpInfo &info); | 61 | void Add(const RuntimeOpInfo &info); |
| 62 | RuntimeOpInfo Get(uint16_t deviceId, uint32_t streamId, uint32_t taskId); | 62 | RuntimeOpInfo Get(uint16_t deviceId, uint32_t streamId, uint32_t taskId); |
| 63 | const std::unordered_map<std::string, RuntimeOpInfo> &GetAll() const; | 63 | const std::unordered_map<std::string, RuntimeOpInfo> &GetAll() const; |
| @@ -68,7 +68,6 @@ class RTAddInfoCenter : public Utils::Singleton<RTAddInfoCenter> | |||
| 68 | 68 | ||
| 69 | private: | 69 | private: |
| 70 | void LoadDB(const std::string &path); | 70 | void LoadDB(const std::string &path); |
| 71 | - void LoadCaptureInfoDB(const std::string &path); | ||
| 72 | void BuildCaptureInfoTimeRange(); | 71 | void BuildCaptureInfoTimeRange(); |
| 73 | std::unordered_map<std::string, RuntimeOpInfo> runtimeOpInfoData_; | 72 | std::unordered_map<std::string, RuntimeOpInfo> runtimeOpInfoData_; |
| 74 | std::vector<RuntimeOpInfo> dumpList_; | 73 | std::vector<RuntimeOpInfo> dumpList_; |
| @@ -0,0 +1,145 @@ | |||
| 1 | +/* ------------------------------------------------------------------------- | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This file is part of the MindStudio project. | ||
| 4 | + * | ||
| 5 | + * MindStudio is licensed under Mulan PSL v2. | ||
| 6 | + * You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 7 | + * You may obtain a copy of Mulan PSL v2 at: | ||
| 8 | + * | ||
| 9 | + * http://license.coscl.org.cn/MulanPSL2 | ||
| 10 | + * | ||
| 11 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 12 | + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 13 | + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 14 | + * See the Mulan PSL v2 for more details. | ||
| 15 | + * -------------------------------------------------------------------------*/ | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | + | ||
| 23 | + | ||
| 24 | + | ||
| 25 | +namespace Analysis | ||
| 26 | +{ | ||
| 27 | +namespace Domain | ||
| 28 | +{ | ||
| 29 | +namespace | ||
| 30 | +{ | ||
| 31 | +std::string FormatModelIds(const std::set<uint32_t> &modelIds) | ||
| 32 | +{ | ||
| 33 | + std::ostringstream stream; | ||
| 34 | + stream << '{'; | ||
| 35 | + bool first = true; | ||
| 36 | + for (auto modelId : modelIds) | ||
| 37 | + { | ||
| 38 | + if (!first) | ||
| 39 | + { | ||
| 40 | + stream << ','; | ||
| 41 | + } | ||
| 42 | + stream << modelId; | ||
| 43 | + first = false; | ||
| 44 | + } | ||
| 45 | + stream << '}'; | ||
| 46 | + return stream.str(); | ||
| 47 | +} | ||
| 48 | +} // namespace | ||
| 49 | + | ||
| 50 | +CaptureStreamInfoDumper::CaptureStreamInfoDumper(const std::string &hostPath) | ||
| 51 | + : BaseDumper<CaptureStreamInfoDumper>(hostPath, "CaptureStreamInfo") | ||
| 52 | +{ | ||
| 53 | + MAKE_SHARED0_NO_OPERATION(database_, StreamInfoDB); | ||
| 54 | +} | ||
| 55 | + | ||
| 56 | +CaptureStreamInfoData CaptureStreamInfoDumper::FormatData(const std::vector<std::shared_ptr<ParserCompactInfo>> &input) | ||
| 57 | +{ | ||
| 58 | + auto sortedInput = input; | ||
| 59 | + std::stable_sort(sortedInput.begin(), sortedInput.end(), | ||
| 60 | + [](const std::shared_ptr<ParserCompactInfo> &left, const std::shared_ptr<ParserCompactInfo> &right) | ||
| 61 | + { | ||
| 62 | + if (!left) | ||
| 63 | + { | ||
| 64 | + return false; | ||
| 65 | + } | ||
| 66 | + if (!right) | ||
| 67 | + { | ||
| 68 | + return true; | ||
| 69 | + } | ||
| 70 | + return left->timeStamp < right->timeStamp; | ||
| 71 | + }); | ||
| 72 | + | ||
| 73 | + CaptureStreamInfoData output; | ||
| 74 | + if (!Utils::Reserve(output, sortedInput.size())) | ||
| 75 | + { | ||
| 76 | + ERROR("CaptureStreamInfoDumper: Reserve data failed."); | ||
| 77 | + return {}; | ||
| 78 | + } | ||
| 79 | + std::set<uint32_t> startModelSet; | ||
| 80 | + std::set<uint32_t> endModelSet; | ||
| 81 | + using RecordKey = std::tuple<uint16_t, uint32_t, uint32_t, uint32_t, uint16_t, uint64_t>; | ||
| 82 | + std::set<RecordKey> seenRecords; | ||
| 83 | + std::map<std::pair<uint16_t, uint32_t>, uint32_t> batchIdMap; | ||
| 84 | + uint64_t repeatedNum = 0; | ||
| 85 | + | ||
| 86 | + for (const auto &info : sortedInput) | ||
| 87 | + { | ||
| 88 | + if (!info) | ||
| 89 | + { | ||
| 90 | + continue; | ||
| 91 | + } | ||
| 92 | + const auto &record = info->data.captureStreamInfo; | ||
| 93 | + const auto deviceStreamKey = std::make_pair(record.deviceId, record.streamId); | ||
| 94 | + const uint32_t batchId = batchIdMap[deviceStreamKey]; | ||
| 95 | + const RecordKey recordKey = std::make_tuple(record.deviceId, record.modelId, record.originalStreamId, | ||
| 96 | + record.streamId, record.captureStatus, info->timeStamp); | ||
| 97 | + if (seenRecords.find(recordKey) != seenRecords.end()) | ||
| 98 | + { | ||
| 99 | + ++repeatedNum; | ||
| 100 | + continue; | ||
| 101 | + } | ||
| 102 | + if (record.captureStatus == 0) | ||
| 103 | + { | ||
| 104 | + startModelSet.insert(record.modelId); | ||
| 105 | + } | ||
| 106 | + if (record.captureStatus == 1 && !endModelSet.insert(record.modelId).second) | ||
| 107 | + { | ||
| 108 | + continue; | ||
| 109 | + } | ||
| 110 | + batchIdMap[deviceStreamKey] = batchId + 1; | ||
| 111 | + seenRecords.insert(recordKey); | ||
| 112 | + output.emplace_back(record.deviceId, record.modelId, record.originalStreamId, record.streamId, batchId, | ||
| 113 | + record.captureStatus, info->timeStamp); | ||
| 114 | + } | ||
| 115 | + | ||
| 116 | + if (startModelSet != endModelSet) | ||
| 117 | + { | ||
| 118 | + WARN("CaptureStreamInfoDumper: Capture start model ids are %, end model ids are %.", | ||
| 119 | + FormatModelIds(startModelSet), FormatModelIds(endModelSet)); | ||
| 120 | + } | ||
| 121 | + if (repeatedNum > 0) | ||
| 122 | + { | ||
| 123 | + WARN("CaptureStreamInfoDumper: There are % duplicate records.", repeatedNum); | ||
| 124 | + } | ||
| 125 | + return output; | ||
| 126 | +} | ||
| 127 | + | ||
| 128 | +CaptureStreamInfoDBData CaptureStreamInfoDumper::GenerateData(const CaptureStreamInfoData &input) | ||
| 129 | +{ | ||
| 130 | + CaptureStreamInfoDBData dbData; | ||
| 131 | + if (!Utils::Reserve(dbData, input.size())) | ||
| 132 | + { | ||
| 133 | + ERROR("CaptureStreamInfoDumper: Reserve DB data failed."); | ||
| 134 | + return {}; | ||
| 135 | + } | ||
| 136 | + for (const auto &item : input) | ||
| 137 | + { | ||
| 138 | + dbData.emplace_back(item.deviceId, item.modelId, item.originalStreamId, item.streamId, item.batchId, | ||
| 139 | + item.captureStatus, item.timeStamp); | ||
| 140 | + } | ||
| 141 | + return dbData; | ||
| 142 | +} | ||
| 143 | + | ||
| 144 | +} // namespace Domain | ||
| 145 | +} // namespace Analysis | ||
| @@ -0,0 +1,71 @@ | |||
| 1 | +/* ------------------------------------------------------------------------- | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This file is part of the MindStudio project. | ||
| 4 | + * | ||
| 5 | + * MindStudio is licensed under Mulan PSL v2. | ||
| 6 | + * You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 7 | + * You may obtain a copy of Mulan PSL v2 at: | ||
| 8 | + * | ||
| 9 | + * http://license.coscl.org.cn/MulanPSL2 | ||
| 10 | + * | ||
| 11 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 12 | + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 13 | + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 14 | + * See the Mulan PSL v2 for more details. | ||
| 15 | + * -------------------------------------------------------------------------*/ | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | + | ||
| 23 | + | ||
| 24 | + | ||
| 25 | + | ||
| 26 | +namespace Analysis | ||
| 27 | +{ | ||
| 28 | +namespace Domain | ||
| 29 | +{ | ||
| 30 | + | ||
| 31 | +struct CaptureStreamInfoDataItem | ||
| 32 | +{ | ||
| 33 | + uint16_t deviceId = 0; | ||
| 34 | + uint32_t modelId = 0; | ||
| 35 | + uint32_t originalStreamId = 0; | ||
| 36 | + uint32_t streamId = 0; | ||
| 37 | + uint32_t batchId = 0; | ||
| 38 | + uint16_t captureStatus = 0; | ||
| 39 | + uint64_t timeStamp = 0; | ||
| 40 | + | ||
| 41 | + CaptureStreamInfoDataItem() = default; | ||
| 42 | + CaptureStreamInfoDataItem(uint16_t deviceIdValue, uint32_t modelIdValue, uint32_t originalStreamIdValue, | ||
| 43 | + uint32_t streamIdValue, uint32_t batchIdValue, uint16_t captureStatusValue, | ||
| 44 | + uint64_t timeStampValue) | ||
| 45 | + : deviceId(deviceIdValue), | ||
| 46 | + modelId(modelIdValue), | ||
| 47 | + originalStreamId(originalStreamIdValue), | ||
| 48 | + streamId(streamIdValue), | ||
| 49 | + batchId(batchIdValue), | ||
| 50 | + captureStatus(captureStatusValue), | ||
| 51 | + timeStamp(timeStampValue) | ||
| 52 | + { | ||
| 53 | + } | ||
| 54 | +}; | ||
| 55 | + | ||
| 56 | +using CaptureStreamInfoData = std::vector<CaptureStreamInfoDataItem>; | ||
| 57 | +using CaptureStreamInfoDBData = | ||
| 58 | + std::vector<std::tuple<uint16_t, uint32_t, uint32_t, uint32_t, uint32_t, uint16_t, uint64_t>>; | ||
| 59 | + | ||
| 60 | +class CaptureStreamInfoDumper : public BaseDumper<CaptureStreamInfoDumper> | ||
| 61 | +{ | ||
| 62 | + public: | ||
| 63 | + explicit CaptureStreamInfoDumper(const std::string &hostPath); | ||
| 64 | + CaptureStreamInfoData FormatData(const std::vector<std::shared_ptr<ParserCompactInfo>> &input); | ||
| 65 | + CaptureStreamInfoDBData GenerateData(const CaptureStreamInfoData &input); | ||
| 66 | +}; | ||
| 67 | + | ||
| 68 | +} // namespace Domain | ||
| 69 | +} // namespace Analysis | ||
| 70 | + | ||
| 71 | + | ||
| @@ -0,0 +1,128 @@ | |||
| 1 | +/* ------------------------------------------------------------------------- | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This file is part of the MindStudio project. | ||
| 4 | + * | ||
| 5 | + * MindStudio is licensed under Mulan PSL v2. | ||
| 6 | + * You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 7 | + * You may obtain a copy of Mulan PSL v2 at: | ||
| 8 | + * | ||
| 9 | + * http://license.coscl.org.cn/MulanPSL2 | ||
| 10 | + * | ||
| 11 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 12 | + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 13 | + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 14 | + * See the Mulan PSL v2 for more details. | ||
| 15 | + * -------------------------------------------------------------------------*/ | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | + | ||
| 23 | +namespace Analysis | ||
| 24 | +{ | ||
| 25 | +namespace Domain | ||
| 26 | +{ | ||
| 27 | +using Host::Cann::MC2_COMM_STREAM_MAX_NUM; | ||
| 28 | +using Host::Cann::MsprofMc2CommInfo; | ||
| 29 | + | ||
| 30 | +namespace | ||
| 31 | +{ | ||
| 32 | +std::string JoinCommStreamIds(const MsprofMc2CommInfo &payload) | ||
| 33 | +{ | ||
| 34 | + if (payload.streamSize > MC2_COMM_STREAM_MAX_NUM) | ||
| 35 | + { | ||
| 36 | + return ""; | ||
| 37 | + } | ||
| 38 | + std::ostringstream stream; | ||
| 39 | + for (uint32_t i = 0; i < payload.streamSize; ++i) | ||
| 40 | + { | ||
| 41 | + if (i > 0) | ||
| 42 | + { | ||
| 43 | + stream << ','; | ||
| 44 | + } | ||
| 45 | + stream << payload.commStreamIds[i]; | ||
| 46 | + } | ||
| 47 | + return stream.str(); | ||
| 48 | +} | ||
| 49 | +} // namespace | ||
| 50 | + | ||
| 51 | +Mc2CommInfoDumper::Mc2CommInfoDumper(const std::string &hostPath) | ||
| 52 | + : BaseDumper<Mc2CommInfoDumper>(hostPath, "Mc2CommInfo") | ||
| 53 | +{ | ||
| 54 | + MAKE_SHARED0_NO_OPERATION(database_, Mc2CommInfoDB); | ||
| 55 | +} | ||
| 56 | + | ||
| 57 | +Mc2CommInfoData Mc2CommInfoDumper::GenerateData(const Mc2CommInfoInput &input) | ||
| 58 | +{ | ||
| 59 | + std::map<uint32_t, std::set<uint32_t>> captureStreamMap; | ||
| 60 | + for (const auto &capture : input.captureData) | ||
| 61 | + { | ||
| 62 | + captureStreamMap[capture.originalStreamId].insert(capture.streamId); | ||
| 63 | + } | ||
| 64 | + | ||
| 65 | + size_t supplementSize = 0; | ||
| 66 | + for (const auto &info : input.mc2Data) | ||
| 67 | + { | ||
| 68 | + if (!info) | ||
| 69 | + { | ||
| 70 | + continue; | ||
| 71 | + } | ||
| 72 | + const auto payload = Utils::ReinterpretConvert<const MsprofMc2CommInfo *>(info->data); | ||
| 73 | + auto captureIt = captureStreamMap.find(payload->streamId); | ||
| 74 | + if (captureIt != captureStreamMap.end()) | ||
| 75 | + { | ||
| 76 | + supplementSize += captureIt->second.size(); | ||
| 77 | + } | ||
| 78 | + } | ||
| 79 | + | ||
| 80 | + Mc2CommInfoData output; | ||
| 81 | + if (!Utils::Reserve(output, input.mc2Data.size() + supplementSize)) | ||
| 82 | + { | ||
| 83 | + ERROR("Mc2CommInfoDumper: Reserve data failed."); | ||
| 84 | + return {}; | ||
| 85 | + } | ||
| 86 | + uint64_t invalidStreamSizeNum = 0; | ||
| 87 | + for (const auto &info : input.mc2Data) | ||
| 88 | + { | ||
| 89 | + if (!info) | ||
| 90 | + { | ||
| 91 | + continue; | ||
| 92 | + } | ||
| 93 | + const auto payload = Utils::ReinterpretConvert<const MsprofMc2CommInfo *>(info->data); | ||
| 94 | + if (payload->streamSize > MC2_COMM_STREAM_MAX_NUM) | ||
| 95 | + { | ||
| 96 | + ++invalidStreamSizeNum; | ||
| 97 | + } | ||
| 98 | + output.emplace_back(std::to_string(payload->groupName), payload->rankSize, payload->rankId, payload->usrRankId, | ||
| 99 | + payload->streamId, JoinCommStreamIds(*payload)); | ||
| 100 | + } | ||
| 101 | + for (const auto &info : input.mc2Data) | ||
| 102 | + { | ||
| 103 | + if (!info) | ||
| 104 | + { | ||
| 105 | + continue; | ||
| 106 | + } | ||
| 107 | + const auto payload = Utils::ReinterpretConvert<const MsprofMc2CommInfo *>(info->data); | ||
| 108 | + auto captureIt = captureStreamMap.find(payload->streamId); | ||
| 109 | + if (captureIt == captureStreamMap.end()) | ||
| 110 | + { | ||
| 111 | + continue; | ||
| 112 | + } | ||
| 113 | + for (auto modelStreamId : captureIt->second) | ||
严重程度: 提示 问题: 第二个循环对 原因: 已用 怎么改: 在第二个循环对非法记录跳过补充生成:
或在补充行也写空时附加标记,便于下游区分损坏与空集。 ![]() ![]() | |||
| 114 | + { | ||
| 115 | + output.emplace_back(std::to_string(payload->groupName), payload->rankSize, payload->rankId, | ||
| 116 | + payload->usrRankId, modelStreamId, JoinCommStreamIds(*payload)); | ||
| 117 | + } | ||
| 118 | + } | ||
| 119 | + if (invalidStreamSizeNum > 0) | ||
| 120 | + { | ||
| 121 | + ERROR("Mc2CommInfoDumper: % records have stream size greater than max stream size %.", invalidStreamSizeNum, | ||
| 122 | + MC2_COMM_STREAM_MAX_NUM); | ||
| 123 | + } | ||
| 124 | + return output; | ||
| 125 | +} | ||
| 126 | + | ||
| 127 | +} // namespace Domain | ||
| 128 | +} // namespace Analysis | ||
| @@ -0,0 +1,58 @@ | |||
| 1 | +/* ------------------------------------------------------------------------- | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This file is part of the MindStudio project. | ||
| 4 | + * | ||
| 5 | + * MindStudio is licensed under Mulan PSL v2. | ||
| 6 | + * You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 7 | + * You may obtain a copy of Mulan PSL v2 at: | ||
| 8 | + * | ||
| 9 | + * http://license.coscl.org.cn/MulanPSL2 | ||
| 10 | + * | ||
| 11 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 12 | + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 13 | + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 14 | + * See the Mulan PSL v2 for more details. | ||
| 15 | + * -------------------------------------------------------------------------*/ | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | + | ||
| 23 | + | ||
| 24 | + | ||
| 25 | + | ||
| 26 | + | ||
| 27 | +namespace Analysis | ||
| 28 | +{ | ||
| 29 | +namespace Domain | ||
| 30 | +{ | ||
| 31 | + | ||
| 32 | +using Mc2CommInfoData = std::vector<std::tuple<std::string, uint32_t, uint32_t, uint32_t, uint32_t, std::string>>; | ||
| 33 | + | ||
| 34 | +struct Mc2CommInfoInput | ||
| 35 | +{ | ||
| 36 | + Mc2CommInfoInput(const std::vector<std::shared_ptr<ParserAdditionalInfo>> &mc2Data, | ||
| 37 | + const CaptureStreamInfoData &captureData) | ||
| 38 | + : mc2Data(mc2Data), captureData(captureData) | ||
| 39 | + { | ||
| 40 | + } | ||
| 41 | + | ||
| 42 | + bool empty() const { return mc2Data.empty(); } | ||
| 43 | + | ||
| 44 | + const std::vector<std::shared_ptr<ParserAdditionalInfo>> &mc2Data; | ||
| 45 | + const CaptureStreamInfoData &captureData; | ||
| 46 | +}; | ||
| 47 | + | ||
| 48 | +class Mc2CommInfoDumper : public BaseDumper<Mc2CommInfoDumper> | ||
| 49 | +{ | ||
| 50 | + public: | ||
| 51 | + explicit Mc2CommInfoDumper(const std::string &hostPath); | ||
| 52 | + Mc2CommInfoData GenerateData(const Mc2CommInfoInput &input); | ||
| 53 | +}; | ||
| 54 | + | ||
| 55 | +} // namespace Domain | ||
| 56 | +} // namespace Analysis | ||
| 57 | + | ||
| 58 | + | ||
| @@ -45,6 +45,7 @@ enum class AdditionalInfoFormat : uint8_t | |||
| 45 | MEMORY_APPLICATION_TYPE, | 45 | MEMORY_APPLICATION_TYPE, |
| 46 | MULTI_THREAD_TYPE, | 46 | MULTI_THREAD_TYPE, |
| 47 | TASK_MEMORY_INFO_TYPE, | 47 | TASK_MEMORY_INFO_TYPE, |
| 48 | + MC2_COMM_INFO_TYPE, | ||
| 48 | }; | 49 | }; |
| 49 | 50 | ||
| 50 | enum class VariableInfoFormat : uint8_t | 51 | enum class VariableInfoFormat : uint8_t |
| @@ -379,13 +379,13 @@ TEST_F(ExportManagerUTest, ShouldBuildProcessorDependencyClosure) | |||
| 379 | TopoGraphBuilder builder; | 379 | TopoGraphBuilder builder; |
| 380 | ASSERT_TRUE(builder.Build(context, {{TopoNodeStage::DATA_PROCESSING, PROCESSOR_NAME_OVERLAP_ANALYSIS}}, processes)); | 380 | ASSERT_TRUE(builder.Build(context, {{TopoNodeStage::DATA_PROCESSING, PROCESSOR_NAME_OVERLAP_ANALYSIS}}, processes)); |
| 381 | 381 | ||
| 382 | - EXPECT_EQ(processes.size(), 6UL); | 382 | + EXPECT_EQ(processes.size(), 5UL); |
| 383 | bool overlapFound = false; | 383 | bool overlapFound = false; |
| 384 | for (const auto& process : processes) { | 384 | for (const auto& process : processes) { |
| 385 | if (process.second.processName == PROCESSOR_NAME_OVERLAP_ANALYSIS) { | 385 | if (process.second.processName == PROCESSOR_NAME_OVERLAP_ANALYSIS) { |
| 386 | overlapFound = true; | 386 | overlapFound = true; |
| 387 | - EXPECT_EQ(process.second.processDependence.size(), 4UL); | 387 | + EXPECT_EQ(process.second.processDependence.size(), 3UL); |
| 388 | - EXPECT_EQ(process.second.paramTypes.size(), 4UL); | 388 | + EXPECT_EQ(process.second.paramTypes.size(), 3UL); |
| 389 | } | 389 | } |
| 390 | EXPECT_NE(process.second.processName, "DataProcessorCollector"); | 390 | EXPECT_NE(process.second.processName, "DataProcessorCollector"); |
| 391 | } | 391 | } |
| @@ -16,6 +16,7 @@ | |||
| 16 | 16 | ||
| 17 | 17 | ||
| 18 | 18 | ||
| 19 | + | ||
| 19 | 20 | ||
| 20 | 21 | ||
| 21 | 22 | ||
| @@ -118,3 +119,22 @@ TEST_F(Mc2CommInfoProcessorUTest, TestRunShouldReturnFalseWhenSaveToDataInventor | |||
| 118 | EXPECT_FALSE(processor.Run(dataInventory, PROCESSOR_MC2_COMM_INFO)); | 119 | EXPECT_FALSE(processor.Run(dataInventory, PROCESSOR_MC2_COMM_INFO)); |
| 119 | MOCKER_CPP(&DataProcessor::SaveToDataInventory<MC2CommInfoData>).reset(); | 120 | MOCKER_CPP(&DataProcessor::SaveToDataInventory<MC2CommInfoData>).reset(); |
| 120 | } | 121 | } |
| 122 | + | ||
| 123 | +TEST_F(Mc2CommInfoProcessorUTest, ShouldPreserveUint32KfcStreamId) | ||
| 124 | +{ | ||
| 125 | + using WideMc2Format = | ||
| 126 | + std::vector<std::tuple<std::string, uint32_t, uint32_t, uint32_t, uint32_t, std::string>>; | ||
| 127 | + WideMc2Format wideData{{"99", 2, 0, 0, 70003, "70004,70005"}}; | ||
| 128 | + DBRunner dbRunner(File::PathJoin({PROF_PATH, HOST, SQLITE, DB_SUFFIX})); | ||
| 129 | + ASSERT_TRUE(dbRunner.InsertData(TABLE_NAME, wideData)); | ||
| 130 | + | ||
| 131 | + DataInventory dataInventory; | ||
| 132 | + auto processor = Mc2CommInfoProcessor(PROF_PATH); | ||
| 133 | + ASSERT_TRUE(processor.Run(dataInventory, PROCESSOR_MC2_COMM_INFO)); | ||
| 134 | + auto result = dataInventory.GetPtr<std::vector<MC2CommInfoData>>(); | ||
| 135 | + ASSERT_TRUE(result); | ||
| 136 | + auto record = std::find_if(result->begin(), result->end(), [](const MC2CommInfoData &item) { | ||
| 137 | + return item.aiCpuKfcStreamId == 70003; | ||
| 138 | + }); | ||
| 139 | + EXPECT_NE(result->end(), record); | ||
| 140 | +} | ||
| @@ -0,0 +1,61 @@ | |||
| 1 | +/* ------------------------------------------------------------------------- | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This file is part of the MindStudio project. | ||
| 4 | + * | ||
| 5 | + * MindStudio is licensed under Mulan PSL v2. | ||
| 6 | + * You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 7 | + * You may obtain a copy of Mulan PSL v2 at: | ||
| 8 | + * | ||
| 9 | + * http://license.coscl.org.cn/MulanPSL2 | ||
| 10 | + * | ||
| 11 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 12 | + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 13 | + * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 14 | + * See the Mulan PSL v2 for more details. | ||
| 15 | + * -------------------------------------------------------------------------*/ | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | +using namespace Analysis::Application; | ||
| 23 | +using namespace Analysis::Domain; | ||
| 24 | +using namespace Analysis::Infra; | ||
| 25 | + | ||
| 26 | +TEST(OverlapAnalysisProcessorUTest, WideMc2StreamIdShouldNotFilterDifferentUint16TaskStream) | ||
| 27 | +{ | ||
| 28 | + AscendTaskData ascendTask; | ||
| 29 | + ascendTask.deviceId = 0; | ||
| 30 | + ascendTask.streamId = 4467; | ||
| 31 | + ascendTask.taskId = 1; | ||
| 32 | + ascendTask.contextId = 2; | ||
| 33 | + ascendTask.batchId = 3; | ||
| 34 | + ascendTask.timestamp = 10; | ||
| 35 | + ascendTask.duration = 10; | ||
| 36 | + | ||
| 37 | + TaskInfoData computeTask; | ||
| 38 | + computeTask.deviceId = ascendTask.deviceId; | ||
| 39 | + computeTask.streamId = ascendTask.streamId; | ||
| 40 | + computeTask.taskId = ascendTask.taskId; | ||
| 41 | + computeTask.contextId = ascendTask.contextId; | ||
| 42 | + computeTask.batchId = ascendTask.batchId; | ||
| 43 | + computeTask.opName = "compute_op"; | ||
| 44 | + | ||
| 45 | + MC2CommInfoData mc2CommInfo; | ||
| 46 | + mc2CommInfo.aiCpuKfcStreamId = 70003; | ||
| 47 | + | ||
| 48 | + DataInventory dataInventory; | ||
| 49 | + ASSERT_TRUE(dataInventory.Inject(std::make_shared<std::vector<AscendTaskData>>(1, ascendTask))); | ||
| 50 | + ASSERT_TRUE(dataInventory.Inject(std::make_shared<std::vector<TaskInfoData>>(1, computeTask))); | ||
| 51 | + ASSERT_TRUE(dataInventory.Inject(std::make_shared<std::vector<MC2CommInfoData>>(1, mc2CommInfo))); | ||
| 52 | + | ||
| 53 | + OverlapAnalysisProcessor processor; | ||
| 54 | + ASSERT_TRUE(processor.Run(dataInventory, PROCESSOR_NAME_OVERLAP_ANALYSIS)); | ||
| 55 | + auto result = dataInventory.GetPtr<std::vector<OverlapAnalysisData>>(); | ||
| 56 | + ASSERT_TRUE(result); | ||
| 57 | + ASSERT_EQ(1UL, result->size()); | ||
| 58 | + EXPECT_EQ(OverlapAnalysisType::COMPUTE, result->front().type); | ||
| 59 | + EXPECT_EQ(10UL, result->front().timestamp); | ||
| 60 | + EXPECT_EQ(10UL, result->front().duration); | ||
| 61 | +} | ||
| @@ -148,4 +148,5 @@ TEST_F(ApiEventParserUTest, TestProduceDataShouldReturnEmptyWhenPopNullptr) | |||
| 148 | auto parser = std::make_shared<ApiEventParser>(File::PathJoin(std::vector<std::string>{DATA_DIR, "host", "data"})); | 148 | auto parser = std::make_shared<ApiEventParser>(File::PathJoin(std::vector<std::string>{DATA_DIR, "host", "data"})); |
| 149 | auto data = parser->ParseData<ParserApi>(); | 149 | auto data = parser->ParseData<ParserApi>(); |
| 150 | EXPECT_EQ(0, data.size()); | 150 | EXPECT_EQ(0, data.size()); |
| 151 | + MOCKER_CPP(&ChunkGenerator::Pop).reset(); | ||
| 151 | } | 152 | } |
| @@ -0,0 +1,157 @@ | |||
| 1 | +/* ------------------------------------------------------------------------- | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This file is part of the MindStudio project. | ||
| 4 | + * | ||
| 5 | + * MindStudio is licensed under Mulan PSL v2. | ||
| 6 | + * You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 7 | + * You may obtain a copy of Mulan PSL v2 at: | ||
| 8 | + * | ||
| 9 | + * http://license.coscl.org.cn/MulanPSL2 | ||
| 10 | + * | ||
| 11 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 12 | + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 13 | + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 14 | + * See the Mulan PSL v2 for more details. | ||
| 15 | + * -------------------------------------------------------------------------*/ | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | + | ||
| 23 | + | ||
| 24 | + | ||
| 25 | +using namespace Analysis::Domain::Host::Cann; | ||
| 26 | +using namespace Analysis::Utils; | ||
| 27 | + | ||
| 28 | +namespace { | ||
| 29 | +const std::string CAPTURE_ROOT = "./capture_stream_parser"; | ||
| 30 | +const std::string CAPTURE_DATA_PATH = File::PathJoin({CAPTURE_ROOT, "host", "data"}); | ||
| 31 | +} | ||
| 32 | + | ||
| 33 | +class CaptureStreamInfoParserUTest : public testing::Test { | ||
| 34 | +protected: | ||
| 35 | + void SetUp() override | ||
| 36 | + { | ||
| 37 | + if (File::Exist(CAPTURE_ROOT)) { | ||
| 38 | + EXPECT_TRUE(File::RemoveDir(CAPTURE_ROOT, 0)); | ||
| 39 | + } | ||
| 40 | + EXPECT_TRUE(File::CreateDir(CAPTURE_ROOT)); | ||
| 41 | + EXPECT_TRUE(File::CreateDir(File::PathJoin({CAPTURE_ROOT, "host"}))); | ||
| 42 | + EXPECT_TRUE(File::CreateDir(CAPTURE_DATA_PATH)); | ||
| 43 | + } | ||
| 44 | + | ||
| 45 | + void TearDown() override | ||
| 46 | + { | ||
| 47 | + if (File::Exist(CAPTURE_ROOT)) { | ||
| 48 | + EXPECT_TRUE(File::RemoveDir(CAPTURE_ROOT, 0)); | ||
| 49 | + } | ||
| 50 | + } | ||
| 51 | +}; | ||
| 52 | + | ||
| 53 | +TEST_F(CaptureStreamInfoParserUTest, ShouldNotLookupV1CaptureSlice) | ||
| 54 | +{ | ||
| 55 | + MsprofCompactInfo info{}; | ||
| 56 | + info.timeStamp = 101; | ||
| 57 | + info.data.captureStreamInfo.deviceId = 2; | ||
| 58 | + info.data.captureStreamInfo.modelId = 3; | ||
| 59 | + info.data.captureStreamInfo.originalStreamId = 4; | ||
| 60 | + info.data.captureStreamInfo.modelStreamId = 5; | ||
| 61 | + info.data.captureStreamInfo.captureStatus = 0; | ||
| 62 | + std::vector<MsprofCompactInfo> input{info}; | ||
| 63 | + ASSERT_TRUE(WriteBin(input, CAPTURE_DATA_PATH, "unaging.compact.capture_stream_info.slice_0")); | ||
| 64 | + | ||
| 65 | + CaptureStreamInfoParser parser(CAPTURE_DATA_PATH); | ||
| 66 | + auto output = parser.ParseData<ParserCompactInfo>(); | ||
| 67 | + ASSERT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR)); | ||
| 68 | + EXPECT_TRUE(output.empty()); | ||
| 69 | +} | ||
| 70 | + | ||
| 71 | +TEST_F(CaptureStreamInfoParserUTest, ShouldNotLookupCaptureSlicesWhenBothVersionsExist) | ||
| 72 | +{ | ||
| 73 | + MsprofCompactInfo v1{}; | ||
| 74 | + v1.timeStamp = 100; | ||
| 75 | + v1.data.captureStreamInfo.modelId = 11; | ||
| 76 | + std::vector<MsprofCompactInfo> v1Input{v1}; | ||
| 77 | + ASSERT_TRUE(WriteBin(v1Input, CAPTURE_DATA_PATH, "aging.compact.capture_stream_info.slice_0")); | ||
| 78 | + | ||
| 79 | + MsprofCompactInfo v2{}; | ||
| 80 | + v2.timeStamp = 200; | ||
| 81 | + v2.data.captureStreamInfo.modelId = 22; | ||
| 82 | + std::vector<MsprofCompactInfo> v2Input{v2}; | ||
| 83 | + ASSERT_TRUE(WriteBin(v2Input, CAPTURE_DATA_PATH, "unaging.compact.capture_stream_info_v2.slice_0")); | ||
| 84 | + | ||
| 85 | + CaptureStreamInfoParser parser(CAPTURE_DATA_PATH); | ||
| 86 | + auto output = parser.ParseData<ParserCompactInfo>(); | ||
| 87 | + ASSERT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR)); | ||
| 88 | + EXPECT_TRUE(output.empty()); | ||
| 89 | +} | ||
| 90 | + | ||
| 91 | +TEST_F(CaptureStreamInfoParserUTest, ShouldNotLookupV2CaptureSlice) | ||
| 92 | +{ | ||
| 93 | + MsprofCompactInfo info{}; | ||
| 94 | + info.timeStamp = 202; | ||
| 95 | + info.data.captureStreamInfoV2.deviceId = 6; | ||
| 96 | + info.data.captureStreamInfoV2.modelId = 70001; | ||
| 97 | + info.data.captureStreamInfoV2.originalStreamId = 70002; | ||
| 98 | + info.data.captureStreamInfoV2.streamId = 70003; | ||
| 99 | + info.data.captureStreamInfoV2.captureStatus = 1; | ||
| 100 | + std::vector<MsprofCompactInfo> input{info}; | ||
| 101 | + ASSERT_TRUE(WriteBin(input, CAPTURE_DATA_PATH, "unaging.compact.capture_stream_info_v2.slice_0")); | ||
| 102 | + | ||
| 103 | + CaptureStreamInfoParser parser(CAPTURE_DATA_PATH); | ||
| 104 | + auto output = parser.ParseData<ParserCompactInfo>(); | ||
| 105 | + ASSERT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR)); | ||
| 106 | + EXPECT_TRUE(output.empty()); | ||
| 107 | +} | ||
| 108 | + | ||
| 109 | +TEST_F(CaptureStreamInfoParserUTest, ShouldNotLookupV2WhenV1MarkerExists) | ||
| 110 | +{ | ||
| 111 | + std::ofstream marker(File::PathJoin({CAPTURE_DATA_PATH, | ||
| 112 | + "unaging.compact.capture_stream_info.slice_0.done"})); | ||
| 113 | + ASSERT_TRUE(marker.good()); | ||
| 114 | + marker.close(); | ||
| 115 | + | ||
| 116 | + MsprofCompactInfo info{}; | ||
| 117 | + info.data.captureStreamInfoV2.modelId = 70001; | ||
| 118 | + std::vector<MsprofCompactInfo> input{info}; | ||
| 119 | + ASSERT_TRUE(WriteBin(input, CAPTURE_DATA_PATH, "unaging.compact.capture_stream_info_v2.slice_0")); | ||
| 120 | + | ||
| 121 | + CaptureStreamInfoParser parser(CAPTURE_DATA_PATH); | ||
| 122 | + auto output = parser.ParseData<ParserCompactInfo>(); | ||
| 123 | + ASSERT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR)); | ||
| 124 | + EXPECT_TRUE(output.empty()); | ||
| 125 | +} | ||
| 126 | + | ||
| 127 | +TEST_F(CaptureStreamInfoParserUTest, ShouldSkipRecordWithInvalidMagic) | ||
| 128 | +{ | ||
| 129 | + MsprofCompactInfo info{}; | ||
| 130 | + info.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM + 1; | ||
| 131 | + std::vector<MsprofCompactInfo> input{info}; | ||
| 132 | + ASSERT_TRUE(WriteBin(input, CAPTURE_DATA_PATH, "unaging.compact.capture_stream_info.slice_0")); | ||
| 133 | + | ||
| 134 | + CaptureStreamInfoParser parser(CAPTURE_DATA_PATH); | ||
| 135 | + auto output = parser.ParseData<ParserCompactInfo>(); | ||
| 136 | + EXPECT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR)); | ||
| 137 | + EXPECT_TRUE(output.empty()); | ||
| 138 | +} | ||
| 139 | + | ||
| 140 | +TEST_F(CaptureStreamInfoParserUTest, ShouldReturnTrueAndEmptyWhenNoFileExists) | ||
| 141 | +{ | ||
| 142 | + CaptureStreamInfoParser parser(CAPTURE_DATA_PATH); | ||
| 143 | + auto output = parser.ParseData<ParserCompactInfo>(); | ||
| 144 | + EXPECT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR)); | ||
| 145 | + EXPECT_TRUE(output.empty()); | ||
| 146 | +} | ||
| 147 | + | ||
| 148 | +TEST_F(CaptureStreamInfoParserUTest, ShouldReturnTrueAndEmptyWhenTruncatedRecordExists) | ||
| 149 | +{ | ||
| 150 | + std::vector<uint8_t> input(sizeof(MsprofCompactInfo) - 1, 0); | ||
| 151 | + ASSERT_TRUE(WriteBin(input, CAPTURE_DATA_PATH, "unaging.compact.capture_stream_info.slice_0")); | ||
| 152 | + | ||
| 153 | + CaptureStreamInfoParser parser(CAPTURE_DATA_PATH); | ||
| 154 | + auto output = parser.ParseData<ParserCompactInfo>(); | ||
| 155 | + EXPECT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR)); | ||
| 156 | + EXPECT_TRUE(output.empty()); | ||
| 157 | +} | ||
| @@ -75,12 +75,13 @@ protected: | |||
| 75 | std::vector<MsprofCompactInfo> agingTraces; | 75 | std::vector<MsprofCompactInfo> agingTraces; |
| 76 | std::vector<MsprofCompactInfo> unAgingTraces; | 76 | std::vector<MsprofCompactInfo> unAgingTraces; |
| 77 | for (uint32_t i = 0; i < DATA_NUM; ++i) { | 77 | for (uint32_t i = 0; i < DATA_NUM; ++i) { |
| 78 | - MsprofCompactInfo info; | 78 | + MsprofCompactInfo info{}; |
| 79 | info.level = level; | 79 | info.level = level; |
| 80 | info.type = static_cast<uint32_t>(type); | 80 | info.type = static_cast<uint32_t>(type); |
| 81 | info.threadId = i; | 81 | info.threadId = i; |
| 82 | info.dataLen = dataLen; | 82 | info.dataLen = dataLen; |
| 83 | info.timeStamp = DATA_NUM + i; | 83 | info.timeStamp = DATA_NUM + i; |
| 84 | + info.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM; | ||
| 84 | if (i >= DATA_NUM - invalidDataNum) { | 85 | if (i >= DATA_NUM - invalidDataNum) { |
| 85 | info.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM + 1; | 86 | info.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM + 1; |
| 86 | } | 87 | } |
| @@ -225,6 +226,7 @@ TEST_F(CompactInfoParserUTest, TestTaskTrackParserProduceDataShouldReturn8Compac | |||
| 225 | Check(compactInfo, EventType::EVENT_TYPE_TASK_TRACK, MSPROF_REPORT_NODE_LEVEL, | 226 | Check(compactInfo, EventType::EVENT_TYPE_TASK_TRACK, MSPROF_REPORT_NODE_LEVEL, |
| 226 | DATA_NUM - flipTaskNum - maintenanceTaskNum); | 227 | DATA_NUM - flipTaskNum - maintenanceTaskNum); |
| 227 | EXPECT_EQ(flipTaskNum, flipTask.size()); | 228 | EXPECT_EQ(flipTaskNum, flipTask.size()); |
| 229 | + MOCKER_CPP(&Context::IsAllExport).reset(); | ||
| 228 | } | 230 | } |
| 229 | 231 | ||
| 230 | TEST_F(CompactInfoParserUTest, TestTaskTrackParserProduceDataShouldReturnEmptyWhenReserveFailed) | 232 | TEST_F(CompactInfoParserUTest, TestTaskTrackParserProduceDataShouldReturnEmptyWhenReserveFailed) |
| @@ -349,12 +351,13 @@ static void GenDpuTrackData(uint16_t dataNum = DATA_NUM, uint16_t invalidDataNum | |||
| 349 | std::vector<MsprofCompactInfo> agingTraces; | 351 | std::vector<MsprofCompactInfo> agingTraces; |
| 350 | std::vector<MsprofCompactInfo> unAgingTraces; | 352 | std::vector<MsprofCompactInfo> unAgingTraces; |
| 351 | for (uint32_t i = 0; i < dataNum; ++i) { | 353 | for (uint32_t i = 0; i < dataNum; ++i) { |
| 352 | - MsprofCompactInfo info; | 354 | + MsprofCompactInfo info{}; |
| 353 | info.level = level; | 355 | info.level = level; |
| 354 | info.type = static_cast<uint32_t>(EventType::EVENT_TYPE_DPU_TASK_TRACK); | 356 | info.type = static_cast<uint32_t>(EventType::EVENT_TYPE_DPU_TASK_TRACK); |
| 355 | info.threadId = i; | 357 | info.threadId = i; |
| 356 | info.dataLen = dataLen; | 358 | info.dataLen = dataLen; |
| 357 | info.timeStamp = dataNum + i; | 359 | info.timeStamp = dataNum + i; |
| 360 | + info.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM; | ||
| 358 | if (i >= dataNum - invalidDataNum) { | 361 | if (i >= dataNum - invalidDataNum) { |
| 359 | info.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM + 1; | 362 | info.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM + 1; |
| 360 | } | 363 | } |
| @@ -1257,4 +1257,4 @@ TEST_F(ContextUTest, TestGetCannVersionShouldReturnEmptyWhenCannVersionInfoIsInv | |||
| 1257 | auto cannVersion = Context::GetInstance().GetCannVersion(HOST_ID, File::PathJoin(std::vector<std::string>{CONTEXT_DIR, TEST_DIR})); | 1257 | auto cannVersion = Context::GetInstance().GetCannVersion(HOST_ID, File::PathJoin(std::vector<std::string>{CONTEXT_DIR, TEST_DIR})); |
| 1258 | EXPECT_EQ(cannVersion.size(), 0); | 1258 | EXPECT_EQ(cannVersion.size(), 0); |
| 1259 | MOCKER_CPP(&Context::CheckInfoValueIsValid).reset(); | 1259 | MOCKER_CPP(&Context::CheckInfoValueIsValid).reset(); |
| 1260 | -} | 1260 | +} |
| @@ -31,6 +31,7 @@ | |||
| 31 | 31 | ||
| 32 | 32 | ||
| 33 | 33 | ||
| 34 | + | ||
| 34 | 35 | ||
| 35 | using namespace Analysis::Utils; | 36 | using namespace Analysis::Utils; |
| 36 | using namespace Analysis::Domain::Host::Cann; | 37 | using namespace Analysis::Domain::Host::Cann; |
| @@ -324,6 +325,43 @@ TEST_F(EventGrouperUTest, TestGroupShouldReturnEmptyWhenDataDirEmpty) | |||
| 324 | EXPECT_EQ(true, File::RemoveDir(fakeDataDir, 0)); | 325 | EXPECT_EQ(true, File::RemoveDir(fakeDataDir, 0)); |
| 325 | } | 326 | } |
| 326 | 327 | ||
| 328 | +TEST_F(EventGrouperUTest, ShouldSkipCaptureAndMc2InGroupWhenCppParserDisabled) | ||
| 329 | +{ | ||
| 330 | + const std::string fakeDataDir = "./fakeCaptureData"; | ||
| 331 | + const std::string hostDataDir = fakeDataDir + "/host/data"; | ||
| 332 | + File::RemoveDir(fakeDataDir, 0); | ||
| 333 | + ASSERT_TRUE(File::CreateDir(fakeDataDir)); | ||
| 334 | + ASSERT_TRUE(File::CreateDir(fakeDataDir + "/host")); | ||
| 335 | + ASSERT_TRUE(File::CreateDir(hostDataDir)); | ||
| 336 | + | ||
| 337 | + MsprofCompactInfo capture{}; | ||
| 338 | + capture.threadId = 9; | ||
| 339 | + capture.timeStamp = 123; | ||
| 340 | + capture.data.captureStreamInfo.deviceId = 1; | ||
| 341 | + capture.data.captureStreamInfo.modelId = 7; | ||
| 342 | + capture.data.captureStreamInfo.originalStreamId = 52; | ||
| 343 | + capture.data.captureStreamInfo.modelStreamId = 70; | ||
| 344 | + capture.data.captureStreamInfo.captureStatus = 0; | ||
| 345 | + std::vector<MsprofCompactInfo> captureRecords{capture}; | ||
| 346 | + ASSERT_TRUE(WriteBin(captureRecords, hostDataDir, | ||
| 347 | + "unaging.compact.capture_stream_info.slice_0")); | ||
| 348 | + | ||
| 349 | + MsprofAdditionalInfo mc2{}; | ||
| 350 | + auto payload = ReinterpretConvert<MsprofMc2CommInfo *>(mc2.data); | ||
| 351 | + payload->groupName = 99; | ||
| 352 | + payload->streamId = 52; | ||
| 353 | + std::vector<MsprofAdditionalInfo> mc2Records{mc2}; | ||
| 354 | + ASSERT_TRUE(WriteBin(mc2Records, hostDataDir, "unaging.additional.mc2_comm_info.slice_0")); | ||
| 355 | + | ||
| 356 | + EventGrouper grouper(hostDataDir); | ||
| 357 | + ASSERT_TRUE(grouper.Group()); | ||
| 358 | + EXPECT_TRUE(grouper.GetCaptureStreamInfoData().empty()); | ||
| 359 | + EXPECT_TRUE(grouper.GetMc2CommInfoData().empty()); | ||
| 360 | + EXPECT_TRUE(grouper.GetThreadIdSet().empty()); | ||
| 361 | + EXPECT_FALSE(grouper.GetGroupEvents().Find(9)); | ||
| 362 | + EXPECT_TRUE(File::RemoveDir(fakeDataDir, 0)); | ||
| 363 | +} | ||
| 364 | + | ||
| 327 | // 测试文件夹中只有Api类型数据 | 365 | // 测试文件夹中只有Api类型数据 |
| 328 | TEST_F(EventGrouperUTest, TestGroupShouldReturnEmptyWhenDataOnlyHasApiBin) | 366 | TEST_F(EventGrouperUTest, TestGroupShouldReturnEmptyWhenDataOnlyHasApiBin) |
| 329 | { | 367 | { |
| @@ -0,0 +1,149 @@ | |||
| 1 | +/* ------------------------------------------------------------------------- | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This file is part of the MindStudio project. | ||
| 4 | + * | ||
| 5 | + * MindStudio is licensed under Mulan PSL v2. | ||
| 6 | + * You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 7 | + * You may obtain a copy of Mulan PSL v2 at: | ||
| 8 | + * | ||
| 9 | + * http://license.coscl.org.cn/MulanPSL2 | ||
| 10 | + * | ||
| 11 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 12 | + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 13 | + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 14 | + * See the Mulan PSL v2 for more details. | ||
| 15 | + * -------------------------------------------------------------------------*/ | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | + | ||
| 23 | + | ||
| 24 | +using namespace Analysis::Domain::Host::Cann; | ||
| 25 | +using namespace Analysis::Utils; | ||
| 26 | + | ||
| 27 | +namespace { | ||
| 28 | +const std::string MC2_ROOT = "./mc2_comm_parser"; | ||
| 29 | +const std::string MC2_DATA_PATH = File::PathJoin({MC2_ROOT, "host", "data"}); | ||
| 30 | + | ||
| 31 | +MsprofAdditionalInfo MakeMc2Info(uint64_t groupName, uint32_t streamSize) | ||
| 32 | +{ | ||
| 33 | + MsprofAdditionalInfo info{}; | ||
| 34 | + auto payload = ReinterpretConvert<MsprofMc2CommInfo *>(info.data); | ||
| 35 | + payload->groupName = groupName; | ||
| 36 | + payload->rankSize = 8; | ||
| 37 | + payload->rankId = 1; | ||
| 38 | + payload->usrRankId = 2; | ||
| 39 | + payload->streamId = 52; | ||
| 40 | + payload->streamSize = streamSize; | ||
| 41 | + for (uint32_t i = 0; i < MC2_COMM_STREAM_MAX_NUM; ++i) { | ||
| 42 | + payload->commStreamIds[i] = 100 + i; | ||
| 43 | + } | ||
| 44 | + return info; | ||
| 45 | +} | ||
| 46 | +} | ||
| 47 | + | ||
| 48 | +class Mc2CommInfoParserUTest : public testing::Test { | ||
| 49 | +protected: | ||
| 50 | + void SetUp() override | ||
| 51 | + { | ||
| 52 | + if (File::Exist(MC2_ROOT)) { | ||
| 53 | + EXPECT_TRUE(File::RemoveDir(MC2_ROOT, 0)); | ||
| 54 | + } | ||
| 55 | + EXPECT_TRUE(File::CreateDir(MC2_ROOT)); | ||
| 56 | + EXPECT_TRUE(File::CreateDir(File::PathJoin({MC2_ROOT, "host"}))); | ||
| 57 | + EXPECT_TRUE(File::CreateDir(MC2_DATA_PATH)); | ||
| 58 | + } | ||
| 59 | + | ||
| 60 | + void TearDown() override | ||
| 61 | + { | ||
| 62 | + if (File::Exist(MC2_ROOT)) { | ||
| 63 | + EXPECT_TRUE(File::RemoveDir(MC2_ROOT, 0)); | ||
| 64 | + } | ||
| 65 | + } | ||
| 66 | +}; | ||
| 67 | + | ||
| 68 | +TEST_F(Mc2CommInfoParserUTest, ShouldNotLookupMc2CommInfoSlice) | ||
| 69 | +{ | ||
| 70 | + std::vector<MsprofAdditionalInfo> input{MakeMc2Info(7466789422691968299ULL, 8)}; | ||
| 71 | + ASSERT_TRUE(WriteBin(input, MC2_DATA_PATH, "unaging.additional.mc2_comm_info.slice_0")); | ||
| 72 | + | ||
| 73 | + Mc2CommInfoParser parser(MC2_DATA_PATH); | ||
| 74 | + auto output = parser.ParseData<ParserAdditionalInfo>(); | ||
| 75 | + ASSERT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR)); | ||
| 76 | + EXPECT_TRUE(output.empty()); | ||
| 77 | +} | ||
| 78 | + | ||
| 79 | +TEST_F(Mc2CommInfoParserUTest, ShouldNotLookupAgingMc2CommInfoSlice) | ||
| 80 | +{ | ||
| 81 | + std::vector<MsprofAdditionalInfo> input{MakeMc2Info(1, 9)}; | ||
| 82 | + ASSERT_TRUE(WriteBin(input, MC2_DATA_PATH, "aging.additional.mc2_comm_info.slice_0")); | ||
| 83 | + | ||
| 84 | + Mc2CommInfoParser parser(MC2_DATA_PATH); | ||
| 85 | + auto output = parser.ParseData<ParserAdditionalInfo>(); | ||
| 86 | + ASSERT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR)); | ||
| 87 | + EXPECT_TRUE(output.empty()); | ||
| 88 | +} | ||
| 89 | + | ||
| 90 | +TEST_F(Mc2CommInfoParserUTest, ShouldNotLookupMc2SlicesInAnyOrder) | ||
| 91 | +{ | ||
| 92 | + std::vector<MsprofAdditionalInfo> unaging0{MakeMc2Info(10, 0)}; | ||
| 93 | + std::vector<MsprofAdditionalInfo> unaging1{MakeMc2Info(11, 1)}; | ||
| 94 | + std::vector<MsprofAdditionalInfo> aging{MakeMc2Info(20, 1)}; | ||
| 95 | + ASSERT_TRUE(WriteBin(unaging1, MC2_DATA_PATH, "unaging.additional.mc2_comm_info.slice_1")); | ||
| 96 | + ASSERT_TRUE(WriteBin(unaging0, MC2_DATA_PATH, "unaging.additional.mc2_comm_info.slice_0")); | ||
| 97 | + ASSERT_TRUE(WriteBin(aging, MC2_DATA_PATH, "aging.additional.mc2_comm_info.slice_0")); | ||
| 98 | + | ||
| 99 | + Mc2CommInfoParser parser(MC2_DATA_PATH); | ||
| 100 | + auto output = parser.ParseData<ParserAdditionalInfo>(); | ||
| 101 | + ASSERT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR)); | ||
| 102 | + EXPECT_TRUE(output.empty()); | ||
| 103 | +} | ||
| 104 | + | ||
| 105 | +TEST_F(Mc2CommInfoParserUTest, ShouldSkipRecordWithInvalidMagic) | ||
| 106 | +{ | ||
| 107 | + auto info = MakeMc2Info(1, 1); | ||
| 108 | + info.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM + 1; | ||
| 109 | + std::vector<MsprofAdditionalInfo> input{info}; | ||
| 110 | + ASSERT_TRUE(WriteBin(input, MC2_DATA_PATH, "unaging.additional.mc2_comm_info.slice_0")); | ||
| 111 | + | ||
| 112 | + Mc2CommInfoParser parser(MC2_DATA_PATH); | ||
| 113 | + auto output = parser.ParseData<ParserAdditionalInfo>(); | ||
| 114 | + EXPECT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR)); | ||
| 115 | + EXPECT_TRUE(output.empty()); | ||
| 116 | +} | ||
| 117 | + | ||
| 118 | +TEST_F(Mc2CommInfoParserUTest, ShouldNotLookupMixedValidAndInvalidMagicRecords) | ||
| 119 | +{ | ||
| 120 | + auto invalid = MakeMc2Info(1, 1); | ||
| 121 | + invalid.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM + 1; | ||
| 122 | + auto valid = MakeMc2Info(2, 1); | ||
| 123 | + std::vector<MsprofAdditionalInfo> input{invalid, valid}; | ||
| 124 | + ASSERT_TRUE(WriteBin(input, MC2_DATA_PATH, "unaging.additional.mc2_comm_info.slice_0")); | ||
| 125 | + | ||
| 126 | + Mc2CommInfoParser parser(MC2_DATA_PATH); | ||
| 127 | + auto output = parser.ParseData<ParserAdditionalInfo>(); | ||
| 128 | + ASSERT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR)); | ||
| 129 | + EXPECT_TRUE(output.empty()); | ||
| 130 | +} | ||
| 131 | + | ||
| 132 | +TEST_F(Mc2CommInfoParserUTest, ShouldReturnTrueAndEmptyWhenNoFileExists) | ||
| 133 | +{ | ||
| 134 | + Mc2CommInfoParser parser(MC2_DATA_PATH); | ||
| 135 | + auto output = parser.ParseData<ParserAdditionalInfo>(); | ||
| 136 | + EXPECT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR)); | ||
| 137 | + EXPECT_TRUE(output.empty()); | ||
| 138 | +} | ||
| 139 | + | ||
| 140 | +TEST_F(Mc2CommInfoParserUTest, ShouldReturnTrueAndEmptyWhenTruncatedRecordExists) | ||
| 141 | +{ | ||
| 142 | + std::vector<uint8_t> input(sizeof(MsprofAdditionalInfo) - 1, 0); | ||
| 143 | + ASSERT_TRUE(WriteBin(input, MC2_DATA_PATH, "unaging.additional.mc2_comm_info.slice_0")); | ||
| 144 | + | ||
| 145 | + Mc2CommInfoParser parser(MC2_DATA_PATH); | ||
| 146 | + auto output = parser.ParseData<ParserAdditionalInfo>(); | ||
| 147 | + EXPECT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR)); | ||
| 148 | + EXPECT_TRUE(output.empty()); | ||
| 149 | +} | ||
| @@ -18,6 +18,8 @@ | |||
| 18 | 18 | ||
| 19 | 19 | ||
| 20 | 20 | ||
| 21 | + | ||
| 22 | + | ||
| 21 | 23 | ||
| 22 | 24 | ||
| 23 | 25 | ||
| @@ -50,6 +52,11 @@ const captureStreamDataFormat CAPTURE_STREAM_DATA{ | |||
| 50 | {0, 49, 44, 11, 0, 0, 27063567062060}, | 52 | {0, 49, 44, 11, 0, 0, 27063567062060}, |
| 51 | {0, 49, 43, 12, 0, 0, 27063567062060} | 53 | {0, 49, 43, 12, 0, 0, 27063567062060} |
| 52 | }; | 54 | }; |
| 55 | +const std::vector<CaptureStreamInfo> DIRECT_CAPTURE_STREAM_DATA{ | ||
| 56 | + {49, 27063567062060, 95, 3, 0, 0, 0}, | ||
| 57 | + {49, 27063567062060, 11, 44, 0, 0, 0}, | ||
| 58 | + {49, 27063567062060, 12, 43, 0, 0, 0}, | ||
| 59 | +}; | ||
| 53 | 60 | ||
| 54 | const std::string PROF_PATH = "./PROF_XXX"; | 61 | const std::string PROF_PATH = "./PROF_XXX"; |
| 55 | const std::string HOST_PATH = File::PathJoin({PROF_PATH, "host"}); | 62 | const std::string HOST_PATH = File::PathJoin({PROF_PATH, "host"}); |
| @@ -89,6 +96,7 @@ protected: | |||
| 89 | 96 | ||
| 90 | void SetUp() override | 97 | void SetUp() override |
| 91 | { | 98 | { |
| 99 | + RTAddInfoCenter::GetInstance().SetCaptureStreamInfoData({}); | ||
| 92 | RtsTrackDB rtsTrackDB; | 100 | RtsTrackDB rtsTrackDB; |
| 93 | std::string opInfo = "RuntimeOpInfo"; | 101 | std::string opInfo = "RuntimeOpInfo"; |
| 94 | CreateDB(rtsTrackDB, opInfo, RTS_TRACK_DB_PATH, DATA); | 102 | CreateDB(rtsTrackDB, opInfo, RTS_TRACK_DB_PATH, DATA); |
| @@ -144,6 +152,7 @@ void CheckRuntimeOpInfo(RuntimeOpInfo info, RuntimeOpInfo expect) | |||
| 144 | TEST_F(RTAddInfoCenterUTest, LoadThenTestGetWhenLoadSuccess) | 152 | TEST_F(RTAddInfoCenterUTest, LoadThenTestGetWhenLoadSuccess) |
| 145 | { | 153 | { |
| 146 | HashData::GetInstance().Load(DATA_PATH); | 154 | HashData::GetInstance().Load(DATA_PATH); |
| 155 | + RTAddInfoCenter::GetInstance().SetCaptureStreamInfoData(DIRECT_CAPTURE_STREAM_DATA); | ||
| 147 | RTAddInfoCenter::GetInstance().Load(SQLITE_PATH); | 156 | RTAddInfoCenter::GetInstance().Load(SQLITE_PATH); |
| 148 | // info 1 2 是有效值,3是空 | 157 | // info 1 2 是有效值,3是空 |
| 149 | auto info1 = RTAddInfoCenter::GetInstance().Get(5, 30, 10); | 158 | auto info1 = RTAddInfoCenter::GetInstance().Get(5, 30, 10); |
| @@ -167,6 +176,27 @@ TEST_F(RTAddInfoCenterUTest, LoadThenTestGetWhenLoadSuccess) | |||
| 167 | EXPECT_EQ(modelId4, 4294967295u); | 176 | EXPECT_EQ(modelId4, 4294967295u); |
| 168 | } | 177 | } |
| 169 | 178 | ||
| 179 | +TEST_F(RTAddInfoCenterUTest, LoadShouldNotReadCaptureStreamInfoFromDb) | ||
| 180 | +{ | ||
| 181 | + RTAddInfoCenter::GetInstance().SetCaptureStreamInfoData({}); | ||
| 182 | + RTAddInfoCenter::GetInstance().Load(SQLITE_PATH); | ||
| 183 | + EXPECT_EQ(DEFAULT_MODEL_ID, RTAddInfoCenter::GetInstance().GetModelId(0, 95, 0, 27063567062061)); | ||
| 184 | +} | ||
| 185 | + | ||
| 186 | +TEST_F(RTAddInfoCenterUTest, SetCaptureInfoShouldPreserveV2OriginalStreamIdWidth) | ||
| 187 | +{ | ||
| 188 | + std::vector<CaptureStreamInfo> wideData{{88, 27063567062070, 70003, 70002, 0, 0, 0}}; | ||
| 189 | + RTAddInfoCenter::GetInstance().SetCaptureStreamInfoData(wideData); | ||
| 190 | + RTAddInfoCenter::GetInstance().Load(SQLITE_PATH); | ||
| 191 | + const auto &captureData = RTAddInfoCenter::GetInstance().captureStreamInfoData_; | ||
| 192 | + auto record = std::find_if(captureData.begin(), captureData.end(), [](const CaptureStreamInfo &item) { | ||
| 193 | + return item.modelId == 88; | ||
| 194 | + }); | ||
| 195 | + ASSERT_NE(captureData.end(), record); | ||
| 196 | + EXPECT_EQ(70002u, record->originalStreamId); | ||
| 197 | + EXPECT_EQ(70003u, record->streamId); | ||
| 198 | +} | ||
| 199 | + | ||
| 170 | TEST_F(RTAddInfoCenterUTest, AddShouldMarkLoadedFromBinaryAndKeepDumpList) | 200 | TEST_F(RTAddInfoCenterUTest, AddShouldMarkLoadedFromBinaryAndKeepDumpList) |
| 171 | { | 201 | { |
| 172 | RuntimeOpInfo info1{9, 101, 1, 0, 0, 0, 21, 7, "AI_CORE", "Add", "op_a", "N/A", "0", | 202 | RuntimeOpInfo info1{9, 101, 1, 0, 0, 0, 21, 7, "AI_CORE", "Add", "op_a", "N/A", "0", |
| @@ -0,0 +1,205 @@ | |||
| 1 | +/* ------------------------------------------------------------------------- | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This file is part of the MindStudio project. | ||
| 4 | + * | ||
| 5 | + * MindStudio is licensed under Mulan PSL v2. | ||
| 6 | + * You can use this software according to the terms and conditions of the Mulan PSL v2. | ||
| 7 | + * You may obtain a copy of Mulan PSL v2 at: | ||
| 8 | + * | ||
| 9 | + * http://license.coscl.org.cn/MulanPSL2 | ||
| 10 | + * | ||
| 11 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, | ||
| 12 | + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, | ||
| 13 | + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. | ||
| 14 | + * See the Mulan PSL v2 for more details. | ||
| 15 | + * -------------------------------------------------------------------------*/ | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | + | ||
| 23 | + | ||
| 24 | +using namespace Analysis::Domain; | ||
| 25 | +using namespace Analysis::Domain::Host::Cann; | ||
| 26 | +using namespace mockcpp; | ||
| 27 | + | ||
| 28 | +namespace { | ||
| 29 | +std::string capturedLogMessage; | ||
| 30 | + | ||
| 31 | +std::shared_ptr<ParserCompactInfo> MakeCaptureInfo(uint16_t deviceId, uint32_t modelId, uint32_t originalStreamId, | ||
| 32 | + uint32_t streamId, uint16_t captureStatus, uint64_t timeStamp) | ||
| 33 | +{ | ||
| 34 | + auto info = std::make_shared<ParserCompactInfo>(); | ||
| 35 | + info->timeStamp = timeStamp; | ||
| 36 | + info->data.captureStreamInfo.deviceId = deviceId; | ||
| 37 | + info->data.captureStreamInfo.modelId = modelId; | ||
| 38 | + info->data.captureStreamInfo.originalStreamId = originalStreamId; | ||
| 39 | + info->data.captureStreamInfo.streamId = streamId; | ||
| 40 | + info->data.captureStreamInfo.captureStatus = captureStatus; | ||
| 41 | + return info; | ||
| 42 | +} | ||
| 43 | + | ||
| 44 | +std::shared_ptr<ParserAdditionalInfo> MakeMc2Info(uint64_t groupName, uint32_t rankSize, uint32_t rankId, | ||
| 45 | + uint32_t usrRankId, uint32_t streamId, uint32_t streamSize) | ||
| 46 | +{ | ||
| 47 | + auto info = std::make_shared<ParserAdditionalInfo>(); | ||
| 48 | + auto payload = Analysis::Utils::ReinterpretConvert<MsprofMc2CommInfo *>(info->data); | ||
| 49 | + payload->groupName = groupName; | ||
| 50 | + payload->rankSize = rankSize; | ||
| 51 | + payload->rankId = rankId; | ||
| 52 | + payload->usrRankId = usrRankId; | ||
| 53 | + payload->streamId = streamId; | ||
| 54 | + payload->streamSize = streamSize; | ||
| 55 | + for (uint32_t i = 0; i < MC2_COMM_STREAM_MAX_NUM; ++i) { | ||
| 56 | + payload->commStreamIds[i] = 100 + i; | ||
| 57 | + } | ||
| 58 | + return info; | ||
| 59 | +} | ||
| 60 | + | ||
| 61 | +void CaptureLogMessage(Analysis::Log *, const std::string &message, const std::string &, const std::string &, | ||
| 62 | + const uint32_t &) | ||
| 63 | +{ | ||
| 64 | + capturedLogMessage = message; | ||
| 65 | +} | ||
| 66 | +} // namespace | ||
| 67 | + | ||
| 68 | +TEST(CaptureStreamInfoDumperUTest, ShouldMatchPythonGlobalModelEndDeduplication) | ||
| 69 | +{ | ||
| 70 | + std::vector<std::shared_ptr<ParserCompactInfo>> input{ | ||
| 71 | + MakeCaptureInfo(0, 1, 10, 20, 0, 20), | ||
| 72 | + MakeCaptureInfo(0, 1, 10, 20, 0, 10), | ||
| 73 | + MakeCaptureInfo(0, 1, 10, 20, 0, 10), | ||
| 74 | + MakeCaptureInfo(0, 1, UINT32_MAX, UINT32_MAX, 1, 30), | ||
| 75 | + MakeCaptureInfo(1, 1, UINT32_MAX, UINT32_MAX, 1, 40), | ||
| 76 | + }; | ||
| 77 | + | ||
| 78 | + CaptureStreamInfoDumper dumper("."); | ||
| 79 | + auto output = dumper.FormatData(input); | ||
| 80 | + ASSERT_EQ(3ul, output.size()); | ||
| 81 | + | ||
| 82 | + EXPECT_EQ(0, output[0].deviceId); | ||
| 83 | + EXPECT_EQ(1u, output[0].modelId); | ||
| 84 | + EXPECT_EQ(10u, output[0].originalStreamId); | ||
| 85 | + EXPECT_EQ(20u, output[0].streamId); | ||
| 86 | + EXPECT_EQ(0u, output[0].batchId); | ||
| 87 | + EXPECT_EQ(0, output[0].captureStatus); | ||
| 88 | + EXPECT_EQ(10ul, output[0].timeStamp); | ||
| 89 | + | ||
| 90 | + EXPECT_EQ(1u, output[1].batchId); | ||
| 91 | + EXPECT_EQ(20ul, output[1].timeStamp); | ||
| 92 | + EXPECT_EQ(1, output[2].captureStatus); | ||
| 93 | + EXPECT_EQ(30ul, output[2].timeStamp); | ||
| 94 | +} | ||
| 95 | + | ||
| 96 | +TEST(CaptureStreamInfoDumperUTest, ShouldKeepSeparateBatchCountersForEachDeviceAndStream) | ||
| 97 | +{ | ||
| 98 | + std::vector<std::shared_ptr<ParserCompactInfo>> input{ | ||
| 99 | + MakeCaptureInfo(0, 1, 10, 20, 0, 10), | ||
| 100 | + MakeCaptureInfo(1, 1, 10, 20, 0, 11), | ||
| 101 | + MakeCaptureInfo(0, 1, 11, 21, 0, 12), | ||
| 102 | + MakeCaptureInfo(0, 1, 10, 20, 0, 13), | ||
| 103 | + }; | ||
| 104 | + | ||
| 105 | + CaptureStreamInfoDumper dumper("."); | ||
| 106 | + auto output = dumper.FormatData(input); | ||
| 107 | + ASSERT_EQ(4ul, output.size()); | ||
| 108 | + EXPECT_EQ(0u, output[0].batchId); | ||
| 109 | + EXPECT_EQ(0u, output[1].batchId); | ||
| 110 | + EXPECT_EQ(0u, output[2].batchId); | ||
| 111 | + EXPECT_EQ(1u, output[3].batchId); | ||
| 112 | +} | ||
| 113 | + | ||
| 114 | +TEST(CaptureStreamInfoDumperUTest, ShouldLogMismatchedStartAndEndModelIds) | ||
| 115 | +{ | ||
| 116 | + const std::string expectedMessage = | ||
| 117 | + "CaptureStreamInfoDumper: Capture start model ids are {7}, end model ids are {}."; | ||
| 118 | + capturedLogMessage.clear(); | ||
| 119 | + MOCKER_CPP(&Analysis::Log::LogMsg).stubs().will(invoke(CaptureLogMessage)); | ||
| 120 | + std::vector<std::shared_ptr<ParserCompactInfo>> input{MakeCaptureInfo(0, 7, 10, 20, 0, 10)}; | ||
| 121 | + | ||
| 122 | + CaptureStreamInfoDumper dumper("."); | ||
| 123 | + auto output = dumper.FormatData(input); | ||
| 124 | + | ||
| 125 | + ASSERT_EQ(1ul, output.size()); | ||
| 126 | + EXPECT_EQ(expectedMessage, capturedLogMessage); | ||
| 127 | + MOCKER_CPP(&Analysis::Log::LogMsg).reset(); | ||
| 128 | +} | ||
| 129 | + | ||
| 130 | +TEST(Mc2CommInfoDumperUTest, ShouldKeepRawRowsAndAppendOneToManyCaptureMappedRowsInDeterministicOrder) | ||
| 131 | +{ | ||
| 132 | + std::vector<std::shared_ptr<ParserAdditionalInfo>> mc2Data{ | ||
| 133 | + MakeMc2Info(5862276093215481612ULL, 2, 0, 0, 20, 2), | ||
| 134 | + MakeMc2Info(9, 4, 1, 1, 99, 1), | ||
| 135 | + }; | ||
| 136 | + CaptureStreamInfoData captureInput{ | ||
| 137 | + {0, 1, 20, 32, 0, 0, 1}, | ||
| 138 | + {0, 1, 20, 21, 0, 0, 2}, | ||
| 139 | + {0, 1, 20, 21, 1, 0, 3}, | ||
| 140 | + {0, 1, 20, 22, 0, 0, 4}, | ||
| 141 | + }; | ||
| 142 | + | ||
| 143 | + Mc2CommInfoDumper dumper("."); | ||
| 144 | + Mc2CommInfoInput input(mc2Data, captureInput); | ||
| 145 | + auto output = dumper.GenerateData(input); | ||
| 146 | + ASSERT_EQ(5ul, output.size()); | ||
| 147 | + EXPECT_EQ("5862276093215481612", std::get<0>(output[0])); | ||
| 148 | + EXPECT_EQ(20u, std::get<4>(output[0])); | ||
| 149 | + EXPECT_EQ("9", std::get<0>(output[1])); | ||
| 150 | + EXPECT_EQ(99u, std::get<4>(output[1])); | ||
| 151 | + | ||
| 152 | + EXPECT_EQ(21u, std::get<4>(output[2])); | ||
| 153 | + EXPECT_EQ(22u, std::get<4>(output[3])); | ||
| 154 | + EXPECT_EQ(32u, std::get<4>(output[4])); | ||
| 155 | + EXPECT_EQ("100,101", std::get<5>(output[4])); | ||
| 156 | +} | ||
| 157 | + | ||
| 158 | +TEST(Mc2CommInfoDumperUTest, ShouldNotAppendCaptureRowsWhenKfcStreamDoesNotMatch) | ||
| 159 | +{ | ||
| 160 | + std::vector<std::shared_ptr<ParserAdditionalInfo>> mc2Data{MakeMc2Info(1, 2, 0, 0, 99, 1)}; | ||
| 161 | + CaptureStreamInfoData captureInput{{0, 1, 20, 32, 0, 0, 1}}; | ||
| 162 | + | ||
| 163 | + Mc2CommInfoDumper dumper("."); | ||
| 164 | + Mc2CommInfoInput input(mc2Data, captureInput); | ||
| 165 | + auto output = dumper.GenerateData(input); | ||
| 166 | + ASSERT_EQ(1ul, output.size()); | ||
| 167 | + EXPECT_EQ(99u, std::get<4>(output[0])); | ||
| 168 | +} | ||
| 169 | + | ||
| 170 | +TEST(Mc2CommInfoDumperUTest, ShouldLogInvalidStreamSizeCountOnce) | ||
| 171 | +{ | ||
| 172 | + const std::string expectedMessage = "Mc2CommInfoDumper: 2 records have stream size greater than max stream size 8."; | ||
| 173 | + capturedLogMessage.clear(); | ||
| 174 | + MOCKER_CPP(&Analysis::Log::LogMsg).stubs().will(invoke(CaptureLogMessage)); | ||
| 175 | + std::vector<std::shared_ptr<ParserAdditionalInfo>> mc2Data{ | ||
| 176 | + MakeMc2Info(1, 2, 0, 0, 10, 9), | ||
| 177 | + MakeMc2Info(2, 2, 0, 0, 11, 10), | ||
| 178 | + }; | ||
| 179 | + CaptureStreamInfoData captureData; | ||
| 180 | + | ||
| 181 | + Mc2CommInfoDumper dumper("."); | ||
| 182 | + Mc2CommInfoInput input(mc2Data, captureData); | ||
| 183 | + auto output = dumper.GenerateData(input); | ||
| 184 | + | ||
| 185 | + ASSERT_EQ(2ul, output.size()); | ||
| 186 | + EXPECT_TRUE(std::get<5>(output[0]).empty()); | ||
| 187 | + EXPECT_EQ(expectedMessage, capturedLogMessage); | ||
| 188 | + MOCKER_CPP(&Analysis::Log::LogMsg).reset(); | ||
| 189 | +} | ||
| 190 | + | ||
| 191 | +TEST(Mc2CommInfoDumperUTest, ShouldMatchPythonWhenInvalidStreamSizeHasCaptureMapping) | ||
| 192 | +{ | ||
| 193 | + std::vector<std::shared_ptr<ParserAdditionalInfo>> mc2Data{MakeMc2Info(1, 2, 0, 0, 20, 9)}; | ||
| 194 | + CaptureStreamInfoData captureData{{0, 1, 20, 32, 0, 0, 1}}; | ||
| 195 | + | ||
| 196 | + Mc2CommInfoDumper dumper("."); | ||
| 197 | + Mc2CommInfoInput input(mc2Data, captureData); | ||
| 198 | + auto output = dumper.GenerateData(input); | ||
| 199 | + | ||
| 200 | + ASSERT_EQ(2ul, output.size()); | ||
| 201 | + EXPECT_EQ(20u, std::get<4>(output[0])); | ||
| 202 | + EXPECT_EQ(32u, std::get<4>(output[1])); | ||
| 203 | + EXPECT_TRUE(std::get<5>(output[0]).empty()); | ||
| 204 | + EXPECT_TRUE(std::get<5>(output[1]).empty()); | ||
| 205 | +} | ||
| @@ -52,4 +52,4 @@ TEST_F(HostTraceWorkerUtest, TestKernelHostTraceWorkerShouldReturnTrueWhenGroupR | |||
| 52 | MOCKER_CPP(&EventGrouper::GetGroupEvents).stubs().will(returnValue(mockWarehouses)); | 52 | MOCKER_CPP(&EventGrouper::GetGroupEvents).stubs().will(returnValue(mockWarehouses)); |
| 53 | auto ret = hostTraceWorker.Run(); | 53 | auto ret = hostTraceWorker.Run(); |
| 54 | EXPECT_EQ(ret, true); | 54 | EXPECT_EQ(ret, true); |
| 55 | -} | 55 | +} |
| @@ -17,13 +17,18 @@ | |||
| 17 | 17 | ||
| 18 | 18 | ||
| 19 | 19 | ||
| 20 | + | ||
| 20 | 21 | ||
| 22 | + | ||
| 21 | 23 | ||
| 22 | 24 | ||
| 23 | 25 | ||
| 24 | 26 | ||
| 25 | 27 | ||
| 26 | - | 28 | +#include "analysis/csrc/domain/services/parser/host/cann/rt_add_info_center.h" |
| 29 | + | ||
| 30 | + | ||
| 31 | + | ||
| 27 | 32 | ||
| 28 | using namespace Analysis::Domain; | 33 | using namespace Analysis::Domain; |
| 29 | 34 | ||
| @@ -38,6 +43,9 @@ class KernelParserWorkerUtest : public testing::Test { | |||
| 38 | protected: | 43 | protected: |
| 39 | virtual void SetUp() | 44 | virtual void SetUp() |
| 40 | { | 45 | { |
| 46 | + if (File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite"}))) { | ||
| 47 | + File::RemoveDir(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite"}), 0); | ||
| 48 | + } | ||
| 41 | File::CreateDir(TEST_HOST_FILE_PATH); | 49 | File::CreateDir(TEST_HOST_FILE_PATH); |
| 42 | File::CreateDir(File::PathJoin({TEST_HOST_FILE_PATH, "data"})); | 50 | File::CreateDir(File::PathJoin({TEST_HOST_FILE_PATH, "data"})); |
| 43 | File::CreateDir(File::PathJoin({TEST_HOST_FILE_PATH, "data", "sqlite"})); | 51 | File::CreateDir(File::PathJoin({TEST_HOST_FILE_PATH, "data", "sqlite"})); |
| @@ -53,17 +61,40 @@ protected: | |||
| 53 | { | 61 | { |
| 54 | GlobalMockObject::verify(); | 62 | GlobalMockObject::verify(); |
| 55 | File::RemoveDir(File::PathJoin({TEST_HOST_FILE_PATH, "data"}), 0); | 63 | File::RemoveDir(File::PathJoin({TEST_HOST_FILE_PATH, "data"}), 0); |
| 64 | + if (File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite"}))) { | ||
| 65 | + File::RemoveDir(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite"}), 0); | ||
| 66 | + } | ||
| 67 | + } | ||
| 68 | + | ||
| 69 | + void StubContextLoadSuccess() | ||
| 70 | + { | ||
| 71 | + MOCKER_CPP(&Analysis::Domain::Environment::Context::Load).stubs().will(returnValue(true)); | ||
| 72 | + } | ||
| 73 | + | ||
| 74 | + void StubPlatformVersion(Analysis::Domain::Environment::Chip chip) | ||
| 75 | + { | ||
| 76 | + StubContextLoadSuccess(); | ||
| 77 | + MOCKER_CPP(&Analysis::Domain::Environment::Context::GetPlatformVersion) | ||
| 78 | + .stubs() | ||
| 79 | + .will(returnValue(static_cast<uint16_t>(chip))); | ||
| 80 | + } | ||
| 81 | + | ||
| 82 | + void UseRealHostTraceWorker() | ||
| 83 | + { | ||
| 84 | + MOCKER_CPP(&HostTraceWorker::Run).reset(); | ||
| 56 | } | 85 | } |
| 57 | }; | 86 | }; |
| 58 | 87 | ||
| 59 | TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnSuccessWhenAllFunctionWell) | 88 | TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnSuccessWhenAllFunctionWell) |
| 60 | { | 89 | { |
| 90 | + StubContextLoadSuccess(); | ||
| 61 | KernelParserWorker kernelParserWorker(TEST_HOST_FILE_PATH); | 91 | KernelParserWorker kernelParserWorker(TEST_HOST_FILE_PATH); |
| 62 | auto res = kernelParserWorker.Run(); | 92 | auto res = kernelParserWorker.Run(); |
| 63 | EXPECT_EQ(res, 0); | 93 | EXPECT_EQ(res, 0); |
| 64 | } | 94 | } |
| 65 | TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnFailedWhenHostTraceWorkerReturnFailed) | 95 | TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnFailedWhenHostTraceWorkerReturnFailed) |
| 66 | { | 96 | { |
| 97 | + StubContextLoadSuccess(); | ||
| 67 | KernelParserWorker kernelParserWorker(TEST_HOST_FILE_PATH); | 98 | KernelParserWorker kernelParserWorker(TEST_HOST_FILE_PATH); |
| 68 | MOCKER_CPP(&HostTraceWorker::Run).reset(); | 99 | MOCKER_CPP(&HostTraceWorker::Run).reset(); |
| 69 | MOCKER_CPP(&HostTraceWorker::Run).stubs().will(returnValue(false)); | 100 | MOCKER_CPP(&HostTraceWorker::Run).stubs().will(returnValue(false)); |
| @@ -73,6 +104,7 @@ TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnFailedWhenHost | |||
| 73 | 104 | ||
| 74 | TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnFailedWhenDumpHashFailed) | 105 | TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnFailedWhenDumpHashFailed) |
| 75 | { | 106 | { |
| 107 | + StubContextLoadSuccess(); | ||
| 76 | KernelParserWorker kernelParserWorker(TEST_HOST_FILE_PATH); | 108 | KernelParserWorker kernelParserWorker(TEST_HOST_FILE_PATH); |
| 77 | MOCKER_CPP(&DBRunner::CreateTable).stubs().will(returnValue(false)); | 109 | MOCKER_CPP(&DBRunner::CreateTable).stubs().will(returnValue(false)); |
| 78 | auto res = kernelParserWorker.Run(); | 110 | auto res = kernelParserWorker.Run(); |
| @@ -81,6 +113,7 @@ TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnFailedWhenDump | |||
| 81 | 113 | ||
| 82 | TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnFailedWhenTypeInfoDumpFailed) | 114 | TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnFailedWhenTypeInfoDumpFailed) |
| 83 | { | 115 | { |
| 116 | + StubContextLoadSuccess(); | ||
| 84 | KernelParserWorker kernelParserWorker(TEST_HOST_FILE_PATH); | 117 | KernelParserWorker kernelParserWorker(TEST_HOST_FILE_PATH); |
| 85 | MOCKER_CPP(&DBRunner::CreateTable).stubs().will(returnValue(false)); | 118 | MOCKER_CPP(&DBRunner::CreateTable).stubs().will(returnValue(false)); |
| 86 | auto res = kernelParserWorker.Run(); | 119 | auto res = kernelParserWorker.Run(); |
| @@ -93,4 +126,189 @@ TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnErrorWhenConte | |||
| 93 | MOCKER_CPP(&Analysis::Domain::Environment::Context::Load).stubs().will(returnValue(false)); | 126 | MOCKER_CPP(&Analysis::Domain::Environment::Context::Load).stubs().will(returnValue(false)); |
| 94 | auto res = kernelParserWorker.Run(); | 127 | auto res = kernelParserWorker.Run(); |
| 95 | EXPECT_EQ(res, 1); | 128 | EXPECT_EQ(res, 1); |
| 96 | -} | 129 | +} |
| 130 | + | ||
| 131 | +TEST_F(KernelParserWorkerUtest, ShouldSkipCaptureParseWhenCppParserDisabledEvenIfTruncatedRecordExists) | ||
| 132 | +{ | ||
| 133 | + // C++ Capture/MC2 默认关闭:截断 bin 不会进入 Host C++ 解析,Host 其它输出仍成功。 | ||
| 134 | + ASSERT_FALSE(Host::Cann::kEnableCaptureStreamMc2CppParser); | ||
| 135 | + UseRealHostTraceWorker(); | ||
| 136 | + std::vector<uint8_t> truncatedInput{0}; | ||
| 137 | + ASSERT_TRUE(WriteBin(truncatedInput, File::PathJoin({TEST_HOST_FILE_PATH, "data"}), | ||
| 138 | + "unaging.compact.capture_stream_info.slice_0")); | ||
| 139 | + StubPlatformVersion(Analysis::Domain::Environment::Chip::CHIP_V3_1_0); | ||
| 140 | + | ||
| 141 | + KernelParserWorker worker(TEST_HOST_FILE_PATH); | ||
| 142 | + EXPECT_EQ(ANALYSIS_OK, worker.Run()); | ||
| 143 | + EXPECT_TRUE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "ge_hash.db"}))); | ||
| 144 | + EXPECT_FALSE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "stream_info.db"}))); | ||
| 145 | +} | ||
| 146 | + | ||
| 147 | +TEST_F(KernelParserWorkerUtest, ShouldSkipMc2ParseWhenCppParserDisabledEvenIfTruncatedRecordExists) | ||
| 148 | +{ | ||
| 149 | + ASSERT_FALSE(Host::Cann::kEnableCaptureStreamMc2CppParser); | ||
| 150 | + UseRealHostTraceWorker(); | ||
| 151 | + std::vector<uint8_t> truncatedInput{0}; | ||
| 152 | + ASSERT_TRUE(WriteBin(truncatedInput, File::PathJoin({TEST_HOST_FILE_PATH, "data"}), | ||
| 153 | + "unaging.additional.mc2_comm_info.slice_0")); | ||
| 154 | + StubPlatformVersion(Analysis::Domain::Environment::Chip::CHIP_V3_3_0); | ||
| 155 | + | ||
| 156 | + KernelParserWorker worker(TEST_HOST_FILE_PATH); | ||
| 157 | + EXPECT_EQ(ANALYSIS_OK, worker.Run()); | ||
| 158 | + EXPECT_TRUE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "ge_hash.db"}))); | ||
| 159 | + EXPECT_FALSE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "mc2_comm_info.db"}))); | ||
| 160 | +} | ||
| 161 | + | ||
| 162 | +TEST_F(KernelParserWorkerUtest, ShouldNotInvokeCaptureDumperWhenCppParserDisabled) | ||
| 163 | +{ | ||
| 164 | + ASSERT_FALSE(Host::Cann::kEnableCaptureStreamMc2CppParser); | ||
| 165 | + UseRealHostTraceWorker(); | ||
| 166 | + MsprofCompactInfo capture{}; | ||
| 167 | + capture.timeStamp = 123; | ||
| 168 | + capture.data.captureStreamInfo.deviceId = 0; | ||
| 169 | + capture.data.captureStreamInfo.modelId = 7; | ||
| 170 | + capture.data.captureStreamInfo.originalStreamId = 52; | ||
| 171 | + capture.data.captureStreamInfo.modelStreamId = 70; | ||
| 172 | + capture.data.captureStreamInfo.captureStatus = 0; | ||
| 173 | + std::vector<MsprofCompactInfo> captureInput{capture}; | ||
| 174 | + ASSERT_TRUE(WriteBin(captureInput, File::PathJoin({TEST_HOST_FILE_PATH, "data"}), | ||
| 175 | + "unaging.compact.capture_stream_info.slice_0")); | ||
| 176 | + StubPlatformVersion(Analysis::Domain::Environment::Chip::CHIP_V3_1_0); | ||
| 177 | + ASSERT_TRUE(File::CreateDir(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite"}))); | ||
| 178 | + // 即使预置与 dumper 冲突的路径,默认关闭时也不会走 Capture dump。 | ||
| 179 | + ASSERT_TRUE(File::CreateDir(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "stream_info.db"}))); | ||
| 180 | + | ||
| 181 | + KernelParserWorker worker(TEST_HOST_FILE_PATH); | ||
| 182 | + EXPECT_EQ(ANALYSIS_OK, worker.Run()); | ||
| 183 | + EXPECT_TRUE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "ge_hash.db"}))); | ||
| 184 | +} | ||
| 185 | + | ||
| 186 | +TEST_F(KernelParserWorkerUtest, ShouldNotInvokeMc2DumperWhenCppParserDisabled) | ||
| 187 | +{ | ||
| 188 | + ASSERT_FALSE(Host::Cann::kEnableCaptureStreamMc2CppParser); | ||
| 189 | + UseRealHostTraceWorker(); | ||
| 190 | + MsprofAdditionalInfo mc2{}; | ||
| 191 | + auto payload = ReinterpretConvert<Analysis::Domain::Host::Cann::MsprofMc2CommInfo *>(mc2.data); | ||
| 192 | + payload->groupName = 99; | ||
| 193 | + payload->rankSize = 2; | ||
| 194 | + payload->streamId = 52; | ||
| 195 | + payload->streamSize = 1; | ||
| 196 | + payload->commStreamIds[0] = 100; | ||
| 197 | + std::vector<MsprofAdditionalInfo> mc2Input{mc2}; | ||
| 198 | + ASSERT_TRUE(WriteBin(mc2Input, File::PathJoin({TEST_HOST_FILE_PATH, "data"}), | ||
| 199 | + "unaging.additional.mc2_comm_info.slice_0")); | ||
| 200 | + StubPlatformVersion(Analysis::Domain::Environment::Chip::CHIP_V3_3_0); | ||
| 201 | + ASSERT_TRUE(File::CreateDir(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite"}))); | ||
| 202 | + ASSERT_TRUE(File::CreateDir(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "mc2_comm_info.db"}))); | ||
| 203 | + | ||
| 204 | + KernelParserWorker worker(TEST_HOST_FILE_PATH); | ||
| 205 | + EXPECT_EQ(ANALYSIS_OK, worker.Run()); | ||
| 206 | + EXPECT_TRUE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "ge_hash.db"}))); | ||
| 207 | +} | ||
| 208 | + | ||
| 209 | +TEST_F(KernelParserWorkerUtest, ShouldSkipCaptureFormattingWhenCppParserDisabled) | ||
| 210 | +{ | ||
| 211 | + ASSERT_FALSE(Host::Cann::kEnableCaptureStreamMc2CppParser); | ||
| 212 | + UseRealHostTraceWorker(); | ||
| 213 | + MsprofCompactInfo capture{}; | ||
| 214 | + capture.timeStamp = 123; | ||
| 215 | + capture.data.captureStreamInfo.modelId = 7; | ||
| 216 | + capture.data.captureStreamInfo.originalStreamId = 52; | ||
| 217 | + capture.data.captureStreamInfo.modelStreamId = 70; | ||
| 218 | + std::vector<MsprofCompactInfo> captureInput{capture}; | ||
| 219 | + ASSERT_TRUE(WriteBin(captureInput, File::PathJoin({TEST_HOST_FILE_PATH, "data"}), | ||
| 220 | + "unaging.compact.capture_stream_info.slice_0")); | ||
| 221 | + StubPlatformVersion(Analysis::Domain::Environment::Chip::CHIP_V3_1_0); | ||
| 222 | + Analysis::Test::StubReserveFailureForVector<CaptureStreamInfoData>(); | ||
| 223 | + | ||
| 224 | + KernelParserWorker worker(TEST_HOST_FILE_PATH); | ||
| 225 | + auto result = worker.Run(); | ||
| 226 | + Analysis::Test::ResetReserveFailureForVector<CaptureStreamInfoData>(); | ||
| 227 | + EXPECT_EQ(ANALYSIS_OK, result); | ||
| 228 | + EXPECT_TRUE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "ge_hash.db"}))); | ||
| 229 | +} | ||
| 230 | + | ||
| 231 | +TEST_F(KernelParserWorkerUtest, ShouldSkipMc2FormattingWhenCppParserDisabled) | ||
| 232 | +{ | ||
| 233 | + ASSERT_FALSE(Host::Cann::kEnableCaptureStreamMc2CppParser); | ||
| 234 | + UseRealHostTraceWorker(); | ||
| 235 | + MsprofAdditionalInfo mc2{}; | ||
| 236 | + auto payload = ReinterpretConvert<Analysis::Domain::Host::Cann::MsprofMc2CommInfo *>(mc2.data); | ||
| 237 | + payload->groupName = 99; | ||
| 238 | + payload->rankSize = 2; | ||
| 239 | + payload->streamId = 52; | ||
| 240 | + payload->streamSize = 1; | ||
| 241 | + payload->commStreamIds[0] = 100; | ||
| 242 | + std::vector<MsprofAdditionalInfo> mc2Input{mc2}; | ||
| 243 | + ASSERT_TRUE(WriteBin(mc2Input, File::PathJoin({TEST_HOST_FILE_PATH, "data"}), | ||
| 244 | + "unaging.additional.mc2_comm_info.slice_0")); | ||
| 245 | + StubPlatformVersion(Analysis::Domain::Environment::Chip::CHIP_V3_3_0); | ||
| 246 | + Analysis::Test::StubReserveFailureForVector<Mc2CommInfoData>(); | ||
| 247 | + | ||
| 248 | + KernelParserWorker worker(TEST_HOST_FILE_PATH); | ||
| 249 | + auto result = worker.Run(); | ||
| 250 | + Analysis::Test::ResetReserveFailureForVector<Mc2CommInfoData>(); | ||
| 251 | + EXPECT_EQ(ANALYSIS_OK, result); | ||
| 252 | + EXPECT_TRUE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "ge_hash.db"}))); | ||
| 253 | +} | ||
| 254 | + | ||
| 255 | +TEST_F(KernelParserWorkerUtest, ShouldNotCreateCaptureAndMc2DatabasesWhenCppParserDisabled) | ||
| 256 | +{ | ||
| 257 | + ASSERT_FALSE(Host::Cann::kEnableCaptureStreamMc2CppParser); | ||
| 258 | + UseRealHostTraceWorker(); | ||
| 259 | + MsprofCompactInfo capture{}; | ||
| 260 | + capture.timeStamp = 123; | ||
| 261 | + capture.data.captureStreamInfoV2.deviceId = 0; | ||
| 262 | + capture.data.captureStreamInfoV2.modelId = 7; | ||
| 263 | + capture.data.captureStreamInfoV2.originalStreamId = 70002; | ||
| 264 | + capture.data.captureStreamInfoV2.streamId = 70003; | ||
| 265 | + capture.data.captureStreamInfoV2.captureStatus = 0; | ||
| 266 | + std::vector<MsprofCompactInfo> captureInput{capture}; | ||
| 267 | + ASSERT_TRUE(WriteBin(captureInput, File::PathJoin({TEST_HOST_FILE_PATH, "data"}), | ||
| 268 | + "unaging.compact.capture_stream_info_v2.slice_0")); | ||
| 269 | + | ||
| 270 | + MsprofAdditionalInfo mc2{}; | ||
| 271 | + auto payload = ReinterpretConvert<Analysis::Domain::Host::Cann::MsprofMc2CommInfo *>(mc2.data); | ||
| 272 | + payload->groupName = 99; | ||
| 273 | + payload->rankSize = 2; | ||
| 274 | + payload->rankId = 0; | ||
| 275 | + payload->usrRankId = 0; | ||
| 276 | + payload->streamId = 70002; | ||
| 277 | + payload->streamSize = 2; | ||
| 278 | + payload->commStreamIds[0] = 70004; | ||
| 279 | + payload->commStreamIds[1] = 70005; | ||
| 280 | + std::vector<MsprofAdditionalInfo> mc2Input{mc2}; | ||
| 281 | + ASSERT_TRUE(WriteBin(mc2Input, File::PathJoin({TEST_HOST_FILE_PATH, "data"}), | ||
| 282 | + "unaging.additional.mc2_comm_info.slice_0")); | ||
| 283 | + StubPlatformVersion(Analysis::Domain::Environment::Chip::CHIP_V3_3_0); | ||
| 284 | + | ||
| 285 | + KernelParserWorker worker(TEST_HOST_FILE_PATH); | ||
| 286 | + ASSERT_EQ(ANALYSIS_OK, worker.Run()); | ||
| 287 | + | ||
| 288 | + EXPECT_FALSE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "stream_info.db"}))); | ||
| 289 | + EXPECT_FALSE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "mc2_comm_info.db"}))); | ||
| 290 | +} | ||
| 291 | + | ||
| 292 | +TEST_F(KernelParserWorkerUtest, ShouldSkipCaptureAndMc2WhenCppParserDisabledOnAnyPlatform) | ||
| 293 | +{ | ||
| 294 | + ASSERT_FALSE(Host::Cann::kEnableCaptureStreamMc2CppParser); | ||
| 295 | + UseRealHostTraceWorker(); | ||
| 296 | + MsprofCompactInfo capture{}; | ||
| 297 | + capture.data.captureStreamInfo.modelStreamId = 70; | ||
| 298 | + std::vector<MsprofCompactInfo> captureInput{capture}; | ||
| 299 | + ASSERT_TRUE(WriteBin(captureInput, File::PathJoin({TEST_HOST_FILE_PATH, "data"}), | ||
| 300 | + "unaging.compact.capture_stream_info.slice_0")); | ||
| 301 | + | ||
| 302 | + MsprofAdditionalInfo mc2{}; | ||
| 303 | + auto payload = ReinterpretConvert<Analysis::Domain::Host::Cann::MsprofMc2CommInfo *>(mc2.data); | ||
| 304 | + payload->streamId = 52; | ||
| 305 | + std::vector<MsprofAdditionalInfo> mc2Input{mc2}; | ||
| 306 | + ASSERT_TRUE(WriteBin(mc2Input, File::PathJoin({TEST_HOST_FILE_PATH, "data"}), | ||
| 307 | + "unaging.additional.mc2_comm_info.slice_0")); | ||
| 308 | + StubPlatformVersion(Analysis::Domain::Environment::Chip::CHIP_V1_1_0); | ||
| 309 | + | ||
| 310 | + KernelParserWorker worker(TEST_HOST_FILE_PATH); | ||
| 311 | + ASSERT_EQ(ANALYSIS_OK, worker.Run()); | ||
| 312 | + EXPECT_FALSE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "stream_info.db"}))); | ||
| 313 | + EXPECT_FALSE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "mc2_comm_info.db"}))); | ||
| 314 | +} | ||
| @@ -20,27 +20,68 @@ function: | |||
| 20 | Copyright Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. | 20 | Copyright Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. |
| 21 | """ | 21 | """ |
| 22 | import unittest | 22 | import unittest |
| 23 | +from unittest import mock | ||
| 23 | 24 | ||
| 25 | +from common_func.cpp_enable_scene import CannCalculatorScene, DeviceParseScene | ||
| 24 | from common_func.info_conf_reader import InfoConfReader | 26 | from common_func.info_conf_reader import InfoConfReader |
| 25 | from common_func.platform.chip_manager import ChipManager | 27 | from common_func.platform.chip_manager import ChipManager |
| 28 | +from common_func.profiling_scene import ProfilingScene | ||
| 26 | from framework.config_data_parsers import ConfigDataParsers | 29 | from framework.config_data_parsers import ConfigDataParsers |
| 27 | from msconfig.config_manager import ConfigManager | 30 | from msconfig.config_manager import ConfigManager |
| 28 | from profiling_bean.prof_enum.chip_model import ChipModel | 31 | from profiling_bean.prof_enum.chip_model import ChipModel |
| 29 | 32 | ||
| 30 | 33 | ||
| 31 | class TestConfigDataParsers(unittest.TestCase): | 34 | class TestConfigDataParsers(unittest.TestCase): |
| 35 | + | ||
| 36 | + def _parser_names(parsers): | ||
| 37 | + return {parser.__name__ for level_parsers in parsers.values() for parser in level_parsers} | ||
| 38 | + | ||
| 32 | def test_get_parsers(self): | 39 | def test_get_parsers(self): |
| 33 | InfoConfReader()._sample_json = {'devices': '0'} | 40 | InfoConfReader()._sample_json = {'devices': '0'} |
| 34 | - ConfigDataParsers.get_parsers(ConfigManager.DATA_CALCULATOR, str(ChipModel.CHIP_V3_1_0.value), False) | 41 | + parsers = ConfigDataParsers.get_parsers( |
| 42 | + ConfigManager.DATA_CALCULATOR, str(ChipModel.CHIP_V3_1_0.value), False) | ||
| 43 | + self.assertIsInstance(parsers, dict) | ||
| 35 | 44 | ||
| 36 | def test_load_can_cpp_parse_or_calculate_host_data(self): | 45 | def test_load_can_cpp_parse_or_calculate_host_data(self): |
| 37 | ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_host_data("NpuMemParser") | 46 | ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_host_data("NpuMemParser") |
| 38 | self.assertFalse(ret) | 47 | self.assertFalse(ret) |
| 39 | ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_host_data("HashDicParser") | 48 | ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_host_data("HashDicParser") |
| 40 | self.assertTrue(ret) | 49 | self.assertTrue(ret) |
| 50 | + ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_host_data("CaptureStreamInfoParser") | ||
| 51 | + self.assertFalse(ret) | ||
| 52 | + ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_host_data("Mc2CommInfoParser") | ||
| 53 | + self.assertFalse(ret) | ||
| 41 | ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_host_data("RuntimeOpInfoParser") | 54 | ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_host_data("RuntimeOpInfoParser") |
| 42 | self.assertTrue(ret) | 55 | self.assertTrue(ret) |
| 43 | 56 | ||
| 57 | + | ||
| 58 | + | ||
| 59 | + def test_get_parsers_should_keep_capture_and_mc2_python_fallback_when_host_cpp_is_disabled( | ||
| 60 | + self, _host_cpp_enable, _device_cpp_enable): | ||
| 61 | + InfoConfReader()._sample_json = {'devices': str(64)} | ||
| 62 | + | ||
| 63 | + with mock.patch.object(ProfilingScene(), "is_all_export", return_value=False): | ||
| 64 | + parsers = ConfigDataParsers.get_parsers( | ||
| 65 | + ConfigManager.DATA_PARSERS, str(ChipModel.CHIP_V3_3_0.value), False) | ||
| 66 | + | ||
| 67 | + parser_names = self._parser_names(parsers) | ||
| 68 | + self.assertIn("CaptureStreamInfoParser", parser_names) | ||
| 69 | + self.assertIn("Mc2CommInfoParser", parser_names) | ||
| 70 | + | ||
| 71 | + | ||
| 72 | + | ||
| 73 | + def test_get_parsers_should_keep_capture_and_mc2_when_host_cpp_is_enabled( | ||
| 74 | + self, _host_cpp_enable, _device_cpp_enable): | ||
| 75 | + InfoConfReader()._sample_json = {'devices': str(64)} | ||
| 76 | + | ||
| 77 | + with mock.patch.object(ProfilingScene(), "is_all_export", return_value=False): | ||
| 78 | + parsers = ConfigDataParsers.get_parsers( | ||
| 79 | + ConfigManager.DATA_PARSERS, str(ChipModel.CHIP_V3_3_0.value), False) | ||
| 80 | + | ||
| 81 | + parser_names = self._parser_names(parsers) | ||
| 82 | + self.assertIn("CaptureStreamInfoParser", parser_names) | ||
| 83 | + self.assertIn("Mc2CommInfoParser", parser_names) | ||
| 84 | + | ||
| 44 | def test_load_can_cpp_parse_or_calculate_device_data_should_return_true_when_given_in_whitelist(self): | 85 | def test_load_can_cpp_parse_or_calculate_device_data_should_return_true_when_given_in_whitelist(self): |
| 45 | ChipManager().chip_id = ChipModel.CHIP_V4_1_0 | 86 | ChipManager().chip_id = ChipModel.CHIP_V4_1_0 |
| 46 | ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_device_data("AscendTaskCalculator") | 87 | ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_device_data("AscendTaskCalculator") |


严重程度: 建议
问题: END 记录仅按
modelId全局去重:if (record.captureStatus == 1 && !endModelSet.insert(record.modelId).second) continue;,某 modelId 的首条 END 之后,任何其它 (device, stream) 上同 modelId 的 END 都被静默丢弃。原因: 被丢弃的 END 不会写入
output,下游RTAddInfoCenter::BuildCaptureInfoTimeRange对应 (deviceId, streamId, batchId) 的TimeRangeInfo末端将保持kInf,使该流上任务GetModelId在时间窗内一直命中,可能错误归属 modelId。UTShouldSortDeduplicateAssignBatchAndDropRepeatedModelEnd编码了当前行为(跨 device 的 dev1 END 被丢),但这是否符合采集语义需确认:PR 描述称“与 Python 逐字段一致”,请确认 Python 对多 device/多 stream 同 modelId 的 END 也是全局只保留首条。怎么改: 若需按会话区分,改为按
(deviceId, streamId, modelId)或(deviceId, streamId)去重:// 例如按 device+stream 去重 END if (record.captureStatus == 1 && !endSeenStreams.insert(deviceStreamKey).second) { continue; }若 Python 确为全局 modelId 去重,建议在注释里写明该约定,便于后续维护者理解。