已合并
[msprof]补充Capture Stream与MC2通信信息Parser C化模块 #398
[msprof]补充Capture Stream与MC2通信信息Parser C化模块 #398
已合并
gcw_648Mce95创建于 8月4日
34 个文件变更+1654-78
@@ -22,7 +22,6 @@
22#include "analysis/csrc/domain/entities/viewer_data/ai_task/include/communication_info_data.h"22#include "analysis/csrc/domain/entities/viewer_data/ai_task/include/communication_info_data.h"
23#include "analysis/csrc/domain/entities/viewer_data/ai_task/include/dpu_data.h"23#include "analysis/csrc/domain/entities/viewer_data/ai_task/include/dpu_data.h"
24#include "analysis/csrc/domain/entities/viewer_data/ai_task/include/kfc_turn_data.h"24#include "analysis/csrc/domain/entities/viewer_data/ai_task/include/kfc_turn_data.h"
25-#include "analysis/csrc/domain/entities/viewer_data/ai_task/include/mc2_comm_info_data.h"
26#include "analysis/csrc/domain/entities/viewer_data/ai_task/include/memcpy_info_data.h"25#include "analysis/csrc/domain/entities/viewer_data/ai_task/include/memcpy_info_data.h"
27#include "analysis/csrc/domain/entities/viewer_data/ai_task/include/msprof_tx_host_data.h"26#include "analysis/csrc/domain/entities/viewer_data/ai_task/include/msprof_tx_host_data.h"
28#include "analysis/csrc/domain/entities/viewer_data/ai_task/include/task_info_data.h"27#include "analysis/csrc/domain/entities/viewer_data/ai_task/include/task_info_data.h"
@@ -123,9 +122,8 @@ REGISTER_DB_SAVER_WITH_DATA(PROCESSOR_NAME_TASK,
123 TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_DEVICE_TX)),122 TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_DEVICE_TX)),
124 std::vector<AscendTaskData>, std::vector<MsprofTxDeviceData>);123 std::vector<AscendTaskData>, std::vector<MsprofTxDeviceData>);
125REGISTER_DB_SAVER_WITH_DATA(PROCESSOR_NAME_COMPUTE_TASK_INFO,124REGISTER_DB_SAVER_WITH_DATA(PROCESSOR_NAME_COMPUTE_TASK_INFO,
126- TOPO_DEPS(TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_COMPUTE_TASK_INFO),125+ TOPO_DEPS(TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_COMPUTE_TASK_INFO)),
127- TOPO_NODE(DATA_PROCESSING, PROCESSOR_MC2_COMM_INFO)),126+ std::vector<TaskInfoData>);
128- std::vector<TaskInfoData>, std::vector<MC2CommInfoData>);
129REGISTER_DB_SAVER_WITH_DATA(PROCESSOR_NAME_MEMCPY_INFO,127REGISTER_DB_SAVER_WITH_DATA(PROCESSOR_NAME_MEMCPY_INFO,
130 TOPO_DEPS(TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_MEMCPY_INFO)),128 TOPO_DEPS(TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_MEMCPY_INFO)),
131 std::vector<MemcpyInfoData>);129 std::vector<MemcpyInfoData>);
@@ -35,7 +35,7 @@ namespace
35const std::set<std::string> FILTER_TYPE = {"KERNEL_AICORE", "KERNEL_AIVEC", "FFTS_PLUS", "KERNEL_MIX_AIC",35const std::set<std::string> FILTER_TYPE = {"KERNEL_AICORE", "KERNEL_AIVEC", "FFTS_PLUS", "KERNEL_MIX_AIC",
36 "KERNEL_MIX_AIV", "PROFILING_ENABLE", "PROFILING_DISABLE"};36 "KERNEL_MIX_AIV", "PROFILING_ENABLE", "PROFILING_DISABLE"};
37 37 
38-void SepOneTask(std::vector<TimeDuration> &times, std::set<uint32_t> &mc2StreamsTable, TaskInfoData &task,38+void SepOneTask(std::vector<TimeDuration> &times, const std::set<uint32_t> &mc2StreamsTable, TaskInfoData &task,
39 std::unordered_map<uint16_t, std::vector<TimeDuration>> &compSections)39 std::unordered_map<uint16_t, std::vector<TimeDuration>> &compSections)
40{40{
41 if (mc2StreamsTable.find(task.streamId) != mc2StreamsTable.end() || EndsWith(task.opName, AICPU_KERNEL) ||41 if (mc2StreamsTable.find(task.streamId) != mc2StreamsTable.end() || EndsWith(task.opName, AICPU_KERNEL) ||
@@ -27,7 +27,6 @@
27#include "analysis/csrc/domain/data_process/ai_task/hccl_statistic_processor.h"27#include "analysis/csrc/domain/data_process/ai_task/hccl_statistic_processor.h"
28#include "analysis/csrc/domain/data_process/ai_task/host_task_processor.h"28#include "analysis/csrc/domain/data_process/ai_task/host_task_processor.h"
29#include "analysis/csrc/domain/data_process/ai_task/kfc_task_processor.h"29#include "analysis/csrc/domain/data_process/ai_task/kfc_task_processor.h"
30-#include "analysis/csrc/domain/data_process/ai_task/mc2_comm_info_processor.h"
31#include "analysis/csrc/domain/data_process/ai_task/memcpy_info_processor.h"30#include "analysis/csrc/domain/data_process/ai_task/memcpy_info_processor.h"
32#include "analysis/csrc/domain/data_process/ai_task/metric_processor.h"31#include "analysis/csrc/domain/data_process/ai_task/metric_processor.h"
33#include "analysis/csrc/domain/data_process/ai_task/model_name_processor.h"32#include "analysis/csrc/domain/data_process/ai_task/model_name_processor.h"
@@ -120,7 +119,6 @@ REGISTER_PROCESSOR(RoCETimelineProcessor, PROCESSOR_NAME_ROCE_TIMELINE, TOPO_DEP
120REGISTER_PROCESSOR(NicProcessor, PROCESSOR_NAME_NIC, TOPO_DEPS());119REGISTER_PROCESSOR(NicProcessor, PROCESSOR_NAME_NIC, TOPO_DEPS());
121REGISTER_PROCESSOR(RoCEProcessor, PROCESSOR_NAME_ROCE, TOPO_DEPS());120REGISTER_PROCESSOR(RoCEProcessor, PROCESSOR_NAME_ROCE, TOPO_DEPS());
122REGISTER_PROCESSOR(QosProcessor, PROCESSOR_NAME_QOS, TOPO_DEPS());121REGISTER_PROCESSOR(QosProcessor, PROCESSOR_NAME_QOS, TOPO_DEPS());
123-REGISTER_PROCESSOR(Mc2CommInfoProcessor, PROCESSOR_MC2_COMM_INFO, TOPO_DEPS());
124REGISTER_PROCESSOR(MetricProcessor, PROCESSOR_PMU, TOPO_DEPS());122REGISTER_PROCESSOR(MetricProcessor, PROCESSOR_PMU, TOPO_DEPS());
125REGISTER_PROCESSOR(MemcpyInfoProcessor, PROCESSOR_NAME_MEMCPY_INFO, TOPO_DEPS());123REGISTER_PROCESSOR(MemcpyInfoProcessor, PROCESSOR_NAME_MEMCPY_INFO, TOPO_DEPS());
126REGISTER_PROCESSOR_WITH_DATA(NpuOpMemProcessor, PROCESSOR_NAME_NPU_OP_MEM,124REGISTER_PROCESSOR_WITH_DATA(NpuOpMemProcessor, PROCESSOR_NAME_NPU_OP_MEM,
@@ -136,10 +134,8 @@ REGISTER_PROCESSOR(HostTaskProcessor, PROCESSOR_HOST_TASK, TOPO_DEPS());
136REGISTER_PROCESSOR_WITH_DATA(OverlapAnalysisProcessor, PROCESSOR_NAME_OVERLAP_ANALYSIS,134REGISTER_PROCESSOR_WITH_DATA(OverlapAnalysisProcessor, PROCESSOR_NAME_OVERLAP_ANALYSIS,
137 TOPO_DEPS(TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_TASK),135 TOPO_DEPS(TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_TASK),
138 TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_COMPUTE_TASK_INFO),136 TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_COMPUTE_TASK_INFO),
139- TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_COMMUNICATION),137+ TOPO_NODE(DATA_PROCESSING, PROCESSOR_NAME_COMMUNICATION)),
140- TOPO_NODE(DATA_PROCESSING, PROCESSOR_MC2_COMM_INFO)),138+ std::vector<AscendTaskData>, std::vector<TaskInfoData>, std::vector<CommunicationOpData>);
141- std::vector<AscendTaskData>, std::vector<TaskInfoData>, std::vector<CommunicationOpData>,
142- std::vector<MC2CommInfoData>);
143REGISTER_PROCESSOR(LowPowerProcessor, PROCESSOR_NAME_LOW_POWER, TOPO_DEPS());139REGISTER_PROCESSOR(LowPowerProcessor, PROCESSOR_NAME_LOW_POWER, TOPO_DEPS());
144REGISTER_PROCESSOR(BiuPerfProcessor, PROCESSOR_NAME_BIU_PERF, TOPO_DEPS());140REGISTER_PROCESSOR(BiuPerfProcessor, PROCESSOR_NAME_BIU_PERF, TOPO_DEPS());
145REGISTER_PROCESSOR(UbProcessor, PROCESSOR_NAME_UB, TOPO_DEPS());141REGISTER_PROCESSOR(UbProcessor, PROCESSOR_NAME_UB, TOPO_DEPS());
@@ -264,6 +264,13 @@ bool ParserAdditionalInfoAdapter::AdapterAdditionalInfo(MsprofAdditionalInfo* ad
264 case AdditionalInfoFormat::TASK_MEMORY_INFO_TYPE:264 case AdditionalInfoFormat::TASK_MEMORY_INFO_TYPE:
265 AdapterMemoryInfo(addition, parsed);265 AdapterMemoryInfo(addition, parsed);
266 return true;266 return true;
267+ case AdditionalInfoFormat::MC2_COMM_INFO_TYPE:
268+ if (memcpy_s(parsed->data, sizeof(parsed->data), addition->data, sizeof(addition->data)) != EOK)
269+ {
270+ ERROR("adapter mc2 comm info data failed.");
271+ return false;
272+ }
273+ return true;
267 default:274 default:
268 ERROR("Unsupported Additional Info: %.", static_cast<uint32_t>(parserType));275 ERROR("Unsupported Additional Info: %.", static_cast<uint32_t>(parserType));
269 return false;276 return false;
@@ -19,11 +19,14 @@
19#include "analysis/csrc/domain/services/association/cann/include/tree_analyzer.h"19#include "analysis/csrc/domain/services/association/cann/include/tree_analyzer.h"
20#include "analysis/csrc/domain/services/association/cann/include/tree_builder.h"20#include "analysis/csrc/domain/services/association/cann/include/tree_builder.h"
21#include "analysis/csrc/domain/services/host_worker/host_cpu_freq_parser.h"21#include "analysis/csrc/domain/services/host_worker/host_cpu_freq_parser.h"
22+#include "analysis/csrc/domain/services/parser/host/cann/capture_mc2_cpp_enable.h"
22#include "analysis/csrc/domain/services/parser/host/cann/rt_add_info_center.h"23#include "analysis/csrc/domain/services/parser/host/cann/rt_add_info_center.h"
23#include "analysis/csrc/domain/services/persistence/host/api_event_db_dumper.h"24#include "analysis/csrc/domain/services/persistence/host/api_event_db_dumper.h"
24#include "analysis/csrc/domain/services/persistence/host/cann_trace_db_dumper.h"25#include "analysis/csrc/domain/services/persistence/host/cann_trace_db_dumper.h"
26+#include "analysis/csrc/domain/services/persistence/host/capture_stream_info_dumper.h"
25#include "analysis/csrc/domain/services/persistence/host/dpu_task_track_db_dumper.h"27#include "analysis/csrc/domain/services/persistence/host/dpu_task_track_db_dumper.h"
26#include "analysis/csrc/domain/services/persistence/host/flip_task_db_dumper.h"28#include "analysis/csrc/domain/services/persistence/host/flip_task_db_dumper.h"
29+#include "analysis/csrc/domain/services/persistence/host/mc2_comm_info_dumper.h"
27#include "analysis/csrc/domain/services/persistence/host/memcpy_info_dumper.h"30#include "analysis/csrc/domain/services/persistence/host/memcpy_info_dumper.h"
28#include "analysis/csrc/domain/services/persistence/host/model_name_db_dumper.h"31#include "analysis/csrc/domain/services/persistence/host/model_name_db_dumper.h"
29#include "analysis/csrc/domain/services/persistence/host/runtime_op_info_dumper.h"32#include "analysis/csrc/domain/services/persistence/host/runtime_op_info_dumper.h"
@@ -42,7 +45,19 @@ bool HostTraceWorker::Run()
42 auto hostDataPath = Utils::File::PathJoin({hostPath_, "data"});45 auto hostDataPath = Utils::File::PathJoin({hostPath_, "data"});
43 std::shared_ptr<EventGrouper> grouper;46 std::shared_ptr<EventGrouper> grouper;
44 MAKE_SHARED_RETURN_VALUE(grouper, EventGrouper, false, hostDataPath);47 MAKE_SHARED_RETURN_VALUE(grouper, EventGrouper, false, hostDataPath);
45- grouper->Group();48+ bool result = grouper->Group();
49+ // CaptureStreamInfoData formattedCaptureData;
50+ // if (Host::Cann::kEnableCaptureStreamMc2CppParser)
51+ // {
52+ // if (!DumpCaptureStreamInfo(grouper, formattedCaptureData))
53+ // {
54+ // result = false;
55+ // }
56+ // if (!DumpMc2CommInfo(grouper, formattedCaptureData))
57+ // {
58+ // result = false;
59+ // }
60+ // }
46 auto sqlitePath = Utils::File::PathJoin({hostPath_, "sqlite"});61 auto sqlitePath = Utils::File::PathJoin({hostPath_, "sqlite"});
47 RTAddInfoCenter::GetInstance().Load(sqlitePath);62 RTAddInfoCenter::GetInstance().Load(sqlitePath);
48 DumpRuntimeOpInfo();63 DumpRuntimeOpInfo();
@@ -75,6 +90,66 @@ bool HostTraceWorker::Run()
75 apiDumpPool.WaitAllTasks();90 apiDumpPool.WaitAllTasks();
76 apiDumpPool.Stop();91 apiDumpPool.Stop();
77 DumpMemcpyInfo(hostDataPath); // 依赖runtime.db中的HostTask, 不能放在pool中92 DumpMemcpyInfo(hostDataPath); // 依赖runtime.db中的HostTask, 不能放在pool中
93+ return result;
94+}
95+ 
96+bool HostTraceWorker::DumpCaptureStreamInfo(const std::shared_ptr<EventGrouper> &grouper,
97+ CaptureStreamInfoData &formattedCaptureData)
98+{
99+ bool result = true;
100+ RTAddInfoCenter::GetInstance().SetCaptureStreamInfoData({});
101+ const auto &captureData = grouper->GetCaptureStreamInfoData();
102+ if (captureData.empty())
103+ {
104+ return true;
105+ }
106+ 
107+ CaptureStreamInfoDumper captureDumper(hostPath_);
108+ formattedCaptureData = captureDumper.FormatData(captureData);
109+ if (formattedCaptureData.empty())
110+ {
111+ ERROR("Format capture stream info failed.");
112+ return false;
113+ }
114+ 
115+ std::vector<Analysis::Domain::CaptureStreamInfo> centerData;
116+ if (!Utils::Reserve(centerData, formattedCaptureData.size()))
117+ {
118+ ERROR("Reserve capture stream info center data failed.");
119+ result = false;
120+ }
121+ else
122+ {
123+ for (const auto &item : formattedCaptureData)
124+ {
125+ centerData.emplace_back(item.modelId, item.timeStamp, item.streamId, item.originalStreamId, item.deviceId,
126+ static_cast<uint16_t>(item.batchId), item.captureStatus);
127+ }
128+ RTAddInfoCenter::GetInstance().SetCaptureStreamInfoData(centerData);
129+ }
130+ if (!captureDumper.DumpData(formattedCaptureData))
131+ {
132+ ERROR("Dump capture stream info failed.");
133+ result = false;
134+ }
135+ return result;
136+}
137+ 
138+bool HostTraceWorker::DumpMc2CommInfo(const std::shared_ptr<EventGrouper> &grouper,
139+ const CaptureStreamInfoData &formattedCaptureData)
140+{
141+ const auto &mc2Data = grouper->GetMc2CommInfoData();
142+ if (mc2Data.empty())
143+ {
144+ return true;
145+ }
146+ Mc2CommInfoDumper mc2Dumper(hostPath_);
147+ Mc2CommInfoInput input(mc2Data, formattedCaptureData);
148+ if (!mc2Dumper.DumpData(input))
149+ {
150+ ERROR("Dump mc2 comm info failed.");
151+ return false;
152+ }
78 return true;153 return true;
79}154}
80 155 
@@ -16,6 +16,7 @@
16 16 
17#ifndef ANALYSIS_WORKER_HOST_TRACE_THREAD_H17#ifndef ANALYSIS_WORKER_HOST_TRACE_THREAD_H
18#define ANALYSIS_WORKER_HOST_TRACE_THREAD_H18#define ANALYSIS_WORKER_HOST_TRACE_THREAD_H
19+#include <cstdint>
19#include <set>20#include <set>
20#include <string>21#include <string>
21#include <utility>22#include <utility>
@@ -23,6 +24,7 @@
23#include "analysis/csrc/domain/entities/tree/include/tree.h"24#include "analysis/csrc/domain/entities/tree/include/tree.h"
24#include "analysis/csrc/domain/services/parser/host/cann/cann_warehouse.h"25#include "analysis/csrc/domain/services/parser/host/cann/cann_warehouse.h"
25#include "analysis/csrc/domain/services/parser/host/cann/event_grouper.h"26#include "analysis/csrc/domain/services/parser/host/cann/event_grouper.h"
27+#include "analysis/csrc/domain/services/persistence/host/capture_stream_info_dumper.h"
26#include "analysis/csrc/infrastructure/utils/safe_unordered_map.h"28#include "analysis/csrc/infrastructure/utils/safe_unordered_map.h"
27#include "analysis/csrc/infrastructure/utils/thread_pool.h"29#include "analysis/csrc/infrastructure/utils/thread_pool.h"
28 30 
@@ -57,6 +59,10 @@ class HostTraceWorker
57 void DumpModelName(ThreadPool &pool, const std::string &hostDataPath);59 void DumpModelName(ThreadPool &pool, const std::string &hostDataPath);
58 void DumpHostSystemProfileData(ThreadPool &pool);60 void DumpHostSystemProfileData(ThreadPool &pool);
59 void DumpMemcpyInfo(const std::string &hostDataPath);61 void DumpMemcpyInfo(const std::string &hostDataPath);
62+ bool DumpCaptureStreamInfo(const std::shared_ptr<EventGrouper> &grouper,
63+ CaptureStreamInfoData &formattedCaptureData);
64+ bool DumpMc2CommInfo(const std::shared_ptr<EventGrouper> &grouper,
65+ const CaptureStreamInfoData &formattedCaptureData);
60 66 
61 private:67 private:
62 const uint32_t poolSize_ = 10;68 const uint32_t poolSize_ = 10;
@@ -32,6 +32,24 @@ namespace Host
32{32{
33namespace Cann33namespace Cann
34{34{
35+ 
36+constexpr uint32_t MC2_COMM_STREAM_MAX_NUM = 8;
37+constexpr uint32_t MC2_COMM_RESERVED_UINT32_NUM = 43;
38+ 
39+struct MsprofMc2CommInfo
40+{
41+ uint64_t groupName;
42+ uint32_t rankSize;
43+ uint32_t rankId;
44+ uint32_t usrRankId;
45+ uint32_t streamId;
46+ uint32_t streamSize;
47+ uint32_t commStreamIds[MC2_COMM_STREAM_MAX_NUM];
48+ uint32_t reserved[MC2_COMM_RESERVED_UINT32_NUM];
49+};
50+ 
51+static_assert(sizeof(MsprofMc2CommInfo) == 232, "MC2 payload must match the 256-byte additional record");
52+ 
35// 该类的作用是Addition数据的解析53// 该类的作用是Addition数据的解析
36class AdditionInfoParser : public BaseParser<AdditionInfoParser>54class AdditionInfoParser : public BaseParser<AdditionInfoParser>
37{55{
@@ -53,7 +71,25 @@ class AdditionInfoParser : public BaseParser<AdditionInfoParser>
53 std::vector<std::shared_ptr<ParserConcatTensorInfo>> concatTensorData_; // not owned71 std::vector<std::shared_ptr<ParserConcatTensorInfo>> concatTensorData_; // not owned
54}; // class AdditionInfoParser72}; // class AdditionInfoParser
55 73 
74+// mc2通信数据的解析,复用Additional数据读取流程
75+class Mc2CommInfoParser final : public AdditionInfoParser
76+{
77+ public:
78+ explicit Mc2CommInfoParser(const std::string &path) : AdditionInfoParser(path, "Mc2CommInfoParser")
79+ {
80+ parserType_ = AdditionalInfoFormat::MC2_COMM_INFO_TYPE;
81+ Init(filePrefix_);
82+ }
83+ 
84+ private:
85+ std::vector<std::string> filePrefix_ = {
86+ // "unaging.additional.mc2_comm_info.slice",
87+ // "aging.additional.mc2_comm_info.slice",
88+ };
89+}; // class Mc2CommInfoParser
90+ 
56// 该类的作用是CtxId数据的解析91// 该类的作用是CtxId数据的解析
92+ 
57class CtxIdParser final : public AdditionInfoParser93class CtxIdParser final : public AdditionInfoParser
58{94{
59 public:95 public:
@@ -24,15 +24,20 @@
24#include "analysis/csrc/domain/entities/tree/include/event.h"24#include "analysis/csrc/domain/entities/tree/include/event.h"
25#include "analysis/csrc/domain/entities/tree/include/event_queue.h"25#include "analysis/csrc/domain/entities/tree/include/event_queue.h"
26 26 
27-namespace Analysis {27+namespace Analysis
28-namespace Domain {28+{
29-namespace Host {29+namespace Domain
30-namespace Cann {30+{
31+namespace Host
32+{
33+namespace Cann
34+{
31 35 
32using EventQueue = Analysis::Domain::EventQueue;36using EventQueue = Analysis::Domain::EventQueue;
33 37 
34// CANN数据仓,用于存储各个Type的Events38// CANN数据仓,用于存储各个Type的Events
35-struct CANNWarehouse {39+struct CANNWarehouse
40+{
36 // API Type只在Model、Node、HCCL Level的Event41 // API Type只在Model、Node、HCCL Level的Event
37 std::shared_ptr<EventQueue> kernelEvents = nullptr;42 std::shared_ptr<EventQueue> kernelEvents = nullptr;
38 std::shared_ptr<EventQueue> graphIdMapEvents = nullptr;43 std::shared_ptr<EventQueue> graphIdMapEvents = nullptr;
@@ -46,8 +51,8 @@ struct CANNWarehouse {
46 std::shared_ptr<EventQueue> hcclOpInfoEvents = nullptr;51 std::shared_ptr<EventQueue> hcclOpInfoEvents = nullptr;
47};52};
48 53 
49-} // namespace Cann54+} // namespace Cann
50-} // namespace Host55+} // namespace Host
51-} // namespace Parser56+} // namespace Domain
52-} // namespace Analysis57+} // namespace Analysis
53-#endif // ANALYSIS_PARSER_HOST_CANN_CANN_WAREHOUSE_H58+#endif // ANALYSIS_PARSER_HOST_CANN_CANN_WAREHOUSE_H
@@ -0,0 +1,36 @@
1+/* -------------------------------------------------------------------------
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This file is part of the MindStudio project.
4+ *
5+ * MindStudio is licensed under Mulan PSL v2.
6+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
7+ * You may obtain a copy of Mulan PSL v2 at:
8+ *
9+ * http://license.coscl.org.cn/MulanPSL2
10+ *
11+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14+ * See the Mulan PSL v2 for more details.
15+ * -------------------------------------------------------------------------*/
16+ 
17+#ifndef ANALYSIS_PARSER_HOST_CANN_CAPTURE_MC2_CPP_ENABLE_H
18+#define ANALYSIS_PARSER_HOST_CANN_CAPTURE_MC2_CPP_ENABLE_H
19+ 
20+#pragma once
21+ 
22+namespace Analysis
23+{
24+namespace Domain
25+{
26+namespace Host
27+{
28+namespace Cann
29+{
30+constexpr bool kEnableCaptureStreamMc2CppParser = false;
31+} // namespace Cann
32+} // namespace Host
33+} // namespace Domain
34+} // namespace Analysis
35+ 
36+#endif // ANALYSIS_PARSER_HOST_CANN_CAPTURE_MC2_CPP_ENABLE_H
@@ -42,6 +42,20 @@ enum class DevType : uint16_t
42 DPU = 1,42 DPU = 1,
43};43};
44 44 
45+const std::vector<std::string> CAPTURE_V1_PREFIXES = {
46+ // "unaging.compact.capture_stream_info.slice",
47+ // "aging.compact.capture_stream_info.slice",
48+};
49+const std::vector<std::string> CAPTURE_V2_PREFIXES = {
50+ // "unaging.compact.capture_stream_info_v2.slice",
51+ // "aging.compact.capture_stream_info_v2.slice",
52+};
53+ 
54+bool HasCaptureData(const std::string &path, const std::vector<std::string> &prefixes)
55+{
56+ return !File::GetOriginData(path, prefixes, {"done", "complete"}).empty();
57+}
58+ 
45constexpr uint16_t kDevTypeShift = 12;59constexpr uint16_t kDevTypeShift = 12;
46constexpr uint16_t kDevTypeMask = 0xF;60constexpr uint16_t kDevTypeMask = 0xF;
47 61 
@@ -107,6 +121,50 @@ int CompactInfoParser::ProduceData()
107 return ANALYSIS_OK;121 return ANALYSIS_OK;
108}122}
109 123 
124+CaptureStreamInfoParser::CaptureStreamInfoParser(const std::string &path)
125+ : CompactInfoParser(path, "CaptureStreamInfoParser")
126+{
127+ isV2_ = !HasCaptureData(path, CAPTURE_V1_PREFIXES) && HasCaptureData(path, CAPTURE_V2_PREFIXES);
128+ Init(isV2_ ? CAPTURE_V2_PREFIXES : CAPTURE_V1_PREFIXES);
129+}
130+ 
131+int CaptureStreamInfoParser::ProduceData()
132+{
133+ if (chunkProducer_->Empty())
134+ {
135+ return ANALYSIS_OK;
136+ }
137+ if (!Reserve(compactData_, chunkProducer_->Size()))
138+ {
139+ ERROR("%: Reserve data failed", parserName_);
140+ return ANALYSIS_ERROR;
141+ }
142+ const auto format = isV2_ ? CaptureStreamFormat::V2 : CaptureStreamFormat::V1;
143+ while (!chunkProducer_->Empty())
144+ {
145+ std::unique_ptr<char[]> chunk(chunkProducer_->Pop());
146+ auto compactInfo = ReinterpretConvert<MsprofCompactInfo *>(chunk.get());
147+ if (!compactInfo)
148+ {
149+ ERROR("%: Pop chunk failed.", parserName_);
150+ return ANALYSIS_ERROR;
151+ }
152+ if (compactInfo->magicNumber != MSPROF_DATA_HEAD_MAGIC_NUM)
153+ {
154+ ERROR("%: The last %th data check failed.", parserName_, chunkProducer_->Size());
155+ continue;
156+ }
157+ auto parserCompactInfo = std::make_shared<ParserCompactInfo>();
158+ if (!ParserCompactInfoAdapter::AdapterCaptureStreamInfo(compactInfo, format, parserCompactInfo.get()))
159+ {
160+ ERROR("%: copy captureStreamInfo data failed.", parserName_);
161+ return ANALYSIS_ERROR;
162+ }
163+ compactData_.emplace_back(std::move(parserCompactInfo));
164+ }
165+ return ANALYSIS_OK;
166+}
167+ 
110int NodeBasicInfoParser::ProduceData()168int NodeBasicInfoParser::ProduceData()
111{169{
112 std::shared_ptr<ChunkGenerator> staticChunkProducer;170 std::shared_ptr<ChunkGenerator> staticChunkProducer;
@@ -24,6 +24,7 @@
24#include "analysis/csrc/domain/services/adapter/flip.h"24#include "analysis/csrc/domain/services/adapter/flip.h"
25#include "analysis/csrc/domain/services/parser/host/base_parser.h"25#include "analysis/csrc/domain/services/parser/host/base_parser.h"
26#include "analysis/csrc/infrastructure/utils/file.h"26#include "analysis/csrc/infrastructure/utils/file.h"
27+#include "analysis/csrc/infrastructure/utils/parser_struct.h"
27#include "analysis/csrc/infrastructure/utils/prof_struct.h"28#include "analysis/csrc/infrastructure/utils/prof_struct.h"
28 29 
29namespace Analysis30namespace Analysis
@@ -53,6 +54,19 @@ class CompactInfoParser : public BaseParser<CompactInfoParser>
53 std::vector<std::shared_ptr<Adapter::FlipTask>> flipTaskData_; // not owned54 std::vector<std::shared_ptr<Adapter::FlipTask>> flipTaskData_; // not owned
54}; // class CompactInfoParser55}; // class CompactInfoParser
55 56 
57+// capture stream info数据的解析,复用Compact数据读取流程
58+class CaptureStreamInfoParser final : public CompactInfoParser
59+{
60+ public:
61+ explicit CaptureStreamInfoParser(const std::string &path);
62+ 
63+ private:
64+ int ProduceData() override;
65+ 
66+ private:
67+ bool isV2_ = false;
68+}; // class CaptureStreamInfoParser
69+ 
56// 该类的作用是node basic info数据的解析70// 该类的作用是node basic info数据的解析
57class NodeBasicInfoParser final : public CompactInfoParser71class NodeBasicInfoParser final : public CompactInfoParser
58{72{
@@ -17,6 +17,7 @@
17#include "analysis/csrc/domain/services/parser/host/cann/event_grouper.h"17#include "analysis/csrc/domain/services/parser/host/cann/event_grouper.h"
18 18 
19#include "analysis/csrc/domain/services/environment/context.h"19#include "analysis/csrc/domain/services/environment/context.h"
20+#include "analysis/csrc/domain/services/parser/host/cann/capture_mc2_cpp_enable.h"
20#include "analysis/csrc/domain/services/parser/host/cann/compact_info_parser.h"21#include "analysis/csrc/domain/services/parser/host/cann/compact_info_parser.h"
21#include "analysis/csrc/domain/services/parser/host/cann/rt_add_info_center.h"22#include "analysis/csrc/domain/services/parser/host/cann/rt_add_info_center.h"
22#include "analysis/csrc/domain/services/parser/host/cann/type_data.h"23#include "analysis/csrc/domain/services/parser/host/cann/type_data.h"
@@ -66,10 +67,47 @@ std::vector<std::shared_ptr<Adapter::FlipTask>> &EventGrouper::GetFlipTasks() {
66std::vector<std::shared_ptr<ParserCompactInfo>> &EventGrouper::GetDpuTrackData() { return dpuTrackData_; }67std::vector<std::shared_ptr<ParserCompactInfo>> &EventGrouper::GetDpuTrackData() { return dpuTrackData_; }
67std::unordered_map<uint64_t, uint64_t> &EventGrouper::GetDpuKernelNameMap() { return dpuKernelNameMap_; }68std::unordered_map<uint64_t, uint64_t> &EventGrouper::GetDpuKernelNameMap() { return dpuKernelNameMap_; }
68 69 
70+const std::vector<std::shared_ptr<ParserCompactInfo>> &EventGrouper::GetCaptureStreamInfoData() const
71+{
72+ return captureStreamInfoData_;
73+}
74+ 
75+const std::vector<std::shared_ptr<ParserAdditionalInfo>> &EventGrouper::GetMc2CommInfoData() const
76+{
77+ return mc2CommInfoData_;
78+}
79+ 
80+void EventGrouper::ParseCaptureStreamInfo()
81+{
82+ Utils::TimeLogger t{"Parse CaptureStreamInfo"};
83+ CaptureStreamInfoParser parser(hostPath_);
84+ captureStreamInfoData_ = parser.ParseData<ParserCompactInfo>();
85+ if (parser.GetStatus() == ParserStatus::ERROR)
86+ {
87+ ERROR("Parse capture stream info failed.");
88+ result_ = false;
89+ }
90+}
91+ 
92+void EventGrouper::ParseMc2CommInfo()
93+{
94+ Utils::TimeLogger t{"Parse Mc2CommInfo"};
95+ Mc2CommInfoParser parser(hostPath_);
96+ mc2CommInfoData_ = parser.ParseData<ParserAdditionalInfo>();
97+ if (parser.GetStatus() == ParserStatus::ERROR)
98+ {
99+ ERROR("Parse mc2 comm info failed.");
100+ result_ = false;
101+ }
102+}
103+ 
69bool EventGrouper::Group()104bool EventGrouper::Group()
70{105{
71 Utils::TimeLogger t{"Group all events"};106 Utils::TimeLogger t{"Group all events"};
72- const uint32_t poolSize = 8;107+ result_ = true;
108+ captureStreamInfoData_.clear();
109+ mc2CommInfoData_.clear();
110+ const uint32_t poolSize = 10;
73 ThreadPool pool(poolSize);111 ThreadPool pool(poolSize);
74 pool.Start();112 pool.Start();
75 GroupTreeEvent(pool);113 GroupTreeEvent(pool);
@@ -78,7 +116,7 @@ bool EventGrouper::Group()
78 pool.Stop();116 pool.Stop();
79 SetApiEventKeys();117 SetApiEventKeys();
80 RecordCANNWareHouses();118 RecordCANNWareHouses();
81- return true;119+ return result_;
82}120}
83 121 
84void EventGrouper::GroupTreeEvent(ThreadPool &pool)122void EventGrouper::GroupTreeEvent(ThreadPool &pool)
@@ -110,6 +148,11 @@ void EventGrouper::GroupTreeEvent(ThreadPool &pool)
110 GroupEvents<NodeAttrInfoParser, ParserCompactInfo, &CANNWarehouse::nodeAttrInfoEvents>(148 GroupEvents<NodeAttrInfoParser, ParserCompactInfo, &CANNWarehouse::nodeAttrInfoEvents>(
111 "NodeAttrInfo", EventType::EVENT_TYPE_NODE_ATTR_INFO);149 "NodeAttrInfo", EventType::EVENT_TYPE_NODE_ATTR_INFO);
112 });150 });
151+ // if (kEnableCaptureStreamMc2CppParser)
152+ // {
153+ // pool.AddTask([this]() { ParseCaptureStreamInfo(); });
154+ // pool.AddTask([this]() { ParseMc2CommInfo(); });
155+ // }
113 pool.AddTask(156 pool.AddTask(
114 [this]()157 [this]()
115 {158 {
@@ -18,6 +18,7 @@
18#define ANALYSIS_PARSER_HOST_CANN_EVENT_GROUPER_H18#define ANALYSIS_PARSER_HOST_CANN_EVENT_GROUPER_H
19 19 
20#include <algorithm>20#include <algorithm>
21+#include <atomic>
21#include <cstdint>22#include <cstdint>
22#include <memory>23#include <memory>
23#include <set>24#include <set>
@@ -75,6 +76,8 @@ class EventGrouper
75 std::vector<std::shared_ptr<ParserCompactInfo>> &GetDpuTrackData();76 std::vector<std::shared_ptr<ParserCompactInfo>> &GetDpuTrackData();
76 // 获取DpuKernelNameMap77 // 获取DpuKernelNameMap
77 std::unordered_map<uint64_t, uint64_t> &GetDpuKernelNameMap();78 std::unordered_map<uint64_t, uint64_t> &GetDpuKernelNameMap();
79+ const std::vector<std::shared_ptr<ParserCompactInfo>> &GetCaptureStreamInfoData() const;
80+ const std::vector<std::shared_ptr<ParserAdditionalInfo>> &GetMc2CommInfoData() const;
78 // ACL层建树白名单81 // ACL层建树白名单
79 bool IsBuildTreeWithAcl(const std::shared_ptr<ParserApi> &trace);82 bool IsBuildTreeWithAcl(const std::shared_ptr<ParserApi> &trace);
80 83 
@@ -82,6 +85,8 @@ class EventGrouper
82 bool isKernelApiEvent(const std::shared_ptr<ParserApi> &trace);85 bool isKernelApiEvent(const std::shared_ptr<ParserApi> &trace);
83 void InitLastKernelTimes(const std::set<uint32_t> &threadIds);86 void InitLastKernelTimes(const std::set<uint32_t> &threadIds);
84 void RecordCANNWareHouses();87 void RecordCANNWareHouses();
88+ void ParseCaptureStreamInfo();
89+ void ParseMc2CommInfo();
85 void SetApiEventKeys();90 void SetApiEventKeys();
86 91 
87 void GroupTreeEvent(ThreadPool &pool);92 void GroupTreeEvent(ThreadPool &pool);
@@ -156,6 +161,9 @@ class EventGrouper
156 std::set<uint32_t> threadIds_;161 std::set<uint32_t> threadIds_;
157 std::string hostPath_;162 std::string hostPath_;
158 CANNWarehouses cannWarehouses_; // 所有threadId的数据163 CANNWarehouses cannWarehouses_; // 所有threadId的数据
164+ std::vector<std::shared_ptr<ParserCompactInfo>> captureStreamInfoData_;
165+ std::vector<std::shared_ptr<ParserAdditionalInfo>> mc2CommInfoData_;
166+ std::atomic<bool> result_{true};
159 // 记录已经处理好的kernelEvents的最晚时间(threadId, level, time)167 // 记录已经处理好的kernelEvents的最晚时间(threadId, level, time)
160 std::unordered_map<uint32_t, std::unordered_map<uint16_t, std::pair<uint64_t, uint64_t>>> lastKernelTimes_;168 std::unordered_map<uint32_t, std::unordered_map<uint16_t, std::pair<uint64_t, uint64_t>>> lastKernelTimes_;
161};169};
@@ -40,13 +40,17 @@ void Cann::RTAddInfoCenter::Load(const std::string& path)
40 {40 {
41 LoadDB(path);41 LoadDB(path);
42 }42 }
43- LoadCaptureInfoDB(path);43+}
44+ 
45+void RTAddInfoCenter::SetCaptureStreamInfoData(const std::vector<CaptureStreamInfo>& data)
46+{
47+ captureStreamInfoData_ = data;
48+ captureInfoTimeRangeDict_.clear();
44 BuildCaptureInfoTimeRange();49 BuildCaptureInfoTimeRange();
45}50}
46 51 
47void RTAddInfoCenter::Add(const RuntimeOpInfo& info)52void RTAddInfoCenter::Add(const RuntimeOpInfo& info)
48{53{
49- // 仅在 EventGrouper::Group() 内单线程调用(见头文件使用场景注释),与后续读取互不重叠
50 std::string key = Utils::Join("_", info.deviceId, info.streamId, info.taskId);54 std::string key = Utils::Join("_", info.deviceId, info.streamId, info.taskId);
51 runtimeOpInfoData_[key] = info;55 runtimeOpInfoData_[key] = info;
52 dumpList_.push_back(info);56 dumpList_.push_back(info);
@@ -116,41 +120,6 @@ void RTAddInfoCenter::BuildCaptureInfoTimeRange()
116 }120 }
117}121}
118 122 
119-void RTAddInfoCenter::LoadCaptureInfoDB(const std::string& path)
120-{
121- StreamInfoDB streamInfoDB;
122- std::string hostDbDirectory = Utils::File::PathJoin({path, streamInfoDB.GetDBName()});
123- DBRunner dbRunner(hostDbDirectory);
124- if (!File::Exist(hostDbDirectory) || !dbRunner.CheckTableExists("CaptureStreamInfo"))
125- {
126- return;
127- }
128- std::string sql{
129- "SELECT model_id, timestamp, stream_id, original_stream_id, device_id, batch_id, capture_status "
130- "FROM CaptureStreamInfo"};
131- using DataFormat = std::vector<std::tuple<uint64_t, uint64_t, uint32_t, uint32_t, uint16_t, uint32_t, uint16_t>>;
132- DataFormat result;
133- if (!dbRunner.QueryData(sql, result))
134- {
135- ERROR("Query capture stream info data failed, db path is %.", hostDbDirectory);
136- return;
137- }
138- 
139- for (const auto& row : result)
140- {
141- uint16_t deviceId;
142- uint32_t originalStreamId;
143- uint32_t batchId;
144- uint16_t captureStatus;
145- uint32_t streamId;
146- uint64_t modelId;
147- uint64_t timeStamp;
148- std::tie(modelId, timeStamp, streamId, originalStreamId, deviceId, batchId, captureStatus) = row;
149- CaptureStreamInfo info{modelId, timeStamp, streamId, originalStreamId, deviceId, batchId, captureStatus};
150- captureStreamInfoData_.push_back(info);
151- }
152-}
153- 
154void RTAddInfoCenter::LoadDB(const std::string& path)123void RTAddInfoCenter::LoadDB(const std::string& path)
155{124{
156 RtsTrackDB rtsTrackDb;125 RtsTrackDB rtsTrackDb;
@@ -29,6 +29,8 @@
29#include "analysis/csrc/domain/entities/hal/include/ascend_obj.h"29#include "analysis/csrc/domain/entities/hal/include/ascend_obj.h"
30#include "analysis/csrc/infrastructure/utils/singleton.h"30#include "analysis/csrc/infrastructure/utils/singleton.h"
31 31 
32+class RTAddInfoCenterUTest;
33+ 
32namespace Analysis34namespace Analysis
33{35{
34namespace Domain36namespace Domain
@@ -48,16 +50,14 @@ static constexpr uint16_t CAPTURE_STATUS_START = 0;
48 50 
49static constexpr uint16_t CAPTURE_STATUS_END = 1;51static constexpr uint16_t CAPTURE_STATUS_END = 1;
50 52 
51-// 该类是runtime 算子补充信息数据单例类,支持二进制解析写入与 db 兜底。53+// 该类是runtime算子补充信息数据单例类
52-// 使用场景(单 PROF 目录、单线程、无并发):
53-// 1. 一次解析生命周期内先由 EventGrouper::Group() 调 Add 写入二进制解析结果;
54-// 2. Group 返回后再调 Load 做 db 兜底,随后只读(Get/GetAll/GetDumpList/...)。
55-// Add 与读取有严格先后、互不重叠,故不加锁;若未来出现多线程或跨 PROF 目录复用,
56-// 需先补并发保护或按目录清空单例状态。
57class RTAddInfoCenter : public Utils::Singleton<RTAddInfoCenter>54class RTAddInfoCenter : public Utils::Singleton<RTAddInfoCenter>
58{55{
56+ friend class ::RTAddInfoCenterUTest;
57+ 
59 public:58 public:
60 void Load(const std::string &path);59 void Load(const std::string &path);
60+ void SetCaptureStreamInfoData(const std::vector<CaptureStreamInfo> &data);
61 void Add(const RuntimeOpInfo &info);61 void Add(const RuntimeOpInfo &info);
62 RuntimeOpInfo Get(uint16_t deviceId, uint32_t streamId, uint32_t taskId);62 RuntimeOpInfo Get(uint16_t deviceId, uint32_t streamId, uint32_t taskId);
63 const std::unordered_map<std::string, RuntimeOpInfo> &GetAll() const;63 const std::unordered_map<std::string, RuntimeOpInfo> &GetAll() const;
@@ -68,7 +68,6 @@ class RTAddInfoCenter : public Utils::Singleton<RTAddInfoCenter>
68 68 
69 private:69 private:
70 void LoadDB(const std::string &path);70 void LoadDB(const std::string &path);
71- void LoadCaptureInfoDB(const std::string &path);
72 void BuildCaptureInfoTimeRange();71 void BuildCaptureInfoTimeRange();
73 std::unordered_map<std::string, RuntimeOpInfo> runtimeOpInfoData_;72 std::unordered_map<std::string, RuntimeOpInfo> runtimeOpInfoData_;
74 std::vector<RuntimeOpInfo> dumpList_;73 std::vector<RuntimeOpInfo> dumpList_;
@@ -0,0 +1,145 @@
1+/* -------------------------------------------------------------------------
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This file is part of the MindStudio project.
4+ *
5+ * MindStudio is licensed under Mulan PSL v2.
6+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
7+ * You may obtain a copy of Mulan PSL v2 at:
8+ *
9+ * http://license.coscl.org.cn/MulanPSL2
10+ *
11+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14+ * See the Mulan PSL v2 for more details.
15+ * -------------------------------------------------------------------------*/
16+ 
17+#include "analysis/csrc/domain/services/persistence/host/capture_stream_info_dumper.h"
18+ 
19+#include <algorithm>
20+#include <map>
21+#include <set>
22+#include <sstream>
23+#include <tuple>
24+ 
25+namespace Analysis
26+{
27+namespace Domain
28+{
29+namespace
30+{
31+std::string FormatModelIds(const std::set<uint32_t> &modelIds)
32+{
33+ std::ostringstream stream;
34+ stream << '{';
35+ bool first = true;
36+ for (auto modelId : modelIds)
37+ {
38+ if (!first)
39+ {
40+ stream << ',';
41+ }
42+ stream << modelId;
43+ first = false;
44+ }
45+ stream << '}';
46+ return stream.str();
47+}
48+} // namespace
49+ 
50+CaptureStreamInfoDumper::CaptureStreamInfoDumper(const std::string &hostPath)
51+ : BaseDumper<CaptureStreamInfoDumper>(hostPath, "CaptureStreamInfo")
52+{
53+ MAKE_SHARED0_NO_OPERATION(database_, StreamInfoDB);
54+}
55+ 
56+CaptureStreamInfoData CaptureStreamInfoDumper::FormatData(const std::vector<std::shared_ptr<ParserCompactInfo>> &input)
57+{
58+ auto sortedInput = input;
59+ std::stable_sort(sortedInput.begin(), sortedInput.end(),
60+ [](const std::shared_ptr<ParserCompactInfo> &left, const std::shared_ptr<ParserCompactInfo> &right)
61+ {
62+ if (!left)
63+ {
64+ return false;
65+ }
66+ if (!right)
67+ {
68+ return true;
69+ }
70+ return left->timeStamp < right->timeStamp;
71+ });
72+ 
73+ CaptureStreamInfoData output;
74+ if (!Utils::Reserve(output, sortedInput.size()))
75+ {
76+ ERROR("CaptureStreamInfoDumper: Reserve data failed.");
77+ return {};
78+ }
79+ std::set<uint32_t> startModelSet;
80+ std::set<uint32_t> endModelSet;
81+ using RecordKey = std::tuple<uint16_t, uint32_t, uint32_t, uint32_t, uint16_t, uint64_t>;
82+ std::set<RecordKey> seenRecords;
83+ std::map<std::pair<uint16_t, uint32_t>, uint32_t> batchIdMap;
84+ uint64_t repeatedNum = 0;
85+ 
86+ for (const auto &info : sortedInput)
87+ {
88+ if (!info)
89+ {
90+ continue;
91+ }
92+ const auto &record = info->data.captureStreamInfo;
93+ const auto deviceStreamKey = std::make_pair(record.deviceId, record.streamId);
94+ const uint32_t batchId = batchIdMap[deviceStreamKey];
95+ const RecordKey recordKey = std::make_tuple(record.deviceId, record.modelId, record.originalStreamId,
96+ record.streamId, record.captureStatus, info->timeStamp);
97+ if (seenRecords.find(recordKey) != seenRecords.end())
98+ {
99+ ++repeatedNum;
100+ continue;
101+ }
102+ if (record.captureStatus == 0)
103+ {
104+ startModelSet.insert(record.modelId);
105+ }
106+ if (record.captureStatus == 1 && !endModelSet.insert(record.modelId).second)
Mrtutu
MrtutuMrtutu8月8日

严重程度: 建议

问题: END 记录仅按 modelId 全局去重:if (record.captureStatus == 1 && !endModelSet.insert(record.modelId).second) continue;,某 modelId 的首条 END 之后,任何其它 (device, stream) 上同 modelId 的 END 都被静默丢弃。

原因: 被丢弃的 END 不会写入 output,下游 RTAddInfoCenter::BuildCaptureInfoTimeRange 对应 (deviceId, streamId, batchId) 的 TimeRangeInfo 末端将保持 kInf,使该流上任务 GetModelId 在时间窗内一直命中,可能错误归属 modelId。UT ShouldSortDeduplicateAssignBatchAndDropRepeatedModelEnd 编码了当前行为(跨 device 的 dev1 END 被丢),但这是否符合采集语义需确认:PR 描述称“与 Python 逐字段一致”,请确认 Python 对多 device/多 stream 同 modelId 的 END 也是全局只保留首条。

怎么改: 若需按会话区分,改为按 (deviceId, streamId, modelId)(deviceId, streamId) 去重:

// 例如按 device+stream 去重 END
if (record.captureStatus == 1 &&
    !endSeenStreams.insert(deviceStreamKey).second) {
    continue;
}

若 Python 确为全局 modelId 去重,建议在注释里写明该约定,便于后续维护者理解。

likedislike
107+ {
108+ continue;
109+ }
110+ batchIdMap[deviceStreamKey] = batchId + 1;
111+ seenRecords.insert(recordKey);
112+ output.emplace_back(record.deviceId, record.modelId, record.originalStreamId, record.streamId, batchId,
113+ record.captureStatus, info->timeStamp);
114+ }
115+ 
116+ if (startModelSet != endModelSet)
117+ {
118+ WARN("CaptureStreamInfoDumper: Capture start model ids are %, end model ids are %.",
119+ FormatModelIds(startModelSet), FormatModelIds(endModelSet));
120+ }
121+ if (repeatedNum > 0)
122+ {
123+ WARN("CaptureStreamInfoDumper: There are % duplicate records.", repeatedNum);
124+ }
125+ return output;
126+}
127+ 
128+CaptureStreamInfoDBData CaptureStreamInfoDumper::GenerateData(const CaptureStreamInfoData &input)
129+{
130+ CaptureStreamInfoDBData dbData;
131+ if (!Utils::Reserve(dbData, input.size()))
132+ {
133+ ERROR("CaptureStreamInfoDumper: Reserve DB data failed.");
134+ return {};
135+ }
136+ for (const auto &item : input)
137+ {
138+ dbData.emplace_back(item.deviceId, item.modelId, item.originalStreamId, item.streamId, item.batchId,
139+ item.captureStatus, item.timeStamp);
140+ }
141+ return dbData;
142+}
143+ 
144+} // namespace Domain
145+} // namespace Analysis
@@ -0,0 +1,71 @@
1+/* -------------------------------------------------------------------------
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This file is part of the MindStudio project.
4+ *
5+ * MindStudio is licensed under Mulan PSL v2.
6+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
7+ * You may obtain a copy of Mulan PSL v2 at:
8+ *
9+ * http://license.coscl.org.cn/MulanPSL2
10+ *
11+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14+ * See the Mulan PSL v2 for more details.
15+ * -------------------------------------------------------------------------*/
16+ 
17+#ifndef ANALYSIS_PERSISTENCE_HOST_CAPTURE_STREAM_INFO_DUMPER_H
18+#define ANALYSIS_PERSISTENCE_HOST_CAPTURE_STREAM_INFO_DUMPER_H
19+ 
20+#include <tuple>
21+#include <vector>
22+ 
23+#include "analysis/csrc/domain/services/persistence/host/base_dumper.h"
24+#include "analysis/csrc/infrastructure/utils/parser_struct.h"
25+ 
26+namespace Analysis
27+{
28+namespace Domain
29+{
30+ 
31+struct CaptureStreamInfoDataItem
32+{
33+ uint16_t deviceId = 0;
34+ uint32_t modelId = 0;
35+ uint32_t originalStreamId = 0;
36+ uint32_t streamId = 0;
37+ uint32_t batchId = 0;
38+ uint16_t captureStatus = 0;
39+ uint64_t timeStamp = 0;
40+ 
41+ CaptureStreamInfoDataItem() = default;
42+ CaptureStreamInfoDataItem(uint16_t deviceIdValue, uint32_t modelIdValue, uint32_t originalStreamIdValue,
43+ uint32_t streamIdValue, uint32_t batchIdValue, uint16_t captureStatusValue,
44+ uint64_t timeStampValue)
45+ : deviceId(deviceIdValue),
46+ modelId(modelIdValue),
47+ originalStreamId(originalStreamIdValue),
48+ streamId(streamIdValue),
49+ batchId(batchIdValue),
50+ captureStatus(captureStatusValue),
51+ timeStamp(timeStampValue)
52+ {
53+ }
54+};
55+ 
56+using CaptureStreamInfoData = std::vector<CaptureStreamInfoDataItem>;
57+using CaptureStreamInfoDBData =
58+ std::vector<std::tuple<uint16_t, uint32_t, uint32_t, uint32_t, uint32_t, uint16_t, uint64_t>>;
59+ 
60+class CaptureStreamInfoDumper : public BaseDumper<CaptureStreamInfoDumper>
61+{
62+ public:
63+ explicit CaptureStreamInfoDumper(const std::string &hostPath);
64+ CaptureStreamInfoData FormatData(const std::vector<std::shared_ptr<ParserCompactInfo>> &input);
65+ CaptureStreamInfoDBData GenerateData(const CaptureStreamInfoData &input);
66+};
67+ 
68+} // namespace Domain
69+} // namespace Analysis
70+ 
71+#endif // ANALYSIS_PERSISTENCE_HOST_CAPTURE_STREAM_INFO_DUMPER_H
@@ -0,0 +1,128 @@
1+/* -------------------------------------------------------------------------
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This file is part of the MindStudio project.
4+ *
5+ * MindStudio is licensed under Mulan PSL v2.
6+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
7+ * You may obtain a copy of Mulan PSL v2 at:
8+ *
9+ * http://license.coscl.org.cn/MulanPSL2
10+ *
11+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14+ * See the Mulan PSL v2 for more details.
15+ * -------------------------------------------------------------------------*/
16+ 
17+#include "analysis/csrc/domain/services/persistence/host/mc2_comm_info_dumper.h"
18+ 
19+#include <map>
20+#include <set>
21+#include <sstream>
22+ 
23+namespace Analysis
24+{
25+namespace Domain
26+{
27+using Host::Cann::MC2_COMM_STREAM_MAX_NUM;
28+using Host::Cann::MsprofMc2CommInfo;
29+ 
30+namespace
31+{
32+std::string JoinCommStreamIds(const MsprofMc2CommInfo &payload)
33+{
34+ if (payload.streamSize > MC2_COMM_STREAM_MAX_NUM)
35+ {
36+ return "";
37+ }
38+ std::ostringstream stream;
39+ for (uint32_t i = 0; i < payload.streamSize; ++i)
40+ {
41+ if (i > 0)
42+ {
43+ stream << ',';
44+ }
45+ stream << payload.commStreamIds[i];
46+ }
47+ return stream.str();
48+}
49+} // namespace
50+ 
51+Mc2CommInfoDumper::Mc2CommInfoDumper(const std::string &hostPath)
52+ : BaseDumper<Mc2CommInfoDumper>(hostPath, "Mc2CommInfo")
53+{
54+ MAKE_SHARED0_NO_OPERATION(database_, Mc2CommInfoDB);
55+}
56+ 
57+Mc2CommInfoData Mc2CommInfoDumper::GenerateData(const Mc2CommInfoInput &input)
58+{
59+ std::map<uint32_t, std::set<uint32_t>> captureStreamMap;
60+ for (const auto &capture : input.captureData)
61+ {
62+ captureStreamMap[capture.originalStreamId].insert(capture.streamId);
63+ }
64+ 
65+ size_t supplementSize = 0;
66+ for (const auto &info : input.mc2Data)
67+ {
68+ if (!info)
69+ {
70+ continue;
71+ }
72+ const auto payload = Utils::ReinterpretConvert<const MsprofMc2CommInfo *>(info->data);
73+ auto captureIt = captureStreamMap.find(payload->streamId);
74+ if (captureIt != captureStreamMap.end())
75+ {
76+ supplementSize += captureIt->second.size();
77+ }
78+ }
79+ 
80+ Mc2CommInfoData output;
81+ if (!Utils::Reserve(output, input.mc2Data.size() + supplementSize))
82+ {
83+ ERROR("Mc2CommInfoDumper: Reserve data failed.");
84+ return {};
85+ }
86+ uint64_t invalidStreamSizeNum = 0;
87+ for (const auto &info : input.mc2Data)
88+ {
89+ if (!info)
90+ {
91+ continue;
92+ }
93+ const auto payload = Utils::ReinterpretConvert<const MsprofMc2CommInfo *>(info->data);
94+ if (payload->streamSize > MC2_COMM_STREAM_MAX_NUM)
95+ {
96+ ++invalidStreamSizeNum;
97+ }
98+ output.emplace_back(std::to_string(payload->groupName), payload->rankSize, payload->rankId, payload->usrRankId,
99+ payload->streamId, JoinCommStreamIds(*payload));
100+ }
101+ for (const auto &info : input.mc2Data)
102+ {
103+ if (!info)
104+ {
105+ continue;
106+ }
107+ const auto payload = Utils::ReinterpretConvert<const MsprofMc2CommInfo *>(info->data);
108+ auto captureIt = captureStreamMap.find(payload->streamId);
109+ if (captureIt == captureStreamMap.end())
110+ {
111+ continue;
112+ }
113+ for (auto modelStreamId : captureIt->second)
Mrtutu
MrtutuMrtutu8月8日

严重程度: 提示

问题: 第二个循环对 streamSize > MC2_COMM_STREAM_MAX_NUM 的非法记录仍按 payload->streamId 查 capture 映射并追加一对多补充行,其 JoinCommStreamIds 返回空串,与“无通信小算子”不可区分。

原因: 已用 invalidStreamSizeNum 聚合 ERROR 日志,但补充行会把空串复制到多个 modelStreamId 行,下游无法区分“记录损坏”与“无 comm stream”。属低概率边界,影响有限,仅作提示。

怎么改: 在第二个循环对非法记录跳过补充生成:

const auto payload = ...;
if (payload->streamSize > MC2_COMM_STREAM_MAX_NUM) {
    continue;  // 非法记录不生成一对多补充行
}

或在补充行也写空时附加标记,便于下游区分损坏与空集。

likedislike
114+ {
115+ output.emplace_back(std::to_string(payload->groupName), payload->rankSize, payload->rankId,
116+ payload->usrRankId, modelStreamId, JoinCommStreamIds(*payload));
117+ }
118+ }
119+ if (invalidStreamSizeNum > 0)
120+ {
121+ ERROR("Mc2CommInfoDumper: % records have stream size greater than max stream size %.", invalidStreamSizeNum,
122+ MC2_COMM_STREAM_MAX_NUM);
123+ }
124+ return output;
125+}
126+ 
127+} // namespace Domain
128+} // namespace Analysis
@@ -0,0 +1,58 @@
1+/* -------------------------------------------------------------------------
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This file is part of the MindStudio project.
4+ *
5+ * MindStudio is licensed under Mulan PSL v2.
6+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
7+ * You may obtain a copy of Mulan PSL v2 at:
8+ *
9+ * http://license.coscl.org.cn/MulanPSL2
10+ *
11+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14+ * See the Mulan PSL v2 for more details.
15+ * -------------------------------------------------------------------------*/
16+ 
17+#ifndef ANALYSIS_PERSISTENCE_HOST_MC2_COMM_INFO_DUMPER_H
18+#define ANALYSIS_PERSISTENCE_HOST_MC2_COMM_INFO_DUMPER_H
19+ 
20+#include <tuple>
21+#include <vector>
22+ 
23+#include "analysis/csrc/domain/services/parser/host/cann/addition_info_parser.h"
24+#include "analysis/csrc/domain/services/persistence/host/base_dumper.h"
25+#include "analysis/csrc/domain/services/persistence/host/capture_stream_info_dumper.h"
26+ 
27+namespace Analysis
28+{
29+namespace Domain
30+{
31+ 
32+using Mc2CommInfoData = std::vector<std::tuple<std::string, uint32_t, uint32_t, uint32_t, uint32_t, std::string>>;
33+ 
34+struct Mc2CommInfoInput
35+{
36+ Mc2CommInfoInput(const std::vector<std::shared_ptr<ParserAdditionalInfo>> &mc2Data,
37+ const CaptureStreamInfoData &captureData)
38+ : mc2Data(mc2Data), captureData(captureData)
39+ {
40+ }
41+ 
42+ bool empty() const { return mc2Data.empty(); }
43+ 
44+ const std::vector<std::shared_ptr<ParserAdditionalInfo>> &mc2Data;
45+ const CaptureStreamInfoData &captureData;
46+};
47+ 
48+class Mc2CommInfoDumper : public BaseDumper<Mc2CommInfoDumper>
49+{
50+ public:
51+ explicit Mc2CommInfoDumper(const std::string &hostPath);
52+ Mc2CommInfoData GenerateData(const Mc2CommInfoInput &input);
53+};
54+ 
55+} // namespace Domain
56+} // namespace Analysis
57+ 
58+#endif // ANALYSIS_PERSISTENCE_HOST_MC2_COMM_INFO_DUMPER_H
@@ -45,6 +45,7 @@ enum class AdditionalInfoFormat : uint8_t
45 MEMORY_APPLICATION_TYPE,45 MEMORY_APPLICATION_TYPE,
46 MULTI_THREAD_TYPE,46 MULTI_THREAD_TYPE,
47 TASK_MEMORY_INFO_TYPE,47 TASK_MEMORY_INFO_TYPE,
48+ MC2_COMM_INFO_TYPE,
48};49};
49 50 
50enum class VariableInfoFormat : uint8_t51enum class VariableInfoFormat : uint8_t
@@ -379,13 +379,13 @@ TEST_F(ExportManagerUTest, ShouldBuildProcessorDependencyClosure)
379 TopoGraphBuilder builder;379 TopoGraphBuilder builder;
380 ASSERT_TRUE(builder.Build(context, {{TopoNodeStage::DATA_PROCESSING, PROCESSOR_NAME_OVERLAP_ANALYSIS}}, processes));380 ASSERT_TRUE(builder.Build(context, {{TopoNodeStage::DATA_PROCESSING, PROCESSOR_NAME_OVERLAP_ANALYSIS}}, processes));
381 381 
382- EXPECT_EQ(processes.size(), 6UL);382+ EXPECT_EQ(processes.size(), 5UL);
383 bool overlapFound = false;383 bool overlapFound = false;
384 for (const auto& process : processes) {384 for (const auto& process : processes) {
385 if (process.second.processName == PROCESSOR_NAME_OVERLAP_ANALYSIS) {385 if (process.second.processName == PROCESSOR_NAME_OVERLAP_ANALYSIS) {
386 overlapFound = true;386 overlapFound = true;
387- EXPECT_EQ(process.second.processDependence.size(), 4UL);387+ EXPECT_EQ(process.second.processDependence.size(), 3UL);
388- EXPECT_EQ(process.second.paramTypes.size(), 4UL);388+ EXPECT_EQ(process.second.paramTypes.size(), 3UL);
389 }389 }
390 EXPECT_NE(process.second.processName, "DataProcessorCollector");390 EXPECT_NE(process.second.processName, "DataProcessorCollector");
391 }391 }
@@ -16,6 +16,7 @@
16 16 
17#include "gtest/gtest.h"17#include "gtest/gtest.h"
18#include "mockcpp/mockcpp.hpp"18#include "mockcpp/mockcpp.hpp"
19+#include <algorithm>
19#include "analysis/csrc/domain/data_process/ai_task/mc2_comm_info_processor.h"20#include "analysis/csrc/domain/data_process/ai_task/mc2_comm_info_processor.h"
20#include "analysis/csrc/application/database/db_constant.h"21#include "analysis/csrc/application/database/db_constant.h"
21#include "analysis/csrc/infrastructure/utils/file.h"22#include "analysis/csrc/infrastructure/utils/file.h"
@@ -118,3 +119,22 @@ TEST_F(Mc2CommInfoProcessorUTest, TestRunShouldReturnFalseWhenSaveToDataInventor
118 EXPECT_FALSE(processor.Run(dataInventory, PROCESSOR_MC2_COMM_INFO));119 EXPECT_FALSE(processor.Run(dataInventory, PROCESSOR_MC2_COMM_INFO));
119 MOCKER_CPP(&DataProcessor::SaveToDataInventory<MC2CommInfoData>).reset();120 MOCKER_CPP(&DataProcessor::SaveToDataInventory<MC2CommInfoData>).reset();
120}121}
122+ 
123+TEST_F(Mc2CommInfoProcessorUTest, ShouldPreserveUint32KfcStreamId)
124+{
125+ using WideMc2Format =
126+ std::vector<std::tuple<std::string, uint32_t, uint32_t, uint32_t, uint32_t, std::string>>;
127+ WideMc2Format wideData{{"99", 2, 0, 0, 70003, "70004,70005"}};
128+ DBRunner dbRunner(File::PathJoin({PROF_PATH, HOST, SQLITE, DB_SUFFIX}));
129+ ASSERT_TRUE(dbRunner.InsertData(TABLE_NAME, wideData));
130+ 
131+ DataInventory dataInventory;
132+ auto processor = Mc2CommInfoProcessor(PROF_PATH);
133+ ASSERT_TRUE(processor.Run(dataInventory, PROCESSOR_MC2_COMM_INFO));
134+ auto result = dataInventory.GetPtr<std::vector<MC2CommInfoData>>();
135+ ASSERT_TRUE(result);
136+ auto record = std::find_if(result->begin(), result->end(), [](const MC2CommInfoData &item) {
137+ return item.aiCpuKfcStreamId == 70003;
138+ });
139+ EXPECT_NE(result->end(), record);
140+}
@@ -0,0 +1,61 @@
1+/* -------------------------------------------------------------------------
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This file is part of the MindStudio project.
4+ *
5+ * MindStudio is licensed under Mulan PSL v2.
6+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
7+ * You may obtain a copy of Mulan PSL v2 at:
8+ *
9+ * http://license.coscl.org.cn/MulanPSL2
10+ *
11+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13+ * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
14+ * See the Mulan PSL v2 for more details.
15+ * -------------------------------------------------------------------------*/
16+ 
17+#include "gtest/gtest.h"
18+ 
19+#include "analysis/csrc/application/database/db_constant.h"
20+#include "analysis/csrc/domain/data_process/ai_task/overlap_analysis_processor.h"
21+ 
22+using namespace Analysis::Application;
23+using namespace Analysis::Domain;
24+using namespace Analysis::Infra;
25+ 
26+TEST(OverlapAnalysisProcessorUTest, WideMc2StreamIdShouldNotFilterDifferentUint16TaskStream)
27+{
28+ AscendTaskData ascendTask;
29+ ascendTask.deviceId = 0;
30+ ascendTask.streamId = 4467;
31+ ascendTask.taskId = 1;
32+ ascendTask.contextId = 2;
33+ ascendTask.batchId = 3;
34+ ascendTask.timestamp = 10;
35+ ascendTask.duration = 10;
36+ 
37+ TaskInfoData computeTask;
38+ computeTask.deviceId = ascendTask.deviceId;
39+ computeTask.streamId = ascendTask.streamId;
40+ computeTask.taskId = ascendTask.taskId;
41+ computeTask.contextId = ascendTask.contextId;
42+ computeTask.batchId = ascendTask.batchId;
43+ computeTask.opName = "compute_op";
44+ 
45+ MC2CommInfoData mc2CommInfo;
46+ mc2CommInfo.aiCpuKfcStreamId = 70003;
47+ 
48+ DataInventory dataInventory;
49+ ASSERT_TRUE(dataInventory.Inject(std::make_shared<std::vector<AscendTaskData>>(1, ascendTask)));
50+ ASSERT_TRUE(dataInventory.Inject(std::make_shared<std::vector<TaskInfoData>>(1, computeTask)));
51+ ASSERT_TRUE(dataInventory.Inject(std::make_shared<std::vector<MC2CommInfoData>>(1, mc2CommInfo)));
52+ 
53+ OverlapAnalysisProcessor processor;
54+ ASSERT_TRUE(processor.Run(dataInventory, PROCESSOR_NAME_OVERLAP_ANALYSIS));
55+ auto result = dataInventory.GetPtr<std::vector<OverlapAnalysisData>>();
56+ ASSERT_TRUE(result);
57+ ASSERT_EQ(1UL, result->size());
58+ EXPECT_EQ(OverlapAnalysisType::COMPUTE, result->front().type);
59+ EXPECT_EQ(10UL, result->front().timestamp);
60+ EXPECT_EQ(10UL, result->front().duration);
61+}
@@ -148,4 +148,5 @@ TEST_F(ApiEventParserUTest, TestProduceDataShouldReturnEmptyWhenPopNullptr)
148 auto parser = std::make_shared<ApiEventParser>(File::PathJoin(std::vector<std::string>{DATA_DIR, "host", "data"}));148 auto parser = std::make_shared<ApiEventParser>(File::PathJoin(std::vector<std::string>{DATA_DIR, "host", "data"}));
149 auto data = parser->ParseData<ParserApi>();149 auto data = parser->ParseData<ParserApi>();
150 EXPECT_EQ(0, data.size());150 EXPECT_EQ(0, data.size());
151+ MOCKER_CPP(&ChunkGenerator::Pop).reset();
151}152}
@@ -0,0 +1,157 @@
1+/* -------------------------------------------------------------------------
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This file is part of the MindStudio project.
4+ *
5+ * MindStudio is licensed under Mulan PSL v2.
6+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
7+ * You may obtain a copy of Mulan PSL v2 at:
8+ *
9+ * http://license.coscl.org.cn/MulanPSL2
10+ *
11+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14+ * See the Mulan PSL v2 for more details.
15+ * -------------------------------------------------------------------------*/
16+ 
17+#include "gtest/gtest.h"
18+ 
19+#include <fstream>
20+ 
21+#include "analysis/csrc/domain/services/parser/host/cann/compact_info_parser.h"
22+#include "analysis/csrc/infrastructure/utils/file.h"
23+#include "test/msprof_cpp/analysis_ut/domain/services/test/fake_generator.h"
24+ 
25+using namespace Analysis::Domain::Host::Cann;
26+using namespace Analysis::Utils;
27+ 
28+namespace {
29+const std::string CAPTURE_ROOT = "./capture_stream_parser";
30+const std::string CAPTURE_DATA_PATH = File::PathJoin({CAPTURE_ROOT, "host", "data"});
31+}
32+ 
33+class CaptureStreamInfoParserUTest : public testing::Test {
34+protected:
35+ void SetUp() override
36+ {
37+ if (File::Exist(CAPTURE_ROOT)) {
38+ EXPECT_TRUE(File::RemoveDir(CAPTURE_ROOT, 0));
39+ }
40+ EXPECT_TRUE(File::CreateDir(CAPTURE_ROOT));
41+ EXPECT_TRUE(File::CreateDir(File::PathJoin({CAPTURE_ROOT, "host"})));
42+ EXPECT_TRUE(File::CreateDir(CAPTURE_DATA_PATH));
43+ }
44+ 
45+ void TearDown() override
46+ {
47+ if (File::Exist(CAPTURE_ROOT)) {
48+ EXPECT_TRUE(File::RemoveDir(CAPTURE_ROOT, 0));
49+ }
50+ }
51+};
52+ 
53+TEST_F(CaptureStreamInfoParserUTest, ShouldNotLookupV1CaptureSlice)
54+{
55+ MsprofCompactInfo info{};
56+ info.timeStamp = 101;
57+ info.data.captureStreamInfo.deviceId = 2;
58+ info.data.captureStreamInfo.modelId = 3;
59+ info.data.captureStreamInfo.originalStreamId = 4;
60+ info.data.captureStreamInfo.modelStreamId = 5;
61+ info.data.captureStreamInfo.captureStatus = 0;
62+ std::vector<MsprofCompactInfo> input{info};
63+ ASSERT_TRUE(WriteBin(input, CAPTURE_DATA_PATH, "unaging.compact.capture_stream_info.slice_0"));
64+ 
65+ CaptureStreamInfoParser parser(CAPTURE_DATA_PATH);
66+ auto output = parser.ParseData<ParserCompactInfo>();
67+ ASSERT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR));
68+ EXPECT_TRUE(output.empty());
69+}
70+ 
71+TEST_F(CaptureStreamInfoParserUTest, ShouldNotLookupCaptureSlicesWhenBothVersionsExist)
72+{
73+ MsprofCompactInfo v1{};
74+ v1.timeStamp = 100;
75+ v1.data.captureStreamInfo.modelId = 11;
76+ std::vector<MsprofCompactInfo> v1Input{v1};
77+ ASSERT_TRUE(WriteBin(v1Input, CAPTURE_DATA_PATH, "aging.compact.capture_stream_info.slice_0"));
78+ 
79+ MsprofCompactInfo v2{};
80+ v2.timeStamp = 200;
81+ v2.data.captureStreamInfo.modelId = 22;
82+ std::vector<MsprofCompactInfo> v2Input{v2};
83+ ASSERT_TRUE(WriteBin(v2Input, CAPTURE_DATA_PATH, "unaging.compact.capture_stream_info_v2.slice_0"));
84+ 
85+ CaptureStreamInfoParser parser(CAPTURE_DATA_PATH);
86+ auto output = parser.ParseData<ParserCompactInfo>();
87+ ASSERT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR));
88+ EXPECT_TRUE(output.empty());
89+}
90+ 
91+TEST_F(CaptureStreamInfoParserUTest, ShouldNotLookupV2CaptureSlice)
92+{
93+ MsprofCompactInfo info{};
94+ info.timeStamp = 202;
95+ info.data.captureStreamInfoV2.deviceId = 6;
96+ info.data.captureStreamInfoV2.modelId = 70001;
97+ info.data.captureStreamInfoV2.originalStreamId = 70002;
98+ info.data.captureStreamInfoV2.streamId = 70003;
99+ info.data.captureStreamInfoV2.captureStatus = 1;
100+ std::vector<MsprofCompactInfo> input{info};
101+ ASSERT_TRUE(WriteBin(input, CAPTURE_DATA_PATH, "unaging.compact.capture_stream_info_v2.slice_0"));
102+ 
103+ CaptureStreamInfoParser parser(CAPTURE_DATA_PATH);
104+ auto output = parser.ParseData<ParserCompactInfo>();
105+ ASSERT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR));
106+ EXPECT_TRUE(output.empty());
107+}
108+ 
109+TEST_F(CaptureStreamInfoParserUTest, ShouldNotLookupV2WhenV1MarkerExists)
110+{
111+ std::ofstream marker(File::PathJoin({CAPTURE_DATA_PATH,
112+ "unaging.compact.capture_stream_info.slice_0.done"}));
113+ ASSERT_TRUE(marker.good());
114+ marker.close();
115+ 
116+ MsprofCompactInfo info{};
117+ info.data.captureStreamInfoV2.modelId = 70001;
118+ std::vector<MsprofCompactInfo> input{info};
119+ ASSERT_TRUE(WriteBin(input, CAPTURE_DATA_PATH, "unaging.compact.capture_stream_info_v2.slice_0"));
120+ 
121+ CaptureStreamInfoParser parser(CAPTURE_DATA_PATH);
122+ auto output = parser.ParseData<ParserCompactInfo>();
123+ ASSERT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR));
124+ EXPECT_TRUE(output.empty());
125+}
126+ 
127+TEST_F(CaptureStreamInfoParserUTest, ShouldSkipRecordWithInvalidMagic)
128+{
129+ MsprofCompactInfo info{};
130+ info.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM + 1;
131+ std::vector<MsprofCompactInfo> input{info};
132+ ASSERT_TRUE(WriteBin(input, CAPTURE_DATA_PATH, "unaging.compact.capture_stream_info.slice_0"));
133+ 
134+ CaptureStreamInfoParser parser(CAPTURE_DATA_PATH);
135+ auto output = parser.ParseData<ParserCompactInfo>();
136+ EXPECT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR));
137+ EXPECT_TRUE(output.empty());
138+}
139+ 
140+TEST_F(CaptureStreamInfoParserUTest, ShouldReturnTrueAndEmptyWhenNoFileExists)
141+{
142+ CaptureStreamInfoParser parser(CAPTURE_DATA_PATH);
143+ auto output = parser.ParseData<ParserCompactInfo>();
144+ EXPECT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR));
145+ EXPECT_TRUE(output.empty());
146+}
147+ 
148+TEST_F(CaptureStreamInfoParserUTest, ShouldReturnTrueAndEmptyWhenTruncatedRecordExists)
149+{
150+ std::vector<uint8_t> input(sizeof(MsprofCompactInfo) - 1, 0);
151+ ASSERT_TRUE(WriteBin(input, CAPTURE_DATA_PATH, "unaging.compact.capture_stream_info.slice_0"));
152+ 
153+ CaptureStreamInfoParser parser(CAPTURE_DATA_PATH);
154+ auto output = parser.ParseData<ParserCompactInfo>();
155+ EXPECT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR));
156+ EXPECT_TRUE(output.empty());
157+}
@@ -75,12 +75,13 @@ protected:
75 std::vector<MsprofCompactInfo> agingTraces;75 std::vector<MsprofCompactInfo> agingTraces;
76 std::vector<MsprofCompactInfo> unAgingTraces;76 std::vector<MsprofCompactInfo> unAgingTraces;
77 for (uint32_t i = 0; i < DATA_NUM; ++i) {77 for (uint32_t i = 0; i < DATA_NUM; ++i) {
78- MsprofCompactInfo info;78+ MsprofCompactInfo info{};
79 info.level = level;79 info.level = level;
80 info.type = static_cast<uint32_t>(type);80 info.type = static_cast<uint32_t>(type);
81 info.threadId = i;81 info.threadId = i;
82 info.dataLen = dataLen;82 info.dataLen = dataLen;
83 info.timeStamp = DATA_NUM + i;83 info.timeStamp = DATA_NUM + i;
84+ info.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM;
84 if (i >= DATA_NUM - invalidDataNum) {85 if (i >= DATA_NUM - invalidDataNum) {
85 info.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM + 1;86 info.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM + 1;
86 }87 }
@@ -225,6 +226,7 @@ TEST_F(CompactInfoParserUTest, TestTaskTrackParserProduceDataShouldReturn8Compac
225 Check(compactInfo, EventType::EVENT_TYPE_TASK_TRACK, MSPROF_REPORT_NODE_LEVEL,226 Check(compactInfo, EventType::EVENT_TYPE_TASK_TRACK, MSPROF_REPORT_NODE_LEVEL,
226 DATA_NUM - flipTaskNum - maintenanceTaskNum);227 DATA_NUM - flipTaskNum - maintenanceTaskNum);
227 EXPECT_EQ(flipTaskNum, flipTask.size());228 EXPECT_EQ(flipTaskNum, flipTask.size());
229+ MOCKER_CPP(&Context::IsAllExport).reset();
228}230}
229 231 
230TEST_F(CompactInfoParserUTest, TestTaskTrackParserProduceDataShouldReturnEmptyWhenReserveFailed)232TEST_F(CompactInfoParserUTest, TestTaskTrackParserProduceDataShouldReturnEmptyWhenReserveFailed)
@@ -349,12 +351,13 @@ static void GenDpuTrackData(uint16_t dataNum = DATA_NUM, uint16_t invalidDataNum
349 std::vector<MsprofCompactInfo> agingTraces;351 std::vector<MsprofCompactInfo> agingTraces;
350 std::vector<MsprofCompactInfo> unAgingTraces;352 std::vector<MsprofCompactInfo> unAgingTraces;
351 for (uint32_t i = 0; i < dataNum; ++i) {353 for (uint32_t i = 0; i < dataNum; ++i) {
352- MsprofCompactInfo info;354+ MsprofCompactInfo info{};
353 info.level = level;355 info.level = level;
354 info.type = static_cast<uint32_t>(EventType::EVENT_TYPE_DPU_TASK_TRACK);356 info.type = static_cast<uint32_t>(EventType::EVENT_TYPE_DPU_TASK_TRACK);
355 info.threadId = i;357 info.threadId = i;
356 info.dataLen = dataLen;358 info.dataLen = dataLen;
357 info.timeStamp = dataNum + i;359 info.timeStamp = dataNum + i;
360+ info.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM;
358 if (i >= dataNum - invalidDataNum) {361 if (i >= dataNum - invalidDataNum) {
359 info.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM + 1;362 info.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM + 1;
360 }363 }
@@ -1257,4 +1257,4 @@ TEST_F(ContextUTest, TestGetCannVersionShouldReturnEmptyWhenCannVersionInfoIsInv
1257 auto cannVersion = Context::GetInstance().GetCannVersion(HOST_ID, File::PathJoin(std::vector<std::string>{CONTEXT_DIR, TEST_DIR}));1257 auto cannVersion = Context::GetInstance().GetCannVersion(HOST_ID, File::PathJoin(std::vector<std::string>{CONTEXT_DIR, TEST_DIR}));
1258 EXPECT_EQ(cannVersion.size(), 0);1258 EXPECT_EQ(cannVersion.size(), 0);
1259 MOCKER_CPP(&Context::CheckInfoValueIsValid).reset();1259 MOCKER_CPP(&Context::CheckInfoValueIsValid).reset();
1260-}1260+}
@@ -31,6 +31,7 @@
31#include "analysis/csrc/infrastructure/utils/prof_struct.h"31#include "analysis/csrc/infrastructure/utils/prof_struct.h"
32#include "test/msprof_cpp/analysis_ut/fake/fake_trace_generator.h"32#include "test/msprof_cpp/analysis_ut/fake/fake_trace_generator.h"
33#include "analysis/csrc/domain/services/parser/host/cann/type_data.h"33#include "analysis/csrc/domain/services/parser/host/cann/type_data.h"
34+#include "test/msprof_cpp/analysis_ut/domain/services/test/fake_generator.h"
34 35 
35using namespace Analysis::Utils;36using namespace Analysis::Utils;
36using namespace Analysis::Domain::Host::Cann;37using namespace Analysis::Domain::Host::Cann;
@@ -324,6 +325,43 @@ TEST_F(EventGrouperUTest, TestGroupShouldReturnEmptyWhenDataDirEmpty)
324 EXPECT_EQ(true, File::RemoveDir(fakeDataDir, 0));325 EXPECT_EQ(true, File::RemoveDir(fakeDataDir, 0));
325}326}
326 327 
328+TEST_F(EventGrouperUTest, ShouldSkipCaptureAndMc2InGroupWhenCppParserDisabled)
329+{
330+ const std::string fakeDataDir = "./fakeCaptureData";
331+ const std::string hostDataDir = fakeDataDir + "/host/data";
332+ File::RemoveDir(fakeDataDir, 0);
333+ ASSERT_TRUE(File::CreateDir(fakeDataDir));
334+ ASSERT_TRUE(File::CreateDir(fakeDataDir + "/host"));
335+ ASSERT_TRUE(File::CreateDir(hostDataDir));
336+ 
337+ MsprofCompactInfo capture{};
338+ capture.threadId = 9;
339+ capture.timeStamp = 123;
340+ capture.data.captureStreamInfo.deviceId = 1;
341+ capture.data.captureStreamInfo.modelId = 7;
342+ capture.data.captureStreamInfo.originalStreamId = 52;
343+ capture.data.captureStreamInfo.modelStreamId = 70;
344+ capture.data.captureStreamInfo.captureStatus = 0;
345+ std::vector<MsprofCompactInfo> captureRecords{capture};
346+ ASSERT_TRUE(WriteBin(captureRecords, hostDataDir,
347+ "unaging.compact.capture_stream_info.slice_0"));
348+ 
349+ MsprofAdditionalInfo mc2{};
350+ auto payload = ReinterpretConvert<MsprofMc2CommInfo *>(mc2.data);
351+ payload->groupName = 99;
352+ payload->streamId = 52;
353+ std::vector<MsprofAdditionalInfo> mc2Records{mc2};
354+ ASSERT_TRUE(WriteBin(mc2Records, hostDataDir, "unaging.additional.mc2_comm_info.slice_0"));
355+ 
356+ EventGrouper grouper(hostDataDir);
357+ ASSERT_TRUE(grouper.Group());
358+ EXPECT_TRUE(grouper.GetCaptureStreamInfoData().empty());
359+ EXPECT_TRUE(grouper.GetMc2CommInfoData().empty());
360+ EXPECT_TRUE(grouper.GetThreadIdSet().empty());
361+ EXPECT_FALSE(grouper.GetGroupEvents().Find(9));
362+ EXPECT_TRUE(File::RemoveDir(fakeDataDir, 0));
363+}
364+ 
327// 测试文件夹中只有Api类型数据365// 测试文件夹中只有Api类型数据
328TEST_F(EventGrouperUTest, TestGroupShouldReturnEmptyWhenDataOnlyHasApiBin)366TEST_F(EventGrouperUTest, TestGroupShouldReturnEmptyWhenDataOnlyHasApiBin)
329{367{
@@ -0,0 +1,149 @@
1+/* -------------------------------------------------------------------------
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This file is part of the MindStudio project.
4+ *
5+ * MindStudio is licensed under Mulan PSL v2.
6+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
7+ * You may obtain a copy of Mulan PSL v2 at:
8+ *
9+ * http://license.coscl.org.cn/MulanPSL2
10+ *
11+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14+ * See the Mulan PSL v2 for more details.
15+ * -------------------------------------------------------------------------*/
16+ 
17+#include "gtest/gtest.h"
18+ 
19+#include "analysis/csrc/domain/services/parser/host/cann/addition_info_parser.h"
20+#include "analysis/csrc/infrastructure/utils/file.h"
21+#include "analysis/csrc/infrastructure/utils/utils.h"
22+#include "test/msprof_cpp/analysis_ut/domain/services/test/fake_generator.h"
23+ 
24+using namespace Analysis::Domain::Host::Cann;
25+using namespace Analysis::Utils;
26+ 
27+namespace {
28+const std::string MC2_ROOT = "./mc2_comm_parser";
29+const std::string MC2_DATA_PATH = File::PathJoin({MC2_ROOT, "host", "data"});
30+ 
31+MsprofAdditionalInfo MakeMc2Info(uint64_t groupName, uint32_t streamSize)
32+{
33+ MsprofAdditionalInfo info{};
34+ auto payload = ReinterpretConvert<MsprofMc2CommInfo *>(info.data);
35+ payload->groupName = groupName;
36+ payload->rankSize = 8;
37+ payload->rankId = 1;
38+ payload->usrRankId = 2;
39+ payload->streamId = 52;
40+ payload->streamSize = streamSize;
41+ for (uint32_t i = 0; i < MC2_COMM_STREAM_MAX_NUM; ++i) {
42+ payload->commStreamIds[i] = 100 + i;
43+ }
44+ return info;
45+}
46+}
47+ 
48+class Mc2CommInfoParserUTest : public testing::Test {
49+protected:
50+ void SetUp() override
51+ {
52+ if (File::Exist(MC2_ROOT)) {
53+ EXPECT_TRUE(File::RemoveDir(MC2_ROOT, 0));
54+ }
55+ EXPECT_TRUE(File::CreateDir(MC2_ROOT));
56+ EXPECT_TRUE(File::CreateDir(File::PathJoin({MC2_ROOT, "host"})));
57+ EXPECT_TRUE(File::CreateDir(MC2_DATA_PATH));
58+ }
59+ 
60+ void TearDown() override
61+ {
62+ if (File::Exist(MC2_ROOT)) {
63+ EXPECT_TRUE(File::RemoveDir(MC2_ROOT, 0));
64+ }
65+ }
66+};
67+ 
68+TEST_F(Mc2CommInfoParserUTest, ShouldNotLookupMc2CommInfoSlice)
69+{
70+ std::vector<MsprofAdditionalInfo> input{MakeMc2Info(7466789422691968299ULL, 8)};
71+ ASSERT_TRUE(WriteBin(input, MC2_DATA_PATH, "unaging.additional.mc2_comm_info.slice_0"));
72+ 
73+ Mc2CommInfoParser parser(MC2_DATA_PATH);
74+ auto output = parser.ParseData<ParserAdditionalInfo>();
75+ ASSERT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR));
76+ EXPECT_TRUE(output.empty());
77+}
78+ 
79+TEST_F(Mc2CommInfoParserUTest, ShouldNotLookupAgingMc2CommInfoSlice)
80+{
81+ std::vector<MsprofAdditionalInfo> input{MakeMc2Info(1, 9)};
82+ ASSERT_TRUE(WriteBin(input, MC2_DATA_PATH, "aging.additional.mc2_comm_info.slice_0"));
83+ 
84+ Mc2CommInfoParser parser(MC2_DATA_PATH);
85+ auto output = parser.ParseData<ParserAdditionalInfo>();
86+ ASSERT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR));
87+ EXPECT_TRUE(output.empty());
88+}
89+ 
90+TEST_F(Mc2CommInfoParserUTest, ShouldNotLookupMc2SlicesInAnyOrder)
91+{
92+ std::vector<MsprofAdditionalInfo> unaging0{MakeMc2Info(10, 0)};
93+ std::vector<MsprofAdditionalInfo> unaging1{MakeMc2Info(11, 1)};
94+ std::vector<MsprofAdditionalInfo> aging{MakeMc2Info(20, 1)};
95+ ASSERT_TRUE(WriteBin(unaging1, MC2_DATA_PATH, "unaging.additional.mc2_comm_info.slice_1"));
96+ ASSERT_TRUE(WriteBin(unaging0, MC2_DATA_PATH, "unaging.additional.mc2_comm_info.slice_0"));
97+ ASSERT_TRUE(WriteBin(aging, MC2_DATA_PATH, "aging.additional.mc2_comm_info.slice_0"));
98+ 
99+ Mc2CommInfoParser parser(MC2_DATA_PATH);
100+ auto output = parser.ParseData<ParserAdditionalInfo>();
101+ ASSERT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR));
102+ EXPECT_TRUE(output.empty());
103+}
104+ 
105+TEST_F(Mc2CommInfoParserUTest, ShouldSkipRecordWithInvalidMagic)
106+{
107+ auto info = MakeMc2Info(1, 1);
108+ info.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM + 1;
109+ std::vector<MsprofAdditionalInfo> input{info};
110+ ASSERT_TRUE(WriteBin(input, MC2_DATA_PATH, "unaging.additional.mc2_comm_info.slice_0"));
111+ 
112+ Mc2CommInfoParser parser(MC2_DATA_PATH);
113+ auto output = parser.ParseData<ParserAdditionalInfo>();
114+ EXPECT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR));
115+ EXPECT_TRUE(output.empty());
116+}
117+ 
118+TEST_F(Mc2CommInfoParserUTest, ShouldNotLookupMixedValidAndInvalidMagicRecords)
119+{
120+ auto invalid = MakeMc2Info(1, 1);
121+ invalid.magicNumber = MSPROF_DATA_HEAD_MAGIC_NUM + 1;
122+ auto valid = MakeMc2Info(2, 1);
123+ std::vector<MsprofAdditionalInfo> input{invalid, valid};
124+ ASSERT_TRUE(WriteBin(input, MC2_DATA_PATH, "unaging.additional.mc2_comm_info.slice_0"));
125+ 
126+ Mc2CommInfoParser parser(MC2_DATA_PATH);
127+ auto output = parser.ParseData<ParserAdditionalInfo>();
128+ ASSERT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR));
129+ EXPECT_TRUE(output.empty());
130+}
131+ 
132+TEST_F(Mc2CommInfoParserUTest, ShouldReturnTrueAndEmptyWhenNoFileExists)
133+{
134+ Mc2CommInfoParser parser(MC2_DATA_PATH);
135+ auto output = parser.ParseData<ParserAdditionalInfo>();
136+ EXPECT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR));
137+ EXPECT_TRUE(output.empty());
138+}
139+ 
140+TEST_F(Mc2CommInfoParserUTest, ShouldReturnTrueAndEmptyWhenTruncatedRecordExists)
141+{
142+ std::vector<uint8_t> input(sizeof(MsprofAdditionalInfo) - 1, 0);
143+ ASSERT_TRUE(WriteBin(input, MC2_DATA_PATH, "unaging.additional.mc2_comm_info.slice_0"));
144+ 
145+ Mc2CommInfoParser parser(MC2_DATA_PATH);
146+ auto output = parser.ParseData<ParserAdditionalInfo>();
147+ EXPECT_TRUE((parser.GetStatus() != Analysis::Domain::ParserStatus::ERROR));
148+ EXPECT_TRUE(output.empty());
149+}
@@ -18,6 +18,8 @@
18#include "gtest/gtest.h"18#include "gtest/gtest.h"
19#include "mockcpp/mockcpp.hpp"19#include "mockcpp/mockcpp.hpp"
20 20 
21+#include <algorithm>
22+ 
21#include "analysis/csrc/domain/services/parser/host/cann/rt_add_info_center.h"23#include "analysis/csrc/domain/services/parser/host/cann/rt_add_info_center.h"
22#include "analysis/csrc/domain/services/parser/host/cann/hash_data.h"24#include "analysis/csrc/domain/services/parser/host/cann/hash_data.h"
23#include "analysis/csrc/infrastructure/utils/file.h"25#include "analysis/csrc/infrastructure/utils/file.h"
@@ -50,6 +52,11 @@ const captureStreamDataFormat CAPTURE_STREAM_DATA{
50 {0, 49, 44, 11, 0, 0, 27063567062060},52 {0, 49, 44, 11, 0, 0, 27063567062060},
51 {0, 49, 43, 12, 0, 0, 27063567062060}53 {0, 49, 43, 12, 0, 0, 27063567062060}
52};54};
55+const std::vector<CaptureStreamInfo> DIRECT_CAPTURE_STREAM_DATA{
56+ {49, 27063567062060, 95, 3, 0, 0, 0},
57+ {49, 27063567062060, 11, 44, 0, 0, 0},
58+ {49, 27063567062060, 12, 43, 0, 0, 0},
59+};
53 60 
54const std::string PROF_PATH = "./PROF_XXX";61const std::string PROF_PATH = "./PROF_XXX";
55const std::string HOST_PATH = File::PathJoin({PROF_PATH, "host"});62const std::string HOST_PATH = File::PathJoin({PROF_PATH, "host"});
@@ -89,6 +96,7 @@ protected:
89 96 
90 void SetUp() override97 void SetUp() override
91 {98 {
99+ RTAddInfoCenter::GetInstance().SetCaptureStreamInfoData({});
92 RtsTrackDB rtsTrackDB;100 RtsTrackDB rtsTrackDB;
93 std::string opInfo = "RuntimeOpInfo";101 std::string opInfo = "RuntimeOpInfo";
94 CreateDB(rtsTrackDB, opInfo, RTS_TRACK_DB_PATH, DATA);102 CreateDB(rtsTrackDB, opInfo, RTS_TRACK_DB_PATH, DATA);
@@ -144,6 +152,7 @@ void CheckRuntimeOpInfo(RuntimeOpInfo info, RuntimeOpInfo expect)
144TEST_F(RTAddInfoCenterUTest, LoadThenTestGetWhenLoadSuccess)152TEST_F(RTAddInfoCenterUTest, LoadThenTestGetWhenLoadSuccess)
145{153{
146 HashData::GetInstance().Load(DATA_PATH);154 HashData::GetInstance().Load(DATA_PATH);
155+ RTAddInfoCenter::GetInstance().SetCaptureStreamInfoData(DIRECT_CAPTURE_STREAM_DATA);
147 RTAddInfoCenter::GetInstance().Load(SQLITE_PATH);156 RTAddInfoCenter::GetInstance().Load(SQLITE_PATH);
148 // info 1 2 是有效值,3是空157 // info 1 2 是有效值,3是空
149 auto info1 = RTAddInfoCenter::GetInstance().Get(5, 30, 10);158 auto info1 = RTAddInfoCenter::GetInstance().Get(5, 30, 10);
@@ -167,6 +176,27 @@ TEST_F(RTAddInfoCenterUTest, LoadThenTestGetWhenLoadSuccess)
167 EXPECT_EQ(modelId4, 4294967295u);176 EXPECT_EQ(modelId4, 4294967295u);
168}177}
169 178 
179+TEST_F(RTAddInfoCenterUTest, LoadShouldNotReadCaptureStreamInfoFromDb)
180+{
181+ RTAddInfoCenter::GetInstance().SetCaptureStreamInfoData({});
182+ RTAddInfoCenter::GetInstance().Load(SQLITE_PATH);
183+ EXPECT_EQ(DEFAULT_MODEL_ID, RTAddInfoCenter::GetInstance().GetModelId(0, 95, 0, 27063567062061));
184+}
185+ 
186+TEST_F(RTAddInfoCenterUTest, SetCaptureInfoShouldPreserveV2OriginalStreamIdWidth)
187+{
188+ std::vector<CaptureStreamInfo> wideData{{88, 27063567062070, 70003, 70002, 0, 0, 0}};
189+ RTAddInfoCenter::GetInstance().SetCaptureStreamInfoData(wideData);
190+ RTAddInfoCenter::GetInstance().Load(SQLITE_PATH);
191+ const auto &captureData = RTAddInfoCenter::GetInstance().captureStreamInfoData_;
192+ auto record = std::find_if(captureData.begin(), captureData.end(), [](const CaptureStreamInfo &item) {
193+ return item.modelId == 88;
194+ });
195+ ASSERT_NE(captureData.end(), record);
196+ EXPECT_EQ(70002u, record->originalStreamId);
197+ EXPECT_EQ(70003u, record->streamId);
198+}
199+ 
170TEST_F(RTAddInfoCenterUTest, AddShouldMarkLoadedFromBinaryAndKeepDumpList)200TEST_F(RTAddInfoCenterUTest, AddShouldMarkLoadedFromBinaryAndKeepDumpList)
171{201{
172 RuntimeOpInfo info1{9, 101, 1, 0, 0, 0, 21, 7, "AI_CORE", "Add", "op_a", "N/A", "0",202 RuntimeOpInfo info1{9, 101, 1, 0, 0, 0, 21, 7, "AI_CORE", "Add", "op_a", "N/A", "0",
@@ -0,0 +1,205 @@
1+/* -------------------------------------------------------------------------
2+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3+ * This file is part of the MindStudio project.
4+ *
5+ * MindStudio is licensed under Mulan PSL v2.
6+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
7+ * You may obtain a copy of Mulan PSL v2 at:
8+ *
9+ * http://license.coscl.org.cn/MulanPSL2
10+ *
11+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
12+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
13+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
14+ * See the Mulan PSL v2 for more details.
15+ * -------------------------------------------------------------------------*/
16+ 
17+#include "gtest/gtest.h"
18+#include "mockcpp/mockcpp.hpp"
19+ 
20+#include "analysis/csrc/domain/services/persistence/host/capture_stream_info_dumper.h"
21+#include "analysis/csrc/domain/services/persistence/host/mc2_comm_info_dumper.h"
22+#include "analysis/csrc/infrastructure/dfx/log.h"
23+ 
24+using namespace Analysis::Domain;
25+using namespace Analysis::Domain::Host::Cann;
26+using namespace mockcpp;
27+ 
28+namespace {
29+std::string capturedLogMessage;
30+ 
31+std::shared_ptr<ParserCompactInfo> MakeCaptureInfo(uint16_t deviceId, uint32_t modelId, uint32_t originalStreamId,
32+ uint32_t streamId, uint16_t captureStatus, uint64_t timeStamp)
33+{
34+ auto info = std::make_shared<ParserCompactInfo>();
35+ info->timeStamp = timeStamp;
36+ info->data.captureStreamInfo.deviceId = deviceId;
37+ info->data.captureStreamInfo.modelId = modelId;
38+ info->data.captureStreamInfo.originalStreamId = originalStreamId;
39+ info->data.captureStreamInfo.streamId = streamId;
40+ info->data.captureStreamInfo.captureStatus = captureStatus;
41+ return info;
42+}
43+ 
44+std::shared_ptr<ParserAdditionalInfo> MakeMc2Info(uint64_t groupName, uint32_t rankSize, uint32_t rankId,
45+ uint32_t usrRankId, uint32_t streamId, uint32_t streamSize)
46+{
47+ auto info = std::make_shared<ParserAdditionalInfo>();
48+ auto payload = Analysis::Utils::ReinterpretConvert<MsprofMc2CommInfo *>(info->data);
49+ payload->groupName = groupName;
50+ payload->rankSize = rankSize;
51+ payload->rankId = rankId;
52+ payload->usrRankId = usrRankId;
53+ payload->streamId = streamId;
54+ payload->streamSize = streamSize;
55+ for (uint32_t i = 0; i < MC2_COMM_STREAM_MAX_NUM; ++i) {
56+ payload->commStreamIds[i] = 100 + i;
57+ }
58+ return info;
59+}
60+ 
61+void CaptureLogMessage(Analysis::Log *, const std::string &message, const std::string &, const std::string &,
62+ const uint32_t &)
63+{
64+ capturedLogMessage = message;
65+}
66+} // namespace
67+ 
68+TEST(CaptureStreamInfoDumperUTest, ShouldMatchPythonGlobalModelEndDeduplication)
69+{
70+ std::vector<std::shared_ptr<ParserCompactInfo>> input{
71+ MakeCaptureInfo(0, 1, 10, 20, 0, 20),
72+ MakeCaptureInfo(0, 1, 10, 20, 0, 10),
73+ MakeCaptureInfo(0, 1, 10, 20, 0, 10),
74+ MakeCaptureInfo(0, 1, UINT32_MAX, UINT32_MAX, 1, 30),
75+ MakeCaptureInfo(1, 1, UINT32_MAX, UINT32_MAX, 1, 40),
76+ };
77+ 
78+ CaptureStreamInfoDumper dumper(".");
79+ auto output = dumper.FormatData(input);
80+ ASSERT_EQ(3ul, output.size());
81+ 
82+ EXPECT_EQ(0, output[0].deviceId);
83+ EXPECT_EQ(1u, output[0].modelId);
84+ EXPECT_EQ(10u, output[0].originalStreamId);
85+ EXPECT_EQ(20u, output[0].streamId);
86+ EXPECT_EQ(0u, output[0].batchId);
87+ EXPECT_EQ(0, output[0].captureStatus);
88+ EXPECT_EQ(10ul, output[0].timeStamp);
89+ 
90+ EXPECT_EQ(1u, output[1].batchId);
91+ EXPECT_EQ(20ul, output[1].timeStamp);
92+ EXPECT_EQ(1, output[2].captureStatus);
93+ EXPECT_EQ(30ul, output[2].timeStamp);
94+}
95+ 
96+TEST(CaptureStreamInfoDumperUTest, ShouldKeepSeparateBatchCountersForEachDeviceAndStream)
97+{
98+ std::vector<std::shared_ptr<ParserCompactInfo>> input{
99+ MakeCaptureInfo(0, 1, 10, 20, 0, 10),
100+ MakeCaptureInfo(1, 1, 10, 20, 0, 11),
101+ MakeCaptureInfo(0, 1, 11, 21, 0, 12),
102+ MakeCaptureInfo(0, 1, 10, 20, 0, 13),
103+ };
104+ 
105+ CaptureStreamInfoDumper dumper(".");
106+ auto output = dumper.FormatData(input);
107+ ASSERT_EQ(4ul, output.size());
108+ EXPECT_EQ(0u, output[0].batchId);
109+ EXPECT_EQ(0u, output[1].batchId);
110+ EXPECT_EQ(0u, output[2].batchId);
111+ EXPECT_EQ(1u, output[3].batchId);
112+}
113+ 
114+TEST(CaptureStreamInfoDumperUTest, ShouldLogMismatchedStartAndEndModelIds)
115+{
116+ const std::string expectedMessage =
117+ "CaptureStreamInfoDumper: Capture start model ids are {7}, end model ids are {}.";
118+ capturedLogMessage.clear();
119+ MOCKER_CPP(&Analysis::Log::LogMsg).stubs().will(invoke(CaptureLogMessage));
120+ std::vector<std::shared_ptr<ParserCompactInfo>> input{MakeCaptureInfo(0, 7, 10, 20, 0, 10)};
121+ 
122+ CaptureStreamInfoDumper dumper(".");
123+ auto output = dumper.FormatData(input);
124+ 
125+ ASSERT_EQ(1ul, output.size());
126+ EXPECT_EQ(expectedMessage, capturedLogMessage);
127+ MOCKER_CPP(&Analysis::Log::LogMsg).reset();
128+}
129+ 
130+TEST(Mc2CommInfoDumperUTest, ShouldKeepRawRowsAndAppendOneToManyCaptureMappedRowsInDeterministicOrder)
131+{
132+ std::vector<std::shared_ptr<ParserAdditionalInfo>> mc2Data{
133+ MakeMc2Info(5862276093215481612ULL, 2, 0, 0, 20, 2),
134+ MakeMc2Info(9, 4, 1, 1, 99, 1),
135+ };
136+ CaptureStreamInfoData captureInput{
137+ {0, 1, 20, 32, 0, 0, 1},
138+ {0, 1, 20, 21, 0, 0, 2},
139+ {0, 1, 20, 21, 1, 0, 3},
140+ {0, 1, 20, 22, 0, 0, 4},
141+ };
142+ 
143+ Mc2CommInfoDumper dumper(".");
144+ Mc2CommInfoInput input(mc2Data, captureInput);
145+ auto output = dumper.GenerateData(input);
146+ ASSERT_EQ(5ul, output.size());
147+ EXPECT_EQ("5862276093215481612", std::get<0>(output[0]));
148+ EXPECT_EQ(20u, std::get<4>(output[0]));
149+ EXPECT_EQ("9", std::get<0>(output[1]));
150+ EXPECT_EQ(99u, std::get<4>(output[1]));
151+ 
152+ EXPECT_EQ(21u, std::get<4>(output[2]));
153+ EXPECT_EQ(22u, std::get<4>(output[3]));
154+ EXPECT_EQ(32u, std::get<4>(output[4]));
155+ EXPECT_EQ("100,101", std::get<5>(output[4]));
156+}
157+ 
158+TEST(Mc2CommInfoDumperUTest, ShouldNotAppendCaptureRowsWhenKfcStreamDoesNotMatch)
159+{
160+ std::vector<std::shared_ptr<ParserAdditionalInfo>> mc2Data{MakeMc2Info(1, 2, 0, 0, 99, 1)};
161+ CaptureStreamInfoData captureInput{{0, 1, 20, 32, 0, 0, 1}};
162+ 
163+ Mc2CommInfoDumper dumper(".");
164+ Mc2CommInfoInput input(mc2Data, captureInput);
165+ auto output = dumper.GenerateData(input);
166+ ASSERT_EQ(1ul, output.size());
167+ EXPECT_EQ(99u, std::get<4>(output[0]));
168+}
169+ 
170+TEST(Mc2CommInfoDumperUTest, ShouldLogInvalidStreamSizeCountOnce)
171+{
172+ const std::string expectedMessage = "Mc2CommInfoDumper: 2 records have stream size greater than max stream size 8.";
173+ capturedLogMessage.clear();
174+ MOCKER_CPP(&Analysis::Log::LogMsg).stubs().will(invoke(CaptureLogMessage));
175+ std::vector<std::shared_ptr<ParserAdditionalInfo>> mc2Data{
176+ MakeMc2Info(1, 2, 0, 0, 10, 9),
177+ MakeMc2Info(2, 2, 0, 0, 11, 10),
178+ };
179+ CaptureStreamInfoData captureData;
180+ 
181+ Mc2CommInfoDumper dumper(".");
182+ Mc2CommInfoInput input(mc2Data, captureData);
183+ auto output = dumper.GenerateData(input);
184+ 
185+ ASSERT_EQ(2ul, output.size());
186+ EXPECT_TRUE(std::get<5>(output[0]).empty());
187+ EXPECT_EQ(expectedMessage, capturedLogMessage);
188+ MOCKER_CPP(&Analysis::Log::LogMsg).reset();
189+}
190+ 
191+TEST(Mc2CommInfoDumperUTest, ShouldMatchPythonWhenInvalidStreamSizeHasCaptureMapping)
192+{
193+ std::vector<std::shared_ptr<ParserAdditionalInfo>> mc2Data{MakeMc2Info(1, 2, 0, 0, 20, 9)};
194+ CaptureStreamInfoData captureData{{0, 1, 20, 32, 0, 0, 1}};
195+ 
196+ Mc2CommInfoDumper dumper(".");
197+ Mc2CommInfoInput input(mc2Data, captureData);
198+ auto output = dumper.GenerateData(input);
199+ 
200+ ASSERT_EQ(2ul, output.size());
201+ EXPECT_EQ(20u, std::get<4>(output[0]));
202+ EXPECT_EQ(32u, std::get<4>(output[1]));
203+ EXPECT_TRUE(std::get<5>(output[0]).empty());
204+ EXPECT_TRUE(std::get<5>(output[1]).empty());
205+}
@@ -52,4 +52,4 @@ TEST_F(HostTraceWorkerUtest, TestKernelHostTraceWorkerShouldReturnTrueWhenGroupR
52 MOCKER_CPP(&EventGrouper::GetGroupEvents).stubs().will(returnValue(mockWarehouses));52 MOCKER_CPP(&EventGrouper::GetGroupEvents).stubs().will(returnValue(mockWarehouses));
53 auto ret = hostTraceWorker.Run();53 auto ret = hostTraceWorker.Run();
54 EXPECT_EQ(ret, true);54 EXPECT_EQ(ret, true);
55-}55+}
@@ -17,13 +17,18 @@
17#include "mockcpp/mockcpp.hpp"17#include "mockcpp/mockcpp.hpp"
18 18 
19#include "analysis/csrc/domain/services/host_worker/kernel_parser_worker.h"19#include "analysis/csrc/domain/services/host_worker/kernel_parser_worker.h"
20+#include "analysis/csrc/domain/services/persistence/host/capture_stream_info_dumper.h"
20#include "analysis/csrc/domain/services/persistence/host/hash_db_dumper.h"21#include "analysis/csrc/domain/services/persistence/host/hash_db_dumper.h"
22+#include "analysis/csrc/domain/services/persistence/host/mc2_comm_info_dumper.h"
21#include "analysis/csrc/domain/services/persistence/host/type_info_db_dumper.h"23#include "analysis/csrc/domain/services/persistence/host/type_info_db_dumper.h"
22#include "analysis/csrc/domain/services/environment/context.h"24#include "analysis/csrc/domain/services/environment/context.h"
23#include "analysis/csrc/domain/services/parser/host/cann/hash_data.h"25#include "analysis/csrc/domain/services/parser/host/cann/hash_data.h"
24#include "analysis/csrc/domain/services/parser/host/cann/type_data.h"26#include "analysis/csrc/domain/services/parser/host/cann/type_data.h"
25#include "analysis/csrc/domain/services/host_worker/host_trace_worker.h"27#include "analysis/csrc/domain/services/host_worker/host_trace_worker.h"
26- 28+#include "analysis/csrc/domain/services/parser/host/cann/rt_add_info_center.h"
29+#include "analysis/csrc/domain/services/parser/host/cann/capture_mc2_cpp_enable.h"
30+#include "test/msprof_cpp/analysis_ut/domain/data_process/test/reserve_mock_utils.h"
31+#include "test/msprof_cpp/analysis_ut/domain/services/test/fake_generator.h"
27 32 
28using namespace Analysis::Domain;33using namespace Analysis::Domain;
29 34 
@@ -38,6 +43,9 @@ class KernelParserWorkerUtest : public testing::Test {
38protected:43protected:
39 virtual void SetUp()44 virtual void SetUp()
40 {45 {
46+ if (File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite"}))) {
47+ File::RemoveDir(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite"}), 0);
48+ }
41 File::CreateDir(TEST_HOST_FILE_PATH);49 File::CreateDir(TEST_HOST_FILE_PATH);
42 File::CreateDir(File::PathJoin({TEST_HOST_FILE_PATH, "data"}));50 File::CreateDir(File::PathJoin({TEST_HOST_FILE_PATH, "data"}));
43 File::CreateDir(File::PathJoin({TEST_HOST_FILE_PATH, "data", "sqlite"}));51 File::CreateDir(File::PathJoin({TEST_HOST_FILE_PATH, "data", "sqlite"}));
@@ -53,17 +61,40 @@ protected:
53 {61 {
54 GlobalMockObject::verify();62 GlobalMockObject::verify();
55 File::RemoveDir(File::PathJoin({TEST_HOST_FILE_PATH, "data"}), 0);63 File::RemoveDir(File::PathJoin({TEST_HOST_FILE_PATH, "data"}), 0);
64+ if (File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite"}))) {
65+ File::RemoveDir(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite"}), 0);
66+ }
67+ }
68+ 
69+ void StubContextLoadSuccess()
70+ {
71+ MOCKER_CPP(&Analysis::Domain::Environment::Context::Load).stubs().will(returnValue(true));
72+ }
73+ 
74+ void StubPlatformVersion(Analysis::Domain::Environment::Chip chip)
75+ {
76+ StubContextLoadSuccess();
77+ MOCKER_CPP(&Analysis::Domain::Environment::Context::GetPlatformVersion)
78+ .stubs()
79+ .will(returnValue(static_cast<uint16_t>(chip)));
80+ }
81+ 
82+ void UseRealHostTraceWorker()
83+ {
84+ MOCKER_CPP(&HostTraceWorker::Run).reset();
56 }85 }
57};86};
58 87 
59TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnSuccessWhenAllFunctionWell)88TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnSuccessWhenAllFunctionWell)
60{89{
90+ StubContextLoadSuccess();
61 KernelParserWorker kernelParserWorker(TEST_HOST_FILE_PATH);91 KernelParserWorker kernelParserWorker(TEST_HOST_FILE_PATH);
62 auto res = kernelParserWorker.Run();92 auto res = kernelParserWorker.Run();
63 EXPECT_EQ(res, 0);93 EXPECT_EQ(res, 0);
64}94}
65TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnFailedWhenHostTraceWorkerReturnFailed)95TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnFailedWhenHostTraceWorkerReturnFailed)
66{96{
97+ StubContextLoadSuccess();
67 KernelParserWorker kernelParserWorker(TEST_HOST_FILE_PATH);98 KernelParserWorker kernelParserWorker(TEST_HOST_FILE_PATH);
68 MOCKER_CPP(&HostTraceWorker::Run).reset();99 MOCKER_CPP(&HostTraceWorker::Run).reset();
69 MOCKER_CPP(&HostTraceWorker::Run).stubs().will(returnValue(false));100 MOCKER_CPP(&HostTraceWorker::Run).stubs().will(returnValue(false));
@@ -73,6 +104,7 @@ TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnFailedWhenHost
73 104 
74TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnFailedWhenDumpHashFailed)105TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnFailedWhenDumpHashFailed)
75{106{
107+ StubContextLoadSuccess();
76 KernelParserWorker kernelParserWorker(TEST_HOST_FILE_PATH);108 KernelParserWorker kernelParserWorker(TEST_HOST_FILE_PATH);
77 MOCKER_CPP(&DBRunner::CreateTable).stubs().will(returnValue(false));109 MOCKER_CPP(&DBRunner::CreateTable).stubs().will(returnValue(false));
78 auto res = kernelParserWorker.Run();110 auto res = kernelParserWorker.Run();
@@ -81,6 +113,7 @@ TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnFailedWhenDump
81 113 
82TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnFailedWhenTypeInfoDumpFailed)114TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnFailedWhenTypeInfoDumpFailed)
83{115{
116+ StubContextLoadSuccess();
84 KernelParserWorker kernelParserWorker(TEST_HOST_FILE_PATH);117 KernelParserWorker kernelParserWorker(TEST_HOST_FILE_PATH);
85 MOCKER_CPP(&DBRunner::CreateTable).stubs().will(returnValue(false));118 MOCKER_CPP(&DBRunner::CreateTable).stubs().will(returnValue(false));
86 auto res = kernelParserWorker.Run();119 auto res = kernelParserWorker.Run();
@@ -93,4 +126,189 @@ TEST_F(KernelParserWorkerUtest, TestKernelParserWorkerShouldReturnErrorWhenConte
93 MOCKER_CPP(&Analysis::Domain::Environment::Context::Load).stubs().will(returnValue(false));126 MOCKER_CPP(&Analysis::Domain::Environment::Context::Load).stubs().will(returnValue(false));
94 auto res = kernelParserWorker.Run();127 auto res = kernelParserWorker.Run();
95 EXPECT_EQ(res, 1);128 EXPECT_EQ(res, 1);
96-}129+}
130+ 
131+TEST_F(KernelParserWorkerUtest, ShouldSkipCaptureParseWhenCppParserDisabledEvenIfTruncatedRecordExists)
132+{
133+ // C++ Capture/MC2 默认关闭:截断 bin 不会进入 Host C++ 解析,Host 其它输出仍成功。
134+ ASSERT_FALSE(Host::Cann::kEnableCaptureStreamMc2CppParser);
135+ UseRealHostTraceWorker();
136+ std::vector<uint8_t> truncatedInput{0};
137+ ASSERT_TRUE(WriteBin(truncatedInput, File::PathJoin({TEST_HOST_FILE_PATH, "data"}),
138+ "unaging.compact.capture_stream_info.slice_0"));
139+ StubPlatformVersion(Analysis::Domain::Environment::Chip::CHIP_V3_1_0);
140+ 
141+ KernelParserWorker worker(TEST_HOST_FILE_PATH);
142+ EXPECT_EQ(ANALYSIS_OK, worker.Run());
143+ EXPECT_TRUE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "ge_hash.db"})));
144+ EXPECT_FALSE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "stream_info.db"})));
145+}
146+ 
147+TEST_F(KernelParserWorkerUtest, ShouldSkipMc2ParseWhenCppParserDisabledEvenIfTruncatedRecordExists)
148+{
149+ ASSERT_FALSE(Host::Cann::kEnableCaptureStreamMc2CppParser);
150+ UseRealHostTraceWorker();
151+ std::vector<uint8_t> truncatedInput{0};
152+ ASSERT_TRUE(WriteBin(truncatedInput, File::PathJoin({TEST_HOST_FILE_PATH, "data"}),
153+ "unaging.additional.mc2_comm_info.slice_0"));
154+ StubPlatformVersion(Analysis::Domain::Environment::Chip::CHIP_V3_3_0);
155+ 
156+ KernelParserWorker worker(TEST_HOST_FILE_PATH);
157+ EXPECT_EQ(ANALYSIS_OK, worker.Run());
158+ EXPECT_TRUE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "ge_hash.db"})));
159+ EXPECT_FALSE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "mc2_comm_info.db"})));
160+}
161+ 
162+TEST_F(KernelParserWorkerUtest, ShouldNotInvokeCaptureDumperWhenCppParserDisabled)
163+{
164+ ASSERT_FALSE(Host::Cann::kEnableCaptureStreamMc2CppParser);
165+ UseRealHostTraceWorker();
166+ MsprofCompactInfo capture{};
167+ capture.timeStamp = 123;
168+ capture.data.captureStreamInfo.deviceId = 0;
169+ capture.data.captureStreamInfo.modelId = 7;
170+ capture.data.captureStreamInfo.originalStreamId = 52;
171+ capture.data.captureStreamInfo.modelStreamId = 70;
172+ capture.data.captureStreamInfo.captureStatus = 0;
173+ std::vector<MsprofCompactInfo> captureInput{capture};
174+ ASSERT_TRUE(WriteBin(captureInput, File::PathJoin({TEST_HOST_FILE_PATH, "data"}),
175+ "unaging.compact.capture_stream_info.slice_0"));
176+ StubPlatformVersion(Analysis::Domain::Environment::Chip::CHIP_V3_1_0);
177+ ASSERT_TRUE(File::CreateDir(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite"})));
178+ // 即使预置与 dumper 冲突的路径,默认关闭时也不会走 Capture dump。
179+ ASSERT_TRUE(File::CreateDir(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "stream_info.db"})));
180+ 
181+ KernelParserWorker worker(TEST_HOST_FILE_PATH);
182+ EXPECT_EQ(ANALYSIS_OK, worker.Run());
183+ EXPECT_TRUE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "ge_hash.db"})));
184+}
185+ 
186+TEST_F(KernelParserWorkerUtest, ShouldNotInvokeMc2DumperWhenCppParserDisabled)
187+{
188+ ASSERT_FALSE(Host::Cann::kEnableCaptureStreamMc2CppParser);
189+ UseRealHostTraceWorker();
190+ MsprofAdditionalInfo mc2{};
191+ auto payload = ReinterpretConvert<Analysis::Domain::Host::Cann::MsprofMc2CommInfo *>(mc2.data);
192+ payload->groupName = 99;
193+ payload->rankSize = 2;
194+ payload->streamId = 52;
195+ payload->streamSize = 1;
196+ payload->commStreamIds[0] = 100;
197+ std::vector<MsprofAdditionalInfo> mc2Input{mc2};
198+ ASSERT_TRUE(WriteBin(mc2Input, File::PathJoin({TEST_HOST_FILE_PATH, "data"}),
199+ "unaging.additional.mc2_comm_info.slice_0"));
200+ StubPlatformVersion(Analysis::Domain::Environment::Chip::CHIP_V3_3_0);
201+ ASSERT_TRUE(File::CreateDir(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite"})));
202+ ASSERT_TRUE(File::CreateDir(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "mc2_comm_info.db"})));
203+ 
204+ KernelParserWorker worker(TEST_HOST_FILE_PATH);
205+ EXPECT_EQ(ANALYSIS_OK, worker.Run());
206+ EXPECT_TRUE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "ge_hash.db"})));
207+}
208+ 
209+TEST_F(KernelParserWorkerUtest, ShouldSkipCaptureFormattingWhenCppParserDisabled)
210+{
211+ ASSERT_FALSE(Host::Cann::kEnableCaptureStreamMc2CppParser);
212+ UseRealHostTraceWorker();
213+ MsprofCompactInfo capture{};
214+ capture.timeStamp = 123;
215+ capture.data.captureStreamInfo.modelId = 7;
216+ capture.data.captureStreamInfo.originalStreamId = 52;
217+ capture.data.captureStreamInfo.modelStreamId = 70;
218+ std::vector<MsprofCompactInfo> captureInput{capture};
219+ ASSERT_TRUE(WriteBin(captureInput, File::PathJoin({TEST_HOST_FILE_PATH, "data"}),
220+ "unaging.compact.capture_stream_info.slice_0"));
221+ StubPlatformVersion(Analysis::Domain::Environment::Chip::CHIP_V3_1_0);
222+ Analysis::Test::StubReserveFailureForVector<CaptureStreamInfoData>();
223+ 
224+ KernelParserWorker worker(TEST_HOST_FILE_PATH);
225+ auto result = worker.Run();
226+ Analysis::Test::ResetReserveFailureForVector<CaptureStreamInfoData>();
227+ EXPECT_EQ(ANALYSIS_OK, result);
228+ EXPECT_TRUE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "ge_hash.db"})));
229+}
230+ 
231+TEST_F(KernelParserWorkerUtest, ShouldSkipMc2FormattingWhenCppParserDisabled)
232+{
233+ ASSERT_FALSE(Host::Cann::kEnableCaptureStreamMc2CppParser);
234+ UseRealHostTraceWorker();
235+ MsprofAdditionalInfo mc2{};
236+ auto payload = ReinterpretConvert<Analysis::Domain::Host::Cann::MsprofMc2CommInfo *>(mc2.data);
237+ payload->groupName = 99;
238+ payload->rankSize = 2;
239+ payload->streamId = 52;
240+ payload->streamSize = 1;
241+ payload->commStreamIds[0] = 100;
242+ std::vector<MsprofAdditionalInfo> mc2Input{mc2};
243+ ASSERT_TRUE(WriteBin(mc2Input, File::PathJoin({TEST_HOST_FILE_PATH, "data"}),
244+ "unaging.additional.mc2_comm_info.slice_0"));
245+ StubPlatformVersion(Analysis::Domain::Environment::Chip::CHIP_V3_3_0);
246+ Analysis::Test::StubReserveFailureForVector<Mc2CommInfoData>();
247+ 
248+ KernelParserWorker worker(TEST_HOST_FILE_PATH);
249+ auto result = worker.Run();
250+ Analysis::Test::ResetReserveFailureForVector<Mc2CommInfoData>();
251+ EXPECT_EQ(ANALYSIS_OK, result);
252+ EXPECT_TRUE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "ge_hash.db"})));
253+}
254+ 
255+TEST_F(KernelParserWorkerUtest, ShouldNotCreateCaptureAndMc2DatabasesWhenCppParserDisabled)
256+{
257+ ASSERT_FALSE(Host::Cann::kEnableCaptureStreamMc2CppParser);
258+ UseRealHostTraceWorker();
259+ MsprofCompactInfo capture{};
260+ capture.timeStamp = 123;
261+ capture.data.captureStreamInfoV2.deviceId = 0;
262+ capture.data.captureStreamInfoV2.modelId = 7;
263+ capture.data.captureStreamInfoV2.originalStreamId = 70002;
264+ capture.data.captureStreamInfoV2.streamId = 70003;
265+ capture.data.captureStreamInfoV2.captureStatus = 0;
266+ std::vector<MsprofCompactInfo> captureInput{capture};
267+ ASSERT_TRUE(WriteBin(captureInput, File::PathJoin({TEST_HOST_FILE_PATH, "data"}),
268+ "unaging.compact.capture_stream_info_v2.slice_0"));
269+ 
270+ MsprofAdditionalInfo mc2{};
271+ auto payload = ReinterpretConvert<Analysis::Domain::Host::Cann::MsprofMc2CommInfo *>(mc2.data);
272+ payload->groupName = 99;
273+ payload->rankSize = 2;
274+ payload->rankId = 0;
275+ payload->usrRankId = 0;
276+ payload->streamId = 70002;
277+ payload->streamSize = 2;
278+ payload->commStreamIds[0] = 70004;
279+ payload->commStreamIds[1] = 70005;
280+ std::vector<MsprofAdditionalInfo> mc2Input{mc2};
281+ ASSERT_TRUE(WriteBin(mc2Input, File::PathJoin({TEST_HOST_FILE_PATH, "data"}),
282+ "unaging.additional.mc2_comm_info.slice_0"));
283+ StubPlatformVersion(Analysis::Domain::Environment::Chip::CHIP_V3_3_0);
284+ 
285+ KernelParserWorker worker(TEST_HOST_FILE_PATH);
286+ ASSERT_EQ(ANALYSIS_OK, worker.Run());
287+ 
288+ EXPECT_FALSE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "stream_info.db"})));
289+ EXPECT_FALSE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "mc2_comm_info.db"})));
290+}
291+ 
292+TEST_F(KernelParserWorkerUtest, ShouldSkipCaptureAndMc2WhenCppParserDisabledOnAnyPlatform)
293+{
294+ ASSERT_FALSE(Host::Cann::kEnableCaptureStreamMc2CppParser);
295+ UseRealHostTraceWorker();
296+ MsprofCompactInfo capture{};
297+ capture.data.captureStreamInfo.modelStreamId = 70;
298+ std::vector<MsprofCompactInfo> captureInput{capture};
299+ ASSERT_TRUE(WriteBin(captureInput, File::PathJoin({TEST_HOST_FILE_PATH, "data"}),
300+ "unaging.compact.capture_stream_info.slice_0"));
301+ 
302+ MsprofAdditionalInfo mc2{};
303+ auto payload = ReinterpretConvert<Analysis::Domain::Host::Cann::MsprofMc2CommInfo *>(mc2.data);
304+ payload->streamId = 52;
305+ std::vector<MsprofAdditionalInfo> mc2Input{mc2};
306+ ASSERT_TRUE(WriteBin(mc2Input, File::PathJoin({TEST_HOST_FILE_PATH, "data"}),
307+ "unaging.additional.mc2_comm_info.slice_0"));
308+ StubPlatformVersion(Analysis::Domain::Environment::Chip::CHIP_V1_1_0);
309+ 
310+ KernelParserWorker worker(TEST_HOST_FILE_PATH);
311+ ASSERT_EQ(ANALYSIS_OK, worker.Run());
312+ EXPECT_FALSE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "stream_info.db"})));
313+ EXPECT_FALSE(File::Exist(File::PathJoin({TEST_HOST_FILE_PATH, "sqlite", "mc2_comm_info.db"})));
314+}
@@ -20,27 +20,68 @@ function:
20Copyright Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.20Copyright Huawei Technologies Co., Ltd. 2020-2021. All rights reserved.
21"""21"""
22import unittest22import unittest
23+from unittest import mock
23 24 
25+from common_func.cpp_enable_scene import CannCalculatorScene, DeviceParseScene
24from common_func.info_conf_reader import InfoConfReader26from common_func.info_conf_reader import InfoConfReader
25from common_func.platform.chip_manager import ChipManager27from common_func.platform.chip_manager import ChipManager
28+from common_func.profiling_scene import ProfilingScene
26from framework.config_data_parsers import ConfigDataParsers29from framework.config_data_parsers import ConfigDataParsers
27from msconfig.config_manager import ConfigManager30from msconfig.config_manager import ConfigManager
28from profiling_bean.prof_enum.chip_model import ChipModel31from profiling_bean.prof_enum.chip_model import ChipModel
29 32 
30 33 
31class TestConfigDataParsers(unittest.TestCase):34class TestConfigDataParsers(unittest.TestCase):
35+ @staticmethod
36+ def _parser_names(parsers):
37+ return {parser.__name__ for level_parsers in parsers.values() for parser in level_parsers}
38+ 
32 def test_get_parsers(self):39 def test_get_parsers(self):
33 InfoConfReader()._sample_json = {'devices': '0'}40 InfoConfReader()._sample_json = {'devices': '0'}
34- ConfigDataParsers.get_parsers(ConfigManager.DATA_CALCULATOR, str(ChipModel.CHIP_V3_1_0.value), False)41+ parsers = ConfigDataParsers.get_parsers(
42+ ConfigManager.DATA_CALCULATOR, str(ChipModel.CHIP_V3_1_0.value), False)
43+ self.assertIsInstance(parsers, dict)
35 44 
36 def test_load_can_cpp_parse_or_calculate_host_data(self):45 def test_load_can_cpp_parse_or_calculate_host_data(self):
37 ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_host_data("NpuMemParser")46 ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_host_data("NpuMemParser")
38 self.assertFalse(ret)47 self.assertFalse(ret)
39 ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_host_data("HashDicParser")48 ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_host_data("HashDicParser")
40 self.assertTrue(ret)49 self.assertTrue(ret)
50+ ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_host_data("CaptureStreamInfoParser")
51+ self.assertFalse(ret)
52+ ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_host_data("Mc2CommInfoParser")
53+ self.assertFalse(ret)
41 ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_host_data("RuntimeOpInfoParser")54 ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_host_data("RuntimeOpInfoParser")
42 self.assertTrue(ret)55 self.assertTrue(ret)
43 56 
57+ @mock.patch.object(DeviceParseScene, "is_cpp_enable", return_value=False)
58+ @mock.patch.object(CannCalculatorScene, "is_cpp_enable", return_value=False)
59+ def test_get_parsers_should_keep_capture_and_mc2_python_fallback_when_host_cpp_is_disabled(
60+ self, _host_cpp_enable, _device_cpp_enable):
61+ InfoConfReader()._sample_json = {'devices': str(64)}
62+ 
63+ with mock.patch.object(ProfilingScene(), "is_all_export", return_value=False):
64+ parsers = ConfigDataParsers.get_parsers(
65+ ConfigManager.DATA_PARSERS, str(ChipModel.CHIP_V3_3_0.value), False)
66+ 
67+ parser_names = self._parser_names(parsers)
68+ self.assertIn("CaptureStreamInfoParser", parser_names)
69+ self.assertIn("Mc2CommInfoParser", parser_names)
70+ 
71+ @mock.patch.object(DeviceParseScene, "is_cpp_enable", return_value=False)
72+ @mock.patch.object(CannCalculatorScene, "is_cpp_enable", return_value=True)
73+ def test_get_parsers_should_keep_capture_and_mc2_when_host_cpp_is_enabled(
74+ self, _host_cpp_enable, _device_cpp_enable):
75+ InfoConfReader()._sample_json = {'devices': str(64)}
76+ 
77+ with mock.patch.object(ProfilingScene(), "is_all_export", return_value=False):
78+ parsers = ConfigDataParsers.get_parsers(
79+ ConfigManager.DATA_PARSERS, str(ChipModel.CHIP_V3_3_0.value), False)
80+ 
81+ parser_names = self._parser_names(parsers)
82+ self.assertIn("CaptureStreamInfoParser", parser_names)
83+ self.assertIn("Mc2CommInfoParser", parser_names)
84+ 
44 def test_load_can_cpp_parse_or_calculate_device_data_should_return_true_when_given_in_whitelist(self):85 def test_load_can_cpp_parse_or_calculate_device_data_should_return_true_when_given_in_whitelist(self):
45 ChipManager().chip_id = ChipModel.CHIP_V4_1_086 ChipManager().chip_id = ChipModel.CHIP_V4_1_0
46 ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_device_data("AscendTaskCalculator")87 ret = ConfigDataParsers._load_can_cpp_parse_or_calculate_device_data("AscendTaskCalculator")