Star121
56
代码
代码
Issues16
Pull Requests17
项目讨论
Wiki
分析
Star121
56
已合并
aicpu persistence落盘问题修复 #458
wangzixuan创建于 22 天前
aicpu persistence落盘问题修复 #458
已合并
wangzixuan创建于 22 天前
从Seanesmhxocism/msprofcDev合入到Ascend/msprofmaster
共 8 个文件变更+144-5
Manalysis/csrc/domain/services/persistence/device/aicpu_persistence.cpp+4-1
@@ -502,7 +502,9 @@ uint32_t AicpuPersistence::GenerateAndSaveData(const std::string& deviceFilePath
502 // 表驱动方式：使用 std::function 适配不同签名的函数502 // 表驱动方式：使用 std::function 适配不同签名的函数
503 using func = std::function<uint32_t()>;503 using func = std::function<uint32_t()>;
504 504 
505- static const std::vector<std::pair<const std::vector<AicpuData> AicpuPersistence::*, func>> dataProcessMap = {505+ // 注意：不能声明为 static。static 会导致首次调用时捕获的 this/deviceFilePath
506+ // 被所有 device 实例共享，后续卡的落盘路径全部错写成第一张卡的目录。
507+ const std::vector<std::pair<const std::vector<AicpuData> AicpuPersistence::*, func>> dataProcessMap = {
506 {&AicpuPersistence::nodeData_, [this, deviceFilePath]() { return GenerateAndSaveNode(deviceFilePath); }},508 {&AicpuPersistence::nodeData_, [this, deviceFilePath]() { return GenerateAndSaveNode(deviceFilePath); }},
507 {&AicpuPersistence::dpData_, [this, deviceFilePath]() { return GenerateAndSaveDp(deviceFilePath); }},509 {&AicpuPersistence::dpData_, [this, deviceFilePath]() { return GenerateAndSaveDp(deviceFilePath); }},
508 {&AicpuPersistence::modelData_, [this, deviceFilePath]() { return GenerateAndSaveModel(deviceFilePath); }},510 {&AicpuPersistence::modelData_, [this, deviceFilePath]() { return GenerateAndSaveModel(deviceFilePath); }},
@@ -628,6 +630,7 @@ void AicpuPersistence::ComputeAicpuBatchId()
628uint32_t AicpuPersistence::ProcessEntry(DataInventory& dataInventory, const Context& context)630uint32_t AicpuPersistence::ProcessEntry(DataInventory& dataInventory, const Context& context)
629{631{
630 const DeviceContext& deviceContext = static_cast<const DeviceContext&>(context);632 const DeviceContext& deviceContext = static_cast<const DeviceContext&>(context);
633+ INFO("Start to process aicpu data for device, path %.", deviceContext.GetDeviceFilePath());
631 auto aicpuData = dataInventory.GetPtr<std::vector<AicpuData>>();634 auto aicpuData = dataInventory.GetPtr<std::vector<AicpuData>>();
632 auto deviceStreamInfo = dataInventory.GetPtr<DeviceStreamInfo>();635 auto deviceStreamInfo = dataInventory.GetPtr<DeviceStreamInfo>();
633 auto hostStreamInfo = dataInventory.GetPtr<HostStreamInfo>();636 auto hostStreamInfo = dataInventory.GetPtr<HostStreamInfo>();
Manalysis/csrc/infrastructure/utils/utils.h+6-4
@@ -19,8 +19,10 @@
19 19 
20#include <algorithm>20#include <algorithm>
21#include <cmath>21#include <cmath>
22+#include <exception>
22#include <memory>23#include <memory>
23#include <string>24#include <string>
25+#include <typeinfo>
24#include <vector>26#include <vector>
25 27 
26#include "analysis/csrc/infrastructure/dfx/log.h"28#include "analysis/csrc/infrastructure/dfx/log.h"
@@ -175,9 +177,9 @@ bool Reserve(std::vector<T> &vec, size_t s)
175 {177 {
176 vec.reserve(s);178 vec.reserve(s);
177 }179 }
178- catch (...)180+ catch (const std::exception &e)
179 {181 {
180- ERROR("Reserve vector failed");182+ ERROR("Reserve vector failed, data type=%, size=%, reason=%", typeid(T).name(), s, e.what());
181 return false;183 return false;
182 }184 }
183 return true;185 return true;
@@ -190,9 +192,9 @@ bool Resize(std::vector<T> &vec, size_t s)
190 {192 {
191 vec.resize(s);193 vec.resize(s);
192 }194 }
193- catch (...)195+ catch (const std::exception &e)
194 {196 {
195- ERROR("Resize vector failed");197+ ERROR("Resize vector failed, data type=%, size=%, reason=%", typeid(T).name(), s, e.what());
196 return false;198 return false;
197 }199 }
198 return true;200 return true;
Mtest/msprof_cpp/analysis_ut/domain/persistence/test/aicpu_persistence_utest.cpp+88-0
@@ -697,6 +697,94 @@ TEST_F(AicpuPersistenceSaveUtest, ShouldOnlyWriteAiCpuDbFileWhenSaveBothNodeAndD
697 EXPECT_TRUE(dbRunner.CheckTableExists("AiCpuDP"));697 EXPECT_TRUE(dbRunner.CheckTableExists("AiCpuDP"));
698}698}
699 699 
700+// =========================================================================
701+// GenerateAndSaveData — 多 device 实例各自落盘到自己的目录
702+// 回归测试：dataProcessMap 不能声明为 static。static 会让后续 device 实例
703+// 复用首次调用捕获的 this/deviceFilePath，导致所有卡的 aicpu 数据
704+// 全部写进第一张卡的 ai_cpu.db（旧 bug：各卡数据累加和 = 单卡 db 条数）。
705+// =========================================================================
706+ 
707+class AicpuPersistenceMultiDeviceSaveUtest : public Test {
708+protected:
709+ void SetUp() override
710+ {
711+ devicePath0_ = "./aicpu_multi_device_0";
712+ devicePath1_ = "./aicpu_multi_device_1";
713+ EXPECT_TRUE(File::CreateDir(devicePath0_));
714+ EXPECT_TRUE(File::CreateDir(File::PathJoin({devicePath0_, "sqlite"})));
715+ EXPECT_TRUE(File::CreateDir(devicePath1_));
716+ EXPECT_TRUE(File::CreateDir(File::PathJoin({devicePath1_, "sqlite"})));
717+ persistence0_ = std::make_shared<AicpuPersistence>();
718+ persistence1_ = std::make_shared<AicpuPersistence>();
719+ // freq=1000, sysCnt=1000, hostMonotonic=1000，时间换算结果确定无溢出
720+ persistence0_->params_ = SyscntConversionParams(1000.0, 1000, 1000);
721+ persistence1_->params_ = SyscntConversionParams(1000.0, 1000, 1000);
722+ }
723+ 
724+ void TearDown() override
725+ {
726+ EXPECT_TRUE(File::RemoveDir(devicePath0_, 0));
727+ EXPECT_TRUE(File::RemoveDir(devicePath1_, 0));
728+ }
729+ 
730+ AicpuData CreateNodeData(uint32_t taskId, uint64_t runStartTick, uint64_t runEndTick)
731+ {
732+ AicpuData data;
733+ data.type = AicpuType::AICPU_NODE;
734+ data.taskId.streamId = 1;
735+ data.taskId.taskId = taskId;
736+ data.node.runStartTick = runStartTick;
737+ data.node.runEndTick = runEndTick;
738+ data.node.computeStartTime = 1000;
739+ data.node.memcpyStartTime = 1200;
740+ data.node.memcpyEndTime = 1500;
741+ data.node.dispatchTime = 300;
742+ data.node.submitTick = 900;
743+ data.node.tickAfterRun = runEndTick + 100;
744+ return data;
745+ }
746+ 
747+protected:
748+ std::string devicePath0_;
749+ std::string devicePath1_;
750+ std::shared_ptr<AicpuPersistence> persistence0_;
751+ std::shared_ptr<AicpuPersistence> persistence1_;
752+};
753+ 
754+TEST_F(AicpuPersistenceMultiDeviceSaveUtest, ShouldSaveEachDeviceDataToItsOwnDbPath)
755+{
756+ // 两张卡各自构造一份 node 数据，taskId 用于区分来源
757+ persistence0_->nodeData_.emplace_back(CreateNodeData(10, 1000, 2000));
758+ persistence1_->nodeData_.emplace_back(CreateNodeData(20, 3000, 4000));
759+ 
760+ ASSERT_EQ(ANALYSIS_OK, persistence0_->GenerateAndSaveData(devicePath0_));
761+ ASSERT_EQ(ANALYSIS_OK, persistence1_->GenerateAndSaveData(devicePath1_));
762+ 
763+ using NodeRow = std::tuple<uint32_t, uint16_t, double, double, std::string, uint64_t, uint64_t, double, uint64_t,
764+ double>;
765+ 
766+ // device0 的 ai_cpu.db 只含 device0 的数据（1 条，task_id=10）
767+ std::string dbPath0 = File::PathJoin({devicePath0_, "sqlite", "ai_cpu.db"});
768+ EXPECT_TRUE(File::Exist(dbPath0));
769+ DBRunner dbRunner0(dbPath0);
770+ EXPECT_TRUE(dbRunner0.CheckTableExists("AiCpuData"));
771+ std::vector<NodeRow> rows0;
772+ EXPECT_TRUE(dbRunner0.QueryData("SELECT * FROM AiCpuData", rows0));
773+ ASSERT_EQ(rows0.size(), 1);
774+ EXPECT_EQ(std::get<1>(rows0[0]), 10);
775+ 
776+ // device1 的 ai_cpu.db 只含 device1 的数据（1 条，task_id=20）。
777+ // 回归点：旧 static bug 下 device1 的数据被写入 device0 的目录，这里不存在。
778+ std::string dbPath1 = File::PathJoin({devicePath1_, "sqlite", "ai_cpu.db"});
779+ EXPECT_TRUE(File::Exist(dbPath1));
780+ DBRunner dbRunner1(dbPath1);
781+ EXPECT_TRUE(dbRunner1.CheckTableExists("AiCpuData"));
782+ std::vector<NodeRow> rows1;
783+ EXPECT_TRUE(dbRunner1.QueryData("SELECT * FROM AiCpuData", rows1));
784+ ASSERT_EQ(rows1.size(), 1);
785+ EXPECT_EQ(std::get<1>(rows1[0]), 20);
786+}
787+ 
700 788 
701} // namespace789} // namespace
702} // namespace Domain790} // namespace Domain
Mtest/msprof_cpp/analysis_ut/infrastructure/utils/test/utils_utest.cpp+46-0
@@ -15,6 +15,8 @@
15 * -------------------------------------------------------------------------*/15 * -------------------------------------------------------------------------*/
16#include "analysis/csrc/infrastructure/utils/utils.h"16#include "analysis/csrc/infrastructure/utils/utils.h"
17 17 
18+#include <limits>
19+ 
18#include "gtest/gtest.h"20#include "gtest/gtest.h"
19#include "analysis/csrc/infrastructure/dfx/error_code.h"21#include "analysis/csrc/infrastructure/dfx/error_code.h"
20#include "analysis/csrc/domain/services/environment/context.h"22#include "analysis/csrc/domain/services/environment/context.h"
@@ -257,4 +259,48 @@ TEST_F(UtilsUTest, TestDivideByPowersOfTenWithPrecisionShouldReturnTrueValue)
257 259 
258 value = 78; // 入参78260 value = 78; // 入参78
259 EXPECT_EQ("0.07", DivideByPowersOfTenWithPrecision(value, 2, 3)); // 长度小于3位，移动3位，精度2位261 EXPECT_EQ("0.07", DivideByPowersOfTenWithPrecision(value, 2, 3)); // 长度小于3位，移动3位，精度2位
262+}
263+ 
264+// =========================================================================
265+// Reserve / Resize — 异常路径看护
266+// 生产中出现超大 size 时，reserve/resize 抛异常必须被捕获并返回 false，
267+// 而不是让异常向上传播导致进程崩溃
268+// =========================================================================
269+ 
270+TEST_F(UtilsUTest, TestReserveShouldReturnTrueWhenSizeIsNormal)
271+{
272+ std::vector<int> vec;
273+ EXPECT_TRUE(Reserve(vec, 10));
274+ EXPECT_GE(vec.capacity(), static_cast<size_t>(10));
275+}
276+ 
277+TEST_F(UtilsUTest, TestReserveShouldReturnFalseWhenSizeExceedsMax)
278+{
279+ std::vector<int> vec;
280+ // 超过 max_size 会抛 std::length_error，应被捕获并返回 false
281+ EXPECT_FALSE(Reserve(vec, std::numeric_limits<size_t>::max()));
282+ EXPECT_TRUE(vec.empty());
283+ // 异常路径不应破坏已存在的数据
284+ vec = {1, 2, 3};
285+ EXPECT_FALSE(Reserve(vec, std::numeric_limits<size_t>::max()));
286+ EXPECT_EQ(vec.size(), static_cast<size_t>(3));
287+}
288+ 
289+TEST_F(UtilsUTest, TestResizeShouldReturnTrueWhenSizeIsNormal)
290+{
291+ std::vector<int> vec;
292+ EXPECT_TRUE(Resize(vec, 10));
293+ EXPECT_EQ(vec.size(), static_cast<size_t>(10));
294+}
295+ 
296+TEST_F(UtilsUTest, TestResizeShouldReturnFalseWhenSizeExceedsMax)
297+{
298+ std::vector<int> vec;
299+ // 超过 max_size 会抛 std::length_error，应被捕获并返回 false
300+ EXPECT_FALSE(Resize(vec, std::numeric_limits<size_t>::max()));
301+ EXPECT_TRUE(vec.empty());
302+ // 异常路径不应破坏已存在的数据
303+ vec = {1, 2, 3};
304+ EXPECT_FALSE(Resize(vec, std::numeric_limits<size_t>::max()));
305+ EXPECT_EQ(vec.size(), static_cast<size_t>(3));
260}306}
Rtest/st/l0_test_ascend_msprof_matmul_basic_api.sh→test/st/l1_test_ascend_msprof_matmul_basic_api.sh+0-0
文件重命名但无更改。
Rtest/st/l0_test_ascend_msprof_matmul_leakyrelu_basic_api.sh→test/st/l1_test_ascend_msprof_matmul_leakyrelu_basic_api.sh+0-0
文件重命名但无更改。
Rtest/st/src/l0_test_ascend_msprof_matmul_basic_api.py→test/st/src/l1_test_ascend_msprof_matmul_basic_api.py+0-0
文件重命名但无更改。
Rtest/st/src/l0_test_ascend_msprof_matmul_leakyrelu_basic_api.py→test/st/src/l1_test_ascend_msprof_matmul_leakyrelu_basic_api.py+0-0
文件重命名但无更改。