| @@ -571,7 +571,7 @@ HcclResult RegisterToClusterMonitor(HcclComm comm) | |||
| 571 | } | 571 | } |
| 572 | hccl::CollComm* collComm = hcclComm->GetCollComm(); | 572 | hccl::CollComm* collComm = hcclComm->GetCollComm(); |
| 573 | CHK_PTR_NULL(collComm); | 573 | CHK_PTR_NULL(collComm); |
| 574 | - CHK_RET(CollCommMgr::GetInstance()->GetClusterMonitor(collComm->GetDeviceLogicId()).RegisterToClusterMonitor(comm)); | 574 | + CHK_RET(CollCommMgr::GetInstance().GetClusterMonitor(collComm->GetDeviceLogicId()).RegisterToClusterMonitor(comm)); |
| 575 | HCCL_INFO("%s Success", __func__); | 575 | HCCL_INFO("%s Success", __func__); |
| 576 | return HCCL_SUCCESS; | 576 | return HCCL_SUCCESS; |
| 577 | } | 577 | } |
| @@ -43,7 +43,7 @@ HcclResult HcomSetAttachedStream(const char* group, u32 graphId, const rtStream_ | |||
| 43 | } | 43 | } |
| 44 | void* attachedStream = const_cast<void*>(static_cast<const void*>(stream[0])); | 44 | void* attachedStream = const_cast<void*>(static_cast<const void*>(stream[0])); |
| 45 | s32 deviceLogicId = Hccl::HrtGetDevice(); | 45 | s32 deviceLogicId = Hccl::HrtGetDevice(); |
| 46 | - auto& mgr = hccl::CollCommMgr::GetInstance()->GetOrderLaunchThreadMgr(deviceLogicId); | 46 | + auto& mgr = hccl::CollCommMgr::GetInstance().GetOrderLaunchThreadMgr(deviceLogicId); |
| 47 | return mgr.SetAttachedStream(std::string(group), graphId, attachedStream); | 47 | return mgr.SetAttachedStream(std::string(group), graphId, attachedStream); |
| 48 | }()); | 48 | }()); |
| 49 | 49 | ||
| @@ -370,7 +370,7 @@ HcclResult HcclDedicatedThreadAcquire( | |||
| 370 | /* 保序场景:委托给 OrderLaunchThreadMgr(进程粒度) */ | 370 | /* 保序场景:委托给 OrderLaunchThreadMgr(进程粒度) */ |
| 371 | if (ORDER_LAUNCH_TYPES.find(useType) != ORDER_LAUNCH_TYPES.end()) { | 371 | if (ORDER_LAUNCH_TYPES.find(useType) != ORDER_LAUNCH_TYPES.end()) { |
| 372 | s32 deviceLogicId = Hccl::HrtGetDevice(); | 372 | s32 deviceLogicId = Hccl::HrtGetDevice(); |
| 373 | - auto& resMgr = hccl::CollCommMgr::GetInstance()->GetOrderLaunchThreadMgr(deviceLogicId); | 373 | + auto& resMgr = hccl::CollCommMgr::GetInstance().GetOrderLaunchThreadMgr(deviceLogicId); |
| 374 | ThreadHandle th = 0; | 374 | ThreadHandle th = 0; |
| 375 | HcclResult ret = resMgr.OrderLaunchThreadAcquire(useType, collComm, commId, notifyNumPerThread, th); | 375 | HcclResult ret = resMgr.OrderLaunchThreadAcquire(useType, collComm, commId, notifyNumPerThread, th); |
| 376 | CHK_PRT_RET( | 376 | CHK_PRT_RET( |
| @@ -10,7 +10,6 @@ | |||
| 10 | 10 | ||
| 11 | 11 | ||
| 12 | 12 | ||
| 13 | - | ||
| 14 | 13 | ||
| 15 | 14 | ||
| 16 | 15 | ||
| @@ -54,7 +53,6 @@ CollComm::~CollComm() | |||
| 54 | 53 | ||
| 55 | // 兜底释放所有team的syncMem本地内存 | 54 | // 兜底释放所有team的syncMem本地内存 |
| 56 | HcclTeamMgr::GetInstance().ClearByCollComm(this); | 55 | HcclTeamMgr::GetInstance().ClearByCollComm(this); |
| 57 | - CollCommMgr::GetInstance()->UnRegisteCollComm(this); | ||
| 58 | HCCL_INFO("[CollComm][~CollComm] collComm deinit"); | 56 | HCCL_INFO("[CollComm][~CollComm] collComm deinit"); |
| 59 | // dpu的兜底上报 - 异常退出时捕获异常避免二次崩溃 | 57 | // dpu的兜底上报 - 异常退出时捕获异常避免二次崩溃 |
| 60 | if (hcclCommDfx_ != nullptr) { | 58 | if (hcclCommDfx_ != nullptr) { |
| @@ -331,7 +329,6 @@ HcclResult CollComm::UpdateSymmetricRemoteMem( | |||
| 331 | HcclResult CollComm::InitKfcAndRegisterCollComm() | 329 | HcclResult CollComm::InitKfcAndRegisterCollComm() |
| 332 | { | 330 | { |
| 333 | myRank_->SetKfcControlTransfer(kfcControlTransferH2D_, kfcStatusTransferD2H_); | 331 | myRank_->SetKfcControlTransfer(kfcControlTransferH2D_, kfcStatusTransferD2H_); |
| 334 | - CollCommMgr::GetInstance()->RegisteCollComm(this); | ||
| 335 | commStatus_ = HcclCommStatus::HCCL_COMM_STATUS_READY; | 332 | commStatus_ = HcclCommStatus::HCCL_COMM_STATUS_READY; |
| 336 | return HCCL_SUCCESS; | 333 | return HCCL_SUCCESS; |
| 337 | } | 334 | } |
| @@ -77,6 +77,9 @@ public: | |||
| 77 | // 获取devicelogicId | 77 | // 获取devicelogicId |
| 78 | s32 GetDeviceLogicId() const { return deviceLogicId_; } | 78 | s32 GetDeviceLogicId() const { return deviceLogicId_; } |
| 79 | 79 | ||
| 80 | + // 是否为全功能模式(fullMode),供外部 owner 在注册/注销前判断是否需要管理 | ||
| 81 | + bool IsFullMode() const { return initMode_ == CollCommInitMode::fullMode; } | ||
| 82 | + | ||
| 80 | // 获取Rank数量 | 83 | // 获取Rank数量 |
| 81 | uint32_t GetRankSize() const | 84 | uint32_t GetRankSize() const |
| 82 | { | 85 | { |
| @@ -142,10 +145,6 @@ private: | |||
| 142 | HcclResult InitFullMode(void* rankGraph, aclrtBinHandle binHandle, HcclMem cclBuffer, uint32_t opExpansionMode); | 145 | HcclResult InitFullMode(void* rankGraph, aclrtBinHandle binHandle, HcclMem cclBuffer, uint32_t opExpansionMode); |
| 143 | HcclResult InitSimpleMode(void* rankGraph, aclrtBinHandle binHandle, HcclMem cclBuffer, uint32_t opExpansionMode); | 146 | HcclResult InitSimpleMode(void* rankGraph, aclrtBinHandle binHandle, HcclMem cclBuffer, uint32_t opExpansionMode); |
| 144 | 147 | ||
| 145 | - /* A2/A3:使用simpleMode兼容模式没有CommV2,使用简化版的CollComm代理rankgraph、myrank对象,其他功能暂不实现 | ||
| 146 | - * A5&&下一代:使用fullMode全功能collComm模式 | ||
| 147 | - */ | ||
| 148 | - bool IsFullMode() const { return initMode_ == CollCommInitMode::fullMode; } | ||
| 149 | HcclResult HcclBinaryUnLoad(); | 148 | HcclResult HcclBinaryUnLoad(); |
| 150 | 149 | ||
| 151 | void* comm_{nullptr}; | 150 | void* comm_{nullptr}; |
| @@ -8,20 +8,25 @@ | |||
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | - | ||
| 12 | 11 | ||
| 12 | + | ||
| 13 | + | ||
| 13 | 14 | ||
| 14 | namespace hccl { | 15 | namespace hccl { |
| 15 | 16 | ||
| 16 | -CollCommMgr* CollCommMgr::instance_ = nullptr; | 17 | +CollCommMgr& CollCommMgr::GetInstance() |
| 17 | -static std::once_flag instanceFlag; | ||
| 18 | - | ||
| 19 | -CollCommMgr* CollCommMgr::GetInstance() | ||
| 20 | { | 18 | { |
| 21 | - std::call_once(instanceFlag, [&] { | 19 | + static CollCommMgr instance; |
| 22 | - instance_ = new CollCommMgr(); | 20 | + return instance; |
| 23 | - }); | 21 | +} |
| 24 | - return instance_; | 22 | + |
| 23 | +CollCommMgr::~CollCommMgr() | ||
复检实读 head a41f1e32 代码,确认 UAF 风险仍在,作者回复"暂不修正"未给出实质理由,补发再拒绝。 【当前代码】coll_comm_mgr.cc:23-30 析构函数: CollCommMgr::~CollCommMgr() { HCCL_INFO("[CollCommMgr][~CollCommMgr] destruct begin."); for (auto &monitor : clusterMonitor_) { (void)monitor.DeInit(); } HCCL_INFO("[CollCommMgr][~CollCommMgr] destruct end."); } 【UAF风险实锤】本 PR 将单例从指针模式(new 不 delete,永不析构)改为 Meyers singleton(static local),析构时机为进程退出。析构函数内:
【旧代码更安全】merge-base 处 GetInstance() 用 std::call_once + new,永不 delete,析构函数不执行——本 PR 引入了该回归。 【建议改法】不在析构函数依赖 static 析构,改为进程退出前主动调用: (1) 提供 CollCommMgr::DeInit() 公共方法,内部遍历 clusterMonitor_ 调 DeInit(),在 HcclFinalize/hcomm 去初始化阶段主动调用; (2) 析构函数仅做 best-effort 兜底(加 isDeInit_ 原子标志防重入),不依赖外部 static(移除 HCCL_INFO 或降级为无依赖的 fprintf); (3) 或维持旧的指针单例(new 不 delete),明确注释"intentional leak to avoid static destruction fiasco"。 ![]() ![]() | |||
| 24 | +{ | ||
| 25 | + HCCL_INFO("[CollCommMgr][~CollCommMgr] destruct begin."); | ||
| 26 | + for (auto& monitor : clusterMonitor_) { | ||
| 27 | + (void)monitor.DeInit(); | ||
| 28 | + } | ||
| 29 | + HCCL_INFO("[CollCommMgr][~CollCommMgr] destruct end."); | ||
| 25 | } | 30 | } |
| 26 | 31 | ||
| 27 | hcomm::ClusterMonitor& CollCommMgr::GetClusterMonitor(s32 deviceLogicId) | 32 | hcomm::ClusterMonitor& CollCommMgr::GetClusterMonitor(s32 deviceLogicId) |
| @@ -83,7 +88,7 @@ void CollCommMgr::RegisteCollComm(CollComm* collComm) | |||
| 83 | std::lock_guard<std::mutex> lock(mutex_); | 88 | std::lock_guard<std::mutex> lock(mutex_); |
| 84 | allCollComms_[collComm->GetCommId()] = collComm; | 89 | allCollComms_[collComm->GetCommId()] = collComm; |
| 85 | // 注册到需要的地方 | 90 | // 注册到需要的地方 |
| 86 | - HcclTaskAbortHandler::GetInstance().Register(collComm); | 91 | + taskAbortHandler_.Register(collComm); |
| 87 | (void)GetOrderLaunchThreadMgr(collComm->GetDeviceLogicId()).RegisterOrderLaunch(collComm->GetCommId()); | 92 | (void)GetOrderLaunchThreadMgr(collComm->GetDeviceLogicId()).RegisterOrderLaunch(collComm->GetCommId()); |
| 88 | } | 93 | } |
| 89 | 94 | ||
| @@ -92,11 +97,69 @@ void CollCommMgr::UnRegisteCollComm(CollComm* collComm) | |||
| 92 | std::lock_guard<std::mutex> lock(mutex_); | 97 | std::lock_guard<std::mutex> lock(mutex_); |
| 93 | allCollComms_.erase(collComm->GetCommId()); | 98 | allCollComms_.erase(collComm->GetCommId()); |
| 94 | // 从通信域里面注销 | 99 | // 从通信域里面注销 |
| 95 | - HcclTaskAbortHandler::GetInstance().UnRegister(collComm); | 100 | + taskAbortHandler_.UnRegister(collComm); |
| 96 | (void)GetClusterMonitor(collComm->GetDeviceLogicId()).UnRegisterToClusterMonitor(collComm); | 101 | (void)GetClusterMonitor(collComm->GetDeviceLogicId()).UnRegisterToClusterMonitor(collComm); |
| 97 | (void)GetOrderLaunchThreadMgr(collComm->GetDeviceLogicId()).UnRegisterOrderLaunch(collComm->GetCommId()); | 102 | (void)GetOrderLaunchThreadMgr(collComm->GetDeviceLogicId()).UnRegisterOrderLaunch(collComm->GetCommId()); |
| 98 | } | 103 | } |
| 99 | 104 | ||
| 100 | -std::unordered_map<std::string, CollComm*> CollCommMgr::GetAllCollComms() { return allCollComms_; } | 105 | +const std::unordered_map<std::string, CollComm*>& CollCommMgr::GetAllCollComms() const { return allCollComms_; } |
| 106 | + | ||
| 107 | +void CollCommMgr::InitBaseCommRes(uint32_t devId) { (void)HcommResMgrInit(devId); } | ||
| 108 | + | ||
| 109 | +HcclOpInfoCtx& CollCommMgr::LegacyGetOpHcomInfo(uint32_t devId) | ||
| 110 | +{ | ||
| 111 | + if (devId >= MAX_MODULE_DEVICE_NUM + 1) { | ||
| 112 | + devId = MAX_MODULE_DEVICE_NUM; | ||
| 113 | + } | ||
| 114 | + // baseCommInited_ 无需加锁:本函数在生产路径中始终由 LegacyGetHcclExistDeviceOpInfoCtx / | ||
| 115 | + // LegacyGetHcclOpInfoCtx 在 opHcomInfosMutex_ 锁内调用 | ||
| 116 | + if (!baseCommInited_[devId]) { | ||
| 117 | + InitBaseCommRes(devId); | ||
| 118 | + baseCommInited_[devId] = true; | ||
| 119 | + } | ||
| 120 | + return opHcomInfos_[devId]; | ||
| 121 | +} | ||
| 122 | + | ||
| 123 | +HcclOpInfoCtx& CollCommMgr::LegacyGetHcclExistDeviceOpInfoCtx(s32& devId) | ||
| 124 | +{ | ||
| 125 | + std::lock_guard<std::mutex> lock(opHcomInfosMutex_); | ||
| 126 | + auto& opHcomInfo = LegacyGetOpHcomInfo(devId); | ||
| 127 | + if (!opHcomInfo.isUsed) { | ||
| 128 | + HCCL_INFO("[LegacyGetHcclOpInfoCtx] Set device, use devId[%d] ", devId); | ||
| 129 | + auto& backUpOpHcomInfo = LegacyGetOpHcomInfo(MAX_MODULE_DEVICE_NUM); | ||
| 130 | + if (backUpOpHcomInfo.isUsed) { | ||
| 131 | + devId = MAX_MODULE_DEVICE_NUM; | ||
| 132 | + HCCL_INFO("[LegacyGetHcclOpInfoCtx] Used cover bottom devId[%d]", devId); | ||
| 133 | + return backUpOpHcomInfo; | ||
| 134 | + } | ||
| 135 | + } | ||
| 136 | + | ||
| 137 | + HCCL_INFO("[LegacyGetHcclExistDeviceOpInfoCtx] use devId[%d] opHcomInfos", devId); | ||
| 138 | + opHcomInfo.isUsed = true; | ||
| 139 | + return opHcomInfo; | ||
| 140 | +} | ||
| 141 | + | ||
| 142 | +HcclOpInfoCtx& CollCommMgr::LegacyGetHcclOpInfoCtx(s32& devId) | ||
| 143 | +{ | ||
| 144 | + if (HcclGetDeviceId() == HCCL_SUCCESS) { | ||
| 145 | + return LegacyGetHcclExistDeviceOpInfoCtx(devId); | ||
| 146 | + } | ||
| 147 | + | ||
| 148 | + std::lock_guard<std::mutex> lock(opHcomInfosMutex_); | ||
| 149 | + for (u32 i = 0; i < MAX_MODULE_DEVICE_NUM; i++) { | ||
| 150 | + auto& opHcomInfo = LegacyGetOpHcomInfo(i); | ||
| 151 | + if (opHcomInfo.isUsed) { | ||
| 152 | + devId = i; | ||
| 153 | + HCCL_INFO("[LegacyGetHcclOpInfoCtx] Not set device, Used devId[%u] ", i); | ||
| 154 | + return opHcomInfo; | ||
| 155 | + } | ||
| 156 | + } | ||
| 157 | + | ||
| 158 | + devId = MAX_MODULE_DEVICE_NUM; | ||
| 159 | + auto& backUpOpHcomInfo = LegacyGetOpHcomInfo(devId); | ||
| 160 | + backUpOpHcomInfo.isUsed = true; | ||
| 161 | + HCCL_INFO("[LegacyGetHcclOpInfoCtx] Used cover bottom devId[%d]", devId); | ||
| 162 | + return backUpOpHcomInfo; | ||
| 163 | +} | ||
| 101 | 164 | ||
| 102 | } // namespace hccl | 165 | } // namespace hccl |
| @@ -18,6 +18,8 @@ | |||
| 18 | 18 | ||
| 19 | 19 | ||
| 20 | 20 | ||
| 21 | + | ||
| 22 | + | ||
| 21 | 23 | ||
| 22 | 24 | ||
| 23 | namespace hccl { | 25 | namespace hccl { |
| @@ -26,21 +28,33 @@ namespace hccl { | |||
| 26 | */ | 28 | */ |
| 27 | class CollCommMgr { | 29 | class CollCommMgr { |
| 28 | public: | 30 | public: |
| 29 | - static CollCommMgr* GetInstance(); | 31 | + static CollCommMgr& GetInstance(); |
| 30 | void RegisteCollComm(CollComm* collComm); | 32 | void RegisteCollComm(CollComm* collComm); |
| 31 | void UnRegisteCollComm(CollComm* collComm); | 33 | void UnRegisteCollComm(CollComm* collComm); |
| 32 | - std::unordered_map<std::string, CollComm*> GetAllCollComms(); | 34 | + const std::unordered_map<std::string, CollComm*>& GetAllCollComms() const; |
| 33 | hcomm::ClusterMonitor& GetClusterMonitor(s32 deviceLogicId); | 35 | hcomm::ClusterMonitor& GetClusterMonitor(s32 deviceLogicId); |
| 34 | HcclResult TryReserveCcuMsComm(s32 deviceLogicId, const std::string& commId, bool& reserved); | 36 | HcclResult TryReserveCcuMsComm(s32 deviceLogicId, const std::string& commId, bool& reserved); |
| 35 | void ReleaseCcuMsComm(s32 deviceLogicId, const std::string& commId); | 37 | void ReleaseCcuMsComm(s32 deviceLogicId, const std::string& commId); |
| 36 | OrderLaunchThreadMgr& GetOrderLaunchThreadMgr(s32 deviceLogicId); | 38 | OrderLaunchThreadMgr& GetOrderLaunchThreadMgr(s32 deviceLogicId); |
| 39 | + HcclTaskAbortHandler& GetTaskAbortHandler() { return taskAbortHandler_; } | ||
| 40 | + void InitBaseCommRes(uint32_t devId); | ||
| 41 | + ~CollCommMgr(); | ||
| 42 | + | ||
| 43 | + // 以下接口以 Legacy 前缀标记,表示用于兼容历史老接口,仅做 bug 修复与兼容维护,不再承接新特性、不再继续演进 | ||
| 44 | + HcclOpInfoCtx& LegacyGetOpHcomInfo(uint32_t devId); | ||
| 45 | + HcclOpInfoCtx& LegacyGetHcclExistDeviceOpInfoCtx(s32& devId); | ||
| 46 | + HcclOpInfoCtx& LegacyGetHcclOpInfoCtx(s32& devId); | ||
| 37 | 47 | ||
| 38 | private: | 48 | private: |
| 39 | - static CollCommMgr* instance_; | ||
| 40 | std::unordered_map<std::string, CollComm*> allCollComms_; | 49 | std::unordered_map<std::string, CollComm*> allCollComms_; |
| 41 | std::array<hcomm::ClusterMonitor, MAX_MODULE_DEVICE_NUM> clusterMonitor_; | 50 | std::array<hcomm::ClusterMonitor, MAX_MODULE_DEVICE_NUM> clusterMonitor_; |
| 42 | std::array<std::string, MAX_MODULE_DEVICE_NUM> ccuMsCommIds_{}; | 51 | std::array<std::string, MAX_MODULE_DEVICE_NUM> ccuMsCommIds_{}; |
| 43 | std::array<OrderLaunchThreadMgr, MAX_MODULE_DEVICE_NUM> orderLaunchThreadMgrs_; | 52 | std::array<OrderLaunchThreadMgr, MAX_MODULE_DEVICE_NUM> orderLaunchThreadMgrs_; |
| 53 | + HcclTaskAbortHandler taskAbortHandler_; | ||
| 54 | + | ||
| 55 | + std::array<HcclOpInfoCtx, MAX_MODULE_DEVICE_NUM + 1> opHcomInfos_; | ||
| 56 | + std::array<bool, MAX_MODULE_DEVICE_NUM + 1> baseCommInited_{}; | ||
| 57 | + std::mutex opHcomInfosMutex_; | ||
| 44 | 58 | ||
| 45 | std::mutex mutex_; | 59 | std::mutex mutex_; |
| 46 | std::mutex ccuMsCommMutex_; | 60 | std::mutex ccuMsCommMutex_; |
| @@ -0,0 +1,46 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +// 本文件仅用于 legacy ascend910 历史兼容,不再演进,不承接新特性;新增能力请落 base_comm/ 或 | ||
| 12 | +// coll_communicator_mgr/ 正式目录。 | ||
| 13 | + | ||
| 14 | + | ||
| 15 | + | ||
| 16 | + | ||
| 17 | + | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | + | ||
| 23 | +struct HcclInfoTag { | ||
| 24 | + HcclCommPtr pComm; | ||
| 25 | + hccl::HcclCommParams params; | ||
| 26 | + hccl::RankTable_t rankTable; | ||
| 27 | + bool cloudFlag = false; // cloudFlag为0即实验室场景,cloudFlag为1则为云场景 | ||
| 28 | + bool isUsed; | ||
| 29 | + std::mutex opGroupMapMutex; | ||
| 30 | + std::unordered_map<std::string, std::shared_ptr<hccl::hcclComm>> opGroup2CommMap; | ||
| 31 | + std::map<std::string, std::shared_ptr<hccl::TopoInfoDetect>> hcclCommTopoInfoDetectServer; | ||
| 32 | + std::map<std::string, std::shared_ptr<hccl::TopoInfoDetect>> hcclCommTopoInfoDetectAgent; | ||
| 33 | + HcclInfoTag() : isUsed(false) {} | ||
| 34 | + | ||
| 35 | + ~HcclInfoTag() | ||
| 36 | + { | ||
| 37 | + pComm = nullptr; | ||
| 38 | + opGroup2CommMap.clear(); | ||
| 39 | + hcclCommTopoInfoDetectServer.clear(); | ||
| 40 | + hcclCommTopoInfoDetectAgent.clear(); | ||
| 41 | + } | ||
| 42 | +}; | ||
| 43 | + | ||
| 44 | +using HcclOpInfoCtx = HcclInfoTag; | ||
| 45 | + | ||
| 46 | + | ||
| @@ -31,7 +31,7 @@ cluster_monitor/ | |||
| 31 | **目录特征**: | 31 | **目录特征**: |
| 32 | 32 | ||
| 33 | - 位于 `coll_communicator_mgr/dfx/`,隶属DFX监控类功能; | 33 | - 位于 `coll_communicator_mgr/dfx/`,隶属DFX监控类功能; |
| 34 | -- 模块自治:单例(`GetInstance(u32 deviceId)`),不依赖目录内其它文件; | 34 | +- 模块实例由 `CollCommMgr` 经 `clusterMonitor_` 数组持有,按 `GetClusterMonitor(deviceId)` 访问;不依赖目录内其它文件; |
| 35 | - 依赖外部:`ring_buffer`、`reference_map`、`hcclCommSocket`、`hccl_communicator`、`hcclCommDfx`、`coll_comm`、`log`、`comm_addr_logger` 等。 | 35 | - 依赖外部:`ring_buffer`、`reference_map`、`hcclCommSocket`、`hccl_communicator`、`hcclCommDfx`、`coll_comm`、`log`、`comm_addr_logger` 等。 |
| 36 | 36 | ||
| 37 | --- | 37 | --- |
| @@ -55,7 +55,7 @@ sequenceDiagram | |||
| 55 | rect rgb(230, 245, 255) | 55 | rect rgb(230, 245, 255) |
| 56 | Note over Caller, CM: 注册阶段 | 56 | Note over Caller, CM: 注册阶段 |
| 57 | Caller->>Mgr: RegisterToClusterMonitor(comm) | 57 | Caller->>Mgr: RegisterToClusterMonitor(comm) |
| 58 | - Mgr->>CM: GetInstance(deviceId) | 58 | + Mgr->>CM: GetClusterMonitor(deviceId) |
| 59 | CM->>CM: GetRemEndpointDescs<br/>(遍历 netLayer,收集 UID 上下文) | 59 | CM->>CM: GetRemEndpointDescs<br/>(遍历 netLayer,收集 UID 上下文) |
| 60 | CM->>CM: GetConnectRank<br/>(排序 + 双Ring) | 60 | CM->>CM: GetConnectRank<br/>(排序 + 双Ring) |
| 61 | CM->>CM: clusterLinkContext_[commId].push(...) | 61 | CM->>CM: clusterLinkContext_[commId].push(...) |
| @@ -253,7 +253,6 @@ classDiagram | |||
| 253 | +DelErrorSocket() | 253 | +DelErrorSocket() |
| 254 | +ProcessExceptionEvent() | 254 | +ProcessExceptionEvent() |
| 255 | +DeInit() | 255 | +DeInit() |
| 256 | - +GetInstance(deviceId)$ static | ||
| 257 | +GetCqeErrInfoFromTaskException(remoteLocalId, status, localEid, remoteEid, remoteInsId) | 256 | +GetCqeErrInfoFromTaskException(remoteLocalId, status, localEid, remoteEid, remoteInsId) |
| 258 | +GetErrStatusVecFromCluserMonitor() | 257 | +GetErrStatusVecFromCluserMonitor() |
| 259 | +PrintEvents(keyEvents) | 258 | +PrintEvents(keyEvents) |
| @@ -299,7 +298,6 @@ classDiagram | |||
| 299 | 298 | ||
| 300 | | 接口 | 说明 | | 299 | | 接口 | 说明 | |
| 301 | |------|------| | 300 | |------|------| |
| 302 | -| `static ClusterMonitor& GetInstance(u32 deviceId)` | 按device取模块单例(通过 `CollCommMgr` 间接持有)。 | | ||
| 303 | | `HcclResult RegisterToClusterMonitor(HcclComm comm)` | 注册一个通信域:建立UID上下文、计算Ring待连接集合、推入 `clusterLinkContext_` 等待后台建链;首次注册会启动 `MonitorThread`。 | | 301 | | `HcclResult RegisterToClusterMonitor(HcclComm comm)` | 注册一个通信域:建立UID上下文、计算Ring待连接集合、推入 `clusterLinkContext_` 等待后台建链;首次注册会启动 `MonitorThread`。 | |
| 304 | | `HcclResult UnRegisterToClusterMonitor(hccl::CollComm* collComm)` | 注销一个通信域:清空该commId在 `clusterLinkContext_`、`commIdMap_`、`monitorLinkStatusMap_`、`uid2SocketRefMap_` 中的引用计数;最后一个commId注销时触发 `DeInit`。 | | 302 | | `HcclResult UnRegisterToClusterMonitor(hccl::CollComm* collComm)` | 注销一个通信域:清空该commId在 `clusterLinkContext_`、`commIdMap_`、`monitorLinkStatusMap_`、`uid2SocketRefMap_` 中的引用计数;最后一个commId注销时触发 `DeInit`。 | |
| 305 | | `void GetCqeErrInfoFromTaskException(u32 remoteLocalId, uint16_t status, std::string localEid, std::string remoteEid, std::string remoteInsId)` | 由AICPU/CCU CQE错误回调调用,记录CQE错误并以广播形式扩散。 | | 303 | | `void GetCqeErrInfoFromTaskException(u32 remoteLocalId, uint16_t status, std::string localEid, std::string remoteEid, std::string remoteInsId)` | 由AICPU/CCU CQE错误回调调用,记录CQE错误并以广播形式扩散。 | |
| @@ -31,7 +31,7 @@ cluster_monitor/ | |||
| 31 | **Directory features**: | 31 | **Directory features**: |
| 32 | 32 | ||
| 33 | - Located in `coll_communicator_mgr/dfx/`, belongs to DFX monitoring functionality; | 33 | - Located in `coll_communicator_mgr/dfx/`, belongs to DFX monitoring functionality; |
| 34 | -- Module autonomous: singleton (`GetInstance(u32 deviceId)`), does not depend on other files in the directory; | 34 | +- Module instances held by `CollCommMgr` via `clusterMonitor_` array, accessed via `GetClusterMonitor(deviceId)`; does not depend on other files in the directory; |
| 35 | - External dependencies: `ring_buffer`, `reference_map`, `hcclCommSocket`, `hccl_communicator`, `hcclCommDfx`, `coll_comm`, `log`, `comm_addr_logger`, etc. | 35 | - External dependencies: `ring_buffer`, `reference_map`, `hcclCommSocket`, `hccl_communicator`, `hcclCommDfx`, `coll_comm`, `log`, `comm_addr_logger`, etc. |
| 36 | 36 | ||
| 37 | --- | 37 | --- |
| @@ -55,7 +55,7 @@ sequenceDiagram | |||
| 55 | rect rgb(230, 245, 255) | 55 | rect rgb(230, 245, 255) |
| 56 | Note over Caller, CM: Registration Phase | 56 | Note over Caller, CM: Registration Phase |
| 57 | Caller->>Mgr: RegisterToClusterMonitor(comm) | 57 | Caller->>Mgr: RegisterToClusterMonitor(comm) |
| 58 | - Mgr->>CM: GetInstance(deviceId) | 58 | + Mgr->>CM: GetClusterMonitor(deviceId) |
| 59 | CM->>CM: GetRemEndpointDescs<br/>(traverse netLayer, collect UID contexts) | 59 | CM->>CM: GetRemEndpointDescs<br/>(traverse netLayer, collect UID contexts) |
| 60 | CM->>CM: GetConnectRank<br/>(sort + dual Ring) | 60 | CM->>CM: GetConnectRank<br/>(sort + dual Ring) |
| 61 | CM->>CM: clusterLinkContext_[commId].push(...) | 61 | CM->>CM: clusterLinkContext_[commId].push(...) |
| @@ -253,7 +253,6 @@ classDiagram | |||
| 253 | +DelErrorSocket() | 253 | +DelErrorSocket() |
| 254 | +ProcessExceptionEvent() | 254 | +ProcessExceptionEvent() |
| 255 | +DeInit() | 255 | +DeInit() |
| 256 | - +GetInstance(deviceId)$ static | ||
| 257 | +GetCqeErrInfoFromTaskException(remoteLocalId, status, localEid, remoteEid, remoteInsId) | 256 | +GetCqeErrInfoFromTaskException(remoteLocalId, status, localEid, remoteEid, remoteInsId) |
| 258 | +GetErrStatusVecFromCluserMonitor() | 257 | +GetErrStatusVecFromCluserMonitor() |
| 259 | +PrintEvents(keyEvents) | 258 | +PrintEvents(keyEvents) |
| @@ -299,7 +298,6 @@ classDiagram | |||
| 299 | 298 | ||
| 300 | | Interface | Description | | 299 | | Interface | Description | |
| 301 | |------|------| | 300 | |------|------| |
| 302 | -| `static ClusterMonitor& GetInstance(u32 deviceId)` | Get module singleton by device (held indirectly via `CollCommMgr`). | | ||
| 303 | | `HcclResult RegisterToClusterMonitor(HcclComm comm)` | Register a communicator: build UID context, calculate Ring connection set, push into `clusterLinkContext_` waiting for background link establishment; first registration starts `MonitorThread`. | | 301 | | `HcclResult RegisterToClusterMonitor(HcclComm comm)` | Register a communicator: build UID context, calculate Ring connection set, push into `clusterLinkContext_` waiting for background link establishment; first registration starts `MonitorThread`. | |
| 304 | | `HcclResult UnRegisterToClusterMonitor(hccl::CollComm* collComm)` | Deregister a communicator: clear reference counts for that commId in `clusterLinkContext_`, `commIdMap_`, `monitorLinkStatusMap_`, `uid2SocketRefMap_`; triggers `DeInit` when the last commId is deregistered. | | 302 | | `HcclResult UnRegisterToClusterMonitor(hccl::CollComm* collComm)` | Deregister a communicator: clear reference counts for that commId in `clusterLinkContext_`, `commIdMap_`, `monitorLinkStatusMap_`, `uid2SocketRefMap_`; triggers `DeInit` when the last commId is deregistered. | |
| 305 | | `void GetCqeErrInfoFromTaskException(u32 remoteLocalId, uint16_t status, std::string localEid, std::string remoteEid, std::string remoteInsId)` | Called by the AICPU/CCU CQE error callback, records CQE errors and propagates them via broadcast. | | 303 | | `void GetCqeErrInfoFromTaskException(u32 remoteLocalId, uint16_t status, std::string localEid, std::string remoteEid, std::string remoteInsId)` | Called by the AICPU/CCU CQE error callback, records CQE errors and propagates them via broadcast. | |
| @@ -760,11 +760,11 @@ HcclResult ClusterMonitor::RegisterToClusterMonitor(HcclComm comm) | |||
| 760 | 760 | ||
| 761 | HcclResult ClusterMonitor::DeInit() | 761 | HcclResult ClusterMonitor::DeInit() |
| 762 | { | 762 | { |
| 763 | - if (isDeInit_) { | 763 | + bool expected = false; |
| 764 | + if (!isDeInit_.compare_exchange_strong(expected, true)) { | ||
| 764 | HCCL_INFO("[%s] already deinit, skip.", __func__); | 765 | HCCL_INFO("[%s] already deinit, skip.", __func__); |
| 765 | return HCCL_SUCCESS; | 766 | return HCCL_SUCCESS; |
| 766 | } | 767 | } |
| 767 | - isDeInit_ = true; | ||
| 768 | HCCL_INFO("[%s] heartbeat deinit begin.", __func__); | 768 | HCCL_INFO("[%s] heartbeat deinit begin.", __func__); |
| 769 | clusterMonitorThreadFlag_ = false; | 769 | clusterMonitorThreadFlag_ = false; |
| 770 | linkThreadRunning_ = false; | 770 | linkThreadRunning_ = false; |
| @@ -903,7 +903,7 @@ void GetCqeErrInfoFromTaskException( | |||
| 903 | return; | 903 | return; |
| 904 | } | 904 | } |
| 905 | return hccl::CollCommMgr::GetInstance() | 905 | return hccl::CollCommMgr::GetInstance() |
| 906 | - ->GetClusterMonitor(locDeviceId) | 906 | + .GetClusterMonitor(locDeviceId) |
| 907 | .GetCqeErrInfoFromTaskException(remoteLocalId, status, localEid, remoteEid, remoteInsId); | 907 | .GetCqeErrInfoFromTaskException(remoteLocalId, status, localEid, remoteEid, remoteInsId); |
| 908 | } | 908 | } |
| 909 | 909 | ||
| @@ -1018,7 +1018,7 @@ std::vector<std::string> ClusterMonitor::GetErrStatusVecFromCluserMonitor() | |||
| 1018 | 1018 | ||
| 1019 | std::vector<std::string> GetErrStatusVecFromCluserMonitor(s32 deviceLogicID) | 1019 | std::vector<std::string> GetErrStatusVecFromCluserMonitor(s32 deviceLogicID) |
| 1020 | { | 1020 | { |
| 1021 | - return hccl::CollCommMgr::GetInstance()->GetClusterMonitor(deviceLogicID).GetErrStatusVecFromCluserMonitor(); | 1021 | + return hccl::CollCommMgr::GetInstance().GetClusterMonitor(deviceLogicID).GetErrStatusVecFromCluserMonitor(); |
| 1022 | } | 1022 | } |
| 1023 | 1023 | ||
| 1024 | __attribute__((constructor)) void ClusterMonitorCallBackInit() | 1024 | __attribute__((constructor)) void ClusterMonitorCallBackInit() |
| @@ -9,6 +9,7 @@ | |||
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | 11 | ||
| 12 | + | ||
| 12 | 13 | ||
| 13 | 14 | ||
| 14 | 15 | ||
| @@ -186,7 +187,6 @@ public: | |||
| 186 | HcclResult RecvFrame(ClusterUIDType rem); | 187 | HcclResult RecvFrame(ClusterUIDType rem); |
| 187 | HcclResult ParseFrame(ClusterMonitorFrame& cmFrame, ClusterUIDType& src); | 188 | HcclResult ParseFrame(ClusterMonitorFrame& cmFrame, ClusterUIDType& src); |
| 188 | HcclResult DeInit(); | 189 | HcclResult DeInit(); |
| 189 | - static ClusterMonitor& GetInstance(u32 deviceId); | ||
| 190 | void GetCqeErrInfoFromTaskException( | 190 | void GetCqeErrInfoFromTaskException( |
| 191 | u32 remoteLocalId, uint16_t status, std::string localEid, std::string remoteEid, std::string remoteInsId); | 191 | u32 remoteLocalId, uint16_t status, std::string localEid, std::string remoteEid, std::string remoteInsId); |
| 192 | std::vector<std::string> GetErrStatusVecFromCluserMonitor(); | 192 | std::vector<std::string> GetErrStatusVecFromCluserMonitor(); |
| @@ -237,7 +237,7 @@ private: | |||
| 237 | // 防止重复初始化 | 237 | // 防止重复初始化 |
| 238 | bool initialized_ = false; | 238 | bool initialized_ = false; |
| 239 | uint32_t lostThreshold_ = 0; | 239 | uint32_t lostThreshold_ = 0; |
| 240 | - bool isDeInit_ = false; | 240 | + std::atomic<bool> isDeInit_{false}; |
| 241 | std::atomic<bool> linkThreadRunning_{false}; | 241 | std::atomic<bool> linkThreadRunning_{false}; |
| 242 | 242 | ||
| 243 | // 防止多线程同时初始化的线程锁 | 243 | // 防止多线程同时初始化的线程锁 |
| @@ -120,14 +120,6 @@ HcclTaskAbortHandler::~HcclTaskAbortHandler() | |||
| 120 | Hccl::HrtDeviceAbortRegCallBack(nullptr, nullptr, name); | 120 | Hccl::HrtDeviceAbortRegCallBack(nullptr, nullptr, name); |
| 121 | } | 121 | } |
| 122 | 122 | ||
| 123 | -HcclTaskAbortHandler& HcclTaskAbortHandler::GetInstance() | ||
| 124 | -{ | ||
| 125 | - // Leaky Singleton: 故意不释放,规避 Static Destruction Order Fiasco, | ||
| 126 | - // 确保析构回调访问 commVector_ 时单例仍有效 | ||
| 127 | - static HcclTaskAbortHandler& handler = *new HcclTaskAbortHandler(); | ||
| 128 | - return handler; | ||
| 129 | -} | ||
| 130 | - | ||
| 131 | HcclResult HcclTaskAbortHandler::Register(CollComm* communicator) | 123 | HcclResult HcclTaskAbortHandler::Register(CollComm* communicator) |
| 132 | { | 124 | { |
| 133 | std::lock_guard<std::mutex> lock(vecMutex_); | 125 | std::lock_guard<std::mutex> lock(vecMutex_); |
| @@ -29,7 +29,6 @@ class HcclTaskAbortHandler { | |||
| 29 | public: | 29 | public: |
| 30 | HcclTaskAbortHandler(); | 30 | HcclTaskAbortHandler(); |
| 31 | ~HcclTaskAbortHandler(); | 31 | ~HcclTaskAbortHandler(); |
| 32 | - static HcclTaskAbortHandler& GetInstance(); | ||
| 33 | HcclResult Register(CollComm* communicator); | 32 | HcclResult Register(CollComm* communicator); |
| 34 | HcclResult UnRegister(CollComm* communicator); | 33 | HcclResult UnRegister(CollComm* communicator); |
| 35 | 34 | ||
| @@ -246,7 +246,7 @@ HcclResult MyRank::ReserveCcuMsCommOrFallback() | |||
| 246 | } | 246 | } |
| 247 | 247 | ||
| 248 | bool reserved = false; | 248 | bool reserved = false; |
| 249 | - CHK_RET(CollCommMgr::GetInstance()->TryReserveCcuMsComm(devLogicId_, config_.GetConfigCommName(), reserved)); | 249 | + CHK_RET(CollCommMgr::GetInstance().TryReserveCcuMsComm(devLogicId_, config_.GetConfigCommName(), reserved)); |
| 250 | if (reserved) { | 250 | if (reserved) { |
| 251 | ccuMsCommReserved_ = true; | 251 | ccuMsCommReserved_ = true; |
| 252 | return HCCL_SUCCESS; | 252 | return HCCL_SUCCESS; |
| @@ -264,7 +264,7 @@ void MyRank::ReleaseCcuMsCommReservation() | |||
| 264 | if (!ccuMsCommReserved_) { | 264 | if (!ccuMsCommReserved_) { |
| 265 | return; | 265 | return; |
| 266 | } | 266 | } |
| 267 | - CollCommMgr::GetInstance()->ReleaseCcuMsComm(devLogicId_, config_.GetConfigCommName()); | 267 | + CollCommMgr::GetInstance().ReleaseCcuMsComm(devLogicId_, config_.GetConfigCommName()); |
| 268 | ccuMsCommReserved_ = false; | 268 | ccuMsCommReserved_ = false; |
| 269 | } | 269 | } |
| 270 | 270 | ||
| @@ -17,6 +17,9 @@ | |||
| 17 | 17 | ||
| 18 | 18 | ||
| 19 | 19 | ||
| 20 | + | ||
| 21 | + | ||
| 22 | + | ||
| 20 | 23 | ||
| 21 | 24 | ||
| 22 | 25 | ||
| @@ -49,6 +52,13 @@ hcclComm::hcclComm(u64 inCCLbufferSize, u64 outCCLbufferSize, std::string identi | |||
| 49 | 52 | ||
| 50 | hcclComm::~hcclComm() | 53 | hcclComm::~hcclComm() |
| 51 | { | 54 | { |
| 55 | + | ||
| 56 | + // collComm_ 为 fullMode 时由 owner(hcclComm) 负责注销,避免在 ~CollComm 中反向依赖 CollCommMgr; | ||
| 57 | + // 此时 collComm_ 尚未析构(成员析构发生在函数体之后),指针有效 | ||
| 58 | + if (collComm_ != nullptr && collComm_->IsFullMode()) { | ||
| 59 | + CollCommMgr::GetInstance().UnRegisteCollComm(collComm_.get()); | ||
| 60 | + } | ||
| 61 | + | ||
| 52 | RealeaseBarrierMemory(); | 62 | RealeaseBarrierMemory(); |
| 53 | (void)UnRegistTaskAbortHandler(); | 63 | (void)UnRegistTaskAbortHandler(); |
| 54 | BinaryUnLoad(); | 64 | BinaryUnLoad(); |
| @@ -24,6 +24,7 @@ | |||
| 24 | 24 | ||
| 25 | 25 | ||
| 26 | 26 | ||
| 27 | + | ||
| 27 | 28 | ||
| 28 | namespace hccl { | 29 | namespace hccl { |
| 29 | HcclResult hcclComm::AllReduce( | 30 | HcclResult hcclComm::AllReduce( |
| @@ -366,6 +367,9 @@ HcclResult hcclComm::InitCollComm( | |||
| 366 | return HCCL_SUCCESS; | 367 | return HCCL_SUCCESS; |
| 367 | } | 368 | } |
| 368 | 369 | ||
| 370 | + // 注册通信域到 CollCommMgr,由 owner(hcclComm) 负责注册/注销,避免 CollComm 反向依赖 CollCommMgr | ||
| 371 | + CollCommMgr::GetInstance().RegisteCollComm(collComm_.get()); | ||
| 372 | + | ||
| 369 | CHK_RET(collComm_->GetHDCommunicate( | 373 | CHK_RET(collComm_->GetHDCommunicate( |
| 370 | commAicpuParam_.kfcControlTransferH2DParams, commAicpuParam_.kfcStatusTransferD2HParams)); | 374 | commAicpuParam_.kfcControlTransferH2DParams, commAicpuParam_.kfcStatusTransferD2HParams)); |
| 371 | commAicpuParam_.userRank = collComm_->GetMyRankId(); | 375 | commAicpuParam_.userRank = collComm_->GetMyRankId(); |
| @@ -28,6 +28,9 @@ | |||
| 28 | 28 | ||
| 29 | 29 | ||
| 30 | 30 | ||
| 31 | + | ||
【软件架构】 #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU)) 拆分成host和device两个不同文件,登记到issue里后关闭pr ![]() ![]() | |||
| 32 | + | ||
| 33 | + | ||
| 31 | 34 | ||
| 32 | 35 | ||
| 33 | 36 | ||
| @@ -214,6 +217,7 @@ void HcomUnSetGroupTopoInfo(const char* group) | |||
| 214 | 217 | ||
| 215 | HcclResult HcomGetCommHandleByGroup(const char* group, HcclComm* commHandle) | 218 | HcclResult HcomGetCommHandleByGroup(const char* group, HcclComm* commHandle) |
| 216 | { | 219 | { |
| 220 | + | ||
| 217 | CHK_PTR_NULL(commHandle); | 221 | CHK_PTR_NULL(commHandle); |
| 218 | CHK_PTR_NULL(group); | 222 | CHK_PTR_NULL(group); |
| 219 | 223 | ||
| @@ -228,7 +232,8 @@ HcclResult HcomGetCommHandleByGroup(const char* group, HcclComm* commHandle) | |||
| 228 | ret); | 232 | ret); |
| 229 | 233 | ||
| 230 | // MC2单算子和动态图下发性能优化,优先查询返回 | 234 | // MC2单算子和动态图下发性能优化,优先查询返回 |
| 231 | - HcclOpInfoCtx& opBaseHcom = GetHcclExistDeviceOpInfoCtx(); | 235 | + s32 devId = HcclGetThreadDeviceId(); |
| 236 | + HcclOpInfoCtx& opBaseHcom = CollCommMgr::GetInstance().LegacyGetHcclExistDeviceOpInfoCtx(devId); | ||
| 232 | std::unique_lock<std::mutex> lock(opBaseHcom.opGroupMapMutex); | 237 | std::unique_lock<std::mutex> lock(opBaseHcom.opGroupMapMutex); |
| 233 | auto iter = opBaseHcom.opGroup2CommMap.find(std::string(group)); | 238 | auto iter = opBaseHcom.opGroup2CommMap.find(std::string(group)); |
| 234 | if (iter != opBaseHcom.opGroup2CommMap.end()) { | 239 | if (iter != opBaseHcom.opGroup2CommMap.end()) { |
| @@ -252,6 +257,7 @@ HcclResult HcomGetCommHandleByGroup(const char* group, HcclComm* commHandle) | |||
| 252 | group, deviceLogicId, HCOM_ERROR_CODE(ret)), | 257 | group, deviceLogicId, HCOM_ERROR_CODE(ret)), |
| 253 | ret); | 258 | ret); |
| 254 | *commHandle = static_cast<HcclComm>(hcclComm.get()); | 259 | *commHandle = static_cast<HcclComm>(hcclComm.get()); |
| 260 | + | ||
| 255 | return HCCL_SUCCESS; | 261 | return HCCL_SUCCESS; |
| 256 | } | 262 | } |
| 257 | 263 | ||
| @@ -17,6 +17,8 @@ | |||
| 17 | 17 | ||
| 18 | 18 | ||
| 19 | 19 | ||
| 20 | +HcclResult HcclGetDeviceId(void); | ||
| 21 | + | ||
| 20 | // Ge适配的类 | 22 | // Ge适配的类 |
| 21 | constexpr u32 SEND_RECEIVE_TASK_NUM = 20; | 23 | constexpr u32 SEND_RECEIVE_TASK_NUM = 20; |
| 22 | constexpr u32 OP_DEFAULT_TASK_NUM = 245; | 24 | constexpr u32 OP_DEFAULT_TASK_NUM = 245; |
| @@ -38,7 +38,7 @@ | |||
| 38 | 38 | ||
| 39 | 39 | ||
| 40 | 40 | ||
| 41 | -#include "hcomm_c_adpt.h" | 41 | +#include "coll_comm_mgr.h" |
| 42 | 42 | ||
| 43 | 43 | ||
| 44 | 44 | ||
| @@ -82,25 +82,8 @@ HcclResult CallMsprofReportHostApi( | |||
| 82 | } | 82 | } |
| 83 | 83 | ||
| 84 | thread_local s32 g_hcclDeviceId = INVALID_INT; | 84 | thread_local s32 g_hcclDeviceId = INVALID_INT; |
| 85 | -std::mutex g_opHcomInfosMutex{}; | ||
| 86 | std::mutex g_opHcomOneSideMutex{}; | 85 | std::mutex g_opHcomOneSideMutex{}; |
| 87 | 86 | ||
| 88 | -HcclOpInfoCtx& GetOpHcomInfo(uint32_t devId) | ||
| 89 | -{ | ||
| 90 | - if (devId >= MAX_MODULE_DEVICE_NUM + 1) { | ||
| 91 | - devId = MAX_MODULE_DEVICE_NUM; | ||
| 92 | - } | ||
| 93 | - | ||
| 94 | - | ||
| 95 | - // 临时方案:声明comm各类基础单例 | ||
| 96 | - // 仅触发空对象声明,不执行业务动作,故不会失败 | ||
| 97 | - (void)HcommResMgrInit(devId); | ||
| 98 | - | ||
| 99 | - | ||
| 100 | - static HcclOpInfoCtx g_opHcomInfos[MAX_MODULE_DEVICE_NUM + 1]; | ||
| 101 | - return g_opHcomInfos[devId]; | ||
| 102 | -} | ||
| 103 | - | ||
| 104 | HcclResult HcclGetDeviceId(void) | 87 | HcclResult HcclGetDeviceId(void) |
| 105 | { | 88 | { |
| 106 | if (g_hcclDeviceId == INVALID_INT) { | 89 | if (g_hcclDeviceId == INVALID_INT) { |
| @@ -127,49 +110,6 @@ s32 HcclGetThreadDeviceId() | |||
| 127 | return g_hcclDeviceId; | 110 | return g_hcclDeviceId; |
| 128 | } | 111 | } |
| 129 | 112 | ||
| 130 | -// 由调用者保证device id已经被set | ||
| 131 | -HcclOpInfoCtx& GetHcclExistDeviceOpInfoCtx(void) | ||
| 132 | -{ | ||
| 133 | - std::lock_guard<std::mutex> lock(g_opHcomInfosMutex); | ||
| 134 | - auto& opHcomInfo = GetOpHcomInfo(g_hcclDeviceId); | ||
| 135 | - if (!opHcomInfo.isUsed) { | ||
| 136 | - HCCL_INFO("[GetHcclOpInfoCtx] Set device, use g_hcclDeviceId[%d] ", g_hcclDeviceId); | ||
| 137 | - auto& backUpOpHcomInfo = GetOpHcomInfo(MAX_MODULE_DEVICE_NUM); | ||
| 138 | - if (backUpOpHcomInfo.isUsed) { | ||
| 139 | - g_hcclDeviceId = MAX_MODULE_DEVICE_NUM; | ||
| 140 | - HCCL_INFO("[GetHcclOpInfoCtx] Used cover bottom g_hcclDeviceId[%d]", g_hcclDeviceId); | ||
| 141 | - return backUpOpHcomInfo; | ||
| 142 | - } | ||
| 143 | - } | ||
| 144 | - | ||
| 145 | - HCCL_INFO("[GetHcclExistDeviceOpInfoCtx] use g_hcclDeviceId[%d] opHcomInfos", g_hcclDeviceId); | ||
| 146 | - opHcomInfo.isUsed = true; | ||
| 147 | - return opHcomInfo; | ||
| 148 | -} | ||
| 149 | - | ||
| 150 | -HcclOpInfoCtx& GetHcclOpInfoCtx(void) | ||
| 151 | -{ | ||
| 152 | - if (HcclGetDeviceId() == HCCL_SUCCESS) { | ||
| 153 | - return GetHcclExistDeviceOpInfoCtx(); | ||
| 154 | - } | ||
| 155 | - | ||
| 156 | - std::lock_guard<std::mutex> lock(g_opHcomInfosMutex); | ||
| 157 | - for (u32 i = 0; i < MAX_MODULE_DEVICE_NUM; i++) { | ||
| 158 | - auto& opHcomInfo = GetOpHcomInfo(i); | ||
| 159 | - if (opHcomInfo.isUsed) { | ||
| 160 | - g_hcclDeviceId = i; | ||
| 161 | - HCCL_INFO("[GetHcclOpInfoCtx] Not set device, Used g_hcclDeviceId[%u] ", i); | ||
| 162 | - return opHcomInfo; | ||
| 163 | - } | ||
| 164 | - } | ||
| 165 | - | ||
| 166 | - g_hcclDeviceId = MAX_MODULE_DEVICE_NUM; | ||
| 167 | - auto& backUpOpHcomInfo = GetOpHcomInfo(g_hcclDeviceId); | ||
| 168 | - backUpOpHcomInfo.isUsed = true; | ||
| 169 | - HCCL_INFO("[GetHcclOpInfoCtx] Used cover bottom g_hcclDeviceId[%d]", g_hcclDeviceId); | ||
| 170 | - return backUpOpHcomInfo; | ||
| 171 | -} | ||
| 172 | - | ||
| 173 | HcclResult | 113 | HcclResult |
| 174 | GetDeviceComm(uint32_t ndev, const HcclRootInfo& rootHandle, const s32 rank, const s32 logicDeviceId, HcclComm& comm) | 114 | GetDeviceComm(uint32_t ndev, const HcclRootInfo& rootHandle, const s32 rank, const s32 logicDeviceId, HcclComm& comm) |
| 175 | { | 115 | { |
| @@ -532,7 +472,7 @@ HcclResult HcclCommInitCollComm(uint32_t rank, void** commV2, const HcclCommConf | |||
| 532 | EXCEPTION_CATCH( | 472 | EXCEPTION_CATCH( |
| 533 | hcclCommPtr = make_shared<hccl::hcclComm>(cclBufferSize, cclBufferSize, commName), return HCCL_E_PTR); | 473 | hcclCommPtr = make_shared<hccl::hcclComm>(cclBufferSize, cclBufferSize, commName), return HCCL_E_PTR); |
| 534 | CommConfig commConfig(commName); | 474 | CommConfig commConfig(commName); |
| 535 | - HcclOpInfoCtx& opBaseHcom = GetHcclOpInfoCtx(); | 475 | + HcclOpInfoCtx& opBaseHcom = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 536 | CHK_RET(CheckOpBasedHcom(opBaseHcom, rank, commConfig)); | 476 | CHK_RET(CheckOpBasedHcom(opBaseHcom, rank, commConfig)); |
| 537 | 477 | ||
| 538 | void* rankGraph = nullptr; | 478 | void* rankGraph = nullptr; |
| @@ -732,6 +672,7 @@ HcclResult InitCommClusterInfo( | |||
| 732 | 672 | ||
| 733 | HcclResult HcclCommInitClusterInfoWrapper(struct hcclAsyncJob* job_) | 673 | HcclResult HcclCommInitClusterInfoWrapper(struct hcclAsyncJob* job_) |
| 734 | { | 674 | { |
| 675 | + | ||
| 735 | struct hcclCommInitRankTableAsyncJob* job = static_cast<hcclCommInitRankTableAsyncJob*>(job_); | 676 | struct hcclCommInitRankTableAsyncJob* job = static_cast<hcclCommInitRankTableAsyncJob*>(job_); |
| 736 | uint32_t rank = job->rank; | 677 | uint32_t rank = job->rank; |
| 737 | HcclComm* comm = job->initComm; | 678 | HcclComm* comm = job->initComm; |
| @@ -791,7 +732,7 @@ HcclResult HcclCommInitClusterInfoWrapper(struct hcclAsyncJob* job_) | |||
| 791 | 732 | ||
| 792 | HCCL_INFO("%s success, clusterInfoRealPath[%s].", __func__, realFilePath.c_str()); | 733 | HCCL_INFO("%s success, clusterInfoRealPath[%s].", __func__, realFilePath.c_str()); |
| 793 | 734 | ||
| 794 | - HcclOpInfoCtx& opBaseHcom = GetHcclOpInfoCtx(); | 735 | + HcclOpInfoCtx& opBaseHcom = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 795 | CHK_RET(CheckOpBasedHcom(opBaseHcom, rank, commConfig)); | 736 | CHK_RET(CheckOpBasedHcom(opBaseHcom, rank, commConfig)); |
| 796 | 737 | ||
| 797 | CHK_RET(InitCommClusterInfo(rankTableM, rank, commConfig, opBaseHcom, comm)); | 738 | CHK_RET(InitCommClusterInfo(rankTableM, rank, commConfig, opBaseHcom, comm)); |
| @@ -800,11 +741,13 @@ HcclResult HcclCommInitClusterInfoWrapper(struct hcclAsyncJob* job_) | |||
| 800 | HCCL_RUN_INFO( | 741 | HCCL_RUN_INFO( |
| 801 | "[HCCL_TRACE]%s success, take time [%lld]us, clusterInfo[%s], rank[%u], deviceLogicId[%d].", __func__, | 742 | "[HCCL_TRACE]%s success, take time [%lld]us, clusterInfo[%s], rank[%u], deviceLogicId[%d].", __func__, |
| 802 | DURATION_US(TIME_NOW() - startut), clusterInfo, rank, deviceLogicId); | 743 | DURATION_US(TIME_NOW() - startut), clusterInfo, rank, deviceLogicId); |
| 744 | + | ||
| 803 | return HCCL_SUCCESS; | 745 | return HCCL_SUCCESS; |
| 804 | } | 746 | } |
| 805 | 747 | ||
| 806 | HcclResult HcclCommInitClusterInfo(const char* clusterInfo, uint32_t rank, HcclComm* comm) | 748 | HcclResult HcclCommInitClusterInfo(const char* clusterInfo, uint32_t rank, HcclComm* comm) |
| 807 | { | 749 | { |
| 750 | + | ||
| 808 | if (hcclGroupDepth > 0) { | 751 | if (hcclGroupDepth > 0) { |
| 809 | HcclResult ret = HCCL_SUCCESS; | 752 | HcclResult ret = HCCL_SUCCESS; |
| 810 | std::shared_ptr<struct hcclCommInitRankTableAsyncJob> job; | 753 | std::shared_ptr<struct hcclCommInitRankTableAsyncJob> job; |
| @@ -866,7 +809,7 @@ HcclResult HcclCommInitClusterInfo(const char* clusterInfo, uint32_t rank, HcclC | |||
| 866 | 809 | ||
| 867 | HCCL_INFO("%s success, clusterInfoRealPath[%s].", __func__, realFilePath.c_str()); | 810 | HCCL_INFO("%s success, clusterInfoRealPath[%s].", __func__, realFilePath.c_str()); |
| 868 | 811 | ||
| 869 | - HcclOpInfoCtx& opBaseHcom = GetHcclOpInfoCtx(); | 812 | + HcclOpInfoCtx& opBaseHcom = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 870 | CHK_RET(CheckOpBasedHcom(opBaseHcom, rank, commConfig)); | 813 | CHK_RET(CheckOpBasedHcom(opBaseHcom, rank, commConfig)); |
| 871 | 814 | ||
| 872 | CHK_RET(InitCommClusterInfo(rankTableM, rank, commConfig, opBaseHcom, comm)); | 815 | CHK_RET(InitCommClusterInfo(rankTableM, rank, commConfig, opBaseHcom, comm)); |
| @@ -875,6 +818,7 @@ HcclResult HcclCommInitClusterInfo(const char* clusterInfo, uint32_t rank, HcclC | |||
| 875 | HCCL_RUN_INFO( | 818 | HCCL_RUN_INFO( |
| 876 | "[HCCL_TRACE]%s success, take time [%lld]us, clusterInfo[%s], rank[%u], deviceLogicId[%d].", __func__, | 819 | "[HCCL_TRACE]%s success, take time [%lld]us, clusterInfo[%s], rank[%u], deviceLogicId[%d].", __func__, |
| 877 | DURATION_US(TIME_NOW() - startut), clusterInfo, rank, deviceLogicId); | 820 | DURATION_US(TIME_NOW() - startut), clusterInfo, rank, deviceLogicId); |
| 821 | + | ||
| 878 | return HCCL_SUCCESS; | 822 | return HCCL_SUCCESS; |
| 879 | } | 823 | } |
| 880 | 824 | ||
| @@ -967,6 +911,7 @@ HcclCommInitClusterInfoMemConfig(const char* rankTableString, uint32_t rank, Hcc | |||
| 967 | 911 | ||
| 968 | HcclResult HcclCommInitClusterInfoConfigWrapper(struct hcclAsyncJob* job_) | 912 | HcclResult HcclCommInitClusterInfoConfigWrapper(struct hcclAsyncJob* job_) |
| 969 | { | 913 | { |
| 914 | + | ||
| 970 | struct hcclCommInitRankTableConfigAsyncJob* job = static_cast<hcclCommInitRankTableConfigAsyncJob*>(job_); | 915 | struct hcclCommInitRankTableConfigAsyncJob* job = static_cast<hcclCommInitRankTableConfigAsyncJob*>(job_); |
| 971 | uint32_t rank = job->rank; | 916 | uint32_t rank = job->rank; |
| 972 | HcclComm* comm = job->initComm; | 917 | HcclComm* comm = job->initComm; |
| @@ -1041,7 +986,7 @@ HcclResult HcclCommInitClusterInfoConfigWrapper(struct hcclAsyncJob* job_) | |||
| 1041 | 986 | ||
| 1042 | HCCL_INFO("%s success, clusterInfoRealPath[%s].", __func__, realFilePath.c_str()); | 987 | HCCL_INFO("%s success, clusterInfoRealPath[%s].", __func__, realFilePath.c_str()); |
| 1043 | 988 | ||
| 1044 | - HcclOpInfoCtx& opBaseHcom = GetHcclOpInfoCtx(); | 989 | + HcclOpInfoCtx& opBaseHcom = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 1045 | CHK_RET(CheckOpBasedHcom(opBaseHcom, rank, commConfig)); | 990 | CHK_RET(CheckOpBasedHcom(opBaseHcom, rank, commConfig)); |
| 1046 | 991 | ||
| 1047 | CHK_RET(InitCommClusterInfo(rankTableM, rank, commConfig, opBaseHcom, comm)); | 992 | CHK_RET(InitCommClusterInfo(rankTableM, rank, commConfig, opBaseHcom, comm)); |
| @@ -1053,11 +998,13 @@ HcclResult HcclCommInitClusterInfoConfigWrapper(struct hcclAsyncJob* job_) | |||
| 1053 | HCCL_RUN_INFO( | 998 | HCCL_RUN_INFO( |
| 1054 | "[HCCL_TRACE]%s success, take time [%lld]us, clusterInfo[%s], rank[%u], deviceLogicId[%d].", __func__, | 999 | "[HCCL_TRACE]%s success, take time [%lld]us, clusterInfo[%s], rank[%u], deviceLogicId[%d].", __func__, |
| 1055 | DURATION_US(TIME_NOW() - startut), clusterInfo, rank, deviceLogicId); | 1000 | DURATION_US(TIME_NOW() - startut), clusterInfo, rank, deviceLogicId); |
| 1001 | + | ||
| 1056 | return HCCL_SUCCESS; | 1002 | return HCCL_SUCCESS; |
| 1057 | } | 1003 | } |
| 1058 | 1004 | ||
| 1059 | HcclResult HcclCommInitClusterInfoConfig(const char* clusterInfo, uint32_t rank, HcclCommConfig* config, HcclComm* comm) | 1005 | HcclResult HcclCommInitClusterInfoConfig(const char* clusterInfo, uint32_t rank, HcclCommConfig* config, HcclComm* comm) |
| 1060 | { | 1006 | { |
| 1007 | + | ||
| 1061 | if (hcclGroupDepth > 0) { | 1008 | if (hcclGroupDepth > 0) { |
| 1062 | HcclResult ret = HCCL_SUCCESS; | 1009 | HcclResult ret = HCCL_SUCCESS; |
| 1063 | std::shared_ptr<struct hcclCommInitRankTableConfigAsyncJob> job; | 1010 | std::shared_ptr<struct hcclCommInitRankTableConfigAsyncJob> job; |
| @@ -1130,7 +1077,7 @@ HcclResult HcclCommInitClusterInfoConfig(const char* clusterInfo, uint32_t rank, | |||
| 1130 | 1077 | ||
| 1131 | HCCL_INFO("%s success, clusterInfoRealPath[%s].", __func__, realFilePath.c_str()); | 1078 | HCCL_INFO("%s success, clusterInfoRealPath[%s].", __func__, realFilePath.c_str()); |
| 1132 | 1079 | ||
| 1133 | - HcclOpInfoCtx& opBaseHcom = GetHcclOpInfoCtx(); | 1080 | + HcclOpInfoCtx& opBaseHcom = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 1134 | CHK_RET(CheckOpBasedHcom(opBaseHcom, rank, commConfig)); | 1081 | CHK_RET(CheckOpBasedHcom(opBaseHcom, rank, commConfig)); |
| 1135 | 1082 | ||
| 1136 | CHK_RET(InitCommClusterInfo(rankTableM, rank, commConfig, opBaseHcom, comm)); | 1083 | CHK_RET(InitCommClusterInfo(rankTableM, rank, commConfig, opBaseHcom, comm)); |
| @@ -1142,6 +1089,7 @@ HcclResult HcclCommInitClusterInfoConfig(const char* clusterInfo, uint32_t rank, | |||
| 1142 | HCCL_RUN_INFO( | 1089 | HCCL_RUN_INFO( |
| 1143 | "[HCCL_TRACE]%s success, take time [%lld]us, clusterInfo[%s], rank[%u], deviceLogicId[%d].", __func__, | 1090 | "[HCCL_TRACE]%s success, take time [%lld]us, clusterInfo[%s], rank[%u], deviceLogicId[%d].", __func__, |
| 1144 | DURATION_US(TIME_NOW() - startut), clusterInfo, rank, deviceLogicId); | 1091 | DURATION_US(TIME_NOW() - startut), clusterInfo, rank, deviceLogicId); |
| 1092 | + | ||
| 1145 | return HCCL_SUCCESS; | 1093 | return HCCL_SUCCESS; |
| 1146 | } | 1094 | } |
| 1147 | 1095 | ||
| @@ -1149,13 +1097,14 @@ HcclResult HcclCreateSubCommConfigInner( | |||
| 1149 | hccl::hcclComm* globalComm, uint32_t rankNum, uint32_t* rankIds, uint32_t subCommRankId, CommConfig& commConfig, | 1097 | hccl::hcclComm* globalComm, uint32_t rankNum, uint32_t* rankIds, uint32_t subCommRankId, CommConfig& commConfig, |
| 1150 | HcclComm* subComm) | 1098 | HcclComm* subComm) |
| 1151 | { | 1099 | { |
| 1100 | + | ||
| 1152 | HcclResult ret = HCCL_SUCCESS; | 1101 | HcclResult ret = HCCL_SUCCESS; |
| 1153 | HcclCommParams globalParams{}; | 1102 | HcclCommParams globalParams{}; |
| 1154 | RankTable_t globalRankTable{}; | 1103 | RankTable_t globalRankTable{}; |
| 1155 | CHK_RET(globalComm->GetCommParams(globalParams)); | 1104 | CHK_RET(globalComm->GetCommParams(globalParams)); |
| 1156 | CHK_RET(globalComm->GetCommRankTable(globalRankTable)); | 1105 | CHK_RET(globalComm->GetCommRankTable(globalRankTable)); |
| 1157 | 1106 | ||
| 1158 | - HcclOpInfoCtx& opBaseHcom = GetHcclOpInfoCtx(); | 1107 | + HcclOpInfoCtx& opBaseHcom = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 1159 | 1108 | ||
| 1160 | const std::string commIdentifier = commConfig.GetConfigCommName(); | 1109 | const std::string commIdentifier = commConfig.GetConfigCommName(); |
| 1161 | auto iter = opBaseHcom.opGroup2CommMap.find(commIdentifier); | 1110 | auto iter = opBaseHcom.opGroup2CommMap.find(commIdentifier); |
| @@ -1302,6 +1251,7 @@ HcclResult HcclCreateSubCommConfigInner( | |||
| 1302 | HCCL_RUN_INFO( | 1251 | HCCL_RUN_INFO( |
| 1303 | "%s success, sub commm identifier[%s], rankNum[%u], rank[%u], server[%s], device[%d].", __func__, | 1252 | "%s success, sub commm identifier[%s], rankNum[%u], rank[%u], server[%s], device[%d].", __func__, |
| 1304 | commIdentifier.c_str(), subRankTable.rankNum, subCommRankId, subParams.serverId.c_str(), subParams.logicDevId); | 1253 | commIdentifier.c_str(), subRankTable.rankNum, subCommRankId, subParams.serverId.c_str(), subParams.logicDevId); |
| 1254 | + | ||
| 1305 | return HCCL_SUCCESS; | 1255 | return HCCL_SUCCESS; |
| 1306 | } | 1256 | } |
| 1307 | 1257 | ||
| @@ -1422,6 +1372,7 @@ HcclResult HcclCreateSubCommConfig( | |||
| 1422 | 1372 | ||
| 1423 | HcclResult HcclGetRootInfo(HcclRootInfo* rootInfo) | 1373 | HcclResult HcclGetRootInfo(HcclRootInfo* rootInfo) |
| 1424 | { | 1374 | { |
| 1375 | + | ||
| 1425 | HcclUs startut = TIME_NOW(); | 1376 | HcclUs startut = TIME_NOW(); |
| 1426 | s32 deviceLogicId = 0; | 1377 | s32 deviceLogicId = 0; |
| 1427 | CHK_RET(HcclDeviceRefresh(deviceLogicId)); | 1378 | CHK_RET(HcclDeviceRefresh(deviceLogicId)); |
| @@ -1463,7 +1414,7 @@ HcclResult HcclGetRootInfo(HcclRootInfo* rootInfo) | |||
| 1463 | HCCL_E_MEMORY); | 1414 | HCCL_E_MEMORY); |
| 1464 | } | 1415 | } |
| 1465 | 1416 | ||
| 1466 | - HcclOpInfoCtx& opBaseInfo = GetHcclOpInfoCtx(); | 1417 | + HcclOpInfoCtx& opBaseInfo = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 1467 | EXCEPTION_CATCH( | 1418 | EXCEPTION_CATCH( |
| 1468 | opBaseInfo.hcclCommTopoInfoDetectServer.insert({rootHandle.identifier, topoDetectServer}), | 1419 | opBaseInfo.hcclCommTopoInfoDetectServer.insert({rootHandle.identifier, topoDetectServer}), |
| 1469 | return HCCL_E_MEMORY); | 1420 | return HCCL_E_MEMORY); |
| @@ -1471,6 +1422,7 @@ HcclResult HcclGetRootInfo(HcclRootInfo* rootInfo) | |||
| 1471 | HCCL_RUN_INFO( | 1422 | HCCL_RUN_INFO( |
| 1472 | "[HCCL_TRACE]HcclGetRootInfo success, take time [%lld]us, identifier[%s]", DURATION_US(TIME_NOW() - startut), | 1423 | "[HCCL_TRACE]HcclGetRootInfo success, take time [%lld]us, identifier[%s]", DURATION_US(TIME_NOW() - startut), |
| 1473 | rootHandle.identifier); | 1424 | rootHandle.identifier); |
| 1425 | + | ||
| 1474 | return HCCL_SUCCESS; | 1426 | return HCCL_SUCCESS; |
| 1475 | } | 1427 | } |
| 1476 | 1428 | ||
| @@ -1509,6 +1461,7 @@ HcclResult HcclGetCommName(HcclComm commHandle, char* commName) | |||
| 1509 | 1461 | ||
| 1510 | HcclResult HcclGetCommHandle(const char* commName, std::shared_ptr<hccl::hcclComm>& comm) | 1462 | HcclResult HcclGetCommHandle(const char* commName, std::shared_ptr<hccl::hcclComm>& comm) |
| 1511 | { | 1463 | { |
| 1464 | + | ||
| 1512 | CHK_PTR_NULL(commName); | 1465 | CHK_PTR_NULL(commName); |
| 1513 | std::string group(commName); | 1466 | std::string group(commName); |
| 1514 | 1467 | ||
| @@ -1521,7 +1474,7 @@ HcclResult HcclGetCommHandle(const char* commName, std::shared_ptr<hccl::hcclCom | |||
| 1521 | return HCCL_SUCCESS; | 1474 | return HCCL_SUCCESS; |
| 1522 | } | 1475 | } |
| 1523 | 1476 | ||
| 1524 | - HcclOpInfoCtx& opBaseHcom = GetHcclOpInfoCtx(); | 1477 | + HcclOpInfoCtx& opBaseHcom = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 1525 | std::unique_lock<std::mutex> lock(opBaseHcom.opGroupMapMutex); | 1478 | std::unique_lock<std::mutex> lock(opBaseHcom.opGroupMapMutex); |
| 1526 | auto iter = opBaseHcom.opGroup2CommMap.find(group); | 1479 | auto iter = opBaseHcom.opGroup2CommMap.find(group); |
| 1527 | if (iter == opBaseHcom.opGroup2CommMap.end()) { | 1480 | if (iter == opBaseHcom.opGroup2CommMap.end()) { |
| @@ -1530,11 +1483,13 @@ HcclResult HcclGetCommHandle(const char* commName, std::shared_ptr<hccl::hcclCom | |||
| 1530 | } else { | 1483 | } else { |
| 1531 | comm = iter->second; | 1484 | comm = iter->second; |
| 1532 | } | 1485 | } |
| 1486 | + | ||
| 1533 | return HCCL_SUCCESS; | 1487 | return HCCL_SUCCESS; |
| 1534 | } | 1488 | } |
| 1535 | 1489 | ||
| 1536 | HcclResult HcclCommGetHandleWithName(const char* commName, HcclComm* comm) | 1490 | HcclResult HcclCommGetHandleWithName(const char* commName, HcclComm* comm) |
| 1537 | { | 1491 | { |
| 1492 | + | ||
| 1538 | CHK_PTR_NULL(commName); | 1493 | CHK_PTR_NULL(commName); |
| 1539 | CHK_PTR_NULL(comm); | 1494 | CHK_PTR_NULL(comm); |
| 1540 | std::string group(commName); | 1495 | std::string group(commName); |
| @@ -1548,7 +1503,7 @@ HcclResult HcclCommGetHandleWithName(const char* commName, HcclComm* comm) | |||
| 1548 | return HCCL_SUCCESS; | 1503 | return HCCL_SUCCESS; |
| 1549 | } | 1504 | } |
| 1550 | 1505 | ||
| 1551 | - HcclOpInfoCtx& opBaseHcom = GetHcclOpInfoCtx(); | 1506 | + HcclOpInfoCtx& opBaseHcom = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 1552 | std::unique_lock<std::mutex> lock(opBaseHcom.opGroupMapMutex); | 1507 | std::unique_lock<std::mutex> lock(opBaseHcom.opGroupMapMutex); |
| 1553 | auto iter = opBaseHcom.opGroup2CommMap.find(group); | 1508 | auto iter = opBaseHcom.opGroup2CommMap.find(group); |
| 1554 | if (iter == opBaseHcom.opGroup2CommMap.end()) { | 1509 | if (iter == opBaseHcom.opGroup2CommMap.end()) { |
| @@ -1557,13 +1512,15 @@ HcclResult HcclCommGetHandleWithName(const char* commName, HcclComm* comm) | |||
| 1557 | } else { | 1512 | } else { |
| 1558 | *comm = static_cast<HcclComm>(iter->second.get()); | 1513 | *comm = static_cast<HcclComm>(iter->second.get()); |
| 1559 | } | 1514 | } |
| 1515 | + | ||
| 1560 | return HCCL_SUCCESS; | 1516 | return HCCL_SUCCESS; |
| 1561 | } | 1517 | } |
| 1562 | 1518 | ||
| 1563 | HcclResult HcclGetCommConnections( | 1519 | HcclResult HcclGetCommConnections( |
| 1564 | const HcclRootHandle& rootHandle, const std::string& identifier, HcclCommConnections& commConnections) | 1520 | const HcclRootHandle& rootHandle, const std::string& identifier, HcclCommConnections& commConnections) |
| 1565 | { | 1521 | { |
| 1566 | - HcclOpInfoCtx& opBaseInfo = GetHcclOpInfoCtx(); | 1522 | +#if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU)) |
| 1523 | + HcclOpInfoCtx& opBaseInfo = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); | ||
| 1567 | auto iterServer = opBaseInfo.hcclCommTopoInfoDetectServer.find(rootHandle.identifier); | 1524 | auto iterServer = opBaseInfo.hcclCommTopoInfoDetectServer.find(rootHandle.identifier); |
| 1568 | if (iterServer == opBaseInfo.hcclCommTopoInfoDetectServer.end()) { | 1525 | if (iterServer == opBaseInfo.hcclCommTopoInfoDetectServer.end()) { |
| 1569 | commConnections.isRoot = false; | 1526 | commConnections.isRoot = false; |
| @@ -1579,15 +1536,17 @@ HcclResult HcclGetCommConnections( | |||
| 1579 | } else { | 1536 | } else { |
| 1580 | CHK_RET(iterAgent->second->GetAgentConnection(commConnections.agentConnection)); | 1537 | CHK_RET(iterAgent->second->GetAgentConnection(commConnections.agentConnection)); |
| 1581 | } | 1538 | } |
| 1539 | + | ||
| 1582 | return HCCL_SUCCESS; | 1540 | return HCCL_SUCCESS; |
| 1583 | } | 1541 | } |
| 1584 | 1542 | ||
| 1585 | void HcclCloseCommConnections(const std::string& identifier) | 1543 | void HcclCloseCommConnections(const std::string& identifier) |
| 1586 | { | 1544 | { |
| 1587 | - HcclOpInfoCtx& opBaseInfo = GetHcclOpInfoCtx(); | 1545 | +#if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU)) |
| 1546 | + HcclOpInfoCtx& opBaseInfo = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); | ||
| 1588 | EXCEPTION_CATCH(opBaseInfo.hcclCommTopoInfoDetectServer.erase(identifier), return); | 1547 | EXCEPTION_CATCH(opBaseInfo.hcclCommTopoInfoDetectServer.erase(identifier), return); |
| 1589 | EXCEPTION_CATCH(opBaseInfo.hcclCommTopoInfoDetectAgent.erase(identifier), return); | 1548 | EXCEPTION_CATCH(opBaseInfo.hcclCommTopoInfoDetectAgent.erase(identifier), return); |
| 1590 | - return; | 1549 | +#endif |
| 1591 | } | 1550 | } |
| 1592 | 1551 | ||
| 1593 | HcclResult SetupHierarchical( | 1552 | HcclResult SetupHierarchical( |
| @@ -1770,10 +1729,11 @@ HcclResult GetTopoDetectInfo( | |||
| 1770 | HcclResult InitCommRootInfo( | 1729 | HcclResult InitCommRootInfo( |
| 1771 | const u32 nRanks, const u32 rank, const HcclRootHandle& rootHandle, const CommConfig& commConfig, HcclComm* comm) | 1730 | const u32 nRanks, const u32 rank, const HcclRootHandle& rootHandle, const CommConfig& commConfig, HcclComm* comm) |
| 1772 | { | 1731 | { |
| 1732 | + | ||
| 1773 | HcclResult ret = HCCL_SUCCESS; | 1733 | HcclResult ret = HCCL_SUCCESS; |
| 1774 | bool errorFlag = false; | 1734 | bool errorFlag = false; |
| 1775 | std::shared_ptr<hccl::hcclComm> pComm; | 1735 | std::shared_ptr<hccl::hcclComm> pComm; |
| 1776 | - HcclOpInfoCtx& opBaseHcom = GetHcclOpInfoCtx(); | 1736 | + HcclOpInfoCtx& opBaseHcom = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 1777 | const std::string commIdentifier = commConfig.GetConfigCommName(); | 1737 | const std::string commIdentifier = commConfig.GetConfigCommName(); |
| 1778 | auto iter = opBaseHcom.opGroup2CommMap.find(commIdentifier); | 1738 | auto iter = opBaseHcom.opGroup2CommMap.find(commIdentifier); |
| 1779 | CHK_PRT_RET( | 1739 | CHK_PRT_RET( |
| @@ -2030,6 +1990,7 @@ HcclResult InitCommRootInfo( | |||
| 2030 | "logicDevId[%d]", | 1990 | "logicDevId[%d]", |
| 2031 | nRanks, rank, commIdentifier.c_str(), params.serverId.c_str(), params.logicDevId); | 1991 | nRanks, rank, commIdentifier.c_str(), params.serverId.c_str(), params.logicDevId); |
| 2032 | 1992 | ||
| 1993 | + | ||
| 2033 | return HCCL_SUCCESS; | 1994 | return HCCL_SUCCESS; |
| 2034 | } | 1995 | } |
| 2035 | 1996 | ||
| @@ -2329,6 +2290,7 @@ HcclResult HcclCommInitRootInfoConfig( | |||
| 2329 | 2290 | ||
| 2330 | HcclResult HcclSetConfig(HcclConfig config, HcclConfigValue configValue) | 2291 | HcclResult HcclSetConfig(HcclConfig config, HcclConfigValue configValue) |
| 2331 | { | 2292 | { |
| 2293 | + | ||
| 2332 | if (config == HCCL_DETERMINISTIC) { | 2294 | if (config == HCCL_DETERMINISTIC) { |
| 2333 | 2295 | ||
| 2334 | HCCLV2_FUNC_RUN(HcclSetConfigV2(config, configValue)); | 2296 | HCCLV2_FUNC_RUN(HcclSetConfigV2(config, configValue)); |
| @@ -2358,12 +2320,13 @@ HcclResult HcclSetConfig(HcclConfig config, HcclConfigValue configValue) | |||
| 2358 | HCCL_WARNING("[HcclSetConfig] HCCL_DETERMINISTIC has been set by Env, so will not be reset again"); | 2320 | HCCL_WARNING("[HcclSetConfig] HCCL_DETERMINISTIC has been set by Env, so will not be reset again"); |
| 2359 | return HCCL_SUCCESS; | 2321 | return HCCL_SUCCESS; |
| 2360 | } | 2322 | } |
| 2361 | - HcclOpInfoCtx& opBaseInfo = GetHcclOpInfoCtx(); | 2323 | + HcclOpInfoCtx& opBaseInfo = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 2362 | // 遍历所有的通信域设置其确定性计算配置参数 | 2324 | // 遍历所有的通信域设置其确定性计算配置参数 |
| 2363 | for (auto it = opBaseInfo.opGroup2CommMap.begin(); it != opBaseInfo.opGroup2CommMap.end(); it++) { | 2325 | for (auto it = opBaseInfo.opGroup2CommMap.begin(); it != opBaseInfo.opGroup2CommMap.end(); it++) { |
| 2364 | CHK_RET(it->second->SetDeterministicConfig(configValue.value)); | 2326 | CHK_RET(it->second->SetDeterministicConfig(configValue.value)); |
| 2365 | } | 2327 | } |
| 2366 | } | 2328 | } |
| 2329 | + | ||
| 2367 | return HCCL_SUCCESS; | 2330 | return HCCL_SUCCESS; |
| 2368 | } | 2331 | } |
| 2369 | 2332 | ||
| @@ -3541,7 +3504,7 @@ HcclResult HcclOneSidedCommDestroy(HcclComm comm, s32 deviceLogicId, HcclUs star | |||
| 3541 | 3504 | ||
| 3542 | return HCCL_SUCCESS; | 3505 | return HCCL_SUCCESS; |
| 3543 | } | 3506 | } |
| 3544 | - | 3507 | +#if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU)) |
| 3545 | static HcclResult ResetDevice(hccl::hcclComm* hcclComm) | 3508 | static HcclResult ResetDevice(hccl::hcclComm* hcclComm) |
| 3546 | { | 3509 | { |
| 3547 | s32 logicDeviceId = 0; | 3510 | s32 logicDeviceId = 0; |
| @@ -3556,9 +3519,11 @@ static HcclResult ResetDevice(hccl::hcclComm* hcclComm) | |||
| 3556 | } | 3519 | } |
| 3557 | return HCCL_SUCCESS; | 3520 | return HCCL_SUCCESS; |
| 3558 | } | 3521 | } |
| 3522 | + | ||
| 3559 | 3523 | ||
| 3560 | HcclResult HcclCommDestroyWrapper(struct hcclAsyncJob* job_) | 3524 | HcclResult HcclCommDestroyWrapper(struct hcclAsyncJob* job_) |
| 3561 | { | 3525 | { |
| 3526 | + | ||
| 3562 | struct hcclCommDestroyAsyncJob* job = static_cast<hcclCommDestroyAsyncJob*>(job_); | 3527 | struct hcclCommDestroyAsyncJob* job = static_cast<hcclCommDestroyAsyncJob*>(job_); |
| 3563 | HcclComm comm = job->initComm; | 3528 | HcclComm comm = job->initComm; |
| 3564 | s32 devId = job->devId; | 3529 | s32 devId = job->devId; |
| @@ -3581,7 +3546,7 @@ HcclResult HcclCommDestroyWrapper(struct hcclAsyncJob* job_) | |||
| 3581 | CHK_RET(HcclCommDestroyV2( | 3546 | CHK_RET(HcclCommDestroyV2( |
| 3582 | commV2)); // 临时处理,dpustream的销毁要在其他资源销毁前完成。待新方案CpuThread上库后,原dpuStream删除可以恢复顺序 | 3547 | commV2)); // 临时处理,dpustream的销毁要在其他资源销毁前完成。待新方案CpuThread上库后,原dpuStream删除可以恢复顺序 |
| 3583 | string group = hcclComm->GetIdentifier(); | 3548 | string group = hcclComm->GetIdentifier(); |
| 3584 | - HcclOpInfoCtx& opBaseHcom = GetHcclOpInfoCtx(); | 3549 | + HcclOpInfoCtx& opBaseHcom = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 3585 | std::unique_lock<std::mutex> lock(opBaseHcom.opGroupMapMutex); | 3550 | std::unique_lock<std::mutex> lock(opBaseHcom.opGroupMapMutex); |
| 3586 | auto iter = opBaseHcom.opGroup2CommMap.find(group); | 3551 | auto iter = opBaseHcom.opGroup2CommMap.find(group); |
| 3587 | if (iter != opBaseHcom.opGroup2CommMap.end()) { | 3552 | if (iter != opBaseHcom.opGroup2CommMap.end()) { |
| @@ -3612,7 +3577,7 @@ HcclResult HcclCommDestroyWrapper(struct hcclAsyncJob* job_) | |||
| 3612 | return HcclOneSidedCommDestroy(comm, deviceLogicId, startut); | 3577 | return HcclOneSidedCommDestroy(comm, deviceLogicId, startut); |
| 3613 | } | 3578 | } |
| 3614 | 3579 | ||
| 3615 | - HcclOpInfoCtx& opBaseHcom = GetHcclOpInfoCtx(); | 3580 | + HcclOpInfoCtx& opBaseHcom = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 3616 | string group; | 3581 | string group; |
| 3617 | if (comm == opBaseHcom.pComm.get()) { | 3582 | if (comm == opBaseHcom.pComm.get()) { |
| 3618 | group = opBaseHcom.pComm->GetIdentifier(); | 3583 | group = opBaseHcom.pComm->GetIdentifier(); |
| @@ -3655,11 +3620,13 @@ HcclResult HcclCommDestroyWrapper(struct hcclAsyncJob* job_) | |||
| 3655 | HCCL_RUN_INFO( | 3620 | HCCL_RUN_INFO( |
| 3656 | "op_base comm destroy complete, take time [%lld]us, group[%s], deviceLogicId[%d].", | 3621 | "op_base comm destroy complete, take time [%lld]us, group[%s], deviceLogicId[%d].", |
| 3657 | DURATION_US(endut - startut), group.c_str(), deviceLogicId); | 3622 | DURATION_US(endut - startut), group.c_str(), deviceLogicId); |
| 3623 | + | ||
| 3658 | return HCCL_SUCCESS; | 3624 | return HCCL_SUCCESS; |
| 3659 | } | 3625 | } |
| 3660 | 3626 | ||
| 3661 | HcclResult HcclCommDestroy(HcclComm comm) | 3627 | HcclResult HcclCommDestroy(HcclComm comm) |
| 3662 | { | 3628 | { |
| 3629 | + | ||
| 3663 | if (hcclGroupDepth > 0) { | 3630 | if (hcclGroupDepth > 0) { |
| 3664 | std::shared_ptr<struct hcclCommDestroyAsyncJob> job; | 3631 | std::shared_ptr<struct hcclCommDestroyAsyncJob> job; |
| 3665 | EXCEPTION_CATCH((job = std::make_shared<struct hcclCommDestroyAsyncJob>()), return HCCL_E_PARA); | 3632 | EXCEPTION_CATCH((job = std::make_shared<struct hcclCommDestroyAsyncJob>()), return HCCL_E_PARA); |
| @@ -3687,7 +3654,7 @@ HcclResult HcclCommDestroy(HcclComm comm) | |||
| 3687 | CHK_RET(HcclCommDestroyV2( | 3654 | CHK_RET(HcclCommDestroyV2( |
| 3688 | commV2)); // 临时处理,dpustream的销毁要在其他资源销毁前完成。待新方案CpuThread上库后,原dpuStream删除可以恢复顺序 | 3655 | commV2)); // 临时处理,dpustream的销毁要在其他资源销毁前完成。待新方案CpuThread上库后,原dpuStream删除可以恢复顺序 |
| 3689 | string group = hcclComm->GetIdentifier(); | 3656 | string group = hcclComm->GetIdentifier(); |
| 3690 | - HcclOpInfoCtx& opBaseHcom = GetHcclOpInfoCtx(); | 3657 | + HcclOpInfoCtx& opBaseHcom = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 3691 | std::unique_lock<std::mutex> lock(opBaseHcom.opGroupMapMutex); | 3658 | std::unique_lock<std::mutex> lock(opBaseHcom.opGroupMapMutex); |
| 3692 | auto iter = opBaseHcom.opGroup2CommMap.find(group); | 3659 | auto iter = opBaseHcom.opGroup2CommMap.find(group); |
| 3693 | if (iter != opBaseHcom.opGroup2CommMap.end()) { | 3660 | if (iter != opBaseHcom.opGroup2CommMap.end()) { |
| @@ -3724,7 +3691,7 @@ HcclResult HcclCommDestroy(HcclComm comm) | |||
| 3724 | } | 3691 | } |
| 3725 | oneSideLock.unlock(); | 3692 | oneSideLock.unlock(); |
| 3726 | 3693 | ||
| 3727 | - HcclOpInfoCtx& opBaseHcom = GetHcclOpInfoCtx(); | 3694 | + HcclOpInfoCtx& opBaseHcom = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 3728 | string group; | 3695 | string group; |
| 3729 | if (comm == opBaseHcom.pComm.get()) { | 3696 | if (comm == opBaseHcom.pComm.get()) { |
| 3730 | group = opBaseHcom.pComm->GetIdentifier(); | 3697 | group = opBaseHcom.pComm->GetIdentifier(); |
| @@ -3767,7 +3734,7 @@ HcclResult HcclCommDestroy(HcclComm comm) | |||
| 3767 | HCCL_RUN_INFO( | 3734 | HCCL_RUN_INFO( |
| 3768 | "Entry-HcclCommDestroy op_base comm destroy complete, take time [%lld]us, group[%s], deviceLogicId[%d].", | 3735 | "Entry-HcclCommDestroy op_base comm destroy complete, take time [%lld]us, group[%s], deviceLogicId[%d].", |
| 3769 | DURATION_US(endut - startut), group.c_str(), deviceLogicId); | 3736 | DURATION_US(endut - startut), group.c_str(), deviceLogicId); |
| 3770 | - | 3737 | +#endif |
| 3771 | return HCCL_SUCCESS; | 3738 | return HCCL_SUCCESS; |
| 3772 | } | 3739 | } |
| 3773 | 3740 | ||
| @@ -22,34 +22,10 @@ | |||
| 22 | 22 | ||
| 23 | 23 | ||
| 24 | 24 | ||
| 25 | - | 25 | +#include "legacy_op_hcom_info.h" |
| 26 | -using HcclOpInfoCtx = struct HcclInfoTag { | ||
| 27 | - HcclCommPtr pComm; | ||
| 28 | - hccl::HcclCommParams params; | ||
| 29 | - hccl::RankTable_t rankTable; | ||
| 30 | - bool cloudFlag = false; // cloudFlag为0即实验室场景,cloudFlag为1则为云场景 | ||
| 31 | - bool isUsed; | ||
| 32 | - std::mutex opGroupMapMutex; | ||
| 33 | - std::unordered_map<std::string, std::shared_ptr<hccl::hcclComm>> opGroup2CommMap; | ||
| 34 | - std::map<std::string, std::shared_ptr<hccl::TopoInfoDetect>> hcclCommTopoInfoDetectServer; | ||
| 35 | - std::map<std::string, std::shared_ptr<hccl::TopoInfoDetect>> hcclCommTopoInfoDetectAgent; | ||
| 36 | - HcclInfoTag() : isUsed(false) {} | ||
| 37 | - | ||
| 38 | - ~HcclInfoTag() | ||
| 39 | - { | ||
| 40 | - pComm = nullptr; | ||
| 41 | - opGroup2CommMap.clear(); | ||
| 42 | - hcclCommTopoInfoDetectServer.clear(); | ||
| 43 | - hcclCommTopoInfoDetectAgent.clear(); | ||
| 44 | - } | ||
| 45 | -}; | ||
| 46 | 26 | ||
| 47 | constexpr uint32_t MAX_HCOM_NUM = 3U; | 27 | constexpr uint32_t MAX_HCOM_NUM = 3U; |
| 48 | 28 | ||
| 49 | -HcclOpInfoCtx& GetHcclExistDeviceOpInfoCtx(void); | ||
| 50 | - | ||
| 51 | -HcclOpInfoCtx& GetHcclOpInfoCtx(void); | ||
| 52 | - | ||
| 53 | HcclResult InitOtherInfo(hccl::HcclCommParams& params, const char* rankTable); | 29 | HcclResult InitOtherInfo(hccl::HcclCommParams& params, const char* rankTable); |
| 54 | 30 | ||
| 55 | HcclResult CallMsprofReportHostApi( | 31 | HcclResult CallMsprofReportHostApi( |
| @@ -309,6 +309,8 @@ set(UT_COMMON_INCLUDE_LIST | |||
| 309 | 309 | ||
| 310 | ${HCCL_CODE_ROOT}/src/legacy/ascend950/interface/ | 310 | ${HCCL_CODE_ROOT}/src/legacy/ascend950/interface/ |
| 311 | ${HCCL_CODE_ROOT}/src/coll_communicator_mgr/dfx | 311 | ${HCCL_CODE_ROOT}/src/coll_communicator_mgr/dfx |
| 312 | + ${HCCL_CODE_ROOT}/src/coll_communicator_mgr/dfx/cluster_monitor | ||
| 313 | + ${HCCL_CODE_ROOT}/src/coll_communicator_mgr/dfx/ns_recovery | ||
| 312 | ${HCCL_CODE_ROOT}/src/coll_communicator_mgr/dfx/profiling | 314 | ${HCCL_CODE_ROOT}/src/coll_communicator_mgr/dfx/profiling |
| 313 | ${HCCL_CODE_ROOT}/src/coll_communicator_mgr/dfx/profiling/aicpu | 315 | ${HCCL_CODE_ROOT}/src/coll_communicator_mgr/dfx/profiling/aicpu |
| 314 | ${HCCL_CODE_ROOT}/src/coll_communicator_mgr/dfx/profiling/host | 316 | ${HCCL_CODE_ROOT}/src/coll_communicator_mgr/dfx/profiling/host |
| @@ -46,6 +46,7 @@ set(src_to_test_list | |||
| 46 | ut_hccl_comm_host.cc | 46 | ut_hccl_comm_host.cc |
| 47 | ut_ind_op_transport_alloc.cc | 47 | ut_ind_op_transport_alloc.cc |
| 48 | ut_aclgraph_communicator.cc | 48 | ut_aclgraph_communicator.cc |
| 49 | + ut_coll_comm_mgr.cc | ||
| 49 | ) | 50 | ) |
| 50 | 51 | ||
| 51 | add_executable(hccl_utest_framework_communicator ${src_to_test_list}) | 52 | add_executable(hccl_utest_framework_communicator ${src_to_test_list}) |
| @@ -9,6 +9,8 @@ | |||
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | 11 | ||
| 12 | + | ||
| 13 | +extern thread_local s32 g_hcclDeviceId; | ||
| 12 | 14 | ||
| 13 | class HcclCommInitAllTest : public BaseInit { | 15 | class HcclCommInitAllTest : public BaseInit { |
| 14 | public: | 16 | public: |
| @@ -22,7 +24,7 @@ public: | |||
| 22 | void TearDown() override | 24 | void TearDown() override |
| 23 | { | 25 | { |
| 24 | // 删除所有拓扑建链的线程 | 26 | // 删除所有拓扑建链的线程 |
| 25 | - HcclOpInfoCtx& opBaseInfo = GetHcclOpInfoCtx(); | 27 | + HcclOpInfoCtx& opBaseInfo = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 26 | opBaseInfo.hcclCommTopoInfoDetectServer.clear(); | 28 | opBaseInfo.hcclCommTopoInfoDetectServer.clear(); |
| 27 | opBaseInfo.hcclCommTopoInfoDetectAgent.clear(); | 29 | opBaseInfo.hcclCommTopoInfoDetectAgent.clear(); |
| 28 | 30 | ||
| @@ -9,6 +9,8 @@ | |||
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | 11 | ||
| 12 | + | ||
| 13 | +extern thread_local s32 g_hcclDeviceId; | ||
| 12 | 14 | ||
| 13 | class HcclCommInitRootInfoConfigTest : public BaseInit { | 15 | class HcclCommInitRootInfoConfigTest : public BaseInit { |
| 14 | public: | 16 | public: |
| @@ -21,7 +23,7 @@ public: | |||
| 21 | void TearDown() override | 23 | void TearDown() override |
| 22 | { | 24 | { |
| 23 | // 删除所有拓扑建链的线程 | 25 | // 删除所有拓扑建链的线程 |
| 24 | - HcclOpInfoCtx& opBaseInfo = GetHcclOpInfoCtx(); | 26 | + HcclOpInfoCtx& opBaseInfo = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 25 | opBaseInfo.hcclCommTopoInfoDetectServer.clear(); | 27 | opBaseInfo.hcclCommTopoInfoDetectServer.clear(); |
| 26 | opBaseInfo.hcclCommTopoInfoDetectAgent.clear(); | 28 | opBaseInfo.hcclCommTopoInfoDetectAgent.clear(); |
| 27 | 29 | ||
| @@ -9,6 +9,8 @@ | |||
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | 11 | ||
| 12 | + | ||
| 13 | +extern thread_local s32 g_hcclDeviceId; | ||
| 12 | 14 | ||
| 13 | class HcclCommInitRootInfoTest : public BaseInit { | 15 | class HcclCommInitRootInfoTest : public BaseInit { |
| 14 | public: | 16 | public: |
| @@ -21,7 +23,7 @@ public: | |||
| 21 | void TearDown() override | 23 | void TearDown() override |
| 22 | { | 24 | { |
| 23 | // 删除所有拓扑建链的线程 | 25 | // 删除所有拓扑建链的线程 |
| 24 | - HcclOpInfoCtx& opBaseInfo = GetHcclOpInfoCtx(); | 26 | + HcclOpInfoCtx& opBaseInfo = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 25 | opBaseInfo.hcclCommTopoInfoDetectServer.clear(); | 27 | opBaseInfo.hcclCommTopoInfoDetectServer.clear(); |
| 26 | opBaseInfo.hcclCommTopoInfoDetectAgent.clear(); | 28 | opBaseInfo.hcclCommTopoInfoDetectAgent.clear(); |
| 27 | 29 | ||
| @@ -9,6 +9,8 @@ | |||
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | 11 | ||
| 12 | + | ||
| 13 | +extern thread_local s32 g_hcclDeviceId; | ||
| 12 | 14 | ||
| 13 | class HcclGetRootInfoTest : public BaseInit { | 15 | class HcclGetRootInfoTest : public BaseInit { |
| 14 | public: | 16 | public: |
| @@ -21,7 +23,7 @@ public: | |||
| 21 | void TearDown() override | 23 | void TearDown() override |
| 22 | { | 24 | { |
| 23 | // 删除所有拓扑建链的线程 | 25 | // 删除所有拓扑建链的线程 |
| 24 | - HcclOpInfoCtx& opBaseInfo = GetHcclOpInfoCtx(); | 26 | + HcclOpInfoCtx& opBaseInfo = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 25 | opBaseInfo.hcclCommTopoInfoDetectServer.clear(); | 27 | opBaseInfo.hcclCommTopoInfoDetectServer.clear(); |
| 26 | opBaseInfo.hcclCommTopoInfoDetectAgent.clear(); | 28 | opBaseInfo.hcclCommTopoInfoDetectAgent.clear(); |
| 27 | 29 | ||
| @@ -0,0 +1,179 @@ | |||
| 1 | +/** | ||
[MEDIUM] 测试用例覆盖不足。仅测试 GetOpHcomInfo/GetHcclExistDeviceOpInfoCtx/GetHcclOpInfoCtx 三个新增方法,未测试:~CollCommMgr() 析构逻辑、InitBaseCommRes() 行为、单例模式变更、taskAbortHandler_ 收编后的注册/注销。建议:补充 Ut_Destructor_Safe、Ut_InitBaseCommRes_CallsCorrectly、Ut_TaskAbortHandler_Register_UnRegister ![]() ![]() | |||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | + | ||
| 12 | + | ||
| 13 | + | ||
| 14 | + | ||
| 15 | + | ||
| 16 | +extern thread_local s32 g_hcclDeviceId; | ||
| 17 | + | ||
| 18 | +// InitBaseCommRes 为 HcommResMgrInit 的透传封装,捕获传入 devId 以断言透传正确 | ||
| 19 | +static u32 g_ut_capturedHcommResMgrInitDevId = 0xFFFFFFFFu; | ||
| 20 | +static HcommResult stub_CollCommMgrTest_HcommResMgrInit(uint32_t devPhyId) | ||
| 21 | +{ | ||
| 22 | + g_ut_capturedHcommResMgrInitDevId = devPhyId; | ||
| 23 | + return static_cast<HcommResult>(HCOMM_SUCCESS); | ||
| 24 | +} | ||
| 25 | + | ||
| 26 | +class CollCommMgrTest : public BaseInit { | ||
| 27 | +public: | ||
| 28 | + void SetUp() override | ||
| 29 | + { | ||
| 30 | + BaseInit::SetUp(); | ||
| 31 | + // Reset all opHcomInfos_ slots to clean state | ||
| 32 | + for (u32 i = 0; i <= MAX_MODULE_DEVICE_NUM; i++) { | ||
| 33 | + CollCommMgr::GetInstance().LegacyGetOpHcomInfo(i).isUsed = false; | ||
| 34 | + } | ||
| 35 | + g_hcclDeviceId = INVALID_INT; | ||
| 36 | + } | ||
| 37 | + | ||
| 38 | + void TearDown() override | ||
| 39 | + { | ||
| 40 | + // Clean up all slots | ||
| 41 | + for (u32 i = 0; i <= MAX_MODULE_DEVICE_NUM; i++) { | ||
| 42 | + CollCommMgr::GetInstance().LegacyGetOpHcomInfo(i).isUsed = false; | ||
| 43 | + } | ||
| 44 | + g_hcclDeviceId = INVALID_INT; | ||
| 45 | + BaseInit::TearDown(); | ||
| 46 | + GlobalMockObject::verify(); | ||
| 47 | + } | ||
| 48 | +}; | ||
| 49 | + | ||
| 50 | +// ===== LegacyGetOpHcomInfo ===== | ||
| 51 | + | ||
| 52 | +TEST_F(CollCommMgrTest, Ut_LegacyGetOpHcomInfo_When_DevIdNormal_Expect_ReturnCorrespondingSlot) | ||
| 53 | +{ | ||
| 54 | + u32 devId = 3; | ||
| 55 | + HcclOpInfoCtx& info = CollCommMgr::GetInstance().LegacyGetOpHcomInfo(devId); | ||
| 56 | + HcclOpInfoCtx& expectedSlot = CollCommMgr::GetInstance().LegacyGetOpHcomInfo(devId); | ||
| 57 | + EXPECT_EQ(&info, &expectedSlot); | ||
| 58 | + EXPECT_EQ(info.isUsed, false); | ||
| 59 | +} | ||
| 60 | + | ||
| 61 | +TEST_F(CollCommMgrTest, Ut_LegacyGetOpHcomInfo_When_DevIdOutOfRange_Expect_ClampToBackupSlot) | ||
| 62 | +{ | ||
| 63 | + u32 outOfRangeDevId = MAX_MODULE_DEVICE_NUM + 10; | ||
| 64 | + HcclOpInfoCtx& info = CollCommMgr::GetInstance().LegacyGetOpHcomInfo(outOfRangeDevId); | ||
| 65 | + HcclOpInfoCtx& backupSlot = CollCommMgr::GetInstance().LegacyGetOpHcomInfo(MAX_MODULE_DEVICE_NUM); | ||
| 66 | + EXPECT_EQ(&info, &backupSlot); | ||
| 67 | +} | ||
| 68 | + | ||
| 69 | +// ===== LegacyGetHcclExistDeviceOpInfoCtx ===== | ||
| 70 | + | ||
| 71 | +TEST_F(CollCommMgrTest, Ut_LegacyGetHcclExistDeviceOpInfoCtx_When_SlotAlreadyUsed_Expect_DirectReturnDevIdUnchanged) | ||
| 72 | +{ | ||
| 73 | + s32 devId = 2; | ||
| 74 | + CollCommMgr::GetInstance().LegacyGetOpHcomInfo(devId).isUsed = true; | ||
| 75 | + s32 originalDevId = devId; | ||
| 76 | + HcclOpInfoCtx& info = CollCommMgr::GetInstance().LegacyGetHcclExistDeviceOpInfoCtx(devId); | ||
| 77 | + EXPECT_EQ(devId, originalDevId); | ||
| 78 | + EXPECT_EQ(info.isUsed, true); | ||
| 79 | +} | ||
| 80 | + | ||
| 81 | +TEST_F(CollCommMgrTest, Ut_LegacyGetHcclExistDeviceOpInfoCtx_When_SlotNotUsedAndBackupUsed_Expect_FallbackToBackup) | ||
| 82 | +{ | ||
| 83 | + s32 devId = 3; | ||
| 84 | + CollCommMgr::GetInstance().LegacyGetOpHcomInfo(MAX_MODULE_DEVICE_NUM).isUsed = true; | ||
| 85 | + HcclOpInfoCtx& info = CollCommMgr::GetInstance().LegacyGetHcclExistDeviceOpInfoCtx(devId); | ||
| 86 | + EXPECT_EQ(devId, static_cast<s32>(MAX_MODULE_DEVICE_NUM)); | ||
| 87 | + EXPECT_EQ(info.isUsed, true); | ||
| 88 | +} | ||
| 89 | + | ||
| 90 | +TEST_F(CollCommMgrTest, Ut_LegacyGetHcclExistDeviceOpInfoCtx_When_SlotNotUsedAndBackupNotUsed_Expect_MarkAndReturn) | ||
| 91 | +{ | ||
| 92 | + s32 devId = 4; | ||
| 93 | + HcclOpInfoCtx& info = CollCommMgr::GetInstance().LegacyGetHcclExistDeviceOpInfoCtx(devId); | ||
| 94 | + EXPECT_EQ(devId, 4); | ||
| 95 | + EXPECT_EQ(info.isUsed, true); | ||
| 96 | +} | ||
| 97 | + | ||
| 98 | +// ===== LegacyGetHcclOpInfoCtx ===== | ||
| 99 | + | ||
| 100 | +TEST_F(CollCommMgrTest, Ut_LegacyGetHcclOpInfoCtx_When_HcclGetDeviceIdSuccess_Expect_DelegateToExistDevice) | ||
| 101 | +{ | ||
| 102 | + g_hcclDeviceId = 0; | ||
| 103 | + HcclOpInfoCtx& info = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); | ||
| 104 | + EXPECT_EQ(g_hcclDeviceId, 0); | ||
| 105 | + EXPECT_EQ(info.isUsed, true); | ||
| 106 | +} | ||
| 107 | + | ||
| 108 | +TEST_F(CollCommMgrTest, Ut_LegacyGetHcclOpInfoCtx_When_HcclGetDeviceIdFailAndHasUsedSlot_Expect_ScanFirstUsed) | ||
| 109 | +{ | ||
| 110 | + g_hcclDeviceId = INVALID_INT; | ||
| 111 | + MOCKER(hrtGetDevice).stubs().with(mockcpp::any()).will(returnValue(HCCL_E_INTERNAL)); | ||
| 112 | + | ||
| 113 | + CollCommMgr::GetInstance().LegacyGetOpHcomInfo(5).isUsed = true; | ||
| 114 | + HcclOpInfoCtx& info = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); | ||
| 115 | + EXPECT_EQ(g_hcclDeviceId, 5); | ||
| 116 | + EXPECT_EQ(info.isUsed, true); | ||
| 117 | +} | ||
| 118 | + | ||
| 119 | +TEST_F(CollCommMgrTest, Ut_LegacyGetHcclOpInfoCtx_When_HcclGetDeviceIdFailAndNoUsedSlot_Expect_FallbackToBackup) | ||
| 120 | +{ | ||
| 121 | + g_hcclDeviceId = INVALID_INT; | ||
| 122 | + MOCKER(hrtGetDevice).stubs().with(mockcpp::any()).will(returnValue(HCCL_E_INTERNAL)); | ||
| 123 | + | ||
| 124 | + HcclOpInfoCtx& info = CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); | ||
| 125 | + EXPECT_EQ(g_hcclDeviceId, static_cast<s32>(MAX_MODULE_DEVICE_NUM)); | ||
| 126 | + EXPECT_EQ(info.isUsed, true); | ||
| 127 | +} | ||
| 128 | + | ||
| 129 | +// ===== InitBaseCommRes ===== | ||
| 130 | +// InitBaseCommRes 仅透传调用 HcommResMgrInit(devId),验证其以传入 devId 调用底层入口。 | ||
| 131 | +// 使用 stubs + invoke 捕获参数,避免 expects(once) 受 TearDown 链路额外调用影响。 | ||
| 132 | + | ||
| 133 | +TEST_F(CollCommMgrTest, Ut_InitBaseCommRes_When_Called_Expect_InvokeHcommResMgrInitWithDevId) | ||
| 134 | +{ | ||
| 135 | + const u32 devId = 7; | ||
| 136 | + g_ut_capturedHcommResMgrInitDevId = 0xFFFFFFFFu; | ||
| 137 | + MOCKER(HcommResMgrInit).stubs().will(invoke(stub_CollCommMgrTest_HcommResMgrInit)); | ||
| 138 | + CollCommMgr::GetInstance().InitBaseCommRes(devId); | ||
| 139 | + EXPECT_EQ(g_ut_capturedHcommResMgrInitDevId, devId); | ||
| 140 | +} | ||
| 141 | + | ||
| 142 | +// ===== RegisteCollComm / UnRegisteCollComm ===== | ||
| 143 | +// taskAbortHandler_ 收编进 CollCommMgr 后,注册/注销为统一入口,同时维护 allCollComms_ 与 taskAbortHandler_。 | ||
| 144 | +// 使用 simpleMode 轻量桩验证 map 状态与往返安全:simpleMode 桩构造不解引用 null comm,析构短路返回; | ||
| 145 | +// taskAbortHandler Register/UnRegister 全程仅存指针/指针比较;UnRegisterToClusterMonitor 在 initialized_==false 下 | ||
| 146 | +// no-op。 | ||
| 147 | + | ||
| 148 | +TEST_F(CollCommMgrTest, Ut_RegisteCollComm_When_NewComm_Expect_AddedToMap) | ||
| 149 | +{ | ||
| 150 | + const std::string commId = "ut_reg_stub"; | ||
| 151 | + CollComm stub(nullptr, 0, commId, ManagerCallbacks{}, CollCommInitMode::simpleMode); | ||
| 152 | + EXPECT_EQ(CollCommMgr::GetInstance().GetAllCollComms().count(commId), 0U); | ||
| 153 | + CollCommMgr::GetInstance().RegisteCollComm(&stub); | ||
| 154 | + EXPECT_EQ(CollCommMgr::GetInstance().GetAllCollComms().count(commId), 1U); | ||
| 155 | + EXPECT_EQ(CollCommMgr::GetInstance().GetAllCollComms().at(commId), &stub); | ||
| 156 | + CollCommMgr::GetInstance().UnRegisteCollComm(&stub); | ||
| 157 | + EXPECT_EQ(CollCommMgr::GetInstance().GetAllCollComms().count(commId), 0U); | ||
| 158 | +} | ||
| 159 | + | ||
| 160 | +TEST_F(CollCommMgrTest, Ut_RegisteCollComm_When_DuplicateRegister_Expect_OverwriteNotGrow) | ||
| 161 | +{ | ||
| 162 | + const std::string commId = "ut_reg_dup"; | ||
| 163 | + CollComm stub(nullptr, 0, commId, ManagerCallbacks{}, CollCommInitMode::simpleMode); | ||
| 164 | + CollCommMgr::GetInstance().RegisteCollComm(&stub); | ||
| 165 | + CollCommMgr::GetInstance().RegisteCollComm(&stub); | ||
| 166 | + EXPECT_EQ(CollCommMgr::GetInstance().GetAllCollComms().count(commId), 1U); | ||
| 167 | + CollCommMgr::GetInstance().UnRegisteCollComm(&stub); | ||
| 168 | + EXPECT_EQ(CollCommMgr::GetInstance().GetAllCollComms().count(commId), 0U); | ||
| 169 | +} | ||
| 170 | + | ||
| 171 | +TEST_F(CollCommMgrTest, Ut_UnRegisteCollComm_When_NotRegistered_Expect_NoCrashNoChange) | ||
| 172 | +{ | ||
| 173 | + const std::string commId = "ut_unreg_noreg"; | ||
| 174 | + CollComm stub(nullptr, 0, commId, ManagerCallbacks{}, CollCommInitMode::simpleMode); | ||
| 175 | + EXPECT_EQ(CollCommMgr::GetInstance().GetAllCollComms().count(commId), 0U); | ||
| 176 | + // 未注册直接注销:map erase no-op,taskAbortHandler UnRegister 仅告警,clusterMonitor initialized_==false no-op | ||
| 177 | + CollCommMgr::GetInstance().UnRegisteCollComm(&stub); | ||
| 178 | + EXPECT_EQ(CollCommMgr::GetInstance().GetAllCollComms().count(commId), 0U); | ||
| 179 | +} | ||
| @@ -23,20 +23,22 @@ using namespace hcomm; | |||
| 23 | 23 | ||
| 24 | class HcclTaskAbortHandlerTest : public testing::Test { | 24 | class HcclTaskAbortHandlerTest : public testing::Test { |
| 25 | public: | 25 | public: |
| 26 | + HcclTaskAbortHandler handler; | ||
| 27 | + | ||
| 26 | static void SetUpTestCase() { std::cout << "HcclTaskAbortHandlerTest SetUP" << std::endl; } | 28 | static void SetUpTestCase() { std::cout << "HcclTaskAbortHandlerTest SetUP" << std::endl; } |
| 27 | 29 | ||
| 28 | static void TearDownTestCase() { std::cout << "HcclTaskAbortHandlerTest TearDown" << std::endl; } | 30 | static void TearDownTestCase() { std::cout << "HcclTaskAbortHandlerTest TearDown" << std::endl; } |
| 29 | 31 | ||
| 30 | virtual void SetUp() | 32 | virtual void SetUp() |
| 31 | { | 33 | { |
| 32 | - HcclTaskAbortHandler::GetInstance().commVector_.clear(); | 34 | + handler.commVector_.clear(); |
| 33 | std::cout << "A Test case in HcclTaskAbortHandlerTest SetUp" << std::endl; | 35 | std::cout << "A Test case in HcclTaskAbortHandlerTest SetUp" << std::endl; |
| 34 | } | 36 | } |
| 35 | 37 | ||
| 36 | virtual void TearDown() | 38 | virtual void TearDown() |
| 37 | { | 39 | { |
| 38 | std::cout << "A Test case in HcclTaskAbortHandlerTest TearDown" << std::endl; | 40 | std::cout << "A Test case in HcclTaskAbortHandlerTest TearDown" << std::endl; |
| 39 | - HcclTaskAbortHandler::GetInstance().commVector_.clear(); | 41 | + handler.commVector_.clear(); |
| 40 | GlobalMockObject::verify(); | 42 | GlobalMockObject::verify(); |
| 41 | } | 43 | } |
| 42 | }; | 44 | }; |
| @@ -50,8 +52,8 @@ TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_pre_succ | |||
| 50 | 52 | ||
| 51 | // 使用 nullptr 作为测试 communicator 的占位符并注册 | 53 | // 使用 nullptr 作为测试 communicator 的占位符并注册 |
| 52 | CollComm* comm = nullptr; | 54 | CollComm* comm = nullptr; |
| 53 | - HcclTaskAbortHandler::GetInstance().Register(comm); | 55 | + handler.Register(comm); |
| 54 | - void* args = reinterpret_cast<void*>(&HcclTaskAbortHandler::GetInstance().commVector_); | 56 | + void* args = reinterpret_cast<void*>(&handler.commVector_); |
| 55 | 57 | ||
| 56 | // 模拟 Suspend 方法返回成功 | 58 | // 模拟 Suspend 方法返回成功 |
| 57 | MOCKER_CPP(&CollComm::Suspend, HcclResult(CollComm::*)()) | 59 | MOCKER_CPP(&CollComm::Suspend, HcclResult(CollComm::*)()) |
| @@ -69,7 +71,7 @@ TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_pre_succ | |||
| 69 | EXPECT_EQ(ret, static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS)); | 71 | EXPECT_EQ(ret, static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS)); |
| 70 | 72 | ||
| 71 | // 清理 | 73 | // 清理 |
| 72 | - HcclTaskAbortHandler::GetInstance().UnRegister(comm); | 74 | + handler.UnRegister(comm); |
| 73 | } | 75 | } |
| 74 | 76 | ||
| 75 | TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_with_null_args) | 77 | TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_with_null_args) |
| @@ -87,11 +89,11 @@ TEST_F(HcclTaskAbortHandlerTest, test_unregister_not_found_comm) | |||
| 87 | CollComm* comm1 = reinterpret_cast<CollComm*>(0x1000); | 89 | CollComm* comm1 = reinterpret_cast<CollComm*>(0x1000); |
| 88 | CollComm* comm2 = reinterpret_cast<CollComm*>(0x2000); | 90 | CollComm* comm2 = reinterpret_cast<CollComm*>(0x2000); |
| 89 | 91 | ||
| 90 | - HcclTaskAbortHandler::GetInstance().Register(comm1); | 92 | + handler.Register(comm1); |
| 91 | - HcclResult ret = HcclTaskAbortHandler::GetInstance().UnRegister(comm2); | 93 | + HcclResult ret = handler.UnRegister(comm2); |
| 92 | EXPECT_EQ(ret, HCCL_SUCCESS); | 94 | EXPECT_EQ(ret, HCCL_SUCCESS); |
| 93 | 95 | ||
| 94 | - HcclTaskAbortHandler::GetInstance().UnRegister(comm1); | 96 | + handler.UnRegister(comm1); |
| 95 | } | 97 | } |
| 96 | 98 | ||
| 97 | TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_pre_suspending) | 99 | TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_pre_suspending) |
| @@ -101,15 +103,15 @@ TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_pre_susp | |||
| 101 | uint32_t timeout = 0U; | 103 | uint32_t timeout = 0U; |
| 102 | 104 | ||
| 103 | CollComm* comm = nullptr; | 105 | CollComm* comm = nullptr; |
| 104 | - HcclTaskAbortHandler::GetInstance().Register(comm); | 106 | + handler.Register(comm); |
| 105 | - void* args = reinterpret_cast<void*>(&HcclTaskAbortHandler::GetInstance().commVector_); | 107 | + void* args = reinterpret_cast<void*>(&handler.commVector_); |
| 106 | 108 | ||
| 107 | MOCKER_CPP(&CollComm::Suspend, HcclResult(CollComm::*)()).stubs().will(returnValue(HCCL_E_SUSPENDING)); | 109 | MOCKER_CPP(&CollComm::Suspend, HcclResult(CollComm::*)()).stubs().will(returnValue(HCCL_E_SUSPENDING)); |
| 108 | 110 | ||
| 109 | auto ret = ProcessTaskAbortHandleCallback(deviceLogicId, stage, timeout, args); | 111 | auto ret = ProcessTaskAbortHandleCallback(deviceLogicId, stage, timeout, args); |
| 110 | EXPECT_EQ(ret, static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS)); | 112 | EXPECT_EQ(ret, static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS)); |
| 111 | 113 | ||
| 112 | - HcclTaskAbortHandler::GetInstance().UnRegister(comm); | 114 | + handler.UnRegister(comm); |
| 113 | } | 115 | } |
| 114 | 116 | ||
| 115 | TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_post_suspending) | 117 | TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_post_suspending) |
| @@ -119,8 +121,8 @@ TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_post_sus | |||
| 119 | uint32_t timeout = 0U; | 121 | uint32_t timeout = 0U; |
| 120 | 122 | ||
| 121 | CollComm* comm = nullptr; | 123 | CollComm* comm = nullptr; |
| 122 | - HcclTaskAbortHandler::GetInstance().Register(comm); | 124 | + handler.Register(comm); |
| 123 | - void* args = reinterpret_cast<void*>(&HcclTaskAbortHandler::GetInstance().commVector_); | 125 | + void* args = reinterpret_cast<void*>(&handler.commVector_); |
| 124 | 126 | ||
| 125 | MOCKER(CcuIsInited).stubs().will(returnValue(true)); | 127 | MOCKER(CcuIsInited).stubs().will(returnValue(true)); |
| 126 | MOCKER(CcuSetTaskKill).stubs().will(returnValue(HCCL_SUCCESS)); | 128 | MOCKER(CcuSetTaskKill).stubs().will(returnValue(HCCL_SUCCESS)); |
| @@ -131,7 +133,7 @@ TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_post_sus | |||
| 131 | auto ret = ProcessTaskAbortHandleCallback(deviceLogicId, stage, timeout, args); | 133 | auto ret = ProcessTaskAbortHandleCallback(deviceLogicId, stage, timeout, args); |
| 132 | EXPECT_EQ(ret, static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS)); | 134 | EXPECT_EQ(ret, static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS)); |
| 133 | 135 | ||
| 134 | - HcclTaskAbortHandler::GetInstance().UnRegister(comm); | 136 | + handler.UnRegister(comm); |
| 135 | } | 137 | } |
| 136 | 138 | ||
| 137 | TEST_F(HcclTaskAbortHandlerTest, test_task_abort_post_ccu_set_task_kill_fail) | 139 | TEST_F(HcclTaskAbortHandlerTest, test_task_abort_post_ccu_set_task_kill_fail) |
| @@ -141,8 +143,8 @@ TEST_F(HcclTaskAbortHandlerTest, test_task_abort_post_ccu_set_task_kill_fail) | |||
| 141 | uint32_t timeout = 0U; | 143 | uint32_t timeout = 0U; |
| 142 | 144 | ||
| 143 | CollComm* comm = nullptr; | 145 | CollComm* comm = nullptr; |
| 144 | - HcclTaskAbortHandler::GetInstance().Register(comm); | 146 | + handler.Register(comm); |
| 145 | - void* args = reinterpret_cast<void*>(&HcclTaskAbortHandler::GetInstance().commVector_); | 147 | + void* args = reinterpret_cast<void*>(&handler.commVector_); |
| 146 | 148 | ||
| 147 | MOCKER(CcuIsInited).stubs().will(returnValue(true)); | 149 | MOCKER(CcuIsInited).stubs().will(returnValue(true)); |
| 148 | MOCKER(CcuSetTaskKill).stubs().will(returnValue(HCCL_E_INTERNAL)); | 150 | MOCKER(CcuSetTaskKill).stubs().will(returnValue(HCCL_E_INTERNAL)); |
| @@ -150,7 +152,7 @@ TEST_F(HcclTaskAbortHandlerTest, test_task_abort_post_ccu_set_task_kill_fail) | |||
| 150 | auto ret = ProcessTaskAbortHandleCallback(deviceLogicId, stage, timeout, args); | 152 | auto ret = ProcessTaskAbortHandleCallback(deviceLogicId, stage, timeout, args); |
| 151 | EXPECT_NE(ret, static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS)); | 153 | EXPECT_NE(ret, static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS)); |
| 152 | 154 | ||
| 153 | - HcclTaskAbortHandler::GetInstance().UnRegister(comm); | 155 | + handler.UnRegister(comm); |
| 154 | } | 156 | } |
| 155 | 157 | ||
| 156 | TEST_F(HcclTaskAbortHandlerTest, test_register_and_unregister_multiple) | 158 | TEST_F(HcclTaskAbortHandlerTest, test_register_and_unregister_multiple) |
| @@ -158,20 +160,20 @@ TEST_F(HcclTaskAbortHandlerTest, test_register_and_unregister_multiple) | |||
| 158 | CollComm* comm1 = reinterpret_cast<CollComm*>(0x1000); | 160 | CollComm* comm1 = reinterpret_cast<CollComm*>(0x1000); |
| 159 | CollComm* comm2 = reinterpret_cast<CollComm*>(0x2000); | 161 | CollComm* comm2 = reinterpret_cast<CollComm*>(0x2000); |
| 160 | 162 | ||
| 161 | - HcclResult ret = HcclTaskAbortHandler::GetInstance().Register(comm1); | 163 | + HcclResult ret = handler.Register(comm1); |
| 162 | EXPECT_EQ(ret, HCCL_SUCCESS); | 164 | EXPECT_EQ(ret, HCCL_SUCCESS); |
| 163 | - ret = HcclTaskAbortHandler::GetInstance().Register(comm2); | 165 | + ret = handler.Register(comm2); |
| 164 | EXPECT_EQ(ret, HCCL_SUCCESS); | 166 | EXPECT_EQ(ret, HCCL_SUCCESS); |
| 165 | 167 | ||
| 166 | - EXPECT_EQ(HcclTaskAbortHandler::GetInstance().commVector_.size(), 2u); | 168 | + EXPECT_EQ(handler.commVector_.size(), 2u); |
| 167 | 169 | ||
| 168 | - ret = HcclTaskAbortHandler::GetInstance().UnRegister(comm1); | 170 | + ret = handler.UnRegister(comm1); |
| 169 | EXPECT_EQ(ret, HCCL_SUCCESS); | 171 | EXPECT_EQ(ret, HCCL_SUCCESS); |
| 170 | - EXPECT_EQ(HcclTaskAbortHandler::GetInstance().commVector_.size(), 1u); | 172 | + EXPECT_EQ(handler.commVector_.size(), 1u); |
| 171 | 173 | ||
| 172 | - ret = HcclTaskAbortHandler::GetInstance().UnRegister(comm2); | 174 | + ret = handler.UnRegister(comm2); |
| 173 | EXPECT_EQ(ret, HCCL_SUCCESS); | 175 | EXPECT_EQ(ret, HCCL_SUCCESS); |
| 174 | - EXPECT_EQ(HcclTaskAbortHandler::GetInstance().commVector_.size(), 0u); | 176 | + EXPECT_EQ(handler.commVector_.size(), 0u); |
| 175 | } | 177 | } |
| 176 | 178 | ||
| 177 | TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_pre_fail) | 179 | TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_pre_fail) |
| @@ -183,8 +185,8 @@ TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_pre_fail | |||
| 183 | 185 | ||
| 184 | // 使用 nullptr 作为测试 communicator 的占位符并注册 | 186 | // 使用 nullptr 作为测试 communicator 的占位符并注册 |
| 185 | CollComm* comm = nullptr; | 187 | CollComm* comm = nullptr; |
| 186 | - HcclTaskAbortHandler::GetInstance().Register(comm); | 188 | + handler.Register(comm); |
| 187 | - void* args = reinterpret_cast<void*>(&HcclTaskAbortHandler::GetInstance().commVector_); | 189 | + void* args = reinterpret_cast<void*>(&handler.commVector_); |
| 188 | 190 | ||
| 189 | // 模拟 Suspend 方法返回失败 | 191 | // 模拟 Suspend 方法返回失败 |
| 190 | MOCKER_CPP(&CollComm::Suspend, HcclResult(CollComm::*)()) | 192 | MOCKER_CPP(&CollComm::Suspend, HcclResult(CollComm::*)()) |
| @@ -202,7 +204,7 @@ TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_pre_fail | |||
| 202 | EXPECT_EQ(ret, static_cast<int>(TaskAbortResult::TASK_ABORT_FAIL)); | 204 | EXPECT_EQ(ret, static_cast<int>(TaskAbortResult::TASK_ABORT_FAIL)); |
| 203 | 205 | ||
| 204 | // 清理 | 206 | // 清理 |
| 205 | - HcclTaskAbortHandler::GetInstance().UnRegister(comm); | 207 | + handler.UnRegister(comm); |
| 206 | } | 208 | } |
| 207 | 209 | ||
| 208 | TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_post_success) | 210 | TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_post_success) |
| @@ -214,8 +216,8 @@ TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_post_suc | |||
| 214 | 216 | ||
| 215 | // 使用 nullptr 作为测试 communicator 的占位符并注册 | 217 | // 使用 nullptr 作为测试 communicator 的占位符并注册 |
| 216 | CollComm* comm = nullptr; | 218 | CollComm* comm = nullptr; |
| 217 | - HcclTaskAbortHandler::GetInstance().Register(comm); | 219 | + handler.Register(comm); |
| 218 | - void* args = reinterpret_cast<void*>(&HcclTaskAbortHandler::GetInstance().commVector_); | 220 | + void* args = reinterpret_cast<void*>(&handler.commVector_); |
| 219 | 221 | ||
| 220 | // 模拟 CCU 已初始化及相关函数返回成功 | 222 | // 模拟 CCU 已初始化及相关函数返回成功 |
| 221 | MOCKER(CcuIsInited).stubs().will(returnValue(true)); | 223 | MOCKER(CcuIsInited).stubs().will(returnValue(true)); |
| @@ -238,7 +240,7 @@ TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_post_suc | |||
| 238 | EXPECT_EQ(ret, static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS)); | 240 | EXPECT_EQ(ret, static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS)); |
| 239 | 241 | ||
| 240 | // 清理 | 242 | // 清理 |
| 241 | - HcclTaskAbortHandler::GetInstance().UnRegister(comm); | 243 | + handler.UnRegister(comm); |
| 242 | } | 244 | } |
| 243 | 245 | ||
| 244 | TEST_F(HcclTaskAbortHandlerTest, test_task_abort_post_skip_taskkill_when_ccu_not_inited) | 246 | TEST_F(HcclTaskAbortHandlerTest, test_task_abort_post_skip_taskkill_when_ccu_not_inited) |
| @@ -250,8 +252,8 @@ TEST_F(HcclTaskAbortHandlerTest, test_task_abort_post_skip_taskkill_when_ccu_not | |||
| 250 | 252 | ||
| 251 | // 使用 nullptr 作为测试 communicator 的占位符并注册 | 253 | // 使用 nullptr 作为测试 communicator 的占位符并注册 |
| 252 | CollComm* comm = nullptr; | 254 | CollComm* comm = nullptr; |
| 253 | - HcclTaskAbortHandler::GetInstance().Register(comm); | 255 | + handler.Register(comm); |
| 254 | - void* args = reinterpret_cast<void*>(&HcclTaskAbortHandler::GetInstance().commVector_); | 256 | + void* args = reinterpret_cast<void*>(&handler.commVector_); |
| 255 | 257 | ||
| 256 | // 模拟 CCU 未初始化,应跳过 TaskKill | 258 | // 模拟 CCU 未初始化,应跳过 TaskKill |
| 257 | MOCKER(CcuIsInited).stubs().will(returnValue(false)); | 259 | MOCKER(CcuIsInited).stubs().will(returnValue(false)); |
| @@ -272,7 +274,7 @@ TEST_F(HcclTaskAbortHandlerTest, test_task_abort_post_skip_taskkill_when_ccu_not | |||
| 272 | EXPECT_EQ(ret, static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS)); | 274 | EXPECT_EQ(ret, static_cast<int>(TaskAbortResult::TASK_ABORT_SUCCESS)); |
| 273 | 275 | ||
| 274 | // 清理 | 276 | // 清理 |
| 275 | - HcclTaskAbortHandler::GetInstance().UnRegister(comm); | 277 | + handler.UnRegister(comm); |
| 276 | } | 278 | } |
| 277 | 279 | ||
| 278 | TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_post_fail) | 280 | TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_post_fail) |
| @@ -284,8 +286,8 @@ TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_post_fai | |||
| 284 | 286 | ||
| 285 | // 使用 nullptr 作为测试 communicator 的占位符并注册 | 287 | // 使用 nullptr 作为测试 communicator 的占位符并注册 |
| 286 | CollComm* comm = nullptr; | 288 | CollComm* comm = nullptr; |
| 287 | - HcclTaskAbortHandler::GetInstance().Register(comm); | 289 | + handler.Register(comm); |
| 288 | - void* args = reinterpret_cast<void*>(&HcclTaskAbortHandler::GetInstance().commVector_); | 290 | + void* args = reinterpret_cast<void*>(&handler.commVector_); |
| 289 | 291 | ||
| 290 | // 模拟 CCU 已初始化及相关函数返回成功 | 292 | // 模拟 CCU 已初始化及相关函数返回成功 |
| 291 | MOCKER(CcuIsInited).stubs().will(returnValue(true)); | 293 | MOCKER(CcuIsInited).stubs().will(returnValue(true)); |
| @@ -308,5 +310,5 @@ TEST_F(HcclTaskAbortHandlerTest, test_task_abort_handle_call_back_stage_post_fai | |||
| 308 | EXPECT_EQ(ret, static_cast<int>(TaskAbortResult::TASK_ABORT_FAIL)); | 310 | EXPECT_EQ(ret, static_cast<int>(TaskAbortResult::TASK_ABORT_FAIL)); |
| 309 | 311 | ||
| 310 | // 清理 | 312 | // 清理 |
| 311 | - HcclTaskAbortHandler::GetInstance().UnRegister(comm); | 313 | + handler.UnRegister(comm); |
| 312 | } | 314 | } |
| @@ -49,7 +49,7 @@ CcuResult StubDestroyCcuInstance(CcuInsHandle ccuInsHandle) | |||
| 49 | EXPECT_EQ(ccuInsHandle, VALID_CCU_INS_HANDLE); | 49 | EXPECT_EQ(ccuInsHandle, VALID_CCU_INS_HANDLE); |
| 50 | bool reserved = false; | 50 | bool reserved = false; |
| 51 | EXPECT_EQ( | 51 | EXPECT_EQ( |
| 52 | - hccl::CollCommMgr::GetInstance()->TryReserveCcuMsComm(DEVICE_0, "instance_destroy_probe", reserved), | 52 | + hccl::CollCommMgr::GetInstance().TryReserveCcuMsComm(DEVICE_0, "instance_destroy_probe", reserved), |
| 53 | HCCL_SUCCESS); | 53 | HCCL_SUCCESS); |
| 54 | g_reservationHeldDuringInstanceDestroy.store(!reserved); | 54 | g_reservationHeldDuringInstanceDestroy.store(!reserved); |
| 55 | return CcuResult::CCU_SUCCESS; | 55 | return CcuResult::CCU_SUCCESS; |
| @@ -60,7 +60,7 @@ CcuResult StubDeinitCcuFeature(int32_t deviceLogicId) | |||
| 60 | EXPECT_EQ(deviceLogicId, DEVICE_0); | 60 | EXPECT_EQ(deviceLogicId, DEVICE_0); |
| 61 | bool reserved = false; | 61 | bool reserved = false; |
| 62 | EXPECT_EQ( | 62 | EXPECT_EQ( |
| 63 | - hccl::CollCommMgr::GetInstance()->TryReserveCcuMsComm(DEVICE_0, "driver_deinit_probe", reserved), HCCL_SUCCESS); | 63 | + hccl::CollCommMgr::GetInstance().TryReserveCcuMsComm(DEVICE_0, "driver_deinit_probe", reserved), HCCL_SUCCESS); |
| 64 | g_reservationHeldDuringDriverDeinit.store(!reserved); | 64 | g_reservationHeldDuringDriverDeinit.store(!reserved); |
| 65 | return CcuResult::CCU_SUCCESS; | 65 | return CcuResult::CCU_SUCCESS; |
| 66 | } | 66 | } |
| @@ -71,7 +71,7 @@ CcuResult StubCreateCcuInstanceFailure(CcuInstanceType ccuInsType, CcuInsHandle* | |||
| 71 | (void)ccuInsHandle; | 71 | (void)ccuInsHandle; |
| 72 | bool reserved = false; | 72 | bool reserved = false; |
| 73 | EXPECT_EQ( | 73 | EXPECT_EQ( |
| 74 | - hccl::CollCommMgr::GetInstance()->TryReserveCcuMsComm(DEVICE_0, "init_failure_probe", reserved), HCCL_SUCCESS); | 74 | + hccl::CollCommMgr::GetInstance().TryReserveCcuMsComm(DEVICE_0, "init_failure_probe", reserved), HCCL_SUCCESS); |
| 75 | g_reservationHeldDuringInitFailure = !reserved; | 75 | g_reservationHeldDuringInitFailure = !reserved; |
| 76 | return CcuResult::CCU_E_PARA; | 76 | return CcuResult::CCU_E_PARA; |
| 77 | } | 77 | } |
| @@ -100,7 +100,7 @@ protected: | |||
| 100 | 100 | ||
| 101 | static void ResetReservations() | 101 | static void ResetReservations() |
| 102 | { | 102 | { |
| 103 | - auto* manager = hccl::CollCommMgr::GetInstance(); | 103 | + auto* manager = &hccl::CollCommMgr::GetInstance(); |
| 104 | std::lock_guard<std::mutex> lock(manager->ccuMsCommMutex_); | 104 | std::lock_guard<std::mutex> lock(manager->ccuMsCommMutex_); |
| 105 | manager->ccuMsCommIds_.fill(""); | 105 | manager->ccuMsCommIds_.fill(""); |
| 106 | } | 106 | } |
| @@ -119,7 +119,7 @@ protected: | |||
| 119 | 119 | ||
| 120 | TEST_F(CcuMsPerDeviceExclusiveTest, Ut_TryReserveCcuMsComm_When_SameDeviceConcurrent_Expect_OnlyOneReserved) | 120 | TEST_F(CcuMsPerDeviceExclusiveTest, Ut_TryReserveCcuMsComm_When_SameDeviceConcurrent_Expect_OnlyOneReserved) |
| 121 | { | 121 | { |
| 122 | - auto* manager = hccl::CollCommMgr::GetInstance(); | 122 | + auto* manager = &hccl::CollCommMgr::GetInstance(); |
| 123 | std::atomic<uint32_t> readyCount{0}; | 123 | std::atomic<uint32_t> readyCount{0}; |
| 124 | std::atomic<bool> start{false}; | 124 | std::atomic<bool> start{false}; |
| 125 | HcclResult firstRet = HCCL_E_INTERNAL; | 125 | HcclResult firstRet = HCCL_E_INTERNAL; |
| @@ -152,7 +152,7 @@ TEST_F(CcuMsPerDeviceExclusiveTest, Ut_TryReserveCcuMsComm_When_DifferentDevices | |||
| 152 | { | 152 | { |
| 153 | bool firstReserved = false; | 153 | bool firstReserved = false; |
| 154 | bool secondReserved = false; | 154 | bool secondReserved = false; |
| 155 | - auto* manager = hccl::CollCommMgr::GetInstance(); | 155 | + auto* manager = &hccl::CollCommMgr::GetInstance(); |
| 156 | 156 | ||
| 157 | EXPECT_EQ(manager->TryReserveCcuMsComm(DEVICE_0, "device_0_owner", firstReserved), HCCL_SUCCESS); | 157 | EXPECT_EQ(manager->TryReserveCcuMsComm(DEVICE_0, "device_0_owner", firstReserved), HCCL_SUCCESS); |
| 158 | EXPECT_EQ(manager->TryReserveCcuMsComm(DEVICE_1, "device_1_owner", secondReserved), HCCL_SUCCESS); | 158 | EXPECT_EQ(manager->TryReserveCcuMsComm(DEVICE_1, "device_1_owner", secondReserved), HCCL_SUCCESS); |
| @@ -162,7 +162,7 @@ TEST_F(CcuMsPerDeviceExclusiveTest, Ut_TryReserveCcuMsComm_When_DifferentDevices | |||
| 162 | 162 | ||
| 163 | TEST_F(CcuMsPerDeviceExclusiveTest, Ut_ReleaseCcuMsComm_When_NonOwnerReleases_Expect_OwnerPreserved) | 163 | TEST_F(CcuMsPerDeviceExclusiveTest, Ut_ReleaseCcuMsComm_When_NonOwnerReleases_Expect_OwnerPreserved) |
| 164 | { | 164 | { |
| 165 | - auto* manager = hccl::CollCommMgr::GetInstance(); | 165 | + auto* manager = &hccl::CollCommMgr::GetInstance(); |
| 166 | bool ownerReserved = false; | 166 | bool ownerReserved = false; |
| 167 | ASSERT_EQ(manager->TryReserveCcuMsComm(DEVICE_0, "owner", ownerReserved), HCCL_SUCCESS); | 167 | ASSERT_EQ(manager->TryReserveCcuMsComm(DEVICE_0, "owner", ownerReserved), HCCL_SUCCESS); |
| 168 | ASSERT_TRUE(ownerReserved); | 168 | ASSERT_TRUE(ownerReserved); |
| @@ -176,7 +176,7 @@ TEST_F(CcuMsPerDeviceExclusiveTest, Ut_ReleaseCcuMsComm_When_NonOwnerReleases_Ex | |||
| 176 | 176 | ||
| 177 | TEST_F(CcuMsPerDeviceExclusiveTest, Ut_ReleaseCcuMsComm_When_OwnerReleases_Expect_NextCanReserve) | 177 | TEST_F(CcuMsPerDeviceExclusiveTest, Ut_ReleaseCcuMsComm_When_OwnerReleases_Expect_NextCanReserve) |
| 178 | { | 178 | { |
| 179 | - auto* manager = hccl::CollCommMgr::GetInstance(); | 179 | + auto* manager = &hccl::CollCommMgr::GetInstance(); |
| 180 | bool ownerReserved = false; | 180 | bool ownerReserved = false; |
| 181 | ASSERT_EQ(manager->TryReserveCcuMsComm(DEVICE_0, "owner", ownerReserved), HCCL_SUCCESS); | 181 | ASSERT_EQ(manager->TryReserveCcuMsComm(DEVICE_0, "owner", ownerReserved), HCCL_SUCCESS); |
| 182 | ASSERT_TRUE(ownerReserved); | 182 | ASSERT_TRUE(ownerReserved); |
| @@ -195,14 +195,13 @@ TEST_F(CcuMsPerDeviceExclusiveTest, Ut_MyRankInit_When_RankNumIsOneAndModeIsMs_E | |||
| 195 | EXPECT_EQ(myRank_->Init(CreateCclBuffer(), CCU_MS_MODE, 1), HCCL_SUCCESS); | 195 | EXPECT_EQ(myRank_->Init(CreateCclBuffer(), CCU_MS_MODE, 1), HCCL_SUCCESS); |
| 196 | 196 | ||
| 197 | bool nextReserved = false; | 197 | bool nextReserved = false; |
| 198 | - EXPECT_EQ( | 198 | + EXPECT_EQ(hccl::CollCommMgr::GetInstance().TryReserveCcuMsComm(DEVICE_0, "next_owner", nextReserved), HCCL_SUCCESS); |
| 199 | - hccl::CollCommMgr::GetInstance()->TryReserveCcuMsComm(DEVICE_0, "next_owner", nextReserved), HCCL_SUCCESS); | ||
| 200 | EXPECT_TRUE(nextReserved); | 199 | EXPECT_TRUE(nextReserved); |
| 201 | } | 200 | } |
| 202 | 201 | ||
| 203 | TEST_F(CcuMsPerDeviceExclusiveTest, Ut_TryInitCcuInstance_When_SameDeviceAlreadyOwned_Expect_FallbackToSched) | 202 | TEST_F(CcuMsPerDeviceExclusiveTest, Ut_TryInitCcuInstance_When_SameDeviceAlreadyOwned_Expect_FallbackToSched) |
| 204 | { | 203 | { |
| 205 | - auto* manager = hccl::CollCommMgr::GetInstance(); | 204 | + auto* manager = &hccl::CollCommMgr::GetInstance(); |
| 206 | bool ownerReserved = false; | 205 | bool ownerReserved = false; |
| 207 | ASSERT_EQ(manager->TryReserveCcuMsComm(DEVICE_0, "existing_owner", ownerReserved), HCCL_SUCCESS); | 206 | ASSERT_EQ(manager->TryReserveCcuMsComm(DEVICE_0, "existing_owner", ownerReserved), HCCL_SUCCESS); |
| 208 | ASSERT_TRUE(ownerReserved); | 207 | ASSERT_TRUE(ownerReserved); |
| @@ -234,8 +233,7 @@ TEST_F(CcuMsPerDeviceExclusiveTest, Ut_TryInitCcuInstance_When_CcuInitFails_Expe | |||
| 234 | EXPECT_FALSE(myRank_->ccuMsCommReserved_); | 233 | EXPECT_FALSE(myRank_->ccuMsCommReserved_); |
| 235 | 234 | ||
| 236 | bool nextReserved = false; | 235 | bool nextReserved = false; |
| 237 | - EXPECT_EQ( | 236 | + EXPECT_EQ(hccl::CollCommMgr::GetInstance().TryReserveCcuMsComm(DEVICE_0, "next_owner", nextReserved), HCCL_SUCCESS); |
| 238 | - hccl::CollCommMgr::GetInstance()->TryReserveCcuMsComm(DEVICE_0, "next_owner", nextReserved), HCCL_SUCCESS); | ||
| 239 | EXPECT_TRUE(nextReserved); | 237 | EXPECT_TRUE(nextReserved); |
| 240 | } | 238 | } |
| 241 | 239 | ||
| @@ -254,8 +252,7 @@ TEST_F(CcuMsPerDeviceExclusiveTest, Ut_TryInitCcuInstance_When_MsFallsBack_Expec | |||
| 254 | EXPECT_FALSE(myRank_->ccuMsCommReserved_); | 252 | EXPECT_FALSE(myRank_->ccuMsCommReserved_); |
| 255 | 253 | ||
| 256 | bool nextReserved = false; | 254 | bool nextReserved = false; |
| 257 | - EXPECT_EQ( | 255 | + EXPECT_EQ(hccl::CollCommMgr::GetInstance().TryReserveCcuMsComm(DEVICE_0, "next_owner", nextReserved), HCCL_SUCCESS); |
| 258 | - hccl::CollCommMgr::GetInstance()->TryReserveCcuMsComm(DEVICE_0, "next_owner", nextReserved), HCCL_SUCCESS); | ||
| 259 | EXPECT_TRUE(nextReserved); | 256 | EXPECT_TRUE(nextReserved); |
| 260 | } | 257 | } |
| 261 | 258 | ||
| @@ -271,14 +268,13 @@ TEST_F(CcuMsPerDeviceExclusiveTest, Ut_TryInitCcuInstance_When_DriverBusyFallsBa | |||
| 271 | EXPECT_FALSE(myRank_->ccuMsCommReserved_); | 268 | EXPECT_FALSE(myRank_->ccuMsCommReserved_); |
| 272 | 269 | ||
| 273 | bool nextReserved = false; | 270 | bool nextReserved = false; |
| 274 | - EXPECT_EQ( | 271 | + EXPECT_EQ(hccl::CollCommMgr::GetInstance().TryReserveCcuMsComm(DEVICE_0, "next_owner", nextReserved), HCCL_SUCCESS); |
| 275 | - hccl::CollCommMgr::GetInstance()->TryReserveCcuMsComm(DEVICE_0, "next_owner", nextReserved), HCCL_SUCCESS); | ||
| 276 | EXPECT_TRUE(nextReserved); | 272 | EXPECT_TRUE(nextReserved); |
| 277 | } | 273 | } |
| 278 | 274 | ||
| 279 | TEST_F(CcuMsPerDeviceExclusiveTest, Ut_MyRankDestructor_When_HoldsMsReservation_Expect_ReleaseAfterCcuCleanup) | 275 | TEST_F(CcuMsPerDeviceExclusiveTest, Ut_MyRankDestructor_When_HoldsMsReservation_Expect_ReleaseAfterCcuCleanup) |
| 280 | { | 276 | { |
| 281 | - auto* manager = hccl::CollCommMgr::GetInstance(); | 277 | + auto* manager = &hccl::CollCommMgr::GetInstance(); |
| 282 | bool ownerReserved = false; | 278 | bool ownerReserved = false; |
| 283 | ASSERT_EQ(manager->TryReserveCcuMsComm(DEVICE_0, "destructor_owner", ownerReserved), HCCL_SUCCESS); | 279 | ASSERT_EQ(manager->TryReserveCcuMsComm(DEVICE_0, "destructor_owner", ownerReserved), HCCL_SUCCESS); |
| 284 | ASSERT_TRUE(ownerReserved); | 280 | ASSERT_TRUE(ownerReserved); |
| @@ -306,7 +302,7 @@ TEST_F(CcuMsPerDeviceExclusiveTest, Ut_MyRankDestructor_When_HoldsMsReservation_ | |||
| 306 | 302 | ||
| 307 | TEST_F(CcuMsPerDeviceExclusiveTest, Ut_TryReserveCcuMsComm_When_DeviceIdInvalid_Expect_ReturnError) | 303 | TEST_F(CcuMsPerDeviceExclusiveTest, Ut_TryReserveCcuMsComm_When_DeviceIdInvalid_Expect_ReturnError) |
| 308 | { | 304 | { |
| 309 | - auto* manager = hccl::CollCommMgr::GetInstance(); | 305 | + auto* manager = &hccl::CollCommMgr::GetInstance(); |
| 310 | bool ownerReserved = false; | 306 | bool ownerReserved = false; |
| 311 | ASSERT_EQ(manager->TryReserveCcuMsComm(DEVICE_0, "device_0_owner", ownerReserved), HCCL_SUCCESS); | 307 | ASSERT_EQ(manager->TryReserveCcuMsComm(DEVICE_0, "device_0_owner", ownerReserved), HCCL_SUCCESS); |
| 312 | ASSERT_TRUE(ownerReserved); | 308 | ASSERT_TRUE(ownerReserved); |
| @@ -25,6 +25,7 @@ | |||
| 25 | 25 | ||
| 26 | 26 | ||
| 27 | 27 | ||
| 28 | + | ||
| 28 | 29 | ||
| 29 | extern thread_local s32 g_hcclDeviceId; | 30 | extern thread_local s32 g_hcclDeviceId; |
| 30 | 31 | ||
| @@ -165,7 +166,7 @@ protected: | |||
| 165 | 166 | ||
| 166 | static void ResetOpBaseState() | 167 | static void ResetOpBaseState() |
| 167 | { | 168 | { |
| 168 | - HcclOpInfoCtx& opBaseHcom = GetHcclOpInfoCtx(); | 169 | + HcclOpInfoCtx& opBaseHcom = hccl::CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 169 | std::lock_guard<std::mutex> lock(opBaseHcom.opGroupMapMutex); | 170 | std::lock_guard<std::mutex> lock(opBaseHcom.opGroupMapMutex); |
| 170 | opBaseHcom.opGroup2CommMap.clear(); | 171 | opBaseHcom.opGroup2CommMap.clear(); |
| 171 | } | 172 | } |
| @@ -199,7 +200,7 @@ TEST_F(HcclCommInitCollCommGuardTest, Ut_HcclCommInitCollComm_When_SetGroupTopoI | |||
| 199 | 200 | ||
| 200 | EXPECT_EQ(InitComm(failedConfig, failedComm), HCCL_E_INTERNAL); | 201 | EXPECT_EQ(InitComm(failedConfig, failedComm), HCCL_E_INTERNAL); |
| 201 | EXPECT_EQ(failedComm, nullptr); | 202 | EXPECT_EQ(failedComm, nullptr); |
| 202 | - HcclOpInfoCtx& opBaseHcom = GetHcclOpInfoCtx(); | 203 | + HcclOpInfoCtx& opBaseHcom = hccl::CollCommMgr::GetInstance().LegacyGetHcclOpInfoCtx(g_hcclDeviceId); |
| 203 | { | 204 | { |
| 204 | std::lock_guard<std::mutex> lock(opBaseHcom.opGroupMapMutex); | 205 | std::lock_guard<std::mutex> lock(opBaseHcom.opGroupMapMutex); |
| 205 | EXPECT_TRUE(opBaseHcom.opGroup2CommMap.empty()); | 206 | EXPECT_TRUE(opBaseHcom.opGroup2CommMap.empty()); |
| @@ -60,6 +60,7 @@ | |||
| 60 | 60 | ||
| 61 | 61 | ||
| 62 | 62 | ||
| 63 | + | ||
| 63 | 64 | ||
| 64 | 65 | ||
| 65 | 66 | ||
| @@ -3765,11 +3766,10 @@ TEST_F(OpbaseTest, ut_HcclAiCpuResourceByTiling_A3) | |||
| 3765 | } | 3766 | } |
| 3766 | 3767 | ||
| 3767 | extern thread_local s32 g_hcclDeviceId; | 3768 | extern thread_local s32 g_hcclDeviceId; |
| 3768 | -extern HcclOpInfoCtx g_opHcomInfos[MAX_MODULE_DEVICE_NUM + 1]; | 3769 | +TEST_F(OpbaseTest, ut_LegacyGetHcclOpInfoCtx_cover_bottom_false) |
| 3769 | -TEST_F(OpbaseTest, ut_GetHcclOpInfoCtx_cover_bottom_false) | ||
| 3770 | { | 3770 | { |
| 3771 | for (int i = 0; i < MAX_MODULE_DEVICE_NUM + 1; i++) { | 3771 | for (int i = 0; i < MAX_MODULE_DEVICE_NUM + 1; i++) { |
| 3772 | - g_opHcomInfos[i].isUsed = false; | 3772 | + CollCommMgr::GetInstance().LegacyGetOpHcomInfo(i).isUsed = false; |
| 3773 | } | 3773 | } |
| 3774 | g_hcclDeviceId = INVALID_INT; | 3774 | g_hcclDeviceId = INVALID_INT; |
| 3775 | HcclResult ret = HCCL_SUCCESS; | 3775 | HcclResult ret = HCCL_SUCCESS; |
| @@ -3785,12 +3785,12 @@ TEST_F(OpbaseTest, ut_GetHcclOpInfoCtx_cover_bottom_false) | |||
| 3785 | GlobalMockObject::verify(); | 3785 | GlobalMockObject::verify(); |
| 3786 | } | 3786 | } |
| 3787 | 3787 | ||
| 3788 | -TEST_F(OpbaseTest, ut_GetHcclOpInfoCtx_cover_bottom_true) | 3788 | +TEST_F(OpbaseTest, ut_LegacyGetHcclOpInfoCtx_cover_bottom_true) |
| 3789 | { | 3789 | { |
| 3790 | for (int i = 0; i < MAX_MODULE_DEVICE_NUM; i++) { | 3790 | for (int i = 0; i < MAX_MODULE_DEVICE_NUM; i++) { |
| 3791 | - g_opHcomInfos[i].isUsed = false; | 3791 | + CollCommMgr::GetInstance().LegacyGetOpHcomInfo(i).isUsed = false; |
| 3792 | } | 3792 | } |
| 3793 | - g_opHcomInfos[MAX_MODULE_DEVICE_NUM].isUsed = true; | 3793 | + CollCommMgr::GetInstance().LegacyGetOpHcomInfo(MAX_MODULE_DEVICE_NUM).isUsed = true; |
| 3794 | g_hcclDeviceId = INVALID_INT; | 3794 | g_hcclDeviceId = INVALID_INT; |
| 3795 | HcclResult ret = HCCL_SUCCESS; | 3795 | HcclResult ret = HCCL_SUCCESS; |
| 3796 | MOCKER(hrtGetDevice) | 3796 | MOCKER(hrtGetDevice) |


[HIGH] 析构函数存在 UAF 风险CollCommMgr::GetInstance() 使用 static local variable,析构时机是进程退出时(static destruction order fiasco)。此时其他全局/static 对象可能已被销毁,monitor.DeInit() 内部可能依赖这些对象(如日志系统、线程同步等)。建议:明确 ClusterMonitor::DeInit() 的调用时机应在进程退出前主动完成,而非依赖静态析构