已合并
[Fix] 修改 HcommChannelNotifyWaitOnThread 的 timeout 参数单位为：ms->s #4625
Yuanhao Ji创建于 22 天前
[Fix] 修改 HcommChannelNotifyWaitOnThread 的 timeout 参数单位为：ms->s #4625
已合并
共 3 个文件变更+16-15
| @@ -36,7 +36,7 @@ constexpr u32 MEM_BLOCK_SIZE = 128; | |||
| 36 | constexpr uint16_t DEFAULT_LISTENING_PORT = 60001; | 36 | constexpr uint16_t DEFAULT_LISTENING_PORT = 60001; |
| 37 | constexpr u32 SEND_RQE_COUNT = 16; | 37 | constexpr u32 SEND_RQE_COUNT = 16; |
| 38 | constexpr u32 HYBRID_QP_STATUS_POLL_SLEEP_US = 1000; | 38 | constexpr u32 HYBRID_QP_STATUS_POLL_SLEEP_US = 1000; |
| 39 | -constexpr u32 DEFAULT_NOTIFY_POLL_TIMEOUT_MS = 30000; | 39 | +constexpr u32 DEFAULT_NOTIFY_WAIT_TIMEOUT_S = 30; |
| 40 | constexpr uint32_t kHostResourceId = 0U; | 40 | constexpr uint32_t kHostResourceId = 0U; |
| 41 | 41 | ||
| 42 | static inline CommMemType HcclMemTypeToCommMemType(HcclMemType type) | 42 | static inline CommMemType HcclMemTypeToCommMemType(HcclMemType type) |
| @@ -967,17 +967,17 @@ HcclResult HostCpuRoceChannel::NotifyWait(const uint32_t localNotifyIdx, const u | |||
| 967 | 967 | ||
| 968 | // 2.轮询rq_cq | 968 | // 2.轮询rq_cq |
| 969 | auto startTime = std::chrono::steady_clock::now(); | 969 | auto startTime = std::chrono::steady_clock::now(); |
| 970 | - auto waitTime = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::milliseconds(timeout)); | 970 | + auto waitTime = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::seconds(timeout)); |
 | |||
| 971 | for (uint32_t i = 0; i < qpInfo.size(); i++) { | 971 | for (uint32_t i = 0; i < qpInfo.size(); i++) { |
| 972 | CHK_PRT_RET( | 972 | CHK_PRT_RET( |
| 973 | - qpInfo[i].recvCq == nullptr, HCCL_ERROR("[HostCpuRoceChannel::%s] recvCq[%u] is null", i, __func__), | 973 | + qpInfo[i].recvCq == nullptr, HCCL_ERROR("[HostCpuRoceChannel::%s] recvCq[%u] is null", __func__, i), |
| 974 | HCCL_E_INTERNAL); | 974 | HCCL_E_INTERNAL); |
| 975 | CHK_PRT_RET( | 975 | CHK_PRT_RET( |
| 976 | - qpInfo[i].qp == nullptr, HCCL_ERROR("[HostCpuRoceChannel::%s] qp[%u] is null", i, __func__), | 976 | + qpInfo[i].qp == nullptr, HCCL_ERROR("[HostCpuRoceChannel::%s] qp[%u] is null", __func__, i), |
| 977 | HCCL_E_INTERNAL); | 977 | HCCL_E_INTERNAL); |
| 978 | CHK_PRT_RET( | 978 | CHK_PRT_RET( |
| 979 | qpInfo[i].recvCq->context == nullptr, | 979 | qpInfo[i].recvCq->context == nullptr, |
| 980 | - HCCL_ERROR("[HostCpuRoceChannel::%s] recvCq[%u]->context is null", i, __func__), HCCL_E_INTERNAL); | 980 | + HCCL_ERROR("[HostCpuRoceChannel::%s] recvCq[%u]->context is null", __func__, i), HCCL_E_INTERNAL); |
| 981 | 981 | ||
| 982 | while (true) { | 982 | while (true) { |
| 983 | auto actualNum = ibv_poll_cq(qpInfo[i].recvCq, 1, &wc); | 983 | auto actualNum = ibv_poll_cq(qpInfo[i].recvCq, 1, &wc); |
| @@ -1905,8 +1905,8 @@ HcclResult HostCpuRoceChannel::NotifyWaitHybrid(uint32_t localNotifyIdx, uint32_ | |||
| 1905 | 1905 | ||
| 1906 | hccl::MemType type = NotifyIdToMemtypeHybird(localNotifyIdx); | 1906 | hccl::MemType type = NotifyIdToMemtypeHybird(localNotifyIdx); |
| 1907 | 1907 | ||
| 1908 | - // 使用配置的超时时间和轮询间隔 | 1908 | + // 使用配置的超时时间和轮询间隔（timeOut单位为秒，默认30s） |
| 1909 | - uint32_t pollTimeout = (timeout == 0) ? DEFAULT_NOTIFY_POLL_TIMEOUT_MS : timeout; | 1909 | + uint32_t pollTimeout = (timeout == 0) ? DEFAULT_NOTIFY_WAIT_TIMEOUT_S : timeout; |
| 1910 | uint32_t pollInterval = 1; | 1910 | uint32_t pollInterval = 1; |
| 1911 | 1911 | ||
| 1912 | // 使用原子操作读取 Notify 内存 | 1912 | // 使用原子操作读取 Notify 内存 |
| @@ -1914,7 +1914,7 @@ HcclResult HostCpuRoceChannel::NotifyWaitHybrid(uint32_t localNotifyIdx, uint32_ | |||
| 1914 | const uint64_t expectedValue = 1; | 1914 | const uint64_t expectedValue = 1; |
| 1915 | 1915 | ||
| 1916 | auto startTime = std::chrono::steady_clock::now(); | 1916 | auto startTime = std::chrono::steady_clock::now(); |
| 1917 | - auto waitTime = std::chrono::milliseconds(pollTimeout); | 1917 | + auto waitTime = std::chrono::seconds(pollTimeout); |
| 1918 | 1918 | ||
| 1919 | while (true) { | 1919 | while (true) { |
| 1920 | // 使用原子操作读取，确保内存可见性 | 1920 | // 使用原子操作读取，确保内存可见性 |
| @@ -38,6 +38,7 @@ namespace hcomm { | |||
| 38 | constexpr u32 MEM_BLOCK_SIZE = 128; | 38 | constexpr u32 MEM_BLOCK_SIZE = 128; |
| 39 | constexpr uint16_t DEFAULT_LISTENING_PORT = 60001; | 39 | constexpr uint16_t DEFAULT_LISTENING_PORT = 60001; |
| 40 | constexpr u32 SEND_RQE_COUNT = 16; | 40 | constexpr u32 SEND_RQE_COUNT = 16; |
| 41 | +constexpr u32 DEFAULT_NOTIFY_WAIT_TIMEOUT_S = 30; // NotifyWait超时默认值（单位：秒） | ||
| 41 | 42 | ||
| 42 | HostCpuRoceChannel::HostCpuRoceChannel(EndpointHandle endpointHandle, HcommChannelDesc channelDesc) | 43 | HostCpuRoceChannel::HostCpuRoceChannel(EndpointHandle endpointHandle, HcommChannelDesc channelDesc) |
| 43 | : endpointHandle_(endpointHandle), | 44 | : endpointHandle_(endpointHandle), |
| @@ -898,7 +899,7 @@ HcclResult HostCpuRoceChannel::NotifyWait(const uint32_t localNotifyIdx, const u | |||
| 898 | 899 | ||
| 899 | // 2.轮询rq_cq | 900 | // 2.轮询rq_cq |
| 900 | auto startTime = std::chrono::steady_clock::now(); | 901 | auto startTime = std::chrono::steady_clock::now(); |
| 901 | - auto waitTime = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::milliseconds(timeout)); | 902 | + auto waitTime = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::seconds(timeout)); |
🟡 Medium Priority changed line → affected behavior → failure mode → suggested fix：
第 902 行将 建议：在非混合模式 NotifyWait 的轮询循环内加入退避睡眠（参考 NotifyWaitHybrid 的 SaluSleep），避免秒级超时下 100% CPU 空转并长时间持有 cq_mutex。   | |||
| 902 | for (uint32_t i = 0; i < qpInfo.size(); i++) { | 903 | for (uint32_t i = 0; i < qpInfo.size(); i++) { |
| 903 | CHK_PRT_RET( | 904 | CHK_PRT_RET( |
| 904 | qpInfo[i].recvCq == nullptr, HCCL_ERROR("[HostCpuRoceChannel::%s] recvCq[%u] is null", __func__, i), | 905 | qpInfo[i].recvCq == nullptr, HCCL_ERROR("[HostCpuRoceChannel::%s] recvCq[%u] is null", __func__, i), |
| @@ -1848,8 +1849,8 @@ HcclResult HostCpuRoceChannel::NotifyWaitHybrid(uint32_t localNotifyIdx, uint32_ | |||
| 1848 | 1849 | ||
| 1849 | hccl::MemType type = NotifyIdToMemtypeHybird(localNotifyIdx); | 1850 | hccl::MemType type = NotifyIdToMemtypeHybird(localNotifyIdx); |
| 1850 | 1851 | ||
| 1851 | - // 使用配置的超时时间和轮询间隔 | 1852 | + // 使用配置的超时时间和轮询间隔（timeOut单位为秒，默认30s） |
| 1852 | - uint32_t pollTimeout = (timeout == 0) ? 30000 : timeout; | 1853 | + uint32_t pollTimeout = (timeout == 0) ? DEFAULT_NOTIFY_WAIT_TIMEOUT_S : timeout; |
| 1853 | uint32_t pollInterval = 1; | 1854 | uint32_t pollInterval = 1; |
| 1854 | 1855 | ||
| 1855 | // 使用原子操作读取 Notify 内存 | 1856 | // 使用原子操作读取 Notify 内存 |
| @@ -1857,7 +1858,7 @@ HcclResult HostCpuRoceChannel::NotifyWaitHybrid(uint32_t localNotifyIdx, uint32_ | |||
| 1857 | const uint64_t expectedValue = 1; | 1858 | const uint64_t expectedValue = 1; |
| 1858 | 1859 | ||
| 1859 | auto startTime = std::chrono::steady_clock::now(); | 1860 | auto startTime = std::chrono::steady_clock::now(); |
| 1860 | - auto waitTime = std::chrono::milliseconds(pollTimeout); | 1861 | + auto waitTime = std::chrono::seconds(pollTimeout); |
| 1861 | 1862 | ||
| 1862 | while (true) { | 1863 | while (true) { |
| 1863 | // 使用原子操作读取，确保内存可见性 | 1864 | // 使用原子操作读取，确保内存可见性 |
| @@ -1289,8 +1289,8 @@ TEST_F(HostCpuRoceChannelTest, Ut_NotifyWaitHybrid_Timeout_Expect_HCCL_E_TIMEOUT | |||
| 1289 | impl_->localMemMsg_[hccl::DATA_NOTIFY_MEM].len = sizeof(std::atomic<uint32_t>); | 1289 | impl_->localMemMsg_[hccl::DATA_NOTIFY_MEM].len = sizeof(std::atomic<uint32_t>); |
| 1290 | impl_->localMemMsg_[hccl::DATA_NOTIFY_MEM].lkey = 0; | 1290 | impl_->localMemMsg_[hccl::DATA_NOTIFY_MEM].lkey = 0; |
| 1291 | 1291 | ||
| 1292 | - // 使用很短的超时时间（1ms）触发超时 | 1292 | + // 使用很短的超时时间（1s）触发超时 |
| 1293 | - HcclResult ret = impl_->NotifyWaitHybrid(1, 1); // 1ms 超时 | 1293 | + HcclResult ret = impl_->NotifyWaitHybrid(1, 1); // 1s 超时 |
| 1294 | EXPECT_EQ(ret, HCCL_E_TIMEOUT); | 1294 | EXPECT_EQ(ret, HCCL_E_TIMEOUT); |
| 1295 | 1295 | ||
| 1296 | impl_->isHybridMode_ = false; // 析构前设置为false避免调用HrtRaMrDereg | 1296 | impl_->isHybridMode_ = false; // 析构前设置为false避免调用HrtRaMrDereg |
| @@ -1314,7 +1314,7 @@ TEST_F(HostCpuRoceChannelTest, Ut_NotifyWaitHybrid_NullMemAddr_Expect_HCCL_E_TIM | |||
| 1314 | impl_->localMemMsg_[hccl::ACK_NOTIFY_MEM].len = sizeof(std::atomic<uint32_t>); | 1314 | impl_->localMemMsg_[hccl::ACK_NOTIFY_MEM].len = sizeof(std::atomic<uint32_t>); |
| 1315 | impl_->localMemMsg_[hccl::ACK_NOTIFY_MEM].lkey = 0; | 1315 | impl_->localMemMsg_[hccl::ACK_NOTIFY_MEM].lkey = 0; |
| 1316 | 1316 | ||
| 1317 | - // 使用1ms超时 | 1317 | + // 使用1s超时 |
| 1318 | HcclResult ret = impl_->NotifyWaitHybrid(0, 1); | 1318 | HcclResult ret = impl_->NotifyWaitHybrid(0, 1); |
| 1319 | EXPECT_EQ(ret, HCCL_E_TIMEOUT); | 1319 | EXPECT_EQ(ret, HCCL_E_TIMEOUT); |
| 1320 | 1320 | ||
🟡 Medium Priority
changed line → affected behavior → failure mode → suggested fix： 第 970 行将
std::chrono::milliseconds(timeout)改为std::chrono::seconds(timeout)，使非混合模式NotifyWait轮询超时上限从毫秒级放大到秒级。但该函数while (true)循环内只有ibv_poll_cq（非阻塞），无任何sleep/yield（对比本文件NotifyWaitHybrid每轮SaluSleep(pollInterval)），且循环整体位于函数开头std::lock_guard<std::mutex> lock(cq_mutex)（第 961 行）持锁范围。 后果：对端 notify 迟迟未到时，本端线程 100% CPU 空转并长时间持有cq_mutex，最坏持续整个超时（默认约 30 分钟）。改动前同值按毫秒解释（约 1.8 秒），问题几乎不可触发；改为秒后缺失退避的忙轮询被真实放大。建议：在非混合模式 NotifyWait 的轮询循环内加入退避睡眠（参考 NotifyWaitHybrid 的 SaluSleep），避免秒级超时下 100% CPU 空转并长时间持有 cq_mutex。