Star429
1015
代码
代码
Issues96
Pull Requests227
Actions
项目讨论
Wiki
分析
Star429
1015
已合并
[feat]resource refresh optimization #2502
chenliang267创建于 6月2日
[feat]resource refresh optimization #2502
已合并
chenliang267创建于 6月2日
从chenliang267/hcommresource_refresh_opt合入到cann/hcommmaster
共 2 个文件变更+186-3
Msrc/legacy/ascend910/framework/communicator/impl/hccl_communicator_host.cc+21-3
@@ -6066,14 +6066,32 @@ namespace hccl
6066 {6066 {
6067 ListCommon *nextHostList = reinterpret_cast<ListCommon *>(headHostList->nextHost);6067 ListCommon *nextHostList = reinterpret_cast<ListCommon *>(headHostList->nextHost);
6068 ListCommon *nextDeviceList = reinterpret_cast<ListCommon *>(headHostList->nextDevice);6068 ListCommon *nextDeviceList = reinterpret_cast<ListCommon *>(headHostList->nextDevice);
6069+ // 该tag已分配过资源，只需刷新单节点（精确匹配）；否则为首次分配，批量拷贝前N个节点
6070+ bool isRefreshSingleNode = (newTagResAlloced_.find(newTag) != newTagResAlloced_.end());
6071+ // ListCommonAddHead把新节点头插，原头节点的preHost/preDevice被改写，两节点都需要刷新到device
6072+ constexpr uint32_t UPDATE_NODE_NUM = 2;
6073+ uint32_t updateNodeCnt = 0;
6069 6074 
6070- while (nextHostList != headHostList) {6075+ while (nextHostList != headHostList && updateNodeCnt < UPDATE_NODE_NUM) {
6071 HCCL_INFO(6076 HCCL_INFO(
6072 "[HcclCommunicator][CopyHostListResToDeviceParam] remote resource, tag[%s], head Host List[%p], next "6077 "[HcclCommunicator][CopyHostListResToDeviceParam] remote resource, tag[%s], head Host List[%p], next "
6073 "Host List[%p],next Device List[%p]",6078 "Host List[%p],next Device List[%p]",
6074 newTag.c_str(), headHostList, nextHostList, nextDeviceList);6079 newTag.c_str(), headHostList, nextHostList, nextDeviceList);
6075- CHK_RET(hrtMemSyncCopy(reinterpret_cast<void *>(nextDeviceList), size, reinterpret_cast<void *>(nextHostList),6080+ if (isRefreshSingleNode) {
6076- size, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));6081+ // 刷新已有资源：遍历链表找到匹配的tag，只拷贝该节点，避免全量刷新
6082+ std::string curTag = (size == sizeof(HccltagLocalResV2)) ? reinterpret_cast<HccltagLocalResV2 *>(nextHostList)->tag :
6083+ reinterpret_cast<HccltagRemoteResV2 *>(nextHostList)->tag;
6084+ if (curTag == newTag) {
6085+ CHK_RET(hrtMemSyncCopy(reinterpret_cast<void *>(nextDeviceList), size, reinterpret_cast<void *>(nextHostList),
6086+ size, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
6087+ break;
6088+ }
6089+ } else {
6090+ // 首分配置：拷贝前UPDATE_NODE_NUM个节点到device，减少H2D次数
6091+ CHK_RET(hrtMemSyncCopy(reinterpret_cast<void *>(nextDeviceList), size, reinterpret_cast<void *>(nextHostList),
6092+ size, HcclRtMemcpyKind::HCCL_RT_MEMCPY_KIND_HOST_TO_DEVICE));
6093+ updateNodeCnt++;
6094+ }
6077 nextDeviceList = reinterpret_cast<ListCommon *>(nextHostList->nextDevice);6095 nextDeviceList = reinterpret_cast<ListCommon *>(nextHostList->nextDevice);
6078 nextHostList = reinterpret_cast<ListCommon *>(nextHostList->nextHost);6096 nextHostList = reinterpret_cast<ListCommon *>(nextHostList->nextHost);
6079 }6097 }
Mtest/ut/framework/communicator/impl/symmetric_memory/ut_hccl_communicator_host.cc+165-0
@@ -19,6 +19,7 @@
19#undef private19#undef private
20#undef protected20#undef protected
21#include "llt_hccl_stub_pub.h"21#include "llt_hccl_stub_pub.h"
22+#include "adapter_rts_common.h"
22 23 
23using namespace std;24using namespace std;
24using namespace hccl;25using namespace hccl;
@@ -341,3 +342,167 @@ TEST_F(HcclCommunicatorHostTest, Ut_HcclGetAlgExecParam_When_Normal_Expect_Retur
341 free(inputPtr);342 free(inputPtr);
342 free(outputPtr);343 free(outputPtr);
343}344}
345+TEST_F(HcclCommunicatorHostTest, Ut_CopyHostListResToDeviceParam_FirstTime_EmptyList_Expect_Success)
346+{
347+ std::unique_ptr<HcclCommunicator> hcclCommunicator(new (std::nothrow) HcclCommunicator());
348+ 
349+ // 空链表：sentinel 指向自身，while 条件立即为 false
350+ ListCommon sentinel;
351+ sentinel.nextHost = reinterpret_cast<u64>(&sentinel);
352+ sentinel.preHost = reinterpret_cast<u64>(&sentinel);
353+ sentinel.nextDevice = reinterpret_cast<u64>(&sentinel);
354+ sentinel.preDevice = reinterpret_cast<u64>(&sentinel);
355+ 
356+ MOCKER(hrtMemSyncCopy).stubs().will(returnValue(HCCL_SUCCESS));
357+ 
358+ std::string newTag = "test_tag";
359+ HcclResult ret = hcclCommunicator->CopyHostListResToDeviceParam(newTag, &sentinel, sizeof(HccltagLocalResV2));
360+ 
361+ EXPECT_EQ(ret, HCCL_SUCCESS);
362+ GlobalMockObject::verify();
363+}
364+ 
365+TEST_F(HcclCommunicatorHostTest, Ut_CopyHostListResToDeviceParam_FirstTime_MultiNode_Expect_CopyFirstTwoOnly)
366+{
367+ std::unique_ptr<HcclCommunicator> hcclCommunicator(new (std::nothrow) HcclCommunicator());
368+ 
369+ // 构建4个节点的循环链表：sentinel → n1 → n2 → n3 → sentinel
370+ ListCommon sentinel;
371+ HccltagLocalResV2 n1, n2, n3;
372+ memset(&n1, 0, sizeof(n1));
373+ memset(&n2, 0, sizeof(n2));
374+ memset(&n3, 0, sizeof(n3));
375+ 
376+ // host 链
377+ sentinel.nextHost = reinterpret_cast<u64>(&n1.nextTagRes);
378+ n1.nextTagRes.nextHost = reinterpret_cast<u64>(&n2.nextTagRes);
379+ n2.nextTagRes.nextHost = reinterpret_cast<u64>(&n3.nextTagRes);
380+ n3.nextTagRes.nextHost = reinterpret_cast<u64>(&sentinel);
381+ 
382+ // device 链（只填充 host 节点的 nextDevice，供 memcpy dst 用；device 节点本身不遍历）
383+ ListCommon dev1, dev2, dev3;
384+ sentinel.nextDevice = reinterpret_cast<u64>(&dev1);
385+ n1.nextTagRes.nextDevice = reinterpret_cast<u64>(&dev2);
386+ n2.nextTagRes.nextDevice = reinterpret_cast<u64>(&dev3);
387+ n3.nextTagRes.nextDevice = reinterpret_cast<u64>(&sentinel);
388+ 
389+ // 首次分配（newTagResAlloced_ 为空），应只拷前2个节点
390+ MOCKER(hrtMemSyncCopy).expects(atMost(2)).will(returnValue(HCCL_SUCCESS));
391+ 
392+ std::string newTag = "test_tag";
393+ HcclResult ret = hcclCommunicator->CopyHostListResToDeviceParam(newTag, &sentinel, sizeof(HccltagLocalResV2));
394+ 
395+ EXPECT_EQ(ret, HCCL_SUCCESS);
396+ GlobalMockObject::verify();
397+}
398+ 
399+TEST_F(HcclCommunicatorHostTest, Ut_CopyHostListResToDeviceParam_RefreshSingleNode_TagFound_Expect_CopyOneOnly)
400+{
401+ std::unique_ptr<HcclCommunicator> hcclCommunicator(new (std::nothrow) HcclCommunicator());
402+ 
403+ // 预先插入 tag，使 isRefreshSingleNode = true
404+ std::string targetTag = "refresh_tag";
405+ hcclCommunicator->newTagResAlloced_.insert(targetTag);
406+ 
407+ // 构建3个节点的链表，第二个节点匹配 targetTag
408+ ListCommon sentinel;
409+ HccltagLocalResV2 n1, n2, n3;
410+ memset(&n1, 0, sizeof(n1));
411+ memset(&n2, 0, sizeof(n2));
412+ memset(&n3, 0, sizeof(n3));
413+ memcpy_s(n1.tag, TAG_MAX_LENGTH, "other_tag", sizeof("other_tag"));
414+ memcpy_s(n2.tag, TAG_MAX_LENGTH, targetTag.c_str(), targetTag.length() + 1);
415+ memcpy_s(n3.tag, TAG_MAX_LENGTH, "another_tag", sizeof("another_tag"));
416+ 
417+ sentinel.nextHost = reinterpret_cast<u64>(&n1.nextTagRes);
418+ n1.nextTagRes.nextHost = reinterpret_cast<u64>(&n2.nextTagRes);
419+ n2.nextTagRes.nextHost = reinterpret_cast<u64>(&n3.nextTagRes);
420+ n3.nextTagRes.nextHost = reinterpret_cast<u64>(&sentinel);
421+ 
422+ ListCommon dev1, dev2, dev3;
423+ sentinel.nextDevice = reinterpret_cast<u64>(&dev1);
424+ n1.nextTagRes.nextDevice = reinterpret_cast<u64>(&dev2);
425+ n2.nextTagRes.nextDevice = reinterpret_cast<u64>(&dev3);
426+ n3.nextTagRes.nextDevice = reinterpret_cast<u64>(&sentinel);
427+ 
428+ // 匹配到第二个节点时 break，只拷1次
429+ MOCKER(hrtMemSyncCopy).expects(atMost(1)).will(returnValue(HCCL_SUCCESS));
430+ 
431+ HcclResult ret = hcclCommunicator->CopyHostListResToDeviceParam(targetTag, &sentinel, sizeof(HccltagLocalResV2));
432+ 
433+ EXPECT_EQ(ret, HCCL_SUCCESS);
434+ GlobalMockObject::verify();
435+}
436+ 
437+TEST_F(HcclCommunicatorHostTest, Ut_CopyHostListResToDeviceParam_RefreshSingleNode_TagNotFound_Expect_NoCopy)
438+{
439+ std::unique_ptr<HcclCommunicator> hcclCommunicator(new (std::nothrow) HcclCommunicator());
440+ 
441+ // 预先插入 tag，使 isRefreshSingleNode = true
442+ std::string targetTag = "refresh_tag";
443+ hcclCommunicator->newTagResAlloced_.insert(targetTag);
444+ 
445+ // 构建3个节点的链表，没有节点匹配 targetTag
446+ ListCommon sentinel;
447+ HccltagLocalResV2 n1, n2, n3;
448+ memset(&n1, 0, sizeof(n1));
449+ memset(&n2, 0, sizeof(n2));
450+ memset(&n3, 0, sizeof(n3));
451+ memcpy_s(n1.tag, TAG_MAX_LENGTH, "tag_a", sizeof("tag_a"));
452+ memcpy_s(n2.tag, TAG_MAX_LENGTH, "tag_b", sizeof("tag_b"));
453+ memcpy_s(n3.tag, TAG_MAX_LENGTH, "tag_c", sizeof("tag_c"));
454+ 
455+ sentinel.nextHost = reinterpret_cast<u64>(&n1.nextTagRes);
456+ n1.nextTagRes.nextHost = reinterpret_cast<u64>(&n2.nextTagRes);
457+ n2.nextTagRes.nextHost = reinterpret_cast<u64>(&n3.nextTagRes);
458+ n3.nextTagRes.nextHost = reinterpret_cast<u64>(&sentinel);
459+ 
460+ ListCommon dev1, dev2, dev3;
461+ sentinel.nextDevice = reinterpret_cast<u64>(&dev1);
462+ n1.nextTagRes.nextDevice = reinterpret_cast<u64>(&dev2);
463+ n2.nextTagRes.nextDevice = reinterpret_cast<u64>(&dev3);
464+ n3.nextTagRes.nextDevice = reinterpret_cast<u64>(&sentinel);
465+ 
466+ // 没有匹配tag，不应触发拷贝
467+ MOCKER(hrtMemSyncCopy).expects(atMost(0)).will(returnValue(HCCL_SUCCESS));
468+ 
469+ HcclResult ret = hcclCommunicator->CopyHostListResToDeviceParam(targetTag, &sentinel, sizeof(HccltagLocalResV2));
470+ 
471+ EXPECT_EQ(ret, HCCL_SUCCESS);
472+ GlobalMockObject::verify();
473+}
474+ 
475+TEST_F(HcclCommunicatorHostTest, Ut_CopyHostListResToDeviceParam_RefreshSingleNode_RemoteRes_TagFound_Expect_CopyOneOnly)
476+{
477+ std::unique_ptr<HcclCommunicator> hcclCommunicator(new (std::nothrow) HcclCommunicator());
478+ 
479+ std::string targetTag = "refresh_tag";
480+ hcclCommunicator->newTagResAlloced_.insert(targetTag);
481+ 
482+ // 使用 HccltagRemoteResV2 类型（size == sizeof(HccltagRemoteResV2) 分支）
483+ ListCommon sentinel;
484+ HccltagRemoteResV2 n1, n2;
485+ memset(&n1, 0, sizeof(n1));
486+ memset(&n2, 0, sizeof(n2));
487+ memcpy_s(n1.tag, TAG_MAX_LENGTH, "other_tag", sizeof("other_tag"));
488+ memcpy_s(n2.tag, TAG_MAX_LENGTH, targetTag.c_str(), targetTag.length() + 1);
489+ 
490+ sentinel.nextHost = reinterpret_cast<u64>(&n1.nextTagRes);
491+ n1.nextTagRes.nextHost = reinterpret_cast<u64>(&n2.nextTagRes);
492+ n2.nextTagRes.nextHost = reinterpret_cast<u64>(&sentinel);
493+ 
494+ ListCommon dev1, dev2;
495+ sentinel.nextDevice = reinterpret_cast<u64>(&dev1);
496+ n1.nextTagRes.nextDevice = reinterpret_cast<u64>(&dev2);
497+ n2.nextTagRes.nextDevice = reinterpret_cast<u64>(&sentinel);
498+ 
499+ // HccltagRemoteResV2 路径，匹配到第二个节点后 break
500+ MOCKER(hrtMemSyncCopy).expects(atMost(1)).will(returnValue(HCCL_SUCCESS));
501+ 
502+ HcclResult ret = hcclCommunicator->CopyHostListResToDeviceParam(
503+ targetTag, &sentinel, sizeof(HccltagRemoteResV2));
504+ 
505+ EXPECT_EQ(ret, HCCL_SUCCESS);
506+ GlobalMockObject::verify();
507+}
508+