已合并
ubx perf bugfix #2727
zhaojiayu创建于 8月22日
ubx perf bugfix #2727
已合并
zhaojiayu创建于 8月22日
共 6 个文件变更+48-6
@@ -498,7 +498,7 @@ SelectorStatus AllReduceAutoSelector::SelectMeshAlgoAicpuUBX(
498 }498 }
499 } else if (isDataTypeOrReduceTypeSpecial) {499 } else if (isDataTypeOrReduceTypeSpecial) {
500 selectAlgName = "AicpuAllReduceSoleNHRAicpuReduce";500 selectAlgName = "AicpuAllReduceSoleNHRAicpuReduce";
501- } else if (isClosNumMultipleOfMeshNum && !IsSmallData(dataSize)) {501+ } else if (isClosNumMultipleOfMeshNum && IsLargeData(dataSize)) {
502 if (opParam.supportSymmetricMemory) {502 if (opParam.supportSymmetricMemory) {
503 selectAlgName = "AicpuAllReducePipeLineUBX";503 selectAlgName = "AicpuAllReducePipeLineUBX";
504 } else {504 } else {
@@ -232,6 +232,10 @@ SelectorStatus BroadcastAutoSelector::SelectAicpuAlgo(
232SelectorStatus BroadcastAutoSelector::SelectMeshAlgoAicpu(232SelectorStatus BroadcastAutoSelector::SelectMeshAlgoAicpu(
233 const TopoInfoWithNetLayerDetails* topoInfo, const OpParam& opParam, std::string& selectAlgName) const233 const TopoInfoWithNetLayerDetails* topoInfo, const OpParam& opParam, std::string& selectAlgName) const
234{234{
235+ u64 perDataSize = DATATYPE_SIZE_TABLE[opParam.DataDes.dataType];
236+ u64 dataSize = opParam.DataDes.count * perDataSize;
237+ HCCL_DEBUG("[SelectMeshAlgoCcuSchedule] dataSize[%llu]", dataSize);
238+ 
235 if (topoInfo->level0Topo == Level0Shape::MESH_1D) {239 if (topoInfo->level0Topo == Level0Shape::MESH_1D) {
236 selectAlgName = "AicpuBroadcastSoleMeshTwoShot";240 selectAlgName = "AicpuBroadcastSoleMeshTwoShot";
237 } else if (topoInfo->level0Topo == Level0Shape::MESH_1D_CLOS) {241 } else if (topoInfo->level0Topo == Level0Shape::MESH_1D_CLOS) {
@@ -240,7 +244,11 @@ SelectorStatus BroadcastAutoSelector::SelectMeshAlgoAicpu(
240 } else if (topoInfo->level0PcieMix) {244 } else if (topoInfo->level0PcieMix) {
241 selectAlgName = "InsBroadcastParallelMesh1DNHRPcie";245 selectAlgName = "InsBroadcastParallelMesh1DNHRPcie";
242 } else {246 } else {
243- selectAlgName = "InsBroadcastParallelMesh1DNHRUBX";247+ if (IsLargeData(dataSize)) {
248+ selectAlgName = "InsBroadcastParallelMesh1DNHRUBX";
249+ } else {
250+ selectAlgName = "AicpuBroadcastSoleNHR";
251+ }
244 }252 }
245 } else if (topoInfo->level0Topo == Level0Shape::CLOS) {253 } else if (topoInfo->level0Topo == Level0Shape::CLOS) {
246 selectAlgName = "AicpuBroadcastSoleNHRTwoShotMultiLink";254 selectAlgName = "AicpuBroadcastSoleNHRTwoShotMultiLink";
@@ -25,7 +25,22 @@ HcclResult InsTempBroadcastNHR::CalcRes(
25 AlgResourceRequest& resourceRequest)25 AlgResourceRequest& resourceRequest)
26{26{
27 std::vector<HcclChannelDesc> level0Channels;27 std::vector<HcclChannelDesc> level0Channels;
28- CHK_RET(CalcChannelRequestNhr(comm, param, topoInfo, subCommRanks_, level0Channels));28+ // MESH_1D_CLOS 层0 CLOS/MESH 实例尺寸对称(GCD>1),Level1Nhr 不可达;本分支仅单级拓扑可达
29+ if (topoInfo->level0Topo == Level0Shape::MESH_1D_CLOS && !topoInfo->level0PcieMix) {
30+ std::vector<HcclChannelDesc> myChannelDescs;
31+ CHK_RET(CalcChannelRequestNhrMultiJetty(comm, param, topoInfo, subCommRanks_, myChannelDescs));
32+ for (auto channel : myChannelDescs) {
33+ if (channel.channelProtocol == COMM_PROTOCOL_UB_CTP) {
34+ level0Channels.push_back(channel);
35+ }
36+ }
37+ HCCL_DEBUG("[InsTempBroadcastNHR::CalcRes] Get Channel Success!");
38+ } else {
39+ CHK_RET(CalcChannelRequestNhr(comm, param, topoInfo, subCommRanks_, level0Channels));
40+ }
41+ CHK_PRT_RET(
42+ level0Channels.empty(), HCCL_ERROR("[InsTempBroadcastNHR::CalcRes] no UB_CTP channel after filter"),
43+ HcclResult::HCCL_E_INTERNAL);
29 resourceRequest.channels.push_back(level0Channels);44 resourceRequest.channels.push_back(level0Channels);
30 channelsPerRank_ = CalcChannelsPerRank(level0Channels);45 channelsPerRank_ = CalcChannelsPerRank(level0Channels);
31 if (channelsPerRank_ > MAX_JETTY_NUM) {46 if (channelsPerRank_ > MAX_JETTY_NUM) {
@@ -350,7 +350,11 @@ SelectorStatus ReduceAutoSelector::SelectMeshAlgoAicpu(
350 if (Is64BitDataType(opParam.DataDes.dataType) || opParam.reduceType == HcclReduceOp::HCCL_REDUCE_PROD) {350 if (Is64BitDataType(opParam.DataDes.dataType) || opParam.reduceType == HcclReduceOp::HCCL_REDUCE_PROD) {
351 selectAlgName = "AicpuReduceSoleNHRAicpuReduce";351 selectAlgName = "AicpuReduceSoleNHRAicpuReduce";
352 } else {352 } else {
353- selectAlgName = "ReduceParallelMesh1DNHRUBX";353+ if (IsLargeData(dataSize)) {
354+ selectAlgName = "ReduceParallelMesh1DNHRUBX";
355+ } else {
356+ selectAlgName = "AicpuReduceSoleNHR";
357+ }
354 }358 }
355 }359 }
356 } else if (topoInfo->level0Topo == Level0Shape::CLOS) {360 } else if (topoInfo->level0Topo == Level0Shape::CLOS) {
@@ -36,7 +36,22 @@ HcclResult ReduceNHR::CalcRes(
36 resourceRequest.notifyNumOnMainThread = threadNum - 1;36 resourceRequest.notifyNumOnMainThread = threadNum - 1;
37 37 
38 std::vector<HcclChannelDesc> level1Channels;38 std::vector<HcclChannelDesc> level1Channels;
39- CHK_RET(CalcChannelRequestNhr(comm, param, topoInfo, subCommRanks_, level1Channels));39+ // MESH_1D_CLOS 层0 CLOS/MESH 实例尺寸对称(GCD>1),Level1Nhr 不可达;本分支仅单级拓扑可达
40+ if (topoInfo->level0Topo == Level0Shape::MESH_1D_CLOS && !topoInfo->level0PcieMix) {
严正行严正行
严正行严正行8月26日

[MEDIUM] CalcRes 新增分支未守护 topoLevelNums,多级拓扑下 AicpuReduceSoleNHR 将因跨 pod 链路非 UB_CTP 而建链失败

问题分析:

  • 第 39 行分支条件只判断 level0Topo == MESH_1D_CLOS && !level0PcieMix,未限制单级拓扑。
  • AicpuReduceSoleNHR 在多级拓扑下同样会被选中(reduce_auto_selector.cc 第 299 行 Level1Nhr(layer0 实例数 GCD==1)路径),此时 TopoMatch1D 会把 infos[0][0] 填成全量 rank,NHR 环上对端可能跨 pod。
  • CalcChannelRequestNhrMultiJetty(channel.cc)只接受 linkAttr.linkProtocol == COMM_PROTOCOL_UB_CTP 的链路,跨 pod 链路通常为 UBOE/ROCE,匹配不到时直接报 "there is no link" 返回 HCCL_E_INTERNAL。
  • 旧路径 CalcChannelRequestNhr 走 GetProtocolByEngine(UB_CTP/PCIE/UBOE 多协议)可跨 pod 建链;本 PR 改动后该场景由可用变为直接报错。

建议修复: 分支条件增加单级拓扑守护(与 SelectMeshAlgoAicpu 仅在 topoLevelNums<=1 时被调用的语义对齐):

if (topoInfo->level0Topo == Level0Shape::MESH_1D_CLOS && !topoInfo->level0PcieMix
    && topoInfo->topoLevelNums == 1) {

若确认 Level1Nhr + MESH_1D_CLOS 组合不可达,请在注释中说明依据。

likedislike
zhaojiayu
zhaojiayu
8月26日 评论:
严正行严正行8月26日

[MEDIUM][复检] 回复称"已在注释说明",但当前 head 未找到该注释

复检事实:

  • 实读 head 8c54062b0 的本文件:第 39 行仅有 // UBX机型 & PCIE机型判断,grep 全文件无 Level1Nhr / 单级拓扑 / 不可达相关注释,与回复"已在注释说明"不符。
  • 技术依据本身经核验成立:SelectMeshAlgoAicpu 仅在 topoLevelNums<=1 时被调用(SelectAicpuAlgo 的 else 分支);多级拓扑下 MESH_1D_CLOS 实例尺寸对称(GCD>1)时 Level1Nhr=false,选择链最终 NOT_MATCH,不会实例化 ReduceNHR。

建议修复(二选一,成本都很低):

  • 按回复所述补一行注释,把依据留在代码里:
// MESH_1D_CLOS 层0 CLOS/MESH 实例尺寸对称(GCD>1),Level1Nhr 不可达;本分支仅单级拓扑可达
if (topoInfo->level0Topo == Level0Shape::MESH_1D_CLOS && !topoInfo->level0PcieMix) {
  • 或直接补守护,防御性更强:分支条件追加 && topoInfo->topoLevelNums == 1。
likedislike
zhaojiayu
zhaojiayu
8月26日 评论:
41+ std::vector<HcclChannelDesc> myChannelDescs;
42+ CHK_RET(CalcChannelRequestNhrMultiJetty(comm, param, topoInfo, subCommRanks_, myChannelDescs));
43+ for (const auto& channel : myChannelDescs) {
44+ if (channel.channelProtocol == COMM_PROTOCOL_UB_CTP) {
45+ level1Channels.push_back(channel);
严正行
严正行严正行8月26日

[MEDIUM] 每对端多条 UB_CTP 通道被全部申请但只使用 at(0),造成通道资源浪费

问题分析:

  • 第 41-46 行调用 CalcChannelRequestNhrMultiJetty,该函数按 multi-jetty 设计可为每个对端返回多条通道(兄弟模板 InsTempAllReduceNHR/InsTempReduceScatterNHR 均按 channelsPerRank 建多线程消费,MAX_JETTY_NUM=4)。
  • ReduceNHR 的线程模型固定单线程:第 33-36 行 threadNum=1、GetThreadNum() 恒返回 1;KernelRun 中(第 188/189/243/244 行)channels.at(...).at(0) 只取每个对端的第一条通道。
  • 若此拓扑下每对端返回多于一条通道,resourceRequest.channels 中多出的通道会被资源层真实创建(占用 jetty/notify 等设备资源)但从不使用,放大通道占用。

建议修复: 若本意只是换选路(选 port 127 的 CLOS UB_CTP 通道),建议每个对端只保留一条,例如过滤时跳过 remoteRank 已存在的通道;或按实际通道数对齐线程模型(参考 InsTempAllReduceNHR 的 channelsPerRank_ 做法)。

likedislike
zhaojiayu
zhaojiayu
8月26日 评论:
46+ }
47+ }
48+ HCCL_INFO("[ReduceNHR::CalcRes] channels total[%zu] ubctp[%zu]", myChannelDescs.size(), level1Channels.size());
49+ } else {
50+ CHK_RET(CalcChannelRequestNhr(comm, param, topoInfo, subCommRanks_, level1Channels));
51+ }
52+ CHK_PRT_RET(
53+ level1Channels.empty(), HCCL_ERROR("[ReduceNHR::CalcRes] no UB_CTP channel after filter"),
54+ HcclResult::HCCL_E_INTERNAL);
40 resourceRequest.channels.push_back(level1Channels);55 resourceRequest.channels.push_back(level1Channels);
41 HCCL_INFO(56 HCCL_INFO(
42 "[ReduceNHR][CalcRes]slaveThreadNum[%u] notifyNumPerThread[%u] notifyNumOnMainThread[%u]"57 "[ReduceNHR][CalcRes]slaveThreadNum[%u] notifyNumPerThread[%u] notifyNumOnMainThread[%u]"
@@ -682,7 +682,7 @@ SelectorStatus ReduceScatterAutoSelector::SelectMeshAlgoAicpuForMesh1DClos(
682 }682 }
683 } else if (Is64BitDataType(opParam.DataDes.dataType) || opParam.reduceType == HcclReduceOp::HCCL_REDUCE_PROD) {683 } else if (Is64BitDataType(opParam.DataDes.dataType) || opParam.reduceType == HcclReduceOp::HCCL_REDUCE_PROD) {
684 selectAlgName = "AicpuReduceScatterSoleNHRAicpuReduce";684 selectAlgName = "AicpuReduceScatterSoleNHRAicpuReduce";
685- } else if (isClosNumMultipleOfMeshNum && !IsSmallData(dataSize)) {685+ } else if (isClosNumMultipleOfMeshNum && IsLargeData(dataSize)) {
686 if (opParam.supportSymmetricMemory) {686 if (opParam.supportSymmetricMemory) {
687 selectAlgName = "AicpuReduceScatterPipeLineUBX";687 selectAlgName = "AicpuReduceScatterPipeLineUBX";
688 } else {688 } else {