已合并
ubx perf bugfix #2727
zhaojiayu创建于 8月22日
ubx perf bugfix #2727
已合并
共 6 个文件变更+48-6
| @@ -498,7 +498,7 @@ SelectorStatus AllReduceAutoSelector::SelectMeshAlgoAicpuUBX( | |||
| 498 | } | 498 | } |
| 499 | } else if (isDataTypeOrReduceTypeSpecial) { | 499 | } else if (isDataTypeOrReduceTypeSpecial) { |
| 500 | selectAlgName = "AicpuAllReduceSoleNHRAicpuReduce"; | 500 | selectAlgName = "AicpuAllReduceSoleNHRAicpuReduce"; |
| 501 | - } else if (isClosNumMultipleOfMeshNum && !IsSmallData(dataSize)) { | 501 | + } else if (isClosNumMultipleOfMeshNum && IsLargeData(dataSize)) { |
| 502 | if (opParam.supportSymmetricMemory) { | 502 | if (opParam.supportSymmetricMemory) { |
| 503 | selectAlgName = "AicpuAllReducePipeLineUBX"; | 503 | selectAlgName = "AicpuAllReducePipeLineUBX"; |
| 504 | } else { | 504 | } else { |
| @@ -232,6 +232,10 @@ SelectorStatus BroadcastAutoSelector::SelectAicpuAlgo( | |||
| 232 | SelectorStatus BroadcastAutoSelector::SelectMeshAlgoAicpu( | 232 | SelectorStatus BroadcastAutoSelector::SelectMeshAlgoAicpu( |
| 233 | const TopoInfoWithNetLayerDetails* topoInfo, const OpParam& opParam, std::string& selectAlgName) const | 233 | const TopoInfoWithNetLayerDetails* topoInfo, const OpParam& opParam, std::string& selectAlgName) const |
| 234 | { | 234 | { |
| 235 | + u64 perDataSize = DATATYPE_SIZE_TABLE[opParam.DataDes.dataType]; | ||
| 236 | + u64 dataSize = opParam.DataDes.count * perDataSize; | ||
| 237 | + HCCL_DEBUG("[SelectMeshAlgoCcuSchedule] dataSize[%llu]", dataSize); | ||
| 238 | + | ||
| 235 | if (topoInfo->level0Topo == Level0Shape::MESH_1D) { | 239 | if (topoInfo->level0Topo == Level0Shape::MESH_1D) { |
| 236 | selectAlgName = "AicpuBroadcastSoleMeshTwoShot"; | 240 | selectAlgName = "AicpuBroadcastSoleMeshTwoShot"; |
| 237 | } else if (topoInfo->level0Topo == Level0Shape::MESH_1D_CLOS) { | 241 | } else if (topoInfo->level0Topo == Level0Shape::MESH_1D_CLOS) { |
| @@ -240,7 +244,11 @@ SelectorStatus BroadcastAutoSelector::SelectMeshAlgoAicpu( | |||
| 240 | } else if (topoInfo->level0PcieMix) { | 244 | } else if (topoInfo->level0PcieMix) { |
| 241 | selectAlgName = "InsBroadcastParallelMesh1DNHRPcie"; | 245 | selectAlgName = "InsBroadcastParallelMesh1DNHRPcie"; |
| 242 | } else { | 246 | } else { |
| 243 | - selectAlgName = "InsBroadcastParallelMesh1DNHRUBX"; | 247 | + if (IsLargeData(dataSize)) { |
| 248 | + selectAlgName = "InsBroadcastParallelMesh1DNHRUBX"; | ||
| 249 | + } else { | ||
| 250 | + selectAlgName = "AicpuBroadcastSoleNHR"; | ||
| 251 | + } | ||
| 244 | } | 252 | } |
| 245 | } else if (topoInfo->level0Topo == Level0Shape::CLOS) { | 253 | } else if (topoInfo->level0Topo == Level0Shape::CLOS) { |
| 246 | selectAlgName = "AicpuBroadcastSoleNHRTwoShotMultiLink"; | 254 | selectAlgName = "AicpuBroadcastSoleNHRTwoShotMultiLink"; |
| @@ -25,7 +25,22 @@ HcclResult InsTempBroadcastNHR::CalcRes( | |||
| 25 | AlgResourceRequest& resourceRequest) | 25 | AlgResourceRequest& resourceRequest) |
| 26 | { | 26 | { |
| 27 | std::vector<HcclChannelDesc> level0Channels; | 27 | std::vector<HcclChannelDesc> level0Channels; |
| 28 | - CHK_RET(CalcChannelRequestNhr(comm, param, topoInfo, subCommRanks_, level0Channels)); | 28 | + // MESH_1D_CLOS 层0 CLOS/MESH 实例尺寸对称(GCD>1),Level1Nhr 不可达;本分支仅单级拓扑可达 |
| 29 | + if (topoInfo->level0Topo == Level0Shape::MESH_1D_CLOS && !topoInfo->level0PcieMix) { | ||
| 30 | + std::vector<HcclChannelDesc> myChannelDescs; | ||
| 31 | + CHK_RET(CalcChannelRequestNhrMultiJetty(comm, param, topoInfo, subCommRanks_, myChannelDescs)); | ||
| 32 | + for (auto channel : myChannelDescs) { | ||
| 33 | + if (channel.channelProtocol == COMM_PROTOCOL_UB_CTP) { | ||
| 34 | + level0Channels.push_back(channel); | ||
| 35 | + } | ||
| 36 | + } | ||
| 37 | + HCCL_DEBUG("[InsTempBroadcastNHR::CalcRes] Get Channel Success!"); | ||
| 38 | + } else { | ||
| 39 | + CHK_RET(CalcChannelRequestNhr(comm, param, topoInfo, subCommRanks_, level0Channels)); | ||
| 40 | + } | ||
| 41 | + CHK_PRT_RET( | ||
| 42 | + level0Channels.empty(), HCCL_ERROR("[InsTempBroadcastNHR::CalcRes] no UB_CTP channel after filter"), | ||
| 43 | + HcclResult::HCCL_E_INTERNAL); | ||
| 29 | resourceRequest.channels.push_back(level0Channels); | 44 | resourceRequest.channels.push_back(level0Channels); |
| 30 | channelsPerRank_ = CalcChannelsPerRank(level0Channels); | 45 | channelsPerRank_ = CalcChannelsPerRank(level0Channels); |
| 31 | if (channelsPerRank_ > MAX_JETTY_NUM) { | 46 | if (channelsPerRank_ > MAX_JETTY_NUM) { |
| @@ -350,7 +350,11 @@ SelectorStatus ReduceAutoSelector::SelectMeshAlgoAicpu( | |||
| 350 | if (Is64BitDataType(opParam.DataDes.dataType) || opParam.reduceType == HcclReduceOp::HCCL_REDUCE_PROD) { | 350 | if (Is64BitDataType(opParam.DataDes.dataType) || opParam.reduceType == HcclReduceOp::HCCL_REDUCE_PROD) { |
| 351 | selectAlgName = "AicpuReduceSoleNHRAicpuReduce"; | 351 | selectAlgName = "AicpuReduceSoleNHRAicpuReduce"; |
| 352 | } else { | 352 | } else { |
| 353 | - selectAlgName = "ReduceParallelMesh1DNHRUBX"; | 353 | + if (IsLargeData(dataSize)) { |
| 354 | + selectAlgName = "ReduceParallelMesh1DNHRUBX"; | ||
| 355 | + } else { | ||
| 356 | + selectAlgName = "AicpuReduceSoleNHR"; | ||
| 357 | + } | ||
| 354 | } | 358 | } |
| 355 | } | 359 | } |
| 356 | } else if (topoInfo->level0Topo == Level0Shape::CLOS) { | 360 | } else if (topoInfo->level0Topo == Level0Shape::CLOS) { |
| @@ -36,7 +36,22 @@ HcclResult ReduceNHR::CalcRes( | |||
| 36 | resourceRequest.notifyNumOnMainThread = threadNum - 1; | 36 | resourceRequest.notifyNumOnMainThread = threadNum - 1; |
| 37 | 37 | ||
| 38 | std::vector<HcclChannelDesc> level1Channels; | 38 | std::vector<HcclChannelDesc> level1Channels; |
| 39 | - CHK_RET(CalcChannelRequestNhr(comm, param, topoInfo, subCommRanks_, level1Channels)); | 39 | + // MESH_1D_CLOS 层0 CLOS/MESH 实例尺寸对称(GCD>1),Level1Nhr 不可达;本分支仅单级拓扑可达 |
| 40 | + if (topoInfo->level0Topo == Level0Shape::MESH_1D_CLOS && !topoInfo->level0PcieMix) { | ||
[MEDIUM][复检] 回复称"已在注释说明",但当前 head 未找到该注释 复检事实:
建议修复(二选一,成本都很低):
![]() ![]() | |||
| 41 | + std::vector<HcclChannelDesc> myChannelDescs; | ||
| 42 | + CHK_RET(CalcChannelRequestNhrMultiJetty(comm, param, topoInfo, subCommRanks_, myChannelDescs)); | ||
| 43 | + for (const auto& channel : myChannelDescs) { | ||
| 44 | + if (channel.channelProtocol == COMM_PROTOCOL_UB_CTP) { | ||
| 45 | + level1Channels.push_back(channel); | ||
[MEDIUM] 每对端多条 UB_CTP 通道被全部申请但只使用 at(0),造成通道资源浪费 问题分析:
建议修复: 若本意只是换选路(选 port 127 的 CLOS UB_CTP 通道),建议每个对端只保留一条,例如过滤时跳过 remoteRank 已存在的通道;或按实际通道数对齐线程模型(参考 InsTempAllReduceNHR 的 channelsPerRank_ 做法)。 ![]() ![]() | |||
| 46 | + } | ||
| 47 | + } | ||
| 48 | + HCCL_INFO("[ReduceNHR::CalcRes] channels total[%zu] ubctp[%zu]", myChannelDescs.size(), level1Channels.size()); | ||
| 49 | + } else { | ||
| 50 | + CHK_RET(CalcChannelRequestNhr(comm, param, topoInfo, subCommRanks_, level1Channels)); | ||
| 51 | + } | ||
| 52 | + CHK_PRT_RET( | ||
| 53 | + level1Channels.empty(), HCCL_ERROR("[ReduceNHR::CalcRes] no UB_CTP channel after filter"), | ||
| 54 | + HcclResult::HCCL_E_INTERNAL); | ||
| 40 | resourceRequest.channels.push_back(level1Channels); | 55 | resourceRequest.channels.push_back(level1Channels); |
| 41 | HCCL_INFO( | 56 | HCCL_INFO( |
| 42 | "[ReduceNHR][CalcRes]slaveThreadNum[%u] notifyNumPerThread[%u] notifyNumOnMainThread[%u]" | 57 | "[ReduceNHR][CalcRes]slaveThreadNum[%u] notifyNumPerThread[%u] notifyNumOnMainThread[%u]" |
| @@ -682,7 +682,7 @@ SelectorStatus ReduceScatterAutoSelector::SelectMeshAlgoAicpuForMesh1DClos( | |||
| 682 | } | 682 | } |
| 683 | } else if (Is64BitDataType(opParam.DataDes.dataType) || opParam.reduceType == HcclReduceOp::HCCL_REDUCE_PROD) { | 683 | } else if (Is64BitDataType(opParam.DataDes.dataType) || opParam.reduceType == HcclReduceOp::HCCL_REDUCE_PROD) { |
| 684 | selectAlgName = "AicpuReduceScatterSoleNHRAicpuReduce"; | 684 | selectAlgName = "AicpuReduceScatterSoleNHRAicpuReduce"; |
| 685 | - } else if (isClosNumMultipleOfMeshNum && !IsSmallData(dataSize)) { | 685 | + } else if (isClosNumMultipleOfMeshNum && IsLargeData(dataSize)) { |
| 686 | if (opParam.supportSymmetricMemory) { | 686 | if (opParam.supportSymmetricMemory) { |
| 687 | selectAlgName = "AicpuReduceScatterPipeLineUBX"; | 687 | selectAlgName = "AicpuReduceScatterPipeLineUBX"; |
| 688 | } else { | 688 | } else { |


[MEDIUM] CalcRes 新增分支未守护 topoLevelNums,多级拓扑下 AicpuReduceSoleNHR 将因跨 pod 链路非 UB_CTP 而建链失败
问题分析:
level0Topo == MESH_1D_CLOS && !level0PcieMix,未限制单级拓扑。Level1Nhr(layer0 实例数 GCD==1)路径),此时 TopoMatch1D 会把 infos[0][0] 填成全量 rank,NHR 环上对端可能跨 pod。linkAttr.linkProtocol == COMM_PROTOCOL_UB_CTP的链路,跨 pod 链路通常为 UBOE/ROCE,匹配不到时直接报 "there is no link" 返回 HCCL_E_INTERNAL。建议修复: 分支条件增加单级拓扑守护(与 SelectMeshAlgoAicpu 仅在 topoLevelNums<=1 时被调用的语义对齐):
if (topoInfo->level0Topo == Level0Shape::MESH_1D_CLOS && !topoInfo->level0PcieMix && topoInfo->topoLevelNums == 1) {若确认 Level1Nhr + MESH_1D_CLOS 组合不可达,请在注释中说明依据。