已合并
a3 back 2 back fix about bc #3433
Eternal创建于 7月6日
a3 back 2 back fix about bc #3433
已合并
共 5 个文件变更+328-2
| @@ -17,6 +17,7 @@ set(src_list | |||
| 17 | ${CMAKE_CURRENT_SOURCE_DIR}/coll_broadcast_mix_executor.cc | 17 | ${CMAKE_CURRENT_SOURCE_DIR}/coll_broadcast_mix_executor.cc |
| 18 | ${CMAKE_CURRENT_SOURCE_DIR}/coll_broadcast_ring_zerocopy_executor.cc | 18 | ${CMAKE_CURRENT_SOURCE_DIR}/coll_broadcast_ring_zerocopy_executor.cc |
| 19 | ${CMAKE_CURRENT_SOURCE_DIR}/coll_broadcast_mesh_aiv_executor.cc | 19 | ${CMAKE_CURRENT_SOURCE_DIR}/coll_broadcast_mesh_aiv_executor.cc |
| 20 | + ${CMAKE_CURRENT_SOURCE_DIR}/coll_broadcast_midcount_for_910_93_executor.cc | ||
| 20 | ) | 21 | ) |
| 21 | 22 | ||
| 22 | if(TARGET hccl_alg) | 23 | if(TARGET hccl_alg) |
| @@ -35,6 +36,7 @@ if(TARGET ccl_kernel) | |||
| 35 | ${CMAKE_CURRENT_SOURCE_DIR}/coll_broadcast_smallcount_executor.cc | 36 | ${CMAKE_CURRENT_SOURCE_DIR}/coll_broadcast_smallcount_executor.cc |
| 36 | ${CMAKE_CURRENT_SOURCE_DIR}/coll_broadcast_mix_executor.cc | 37 | ${CMAKE_CURRENT_SOURCE_DIR}/coll_broadcast_mix_executor.cc |
| 37 | ${CMAKE_CURRENT_SOURCE_DIR}/coll_broadcast_ring_zerocopy_executor.cc | 38 | ${CMAKE_CURRENT_SOURCE_DIR}/coll_broadcast_ring_zerocopy_executor.cc |
| 39 | + ${CMAKE_CURRENT_SOURCE_DIR}/coll_broadcast_midcount_for_910_93_executor.cc | ||
| 38 | ) | 40 | ) |
| 39 | target_include_directories(ccl_kernel PRIVATE | 41 | target_include_directories(ccl_kernel PRIVATE |
| 40 | ${CMAKE_CURRENT_SOURCE_DIR} | 42 | ${CMAKE_CURRENT_SOURCE_DIR} |
| @@ -0,0 +1,257 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | + | ||
| 12 | + | ||
| 13 | +namespace hccl { | ||
| 14 | +CollBroadcastMidCountFor91093Executor::CollBroadcastMidCountFor91093Executor(const HcclDispatcher dispatcher, | ||
| 15 | + std::unique_ptr<TopoMatcher> &topoMatcher) | ||
| 16 | + : CollBroadcastExecutor(dispatcher, topoMatcher) | ||
| 17 | +{ | ||
| 18 | + desc_.level1SupportedAlgos = { | ||
| 19 | + AlgTypeLevel1::ALG_LEVEL1_NHR, | ||
| 20 | + }; | ||
| 21 | + desc_.level2SupportedAlgos = { | ||
| 22 | + AlgTypeLevel2::ALG_LEVEL2_NHR, | ||
| 23 | + }; | ||
| 24 | +} | ||
| 25 | +HcclResult CollBroadcastMidCountFor91093Executor::CalcStreamNum(u32& streamNum) | ||
| 26 | +{ | ||
| 27 | + streamNum = 0; | ||
| 28 | + HCCL_INFO("[CollBroadcastCommExecutor][CalcStreamNum]tag[%s] streamNum_ is [%u]", tag_.c_str(), streamNum); | ||
| 29 | + return HCCL_SUCCESS; | ||
| 30 | +} | ||
| 31 | + | ||
| 32 | +HcclResult CollBroadcastMidCountFor91093Executor::CalcCommInfo(std::vector<LevelNSubCommTransport>& opTransport) | ||
| 33 | +{ | ||
| 34 | + TransportMemType inputType = TransportMemType::RESERVED; | ||
| 35 | + TransportMemType outputType = TransportMemType::RESERVED; | ||
| 36 | + CHK_RET(CalcTransportMemType(inputType, outputType)); | ||
| 37 | + CHK_RET(CalcLevel1CommInfo(inputType, outputType, opTransport)); | ||
| 38 | + CHK_RET(CalcLevel2CommInfo(inputType, outputType, opTransport)); | ||
| 39 | + return HCCL_SUCCESS; | ||
| 40 | +} | ||
| 41 | + | ||
| 42 | +HcclResult CollBroadcastMidCountFor91093Executor::CalcTransportMemType(TransportMemType &inputType, | ||
| 43 | + TransportMemType &outputType) const | ||
| 44 | +{ | ||
| 45 | + if (workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) { | ||
| 46 | + inputType = TransportMemType::CCL_INPUT; | ||
| 47 | + outputType = TransportMemType::CCL_OUTPUT; | ||
| 48 | + } else { | ||
| 49 | + HCCL_ERROR("BroadcastMidCountFor91093Executor do not support offload mode"); | ||
| 50 | + return HCCL_E_UNAVAIL; | ||
| 51 | + } | ||
| 52 | + HCCL_INFO("[CollBroadcastMidCountFor91093Executor][CalcTransportMemType] tag[%s] inputType[%d], outputType[%d]", | ||
| 53 | + tag_.c_str(), inputType, outputType); | ||
| 54 | + return HCCL_SUCCESS; | ||
| 55 | +} | ||
| 56 | + | ||
| 57 | +HcclResult CollBroadcastMidCountFor91093Executor::CalcLevel1CommInfo(TransportMemType inputType, | ||
| 58 | + TransportMemType outputType, | ||
| 59 | + std::vector<LevelNSubCommTransport>& opTransport) | ||
| 60 | +{ | ||
| 61 | + CommParaInfo commParaCombineL1(COMM_COMBINE_L1, CommType::COMM_TAG_NONUNIFORM_HIERARCHICAL_RING); | ||
| 62 | + CHK_RET(CalcCommPlaneInfo(tag_, commParaCombineL1, opTransport[COMM_COMBINE_L1], inputType, outputType)); | ||
| 63 | + return HCCL_SUCCESS; | ||
| 64 | +} | ||
| 65 | + | ||
| 66 | +HcclResult CollBroadcastMidCountFor91093Executor::CalcLevel2CommInfo(TransportMemType inputType, TransportMemType outputType, | ||
| 67 | + std::vector<LevelNSubCommTransport>& opTransport) | ||
| 68 | +{ | ||
| 69 | + CommParaInfo commParaLevel2(COMM_LEVEL2, CommType::COMM_TAG_NONUNIFORM_HIERARCHICAL_RING); | ||
| 70 | + CHK_RET(CalcCommPlaneInfo(tag_, commParaLevel2, opTransport[COMM_LEVEL2], inputType, outputType)); | ||
| 71 | + return HCCL_SUCCESS; | ||
| 72 | +} | ||
| 73 | + | ||
| 74 | +u64 CollBroadcastMidCountFor91093Executor::CalcLoopMaxCount(const u64 cclBuffSize, const u32 unitSize) | ||
| 75 | +{ | ||
| 76 | + u64 maxCountPerLoop = cclBuffSize / HCCL_MIN_SLICE_ALIGN_910_93 * HCCL_MIN_SLICE_ALIGN_910_93 / unitSize ; | ||
| 77 | + if (maxCountPerLoop == 0) { | ||
| 78 | + HCCL_ERROR("[CollBroadcastMidCountFor91093Executor][CalcLoopMaxCount] cclbuffer size is too small"); | ||
| 79 | + } | ||
| 80 | + return maxCountPerLoop; | ||
| 81 | +} | ||
| 82 | + | ||
| 83 | +HcclResult CollBroadcastMidCountFor91093Executor::RunLevel2ByNHR(const OpParam ¶m, ExecMem &execMem, | ||
| 84 | + SubCommInfo &level1CommInfo, SubCommInfo &level2CommInfo) const | ||
| 85 | +{ | ||
| 86 | + HCCL_CONFIG_INFO(HCCL_ALG, "[MidCountBroadcast][RunLevel2ByNHR] userRank[%u] starts.", topoAttr_.userRank); | ||
| 87 | + | ||
| 88 | + u32 unitSize = 0; | ||
| 89 | + const HcclDataType dataType = param.GetDataType(); | ||
| 90 | + CHK_RET(SalGetDataTypeSize(dataType, unitSize)); | ||
| 91 | + | ||
| 92 | + std::unique_ptr<AlgTemplateBase> tempAlg = AlgTemplateRegistry::Instance().GetAlgTemplate( | ||
| 93 | + TemplateType::TEMPLATE_BROADCAST_NHR_ONESHOT, dispatcher_); | ||
| 94 | + HCCL_CONFIG_INFO(HCCL_ALG, "[%s] Run TEMPLATE_BROADCAST_NHR_ONESHOT in COMM_LEVEL2", __func__); | ||
| 95 | + CHK_SMART_PTR_NULL(tempAlg); | ||
| 96 | + | ||
| 97 | + // 获取root | ||
| 98 | + u32 rootRank = param.root / level1CommInfo.localRankSize; | ||
| 99 | + CHK_RET(tempAlg->Prepare(execMem.inputMem, execMem.inputMem, execMem.inputMem, execMem.count, | ||
 | |||
| 100 | + param.DataDes.dataType, param.stream, HCCL_REDUCE_RESERVED, rootRank)); | ||
🟠 High Priority 在 触发条件：当 broadcast 的 root 不在 superpod 0 中时（例如 root 在 superpod 1），level1 广播完成后只有 root 所在 superpod 内的 rank 持有数据。level2 通信子中 rank 0 对应 superpod 0，它并不持有数据，但   | |||
| 101 | + | ||
| 102 | + CHK_RET(RunTemplate(tempAlg, level2CommInfo)); | ||
| 103 | + | ||
| 104 | + HCCL_INFO("MidCountBroadcast run success in level2"); | ||
| 105 | + return HCCL_SUCCESS; | ||
| 106 | +} | ||
| 107 | + | ||
| 108 | +HcclResult CollBroadcastMidCountFor91093Executor::RunLevel1ByNHR(const OpParam ¶m, ExecMem &execMem, | ||
| 109 | + SubCommInfo &level1CommInfo, SubCommInfo &level2CommInfo) | ||
| 110 | +{ | ||
| 111 | + HCCL_CONFIG_INFO(HCCL_ALG, "[MidCountBroadcast][RunLevel1ByNHR] userRank[%u] starts.", topoAttr_.userRank); | ||
| 112 | + u32 unitSize = 0; | ||
| 113 | + const HcclDataType dataType = param.GetDataType(); | ||
| 114 | + CHK_RET(SalGetDataTypeSize(dataType, unitSize)); | ||
| 115 | + | ||
| 116 | + std::unique_ptr<AlgTemplateBase> tempAlg = AlgTemplateRegistry::Instance().GetAlgTemplate( | ||
| 117 | + TemplateType::TEMPLATE_BROADCAST_NHR_ONESHOT, dispatcher_); | ||
| 118 | + HCCL_CONFIG_INFO(HCCL_ALG, "[%s] Run TEMPLATE_BROADCAST_NHR_ONESHOT in COMM_COMBINE_L1", __func__); | ||
| 119 | + CHK_SMART_PTR_NULL(tempAlg); | ||
| 120 | + | ||
| 121 | + // 获取root | ||
| 122 | + u32 rootRank = 0; | ||
| 123 | + CHK_RET(GetRankByUserRank(COMM_COMBINE_L1, COMM_INDEX_0, param.root, rootRank)); | ||
| 124 | + CHK_RET(tempAlg->Prepare(execMem.inputMem, execMem.inputMem, execMem.inputMem, execMem.count, | ||
| 125 | + param.DataDes.dataType, param.stream, HCCL_REDUCE_RESERVED, rootRank)); | ||
| 126 | + | ||
| 127 | + CHK_RET(RunTemplate(tempAlg, level1CommInfo)); | ||
| 128 | + | ||
| 129 | + HCCL_INFO("MidCountBroadcast run success in level1"); | ||
| 130 | + return HCCL_SUCCESS; | ||
| 131 | +} | ||
| 132 | + | ||
| 133 | +HcclResult CollBroadcastMidCountFor91093Executor::KernelRun(const OpParam ¶m, ExecMem &execMem) | ||
| 134 | +{ | ||
| 135 | + HCCL_CONFIG_INFO(HCCL_ALG, "[%s] The MidCountFor91093Executor starts", __func__); | ||
| 136 | + | ||
| 137 | + SubCommInfo level1CommInfo; | ||
| 138 | + SubCommInfo level2CommInfo; | ||
| 139 | + CHK_RET(CheckCommSize(COMM_COMBINE_L1, COMM_INDEX_0 + 1)); | ||
| 140 | + CHK_RET(CheckCommSize(COMM_LEVEL2, COMM_INDEX_0 + 1)); | ||
| 141 | + level1CommInfo = GetSubCommInfo(COMM_COMBINE_L1, COMM_INDEX_0); | ||
| 142 | + level2CommInfo = GetSubCommInfo(COMM_LEVEL2, COMM_INDEX_0); | ||
| 143 | + | ||
| 144 | + u32 unitSize = 0; | ||
| 145 | + const HcclDataType dataType = param.GetDataType(); | ||
| 146 | + CHK_RET(SalGetDataTypeSize(dataType, unitSize)); | ||
| 147 | + | ||
| 148 | + u32 rootSideIndex = param.root / level1CommInfo.localRankSize; | ||
| 149 | + u32 rankSideIndex = topoAttr_.userRank / level1CommInfo.localRankSize; | ||
| 150 | + if (rootSideIndex == rankSideIndex) { | ||
| 151 | + CHK_RET(RunLevel1ByNHR(param, execMem, level1CommInfo, level2CommInfo)); | ||
| 152 | + } | ||
| 153 | + CHK_RET(RunLevel2ByNHR(param, execMem, level1CommInfo, level2CommInfo)); | ||
| 154 | + | ||
| 155 | + HCCL_INFO("MidCountBroadcast run success."); | ||
| 156 | + return HCCL_SUCCESS; | ||
| 157 | +} | ||
| 158 | + | ||
| 159 | +HcclResult CollBroadcastMidCountFor91093Executor::RunLoopInner(OpParam ¶m, ExecMem &execMem) | ||
| 160 | +{ | ||
| 161 | + u32 unitSize = SIZE_TABLE[param.DataDes.dataType]; | ||
| 162 | + u64 totalSize = unitSize * param.DataDes.count; | ||
| 163 | + bool isRootRank = param.root == topoAttr_.realUserRank ? true : false; | ||
| 164 | + u64 curSize = execMem.count * unitSize; // 单位：字节 | ||
| 165 | + auto inCCLbufferSize = execMem.inputMem.size(); | ||
| 166 | + | ||
| 167 | + HCCL_DEBUG("[CollBroadcastMidCountFor91093Executor][RunLoopInner]inputMem[%p], outputMem[%p]" \ | ||
| 168 | + "intputPtr[%p], curCount[%llu], curSize[%llu]", | ||
| 169 | + execMem.inputMem.ptr(), execMem.outputMem.ptr(), execMem.inputPtr, execMem.count, curSize); | ||
| 170 | + CHK_PRT_RET((execMem.count == 0), | ||
| 171 | + HCCL_ERROR("[CollBroadcastMidCountFor91093Executor][RunLoop]In OP_BASE curCount is zero."), HCCL_E_PARA); | ||
| 172 | + | ||
| 173 | + bool hugeData = (inCCLbufferSize / topoAttr_.deviceNumPerAggregation > RDMA_SEND_MAX_SIZE) || | ||
| 174 | + (curSize > SDMA_SEND_MAX_SIZE); | ||
| 175 | + bool isSmallData = IsBroadcastSmallData(curSize, totalSize); | ||
| 176 | + u64 sliceNum = 0; | ||
| 177 | + CHK_RET(GetSliceNum(curSize, isSmallData, sliceNum)); | ||
| 178 | + CopyPattern copy = DMAReduceFlag_? CopyPattern::ZCOPY : CopyPattern::BCOPY; | ||
| 179 | + auto meta = HcclOpMetaInfo::GetOneForBroadcast(isRootRank, param.root, hugeData, isSmallData, sliceNum, copy); | ||
| 180 | + CHK_RET(InitTask(dispatcher_, param.stream, meta.isEnableCache, meta.GetCacheKey())); | ||
| 181 | + | ||
| 182 | + // 执行 | ||
| 183 | + HcclResult ret; | ||
| 184 | + // 如果使用in CCL buffer，需要将user buffer in中的结果拷贝到CCL buffer in | ||
| 185 | + DeviceMem inCommMem = execMem.inputMem.range(0, curSize); | ||
| 186 | + DeviceMem inMem(execMem.inputPtr, curSize); | ||
| 187 | + if (topoAttr_.userRank == param.root) { | ||
| 188 | + CHK_RET(HcclD2DMemcpyAsync(dispatcher_, inCommMem, inMem, param.stream)); | ||
| 189 | + } | ||
| 190 | + | ||
| 191 | + ret = KernelRun(param, execMem); | ||
| 192 | + if (topoAttr_.realUserRank != param.root) { | ||
| 193 | + CHK_RET(HcclD2DMemcpyAsync(dispatcher_, inMem, inCommMem, param.stream)); | ||
| 194 | + } | ||
| 195 | + | ||
| 196 | + CHK_PRT_RET(ret != HCCL_SUCCESS, | ||
| 197 | + HCCL_ERROR("[CollBroadcastMidCountFor91093Executor][RunLoop]errNo[0x%016llx]kernel run error, tag[%s], " \ | ||
| 198 | + "inputMem ptr[%p], count[%llu], dataType[%d]", | ||
| 199 | + HCCL_ERROR_CODE(ret), param.tag.c_str(), execMem.inputMem.ptr(), | ||
| 200 | + execMem.count, param.DataDes.dataType), ret); | ||
| 201 | + | ||
| 202 | + CHK_RET(LaunchTaskExtend(dispatcher_, param.stream, algResResp_->slaveStreams)); | ||
| 203 | + return ret; | ||
| 204 | +} | ||
| 205 | + | ||
| 206 | +HcclResult CollBroadcastMidCountFor91093Executor::Orchestrate(OpParam& param, AlgResourceResponse& algRes) | ||
| 207 | +{ | ||
| 208 | + tag_ = param.tag; | ||
| 209 | + algResResp_ = &algRes; | ||
| 210 | + /* ------------执行算法-------------- */ | ||
| 211 | + HcclUs startut = TIME_NOW(); | ||
| 212 | + | ||
| 213 | + // 图模式和单卡场景下不需要Loop | ||
| 214 | + ExecMem execMem; | ||
| 215 | + execMem.count = param.DataDes.count; | ||
| 216 | + execMem.inputPtr = param.inputPtr; | ||
| 217 | + execMem.outputPtr = param.inputPtr; | ||
| 218 | + HCCL_INFO("Orchestrate UserRank[%u], devicePhyId[%u], inputPtr[%p], outputPtr[%p], root[%u]", | ||
| 219 | + topoAttr_.userRank, topoAttr_.devicePhyId, param.inputPtr, param.outputPtr, param.root); | ||
| 220 | + | ||
| 221 | + u32 unitSize = SIZE_TABLE[param.DataDes.dataType]; | ||
| 222 | + u8 *curInputPtr = static_cast<u8 *>(param.inputPtr); | ||
| 223 | + CHK_PTR_NULL(curInputPtr); | ||
| 224 | + u64 maxCountPerLoop = CalcLoopMaxCount(algRes.cclInputMem.size(), unitSize); | ||
| 225 | + HCCL_DEBUG("[CollBroadcastMidCountFor91093Executor][RunLoop]tag[%s], userRankSize is [%u], maxCountPerLoop is [%llu].", | ||
| 226 | + param.tag.c_str(), topoAttr_.userRankSize, maxCountPerLoop); | ||
| 227 | + | ||
| 228 | + u64 totalCount = param.DataDes.count; | ||
| 229 | + for (u64 countLeft = totalCount, curCount = 0, inputOffset = 0; | ||
| 230 | + countLeft > 0; countLeft -= curCount) { | ||
| 231 | + curInputPtr += inputOffset; | ||
| 232 | + // 判断剩余数据量对应的output size是否大于中转output size | ||
| 233 | + curCount = (countLeft > maxCountPerLoop) ? maxCountPerLoop : countLeft; | ||
| 234 | + u64 curSize = curCount * unitSize; // 单位：字节 | ||
| 235 | + | ||
| 236 | + ExecMem execMem; | ||
| 237 | + execMem.count = curCount; | ||
| 238 | + execMem.inputMem = algRes.cclOutputMem; | ||
| 239 | + execMem.outputMem = algRes.cclOutputMem;// ccl buffer 均只使用out buffer | ||
| 240 | + execMem.inputPtr = curInputPtr; | ||
| 241 | + HCCL_DEBUG("[CollBroadcastMidCountFor91093Executor] RunLoop tag[%s], inputOffset[%llu], " \ | ||
| 242 | + "curInputPtr[%p], sendCount[%llu], sendSize[%llu], dataType[%s], realUserRank[%u]", | ||
| 243 | + param.tag.c_str(), inputOffset, curInputPtr, curCount, curSize, | ||
| 244 | + GetDataTypeEnumStr(param.DataDes.dataType).c_str(), topoAttr_.realUserRank); | ||
| 245 | + | ||
| 246 | + CHK_RET(RunLoopInner(param, execMem)); | ||
| 247 | + | ||
| 248 | + inputOffset = curSize; | ||
| 249 | + } | ||
| 250 | + | ||
| 251 | + HCCL_INFO("tag[%s], Broadcast executor orchestrate success, take time [%lld]us.", | ||
| 252 | + param.tag.c_str(), DURATION_US(TIME_NOW() - startut)); | ||
| 253 | + return HCCL_SUCCESS; | ||
| 254 | +} | ||
| 255 | + | ||
| 256 | +REGISTER_EXEC("BroadcastMidCountFor91093Executor", BroadcastMidCountFor91093, CollBroadcastMidCountFor91093Executor); | ||
| 257 | +} // namespace hccl | ||
| @@ -0,0 +1,39 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2025 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | + | ||
| 12 | + | ||
| 13 | + | ||
| 14 | +namespace hccl { | ||
| 15 | +class CollBroadcastMidCountFor91093Executor : public CollBroadcastExecutor { | ||
| 16 | +public: | ||
| 17 | + explicit CollBroadcastMidCountFor91093Executor(const HcclDispatcher dispatcher, std::unique_ptr<TopoMatcher> &topoMatcher); | ||
| 18 | + ~CollBroadcastMidCountFor91093Executor() override = default; | ||
| 19 | + HcclResult Orchestrate(OpParam& param, AlgResourceResponse& algRes) override; | ||
| 20 | +private: | ||
| 21 | + /* *************** 资源计算 *************** */ | ||
| 22 | + HcclResult CalcStreamNum(u32& streamNum) override; | ||
| 23 | + HcclResult CalcCommInfo(std::vector<LevelNSubCommTransport>& opTransport) override; | ||
| 24 | + HcclResult CalcTransportMemType(TransportMemType &inputType, TransportMemType &outputType) const; | ||
| 25 | + HcclResult CalcLevel1CommInfo(TransportMemType inputType, TransportMemType outputType, | ||
| 26 | + std::vector<LevelNSubCommTransport>& opTransport) override; | ||
| 27 | + HcclResult CalcLevel2CommInfo(TransportMemType inputType, TransportMemType outputType, | ||
| 28 | + std::vector<LevelNSubCommTransport>& opTransport) override; | ||
| 29 | + u64 CalcLoopMaxCount(const u64 cclBuffSize, const u32 unitSize) override; | ||
| 30 | + /* *************** 算法编排 *************** */ | ||
| 31 | + HcclResult RunLevel2ByNHR(const OpParam ¶m, ExecMem &execMem, SubCommInfo &level1CommInfo, SubCommInfo &level2CommInfo) const; | ||
| 32 | + HcclResult RunLevel1ByNHR(const OpParam ¶m, ExecMem &execMem, SubCommInfo &level1CommInfo, SubCommInfo &level2CommInfo); | ||
| 33 | + HcclResult KernelRun(const OpParam ¶m, ExecMem &execMem) override; | ||
| 34 | + HcclResult RunLoopInner(OpParam ¶m, ExecMem &execMem); | ||
| 35 | +}; | ||
| 36 | + | ||
| 37 | +} // namespace hccl | ||
| 38 | + | ||
| 39 | + | ||
| @@ -218,9 +218,11 @@ HcclResult BroadCastOperator::SelectAlgfor91093(const OpParam& param, std::strin | |||
| 218 | bool smallCountOptimMultiServer = | 218 | bool smallCountOptimMultiServer = |
| 219 | (deviceNumPerAggregation_ > HCCL_DEVICE_NUM_TWO) && (serverNum_ != 1) && (superPodNum_ == 1) && | 219 | (deviceNumPerAggregation_ > HCCL_DEVICE_NUM_TWO) && (serverNum_ != 1) && (superPodNum_ == 1) && |
| 220 | (param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] <= HCCL_SMALL_COUNT_1_MB * deviceNumPerAggregation_); | 220 | (param.DataDes.count * SIZE_TABLE[param.DataDes.dataType] <= HCCL_SMALL_COUNT_1_MB * deviceNumPerAggregation_); |
| 221 | - bool smallCountOptimMultiPod = (superPodNum_ > 1 || (GetExternalInputInterHccsDisable() && serverNum_ > 1)) && | 221 | + bool is2Pod2ServerTopo = (superPodNum_ == 2 && serverNum_ == 2);// 针对 A3背靠背机型 |
| 222 | + bool smallCountOptimMultiPod = (superPodNum_ > 1 || (GetExternalInputInterHccsDisable() && serverNum_ > 1)) && !is2Pod2ServerTopo && | ||
| 222 | (param.DataDes.count * unitSize <= HCCL_SMALL_COUNT_16_KB * deviceNumPerAggregation_) && !retryEnable_; // 涉及ROCE平面 | 223 | (param.DataDes.count * unitSize <= HCCL_SMALL_COUNT_16_KB * deviceNumPerAggregation_) && !retryEnable_; // 涉及ROCE平面 |
| 223 | - | 224 | + bool isBack2BackFor91093 = is2Pod2ServerTopo && (workflowMode_ == HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE) && |
| 225 | + (param.DataDes.count * unitSize <= HCCL_SMALL_COUNT_16_KB * deviceNumPerAggregation_) && !retryEnable_; | ||
| 224 | if (isAivMode_) { | 226 | if (isAivMode_) { |
| 225 | algName = "BroadcastMeshAivExecutor"; | 227 | algName = "BroadcastMeshAivExecutor"; |
| 226 | } else if (multiModuleDiffDeviceNumMode_ || multiSuperPodDiffServerNumMode_) { | 228 | } else if (multiModuleDiffDeviceNumMode_ || multiSuperPodDiffServerNumMode_) { |
| @@ -228,6 +230,8 @@ HcclResult BroadCastOperator::SelectAlgfor91093(const OpParam& param, std::strin | |||
| 228 | } else if (smallCountOptimMultiServer || smallCountOptimMultiPod) { | 230 | } else if (smallCountOptimMultiServer || smallCountOptimMultiPod) { |
| 229 | algName = "BroadCastComm"; | 231 | algName = "BroadCastComm"; |
| 230 | algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_NHR; | 232 | algType_.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_NHR; |
| 233 | + } else if (isBack2BackFor91093) { | ||
| 234 | + algName = "BroadcastMidCountFor91093Executor"; | ||
| 231 | } else if (smallCountOptimSingleServer) { | 235 | } else if (smallCountOptimSingleServer) { |
| 232 | algName = "BroadCastSmallCountExecutor"; | 236 | algName = "BroadCastSmallCountExecutor"; |
| 233 | } else if (param.supportZeroCopy && | 237 | } else if (param.supportZeroCopy && |
| @@ -753,4 +753,28 @@ TEST_F(BroadcastTest, broadcast_aivsmallcount) | |||
| 753 | Checker checker; | 753 | Checker checker; |
| 754 | HcclResult ret; | 754 | HcclResult ret; |
| 755 | ret = checker.Check(checkerOpParam, topoMeta); | 755 | ret = checker.Check(checkerOpParam, topoMeta); |
| 756 | +} | ||
| 757 | + | ||
| 758 | + | ||
| 759 | +TEST_F(BroadcastTest, broadcast_BroadcastMidCountFor91093Executor) | ||
| 760 | +{ | ||
| 761 | + setenv("HCCL_ALGO","level1:NHR",1); | ||
| 762 | + RankTable_For_LLT gen; | ||
| 763 | + TopoMeta topoMeta; | ||
| 764 | + gen.GenTopoMeta(topoMeta, 2, 1, 4); | ||
| 765 | + | ||
| 766 | + CheckerOpParam checkerOpParam; | ||
| 767 | + checkerOpParam.opType = CheckerOpType::BROADCAST; | ||
| 768 | + checkerOpParam.tag = "broadcast"; | ||
| 769 | + checkerOpParam.opMode = CheckerOpMode::OPBASE; | ||
| 770 | + checkerOpParam.DataDes.count = 10; | ||
| 771 | + checkerOpParam.root = 0; | ||
| 772 | + checkerOpParam.DataDes.dataType = CheckerDataType::DATA_TYPE_FP16; | ||
| 773 | + checkerOpParam.devtype = CheckerDevType::DEV_TYPE_910_93; | ||
| 774 | + checkerOpParam.algName = "BroadcastMidCountFor91093Executor"; // BroadCastComm BroadcastMidCountFor91093Executor | ||
| 775 | + | ||
| 776 | + Checker checker; | ||
| 777 | + HcclResult ret; | ||
| 778 | + checker.EnableTaskPrint(); | ||
| 779 | + ret = checker.Check(checkerOpParam, topoMeta); | ||
| 756 | } | 780 | } |
🟠 High Priority
在
coll_broadcast_midcount_for_910_93_ecexutor.cc第 99 行，RunLevel2ByNHR方法将rootRank硬编码为0，直接将未经转换的 rank 0 传入tempAlg->Prepare(...)作为 root。对比同类方法：
RunLevel1ByNHR（第 125 行）正确地调用了GetRankByUserRank(COMM_COMBINE_L1, COMM_INDEX_0, param.root, rootRank)将用户 root rank 映射为 level1 子通信域内的 rank。同时，CollBroadcastCommExecutor::KernelRun（coll_broadcast_comm_executor.cc第 125-126 行）也展示了正确的模式。故障场景：在 A3 背靠背拓扑中（2 superpod × 2 server），level2 通信平面连接两台服务器（跨 superpod）。若
param.root所在设备映射到 level2 通信域中的 rank 1（而非 rank 0），使用硬编码的rootRank=0会导致数据从错误的服务器广播，造成数据损坏。修复：在第 99 行将
u32 rootRank = 0;替换为：