Merged
perf(hccl): optimize parallel AllGather CCU data movement and scheduling #7357
perf(hccl): optimize parallel AllGather CCU data movement and scheduling #7357
Merged
xfou@hwAdd on 12 days ago
Total: 3 files changed+173-6
@@ -273,6 +273,71 @@ static CcuResult DoRepeatAllGather(KfcAllGatherMesh1DMem2MemContext& ctx, bool u
273 return CCU_SUCCESS;273 return CCU_SUCCESS;
274}274}
275 275 
276+static CcuResult DoParallelAllGatherGroupCopy(KfcAllGatherMesh1DMem2MemContext& ctx)
277+{
278+ CCU_IF(ctx.isInputOutputEqual == 0)
279+ {
280+ CCU_IF(ctx.localCopyRepeatNum != UINT64_MAX)
281+ {
282+ ctx.repeatTimeFlag = 0;
283+ CCU_WHILE(ctx.localCopyRepeatNum != UINT64_MAX)
284+ {
285+ ctx.localCopyRepeatNum += ctx.constVar1;
286+ CCU_IF(ctx.repeatTimeFlag != 0)
287+ {
288+ ctx.localDst.addr += ctx.outputRepeatStride;
289+ ctx.srcLocCopy.addr += ctx.inputRepeatStride;
290+ }
291+ CCU_CHK_RET(GroupCopy(ctx, ctx.localDst, ctx.srcLocCopy, ctx.goSize, GetCcuVersion()));
292+ ctx.repeatTimeFlag = 1;
293+ }
294+ }
295+ }
296+ return CCU_SUCCESS;
297+}
298+ 
299+static CcuResult DoParallelRepeatAllGather(KfcAllGatherMesh1DMem2MemContext& ctx)
300+{
301+ CCU_CHK_RET(InitAllGatherAddr(ctx));
302+ ctx.waitRepeatNum = ctx.tmpRepeatNum;
303+ ctx.localCopyRepeatNum = ctx.tmpRepeatNum;
304+ 
305+ // Phase 1: 先下发所有WriteNb(非阻塞,event错开),不包含GroupCopy
306+ CCU_IF(ctx.tmpRepeatNum != UINT64_MAX)
307+ {
308+ ctx.tmpRepeatNum += ctx.constVar1;
309+ CCU_CHK_RET(DoAllGatherWrite(ctx, ctx.src, ctx.dst, ctx.normalSliceSize, 0));
310+ }
311+ 
312+ for (uint32_t i = 1; i < AG_M2M_UNROLL_NUM; i++) {
313+ CCU_IF(ctx.tmpRepeatNum != UINT64_MAX)
314+ {
315+ ctx.tmpRepeatNum += ctx.constVar1;
316+ ctx.src.addr += ctx.inputRepeatStride;
317+ for (uint32_t rankIdx = 0; rankIdx < ctx.rankSize; rankIdx++) {
318+ if (rankIdx != ctx.rankId) {
319+ ctx.dst[rankIdx].addr += ctx.outputRepeatStride;
320+ }
321+ }
322+ CCU_CHK_RET(DoAllGatherWrite(ctx, ctx.src, ctx.dst, ctx.normalSliceSize, i));
323+ }
324+ }
325+ 
326+ // Phase 2: GroupCopy使用CCU_WHILE
327+ CCU_CHK_RET(DoParallelAllGatherGroupCopy(ctx));
328+ 
329+ // Phase 3: 批量WaitEvent
330+ for (uint32_t i = 0; i < AG_M2M_UNROLL_NUM; i++) {
331+ CCU_IF(ctx.waitRepeatNum != UINT64_MAX)
332+ {
333+ ctx.waitRepeatNum += ctx.constVar1;
334+ CCU_CHK_RET(DoAllGatherWait(ctx, i));
335+ }
336+ }
337+ 
338+ return CCU_SUCCESS;
339+}
340+ 
276static void AddVariableNTimes(ccu::Variable& result, const ccu::Variable& value, uint32_t times)341static void AddVariableNTimes(ccu::Variable& result, const ccu::Variable& value, uint32_t times)
277{342{
278 result = 0U;343 result = 0U;
@@ -321,6 +386,12 @@ CcuResult CcuKfcParallelAllGatherMesh1DMem2MemKernel(
321 ccu::Variable part1GoSize3, const ChannelHandle channels[], uint32_t channelCount, uint32_t rankSizeLevel0,386 ccu::Variable part1GoSize3, const ChannelHandle channels[], uint32_t channelCount, uint32_t rankSizeLevel0,
322 uint32_t rankIdxLevel0, uint32_t rankSizeLevel1, uint32_t rankIdxLevel1)387 uint32_t rankIdxLevel0, uint32_t rankSizeLevel1, uint32_t rankIdxLevel1)
323{388{
389+ // The HCCL sequence emits at most AG_M2M_UNROLL_NUM repeats.
390+ // Parallel Mesh stage 1 repeats once per level-1 rank.
391+ if (rankSizeLevel1 > AG_M2M_UNROLL_NUM) {
392+ HCCL_ERROR("Parallel Mesh level-1 size[%u] exceeds unroll[%u]", rankSizeLevel1, AG_M2M_UNROLL_NUM);
393+ return CcuResult::CCU_E_PARA;
394+ }
atomgit-bot
atomgit-botatomgit-bot12 days ago

🟡 Medium Priority

变更点:ccu_kernel_kfc_all_gather_mesh1d_mem2mem.cc:391-393 新增 if (rankSizeLevel1 > AG_M2M_UNROLL_NUM) { HCCL_ERROR(...); return CcuResult::CCU_E_PARA; },同时 430/447 行把原来的 DoRepeatAllGather(含外层 CCU_WHILE 批处理,可支持任意 repeat 数)换成不含外层循环的 DoParallelRepeatAllGather(最多只会发射 AG_M2M_UNROLL_NUM=4 次 repeat)。

证据链:

  1. 并行 Mesh 第二阶段的 repeat 次数来自 phase1RepeatNumInv = UINT64_MAX - rankSizeLevel1(第 420 行),即 repeat 数 == rankSizeLevel1;DoParallelRepeatAllGather 只有 for (i = 0; i < AG_M2M_UNROLL_NUM; ++i) 的展开,没有外层 while,因此 repeat > 4 时会漏发(这正是新守卫想拦的条件)。
  2. 但唯一的构建期调用点 impl/adv_api/detail/hccl/cc/src/ops/kfc_server/template/ccu/kernel/ccu_kernel_kfc_server.cc:245 直接调用 CcuKfcParallelAllGatherMesh1DMem2MemKernel(...) 且完全丢弃返回值(没有 CCU_CHK_RET/判错),所以新增的 CCU_E_PARA 不会传播成 HCCL_E_PARA,也不会触发算法回退。
  3. 早退位置在任何指令发射之前,函数尾部(第 434 行)的 ccu::Store(meshPhaseDoneAddr, one) 以及 PreSync/PostSync/所有 Write/GroupCopy 都不会被发射。

失败模式:当选中 CcuSchedAllGatherParallelMeshNHRMultiLink(UBX + MESH_1D_CLOS 单层拓扑)且 level-1 卡数 > 4(例如 mesh 2 × clos 8 之类的矩形配置)时,Mesh mission 的 CCU 程序为空,不写 meshPhaseDoneAddr;NHR mission 在 nhr kernel 第 497 行 CCU_WHILE(meshDone != 1U) 处无限空转,同时全局输出缺少 Mesh 阶段写回的数据 → 表现为通信挂死/超时或结果错误,而不是一个可诊断的参数错误。相对修改前(DoRepeatAllGather 的外层 WHILE 支持任意 repeat 数)这是功能回归,且新增上限没有任何测试覆盖。

建议:把容量校验上移到 host 侧并让并行算法返回 HCCL_E_PARA 以便回退(或恢复 DoParallelRepeatAllGather 外层 CCU_WHILE 以支持任意 repeat 数);若保留早退,必须让 ccu_kernel_kfc_server.cc 的调用点检查返回值,避免 mesh 程序残缺后 NHR 阶段死等 meshPhaseDoneAddr。

likedislike
Incorrect?
xfou@hw
xfou@hw
12 days ago Comments:
likedislike
324 KfcAllGatherMesh1DMem2MemContext ctx;395 KfcAllGatherMesh1DMem2MemContext ctx;
325 ctx.channels = channels;396 ctx.channels = channels;
326 ctx.channelCount = channelCount;397 ctx.channelCount = channelCount;
@@ -356,7 +427,7 @@ CcuResult CcuKfcParallelAllGatherMesh1DMem2MemKernel(
356 part0GoSize2, part0GoSize3, zero, phase0RepeatNumInv, zero, zero, part0Size, zero));427 part0GoSize2, part0GoSize3, zero, phase0RepeatNumInv, zero, zero, part0Size, zero));
357 CCU_CHK_RET(PreSync(ctx));428 CCU_CHK_RET(PreSync(ctx));
358 ctx.sliceSize = part0Size;429 ctx.sliceSize = part0Size;
359- CCU_IF(part0Size != 0) { CCU_CHK_RET(DoRepeatAllGather(ctx, false)); }430+ CCU_IF(part0Size != 0) { CCU_CHK_RET(DoParallelRepeatAllGather(ctx)); }
360 CCU_CHK_RET(PostSync(ctx));431 CCU_CHK_RET(PostSync(ctx));
361 432 
362 // Both missions must finish stage 0 before either reads its stage-0 output.433 // Both missions must finish stage 0 before either reads its stage-0 output.
@@ -373,7 +444,7 @@ CcuResult CcuKfcParallelAllGatherMesh1DMem2MemKernel(
373 part1Size, one));444 part1Size, one));
374 CCU_CHK_RET(PreSync(ctx));445 CCU_CHK_RET(PreSync(ctx));
375 ctx.sliceSize = part1Size;446 ctx.sliceSize = part1Size;
376- CCU_IF(part1Size != 0) { CCU_CHK_RET(DoRepeatAllGather(ctx, false)); }447+ CCU_IF(part1Size != 0) { CCU_CHK_RET(DoParallelRepeatAllGather(ctx)); }
377 CCU_CHK_RET(PostSync(ctx));448 CCU_CHK_RET(PostSync(ctx));
378 return CCU_SUCCESS;449 return CCU_SUCCESS;
379}450}
@@ -14,8 +14,8 @@
14 14 
15namespace mc2_ops_hccl {15namespace mc2_ops_hccl {
16 16 
17-// 每个 repeat 批次最多展开 16 次 Write/LocalCopy/Wait 三阶段流水。17+// 每个 repeat 批次最多展开 4 次 Write/LocalCopy/Wait 三阶段流水。
18-constexpr uint32_t AG_M2M_UNROLL_NUM = 1;18+constexpr uint32_t AG_M2M_UNROLL_NUM = 4;
19 19 
20CcuResult CcuKfcAllGatherMesh1DMem2MemKernel(20CcuResult CcuKfcAllGatherMesh1DMem2MemKernel(
21 ccu::Variable inputAddr, ccu::Variable outputAddr, ccu::Variable tokenInfo, ccu::Variable outputOffset,21 ccu::Variable inputAddr, ccu::Variable outputAddr, ccu::Variable tokenInfo, ccu::Variable outputOffset,
@@ -278,6 +278,102 @@ CcuResult RunNhr(KfcAllGatherNhrContext& ctx)
278 return CCU_SUCCESS;278 return CCU_SUCCESS;
279}279}
280 280 
281+CcuResult RunParallelStep(KfcAllGatherNhrContext& ctx, const KfcNhrStepInfo& stepInfo)
282+{
283+ const uint32_t toChannelIdx = ctx.rank2ChannelIdx->at(stepInfo.toRank);
284+ const uint32_t fromChannelIdx = ctx.rank2ChannelIdx->at(stepInfo.fromRank);
285+ const ChannelHandle sendChannel = ctx.channels[toChannelIdx];
286+ const ChannelHandle recvChannel = ctx.channels[fromChannelIdx];
287+ const uint32_t localIdx = ctx.channelCount;
288+ 
289+ CCU_CHK_RET(ccu::NotifyRecord(recvChannel, NHR_CKE_IDX, NHR_STEP_PRE_SYNC_BIT));
290+ CCU_CHK_RET(ccu::NotifyWait(sendChannel, NHR_CKE_IDX, NHR_STEP_PRE_SYNC_BIT));
291+ 
292+ // Match HCCL: slice outer loop, repeat inner loop, wait after each slice.
293+ for (uint32_t sliceIdx : stepInfo.txSliceIdxs) {
294+ ccu::LocalAddr src;
295+ if (sliceIdx == ctx.rankId) {
296+ src.addr = ctx.input;
297+ src.addr += ctx.myRankInputSliceOffset;
298+ } else {
299+ src.addr = ctx.output[localIdx];
300+ src.addr += ctx.outputSliceOffset[sliceIdx];
301+ }
302+ src.token = ctx.token[localIdx];
303+ ccu::RemoteAddr dst;
304+ dst.addr = ctx.output[toChannelIdx];
305+ dst.addr += ctx.outputSliceOffset[sliceIdx];
306+ dst.token = ctx.token[toChannelIdx];
307+ 
308+ ctx.repeatFlag = 0;
309+ ctx.repeatNum = ctx.repeatNumInv;
310+ CCU_WHILE(ctx.repeatNum != UINT64_MAX)
311+ {
312+ ctx.repeatNum += ctx.constVar1;
313+ CCU_IF(ctx.repeatFlag == 1)
314+ {
315+ src.addr += ctx.inputRepeatStride;
316+ dst.addr += ctx.outputRepeatStride;
317+ }
318+ CCU_CHK_RET(WriteSlice(ctx, stepInfo.toRank, src, dst));
319+ ctx.repeatFlag = 1U;
320+ }
321+ }
atomgit-bot
atomgit-botatomgit-bot12 days ago

🟡 Medium Priority

变更点:新增的 RunParallelStep(第 293-321 行)固定采用"slice 外层、repeat 内层、每片每轮都 WriteSlice 并在片内等待"的串行结构;并行 kernel 的 491/505 行把原本调用的 RunNhr(内部走 RunStep 的步内流水分支)替换为 RunParallelNhr → RunParallelStep。

证据链:

  1. RunStep(第 157-193 行)在同文件里已实现步内流水:sliceNum * jettyNum <= NHR_MJ_EVENT_BIT_BUDGET(16) 时,一轮 repeat 内先发射全部 slice(各占独立事件位),轮末只做一次 ccu::EventWait(ctx.event, waitMask);注释也写明"消除'发一个等一个'的完成等待空转间隙"。
  2. RunParallelStep 无条件走 WriteSlice(第 134-139 行),即 IssueSlice 后立即 EventWait((1<<jettyNum)-1),既不再使用 NHR_MJ_EVENT_BIT_BUDGET 分流,也不做步末批量等待。

失败模式(性能回归,非数据错误):每个 NHR step 的等待次数从"repeat 轮数"放大为"slice 数 × repeat 轮数"。NHR AllGather 中每个 step 的 txSliceIdxs 随 step 递增(累计片数可达数片),并行多 jetty 场景下每片的往返完成延迟无法被后续片的发射掩盖,正是本 PR(perf 目标)要消除的空转;对 level-1 卡数较大、slice 数较多的场景,并行 NHR 阶段耗时可能比修改前的 RunNhr 更差。PR 描述的"性能与 HCCL 不相上下"是在 NHR 分叉之前的版本上验证的,分叉后的顺序尚未上机。

建议:让 RunParallelStep 保留 RunStep 的步内流水(slice 内层 + 轮末一次 EventWait,事件位按 sliceSeq*jettyNum 展开),只在超出 16 位事件预算时回退为逐片 WriteSlice;避免并行路径丢失已有的批量等待优化。

likedislike
Incorrect?
xfou@hw
xfou@hw
12 days ago Comments:
likedislike
322+ CCU_CHK_RET(ccu::NotifyRecord(sendChannel, NHR_CKE_IDX, NHR_STEP_POST_SYNC_BIT));
323+ CCU_CHK_RET(ccu::NotifyWait(recvChannel, NHR_CKE_IDX, NHR_STEP_POST_SYNC_BIT));
324+ return CCU_SUCCESS;
325+}
326+ 
327+CcuResult RunParallelNhr(KfcAllGatherNhrContext& ctx)
328+{
329+ ccu::Variable sliceOffset;
330+ sliceOffset = 0;
331+ ctx.myRankInputSliceOffset = 0;
332+ for (uint32_t rank = 0; rank < ctx.rankId; ++rank) {
333+ ctx.myRankInputSliceOffset += ctx.inputSliceStride;
334+ }
335+ for (uint32_t rank = 0; rank < ctx.rankSize; ++rank) {
336+ ctx.outputSliceOffset[rank] = sliceOffset;
337+ sliceOffset += ctx.outputSliceStride;
338+ }
339+ for (const auto& stepInfo : *ctx.stepInfoVector) {
340+ CCU_CHK_RET(RunParallelStep(ctx, stepInfo));
341+ }
342+ 
343+ const uint32_t localIdx = ctx.channelCount;
344+ ccu::LocalAddr src;
345+ src.addr = ctx.input;
346+ src.addr += ctx.myRankInputSliceOffset;
347+ src.token = ctx.token[localIdx];
348+ ccu::LocalAddr dst;
349+ dst.addr = ctx.output[localIdx];
350+ dst.addr += ctx.outputSliceOffset[ctx.rankId];
351+ dst.token = ctx.token[localIdx];
352+ ctx.repeatFlag = 0;
353+ ctx.repeatNum = ctx.repeatNumInv;
354+ CCU_WHILE(ctx.repeatNum != UINT64_MAX)
355+ {
356+ ctx.repeatNum += ctx.constVar1;
357+ CCU_IF(ctx.repeatFlag != 0)
358+ {
359+ src.addr += ctx.inputRepeatStride;
360+ dst.addr += ctx.outputRepeatStride;
361+ }
362+ // WriteSlice 已等待并清除 event;本地 copy 每次也立即等待,复用 bit0。
363+ // 不能用 1<<rankId:NHR 子域可以超过 16 rank,而 event mask 只有 16 位。
364+ constexpr uint16_t rankMask = 1U;
365+ CCU_IF(ctx.isInputOutputEqual == 0)
366+ {
367+ CCU_CHK_RET(GroupCopy(ctx, dst, src, ctx.goSize, GetCcuVersion()));
368+ CCU_CHK_RET(ccu::EventRecord(ctx.event, rankMask));
369+ }
370+ CCU_ELSE { CCU_CHK_RET(ccu::EventRecord(ctx.event, rankMask)); }
371+ CCU_CHK_RET(ccu::EventWait(ctx.event, rankMask));
372+ ctx.repeatFlag = 1U;
373+ }
374+ return CCU_SUCCESS;
375+}
376+ 
281CcuResult LoadNhrArgs(377CcuResult LoadNhrArgs(
282 KfcAllGatherNhrContext& ctx, ccu::Variable input, ccu::Variable output, ccu::Variable token,378 KfcAllGatherNhrContext& ctx, ccu::Variable input, ccu::Variable output, ccu::Variable token,
283 ccu::Variable sliceSize, ccu::Variable sliceSizePerJetty, ccu::Variable lastSliceSizePerJetty,379 ccu::Variable sliceSize, ccu::Variable sliceSizePerJetty, ccu::Variable lastSliceSizePerJetty,
@@ -392,7 +488,7 @@ CcuResult CcuKfcParallelAllGatherNHR1DMultiJettyMem2MemKernel(
392 phase0RepeatNumInv, zero, level1Stride, zero, zero, zero, part1GoSize0, part1GoSize1, part1GoSize2,488 phase0RepeatNumInv, zero, level1Stride, zero, zero, zero, part1GoSize0, part1GoSize1, part1GoSize2,
393 part1GoSize3));489 part1GoSize3));
394 CCU_CHK_RET(PreSync(ctx));490 CCU_CHK_RET(PreSync(ctx));
395- CCU_IF(part1Size != 0) { CCU_CHK_RET(RunNhr(ctx)); }491+ CCU_IF(part1Size != 0) { CCU_CHK_RET(RunParallelNhr(ctx)); }
396 CCU_CHK_RET(PostSync(ctx));492 CCU_CHK_RET(PostSync(ctx));
397 493 
398 CCU_CHK_RET(ccu::Store(nhrPhaseDoneAddr, one));494 CCU_CHK_RET(ccu::Store(nhrPhaseDoneAddr, one));
@@ -406,7 +502,7 @@ CcuResult CcuKfcParallelAllGatherNHR1DMultiJettyMem2MemKernel(
406 phase1RepeatNumInv, level1Stride, level1Stride, outputStride, outputStride, one, part0GoSize0, part0GoSize1,502 phase1RepeatNumInv, level1Stride, level1Stride, outputStride, outputStride, one, part0GoSize0, part0GoSize1,
407 part0GoSize2, part0GoSize3));503 part0GoSize2, part0GoSize3));
408 CCU_CHK_RET(PreSync(ctx));504 CCU_CHK_RET(PreSync(ctx));
409- CCU_IF(part0Size != 0) { CCU_CHK_RET(RunNhr(ctx)); }505+ CCU_IF(part0Size != 0) { CCU_CHK_RET(RunParallelNhr(ctx)); }
410 CCU_CHK_RET(PostSync(ctx));506 CCU_CHK_RET(PostSync(ctx));
411 return CCU_SUCCESS;507 return CCU_SUCCESS;
412}508}