Merged
perf(hccl): optimize parallel AllGather CCU data movement and scheduling #7357
xfou@hwAdd on 12 days ago
perf(hccl): optimize parallel AllGather CCU data movement and scheduling #7357
Merged
Total: 3 files changed+173-6
| @@ -273,6 +273,71 @@ static CcuResult DoRepeatAllGather(KfcAllGatherMesh1DMem2MemContext& ctx, bool u | |||
| 273 | return CCU_SUCCESS; | 273 | return CCU_SUCCESS; |
| 274 | } | 274 | } |
| 275 | 275 | ||
| 276 | +static CcuResult DoParallelAllGatherGroupCopy(KfcAllGatherMesh1DMem2MemContext& ctx) | ||
| 277 | +{ | ||
| 278 | + CCU_IF(ctx.isInputOutputEqual == 0) | ||
| 279 | + { | ||
| 280 | + CCU_IF(ctx.localCopyRepeatNum != UINT64_MAX) | ||
| 281 | + { | ||
| 282 | + ctx.repeatTimeFlag = 0; | ||
| 283 | + CCU_WHILE(ctx.localCopyRepeatNum != UINT64_MAX) | ||
| 284 | + { | ||
| 285 | + ctx.localCopyRepeatNum += ctx.constVar1; | ||
| 286 | + CCU_IF(ctx.repeatTimeFlag != 0) | ||
| 287 | + { | ||
| 288 | + ctx.localDst.addr += ctx.outputRepeatStride; | ||
| 289 | + ctx.srcLocCopy.addr += ctx.inputRepeatStride; | ||
| 290 | + } | ||
| 291 | + CCU_CHK_RET(GroupCopy(ctx, ctx.localDst, ctx.srcLocCopy, ctx.goSize, GetCcuVersion())); | ||
| 292 | + ctx.repeatTimeFlag = 1; | ||
| 293 | + } | ||
| 294 | + } | ||
| 295 | + } | ||
| 296 | + return CCU_SUCCESS; | ||
| 297 | +} | ||
| 298 | + | ||
| 299 | +static CcuResult DoParallelRepeatAllGather(KfcAllGatherMesh1DMem2MemContext& ctx) | ||
| 300 | +{ | ||
| 301 | + CCU_CHK_RET(InitAllGatherAddr(ctx)); | ||
| 302 | + ctx.waitRepeatNum = ctx.tmpRepeatNum; | ||
| 303 | + ctx.localCopyRepeatNum = ctx.tmpRepeatNum; | ||
| 304 | + | ||
| 305 | + // Phase 1: 先下发所有WriteNb(非阻塞,event错开),不包含GroupCopy | ||
| 306 | + CCU_IF(ctx.tmpRepeatNum != UINT64_MAX) | ||
| 307 | + { | ||
| 308 | + ctx.tmpRepeatNum += ctx.constVar1; | ||
| 309 | + CCU_CHK_RET(DoAllGatherWrite(ctx, ctx.src, ctx.dst, ctx.normalSliceSize, 0)); | ||
| 310 | + } | ||
| 311 | + | ||
| 312 | + for (uint32_t i = 1; i < AG_M2M_UNROLL_NUM; i++) { | ||
| 313 | + CCU_IF(ctx.tmpRepeatNum != UINT64_MAX) | ||
| 314 | + { | ||
| 315 | + ctx.tmpRepeatNum += ctx.constVar1; | ||
| 316 | + ctx.src.addr += ctx.inputRepeatStride; | ||
| 317 | + for (uint32_t rankIdx = 0; rankIdx < ctx.rankSize; rankIdx++) { | ||
| 318 | + if (rankIdx != ctx.rankId) { | ||
| 319 | + ctx.dst[rankIdx].addr += ctx.outputRepeatStride; | ||
| 320 | + } | ||
| 321 | + } | ||
| 322 | + CCU_CHK_RET(DoAllGatherWrite(ctx, ctx.src, ctx.dst, ctx.normalSliceSize, i)); | ||
| 323 | + } | ||
| 324 | + } | ||
| 325 | + | ||
| 326 | + // Phase 2: GroupCopy使用CCU_WHILE | ||
| 327 | + CCU_CHK_RET(DoParallelAllGatherGroupCopy(ctx)); | ||
| 328 | + | ||
| 329 | + // Phase 3: 批量WaitEvent | ||
| 330 | + for (uint32_t i = 0; i < AG_M2M_UNROLL_NUM; i++) { | ||
| 331 | + CCU_IF(ctx.waitRepeatNum != UINT64_MAX) | ||
| 332 | + { | ||
| 333 | + ctx.waitRepeatNum += ctx.constVar1; | ||
| 334 | + CCU_CHK_RET(DoAllGatherWait(ctx, i)); | ||
| 335 | + } | ||
| 336 | + } | ||
| 337 | + | ||
| 338 | + return CCU_SUCCESS; | ||
| 339 | +} | ||
| 340 | + | ||
| 276 | static void AddVariableNTimes(ccu::Variable& result, const ccu::Variable& value, uint32_t times) | 341 | static void AddVariableNTimes(ccu::Variable& result, const ccu::Variable& value, uint32_t times) |
| 277 | { | 342 | { |
| 278 | result = 0U; | 343 | result = 0U; |
| @@ -321,6 +386,12 @@ CcuResult CcuKfcParallelAllGatherMesh1DMem2MemKernel( | |||
| 321 | ccu::Variable part1GoSize3, const ChannelHandle channels[], uint32_t channelCount, uint32_t rankSizeLevel0, | 386 | ccu::Variable part1GoSize3, const ChannelHandle channels[], uint32_t channelCount, uint32_t rankSizeLevel0, |
| 322 | uint32_t rankIdxLevel0, uint32_t rankSizeLevel1, uint32_t rankIdxLevel1) | 387 | uint32_t rankIdxLevel0, uint32_t rankSizeLevel1, uint32_t rankIdxLevel1) |
| 323 | { | 388 | { |
| 389 | + // The HCCL sequence emits at most AG_M2M_UNROLL_NUM repeats. | ||
| 390 | + // Parallel Mesh stage 1 repeats once per level-1 rank. | ||
| 391 | + if (rankSizeLevel1 > AG_M2M_UNROLL_NUM) { | ||
| 392 | + HCCL_ERROR("Parallel Mesh level-1 size[%u] exceeds unroll[%u]", rankSizeLevel1, AG_M2M_UNROLL_NUM); | ||
| 393 | + return CcuResult::CCU_E_PARA; | ||
| 394 | + } | ||
| 324 | KfcAllGatherMesh1DMem2MemContext ctx; | 395 | KfcAllGatherMesh1DMem2MemContext ctx; |
| 325 | ctx.channels = channels; | 396 | ctx.channels = channels; |
| 326 | ctx.channelCount = channelCount; | 397 | ctx.channelCount = channelCount; |
| @@ -356,7 +427,7 @@ CcuResult CcuKfcParallelAllGatherMesh1DMem2MemKernel( | |||
| 356 | part0GoSize2, part0GoSize3, zero, phase0RepeatNumInv, zero, zero, part0Size, zero)); | 427 | part0GoSize2, part0GoSize3, zero, phase0RepeatNumInv, zero, zero, part0Size, zero)); |
| 357 | CCU_CHK_RET(PreSync(ctx)); | 428 | CCU_CHK_RET(PreSync(ctx)); |
| 358 | ctx.sliceSize = part0Size; | 429 | ctx.sliceSize = part0Size; |
| 359 | - CCU_IF(part0Size != 0) { CCU_CHK_RET(DoRepeatAllGather(ctx, false)); } | 430 | + CCU_IF(part0Size != 0) { CCU_CHK_RET(DoParallelRepeatAllGather(ctx)); } |
| 360 | CCU_CHK_RET(PostSync(ctx)); | 431 | CCU_CHK_RET(PostSync(ctx)); |
| 361 | 432 | ||
| 362 | // Both missions must finish stage 0 before either reads its stage-0 output. | 433 | // Both missions must finish stage 0 before either reads its stage-0 output. |
| @@ -373,7 +444,7 @@ CcuResult CcuKfcParallelAllGatherMesh1DMem2MemKernel( | |||
| 373 | part1Size, one)); | 444 | part1Size, one)); |
| 374 | CCU_CHK_RET(PreSync(ctx)); | 445 | CCU_CHK_RET(PreSync(ctx)); |
| 375 | ctx.sliceSize = part1Size; | 446 | ctx.sliceSize = part1Size; |
| 376 | - CCU_IF(part1Size != 0) { CCU_CHK_RET(DoRepeatAllGather(ctx, false)); } | 447 | + CCU_IF(part1Size != 0) { CCU_CHK_RET(DoParallelRepeatAllGather(ctx)); } |
| 377 | CCU_CHK_RET(PostSync(ctx)); | 448 | CCU_CHK_RET(PostSync(ctx)); |
| 378 | return CCU_SUCCESS; | 449 | return CCU_SUCCESS; |
| 379 | } | 450 | } |
| @@ -14,8 +14,8 @@ | |||
| 14 | 14 | ||
| 15 | namespace mc2_ops_hccl { | 15 | namespace mc2_ops_hccl { |
| 16 | 16 | ||
| 17 | -// 每个 repeat 批次最多展开 16 次 Write/LocalCopy/Wait 三阶段流水。 | 17 | +// 每个 repeat 批次最多展开 4 次 Write/LocalCopy/Wait 三阶段流水。 |
| 18 | -constexpr uint32_t AG_M2M_UNROLL_NUM = 1; | 18 | +constexpr uint32_t AG_M2M_UNROLL_NUM = 4; |
| 19 | 19 | ||
| 20 | CcuResult CcuKfcAllGatherMesh1DMem2MemKernel( | 20 | CcuResult CcuKfcAllGatherMesh1DMem2MemKernel( |
| 21 | ccu::Variable inputAddr, ccu::Variable outputAddr, ccu::Variable tokenInfo, ccu::Variable outputOffset, | 21 | ccu::Variable inputAddr, ccu::Variable outputAddr, ccu::Variable tokenInfo, ccu::Variable outputOffset, |
| @@ -278,6 +278,102 @@ CcuResult RunNhr(KfcAllGatherNhrContext& ctx) | |||
| 278 | return CCU_SUCCESS; | 278 | return CCU_SUCCESS; |
| 279 | } | 279 | } |
| 280 | 280 | ||
| 281 | +CcuResult RunParallelStep(KfcAllGatherNhrContext& ctx, const KfcNhrStepInfo& stepInfo) | ||
| 282 | +{ | ||
| 283 | + const uint32_t toChannelIdx = ctx.rank2ChannelIdx->at(stepInfo.toRank); | ||
| 284 | + const uint32_t fromChannelIdx = ctx.rank2ChannelIdx->at(stepInfo.fromRank); | ||
| 285 | + const ChannelHandle sendChannel = ctx.channels[toChannelIdx]; | ||
| 286 | + const ChannelHandle recvChannel = ctx.channels[fromChannelIdx]; | ||
| 287 | + const uint32_t localIdx = ctx.channelCount; | ||
| 288 | + | ||
| 289 | + CCU_CHK_RET(ccu::NotifyRecord(recvChannel, NHR_CKE_IDX, NHR_STEP_PRE_SYNC_BIT)); | ||
| 290 | + CCU_CHK_RET(ccu::NotifyWait(sendChannel, NHR_CKE_IDX, NHR_STEP_PRE_SYNC_BIT)); | ||
| 291 | + | ||
| 292 | + // Match HCCL: slice outer loop, repeat inner loop, wait after each slice. | ||
| 293 | + for (uint32_t sliceIdx : stepInfo.txSliceIdxs) { | ||
| 294 | + ccu::LocalAddr src; | ||
| 295 | + if (sliceIdx == ctx.rankId) { | ||
| 296 | + src.addr = ctx.input; | ||
| 297 | + src.addr += ctx.myRankInputSliceOffset; | ||
| 298 | + } else { | ||
| 299 | + src.addr = ctx.output[localIdx]; | ||
| 300 | + src.addr += ctx.outputSliceOffset[sliceIdx]; | ||
| 301 | + } | ||
| 302 | + src.token = ctx.token[localIdx]; | ||
| 303 | + ccu::RemoteAddr dst; | ||
| 304 | + dst.addr = ctx.output[toChannelIdx]; | ||
| 305 | + dst.addr += ctx.outputSliceOffset[sliceIdx]; | ||
| 306 | + dst.token = ctx.token[toChannelIdx]; | ||
| 307 | + | ||
| 308 | + ctx.repeatFlag = 0; | ||
| 309 | + ctx.repeatNum = ctx.repeatNumInv; | ||
| 310 | + CCU_WHILE(ctx.repeatNum != UINT64_MAX) | ||
| 311 | + { | ||
| 312 | + ctx.repeatNum += ctx.constVar1; | ||
| 313 | + CCU_IF(ctx.repeatFlag == 1) | ||
| 314 | + { | ||
| 315 | + src.addr += ctx.inputRepeatStride; | ||
| 316 | + dst.addr += ctx.outputRepeatStride; | ||
| 317 | + } | ||
| 318 | + CCU_CHK_RET(WriteSlice(ctx, stepInfo.toRank, src, dst)); | ||
| 319 | + ctx.repeatFlag = 1U; | ||
| 320 | + } | ||
| 321 | + } | ||
🟡 Medium Priority 变更点:新增的 证据链:
失败模式(性能回归,非数据错误):每个 NHR step 的等待次数从"repeat 轮数"放大为"slice 数 × repeat 轮数"。NHR AllGather 中每个 step 的 建议:让 RunParallelStep 保留 RunStep 的步内流水(slice 内层 + 轮末一次 EventWait,事件位按 sliceSeq*jettyNum 展开),只在超出 16 位事件预算时回退为逐片 WriteSlice;避免并行路径丢失已有的批量等待优化。 ![]() ![]() Incorrect? | |||
| 322 | + CCU_CHK_RET(ccu::NotifyRecord(sendChannel, NHR_CKE_IDX, NHR_STEP_POST_SYNC_BIT)); | ||
| 323 | + CCU_CHK_RET(ccu::NotifyWait(recvChannel, NHR_CKE_IDX, NHR_STEP_POST_SYNC_BIT)); | ||
| 324 | + return CCU_SUCCESS; | ||
| 325 | +} | ||
| 326 | + | ||
| 327 | +CcuResult RunParallelNhr(KfcAllGatherNhrContext& ctx) | ||
| 328 | +{ | ||
| 329 | + ccu::Variable sliceOffset; | ||
| 330 | + sliceOffset = 0; | ||
| 331 | + ctx.myRankInputSliceOffset = 0; | ||
| 332 | + for (uint32_t rank = 0; rank < ctx.rankId; ++rank) { | ||
| 333 | + ctx.myRankInputSliceOffset += ctx.inputSliceStride; | ||
| 334 | + } | ||
| 335 | + for (uint32_t rank = 0; rank < ctx.rankSize; ++rank) { | ||
| 336 | + ctx.outputSliceOffset[rank] = sliceOffset; | ||
| 337 | + sliceOffset += ctx.outputSliceStride; | ||
| 338 | + } | ||
| 339 | + for (const auto& stepInfo : *ctx.stepInfoVector) { | ||
| 340 | + CCU_CHK_RET(RunParallelStep(ctx, stepInfo)); | ||
| 341 | + } | ||
| 342 | + | ||
| 343 | + const uint32_t localIdx = ctx.channelCount; | ||
| 344 | + ccu::LocalAddr src; | ||
| 345 | + src.addr = ctx.input; | ||
| 346 | + src.addr += ctx.myRankInputSliceOffset; | ||
| 347 | + src.token = ctx.token[localIdx]; | ||
| 348 | + ccu::LocalAddr dst; | ||
| 349 | + dst.addr = ctx.output[localIdx]; | ||
| 350 | + dst.addr += ctx.outputSliceOffset[ctx.rankId]; | ||
| 351 | + dst.token = ctx.token[localIdx]; | ||
| 352 | + ctx.repeatFlag = 0; | ||
| 353 | + ctx.repeatNum = ctx.repeatNumInv; | ||
| 354 | + CCU_WHILE(ctx.repeatNum != UINT64_MAX) | ||
| 355 | + { | ||
| 356 | + ctx.repeatNum += ctx.constVar1; | ||
| 357 | + CCU_IF(ctx.repeatFlag != 0) | ||
| 358 | + { | ||
| 359 | + src.addr += ctx.inputRepeatStride; | ||
| 360 | + dst.addr += ctx.outputRepeatStride; | ||
| 361 | + } | ||
| 362 | + // WriteSlice 已等待并清除 event;本地 copy 每次也立即等待,复用 bit0。 | ||
| 363 | + // 不能用 1<<rankId:NHR 子域可以超过 16 rank,而 event mask 只有 16 位。 | ||
| 364 | + constexpr uint16_t rankMask = 1U; | ||
| 365 | + CCU_IF(ctx.isInputOutputEqual == 0) | ||
| 366 | + { | ||
| 367 | + CCU_CHK_RET(GroupCopy(ctx, dst, src, ctx.goSize, GetCcuVersion())); | ||
| 368 | + CCU_CHK_RET(ccu::EventRecord(ctx.event, rankMask)); | ||
| 369 | + } | ||
| 370 | + CCU_ELSE { CCU_CHK_RET(ccu::EventRecord(ctx.event, rankMask)); } | ||
| 371 | + CCU_CHK_RET(ccu::EventWait(ctx.event, rankMask)); | ||
| 372 | + ctx.repeatFlag = 1U; | ||
| 373 | + } | ||
| 374 | + return CCU_SUCCESS; | ||
| 375 | +} | ||
| 376 | + | ||
| 281 | CcuResult LoadNhrArgs( | 377 | CcuResult LoadNhrArgs( |
| 282 | KfcAllGatherNhrContext& ctx, ccu::Variable input, ccu::Variable output, ccu::Variable token, | 378 | KfcAllGatherNhrContext& ctx, ccu::Variable input, ccu::Variable output, ccu::Variable token, |
| 283 | ccu::Variable sliceSize, ccu::Variable sliceSizePerJetty, ccu::Variable lastSliceSizePerJetty, | 379 | ccu::Variable sliceSize, ccu::Variable sliceSizePerJetty, ccu::Variable lastSliceSizePerJetty, |
| @@ -392,7 +488,7 @@ CcuResult CcuKfcParallelAllGatherNHR1DMultiJettyMem2MemKernel( | |||
| 392 | phase0RepeatNumInv, zero, level1Stride, zero, zero, zero, part1GoSize0, part1GoSize1, part1GoSize2, | 488 | phase0RepeatNumInv, zero, level1Stride, zero, zero, zero, part1GoSize0, part1GoSize1, part1GoSize2, |
| 393 | part1GoSize3)); | 489 | part1GoSize3)); |
| 394 | CCU_CHK_RET(PreSync(ctx)); | 490 | CCU_CHK_RET(PreSync(ctx)); |
| 395 | - CCU_IF(part1Size != 0) { CCU_CHK_RET(RunNhr(ctx)); } | 491 | + CCU_IF(part1Size != 0) { CCU_CHK_RET(RunParallelNhr(ctx)); } |
| 396 | CCU_CHK_RET(PostSync(ctx)); | 492 | CCU_CHK_RET(PostSync(ctx)); |
| 397 | 493 | ||
| 398 | CCU_CHK_RET(ccu::Store(nhrPhaseDoneAddr, one)); | 494 | CCU_CHK_RET(ccu::Store(nhrPhaseDoneAddr, one)); |
| @@ -406,7 +502,7 @@ CcuResult CcuKfcParallelAllGatherNHR1DMultiJettyMem2MemKernel( | |||
| 406 | phase1RepeatNumInv, level1Stride, level1Stride, outputStride, outputStride, one, part0GoSize0, part0GoSize1, | 502 | phase1RepeatNumInv, level1Stride, level1Stride, outputStride, outputStride, one, part0GoSize0, part0GoSize1, |
| 407 | part0GoSize2, part0GoSize3)); | 503 | part0GoSize2, part0GoSize3)); |
| 408 | CCU_CHK_RET(PreSync(ctx)); | 504 | CCU_CHK_RET(PreSync(ctx)); |
| 409 | - CCU_IF(part0Size != 0) { CCU_CHK_RET(RunNhr(ctx)); } | 505 | + CCU_IF(part0Size != 0) { CCU_CHK_RET(RunParallelNhr(ctx)); } |
| 410 | CCU_CHK_RET(PostSync(ctx)); | 506 | CCU_CHK_RET(PostSync(ctx)); |
| 411 | return CCU_SUCCESS; | 507 | return CCU_SUCCESS; |
| 412 | } | 508 | } |


🟡 Medium Priority
变更点:ccu_kernel_kfc_all_gather_mesh1d_mem2mem.cc:391-393 新增
if (rankSizeLevel1 > AG_M2M_UNROLL_NUM) { HCCL_ERROR(...); return CcuResult::CCU_E_PARA; },同时 430/447 行把原来的DoRepeatAllGather(含外层CCU_WHILE批处理,可支持任意 repeat 数)换成不含外层循环的DoParallelRepeatAllGather(最多只会发射 AG_M2M_UNROLL_NUM=4 次 repeat)。证据链:
phase1RepeatNumInv = UINT64_MAX - rankSizeLevel1(第 420 行),即 repeat 数 == rankSizeLevel1;DoParallelRepeatAllGather只有for (i = 0; i < AG_M2M_UNROLL_NUM; ++i)的展开,没有外层 while,因此 repeat > 4 时会漏发(这正是新守卫想拦的条件)。impl/adv_api/detail/hccl/cc/src/ops/kfc_server/template/ccu/kernel/ccu_kernel_kfc_server.cc:245直接调用CcuKfcParallelAllGatherMesh1DMem2MemKernel(...)且完全丢弃返回值(没有 CCU_CHK_RET/判错),所以新增的 CCU_E_PARA 不会传播成 HCCL_E_PARA,也不会触发算法回退。ccu::Store(meshPhaseDoneAddr, one)以及 PreSync/PostSync/所有 Write/GroupCopy 都不会被发射。失败模式:当选中
CcuSchedAllGatherParallelMeshNHRMultiLink(UBX + MESH_1D_CLOS 单层拓扑)且 level-1 卡数 > 4(例如 mesh 2 × clos 8 之类的矩形配置)时,Mesh mission 的 CCU 程序为空,不写meshPhaseDoneAddr;NHR mission 在 nhr kernel 第 497 行CCU_WHILE(meshDone != 1U)处无限空转,同时全局输出缺少 Mesh 阶段写回的数据 → 表现为通信挂死/超时或结果错误,而不是一个可诊断的参数错误。相对修改前(DoRepeatAllGather的外层 WHILE 支持任意 repeat 数)这是功能回归,且新增上限没有任何测试覆盖。建议:把容量校验上移到 host 侧并让并行算法返回 HCCL_E_PARA 以便回退(或恢复 DoParallelRepeatAllGather 外层 CCU_WHILE 以支持任意 repeat 数);若保留早退,必须让 ccu_kernel_kfc_server.cc 的调用点检查返回值,避免 mesh 程序残缺后 NHR 阶段死等 meshPhaseDoneAddr。