| @@ -380,6 +380,10 @@ bool AdvanceStepTilingHelper::Tiling4Seqs() | |||
| 380 | int64_t perCoreInBufSize = 4 * AlignBytes(this->tokenEachReqs_ + | 380 | int64_t perCoreInBufSize = 4 * AlignBytes(this->tokenEachReqs_ + |
| 381 | ONE_BLOCK_BYTE / static_cast<int64_t>(sizeof(int64_t))) + | 381 | ONE_BLOCK_BYTE / static_cast<int64_t>(sizeof(int64_t))) + |
| 382 | AlignBytes(this->tokenEachReqs_, static_cast<int64_t>(sizeof(int32_t))) + ONE_BLOCK_BYTE; | 382 | AlignBytes(this->tokenEachReqs_, static_cast<int64_t>(sizeof(int32_t))) + ONE_BLOCK_BYTE; |
| 383 | + // 独立队列整改(与 kernel buffer4/buffer5 逐项对齐):sampledTokenIds/specToken 专用队列 | ||
| 384 | + // 各占 (AlignBytes(行宽,8)+32B)/seq,追加后 perLoopMaxSeqs 相应收紧、接受面不变 | ||
| 385 | + perCoreInBufSize += AlignBytes(this->tokenEachReqs_ + ONE_BLOCK_BYTE / static_cast<int64_t>(sizeof(int64_t))) + | ||
| 386 | + AlignBytes(this->tokenEachReqs_ - 1 + ONE_BLOCK_BYTE / static_cast<int64_t>(sizeof(int64_t))); | ||
| 383 | // 向量化行展开 scratch(kernel rowIdxBuf_/byteOffBuf_):2 x rowStride x 4B + acc32 4B; | 387 | // 向量化行展开 scratch(kernel rowIdxBuf_/byteOffBuf_):2 x rowStride x 4B + acc32 4B; |
| 384 | // rowStride 按 align 上界 AlignBytes(te,8)/8 + 4 元素计(与 kernel InitBuffer 预留一致) | 388 | // rowStride 按 align 上界 AlignBytes(te,8)/8 + 4 元素计(与 kernel InitBuffer 预留一致) |
| 385 | perCoreInBufSize += 2 * | 389 | perCoreInBufSize += 2 * |
| @@ -67,8 +67,16 @@ private: | |||
| 67 | 67 | ||
| 68 | TQueBind<QuePosition::VECIN, QuePosition::VECOUT, 1> buffer1; | 68 | TQueBind<QuePosition::VECIN, QuePosition::VECOUT, 1> buffer1; |
| 69 | TQueBind<QuePosition::VECIN, QuePosition::VECOUT, 1> buffer2; | 69 | TQueBind<QuePosition::VECIN, QuePosition::VECOUT, 1> buffer2; |
| 70 | - TQue<QuePosition::VECIN, 1> buffer3; | 70 | + // buffer3(acceptedNum→中间暂存→lastTokens,经 MTE3 拷出):同按绑定的 VECOUT(dst) |
| 71 | + // 侧释放,消除 lastToken MTE3 读与下轮 MTE2 写的跨 loop WAR 窗口 | ||
| 72 | + TQueBind<QuePosition::VECIN, QuePosition::VECOUT, 1> buffer3; | ||
| 71 | TQue<QuePosition::VECIN, 1> tempQue_; | 73 | TQue<QuePosition::VECIN, 1> tempQue_; |
| 74 | + // 独立队列整改:sampledTokenIds/specToken 不再复用 buffer1/2(同一 loop 两轮 MTE2 写 | ||
| 75 | + // 同一 UB 槽位被 racecheck 判定 WAW)。队列类型与原 buffer1/2 一致取 | ||
| 76 | + // TQueBind<VECIN,VECOUT>:FreeTensor 走绑定的 VECOUT(dst) 侧释放,MTE3 拷出与 | ||
| 77 | + // 下轮 MTE2 搬入按硬件 token 序(TQue<VECIN> 的释放走 MTE2 侧、丢失该次序) | ||
| 78 | + TQueBind<QuePosition::VECIN, QuePosition::VECOUT, 1> buffer4; | ||
| 79 | + TQueBind<QuePosition::VECIN, QuePosition::VECOUT, 1> buffer5; | ||
| 72 | 80 | ||
| 73 | TBuf<QuePosition::VECCALC> sampledTokenIdsBuf_; | 81 | TBuf<QuePosition::VECCALC> sampledTokenIdsBuf_; |
| 74 | TBuf<QuePosition::VECCALC> outputInt32Buf_; | 82 | TBuf<QuePosition::VECCALC> outputInt32Buf_; |
| @@ -141,6 +149,8 @@ __aicore__ inline void KernelAdvanceStepSpec<T>::Init(GM_ADDR input_tokens, GM_A | |||
| 141 | pipe_->InitBuffer(buffer3, 1, | 149 | pipe_->InitBuffer(buffer3, 1, |
| 142 | (AlignBytes(tokenEachReqs_, sizeof(T)) + BLOCK_BYTES) * perLoopSeqs_ + VEC_SCRATCH_ALIGN_SLACK); | 150 | (AlignBytes(tokenEachReqs_, sizeof(T)) + BLOCK_BYTES) * perLoopSeqs_ + VEC_SCRATCH_ALIGN_SLACK); |
| 143 | pipe_->InitBuffer(tempQue_, 1, (AlignBytes(tokenEachReqs_, sizeof(T)) + BLOCK_BYTES) * perLoopSeqs_); | 151 | pipe_->InitBuffer(tempQue_, 1, (AlignBytes(tokenEachReqs_, sizeof(T)) + BLOCK_BYTES) * perLoopSeqs_); |
| 152 | + pipe_->InitBuffer(buffer4, 1, (AlignBytes(tokenEachReqs_, sizeof(T)) + BLOCK_BYTES) * perLoopSeqs_); | ||
| 153 | + pipe_->InitBuffer(buffer5, 1, (AlignBytes(specNum_, sizeof(T)) + BLOCK_BYTES) * perLoopSeqs_); | ||
| 144 | 154 | ||
| 145 | pipe_->InitBuffer(sampledTokenIdsBuf_, AlignBytes(tokenEachReqs_ + 1, sizeof(float))); | 155 | pipe_->InitBuffer(sampledTokenIdsBuf_, AlignBytes(tokenEachReqs_ + 1, sizeof(float))); |
| 146 | pipe_->InitBuffer(outputInt32Buf_, (AlignBytes(tokenEachReqs_, sizeof(int32_t)) + BLOCK_BYTES) * perLoopSeqs_); | 156 | pipe_->InitBuffer(outputInt32Buf_, (AlignBytes(tokenEachReqs_, sizeof(int32_t)) + BLOCK_BYTES) * perLoopSeqs_); |
| @@ -286,7 +296,11 @@ __aicore__ inline void KernelAdvanceStepSpec<T>::CopyOut(int64_t loop, int64_t s | |||
| 286 | DataCopyPad(inputPositionsGm_[offset], inputPositionsLocal, dataCopyParams); | 296 | DataCopyPad(inputPositionsGm_[offset], inputPositionsLocal, dataCopyParams); |
| 287 | DataCopyPad(seqLensGm_[offset], seqLensLocal, dataCopyParams); | 297 | DataCopyPad(seqLensGm_[offset], seqLensLocal, dataCopyParams); |
| 288 | SetWaitFlag<HardEvent::MTE3_V>(HardEvent::MTE3_V); | 298 | SetWaitFlag<HardEvent::MTE3_V>(HardEvent::MTE3_V); |
| 289 | - buffer1.EnQue(inputPositionsLocal); | 299 | + // 生命周期收口:positions 拷出后本轮不再有 DeQue 消费者,FreeTensor 走绑定的 |
| 300 | + // VECOUT(dst) 侧按 MTE3 序归还槽位(EnQue 仅投递消费者事件、不释放,见 | ||
| 301 | + // kernel_tquebind_impl.h FreeBuffer);否则第二轮起 AllocTensor 等待未注册的 | ||
| 302 | + // freeBufEvt 或无序抢占在途槽位(racecheck RAW/WAR UB 来源) | ||
| 303 | + buffer1.FreeTensor(inputPositionsLocal); | ||
| 290 | buffer2.EnQue(seqLensLocal); | 304 | buffer2.EnQue(seqLensLocal); |
| 291 | } | 305 | } |
| 292 | 306 | ||
| @@ -512,14 +526,15 @@ __aicore__ inline void KernelAdvanceStepSpec<T>::CopyOutSlotMapping(int64_t loop | |||
| 512 | int64_t offset = loop * perLoopSeqs_ * tokenEachReqs_; | 526 | int64_t offset = loop * perLoopSeqs_ * tokenEachReqs_; |
| 513 | DataCopyPad(slotMappingGm_[offset], slotMappingLocal, dataCopyParams); | 527 | DataCopyPad(slotMappingGm_[offset], slotMappingLocal, dataCopyParams); |
| 514 | SetWaitFlag<HardEvent::MTE3_MTE2>(HardEvent::MTE3_MTE2); | 528 | SetWaitFlag<HardEvent::MTE3_MTE2>(HardEvent::MTE3_MTE2); |
| 515 | - buffer2.EnQue(slotMappingLocal); | 529 | + // 同 CopyOut 的 buffer1:本轮终点无后续 DeQue 消费者,FreeTensor 按 MTE3 序归还 |
| 530 | + buffer2.FreeTensor(slotMappingLocal); | ||
| 516 | } | 531 | } |
| 517 | 532 | ||
| 518 | template <typename T> | 533 | template <typename T> |
| 519 | __aicore__ inline void KernelAdvanceStepSpec<T>::CopyInInputTokens(int64_t loop, int64_t seqsNum) | 534 | __aicore__ inline void KernelAdvanceStepSpec<T>::CopyInInputTokens(int64_t loop, int64_t seqsNum) |
| 520 | { | 535 | { |
| 521 | - LocalTensor<T> sampledTokenIdsLocal = buffer1.DeQue<T>(); | 536 | + LocalTensor<T> sampledTokenIdsLocal = buffer4.AllocTensor<T>(); |
| 522 | - LocalTensor<T> specTokenLocal = buffer2.DeQue<T>(); | 537 | + LocalTensor<T> specTokenLocal = buffer5.AllocTensor<T>(); |
| 523 | 538 | ||
| 524 | // 搬运sampledTokenIds,其第2维是specNum+1 | 539 | // 搬运sampledTokenIds,其第2维是specNum+1 |
| 525 | int64_t offset = loop * perLoopSeqs_ * tokenEachReqs_; | 540 | int64_t offset = loop * perLoopSeqs_ * tokenEachReqs_; |
| @@ -536,14 +551,14 @@ __aicore__ inline void KernelAdvanceStepSpec<T>::CopyInInputTokens(int64_t loop, | |||
| 536 | DataCopyPadExtParams<T> specTokenDataCopyPadParams{false, 0, 0, 0}; | 551 | DataCopyPadExtParams<T> specTokenDataCopyPadParams{false, 0, 0, 0}; |
| 537 | DataCopyPad(specTokenLocal, specTokenGm_[offset], specTokenDataCopyParams, specTokenDataCopyPadParams); | 552 | DataCopyPad(specTokenLocal, specTokenGm_[offset], specTokenDataCopyParams, specTokenDataCopyPadParams); |
| 538 | 553 | ||
| 539 | - buffer1.EnQue(sampledTokenIdsLocal); | 554 | + buffer4.EnQue(sampledTokenIdsLocal); |
| 540 | - buffer2.EnQue(specTokenLocal); | 555 | + buffer5.EnQue(specTokenLocal); |
| 541 | } | 556 | } |
| 542 | 557 | ||
| 543 | template <typename T> | 558 | template <typename T> |
| 544 | __aicore__ inline void KernelAdvanceStepSpec<T>::ComputeInputTokens(int64_t loop, int64_t seqsNum) | 559 | __aicore__ inline void KernelAdvanceStepSpec<T>::ComputeInputTokens(int64_t loop, int64_t seqsNum) |
| 545 | { | 560 | { |
| 546 | - LocalTensor<T> sampledTokenIdsLocalInt64 = buffer1.DeQue<T>(); | 561 | + LocalTensor<T> sampledTokenIdsLocalInt64 = buffer4.DeQue<T>(); |
| 547 | LocalTensor<T> lastTokensInt64 = buffer3.DeQue<T>(); | 562 | LocalTensor<T> lastTokensInt64 = buffer3.DeQue<T>(); |
| 548 | 563 | ||
| 549 | LocalTensor<float> sampledTokenIdsLocalFp32 = sampledTokenIdsBuf_.Get<float>(); | 564 | LocalTensor<float> sampledTokenIdsLocalFp32 = sampledTokenIdsBuf_.Get<float>(); |
| @@ -574,13 +589,13 @@ __aicore__ inline void KernelAdvanceStepSpec<T>::ComputeInputTokens(int64_t loop | |||
| 574 | // 队列事件不覆盖 S→MTE3 依赖,需显式同步(参照 layer_norm_v4_single_read.h 同步范式) | 589 | // 队列事件不覆盖 S→MTE3 依赖,需显式同步(参照 layer_norm_v4_single_read.h 同步范式) |
| 575 | SetWaitFlag<HardEvent::S_MTE3>(HardEvent::S_MTE3); | 590 | SetWaitFlag<HardEvent::S_MTE3>(HardEvent::S_MTE3); |
| 576 | buffer3.EnQue(lastTokensInt64); | 591 | buffer3.EnQue(lastTokensInt64); |
| 577 | - buffer1.FreeTensor(sampledTokenIdsLocalInt64); | 592 | + buffer4.FreeTensor(sampledTokenIdsLocalInt64); |
🟡 Medium Priority 变更点: 影响:改前每轮循环的队列所有权是严格配平的 —— buffer1 在 失败模式:Ascend C 队列深度为 1 时,缓冲池为空后 ![]() ![]() Incorrect? | |||
| 578 | } | 593 | } |
| 579 | 594 | ||
| 580 | template <typename T> | 595 | template <typename T> |
| 581 | __aicore__ inline void KernelAdvanceStepSpec<T>::CopyOutInputTokens(int64_t loop, int64_t seqsNum) | 596 | __aicore__ inline void KernelAdvanceStepSpec<T>::CopyOutInputTokens(int64_t loop, int64_t seqsNum) |
| 582 | { | 597 | { |
| 583 | - LocalTensor<T> specTokenLocal = buffer2.DeQue<T>(); | 598 | + LocalTensor<T> specTokenLocal = buffer5.DeQue<T>(); |
| 584 | LocalTensor<T> lastTokensLocal = buffer3.DeQue<T>(); | 599 | LocalTensor<T> lastTokensLocal = buffer3.DeQue<T>(); |
| 585 | // 搬运lastToken到inputTokens的第1列 | 600 | // 搬运lastToken到inputTokens的第1列 |
| 586 | int64_t offset = loop * perLoopSeqs_ * tokenEachReqs_; | 601 | int64_t offset = loop * perLoopSeqs_ * tokenEachReqs_; |
| @@ -594,7 +609,7 @@ __aicore__ inline void KernelAdvanceStepSpec<T>::CopyOutInputTokens(int64_t loop | |||
| 594 | static_cast<uint32_t>(sizeof(T)), 0}; | 609 | static_cast<uint32_t>(sizeof(T)), 0}; |
| 595 | DataCopyPad(inputTokensGm_[offset + 1], specTokenLocal, specTokenDataCopyParams); | 610 | DataCopyPad(inputTokensGm_[offset + 1], specTokenLocal, specTokenDataCopyParams); |
| 596 | 611 | ||
| 597 | - buffer2.FreeTensor(specTokenLocal); | 612 | + buffer5.FreeTensor(specTokenLocal); |
| 598 | buffer3.FreeTensor(lastTokensLocal); | 613 | buffer3.FreeTensor(lastTokensLocal); |
| 599 | } | 614 | } |
| 600 | 615 | ||


【review】 问题分类:正确性 位置:optim/advance_step/op_host/advance_step_tiling.cpp:385,代码:perCoreInBufSize += AlignBytes(this->tokenEachReqs_ + ONE_BLOCK_BYTE / static_cast<int64_t>(sizeof(int64_t))) + 问题:新队列占用直接叠加到唯一的全载 tiling,导致实际支持的 specNum 上限下降,与上一行的“接受面不变”不符。以现有 Host UT 的 196608 字节 UB 为例,numSeqs=numQueries=1、specNum=3200、tokenEachReqs=3201 时,扣除 temp 和固定预留后可用 178912 字节。变更前单序列估算为 141204 字节,可分到一条序列;变更后为 192500 字节,第 403 行直接拒绝。 可能导致的结果:已有能落到全载路径的 V2 宽行输入会在 tiling 阶段新增失败;缩小每循环的序列数无法缓解单条序列已超预算。 建议的修改方法:为独立队列装不下的单宽行增加分段搬运或能保证事件顺序的低内存回退,并加入上述边界回归。若必须收窄支持范围,应更新接口约束及说明,并显式测试新的上限。