* Copyright (c) 2026 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
* \file mod_bcast_impl.h
* \brief Mod<T,ST,OT> 融合广播 (fused general-broadcast) 成员定义 (arch22-only)。
*
* 融合广播:OUTER 行广播每核只把 x2 的 [INNER] 行读一次 -> UB 常驻复用 (消 mte2 冗余);INNER 列广播每行
* Duplicate (避 GatherMask)。物化广播 divisor 到 fp32,直接跑 RemainderAdaptive -> 精简计算/buffer
* (perElem ~44 B/elem)。
*
* 隔离:全 `#if MOD_ENH_ARCH22` 守卫;仅当 host 判定资格 (fp32/fp16/bf16/int16 同 dtype + collapse-2D)
* 置 bcastFusedMode_ != 0 才进入。通用 ProcessBroadcast / mixed / int32 广播 全部走不到融合 (独立运行时
* 分支、风险隔离)。融合分支自带整套精简 buffer,InitConstants 对融合路跳过 (精简核不用 inf/nan/zero 常量)。
* 非 arch22 -> 整块编译移除,Process() 回落通用 ProcessBroadcast。
*
* 0811 tile 塌陷修复 (评委 0811 复测 Test_036/044 = 24.4×/35.9× 回退的根因):
* 原资格要求 INNER 32B 对齐 -> 非对齐 inner (如 95/5) 的 same-dtype 广播 + 全部 int16 广播落官方通用
* ProcessBroadcast,tile 被 x2 连续段长卡死 (tile=inner=5/95 元素) -> 每 tile 队列/GM 固定开销不摊薄。
* 修复 = padding 行布局:UB 内 self/out/otherF32 统一 [rows, bcIpad_] 排布 (bcIpad_ =
* ceil(inner*sizeof(dtype)/32)*32/sizeof(dtype),与 DataCopyPad blockCount 模式自动 padding 落块 lockstep),
* 行首恒 32B 对齐 -> 任意 inner 可融合;int16 same-dtype 入列 (fp32 域 RemainderAdaptive 对 int16 输入
* 精确, 下行 CAST_RINT 与 K2 现用法一致 -> 与通用路 bit 级一致)。pad 车道消毒:x1 两 ping-pong 槽每核
* priming 置 0 (防 fp 脏车道 inf/nan 污染 maxAbsA 探针);x2 恒预填 1.0 (pad 商=0/1=0 良性);pad 结果被
* CopyOut 的 blockLen=inner 丢弃,永不出 UB。
*
* 数值等价:精简核直接 RemainderAdaptive 直算 fp32 域,cast 收尾 CAST_RINT。RemainderAdaptive/AlgoA 不改写
* aIn/bIn -> OUTER 物化 divisor 常驻块跨 tile 复用安全。inf/nan 由 naive/AlgoA 自然传播。
*
* 与 mod_flat_impl.h / mod_compute_impl.h 同惯例:本文件不自带 `namespace ModNs`,从 mod.h 的 namespace 块内
* #include -> 定义附着到 ModNs::Mod<T,ST,OT>。
*/
#ifndef MOD_BCAST_IMPL_H
#define MOD_BCAST_IMPL_H
#if MOD_ENH_ARCH22
template <typename T, typename ST, typename OT>
__aicore__ inline void Mod<T, ST, OT>::InitFusedBcastBuffers()
{
pipe.InitBuffer(inputx1Queue, bufferNum, actualMaxDataCount * sizeof(ST));
pipe.InitBuffer(outputQueue, bufferNum, actualMaxDataCount * sizeof(T));
uint64_t rawElems = (bcastFusedMode_ == 1U) ? bcInner_ : bcUbFormer_;
uint32_t rawAligned = static_cast<uint32_t>((rawElems + DATA_BLOCK - 1) / DATA_BLOCK * DATA_BLOCK);
if (rawAligned < DATA_BLOCK) {
rawAligned = DATA_BLOCK;
}
pipe.InitBuffer(inputx2Queue, bufferNum, rawAligned * sizeof(OT));
pipe.InitBuffer(bcastOtherBuf_, actualMaxDataCount * sizeof(float));
pipe.InitBuffer(ResQuotTensorBuff, maxDataCount * sizeof(float));
pipe.InitBuffer(ResRemTensorBuff, maxDataCount * sizeof(float));
pipe.InitBuffer(A1Buff, maxDataCount * sizeof(float));
pipe.InitBuffer(A2Buff, maxDataCount * sizeof(float));
pipe.InitBuffer(A3Buff, maxDataCount * sizeof(float));
pipe.InitBuffer(A4Buff, maxDataCount * sizeof(float));
if constexpr (NEED_FP32_IO_BUF) {
pipe.InitBuffer(A5Buff, maxDataCount * sizeof(float));
pipe.InitBuffer(x1TensorFP32Buff, maxDataCount * sizeof(float));
pipe.InitBuffer(x2TensorFP32Buff, rawAligned * sizeof(float));
}
}
template <typename T, typename ST, typename OT>
__aicore__ inline void Mod<T, ST, OT>::ProcessFusedBcast()
{
if (coreRows_ == 0) {
return;
}
{
LocalTensor<ST> p0 = inputx1Queue.AllocTensor<ST>();
LocalTensor<ST> p1 = inputx1Queue.AllocTensor<ST>();
Duplicate(p0, static_cast<ST>(0), static_cast<int32_t>(actualMaxDataCount));
Duplicate(p1, static_cast<ST>(0), static_cast<int32_t>(actualMaxDataCount));
inputx1Queue.EnQue(p0);
inputx1Queue.EnQue(p1);
LocalTensor<ST> d0 = inputx1Queue.DeQue<ST>();
LocalTensor<ST> d1 = inputx1Queue.DeQue<ST>();
inputx1Queue.FreeTensor(d0);
inputx1Queue.FreeTensor(d1);
PipeBarrier<PIPE_ALL>();
}
if (bcastFusedMode_ == 1U) {
BuildOuterOtherFused();
}
uint64_t rowOffset = coreRowBase_;
uint64_t rowsLeft = coreRows_;
while (rowsLeft > 0) {
const uint64_t rows = (rowsLeft < bcUbFormer_) ? rowsLeft : bcUbFormer_;
CopyInFusedBcast(rowOffset, rows);
ComputeFusedBcast(rows);
CopyOutFusedBcast(rowOffset, rows);
rowOffset += rows;
rowsLeft -= rows;
}
}
template <typename T, typename ST, typename OT>
__aicore__ inline void Mod<T, ST, OT>::BuildOuterOtherFused()
{
LocalTensor<OT> otherLocal = inputx2Queue.AllocTensor<OT>();
DataCopyParams p;
p.blockCount = 1;
p.blockLen = static_cast<uint32_t>(bcInner_ * sizeof(OT));
p.srcStride = 0;
p.dstStride = 0;
DataCopyPad(otherLocal, inputx2GM[0], p, {false, 0, 0, 0});
inputx2Queue.EnQue(otherLocal);
LocalTensor<OT> otherIn = inputx2Queue.DeQue<OT>();
LocalTensor<float> otherF32 = bcastOtherBuf_.Get<float>();
const int32_t innerCnt = static_cast<int32_t>(bcInner_);
LocalTensor<float> srcRow;
if constexpr (NEED_FP32_IO_BUF) {
srcRow = x2TensorFP32Buff.Get<float>();
Cast(srcRow, otherIn, RoundMode::CAST_NONE, innerCnt);
PipeBarrier<PIPE_ALL>();
} else {
srcRow = otherIn.template ReinterpretCast<float>();
}
Duplicate(otherF32, 1.0f, static_cast<int32_t>(bcUbFormer_ * bcIpad_));
for (uint64_t r = 0; r < bcUbFormer_; ++r) {
Muls(otherF32[r * bcIpad_], srcRow, 1.0f, innerCnt);
}
inputx2Queue.FreeTensor(otherIn);
PipeBarrier<PIPE_ALL>();
}
template <typename T, typename ST, typename OT>
__aicore__ inline void Mod<T, ST, OT>::CopyInFusedBcast(uint64_t rowOffset, uint64_t rows)
{
LocalTensor<ST> selfLocal = inputx1Queue.AllocTensor<ST>();
DataCopyParams sp;
sp.blockCount = static_cast<uint16_t>(rows);
sp.blockLen = static_cast<uint32_t>(bcInner_ * sizeof(ST));
sp.srcStride = 0;
sp.dstStride = 0;
DataCopyPad(selfLocal, inputx1GM[rowOffset * bcInner_], sp, {false, 0, 0, 0});
inputx1Queue.EnQue(selfLocal);
if (bcastFusedMode_ == 2U) {
LocalTensor<OT> otherLocal = inputx2Queue.AllocTensor<OT>();
DataCopyParams op;
op.blockCount = 1;
op.blockLen = static_cast<uint32_t>(rows * sizeof(OT));
op.srcStride = 0;
op.dstStride = 0;
DataCopyPad(otherLocal, inputx2GM[rowOffset], op, {false, 0, 0, 0});
inputx2Queue.EnQue(otherLocal);
}
}
template <typename T, typename ST, typename OT>
__aicore__ inline void Mod<T, ST, OT>::ComputeFusedBcast(uint64_t rows)
{
const int32_t cnt = static_cast<int32_t>(rows * bcIpad_);
LocalTensor<ST> selfLocal = inputx1Queue.DeQue<ST>();
LocalTensor<float> otherF32 = bcastOtherBuf_.Get<float>();
if (bcastFusedMode_ == 2U) {
LocalTensor<OT> otherLocal = inputx2Queue.DeQue<OT>();
const int32_t innerCnt = static_cast<int32_t>(bcInner_);
LocalTensor<float> otherScalarF32;
if constexpr (NEED_FP32_IO_BUF) {
otherScalarF32 = x2TensorFP32Buff.Get<float>();
Cast(otherScalarF32, otherLocal, RoundMode::CAST_NONE, static_cast<int32_t>(rows));
event_t evVS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S));
SetFlag<HardEvent::V_S>(evVS);
WaitFlag<HardEvent::V_S>(evVS);
} else {
event_t evMte2S = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_S));
SetFlag<HardEvent::MTE2_S>(evMte2S);
WaitFlag<HardEvent::MTE2_S>(evMte2S);
otherScalarF32 = otherLocal.template ReinterpretCast<float>();
}
Duplicate(otherF32, 1.0f, cnt);
for (uint64_t r = 0; r < rows; ++r) {
const float sc = otherScalarF32.GetValue(r);
Duplicate(otherF32[r * bcIpad_], sc, innerCnt);
}
inputx2Queue.FreeTensor(otherLocal);
PipeBarrier<PIPE_V>();
}
LocalTensor<float> selfF32;
if constexpr (NEED_FP32_IO_BUF) {
selfF32 = x1TensorFP32Buff.Get<float>();
Cast(selfF32, selfLocal, RoundMode::CAST_NONE, cnt);
} else {
selfF32 = selfLocal.template ReinterpretCast<float>();
}
LocalTensor<float> w0 = ResQuotTensorBuff.Get<float>();
LocalTensor<float> w1 = ResRemTensorBuff.Get<float>();
LocalTensor<float> w2 = A1Buff.Get<float>();
LocalTensor<float> w3 = A2Buff.Get<float>();
LocalTensor<float> w4 = A3Buff.Get<float>();
LocalTensor<float> w5 = A4Buff.Get<float>();
LocalTensor<T> outLocal = outputQueue.AllocTensor<T>();
if constexpr (NEED_FP32_IO_BUF) {
LocalTensor<float> rF32 = A5Buff.Get<float>();
RemainderRouted(rF32, selfF32, otherF32, w0, w1, w2, w3, w4, w5, cnt);
Cast(outLocal, rF32, RoundMode::CAST_RINT, cnt);
} else {
LocalTensor<float> outF32 = outLocal.template ReinterpretCast<float>();
RemainderRouted(outF32, selfF32, otherF32, w0, w1, w2, w3, w4, w5, cnt);
}
inputx1Queue.FreeTensor(selfLocal);
outputQueue.EnQue(outLocal);
}
template <typename T, typename ST, typename OT>
__aicore__ inline void Mod<T, ST, OT>::CopyOutFusedBcast(uint64_t rowOffset, uint64_t rows)
{
LocalTensor<T> dstLocal = outputQueue.DeQue<T>();
DataCopyParams cp;
cp.blockCount = static_cast<uint16_t>(rows);
cp.blockLen = static_cast<uint32_t>(bcInner_ * sizeof(T));
cp.srcStride = 0;
cp.dstStride = 0;
DataCopyPad(outputGM[rowOffset * bcInner_], dstLocal, cp);
outputQueue.FreeTensor(dstLocal);
}
#endif
#endif