* Copyright (c) 2026 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
* \file broadcast_common.h
* \brief AscendC 广播算子通用件:常量 / TilingData / 切分辅助 helper / 四类广播原语。
* 纯 AscendC API,不依赖任何模板调度框架,可直接拷入自定义算子工程裁剪使用。
* 平台:DAV_3510(A5 / Ascend950,C310)。
*
* 四类广播实现:
* ④ OneDim(前置) —— 合轴塌一维时整算子走此路:标量首块 Duplicate 铺满、满输入直搬,tiling 退化 1D
* ① BroadcastNddma —— GM→UB 多维 DMA,广播轴 srcStride=0 由引擎复制(不占 Vector)
* ② DataCopyPadCompact —— 每轮紧凑搬入,广播由 GetGmOffset 的 stride=0 寻址实现(外层广播轴用);
* offset 相同时复用为可选优化(本样例未启用)
* ③ UbBroadcast —— DataCopyPad 搬紧凑源后用 Broadcast 矢量指令在 UB 内展开
*/
#ifndef BROADCAST_COMMON_H
#define BROADCAST_COMMON_H
#include "kernel_operator.h"
namespace BrcDemo {
using namespace AscendC;
constexpr int64_t BROADCAST_MAX_DIMS = 8;
constexpr int64_t NDDMA_DIM = 5;
constexpr int64_t BLOCK_LENGTH = 32;
constexpr int64_t DOUBLE_BUFFER_NUM = 2;
enum BrcMode { BRC_NONE = 0, BRC_NDDMA = 1, BRC_DATACOPYPAD = 2, BRC_UB = 3 };
template <int IN_NUM>
struct BroadcastTilingData {
int32_t brcMode[IN_NUM];
int32_t shapeLen;
int32_t ubSplitAxis;
int32_t ubFormer, ubTail;
int64_t ubOuter;
int64_t blockNum;
int64_t blockFormer, blockTail;
int64_t dimProductBeforeUbInner;
int64_t elemNum;
int64_t outputDims[BROADCAST_MAX_DIMS];
int64_t outputStrides[BROADCAST_MAX_DIMS];
int64_t inputDims[IN_NUM][BROADCAST_MAX_DIMS];
int64_t inputStrides[IN_NUM][BROADCAST_MAX_DIMS];
};
static inline int64_t BrcCeilDiv(int64_t a, int64_t b)
{
int64_t d = (b > 0) ? b : 1;
return (a + d - 1) / d;
}
static inline int64_t BrcAlignDown(int64_t a, int64_t al)
{
int64_t d = (al > 0) ? al : 1;
return (a / d) * d;
}
template <int IN_NUM>
inline void ComputeTiling(BroadcastTilingData<IN_NUM>& td, int dtSize, int64_t coreNum, int64_t ubSize,
int64_t aliveBuf)
{
int rank = td.shapeLen;
int64_t dtBytes = (dtSize > 0) ? dtSize : 1;
int64_t alignEle = BLOCK_LENGTH / dtBytes;
td.elemNum = BrcAlignDown(ubSize / (aliveBuf * dtBytes * DOUBLE_BUFFER_NUM ), alignEle);
if (td.elemNum < alignEle)
td.elemNum = alignEle;
int splitAxis = 0;
int64_t innerBelow = 1;
for (int ax = rank - 1; ax >= 0; ax--) {
if (innerBelow * td.outputDims[ax] > td.elemNum) {
splitAxis = ax;
break;
}
innerBelow *= td.outputDims[ax];
splitAxis = ax;
}
int64_t ubFormer;
if (innerBelow > td.elemNum) {
td.ubSplitAxis = rank - 1;
innerBelow = 1;
ubFormer = td.elemNum;
} else {
td.ubSplitAxis = splitAxis;
ubFormer = td.elemNum / innerBelow;
}
int64_t splitDim = td.outputDims[td.ubSplitAxis];
if (ubFormer > splitDim)
ubFormer = splitDim;
if (ubFormer < 1)
ubFormer = 1;
td.ubFormer = ubFormer;
td.ubOuter = BrcCeilDiv(splitDim, ubFormer);
td.ubTail = splitDim - (td.ubOuter - 1) * ubFormer;
int64_t outerProd = 1;
for (int j = 0; j < td.ubSplitAxis; j++)
outerProd *= td.outputDims[j];
td.dimProductBeforeUbInner = outerProd * td.ubOuter;
int64_t totalTiles = td.dimProductBeforeUbInner;
int64_t cn = (coreNum < 1) ? 1 : coreNum;
td.blockFormer = BrcCeilDiv(totalTiles, cn);
if (td.blockFormer < 1)
td.blockFormer = 1;
td.blockNum = BrcCeilDiv(totalTiles, td.blockFormer);
td.blockTail = totalTiles - (td.blockNum - 1) * td.blockFormer;
}
__aicore__ inline void GetAxesIndices(int64_t (&axes)[BROADCAST_MAX_DIMS], int64_t tileLinear,
const int64_t (&outputDims)[BROADCAST_MAX_DIMS], int64_t ubSplitAxis,
int64_t tileTotal)
{
int64_t rem = tileTotal;
for (int64_t idx = 0; idx < ubSplitAxis; idx++) {
rem /= outputDims[idx];
axes[idx] = tileLinear / rem;
tileLinear -= axes[idx] * rem;
}
axes[ubSplitAxis] = tileLinear;
}
__aicore__ inline void UpdateAxesIndices(int64_t (&axes)[BROADCAST_MAX_DIMS],
const int64_t (&outputDims)[BROADCAST_MAX_DIMS], int64_t ubSplitAxis,
int64_t ubOuter)
{
axes[ubSplitAxis]++;
if (axes[ubSplitAxis] == ubOuter) {
axes[ubSplitAxis] = 0;
if (ubSplitAxis > 0)
axes[ubSplitAxis - 1]++;
}
for (int64_t idx = ubSplitAxis - 1; idx >= 1; idx--) {
if (axes[idx] == outputDims[idx]) {
axes[idx] = 0;
axes[idx - 1]++;
}
}
}
__aicore__ inline int64_t GetGmOffset(const int64_t (&axes)[BROADCAST_MAX_DIMS],
const int64_t (&strides)[BROADCAST_MAX_DIMS], int64_t ubSplitAxis,
int64_t ubFormer)
{
int64_t off = 0;
for (int64_t idx = 0; idx < ubSplitAxis; idx++)
off += axes[idx] * strides[idx];
off += axes[ubSplitAxis] * strides[ubSplitAxis] * ubFormer;
return off;
}
__aicore__ inline void FillUbShape(uint32_t (&dstShape)[BROADCAST_MAX_DIMS], uint32_t (&srcShape)[BROADCAST_MAX_DIMS],
const int64_t (&outputDims)[BROADCAST_MAX_DIMS],
const int64_t (&inDims)[BROADCAST_MAX_DIMS], int64_t ubSplitAxis, int64_t shapeLen,
int64_t curRows)
{
dstShape[0] = (uint32_t)curRows;
srcShape[0] = (inDims[ubSplitAxis] == 1) ? 1u : (uint32_t)curRows;
for (int64_t i = ubSplitAxis + 1, j = 1; i < shapeLen; i++, j++) {
dstShape[j] = (uint32_t)outputDims[i];
srcShape[j] = (uint32_t)inDims[i];
}
}
__aicore__ inline int64_t UbSrcLen(const int64_t (&inStrides)[BROADCAST_MAX_DIMS],
const int64_t (&inDims)[BROADCAST_MAX_DIMS], int64_t ubSplitAxis, int64_t shapeLen,
int64_t rows)
{
int64_t srcRows = (inStrides[ubSplitAxis] != 0) ? rows : 1;
int64_t inner = 1;
for (int64_t j = ubSplitAxis + 1; j < shapeLen; j++)
inner *= inDims[j];
return srcRows * inner;
}
template <typename T>
__aicore__ inline AscendC::MultiCopyParams<T, NDDMA_DIM> MakeNddmaParams(
const int64_t (&outputDims)[BROADCAST_MAX_DIMS], const int64_t (&outputStrides)[BROADCAST_MAX_DIMS],
const int64_t (&inputStrides)[BROADCAST_MAX_DIMS], int64_t shapeLen, int64_t ubSplitAxis, int64_t ubSplitSize)
{
AscendC::MultiCopyLoopInfo<NDDMA_DIM> loop;
int64_t axisInsideUb = NDDMA_DIM - (shapeLen - ubSplitAxis);
if (axisInsideUb < 0)
axisInsideUb = 0;
for (int64_t i = 0; i < axisInsideUb; i++) {
loop.loopSize[NDDMA_DIM - 1 - i] = 1;
loop.loopSrcStride[NDDMA_DIM - 1 - i] = inputStrides[ubSplitAxis];
loop.loopDstStride[NDDMA_DIM - 1 - i] = outputStrides[ubSplitAxis];
}
loop.loopSize[NDDMA_DIM - 1 - axisInsideUb] = ubSplitSize;
loop.loopSrcStride[NDDMA_DIM - 1 - axisInsideUb] = inputStrides[ubSplitAxis];
loop.loopDstStride[NDDMA_DIM - 1 - axisInsideUb] = outputStrides[ubSplitAxis];
for (int64_t i = axisInsideUb + 1; i < NDDMA_DIM; i++) {
int64_t axis = ubSplitAxis + i - axisInsideUb;
loop.loopSize[NDDMA_DIM - 1 - i] = outputDims[axis];
loop.loopSrcStride[NDDMA_DIM - 1 - i] = inputStrides[axis];
loop.loopDstStride[NDDMA_DIM - 1 - i] = outputStrides[axis];
}
return AscendC::MultiCopyParams<T, NDDMA_DIM>{loop, (T)0};
}
template <typename T>
__aicore__ inline void BroadcastNddma(const GlobalTensor<T>& gm, const LocalTensor<T>& ub,
const int64_t (&outputDims)[BROADCAST_MAX_DIMS],
const int64_t (&outputStrides)[BROADCAST_MAX_DIMS],
const int64_t (&inputStrides)[BROADCAST_MAX_DIMS], int64_t shapeLen,
int64_t ubSplitAxis, int64_t ubSplitSize)
{
if (outputStrides[ubSplitAxis] != inputStrides[ubSplitAxis]) {
auto params = MakeNddmaParams<T>(outputDims, outputStrides, inputStrides, shapeLen, ubSplitAxis, ubSplitSize);
static constexpr AscendC::MultiCopyConfig cfg = {false, 0, 0, false};
AscendC::DataCopy<T, NDDMA_DIM, cfg>(ub, gm, params);
} else {
AscendC::DataCopyExtParams ext{1, (uint32_t)(ubSplitSize * inputStrides[ubSplitAxis] * sizeof(T)), 0, 0, 0};
AscendC::DataCopyPadExtParams<T> pad{false, 0, 0, 0};
AscendC::DataCopyPad(ub, gm, ext, pad);
}
}
template <typename T>
__aicore__ inline void DataCopyPadCompact(const GlobalTensor<T>& gm, const LocalTensor<T>& ub, int64_t lenEle)
{
AscendC::DataCopyExtParams ext{1, (uint32_t)(lenEle * sizeof(T)), 0, 0, 0};
AscendC::DataCopyPadExtParams<T> pad{false, 0, 0, 0};
AscendC::DataCopyPad(ub, gm, ext, pad);
}
template <typename T>
__aicore__ inline void UbBroadcast(const LocalTensor<T>& dst, const LocalTensor<T>& src,
const int64_t (&outputDims)[BROADCAST_MAX_DIMS],
const int64_t (&inDims)[BROADCAST_MAX_DIMS], int64_t ubSplitAxis, int64_t shapeLen,
int64_t curRows)
{
uint32_t dShape[BROADCAST_MAX_DIMS], sShape[BROADCAST_MAX_DIMS];
FillUbShape(dShape, sShape, outputDims, inDims, ubSplitAxis, shapeLen, curRows);
int64_t rank = shapeLen - ubSplitAxis;
bool anyExpand = false;
int64_t cnt = 1;
for (int64_t k = 0; k < rank; k++) {
if (dShape[k] > sShape[k])
anyExpand = true;
cnt *= dShape[k];
}
if (!anyExpand) {
int32_t alignEle = BLOCK_LENGTH / sizeof(T);
int32_t cntAlign = ((int32_t)cnt + alignEle - 1) / alignEle * alignEle;
AscendC::DataCopy(dst, src, cntAlign);
return;
}
AscendC::BroadcastTiling bt;
AscendC::GetBroadcastTilingInfo<T>(rank, dShape, sShape, false, bt);
AscendC::Broadcast<T>(dst, src, dShape, sShape, &bt);
}
struct OneDimTilingData {
int64_t dimLen;
int32_t tileNum;
int32_t blockNum;
int32_t scalarFlag;
};
template <int IN_NUM>
inline bool TryOneDim(const BroadcastTilingData<IN_NUM>& td, int64_t& dimLen, int32_t& scalarFlag)
{
dimLen = 1;
for (int j = 0; j < td.shapeLen; j++)
dimLen *= td.outputDims[j];
scalarFlag = 0;
for (int i = 0; i < IN_NUM; i++) {
int64_t cnt = 1;
for (int j = 0; j < td.shapeLen; j++)
cnt *= td.inputDims[i][j];
if (cnt == 1)
scalarFlag |= (1 << i);
else if (cnt != dimLen)
return false;
}
return true;
}
inline void ComputeOneDimTiling(OneDimTilingData& t, int64_t dimLen, int32_t scalarFlag, int dtSize, int64_t coreNum,
int64_t ubSize, int64_t aliveBuf)
{
t.dimLen = dimLen;
t.scalarFlag = scalarFlag;
int64_t dtBytes = (dtSize > 0) ? dtSize : 1;
int64_t alignEle = BLOCK_LENGTH / dtBytes;
int64_t tileNum = BrcAlignDown(ubSize / (aliveBuf * dtBytes * DOUBLE_BUFFER_NUM ), alignEle);
if (tileNum < alignEle)
tileNum = alignEle;
int64_t ubOuter = BrcCeilDiv(dimLen, tileNum);
int64_t cn = (coreNum < 1) ? 1 : coreNum;
if (ubOuter < cn)
cn = (ubOuter < 1) ? 1 : ubOuter;
t.tileNum = (int32_t)tileNum;
t.blockNum = (int32_t)cn;
}
struct OneDimCoreParam {
int64_t baseOffset;
int64_t loops;
int64_t tailLen;
};
__aicore__ inline OneDimCoreParam OneDimCalcCore(int64_t dimLen, int64_t tileNum, int64_t blockNum)
{
int64_t tn = (tileNum > 0) ? tileNum : 1;
int64_t bn = (blockNum > 0) ? blockNum : 1;
int64_t ubOuter = (dimLen + tn - 1) / tn;
int64_t ubTail = dimLen - (ubOuter - 1) * tn;
int64_t blockFormer = (ubOuter + bn - 1) / bn;
int64_t blockTail = ubOuter - (bn - 1) * blockFormer;
int64_t bid = AscendC::GetBlockIdx();
OneDimCoreParam p;
p.baseOffset = blockFormer * tn * bid;
bool lastCore = (bid == bn - 1);
p.loops = lastCore ? blockTail : blockFormer;
p.tailLen = lastCore ? ubTail : tn;
return p;
}
template <typename T>
__aicore__ inline void OneDimLoadInput(const LocalTensor<T>& dst, const GlobalTensor<T>& gm, int64_t offset,
int64_t len, bool isScalar, bool firstTile)
{
if (isScalar) {
if (firstTile)
AscendC::Duplicate(dst, gm.GetValue(0), len);
} else {
DataCopyPadCompact(gm[offset], dst, len);
}
}
}
#endif