* Copyright (c) 2026 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*
* Generated By CANNBot
*/
* \file bessel_i0e.cpp
* \brief BesselI0e 算子 Kernel 实现(全 dtype:FP32 + FP16 + BF16)
*
* 数学公式: i0e(x) = exp(-|x|) * I0(x)
* 实现策略: A&S 7/9 项 Horner 多项式分段逼近
* - 小区间 |x|≤3.75: Horner 7 项 P(t²) 逼近 I0,除以 exp(|x|)
* - 大区间 |x|>3.75: Horner 9 项 Q(s) 逼近,除以 sqrt(|x|)
* - 取 min(i0e_before, i0e_after) 保证上界
*/
#include "kernel_operator.h"
#include "kernel_tiling/kernel_tiling.h"
#include "arch35/bessel_i0e_tiling_data.h"
#include "arch35/bessel_i0e_tiling_key.h"
#include <type_traits>
namespace NsBesselI0e {
using namespace AscendC;
constexpr float BOUNDARY = 3.75f;
constexpr float INV_BOUNDARY = 0.26666667f;
constexpr float ZERO_PROTECT_EPS = 1e-30f;
static constexpr float SMALL_COEFFS[7] = {1.0f, 3.5156229f, 3.0899424f, 1.2067492f, 0.2659732f, 0.0360768f, 0.0045813f};
static constexpr float LARGE_COEFFS[9] = {0.39894228f, 0.01328592f, 0.00225319f, -0.00157565f, 0.00916281f,
-0.02057706f, 0.02635537f, -0.01647633f, 0.00392377f};
template <typename T>
class BesselI0eKernel {
static constexpr int BUFFER_NUM = 1;
static constexpr bool NEED_CAST = !std::is_same_v<T, float>;
public:
__aicore__ inline BesselI0eKernel() {}
__aicore__ inline void Init(GM_ADDR x, GM_ADDR y, const BesselI0eTilingData* tilingData);
__aicore__ inline void Process();
private:
__aicore__ inline void CopyIn(int64_t progress, int64_t currentNum);
__aicore__ inline void Compute(LocalTensor<float> xLocal, LocalTensor<float> yLocal, int64_t currentNum);
__aicore__ inline void ComputeSmallInterval(LocalTensor<float> absX, LocalTensor<float> i0eBefore,
int64_t currentNum);
__aicore__ inline void ComputeLargeInterval(LocalTensor<float> absX, LocalTensor<float> i0eAfter,
int64_t currentNum);
__aicore__ inline void CopyOut(int64_t progress, int64_t currentNum);
TPipe pipe;
TQue<TPosition::VECIN, BUFFER_NUM> inputQueue;
TQue<TPosition::VECOUT, BUFFER_NUM> outputQueue;
GlobalTensor<T> inputGM;
GlobalTensor<T> outputGM;
TBuf<TPosition::VECCALC> absXBuf;
TBuf<TPosition::VECCALC> tmpBuf1;
TBuf<TPosition::VECCALC> i0eBeforeBuf;
TBuf<TPosition::VECCALC> tmpBuf2;
TBuf<TPosition::VECCALC> i0eAfterBuf;
TBuf<TPosition::VECCALC> castInBuf;
TBuf<TPosition::VECCALC> resultFp32Buf;
int64_t blockLength_ = 0;
int64_t ubLength_ = 0;
};
template <typename T>
__aicore__ inline void BesselI0eKernel<T>::Init(GM_ADDR x, GM_ADDR y, const BesselI0eTilingData* tilingData)
{
int64_t remainder = tilingData->totalNum - tilingData->blockFactor * GetBlockIdx();
blockLength_ = (remainder > tilingData->blockFactor) ? tilingData->blockFactor : remainder;
ubLength_ = tilingData->ubFactor;
inputGM.SetGlobalBuffer((__gm__ T*)x + tilingData->blockFactor * GetBlockIdx(), blockLength_);
outputGM.SetGlobalBuffer((__gm__ T*)y + tilingData->blockFactor * GetBlockIdx(), blockLength_);
pipe.InitBuffer(inputQueue, BUFFER_NUM, ubLength_ * sizeof(T));
pipe.InitBuffer(outputQueue, BUFFER_NUM, ubLength_ * sizeof(T));
pipe.InitBuffer(absXBuf, ubLength_ * sizeof(float));
pipe.InitBuffer(tmpBuf1, ubLength_ * sizeof(float));
pipe.InitBuffer(i0eBeforeBuf, ubLength_ * sizeof(float));
pipe.InitBuffer(tmpBuf2, ubLength_ * sizeof(float));
pipe.InitBuffer(i0eAfterBuf, ubLength_ * sizeof(float));
if constexpr (NEED_CAST) {
pipe.InitBuffer(castInBuf, ubLength_ * sizeof(float));
pipe.InitBuffer(resultFp32Buf, ubLength_ * sizeof(float));
}
}
template <typename T>
__aicore__ inline void BesselI0eKernel<T>::CopyIn(int64_t progress, int64_t currentNum)
{
LocalTensor<T> xLocal = inputQueue.template AllocTensor<T>();
DataCopyExtParams copyParams;
copyParams.blockCount = 1;
copyParams.blockLen = currentNum * sizeof(T);
copyParams.srcStride = 0;
copyParams.dstStride = 0;
DataCopyPad(xLocal, inputGM[progress * ubLength_], copyParams, {false, 0, 0, 0});
inputQueue.EnQue(xLocal);
}
template <typename T>
__aicore__ inline void BesselI0eKernel<T>::CopyOut(int64_t progress, int64_t currentNum)
{
LocalTensor<T> yLocal = outputQueue.template DeQue<T>();
DataCopyExtParams copyParams;
copyParams.blockCount = 1;
copyParams.blockLen = currentNum * sizeof(T);
copyParams.srcStride = 0;
copyParams.dstStride = 0;
DataCopyPad(outputGM[progress * ubLength_], yLocal, copyParams);
outputQueue.FreeTensor(yLocal);
}
template <typename T>
__aicore__ inline void BesselI0eKernel<T>::ComputeSmallInterval(LocalTensor<float> absX, LocalTensor<float> i0eBefore,
int64_t currentNum)
{
LocalTensor<float> tmp1 = tmpBuf1.Get<float>();
LocalTensor<float> tmp2 = tmpBuf2.Get<float>();
Duplicate(tmp1, BOUNDARY, currentNum);
Min(tmp1, absX, tmp1, currentNum);
Muls(i0eBefore, tmp1, INV_BOUNDARY, currentNum);
Mul(tmp2, i0eBefore, i0eBefore, currentNum);
Duplicate(i0eBefore, SMALL_COEFFS[6], currentNum);
Mul(i0eBefore, i0eBefore, tmp2, currentNum);
Adds(i0eBefore, i0eBefore, SMALL_COEFFS[5], currentNum);
Mul(i0eBefore, i0eBefore, tmp2, currentNum);
Adds(i0eBefore, i0eBefore, SMALL_COEFFS[4], currentNum);
Mul(i0eBefore, i0eBefore, tmp2, currentNum);
Adds(i0eBefore, i0eBefore, SMALL_COEFFS[3], currentNum);
Mul(i0eBefore, i0eBefore, tmp2, currentNum);
Adds(i0eBefore, i0eBefore, SMALL_COEFFS[2], currentNum);
Mul(i0eBefore, i0eBefore, tmp2, currentNum);
Adds(i0eBefore, i0eBefore, SMALL_COEFFS[1], currentNum);
Mul(i0eBefore, i0eBefore, tmp2, currentNum);
Adds(i0eBefore, i0eBefore, SMALL_COEFFS[0], currentNum);
Exp(tmp1, tmp1, currentNum);
Div(i0eBefore, i0eBefore, tmp1, currentNum);
}
template <typename T>
__aicore__ inline void BesselI0eKernel<T>::ComputeLargeInterval(LocalTensor<float> absX, LocalTensor<float> i0eAfter,
int64_t currentNum)
{
LocalTensor<float> tmp1 = tmpBuf1.Get<float>();
LocalTensor<float> tmp2 = tmpBuf2.Get<float>();
Duplicate(tmp1, ZERO_PROTECT_EPS, currentNum);
Max(tmp1, absX, tmp1, currentNum);
Duplicate(tmp2, BOUNDARY, currentNum);
Div(tmp2, tmp2, tmp1, currentNum);
Duplicate(i0eAfter, LARGE_COEFFS[8], currentNum);
Mul(i0eAfter, i0eAfter, tmp2, currentNum);
Adds(i0eAfter, i0eAfter, LARGE_COEFFS[7], currentNum);
Mul(i0eAfter, i0eAfter, tmp2, currentNum);
Adds(i0eAfter, i0eAfter, LARGE_COEFFS[6], currentNum);
Mul(i0eAfter, i0eAfter, tmp2, currentNum);
Adds(i0eAfter, i0eAfter, LARGE_COEFFS[5], currentNum);
Mul(i0eAfter, i0eAfter, tmp2, currentNum);
Adds(i0eAfter, i0eAfter, LARGE_COEFFS[4], currentNum);
Mul(i0eAfter, i0eAfter, tmp2, currentNum);
Adds(i0eAfter, i0eAfter, LARGE_COEFFS[3], currentNum);
Mul(i0eAfter, i0eAfter, tmp2, currentNum);
Adds(i0eAfter, i0eAfter, LARGE_COEFFS[2], currentNum);
Mul(i0eAfter, i0eAfter, tmp2, currentNum);
Adds(i0eAfter, i0eAfter, LARGE_COEFFS[1], currentNum);
Mul(i0eAfter, i0eAfter, tmp2, currentNum);
Adds(i0eAfter, i0eAfter, LARGE_COEFFS[0], currentNum);
Sqrt(tmp2, absX, currentNum);
Div(i0eAfter, i0eAfter, tmp2, currentNum);
}
template <typename T>
__aicore__ inline void BesselI0eKernel<T>::Compute(LocalTensor<float> xLocal, LocalTensor<float> yLocal,
int64_t currentNum)
{
LocalTensor<float> absX = absXBuf.Get<float>();
LocalTensor<float> i0eBefore = i0eBeforeBuf.Get<float>();
LocalTensor<float> i0eAfter = i0eAfterBuf.Get<float>();
Abs(absX, xLocal, currentNum);
ComputeSmallInterval(absX, i0eBefore, currentNum);
ComputeLargeInterval(absX, i0eAfter, currentNum);
Min(yLocal, i0eBefore, i0eAfter, currentNum);
}
template <typename T>
__aicore__ inline void BesselI0eKernel<T>::Process()
{
if (ubLength_ == 0 || blockLength_ == 0) {
return;
}
int64_t loopCount = (blockLength_ + ubLength_ - 1) / ubLength_;
if constexpr (std::is_same_v<T, float>) {
for (int64_t i = 0; i < loopCount; i++) {
int64_t currentNum = (i == (loopCount - 1)) ? (blockLength_ - ubLength_ * i) : ubLength_;
CopyIn(i, currentNum);
LocalTensor<float> xLocal = inputQueue.template DeQue<float>();
LocalTensor<float> yLocal = outputQueue.template AllocTensor<float>();
Compute(xLocal, yLocal, currentNum);
outputQueue.template EnQue<float>(yLocal);
inputQueue.FreeTensor(xLocal);
CopyOut(i, currentNum);
}
} else {
for (int64_t i = 0; i < loopCount; i++) {
int64_t currentNum = (i == (loopCount - 1)) ? (blockLength_ - ubLength_ * i) : ubLength_;
CopyIn(i, currentNum);
LocalTensor<T> xInput = inputQueue.template DeQue<T>();
LocalTensor<float> xFp32 = castInBuf.Get<float>();
Cast<float, T>(xFp32, xInput, RoundMode::CAST_NONE, currentNum);
LocalTensor<float> resultFp32 = resultFp32Buf.Get<float>();
Compute(xFp32, resultFp32, currentNum);
LocalTensor<T> yOutput = outputQueue.template AllocTensor<T>();
Cast<T, float>(yOutput, resultFp32, RoundMode::CAST_ROUND, currentNum);
outputQueue.template EnQue<T>(yOutput);
inputQueue.FreeTensor(xInput);
CopyOut(i, currentNum);
}
}
}
}
template <uint32_t schMode>
__global__ __aicore__ void bessel_i0e(GM_ADDR x, GM_ADDR y, GM_ADDR workspace, GM_ADDR tiling)
{
KERNEL_TASK_TYPE_DEFAULT(KERNEL_TYPE_AIV_ONLY);
REGISTER_TILING_DEFAULT(BesselI0eTilingData);
GET_TILING_DATA_WITH_STRUCT(BesselI0eTilingData, tilingData, tiling);
(void)workspace;
(void)schMode;
NsBesselI0e::BesselI0eKernel<DTYPE_X> op;
op.Init(x, y, &tilingData);
op.Process();
}