* Copyright (c) 2026 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*
* Generated By CANNBot
*/
#ifndef DAWSN_KERNEL_H
#define DAWSN_KERNEL_H
#include "kernel_operator.h"
#include "kernel_tiling/kernel_tiling.h"
#include "dawsn_tiling_data.h"
#include "dawsn_tiling_key.h"
namespace NsDawsn {
using namespace AscendC;
static constexpr float S1_AN[] = {1.13681498971755967716E-11f, 8.49262267667473820670E-10f,
1.94434204175553062245E-08f, 9.53151741254484376186E-07f,
3.07828309874913215953E-06f, 3.52513368520288765742E-04f,
-8.50149846724410876604E-04f, 4.22618223005546619908E-02f,
-9.17480371773452402229E-02f, 1.00000000000000000000E+00f};
static constexpr float S1_AD[] = {2.40372073066762607321E-11f, 1.48864681368493399468E-09f, 5.21265281010541632356E-08f,
1.27258478273186967251E-06f, 2.32490249820789501151E-05f, 3.25524741826057934131E-04f,
3.48805814657162597398E-03f, 2.79448531198828974131E-02f, 1.58874241960120565764E-01f,
5.74918629489320376358E-01f, 1.00000000000000000000E+00f};
static constexpr float S2_BN[] = {
5.08955156417900900223E-01f, -2.44754418142697854899E-01f, 9.41512335303534425579E-02f,
-2.18711255142039033561E-02f, 3.66207612329569201851E-03f, -4.23209114460388778275E-04f,
3.59641304793896659001E-05f, -2.14640351719968955485E-06f, 9.10010780076391391280E-08f,
-2.40274520828250976051E-09f, 3.59233385440928385474E-11f};
static constexpr float S2_BD[] = {-6.31839869873368153996E-01f, 2.36706788228248699024E-01f,
-5.31806367003223243173E-02f, 8.48041718586295401039E-03f,
-9.47996768486665357389E-04f, 7.81025592944552368886E-05f,
-4.55875153252442628812E-06f, 1.89100358111421858017E-07f,
-4.91324691331920598557E-09f, 7.18466403235734505828E-11f};
static constexpr float S3_CN[] = {-5.90592860534773222980E-01f, 6.29235242724368815104E-01f,
-1.72858975380388135701E-01f, 1.64837047825189646211E-02f,
-4.86827613020462685988E-04f};
static constexpr float S3_CD[] = {-2.69820057197544915795E+00f, 1.73270799045947843098E+00f,
-3.93708582281939500902E-01f, 3.44278924041233405640E-02f,
-9.73655226040941201328E-04f};
static constexpr int32_t F32_NAN = 0x7fc00000;
static constexpr float FLT_MAX_VAL = 3.4028235e+38f;
static constexpr float SEG1_UPPER_BOUND = 3.25f;
static constexpr float SEG2_UPPER_BOUND = 6.25f;
static constexpr float SEG3_UPPER_BOUND = 1.0e9f;
static constexpr float DAWSON_HALF = 0.5f;
static constexpr uint32_t VF_REG_SIZE_BYTES = 256;
static constexpr AscendC::Reg::CastTrait WIDEN_F16_TO_F32 = {
AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::UNKNOWN, AscendC::Reg::MaskMergeMode::ZEROING,
AscendC::RoundMode::UNKNOWN};
static constexpr AscendC::Reg::CastTrait WIDEN_BF16_TO_F32 = {
AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::UNKNOWN, AscendC::Reg::MaskMergeMode::ZEROING,
AscendC::RoundMode::UNKNOWN};
static constexpr AscendC::Reg::CastTrait NARROW_F32_TO_F16 = {
AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::NO_SAT, AscendC::Reg::MaskMergeMode::ZEROING,
AscendC::RoundMode::CAST_ROUND};
static constexpr AscendC::Reg::CastTrait NARROW_F32_TO_BF16 = {
AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::NO_SAT, AscendC::Reg::MaskMergeMode::ZEROING,
AscendC::RoundMode::CAST_ROUND};
template <typename T>
__simd_vf__ inline void DawsnVF(__ubuf__ T* dstAddr, __ubuf__ T* srcAddr, uint32_t count, uint32_t oneRepeatSize,
uint16_t repeatTimes);
template <typename T>
class DawsnKernel {
public:
__aicore__ inline DawsnKernel() {}
__aicore__ inline void Init(GM_ADDR x, GM_ADDR y, const DawsnTilingData* tilingData);
__aicore__ inline void Process();
private:
__aicore__ inline void CopyIn(int64_t progress, int64_t currentNum);
__aicore__ inline void Compute(int64_t currentNum);
__aicore__ inline void CopyOut(int64_t progress, int64_t currentNum);
TPipe pipe;
TQue<QuePosition::VECIN, 2> inputQueueX;
TQue<QuePosition::VECOUT, 2> outputQueueY;
GlobalTensor<T> inputGMX;
GlobalTensor<T> outputGMY;
int64_t blockLength_ = 0;
int64_t ubLength_ = 0;
};
template <typename T>
__aicore__ inline void DawsnKernel<T>::Init(GM_ADDR x, GM_ADDR y, const DawsnTilingData* tilingData)
{
int64_t blockIdx = AscendC::GetBlockIdx();
int64_t remainderLength = tilingData->dim0 - tilingData->blockFormer * blockIdx;
blockLength_ = (remainderLength > tilingData->blockFormer) ? tilingData->blockFormer : remainderLength;
ubLength_ = tilingData->ubFormer;
inputGMX.SetGlobalBuffer((__gm__ T*)x + tilingData->blockFormer * blockIdx, blockLength_);
outputGMY.SetGlobalBuffer((__gm__ T*)y + tilingData->blockFormer * blockIdx, blockLength_);
pipe.InitBuffer(inputQueueX, 2, ubLength_ * sizeof(T));
pipe.InitBuffer(outputQueueY, 2, ubLength_ * sizeof(T));
}
template <typename T>
__aicore__ inline void DawsnKernel<T>::CopyIn(int64_t progress, int64_t currentNum)
{
AscendC::LocalTensor<T> xLocal = inputQueueX.template AllocTensor<T>();
AscendC::DataCopyParams copyParams;
copyParams.blockCount = 1;
copyParams.blockLen = currentNum * sizeof(T);
copyParams.srcStride = 0;
copyParams.dstStride = 0;
AscendC::DataCopyPad(xLocal, inputGMX[progress * ubLength_], copyParams, {false, 0, 0, 0});
inputQueueX.EnQue(xLocal);
}
template <typename T>
__aicore__ inline void DawsnKernel<T>::CopyOut(int64_t progress, int64_t currentNum)
{
AscendC::LocalTensor<T> yLocal = outputQueueY.template DeQue<T>();
AscendC::DataCopyParams copyParams;
copyParams.blockCount = 1;
copyParams.blockLen = currentNum * sizeof(T);
copyParams.srcStride = 0;
copyParams.dstStride = 0;
AscendC::DataCopyPad(outputGMY[progress * ubLength_], yLocal, copyParams);
outputQueueY.FreeTensor(yLocal);
}
template <typename T>
__aicore__ inline void DawsnKernel<T>::Compute(int64_t currentNum)
{
AscendC::LocalTensor<T> xLocal = inputQueueX.template DeQue<T>();
AscendC::LocalTensor<T> yLocal = outputQueueY.template AllocTensor<T>();
constexpr uint32_t oneRepeatSize = AscendC::GetVecLen() / sizeof(T);
uint16_t repeatTimes = AscendC::CeilDivision(static_cast<uint32_t>(currentNum), oneRepeatSize);
asc_vf_call<DawsnVF<T>>((__ubuf__ T*)yLocal.GetPhyAddr(), (__ubuf__ T*)xLocal.GetPhyAddr(),
static_cast<uint32_t>(currentNum), oneRepeatSize, repeatTimes);
outputQueueY.template EnQue<T>(yLocal);
inputQueueX.FreeTensor(xLocal);
}
template <typename T>
__aicore__ inline void DawsnKernel<T>::Process()
{
if (blockLength_ <= 0) {
return;
}
int64_t loopCount = (blockLength_ + ubLength_ - 1) / ubLength_;
if (loopCount == 1) {
CopyIn(0, blockLength_);
Compute(blockLength_);
CopyOut(0, blockLength_);
return;
}
CopyIn(0, ubLength_);
int64_t currentNum = ubLength_;
for (int64_t i = 0; i < loopCount - 1; i++) {
int64_t nextNum = (i + 1 == loopCount - 1) ? (blockLength_ - ubLength_ * (i + 1)) : ubLength_;
Compute(currentNum);
CopyIn(i + 1, nextNum);
CopyOut(i, currentNum);
currentNum = nextNum;
}
Compute(currentNum);
CopyOut(loopCount - 1, currentNum);
}
#define DAWSN_HORNER_AN(u, pReg, tmpReg, mask) \
AscendC::Reg::Duplicate(pReg, S1_AN[0], mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AN[1], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AN[2], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AN[3], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AN[4], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AN[5], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AN[6], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AN[7], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AN[8], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AN[9], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask);
#define DAWSN_HORNER_AD(u, qReg, tmpReg, mask) \
AscendC::Reg::Duplicate(qReg, S1_AD[0], mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AD[1], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AD[2], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AD[3], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AD[4], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AD[5], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AD[6], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AD[7], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AD[8], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AD[9], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S1_AD[10], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask);
#define DAWSN_HORNER_BN(u, pReg, tmpReg, mask) \
AscendC::Reg::Duplicate(pReg, S2_BN[0], mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BN[1], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BN[2], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BN[3], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BN[4], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BN[5], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BN[6], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BN[7], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BN[8], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BN[9], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BN[10], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask);
#define DAWSN_HORNER_BD(u, qReg, tmpReg, mask) \
AscendC::Reg::Duplicate(qReg, S2_BD[0], mask); \
AscendC::Reg::Add(qReg, qReg, u, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BD[1], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BD[2], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BD[3], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BD[4], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BD[5], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BD[6], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BD[7], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BD[8], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S2_BD[9], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask);
#define DAWSN_HORNER_CN(u, pReg, tmpReg, mask) \
AscendC::Reg::Duplicate(pReg, S3_CN[0], mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S3_CN[1], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S3_CN[2], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S3_CN[3], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(pReg, pReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S3_CN[4], mask); \
AscendC::Reg::Add(pReg, pReg, tmpReg, mask);
#define DAWSN_HORNER_CD(u, qReg, tmpReg, mask) \
AscendC::Reg::Duplicate(qReg, S3_CD[0], mask); \
AscendC::Reg::Add(qReg, qReg, u, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S3_CD[1], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S3_CD[2], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S3_CD[3], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask); \
AscendC::Reg::Mul(qReg, qReg, u, mask); \
AscendC::Reg::Duplicate(tmpReg, S3_CD[4], mask); \
AscendC::Reg::Add(qReg, qReg, tmpReg, mask);
#define DAWSN_SEG23_FORMULA(invXReg, invX2Reg, pReg, qReg, absReg, segResult, tmpReg, tmp2Reg, mask) \
AscendC::Reg::Mul(tmpReg, qReg, absReg, mask); \
AscendC::Reg::Div(tmpReg, pReg, tmpReg, mask); \
AscendC::Reg::Mul(segResult, invX2Reg, tmpReg, mask); \
AscendC::Reg::Add(segResult, invXReg, segResult, mask); \
AscendC::Reg::Duplicate(tmpReg, DAWSON_HALF, mask); \
AscendC::Reg::Mul(segResult, segResult, tmpReg, mask);
#define DAWSN_COMPUTE(xReg, absReg, xSqReg, pReg, qReg, tmpReg, resultReg, invXReg, segResult, invX2Reg, mask, \
maskSeg1, maskSeg2, maskSeg3, maskSeg4, maskZero, maskNeg, maskNan, maskInf, maskSpecial) \
AscendC::Reg::Abs(absReg, xReg, mask); \
AscendC::Reg::Compare<float, AscendC::CMPMODE::NE>(maskNan, absReg, absReg, mask); \
AscendC::Reg::Duplicate(tmpReg, FLT_MAX_VAL, mask); \
AscendC::Reg::Compare<float, AscendC::CMPMODE::GT>(maskInf, absReg, tmpReg, mask); \
AscendC::Reg::MaskOr(maskSpecial, maskNan, maskInf, mask); \
AscendC::Reg::Duplicate(tmpReg, 0.0f, mask); \
AscendC::Reg::Compare<float, AscendC::CMPMODE::EQ>(maskZero, absReg, tmpReg, mask); \
AscendC::Reg::Duplicate(tmpReg, 1.0f, mask); \
AscendC::Reg::Select<float>(absReg, tmpReg, absReg, maskZero); \
AscendC::Reg::Select<float>(absReg, tmpReg, absReg, maskSpecial); \
AscendC::Reg::Mul(xSqReg, absReg, absReg, mask); \
AscendC::Reg::Duplicate(tmpReg, SEG1_UPPER_BOUND, mask); \
AscendC::Reg::Compare<float, AscendC::CMPMODE::LT>(maskSeg1, absReg, tmpReg, mask); \
AscendC::Reg::Duplicate(tmpReg, SEG2_UPPER_BOUND, mask); \
AscendC::Reg::Compare<float, AscendC::CMPMODE::LT>(maskSeg2, absReg, tmpReg, mask); \
AscendC::Reg::Duplicate(tmpReg, SEG3_UPPER_BOUND, mask); \
AscendC::Reg::Compare<float, AscendC::CMPMODE::LE>(maskSeg3, absReg, tmpReg, mask); \
AscendC::Reg::Compare<float, AscendC::CMPMODE::GT>(maskSeg4, absReg, tmpReg, mask); \
AscendC::Reg::Duplicate(tmpReg, 1.0f, mask); \
AscendC::Reg::Div(invXReg, tmpReg, absReg, mask); \
AscendC::Reg::Mul(invX2Reg, invXReg, invXReg, mask); \
AscendC::Reg::Duplicate(tmpReg, 0.0f, mask); \
AscendC::Reg::Select<float>(invX2Reg, tmpReg, invX2Reg, maskSeg1); \
DAWSN_HORNER_AN(xSqReg, pReg, tmpReg, mask) \
DAWSN_HORNER_AD(xSqReg, qReg, tmpReg, mask) \
AscendC::Reg::Mul(segResult, absReg, pReg, mask); \
AscendC::Reg::Div(segResult, segResult, qReg, mask); \
AscendC::Reg::Duplicate(resultReg, 0.0f, mask); \
AscendC::Reg::Select<float>(resultReg, segResult, resultReg, maskSeg1); \
DAWSN_HORNER_BN(invX2Reg, pReg, tmpReg, mask) \
DAWSN_HORNER_BD(invX2Reg, qReg, tmpReg, mask) \
DAWSN_SEG23_FORMULA(invXReg, invX2Reg, pReg, qReg, absReg, segResult, tmpReg, resultReg, mask) \
AscendC::Reg::Select<float>(resultReg, segResult, resultReg, maskSeg2); \
DAWSN_HORNER_CN(invX2Reg, pReg, tmpReg, mask) \
DAWSN_HORNER_CD(invX2Reg, qReg, tmpReg, mask) \
DAWSN_SEG23_FORMULA(invXReg, invX2Reg, pReg, qReg, absReg, segResult, tmpReg, resultReg, mask) \
AscendC::Reg::Select<float>(resultReg, segResult, resultReg, maskSeg3); \
\
\
\
DAWSN_HORNER_BN(invX2Reg, pReg, tmpReg, mask) \
DAWSN_HORNER_BD(invX2Reg, qReg, tmpReg, mask) \
DAWSN_SEG23_FORMULA(invXReg, invX2Reg, pReg, qReg, absReg, segResult, tmpReg, resultReg, mask) \
AscendC::Reg::Select<float>(resultReg, segResult, resultReg, maskSeg2); \
AscendC::Reg::Duplicate(tmpReg, DAWSON_HALF, mask); \
AscendC::Reg::Div(segResult, tmpReg, absReg, mask); \
AscendC::Reg::Select<float>(resultReg, segResult, resultReg, maskSeg4); \
DAWSN_HORNER_AN(xSqReg, pReg, tmpReg, mask) \
DAWSN_HORNER_AD(xSqReg, qReg, tmpReg, mask) \
AscendC::Reg::Mul(segResult, absReg, pReg, mask); \
AscendC::Reg::Div(segResult, segResult, qReg, mask); \
AscendC::Reg::Select<float>(resultReg, segResult, resultReg, maskSeg1); \
AscendC::Reg::Duplicate(tmpReg, 0.0f, mask); \
AscendC::Reg::Select<float>(resultReg, tmpReg, resultReg, maskZero); \
AscendC::Reg::Duplicate(tmpReg, 0.0f, mask); \
AscendC::Reg::Compare<float, AscendC::CMPMODE::LT>(maskNeg, xReg, tmpReg, mask); \
AscendC::Reg::Neg(segResult, resultReg, mask); \
AscendC::Reg::Select<float>(resultReg, segResult, resultReg, maskNeg); \
AscendC::Reg::Duplicate(tmpReg, 0.0f, mask); \
AscendC::Reg::Select<float>(resultReg, tmpReg, resultReg, maskInf); \
AscendC::Reg::Duplicate<int32_t, AscendC::Reg::MaskMergeMode::MERGING>( \
(AscendC::Reg::RegTensor<int32_t>&)resultReg, F32_NAN, maskNan);
template <>
__simd_vf__ inline void DawsnVF<float>(__ubuf__ float* dstAddr, __ubuf__ float* srcAddr, uint32_t count,
uint32_t oneRepeatSize, uint16_t repeatTimes)
{
AscendC::Reg::RegTensor<float> xReg, absReg, xSqReg, pReg, qReg, tmpReg;
AscendC::Reg::RegTensor<float> resultReg, invXReg, segResult, invX2Reg;
AscendC::Reg::MaskReg mask, maskSeg1, maskSeg2, maskSeg3, maskSeg4, maskZero, maskNeg;
AscendC::Reg::MaskReg maskNan, maskInf, maskSpecial;
for (uint16_t i = 0; i < repeatTimes; ++i) {
mask = AscendC::Reg::UpdateMask<float>(count);
AscendC::Reg::LoadAlign(xReg, srcAddr + i * oneRepeatSize);
DAWSN_COMPUTE(xReg, absReg, xSqReg, pReg, qReg, tmpReg, resultReg, invXReg, segResult, invX2Reg, mask, maskSeg1,
maskSeg2, maskSeg3, maskSeg4, maskZero, maskNeg, maskNan, maskInf, maskSpecial)
AscendC::Reg::StoreAlign(dstAddr + i * oneRepeatSize, resultReg, mask);
}
}
template <>
__simd_vf__ inline void DawsnVF<half>(__ubuf__ half* dstAddr, __ubuf__ half* srcAddr, uint32_t count,
uint32_t oneRepeatSize, uint16_t repeatTimes)
{
AscendC::Reg::RegTensor<half> xHalfReg, yHalfReg;
AscendC::Reg::RegTensor<float> xFloatReg;
AscendC::Reg::RegTensor<float> absReg, xSqReg, pReg, qReg, tmpReg;
AscendC::Reg::RegTensor<float> resultReg, invXReg, segResult, invX2Reg;
AscendC::Reg::MaskReg mask;
AscendC::Reg::MaskReg maskSeg1, maskSeg2, maskSeg3, maskSeg4, maskZero, maskNeg;
AscendC::Reg::MaskReg maskNan, maskInf, maskSpecial;
constexpr uint32_t fOneRepeatSize = VF_REG_SIZE_BYTES / sizeof(float);
uint16_t fRepeatTimes = AscendC::CeilDivision(count, fOneRepeatSize);
for (uint16_t i = 0; i < fRepeatTimes; ++i) {
mask = AscendC::Reg::UpdateMask<float>(count);
AscendC::Reg::LoadAlign<half, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(xHalfReg, srcAddr + i * fOneRepeatSize);
AscendC::Reg::Cast<float, half, WIDEN_F16_TO_F32>(xFloatReg, xHalfReg, mask);
DAWSN_COMPUTE(xFloatReg, absReg, xSqReg, pReg, qReg, tmpReg, resultReg, invXReg, segResult, invX2Reg, mask,
maskSeg1, maskSeg2, maskSeg3, maskSeg4, maskZero, maskNeg, maskNan, maskInf, maskSpecial)
AscendC::Reg::Cast<half, float, NARROW_F32_TO_F16>(yHalfReg, resultReg, mask);
AscendC::Reg::StoreAlign<half, AscendC::Reg::StoreDist::DIST_PACK_B32>(dstAddr + i * fOneRepeatSize, yHalfReg,
mask);
}
}
template <>
__simd_vf__ inline void DawsnVF<bfloat16_t>(__ubuf__ bfloat16_t* dstAddr, __ubuf__ bfloat16_t* srcAddr, uint32_t count,
uint32_t oneRepeatSize, uint16_t repeatTimes)
{
AscendC::Reg::RegTensor<bfloat16_t> xBfReg, yBfReg;
AscendC::Reg::RegTensor<float> xFloatReg;
AscendC::Reg::RegTensor<float> absReg, xSqReg, pReg, qReg, tmpReg;
AscendC::Reg::RegTensor<float> resultReg, invXReg, segResult, invX2Reg;
AscendC::Reg::MaskReg mask;
AscendC::Reg::MaskReg maskSeg1, maskSeg2, maskSeg3, maskSeg4, maskZero, maskNeg;
AscendC::Reg::MaskReg maskNan, maskInf, maskSpecial;
constexpr uint32_t fOneRepeatSize = VF_REG_SIZE_BYTES / sizeof(float);
uint16_t fRepeatTimes = AscendC::CeilDivision(count, fOneRepeatSize);
for (uint16_t i = 0; i < fRepeatTimes; ++i) {
mask = AscendC::Reg::UpdateMask<float>(count);
AscendC::Reg::LoadAlign<bfloat16_t, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(xBfReg,
srcAddr + i * fOneRepeatSize);
AscendC::Reg::Cast<float, bfloat16_t, WIDEN_BF16_TO_F32>(xFloatReg, xBfReg, mask);
DAWSN_COMPUTE(xFloatReg, absReg, xSqReg, pReg, qReg, tmpReg, resultReg, invXReg, segResult, invX2Reg, mask,
maskSeg1, maskSeg2, maskSeg3, maskSeg4, maskZero, maskNeg, maskNan, maskInf, maskSpecial)
AscendC::Reg::Cast<bfloat16_t, float, NARROW_F32_TO_BF16>(yBfReg, resultReg, mask);
AscendC::Reg::StoreAlign<bfloat16_t, AscendC::Reg::StoreDist::DIST_PACK_B32>(dstAddr + i * fOneRepeatSize,
yBfReg, mask);
}
}
}
#endif