Star674
983
代码
代码
Issues120
Pull Requests182
Actions
项目讨论
Wiki
分析
Star674
983
已合并
kirin adv api update #170
乙酸氧铍创建于 1月29日
kirin adv api update #170
已合并
乙酸氧铍创建于 1月29日
从liangyunke/asc-devkitmaster合入到cann/asc-devkitmaster
共 37 个文件变更+1-12672
Dimpl/adv_api/detail/activation/geglu/geglu_l300_impl.h+0-102
@@ -1,102 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10-
11-/*!
12- * \file geglu_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_ACTIVATION_GEGLU_GEGLU_IMPL_L300_H
16-#define IMPL_ACTIVATION_GEGLU_GEGLU_IMPL_L300_H
17- 
18-#include "kernel_tensor.h"
19-#include "kernel_operator_intf.h"
20-#include "../../common/check.h"
21-#include "../../common/common.h"
22- 
23-namespace AscendC {
24-namespace Internal {
25-constexpr float gegluConstantA = 22.36386;
26-constexpr float gegluConstantB = -0.071354814;
27-template <typename T>
28-__simd_vf__ inline void GeGLUImplVF(
29- __ubuf__ T* dst, __ubuf__ T* src0, __ubuf__ T* src1, uint32_t count, const uint16_t repeatTimes)
30-{
31- MicroAPI::RegTensor<half> srcOrigin0;
32- MicroAPI::RegTensor<half> srcOrigin1;
33- MicroAPI::RegTensor<float> srcVreg0;
34- MicroAPI::RegTensor<float> srcVreg1;
35- MicroAPI::RegTensor<float> tmpReg0;
36- MicroAPI::RegTensor<float> tmpReg1;
37- MicroAPI::RegTensor<float> dstVreg;
38- MicroAPI::MaskReg mask;
39- constexpr uint32_t oneRepElm = static_cast<uint32_t>(GetVecLen() / sizeof(float));
40- for (uint16_t i = 0; i < repeatTimes; ++i) {
41- mask = MicroAPI::UpdateMask<float>(count);
42- if constexpr (sizeof(T) == sizeof(half)) {
43- MicroAPI::DataCopy<half, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcOrigin0, src0 + i * oneRepElm);
44- MicroAPI::Cast<float, half, castTraitB16ToB32>(srcVreg0, srcOrigin0, mask);
45- MicroAPI::DataCopy<half, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcOrigin1, src1 + i * oneRepElm);
46- MicroAPI::Cast<float, half, castTraitB16ToB32>(srcVreg1, srcOrigin1, mask);
47- } else {
48- MicroAPI::DataCopy(srcVreg0, src0 + i * oneRepElm);
49- MicroAPI::DataCopy(srcVreg1, src1 + i * oneRepElm);
50- }
51- MicroAPI::Mul(tmpReg0, srcVreg1, srcVreg1, mask);
52- MicroAPI::Adds(tmpReg0, tmpReg0, gegluConstantA, mask);
53- MicroAPI::Mul(tmpReg0, tmpReg0, srcVreg1, mask);
54- MicroAPI::Muls(tmpReg0, tmpReg0, gegluConstantB, mask);
55- MicroAPI::Exp(tmpReg1, tmpReg0, mask);
56- MicroAPI::Adds(tmpReg1, tmpReg1, 1.0f, mask);
57- MicroAPI::Div(tmpReg1, srcVreg1, tmpReg1, mask);
58- MicroAPI::Mul(dstVreg, srcVreg0, tmpReg1, mask);
59- if constexpr (sizeof(T) == sizeof(half)) {
60- MicroAPI::Cast<half, float, castTraitB32ToB16>((MicroAPI::RegTensor<half>&)dstVreg, dstVreg, mask);
61- MicroAPI::DataCopy<half, MicroAPI::StoreDist::DIST_PACK_B32>(
62- dst + i * oneRepElm, (MicroAPI::RegTensor<half>&)dstVreg, mask);
63- } else {
64- MicroAPI::DataCopy(dst + i * oneRepElm, dstVreg, mask);
65- }
66- }
67-}
68-} // namespace Internal
69- 
70-template <typename T, bool isReuseSource = false>
71-__aicore__ inline void GeGLUImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor0,
72- const LocalTensor<T>& srcTensor1, const uint32_t count)
73-{
74- // Only for AI Vector Core.
75- if ASCEND_IS_AIC {
76- return;
77- }
78- static_assert(SupportType<T, half, float>(), "GeGLU only support half/float data type on current device!");
79- ASCENDC_ASSERT((srcTensor0.GetSize() == srcTensor1.GetSize()),
80- { KERNEL_LOG(KERNEL_ERROR, "Input params.GetSize must be equal with each other!"); });
81- CheckTensorPosition(dstTensor, "dstTensor", "VECIN, VECOUT, VECCALC");
82- CheckTensorPosition(srcTensor0, "srcTensor0", "VECIN, VECOUT, VECCALC");
83- CheckTensorPosition(srcTensor1, "srcTensor1", "VECIN, VECOUT, VECCALC");
84- CheckCalCount(count, "count", dstTensor, "dstTensor", "GeGLU");
85- CheckCalCount(count, "count", srcTensor0, "srcTensor0", "GeGLU");
86- CheckCalCount(count, "count", srcTensor1, "srcTensor1", "GeGLU");
87- constexpr uint32_t oneRepElm = static_cast<uint32_t>(GetVecLen() / sizeof(float));
88- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(count, oneRepElm));
89- Internal::GeGLUImplVF<T>((__ubuf__ T*)dstTensor.GetPhyAddr(), (__ubuf__ T*)srcTensor0.GetPhyAddr(),
90- (__ubuf__ T*)srcTensor1.GetPhyAddr(), count, repeatTimes);
91-}
92- 
93-template <typename T, bool isReuseSource = false>
94-__aicore__ inline void GeGLUImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor0,
95- const LocalTensor<T>& srcTensor1, const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t count)
96-{
97- CheckTensorPosition(sharedTmpBuffer, "sharedTmpBuffer", "VECIN, VECOUT, VECCALC");
98- GeGLUImpl<T, isReuseSource>(dstTensor, srcTensor0, srcTensor1, count);
99-}
100- 
101-} // namespace AscendC
102-#endif // IMPL_ACTIVATION_GEGLU_GEGLU_IMPL_C310_H
Dimpl/adv_api/detail/activation/gelu/gelu_l300_impl.h+0-352
@@ -1,352 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10-
11-/*!
12- * \file gelu_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_ACTIVATION_GELU_GELU_IMPL_L300_H
16-#define IMPL_ACTIVATION_GELU_GELU_IMPL_L300_H
17- 
18-#include "kernel_tensor.h"
19-#include "kernel_operator_intf.h"
20-#include "../../common/check.h"
21-#include "../../common/common.h"
22- 
23-namespace AscendC {
24-namespace Internal {
25- 
26-template <typename T, bool highPrecision>
27-__simd_vf__ inline void GeluImplVF(__ubuf__ T* dst, __ubuf__ T* src, uint32_t count, const uint16_t repeatTimes)
28-{
29- constexpr uint32_t oneRepElm = static_cast<uint32_t>(GetVecLen() / sizeof(T));
30- constexpr float coefficientsA = 0.044715;
31- constexpr float coefficientsB = 1.5957691216057308;
32- MicroAPI::RegTensor<T> srcVreg;
33- MicroAPI::RegTensor<T> dstVreg;
34- MicroAPI::RegTensor<T> tmpReg0;
35- MicroAPI::RegTensor<T> tmpReg1;
36- MicroAPI::RegTensor<T> tmpReg2;
37- MicroAPI::RegTensor<T> tmpReg3;
38- MicroAPI::MaskReg mask;
39- for (uint16_t i = 0; i < repeatTimes; ++i) {
40- mask = MicroAPI::UpdateMask<T>(count);
41- if constexpr (highPrecision) {
42- MicroAPI::DataCopy<half, MicroAPI::LoadDist::DIST_UNPACK_B16>(
43- (MicroAPI::RegTensor<half>&)srcVreg, (__ubuf__ half*)src + i * oneRepElm);
44- MicroAPI::Cast<float, half, castTraitB16ToB32>(srcVreg, (MicroAPI::RegTensor<half>&)srcVreg, mask);
45- } else {
46- MicroAPI::DataCopy(srcVreg, src + i * oneRepElm);
47- }
48- // y = (input_x + 0.044715 * input_x ^ 3) * 1.5957691
49- MicroAPI::Mul(tmpReg0, srcVreg, srcVreg, mask);
50- MicroAPI::Mul(tmpReg0, tmpReg0, srcVreg, mask);
51- MicroAPI::Muls(tmpReg0, tmpReg0, coefficientsA, mask);
52- MicroAPI::Add(tmpReg0, tmpReg0, srcVreg, mask);
53- MicroAPI::Muls(tmpReg0, tmpReg0, coefficientsB, mask);
54- // exp(min(y, 0))
55- MicroAPI::Mins(tmpReg1, tmpReg0, 0.0f, mask);
56- MicroAPI::Exp(tmpReg1, tmpReg1, mask);
57- // x / (exp^(-abs(y)) + 1)
58- MicroAPI::Abs(tmpReg2, tmpReg0, mask);
59- MicroAPI::Muls(tmpReg2, tmpReg2, -1.0f, mask);
60- MicroAPI::Exp(tmpReg3, tmpReg2, mask);
61- MicroAPI::Adds(tmpReg3, tmpReg3, 1.0f, mask);
62- MicroAPI::Div(tmpReg3, srcVreg, tmpReg3, mask);
63- // x / (exp^(-abs(y)) + 1) * exp(min(y, 0))
64- MicroAPI::Mul(dstVreg, tmpReg1, tmpReg3, mask);
65- if constexpr (highPrecision) {
66- MicroAPI::Cast<half, float, castTraitB32ToB16>((MicroAPI::RegTensor<half>&)dstVreg, dstVreg, mask);
67- MicroAPI::DataCopy<half, MicroAPI::StoreDist::DIST_PACK_B32>(
68- (__ubuf__ half*)dst + i * oneRepElm, (MicroAPI::RegTensor<half>&)dstVreg, mask);
69- } else {
70- MicroAPI::DataCopy(dst + i * oneRepElm, dstVreg, mask);
71- }
72- }
73-}
74- 
75-template <typename T>
76-__simd_callee__ inline void FastGeluCoreAlg(MicroAPI::RegTensor<T>& dstVreg,
77- MicroAPI::RegTensor<T>& srcVreg, MicroAPI::MaskReg& mask, MicroAPI::RegTensor<T>& stackVreg)
78-{
79- constexpr float coefficients = -1.702f;
80- constexpr float oneFloatScalar = 1.0f;
81- MicroAPI::Muls(stackVreg, srcVreg, coefficients, mask);
82- MicroAPI::Exp(stackVreg, stackVreg, mask);
83- MicroAPI::Adds(stackVreg, stackVreg, oneFloatScalar, mask);
84- MicroAPI::Div(dstVreg, srcVreg, stackVreg, mask);
85-}
86- 
87-template <typename T = half>
88-__simd_vf__ inline void FastGeluHighPrecisionAlgVF(__ubuf__ T* dst, __ubuf__ T* src,
89- const uint32_t dataSize)
90-{
91- MicroAPI::RegTensor<T> srcVreg;
92- MicroAPI::RegTensor<float> srcVregFloat;
93- MicroAPI::RegTensor<T> dstVreg;
94- MicroAPI::RegTensor<float> dstVregFloat;
95- 
96- constexpr uint32_t stackSize = GetVecLen() / sizeof(float);
97- uint32_t sreg = dataSize;
98- 
99- MicroAPI::RegTensor<float> stackVregFloat;
100-
101- MicroAPI::MaskReg mask;
102- 
103- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(dataSize, stackSize));
104- for (uint16_t i = 0; i < repeatTimes; ++i) {
105- mask = MicroAPI::UpdateMask<float>(sreg);
106- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcVreg, src + i * stackSize);
107- MicroAPI::Cast<float, half, castTraitB16ToB32>(srcVregFloat, srcVreg, mask);
108- 
109- FastGeluCoreAlg<float>(dstVregFloat, srcVregFloat, mask, stackVregFloat);
110- 
111- MicroAPI::Cast<half, float, castTraitB32ToB16>(dstVreg, dstVregFloat, mask);
112- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(dst + i * stackSize, dstVreg, mask);
113- }
114-}
115- 
116-template <typename T = half>
117-__aicore__ inline void FastGeluHighPrecisionAlg(const LocalTensor<half>& dstLocal, const LocalTensor<half>& srcLocal,
118- const uint32_t dataSize)
119-{
120- __ubuf__ T* src = (__ubuf__ T *)srcLocal.GetPhyAddr();
121- __ubuf__ T* dst = (__ubuf__ T *)dstLocal.GetPhyAddr();
122- 
123- FastGeluHighPrecisionAlgVF<T>(dst, src, dataSize);
124-}
125- 
126-template <typename T>
127-__simd_vf__ inline void FastGeluAlgVF(__ubuf__ T* dst, __ubuf__ T* src,
128- const uint32_t dataSize)
129-{
130- MicroAPI::RegTensor<T> srcVreg;
131- MicroAPI::RegTensor<T> dstVreg;
132- constexpr uint32_t stackSize = GetVecLen() / sizeof(T);
133- uint32_t sreg = dataSize;
134- MicroAPI::RegTensor<T> stackVreg;
135- MicroAPI::MaskReg mask;
136- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(dataSize, stackSize));
137- for (uint16_t i = 0; i < repeatTimes; ++i) {
138- mask = MicroAPI::UpdateMask<T>(sreg);
139- MicroAPI::DataCopy<T>(srcVreg, src + i * stackSize);
140- FastGeluCoreAlg<T>(dstVreg, srcVreg, mask, stackVreg);
141- MicroAPI::DataCopy<T>(dst + i * stackSize, dstVreg, mask);
142- }
143-}
144- 
145-template <typename T>
146-__aicore__ inline void FastGeluAlg(const LocalTensor<T>& dstLocal, const LocalTensor<T>& srcLocal,
147- const uint32_t dataSize)
148-{
149- __ubuf__ T* src = (__ubuf__ T *)srcLocal.GetPhyAddr();
150- __ubuf__ T* dst = (__ubuf__ T *)dstLocal.GetPhyAddr();
151- 
152- FastGeluAlgVF<T>(dst, src, dataSize);
153-}
154- 
155-template <typename T>
156-__simd_callee__ inline void FastGeluV2CoreAlg(MicroAPI::RegTensor<T>& dstVreg,
157- MicroAPI::RegTensor<T>& srcVreg, MicroAPI::MaskReg& mask, MicroAPI::RegTensor<T>& stackVregA,
158- MicroAPI::RegTensor<T>& stackVregB, MicroAPI::RegTensor<T>& stackVregC)
159-{
160- constexpr float coefficients = 0.000000000001;
161- constexpr float coefficientsHalf = 0.5;
162- constexpr float coefficientsA = -0.1444;
163- constexpr float coefficientsB = -1.769;
164- constexpr float coefficientsBInv = 1.769;
165- constexpr float coefficientsC = 0.7071;
166- constexpr float coefficientsD = 0.5;
167- MicroAPI::Muls(stackVregA, srcVreg, coefficientsC, mask);
168- MicroAPI::Abs(stackVregA, stackVregA, mask);
169- MicroAPI::Mins(stackVregA, stackVregA, coefficientsBInv, mask);
170- MicroAPI::Adds(stackVregA, stackVregA, coefficientsB, mask);
171- MicroAPI::Mul(stackVregA, stackVregA, stackVregA, mask);
172- MicroAPI::Muls(stackVregA, stackVregA, coefficientsA, mask);
173- MicroAPI::Adds(stackVregA, stackVregA, coefficientsD, mask);
174- 
175- MicroAPI::Adds(stackVregB, srcVreg, coefficients, mask);
176- MicroAPI::Abs(stackVregC, stackVregB, mask);
177- MicroAPI::Div(stackVregB, stackVregB, stackVregC, mask);
178- 
179- MicroAPI::Mul(stackVregA, stackVregA, stackVregB, mask);
180- MicroAPI::Adds(stackVregA, stackVregA, coefficientsHalf, mask);
181- 
182- MicroAPI::Mul(dstVreg, srcVreg, stackVregA, mask);
183-}
184- 
185-template <typename T = half>
186-__simd_vf__ inline void FastGeluV2HighPrecisionAlgVF(__ubuf__ T* dst, __ubuf__ T* src,
187- const uint32_t dataSize)
188-{
189- MicroAPI::RegTensor<T> srcVreg;
190- MicroAPI::RegTensor<float> srcVregFloat;
191- MicroAPI::RegTensor<T> dstVreg;
192- MicroAPI::RegTensor<float> dstVregFloat;
193- 
194- constexpr uint32_t stackSize = GetVecLen() / sizeof(float);
195- uint32_t sreg = dataSize;
196- 
197- MicroAPI::RegTensor<float> stackVregFloat;
198-
199- MicroAPI::MaskReg mask;
200- 
201- MicroAPI::RegTensor<float> stackVregA;
202- MicroAPI::RegTensor<float> stackVregB;
203- MicroAPI::RegTensor<float> stackVregC;
204- 
205- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(dataSize, stackSize));
206- for (uint16_t i = 0; i < repeatTimes; ++i) {
207- mask = MicroAPI::UpdateMask<float>(sreg);
208- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcVreg, src + i * stackSize);
209- MicroAPI::Cast<float, half, castTraitB16ToB32>(srcVregFloat, srcVreg, mask);
210- 
211- FastGeluV2CoreAlg<float>(dstVregFloat, srcVregFloat, mask, stackVregA, stackVregB, stackVregC);
212- 
213- MicroAPI::Cast<half, float, castTraitB32ToB16>(dstVreg, dstVregFloat, mask);
214- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(dst + i * stackSize, dstVreg, mask);
215- }
216-}
217- 
218-template <typename T = half>
219-__aicore__ inline void FastGeluV2HighPrecisionAlg(const LocalTensor<half>& dstLocal, const LocalTensor<half>& srcLocal,
220- const uint32_t dataSize)
221-{
222- __ubuf__ T* src = (__ubuf__ T *)srcLocal.GetPhyAddr();
223- __ubuf__ T* dst = (__ubuf__ T *)dstLocal.GetPhyAddr();
224- 
225- FastGeluV2HighPrecisionAlgVF<T>(dst, src, dataSize);
226-}
227- 
228-template <typename T>
229-__simd_vf__ inline void FastGeluV2AlgVF(__ubuf__ T* dst, __ubuf__ T* src,
230- const uint32_t dataSize)
231-{
232- MicroAPI::RegTensor<T> srcVreg;
233- MicroAPI::RegTensor<T> dstVreg;
234- constexpr uint32_t stackSize = GetVecLen() / sizeof(T);
235- uint32_t sreg = dataSize;
236- 
237- MicroAPI::RegTensor<T> stackVregA;
238- MicroAPI::RegTensor<T> stackVregB;
239- MicroAPI::RegTensor<T> stackVregC;
240- MicroAPI::MaskReg mask;
241- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(dataSize, stackSize));
242- for (uint16_t i = 0; i < repeatTimes; ++i) {
243- mask = MicroAPI::UpdateMask<T>(sreg);
244- MicroAPI::DataCopy<T>(srcVreg, src + i * stackSize);
245- FastGeluV2CoreAlg<T>(dstVreg, srcVreg, mask, stackVregA, stackVregB, stackVregC);
246- MicroAPI::DataCopy<T>(dst + i * stackSize, dstVreg, mask);
247- }
248-}
249- 
250-template <typename T>
251-__aicore__ inline void FastGeluV2Alg(const LocalTensor<T>& dstLocal, const LocalTensor<T>& srcLocal,
252- const uint32_t dataSize)
253-{
254- __ubuf__ T* src = (__ubuf__ T *)srcLocal.GetPhyAddr();
255- __ubuf__ T* dst = (__ubuf__ T *)dstLocal.GetPhyAddr();
256- 
257- FastGeluV2AlgVF<T>(dst, src, dataSize);
258-}
259-} // namespace Internal
260- 
261-template <typename T, bool highPrecision = false, bool highPerformance = false>
262-__aicore__ inline void GeluImpl(const LocalTensor<T>& dstLocal, const LocalTensor<T>& srcLocal, const uint32_t count)
263-{
264- // Only for AI Vector Core.
265- if ASCEND_IS_AIC {
266- return;
267- }
268- (void)highPerformance;
269- static_assert(SupportType<T, half, float>(), "Gelu only support half/float data type on current device!");
270- CheckTensorPosition(dstLocal, "dstLocal", "VECIN, VECOUT, VECCALC");
271- CheckTensorPosition(srcLocal, "srcLocal", "VECIN, VECOUT, VECCALC");
272- CheckCalCount(count, "calCount", dstLocal, "dstLocal", "Gelu");
273- CheckCalCount(count, "calCount", srcLocal, "srcLocal", "Gelu");
274- if constexpr (highPrecision && sizeof(T) == sizeof(half)) {
275- constexpr uint32_t oneRepElm = static_cast<uint32_t>(GetVecLen() / sizeof(float));
276- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(count, oneRepElm));
277- Internal::GeluImplVF<float, true>(
278- (__ubuf__ float*)dstLocal.GetPhyAddr(), (__ubuf__ float*)srcLocal.GetPhyAddr(), count, repeatTimes);
279- } else {
280- constexpr uint32_t oneRepElm = static_cast<uint32_t>(GetVecLen() / sizeof(T));
281- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(count, oneRepElm));
282- Internal::GeluImplVF<T, false>(
283- (__ubuf__ T*)dstLocal.GetPhyAddr(), (__ubuf__ T*)srcLocal.GetPhyAddr(), count, repeatTimes);
284- }
285-}
286- 
287-template <typename T, bool highPrecision = false, bool highPerformance = false>
288-__aicore__ inline void GeluImpl(const LocalTensor<T>& dstLocal, const LocalTensor<T>& srcLocal,
289- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t count)
290-{
291- CheckTensorPosition(sharedTmpBuffer, "sharedTmpBuffer", "VECIN, VECOUT, VECCALC");
292- GeluImpl<T, highPrecision, highPerformance>(dstLocal, srcLocal, count);
293-}
294- 
295-template <typename T, bool highPrecision = false, bool highPerformance = false>
296-__aicore__ inline void FasterGeluImpl(const LocalTensor<T>& dstLocal, const LocalTensor<T>& srcLocal,
297- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t dataSize)
298-{
299- (void)sharedTmpBuffer;
300- (void)highPerformance;
301- static_assert((SupportType<T, half, float>()), "current data type is not supported on current device!");
302-#if ASCENDC_CPU_DEBUG
303- bool ret = (dataSize <= srcLocal.GetSize()) && (dataSize <= dstLocal.GetSize()) && (dataSize > 0);
304- ASCENDC_ASSERT(
305- ret, { KERNEL_LOG(KERNEL_ERROR, "DataSize must bigger than 0 and smaller than or equal to src&dst tensor."); });
306-#endif
307- 
308- if constexpr (highPrecision && (IsSameType<T, half>::value)) {
309- Internal::FastGeluHighPrecisionAlg(dstLocal, srcLocal, dataSize);
310- } else {
311- Internal::FastGeluAlg(dstLocal, srcLocal, dataSize);
312- }
313-}
314- 
315-template <typename T, bool highPrecision = false, bool highPerformance = false>
316-__aicore__ inline void FasterGeluImpl(const LocalTensor<T>& dstLocal, const LocalTensor<T>& srcLocal,
317- const uint32_t dataSize)
318-{
319- LocalTensor<uint8_t> sharedTmpBuffer;
320- FasterGeluImpl<T, highPrecision, highPerformance>(dstLocal, srcLocal, sharedTmpBuffer, dataSize);
321-}
322- 
323-template <typename T, bool highPrecision = false, bool highPerformance = false>
324-__aicore__ inline void FasterGeluV2Impl(const LocalTensor<T>& dstLocal, const LocalTensor<T>& srcLocal,
325- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t dataSize)
326-{
327- (void)sharedTmpBuffer;
328- (void)highPerformance;
329- static_assert((SupportType<T, half, float>()), "current data type is not supported on current device!");
330-#if ASCENDC_CPU_DEBUG
331- bool ret = (dataSize <= srcLocal.GetSize()) && (dataSize <= dstLocal.GetSize()) && (dataSize > 0);
332- ASCENDC_ASSERT(
333- ret, { KERNEL_LOG(KERNEL_ERROR, "DataSize must bigger than 0 and smaller than or equal to src&dst tensor."); });
334-#endif
335- 
336- if constexpr (highPrecision && (IsSameType<T, half>::value)) {
337- Internal::FastGeluV2HighPrecisionAlg(dstLocal, srcLocal, dataSize);
338- } else {
339- Internal::FastGeluV2Alg(dstLocal, srcLocal, dataSize);
340- }
341-}
342- 
343-template <typename T, bool highPrecision = false, bool highPerformance = false>
344-__aicore__ inline void FasterGeluV2Impl(const LocalTensor<T>& dstLocal, const LocalTensor<T>& srcLocal,
345- const uint32_t dataSize)
346-{
347- LocalTensor<uint8_t> sharedTmpBuffer;
348- FasterGeluV2Impl<T, highPrecision, highPerformance>(dstLocal, srcLocal, sharedTmpBuffer, dataSize);
349-}
350-#pragma end_pipe
351-} // namespace AscendC
352-#endif // IMPL_ACTIVATION_GELU_GELU_IMPL_C310_H
Dimpl/adv_api/detail/activation/reglu/reglu_l300_impl.h+0-90
@@ -1,90 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10-
11-/*!
12- * \file reglu_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_ACTIVATION_REGLU_REGLU_IMPL_L300_H
16-#define IMPL_ACTIVATION_REGLU_REGLU_IMPL_L300_H
17- 
18-#include "kernel_tensor.h"
19-#include "kernel_operator_intf.h"
20-#include "../../common/check.h"
21-#include "../../common/common.h"
22- 
23-namespace AscendC {
24-namespace Internal {
25-template <typename T>
26-__simd_vf__ inline void ReGluImplVF(
27- __ubuf__ T* dst, __ubuf__ T* src0, __ubuf__ T* src1, uint32_t count, const uint16_t repeatTimes)
28-{
29- MicroAPI::RegTensor<T> srcOrigin0;
30- MicroAPI::RegTensor<T> srcOrigin1;
31- MicroAPI::RegTensor<float> srcVreg0;
32- MicroAPI::RegTensor<float> srcVreg1;
33- MicroAPI::RegTensor<float> tmpReg0;
34- MicroAPI::RegTensor<float> dstVreg;
35- MicroAPI::MaskReg mask;
36- constexpr uint32_t oneRepElm = static_cast<uint32_t>(GetVecLen() / sizeof(float));
37- for (uint16_t i = 0; i < repeatTimes; ++i) {
38- mask = MicroAPI::UpdateMask<float>(count);
39- if constexpr (sizeof(T) == sizeof(half)) {
40- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcOrigin0, src0 + i * oneRepElm);
41- MicroAPI::Cast<float, T, castTraitB16ToB32>(srcVreg0, srcOrigin0, mask);
42- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcOrigin1, src1 + i * oneRepElm);
43- MicroAPI::Cast<float, T, castTraitB16ToB32>(srcVreg1, srcOrigin1, mask);
44- } else {
45- MicroAPI::DataCopy(srcVreg0, src0 + i * oneRepElm);
46- MicroAPI::DataCopy(srcVreg1, src1 + i * oneRepElm);
47- }
48- MicroAPI::Maxs(tmpReg0, srcVreg1, 0.0f, mask);
49- MicroAPI::Mul(dstVreg, srcVreg0, tmpReg0, mask);
50- if constexpr (sizeof(T) == sizeof(half)) {
51- MicroAPI::Cast<T, float, castTraitB32ToB16>((MicroAPI::RegTensor<T>&)dstVreg, dstVreg, mask);
52- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(
53- dst + i * oneRepElm, (MicroAPI::RegTensor<T>&)dstVreg, mask);
54- } else {
55- MicroAPI::DataCopy(dst + i * oneRepElm, dstVreg, mask);
56- }
57- }
58-}
59-} // namespace Internal
60- 
61-template <typename T, bool isReuseSource = false>
62-__aicore__ inline void ReGluImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor0,
63- const LocalTensor<T>& srcTensor1, const uint32_t count)
64-{
65- // Only for AI Vector Core.
66- if ASCEND_IS_AIC {
67- return;
68- }
69- static_assert(SupportType<T, half, float, bfloat16_t>(), "ReGlu only support half/float/bfloat16_t data type on current device!");
70- CheckTensorPosition(dstTensor, "dstTensor", "VECIN, VECOUT, VECCALC");
71- CheckTensorPosition(srcTensor0, "srcTensor0", "VECIN, VECOUT, VECCALC");
72- CheckTensorPosition(srcTensor1, "srcTensor1", "VECIN, VECOUT, VECCALC");
73- CheckCalCount(count, "count", dstTensor, "dstTensor", "ReGlu");
74- CheckCalCount(count, "count", srcTensor0, "srcTensor0", "ReGlu");
75- CheckCalCount(count, "count", srcTensor1, "srcTensor1", "ReGlu");
76- constexpr uint32_t oneRepElm = static_cast<uint32_t>(GetVecLen() / sizeof(float));
77- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(count, oneRepElm));
78- Internal::ReGluImplVF<T>((__ubuf__ T*)dstTensor.GetPhyAddr(), (__ubuf__ T*)srcTensor0.GetPhyAddr(),
79- (__ubuf__ T*)srcTensor1.GetPhyAddr(), count, repeatTimes);
80-}
81- 
82-template <typename T, bool isReuseSource = false>
83-__aicore__ inline void ReGluImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor0,
84- const LocalTensor<T>& srcTensor1, const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t count)
85-{
86- CheckTensorPosition(sharedTmpBuffer, "sharedTmpBuffer", "VECIN, VECOUT, VECCALC");
87- ReGluImpl<T, isReuseSource>(dstTensor, srcTensor0, srcTensor1, count);
88-}
89-} // namespace AscendC
90-#endif // IMPL_ACTIVATION_REGLU_REGLU_IMPL_L300_H
Dimpl/adv_api/detail/activation/sigmoid/sigmoid_l300_impl.h+0-89
@@ -1,89 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/*!
12- * \file sigmoid_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_ACTIVATION_SIGMOID_L300_IMPL_H
16-#define IMPL_ACTIVATION_SIGMOID_L300_IMPL_H
17- 
18-#include "kernel_tensor.h"
19-#include "kernel_pop_stack_buffer.h"
20-#include "../../common/common.h"
21- 
22-namespace AscendC {
23-namespace Internal {
24-/*
25- * Formula is y= 1 / (1 + exp(-x))
26-*/
27-template<typename T>
28-__simd_vf__ inline void SigmoidImplVF(__ubuf__ T* dstUb, __ubuf__ T* srcUb, uint32_t count, const uint16_t repeatTimes)
29-{
30- uint32_t sreg = count;
31- MicroAPI::MaskReg preg;
32- MicroAPI::RegTensor<T> srcReg;
33- MicroAPI::RegTensor<float> castReg;
34- MicroAPI::RegTensor<float> tmpReg;
35- MicroAPI::RegTensor<float> dstReg;
36- 
37- for (uint16_t i = 0; i < repeatTimes; ++i) {
38- preg = MicroAPI::UpdateMask<float>(sreg);
39- if constexpr (sizeof(T) == sizeof(half)) {
40- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcReg, srcUb + i * B32_DATA_NUM_PER_REPEAT);
41- MicroAPI::Cast<float, T, castTraitB16ToB32>(castReg, srcReg, preg);
42- } else {
43- MicroAPI::DataCopy(castReg, srcUb + i * B32_DATA_NUM_PER_REPEAT);
44- }
45- MicroAPI::Muls(tmpReg, castReg, -1.0f, preg);
46- MicroAPI::Exp(tmpReg, tmpReg, preg);
47- 
48- MicroAPI::Adds(tmpReg, tmpReg, 1.0f, preg);
49- MicroAPI::Duplicate(dstReg, 1.0f, preg);
50- MicroAPI::Div(dstReg, dstReg, tmpReg, preg);
51- if constexpr (sizeof(T) == sizeof(half)) {
52- MicroAPI::Cast<T, float, castTraitB32ToB16>(srcReg, dstReg, preg);
53- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(dstUb + i * B32_DATA_NUM_PER_REPEAT, srcReg, preg);
54- } else {
55- MicroAPI::DataCopy(dstUb + i * B32_DATA_NUM_PER_REPEAT, dstReg, preg);
56- }
57- }
58-}
59-} // namespace Internal
60- 
61-template<typename T, bool isReuseSource = false>
62-__aicore__ inline void SigmoidImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
63- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
64-{
65- // Only for AI Vector Core.
66- if ASCEND_IS_AIC {
67- return;
68- }
69- static_assert(SupportType<T, half, float>(), "current data type is not supported on current device!");
70- CheckTensorPos<T>(dstTensor, Hardware::UB, "dstTensor", "VECIN / VECCALC / VECOUT", "Sigmoid");
71- CheckTensorPos<T>(srcTensor, Hardware::UB, "srcTensor", "VECIN / VECCALC / VECOUT", "Sigmoid");
72- CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN / VECCALC / VECOUT", "Sigmoid");
73- ASCENDC_ASSERT((calCount <= srcTensor.GetSize()), {
74- KERNEL_LOG(KERNEL_ERROR, "calCount is %u, which should not be larger than srcTensor length %u", calCount,
75- srcTensor.GetSize());
76- });
77- ASCENDC_ASSERT((calCount <= dstTensor.GetSize()), {
78- KERNEL_LOG(KERNEL_ERROR, "calCount is %u, which should not be larger than dstTensor length %u", calCount,
79- dstTensor.GetSize());
80- });
81- 
82- __ubuf__ T *dstUb = (__ubuf__ T *)dstTensor.GetPhyAddr();
83- __ubuf__ T *srcUb = (__ubuf__ T *)srcTensor.GetPhyAddr();
84- uint16_t repeatTimes = CeilDivision(calCount, B32_DATA_NUM_PER_REPEAT);
85- Internal::SigmoidImplVF<T>(dstUb, srcUb, calCount, repeatTimes);
86-}
87-} // namespace AscendC
88- 
89-#endif // IMPL_ACTIVATION_SIGMOID_L300_IMPL_H
Dimpl/adv_api/detail/activation/silu/silu_l300_impl.h+0-62
@@ -1,62 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/*!
12- * \file silu_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_MATH_SILU_SILU_L300_IMPL_H
16-#define IMPL_MATH_SILU_SILU_L300_IMPL_H
17-#include "kernel_tensor.h"
18-#include "../../common/check.h"
19- 
20-namespace AscendC {
21-namespace Internal {
22-template<typename T>
23-__simd_vf__ inline void SiluComputeVF(__ubuf__ T* dst, __ubuf__ T* src, uint32_t count, const uint16_t repeatTimes)
24-{
25- constexpr uint32_t oneRepElm = static_cast<uint32_t>(GetVecLen() / sizeof(T));
26- MicroAPI::RegTensor<T> srcVreg;
27- MicroAPI::RegTensor<T> tmpReg0;
28- MicroAPI::RegTensor<T> dstVreg;
29- MicroAPI::MaskReg mask;
30- for (uint16_t i = 0; i < repeatTimes; ++i) {
31- mask = MicroAPI::UpdateMask<T>(count);
32- MicroAPI::DataCopy(srcVreg, src + i * oneRepElm);
33- MicroAPI::Muls(tmpReg0, srcVreg, -1.0f, mask);
34- MicroAPI::Exp(tmpReg0, tmpReg0, mask);
35- MicroAPI::Adds(tmpReg0, tmpReg0, 1.0f, mask);
36- MicroAPI::Div(dstVreg, srcVreg, tmpReg0, mask);
37- MicroAPI::DataCopy(dst + i * oneRepElm, dstVreg, mask);
38- }
39-}
40-} // namespace Internal
41- 
42-template <typename T, bool isReuseSource = false>
43-__aicore__ inline void SiluCompute(const LocalTensor<T>& dstLocal, const LocalTensor<T>& srcLocal,
44- const uint32_t count)
45-{
46- // Only for AI Vector Core.
47- if ASCEND_IS_AIC {
48- return;
49- }
50- static_assert(SupportType<T, half, float>(), "Silu only support half/float data type on current device!");
51- CheckTensorPosition(dstLocal, "dstLocal", "VECIN, VECOUT, VECCALC");
52- CheckTensorPosition(srcLocal, "srcLocal", "VECIN, VECOUT, VECCALC");
53- CheckCalCount(count, "count", dstLocal, "dstLocal", "Silu");
54- CheckCalCount(count, "count", srcLocal, "srcLocal", "Silu");
55- 
56- constexpr uint32_t oneRepElm = static_cast<uint32_t>(GetVecLen() / sizeof(T));
57- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(count, oneRepElm));
58- Internal::SiluComputeVF<T>(
59- (__ubuf__ T*)dstLocal.GetPhyAddr(), (__ubuf__ T*)srcLocal.GetPhyAddr(), count, repeatTimes);
60-}
61-} // namespace AscendC
62-#endif // IMPL_MATH_SILU_SILU_C310_IMPL_H
Dimpl/adv_api/detail/activation/swiglu/swiglu_l300_impl.h+0-113
@@ -1,113 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10-
11-/*!
12- * \file swiglu_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_ACTIVATION_SWIGLU_SWIGLU_IMPL_L300_H
16-#define IMPL_ACTIVATION_SWIGLU_SWIGLU_IMPL_L300_H
17- 
18-#include "kernel_tensor.h"
19-#include "kernel_operator_intf.h"
20-#include "../../common/check.h"
21-#include "../../common/common.h"
22- 
23-namespace AscendC {
24-namespace Internal {
25-template <typename T>
26-__simd_vf__ inline void SwiGLUImplVF(__ubuf__ T* dst, __ubuf__ T* src0, __ubuf__ T* src1,
27- const float scalarValue, uint32_t count, const uint16_t repeatTimes)
28-{
29- MicroAPI::RegTensor<float> srcVreg0;
30- MicroAPI::RegTensor<float> srcVreg1;
31- MicroAPI::RegTensor<half> srcOrigin0;
32- MicroAPI::RegTensor<half> srcOrigin1;
33- MicroAPI::RegTensor<float> tmpReg0;
34- MicroAPI::RegTensor<float> tmpReg1;
35- MicroAPI::RegTensor<float> dstVreg;
36- MicroAPI::MaskReg mask;
37- constexpr uint32_t oneRepElm = static_cast<uint32_t>(GetVecLen() / sizeof(float));
38- for (uint16_t i = 0; i < repeatTimes; ++i) {
39- mask = MicroAPI::UpdateMask<float>(count);
40- if constexpr (sizeof(T) == sizeof(half)) {
41- MicroAPI::DataCopy<half, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcOrigin0, src0 + i * oneRepElm);
42- MicroAPI::Cast<float, half, castTraitB16ToB32>(srcVreg0, srcOrigin0, mask);
43- MicroAPI::DataCopy<half, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcOrigin1, src1 + i * oneRepElm);
44- MicroAPI::Cast<float, half, castTraitB16ToB32>(srcVreg1, srcOrigin1, mask);
45- } else {
46- MicroAPI::DataCopy(srcVreg0, src0 + i * oneRepElm);
47- MicroAPI::DataCopy(srcVreg1, src1 + i * oneRepElm);
48- }
49- MicroAPI::Muls(tmpReg0, srcVreg1, scalarValue, mask);
50- MicroAPI::Exp(tmpReg1, tmpReg0, mask);
51- MicroAPI::Adds(tmpReg1, tmpReg1, 1.0f, mask);
52- MicroAPI::Div(tmpReg1, srcVreg1, tmpReg1, mask);
53- MicroAPI::Mul(dstVreg, srcVreg0, tmpReg1, mask);
54- if constexpr (sizeof(T) == sizeof(half)) {
55- MicroAPI::Cast<half, float, castTraitB32ToB16>((MicroAPI::RegTensor<half>&)dstVreg, dstVreg, mask);
56- MicroAPI::DataCopy<half, MicroAPI::StoreDist::DIST_PACK_B32>(dst + i * oneRepElm, (MicroAPI::RegTensor<half>&)dstVreg, mask);
57- } else {
58- MicroAPI::DataCopy(dst + i * oneRepElm, dstVreg, mask);
59- }
60- }
61-}
62-} // namespace Internal
63- 
64-template <typename T, bool isReuseSource = false>
65-__aicore__ inline void SwiGLUImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor0,
66- const LocalTensor<T>& srcTensor1, const float scalarValue, const uint32_t count)
67-{
68- // Only for AI Vector Core.
69- if ASCEND_IS_AIC {
70- return;
71- }
72- static_assert(SupportType<T, half, float>(), "SwiGLU only support half/float data type on current device!");
73- ASCENDC_ASSERT((srcTensor0.GetSize() == srcTensor1.GetSize()),
74- { KERNEL_LOG(KERNEL_ERROR, "Input params.GetSize must be equal with each other!"); });
75- CheckTensorPosition(dstTensor, "dstTensor", "VECIN, VECOUT, VECCALC");
76- CheckTensorPosition(srcTensor0, "srcTensor0", "VECIN, VECOUT, VECCALC");
77- CheckTensorPosition(srcTensor1, "srcTensor1", "VECIN, VECOUT, VECCALC");
78- CheckCalCount(count, "count", dstTensor, "dstTensor", "SwiGLU");
79- CheckCalCount(count, "count", srcTensor0, "srcTensor0", "SwiGLU");
80- CheckCalCount(count, "count", srcTensor1, "srcTensor1", "SwiGLU");
81- constexpr uint32_t oneRepElm = static_cast<uint32_t>(GetVecLen() / sizeof(float));
82- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(count, oneRepElm));
83- const float scalar = static_cast<float>(static_cast<float>(-1.0) * static_cast<float>(scalarValue));
84- Internal::SwiGLUImplVF<T>((__ubuf__ T*)dstTensor.GetPhyAddr(), (__ubuf__ T*)srcTensor0.GetPhyAddr(),
85- (__ubuf__ T*)srcTensor1.GetPhyAddr(), scalar, count, repeatTimes);
86-}
87- 
88-template <typename T, bool isReuseSource = false>
89-__aicore__ inline void SwiGLUImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor0,
90- const LocalTensor<T>& srcTensor1, const float scalarValue, const LocalTensor<uint8_t>& sharedTmpBuffer,
91- const uint32_t count)
92-{
93- CheckTensorPosition(sharedTmpBuffer, "sharedTmpBuffer", "VECIN, VECOUT, VECCALC");
94- SwiGLUImpl<T, isReuseSource>(dstTensor, srcTensor0, srcTensor1, scalarValue, count);
95-}
96- 
97-template <typename T, bool isReuseSource = false>
98-__aicore__ inline void SwiGLUImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor0,
99- const LocalTensor<T>& srcTensor1, const float scalarValue, const LocalTensor<uint8_t>& sharedTmpBuffer)
100-{
101- CheckTensorPosition(sharedTmpBuffer, "sharedTmpBuffer", "VECIN, VECOUT, VECCALC");
102- SwiGLUImpl<T, isReuseSource>(dstTensor, srcTensor0, srcTensor1, scalarValue, srcTensor0.GetSize());
103-}
104- 
105-template <typename T, bool isReuseSource = false>
106-__aicore__ inline void SwiGLUImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor0,
107- const LocalTensor<T>& srcTensor1, const float scalarValue)
108-{
109- SwiGLUImpl<T, isReuseSource>(dstTensor, srcTensor0, srcTensor1, scalarValue, srcTensor0.GetSize());
110-}
111- 
112-} // namespace AscendC
113-#endif // IMPL_ACTIVATION_SWIGLU_SWIGLU_IMPL_C310_H
Dimpl/adv_api/detail/filter/dropout/dropout_l300_impl.h+0-317
@@ -1,317 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11- 
12-/* !
13- * \file dropout_l300_impl.h
14- * \brief
15- */
16-#ifndef LIB_DROPOUT_DROPOUT_L300_IMPL_H
17-#define LIB_DROPOUT_DROPOUT_L300_IMPL_H
18- 
19-#include "kernel_tensor.h"
20-#include "kernel_operator_intf.h"
21-#include "kernel_pop_stack_buffer.h"
22- 
23-namespace AscendC {
24-namespace Internal {
25-__simd_callee__ inline void DropOutBitModeFP32Main(__ubuf__ float* dstUb, __ubuf__ float* srcUb,
26- __ubuf__ uint8_t* maskUb, MicroAPI::RegTensor<float>& vDivValueReg, uint32_t sreg, uint32_t newRepeatTimes,
27- uint16_t loopH, uint32_t srcLastAxis, uint32_t maskLastAxis)
28-{
29- constexpr uint32_t unRollConstant = 2;
30- constexpr uint32_t maskBitToByte = 8;
31- constexpr uint32_t repeatElm = GetVecLen() / sizeof(float);
32- constexpr uint32_t selOffset = repeatElm / maskBitToByte * unRollConstant / (sizeof(float) / sizeof(uint8_t));
33- MicroAPI::RegTensor<float> src0Reg;
34- MicroAPI::RegTensor<float> src1Reg;
35- MicroAPI::RegTensor<float> scalarReg;
36- MicroAPI::RegTensor<float> dst0Reg;
37- MicroAPI::RegTensor<float> dst1Reg;
38- MicroAPI::MaskReg maskReg;
39- MicroAPI::MaskReg selMask0;
40- MicroAPI::MaskReg selMask1;
41- MicroAPI::MaskReg tmpMask0;
42- MicroAPI::MaskReg tmpMask1 = MicroAPI::CreateMask<uint8_t, MicroAPI::MaskPattern::ALL>();
43- MicroAPI::Duplicate(scalarReg, (const float&)0);
44- for (uint16_t i = 0; i < static_cast<uint16_t>(newRepeatTimes); ++i) {
45- MicroAPI::DataCopy<uint32_t, MicroAPI::MaskDist::DIST_US>(
46- tmpMask0, (__ubuf__ uint32_t*)maskUb + loopH * (maskLastAxis >> 2) + i * selOffset);
47- MicroAPI::MaskInterleave<uint16_t>(selMask0, selMask1, tmpMask0, tmpMask1);
48- maskReg = MicroAPI::UpdateMask<float>(sreg);
49- MicroAPI::DataCopy<float>(src0Reg, srcUb + loopH * srcLastAxis + i * unRollConstant * repeatElm);
50- MicroAPI::Select(dst0Reg, src0Reg, scalarReg, selMask0);
51- MicroAPI::Mul(dst0Reg, dst0Reg, vDivValueReg, selMask0);
52- MicroAPI::DataCopy<float>(dstUb + loopH * srcLastAxis + i * unRollConstant * repeatElm, dst0Reg, maskReg);
53- maskReg = MicroAPI::UpdateMask<float>(sreg);
54- MicroAPI::DataCopy<float>(src1Reg, srcUb + loopH * srcLastAxis + (i * unRollConstant + 1) * repeatElm);
55- MicroAPI::Select(dst1Reg, src1Reg, scalarReg, selMask1);
56- MicroAPI::Mul(dst1Reg, dst1Reg, vDivValueReg, selMask1);
57- MicroAPI::DataCopy<float>(dstUb + loopH * srcLastAxis + (i * unRollConstant + 1) * repeatElm, dst1Reg, maskReg);
58- }
59-}
60- 
61-template <typename T>
62-__simd_vf__ inline void VFDropOutBitModeCalc(__ubuf__ T* dstUb, __ubuf__ T* srcUb,
63- __ubuf__ uint8_t* maskUb, const T divValue, const uint32_t dataSize)
64-{
65- MicroAPI::RegTensor<T> vDivValueReg;
66- constexpr uint32_t repeatElm = GetVecLen() / sizeof(T);
67- uint32_t repeatTimes = CeilDivision(dataSize, repeatElm);
68- uint32_t sreg = dataSize;
69- MicroAPI::Duplicate(vDivValueReg, divValue);
70- uint32_t tail = repeatTimes & 1;
71- uint32_t newRepeatTimes = repeatTimes >> 1;
72- if constexpr (sizeof(T) == 4) {
73- DropOutBitModeFP32Main(dstUb, srcUb, maskUb, vDivValueReg, sreg, newRepeatTimes, 0, 0, 0);
74- MicroAPI::MaskReg maskReg;
75- MicroAPI::MaskReg selMask2;
76- MicroAPI::RegTensor<float> src2Reg;
77- MicroAPI::RegTensor<float> dst2Reg;
78- MicroAPI::RegTensor<float> scalarReg;
79- MicroAPI::Duplicate(scalarReg, (const T&)0);
80- uint32_t offset = newRepeatTimes * 2 * repeatElm;
81- uint32_t selOffset = newRepeatTimes * 4;
82- for (uint16_t i = 0; i < static_cast<uint16_t>(tail); ++i) {
83- MicroAPI::DataCopy<uint32_t, MicroAPI::MaskDist::DIST_US>(selMask2, (__ubuf__ uint32_t*)maskUb + selOffset);
84- MicroAPI::MaskUnPack(selMask2, selMask2);
85- maskReg = MicroAPI::UpdateMask<float>(sreg);
86- MicroAPI::DataCopy<float>(src2Reg, srcUb + offset);
87- MicroAPI::Select(dst2Reg, src2Reg, scalarReg, selMask2);
88- MicroAPI::Mul(dst2Reg, dst2Reg, vDivValueReg, selMask2);
89- MicroAPI::DataCopy<float>(dstUb + offset, dst2Reg, maskReg);
90- }
91- } else {
92- MicroAPI::RegTensor<T> src0Reg;
93- MicroAPI::RegTensor<T> src1Reg;
94- MicroAPI::RegTensor<T> dstReg;
95- MicroAPI::MaskReg maskReg;
96- MicroAPI::MaskReg selMask;
97- MicroAPI::Duplicate(src1Reg, (const T&)0);
98- for (uint16_t i = 0; i < static_cast<uint16_t>(repeatTimes); ++i) {
99- MicroAPI::DataCopy<uint32_t, MicroAPI::MaskDist::DIST_US>(selMask, (__ubuf__ uint32_t*)maskUb + i * 4);
100- maskReg = MicroAPI::UpdateMask<T>(sreg);
101- MicroAPI::DataCopy<T>(src0Reg, srcUb + i * repeatElm);
102- MicroAPI::Select(dstReg, src0Reg, src1Reg, selMask);
103- MicroAPI::Mul(dstReg, dstReg, vDivValueReg, selMask);
104- MicroAPI::DataCopy<T>(dstUb + i * repeatElm, dstReg, maskReg);
105- }
106- }
107-}
108- 
109-template <typename T>
110-__simd_vf__ inline void VFDropOutBitModeCalcInfo(__ubuf__ T* dstUb, __ubuf__ T* srcUb,
111- __ubuf__ uint8_t* maskUb, const T divValue, const DropOutShapeInfo info)
112-{
113- MicroAPI::RegTensor<T> vDivValueReg;
114- constexpr uint32_t repeatElm = GetVecLen() / sizeof(T);
115- uint32_t repeatTimes = CeilDivision(info.srcLastAxis, repeatElm);
116- MicroAPI::Duplicate(vDivValueReg, divValue);
117- uint32_t tail = repeatTimes & 1;
118- uint32_t newRepeatTimes = repeatTimes >> 1;
119- for (uint16_t loopH = 0; loopH < static_cast<uint16_t>(info.firstAxis); ++loopH) {
120- uint32_t width = info.srcLastAxis;
121- if constexpr (sizeof(T) == 4) {
122- DropOutBitModeFP32Main(dstUb, srcUb, maskUb, vDivValueReg, width, newRepeatTimes, loopH,
123- info.srcLastAxis, info.maskLastAxis);
124- } else {
125- MicroAPI::RegTensor<T> src0Reg;
126- MicroAPI::RegTensor<T> scalarReg;
127- MicroAPI::RegTensor<T> dstReg;
128- MicroAPI::MaskReg maskReg;
129- MicroAPI::MaskReg selMask;
130- MicroAPI::Duplicate(scalarReg, (const T&)0);
131- for (uint16_t i = 0; i < static_cast<uint16_t>(repeatTimes); ++i) {
132- MicroAPI::DataCopy<uint32_t, MicroAPI::MaskDist::DIST_US>(
133- selMask, (__ubuf__ uint32_t*)maskUb + loopH * (info.maskLastAxis >> 2) + i * 4);
134- maskReg = MicroAPI::UpdateMask<T>(width);
135- MicroAPI::DataCopy<T>(src0Reg, srcUb + loopH * info.srcLastAxis + i * repeatElm);
136- MicroAPI::Select(dstReg, src0Reg, scalarReg, selMask);
137- MicroAPI::Mul(dstReg, dstReg, vDivValueReg, selMask);
138- MicroAPI::DataCopy<T>(dstUb + loopH * info.srcLastAxis + i * repeatElm, dstReg, maskReg);
139- }
140- }
141- }
142- if constexpr (sizeof(T) == 4) {
143- if (tail != 0) {
144- for (uint16_t loopH = 0; loopH < static_cast<uint16_t>(info.firstAxis); ++loopH) {
145- uint32_t selOffset = newRepeatTimes * 4;
146- uint32_t offset = newRepeatTimes * 2 * repeatElm;
147- uint32_t sreg = info.srcLastAxis - offset;
148- MicroAPI::MaskReg maskReg;
149- MicroAPI::MaskReg selMask2;
150- MicroAPI::RegTensor<float> src2Reg;
151- MicroAPI::RegTensor<float> dst2Reg;
152- MicroAPI::RegTensor<float> scalarReg;
153- MicroAPI::Duplicate(scalarReg, (const T&)0);
154- MicroAPI::DataCopy<uint32_t, MicroAPI::MaskDist::DIST_US>(
155- selMask2, (__ubuf__ uint32_t*)maskUb + selOffset + loopH * (info.maskLastAxis >> 2));
156- MicroAPI::MaskUnPack(selMask2, selMask2);
157- maskReg = MicroAPI::UpdateMask<float>(sreg);
158- MicroAPI::DataCopy<float>(src2Reg, srcUb + offset + loopH * info.srcLastAxis);
159- MicroAPI::Select(dst2Reg, src2Reg, scalarReg, selMask2);
160- MicroAPI::Mul(dst2Reg, dst2Reg, vDivValueReg, selMask2);
161- MicroAPI::DataCopy<float>(dstUb + offset + loopH * info.srcLastAxis, dst2Reg, maskReg);
162- }
163- }
164- }
165-}
166- 
167-template <typename T>
168-__simd_vf__ inline void VFDropOutByteModeCalc(__ubuf__ T* dstUb, __ubuf__ T* srcUb,
169- __ubuf__ uint8_t* maskUb, const T divValue, const uint32_t dataSize)
170-{
171- MicroAPI::RegTensor<T> vSrcReg;
172- MicroAPI::RegTensor<T> vDstReg;
173- MicroAPI::RegTensor<T> vDivValueReg;
174- MicroAPI::RegTensor<uint8_t> vMaskReg;
175- MicroAPI::RegTensor<half> vFP16Reg;
176- MicroAPI::RegTensor<float> vFP32Reg;
177-#if defined(__NPU_ARCH__) && (__NPU_ARCH__ != 3003) && (__NPU_ARCH__ != 3113)
178- MicroAPI::RegTensor<bfloat16_t> vBF16Reg;
179-#endif
180- MicroAPI::MaskReg maskReg;
181- constexpr uint32_t repeatElm = GetVecLen() / sizeof(T);
182- uint32_t sreg = dataSize;
183- uint32_t repeatTimes = CeilDivision(dataSize, repeatElm);
184- MicroAPI::Duplicate(vDivValueReg, divValue);
185- for (uint16_t i = 0; i < static_cast<uint16_t>(repeatTimes); ++i) {
186- maskReg = MicroAPI::UpdateMask<T>(sreg);
187- MicroAPI::DataCopy(vSrcReg, srcUb + i * repeatElm);
188- if constexpr (sizeof(T) == 2) {
189- MicroAPI::DataCopy<uint8_t, MicroAPI::LoadDist::DIST_UNPACK_B8>(vMaskReg, maskUb + i * repeatElm);
190- MicroAPI::Cast<half, uint8_t, layoutZMrgZ>(vFP16Reg, vMaskReg, maskReg);
191-#if defined(__NPU_ARCH__) && (__NPU_ARCH__ != 3003) && (__NPU_ARCH__ != 3113)
192- if constexpr (SupportType<T, half>()) {
193- MicroAPI::Mul(vDstReg, vFP16Reg, vSrcReg, maskReg);
194- } else {
195- MicroAPI::Cast<bfloat16_t, half, MrgZRndR>(vBF16Reg, vFP16Reg, maskReg);
196- MicroAPI::Mul(vDstReg, vBF16Reg, vSrcReg, maskReg);
197- }
198-#else
199- MicroAPI::Mul(vDstReg, vFP16Reg, vSrcReg, maskReg);
200-#endif
201- } else {
202- MicroAPI::DataCopy<uint8_t, MicroAPI::LoadDist::DIST_UNPACK4_B8>(vMaskReg, maskUb + i * repeatElm);
203- MicroAPI::Cast<half, uint8_t, layoutZMrgZ>(vFP16Reg, vMaskReg, maskReg);
204- MicroAPI::Cast<float, half, layoutZMrgZ>(vFP32Reg, vFP16Reg, maskReg);
205- MicroAPI::Mul(vDstReg, vFP32Reg, vSrcReg, maskReg);
206- }
207- MicroAPI::Mul(vDstReg, vDivValueReg, vDstReg, maskReg);
208- MicroAPI::DataCopy(dstUb + i * repeatElm, vDstReg, maskReg);
209- }
210-}
211- 
212-template <typename T>
213-__simd_vf__ inline void VFDropOutByteModeCalcInfo(__ubuf__ T* dstUb, __ubuf__ T* srcUb,
214- __ubuf__ uint8_t* maskUb, const T divValue, const DropOutShapeInfo info)
215-{
216- MicroAPI::RegTensor<T> vSrcReg;
217- MicroAPI::RegTensor<T> vDstReg;
218- MicroAPI::RegTensor<T> vDivValueReg;
219- MicroAPI::RegTensor<uint8_t> vMaskReg;
220- MicroAPI::RegTensor<half> vFP16Reg;
221- MicroAPI::RegTensor<float> vFP32Reg;
222-#if defined(__NPU_ARCH__) && (__NPU_ARCH__ != 3003) && (__NPU_ARCH__ != 3113)
223- MicroAPI::RegTensor<bfloat16_t> vBF16Reg;
224-#endif
225- MicroAPI::MaskReg maskReg;
226- constexpr uint32_t repeatElm = GetVecLen() / sizeof(T);
227- uint32_t loopWNum = CeilDivision(info.srcLastAxis, repeatElm);
228- MicroAPI::Duplicate(vDivValueReg, divValue);
229- for (uint16_t loopH = 0; loopH < static_cast<uint16_t>(info.firstAxis); ++loopH) {
230- uint32_t width = info.srcLastAxis;
231- for (uint16_t loopW = 0; loopW < static_cast<uint16_t>(loopWNum); ++loopW) {
232- maskReg = MicroAPI::UpdateMask<T>(width);
233- MicroAPI::DataCopy<T>(vSrcReg, srcUb + loopH * info.srcLastAxis + loopW * repeatElm);
234- if constexpr (sizeof(T) == 2) {
235- MicroAPI::DataCopy<uint8_t, MicroAPI::LoadDist::DIST_UNPACK_B8>(
236- vMaskReg, maskUb + loopH * info.maskLastAxis + loopW * repeatElm);
237- MicroAPI::Cast<half, uint8_t, layoutZMrgZ>(vFP16Reg, vMaskReg, maskReg);
238-#if defined(__NPU_ARCH__) && (__NPU_ARCH__ != 3003) && (__NPU_ARCH__ != 3113)
239- if constexpr (SupportType<T, half>()) {
240- MicroAPI::Mul(vDstReg, vFP16Reg, vSrcReg, maskReg);
241- } else {
242- MicroAPI::Cast<bfloat16_t, half, MrgZRndR>(vBF16Reg, vFP16Reg, maskReg);
243- MicroAPI::Mul(vDstReg, vBF16Reg, vSrcReg, maskReg);
244- }
245-#else
246- MicroAPI::Mul(vDstReg, vFP16Reg, vSrcReg, maskReg);
247-#endif
248- } else {
249- MicroAPI::DataCopy<uint8_t, MicroAPI::LoadDist::DIST_UNPACK4_B8>(
250- vMaskReg, maskUb + loopH * info.maskLastAxis + loopW * repeatElm);
251- MicroAPI::Cast<half, uint8_t, layoutZMrgZ>(vFP16Reg, vMaskReg, maskReg);
252- MicroAPI::Cast<float, half, layoutZMrgZ>(vFP32Reg, vFP16Reg, maskReg);
253- MicroAPI::Mul(vDstReg, vFP32Reg, vSrcReg, maskReg);
254- }
255- MicroAPI::Mul(vDstReg, vDivValueReg, vDstReg, maskReg);
256- MicroAPI::DataCopy(dstUb + loopH * info.srcLastAxis + loopW * repeatElm, vDstReg, maskReg);
257- }
258- }
259-}
260-} // namespace Internal
261- 
262-template <typename T, bool isInitBitMode = false>
263-__aicore__ inline void DropOutBitMode(const LocalTensor<T>& dstLocal, const LocalTensor<T>& srcLocal,
264- const LocalTensor<uint8_t>& maskLocal, const LocalTensor<uint8_t>& sharedTmpBuffer, const T divValue,
265- const uint32_t dataSize)
266-{
267- static_assert(SupportType<T, half, float, bfloat16_t>(), "current data type is not supported on current device!");
268- (void)sharedTmpBuffer;
269- __ubuf__ T *srcUb = (__ubuf__ T *)srcLocal.GetPhyAddr();
270- __ubuf__ T *dstUb = (__ubuf__ T *)dstLocal.GetPhyAddr();
271- __ubuf__ uint8_t *maskUb = (__ubuf__ uint8_t *)maskLocal.GetPhyAddr();
272- 
273- Internal::VFDropOutBitModeCalc<T>(dstUb, srcUb, maskUb, divValue, dataSize);
274-}
275- 
276-template <typename T, bool isInitBitMode = false>
277-__aicore__ inline void DropOutBitMode(const LocalTensor<T>& dstLocal, const LocalTensor<T>& srcLocal,
278- const LocalTensor<uint8_t>& maskLocal, const LocalTensor<uint8_t>& sharedTmpBuffer, const T divValue,
279- const DropOutShapeInfo& info)
280-{
281- static_assert(SupportType<T, half, float, bfloat16_t>(), "current data type is not supported on current device!");
282- __ubuf__ T *srcUb = (__ubuf__ T *)srcLocal.GetPhyAddr();
283- __ubuf__ T *dstUb = (__ubuf__ T *)dstLocal.GetPhyAddr();
284- __ubuf__ uint8_t *maskUb = (__ubuf__ uint8_t *)maskLocal.GetPhyAddr();
285- 
286- Internal::VFDropOutBitModeCalcInfo<T>(dstUb, srcUb, maskUb, divValue, info);
287-}
288- 
289-template <typename T>
290-__aicore__ inline void DropOutByteMode(const LocalTensor<T>& dstLocal, const LocalTensor<T>& srcLocal,
291- const LocalTensor<uint8_t>& maskLocal, const LocalTensor<uint8_t>& sharedTmpBuffer, const T divValue,
292- const uint32_t dataSize)
293-{
294- static_assert(SupportType<T, half, float, bfloat16_t>(), "current data type is not supported on current device!");
295- (void)sharedTmpBuffer;
296- __ubuf__ T *srcUb = (__ubuf__ T *)srcLocal.GetPhyAddr();
297- __ubuf__ T *dstUb = (__ubuf__ T *)dstLocal.GetPhyAddr();
298- __ubuf__ uint8_t *maskUb = (__ubuf__ uint8_t *)maskLocal.GetPhyAddr();
299- 
300- Internal::VFDropOutByteModeCalc<T>(dstUb, srcUb, maskUb, divValue, dataSize);
301-}
302- 
303-template <typename T>
304-__aicore__ inline void DropOutByteMode(const LocalTensor<T>& dstLocal, const LocalTensor<T>& srcLocal,
305- const LocalTensor<uint8_t>& maskLocal, const LocalTensor<uint8_t>& sharedTmpBuffer, const T divValue,
306- const DropOutShapeInfo& info)
307-{
308- static_assert(SupportType<T, half, float, bfloat16_t>(), "current data type is not supported on current device!");
309- (void)sharedTmpBuffer;
310- __ubuf__ T *srcUb = (__ubuf__ T *)srcLocal.GetPhyAddr();
311- __ubuf__ T *dstUb = (__ubuf__ T *)dstLocal.GetPhyAddr();
312- __ubuf__ uint8_t *maskUb = (__ubuf__ uint8_t *)maskLocal.GetPhyAddr();
313- 
314- Internal::VFDropOutByteModeCalcInfo<T>(dstUb, srcUb, maskUb, divValue, info);
315-}
316-} // namespace AscendC
317-#endif // LIB_DROPOUT_DROPOUT_C310_IMPL_H
Dimpl/adv_api/detail/index/arithprogression/arithprogression_l300_impl.h+0-85
@@ -1,85 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/*!
12- * \file arithprogression_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_INDEX_ARITHPROGRESSION_ARITHPROGRESSION_L300_IMPL_H
16-#define IMPL_INDEX_ARITHPROGRESSION_ARITHPROGRESSION_L300_IMPL_H
17- 
18-#include "kernel_tensor.h"
19-#include "kernel_utils.h"
20-#include "kernel_log.h"
21-namespace AscendC {
22-// Generating an underlying arithmetic sequence through scalar operations.
23-template <typename RegT, typename ScalarT>
24-__simd_callee__ inline void GetBaseArithProgression(RegT &dstReg, const ScalarT firstValue, const ScalarT diffValue)
25-{
26- MicroAPI::MaskReg fullMask = MicroAPI::CreateMask<uint8_t>();
27- MicroAPI::Arange(dstReg, ScalarT(0));
28- MicroAPI::Muls(dstReg, dstReg, diffValue, fullMask);
29- MicroAPI::Adds(dstReg, dstReg, firstValue, fullMask);
30-}
31- 
32-template <typename T, const MicroAPI::RegTrait &regTrait>
33-__simd_vf__ inline void VfCallArithProgression(__ubuf__ T *dstLocalAddr, const T firstValue, const T diffValue,
34- const int32_t count, const uint16_t repeatTimes)
35-{
36- MicroAPI::RegTensor<T, regTrait> tmpReg;
37- MicroAPI::RegTensor<T, regTrait> stepReg;
38- MicroAPI::MaskReg fullMask = MicroAPI::CreateMask<T, MicroAPI::MaskPattern::ALL, regTrait>();
39- GetBaseArithProgression(tmpReg, firstValue, diffValue);
40- uint32_t sreg = static_cast<uint32_t>(count);
41- MicroAPI::MaskReg preg;
42- const uint32_t sregLower = static_cast<uint32_t>(regTrait.REG_NUM * ONE_REPEAT_BYTE_SIZE / sizeof(T));
43-#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3003 || __NPU_ARCH__ == 3113)
44- MicroAPI::Duplicate(stepReg, static_cast<T>(static_cast<int32_t>(sregLower)));
45-#else
46- MicroAPI::Duplicate(stepReg, static_cast<T>(sregLower));
47-#endif
48- MicroAPI::Muls(stepReg, stepReg, diffValue, fullMask);
49- for (uint16_t i = 0; i < repeatTimes; ++i) {
50- preg = MicroAPI::UpdateMask<T, regTrait>(sreg);
51- MicroAPI::DataCopy(dstLocalAddr + i * sregLower, tmpReg, preg);
52- MicroAPI::Add(tmpReg, tmpReg, stepReg, fullMask);
53- }
54-}
55- 
56-template <typename T>
57-__aicore__ inline void ArithProgressionImpl(const LocalTensor<T> &dstLocal, const T firstValue, const T diffValue,
58- const int32_t count)
59-{
60- ASCENDC_ASSERT((dstLocal.GetSize() >= count),
61- { KERNEL_LOG(KERNEL_ERROR, "dst length must equal with Arange length"); });
62- ASCENDC_ASSERT((static_cast<float>(diffValue) >= static_cast<float>(0)),
63- { KERNEL_LOG(KERNEL_ERROR, "diff value mast bigger then 0"); });
64- static_assert(SupportType<T, int16_t, int32_t, half, float, int64_t>(),
65- "current data type is not supported on current device!");
66- 
67- __ubuf__ T* dstLocalAddr = (__ubuf__ T*)dstLocal.GetPhyAddr();
68- if constexpr(sizeof(T) != 8) {
69- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(count, ONE_REPEAT_BYTE_SIZE / sizeof(T)));
70- VfCallArithProgression<T, MicroAPI::RegTraitNumOne>(dstLocalAddr, firstValue, diffValue, count, repeatTimes);
71- } else {
72- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(count, 2 * ONE_REPEAT_BYTE_SIZE / sizeof(T)));
73- VfCallArithProgression<T, MicroAPI::RegTraitNumTwo>(dstLocalAddr, firstValue, diffValue, count, repeatTimes);
74- }
75-}
76- 
77-template <typename T>
78-__aicore__ inline __in_pipe__(S) __out_pipe__(V, S) void ArithProgression(const LocalTensor<T> &dstLocal,
79- const T firstValue, const T diffValue, const int32_t count)
80-{
81- ArithProgressionImpl(dstLocal, firstValue, diffValue, count);
82-}
83-} // namespace AscendC
84- 
85-#endif // IMPL_INDEX_ARITHPROGRESSION_ARITHPROGRESSION_C310_IMPL_H
Dimpl/adv_api/detail/math/acos/acos_l300_impl.h+0-58
@@ -1,58 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/* !
12- * \file acos_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_MATH_ACOS_ACOS_L300_IMPL_H
16-#define IMPL_MATH_ACOS_ACOS_L300_IMPL_H
17-#include "kernel_tensor.h"
18-#include "../asin/asin_l300_impl.h"
19-#include "../../common/check.h"
20- 
21-namespace AscendC {
22-// Compute acos values according to formula: arccos(x) = PI*0.5 - arcsin(x).
23-template <typename T>
24-__aicore__ inline void AcosCompute(const LocalTensor<T>& dst, const LocalTensor<T>& src, uint32_t calSize)
25-{
26- constexpr bool convertToCos = true;
27- AsinCompute<T, convertToCos>(dst, src, calSize);
28-}
29- 
30-template <typename T, bool isReuseSource = false>
31-__aicore__ inline void AcosImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
32- const uint32_t calCount)
33-{
34- // Only for AI Vector Core.
35- if ASCEND_IS_AIC {
36- return;
37- }
38- static_assert(SupportType<T, half, float>(), "Acos only support half/float data type on current device!");
39- CheckTensorPosition(dstTensor, "dstTensor", "VECIN, VECOUT, VECCALC");
40- CheckTensorPosition(srcTensor, "srcTensor", "VECIN, VECOUT, VECCALC");
41- CheckCalCount(calCount, "calCount", srcTensor, "srcTensor", "Acos");
42- CheckCalCount(calCount, "calCount", dstTensor, "dstTensor", "Acos");
43- AcosCompute(dstTensor, srcTensor, calCount);
44-}
45- 
46-template <typename T, bool isReuseSource = false>
47-__aicore__ inline void AcosImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
48- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
49-{
50- // Only for AI Vector Core.
51- if ASCEND_IS_AIC {
52- return;
53- }
54- CheckTensorPosition(sharedTmpBuffer, "sharedTmpBuffer", "VECIN, VECOUT, VECCALC");
55- AcosImpl<T, isReuseSource>(dstTensor, srcTensor, calCount);
56-}
57-} // namespace AscendC
58-#endif // IMPL_MATH_ACOS_ACOS_L300_IMPL_H
Dimpl/adv_api/detail/math/asin/asin_l300_impl.h+0-284
@@ -1,284 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/* !
12- * \file asin_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_MATH_ASIN_ASIN_L300_IMPL_H
16-#define IMPL_MATH_ASIN_ASIN_L300_IMPL_H
17-#include "kernel_tensor.h"
18-#include "../math_constant_util.h"
19-#include "../../common/check.h"
20- 
21-namespace AscendC {
22-namespace Internal {
23-constexpr MicroAPI::CastTrait ASIN_CAST_TRAIT_NONE = {MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::SAT,
24- MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_NONE};
25- 
26-constexpr MicroAPI::CastTrait ASIN_CAST_TRAIT_FLOOR = {MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::SAT,
27- MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_FLOOR};
28- 
29-constexpr MicroAPI::CastTrait ASIN_CAST_TRAIT_RINT = {MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::SAT,
30- MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};
31- 
32-// Calculate Taylor Expansion according to (((k_nx^2 + k_n) * x^2 + k_(n-1)) * x^2 +k_(n-2) ……)*x^2 +k_0)*x.
33-template <typename T, typename RegT>
34-__simd_callee__ inline void AsinTaylorComputeInner(RegT& dstReg, RegT& srcReg, MicroAPI::MaskReg& mask)
35-{
36- MicroAPI::Muls(dstReg, dstReg, static_cast<T>(kCOEF[ASIN_TAYLOR_EXPAND_COUNT]), mask);
37- MicroAPI::Adds(dstReg, dstReg, static_cast<T>(kCOEF[6]), mask);
38- MicroAPI::Mul(dstReg, dstReg, srcReg, mask);
39- MicroAPI::Adds(dstReg, dstReg, static_cast<T>(kCOEF[5]), mask);
40- MicroAPI::Mul(dstReg, dstReg, srcReg, mask);
41- MicroAPI::Adds(dstReg, dstReg, static_cast<T>(kCOEF[4]), mask);
42- MicroAPI::Mul(dstReg, dstReg, srcReg, mask);
43- MicroAPI::Adds(dstReg, dstReg, static_cast<T>(kCOEF[3]), mask);
44- MicroAPI::Mul(dstReg, dstReg, srcReg, mask);
45- MicroAPI::Adds(dstReg, dstReg, static_cast<T>(kCOEF[2]), mask);
46- MicroAPI::Mul(dstReg, dstReg, srcReg, mask);
47- MicroAPI::Adds(dstReg, dstReg, static_cast<T>(kCOEF[1]), mask);
48- MicroAPI::Mul(dstReg, dstReg, srcReg, mask);
49- MicroAPI::Adds(dstReg, dstReg, static_cast<T>(kCOEF[0]), mask);
50-}
51- 
52-template <typename T, typename RegT>
53-__simd_callee__ inline void AsinTaylorCompute(RegT& dstReg, RegT& srcReg, MicroAPI::MaskReg& mask)
54-{
55- MicroAPI::RegTensor<T> tmpReg;
56- MicroAPI::Mul(dstReg, srcReg, srcReg, mask);
57- MicroAPI::Mul(tmpReg, srcReg, srcReg, mask);
58- AsinTaylorComputeInner<T>(dstReg, tmpReg, mask);
59- MicroAPI::Mul(dstReg, dstReg, srcReg, mask);
60-}
61- 
62-// Calculate Taylor Expansion of Asin based on its square value, and set the source to be sqrt(x).
63-template <typename T, typename RegT>
64-__simd_callee__ inline void AsinTaylorComputeBySquareValue(RegT& dstReg, RegT& srcReg, MicroAPI::MaskReg& mask)
65-{
66- MicroAPI::Muls(dstReg, srcReg, static_cast<T>(NUM_ONE), mask);
67- AsinTaylorComputeInner<T>(dstReg, srcReg, mask);
68- // Update src to be sqrt(x).
69- MicroAPI::Sqrt(srcReg, srcReg, mask);
70- MicroAPI::Mul(dstReg, dstReg, srcReg, mask);
71-}
72- 
73-template <typename T, typename RegT>
74-__simd_callee__ inline void CalRes2(RegT& resReg, RegT& srcReg, MicroAPI::MaskReg& mask)
75-{
76- MicroAPI::RegTensor<T> tmpReg;
77- MicroAPI::Mul(tmpReg, srcReg, srcReg, mask);
78- MicroAPI::Muls(tmpReg, tmpReg, NEG_ONE, mask);
79- MicroAPI::Adds(tmpReg, tmpReg, NUM_ONE, mask);
80- MicroAPI::Sqrt(tmpReg, tmpReg, mask);
81- AsinTaylorCompute<T>(resReg, tmpReg, mask);
82- MicroAPI::Muls(resReg, resReg, NEG_ONE, mask);
83- MicroAPI::Adds(resReg, resReg, HALF_PI, mask);
84-}
85- 
86-template <typename T, typename RegT>
87-__simd_callee__ inline void ProcessBranch(RegT& resReg1, RegT& resReg2, RegT& tmpReg, MicroAPI::MaskReg& mask)
88-{
89- MicroAPI::RegTensor<int32_t> s32Reg;
90- MicroAPI::Mins(tmpReg, tmpReg, BOUNDARY, mask);
91- MicroAPI::Adds(tmpReg, tmpReg, -BOUNDARY, mask);
92- MicroAPI::Cast<int32_t, T, ASIN_CAST_TRAIT_FLOOR>(s32Reg, tmpReg, mask);
93- MicroAPI::Cast<T, int32_t, ASIN_CAST_TRAIT_RINT>(tmpReg, s32Reg, mask);
94- MicroAPI::Muls(tmpReg, tmpReg, NEG_ONE, mask);
95- MicroAPI::Mul(resReg1, resReg1, tmpReg, mask);
96- MicroAPI::Muls(tmpReg, tmpReg, NEG_ONE, mask);
97- MicroAPI::Adds(tmpReg, tmpReg, NUM_ONE, mask);
98- MicroAPI::Mul(resReg2, resReg2, tmpReg, mask);
99- MicroAPI::Add(resReg1, resReg1, resReg2, mask);
100-}
101- 
102-// Calculate the sign of given values.
103-// Algorithm:
104-// FP16: sign(x) = 2^(15) * x /(2^(-15) + 2^(15) *|x|)
105-// FP32: sign(x) = 2^(62) * x /(2^(-62) + 2^(62) *|x|)
106-template <typename T, typename RegT>
107-__simd_callee__ inline void GetSign(RegT& dstReg, RegT& srcReg, MicroAPI::MaskReg& mask)
108-{
109- MicroAPI::RegTensor<T> denominatorReg;
110- constexpr float FP16_MAX = 32768; // 2^15
111- constexpr float FP16_MIN = 3.0517578125e-05; // 2^-15
112- constexpr float FP32_MAX = 4611686018427387904; // 2^62
113- constexpr float FP32_MIN = 2.168404344971009e-19; // 2^-62
114- constexpr float kFpMax = sizeof(T) == sizeof(float) ? FP32_MAX : FP16_MAX;
115- constexpr float kFpMin = sizeof(T) == sizeof(float) ? FP32_MIN : FP16_MIN;
116- MicroAPI::Muls(dstReg, srcReg, static_cast<T>(kFpMax), mask);
117- MicroAPI::Abs(denominatorReg, dstReg, mask);
118- MicroAPI::Adds(denominatorReg, denominatorReg, static_cast<T>(kFpMin), mask);
119- MicroAPI::Div(dstReg, dstReg, denominatorReg, mask);
120-}
121- 
122-// Compute asin values based on input types.
123-// asin(x) = arcsin(sqrt(1-x^2)) - PI*0.5 when x belongs to (-1, -2^(-0.5))
124-// asin(x) = the 15th order taylor expansion when x belongs to (-2^(-0.5), 2^(-0.5))
125-// asin(x) = PI*0.5 - arcsin(sqrt(1-x^2)) when x belongs to (2^(-0.5), 1)
126-template <typename T, bool convertToAcos = false>
127-__simd_vf__ inline void AsinComputeVFF32(__ubuf__ T* dstUb, __ubuf__ T* srcUb, uint32_t calSize,
128- uint16_t repeatTimes, uint16_t stride)
129-{
130- MicroAPI::MaskReg mask;
131- MicroAPI::RegTensor<T> srcReg;
132- MicroAPI::RegTensor<T> dstReg;
133- MicroAPI::RegTensor<T> resReg1;
134- MicroAPI::RegTensor<T> resReg2;
135- MicroAPI::RegTensor<T> signReg;
136- MicroAPI::RegTensor<T> tmpReg;
137- for (uint16_t i = 0; i < repeatTimes; ++i) {
138- mask = MicroAPI::UpdateMask<T>(calSize);
139- MicroAPI::DataCopy(srcReg, srcUb + i * stride);
140- // Calculate res2 = PI*0.5 - taylor_compute(sqrt(1 - x^2)) -> resReg2.
141- CalRes2<T>(resReg2, srcReg, mask);
142- // Calculate res1 = taylor_compute(abs(x)) -> dst, abs(x) -> resReg1.
143- MicroAPI::Mul(tmpReg, srcReg, srcReg, mask);
144- AsinTaylorComputeBySquareValue<T>(resReg1, tmpReg, mask);
145- // As NPU are not good at scalar process like CPU for if-else statement, the solution here used for handling above
146- // 3 scenarios is to calculate 0/1 choices combining the results on both options.
147- // e.g.
148- // Step1: Calculate both option results of x, no matter which range it's at.
149- // result1(x), result2(x)
150- // Step2: Calculate 0/1 choices of both option results of x, no matter which range it's at.
151- // choice1(x), choice2(x)
152- // Step3: Combine choice result and options results, since at least one choice should be zero.
153- // Result = choice1(x) * result1(x) + choice2(x) * result2(x)
154- // choice1 = -Floor(min(abs(x), BOUNDARY) - BOUNDARY).
155- // choice2 = 1 - choice1
156- // res = res1 * choice1 + res2 * choice2
157- ProcessBranch<T>(resReg1, resReg2, tmpReg, mask);
158- GetSign<T>(signReg, srcReg, mask);
159- MicroAPI::Mul(dstReg, resReg1, signReg, mask);
160- if constexpr (convertToAcos) {
161- // Compute acos values according to formula: arccos(x) = PI*0.5 - arcsin(x).
162- MicroAPI::Adds(dstReg, dstReg, static_cast<T>(-HALF_PI), mask);
163- MicroAPI::Muls(dstReg, dstReg, static_cast<T>(NEG_ONE), mask);
164- }
165- MicroAPI::DataCopy(dstUb + i * stride, dstReg, mask);
166- }
167-}
168- 
169-template <typename T, bool convertToAcos = false>
170-__simd_vf__ inline void AsinComputeVFF16(__ubuf__ T* dstUb, __ubuf__ T* srcUb, uint32_t calSize,
171- uint16_t repeatTimes, uint16_t stride)
172-{
173- MicroAPI::MaskReg mask;
174- MicroAPI::RegTensor<half> srcReg;
175- MicroAPI::RegTensor<half> dstReg;
176- MicroAPI::RegTensor<half> halfReg1;
177- MicroAPI::RegTensor<half> halfReg2;
178- MicroAPI::RegTensor<half> tmpReg;
179- MicroAPI::RegTensor<float> floatReg1;
180- MicroAPI::RegTensor<float> floatReg2;
181- MicroAPI::RegTensor<int8_t> s8Reg;
182- for (uint16_t i = 0; i < repeatTimes; ++i) {
183- mask = MicroAPI::UpdateMask<float>(calSize);
184- // Cast src from half to float type for getting more precise results, but only computes by finishing
185- // taylor expansion computation as it's the majority reason of precision loss.
186- MicroAPI::DataCopy<half, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcReg, srcUb + i * stride);
187- MicroAPI::Cast<float, half, ASIN_CAST_TRAIT_NONE>(floatReg2, srcReg, mask);
188- // Calculate res2 = PI*0.5 - taylor_compute(sqrt(1 - x^2)).
189- MicroAPI::Mul(floatReg2, floatReg2, floatReg2, mask);
190- MicroAPI::Muls(floatReg2, floatReg2, NEG_ONE, mask);
191- MicroAPI::Adds(floatReg2, floatReg2, NUM_ONE, mask);
192- AsinTaylorComputeBySquareValue<half>(floatReg1, floatReg2, mask);
193- MicroAPI::Muls(floatReg1, floatReg1, NEG_ONE, mask);
194- MicroAPI::Adds(floatReg1, floatReg1, HALF_PI, mask);
195- 
196- // Calculate res1 = taylor_compute(abs(x)).
197- MicroAPI::Abs(halfReg2, srcReg, mask);
198- AsinTaylorCompute<half>(dstReg, halfReg2, mask);
199- 
200- // As NPU are not good at scalar process like CPU for if-else statement, the solution here used for handling above
201- // 3 scenarios is to calculate 0/1 choices combining the results on both options.
202- // e.g.
203- // Step1: Calculate both option results of x, no matter which range it's at.
204- // result1(x), result2(x)
205- // Step2: Calculate 0/1 choices of both option results of x, no matter which range it's at.
206- // choice1(x), choice2(x)
207- // Step3: Combine choice result and optional result, since at least one choice should be zero.
208- // Result = choice1(x) * result1(x) + choice2(x) * result2(x)
209- // choice1 = -Floor(min(abs(x), BOUNDARY) - BOUNDARY).
210- // choice2 = 1 - choice1
211- // res = res1 * choice1 + res2 * choice2
212- MicroAPI::Mins(halfReg2, halfReg2, static_cast<half>(BOUNDARY), mask);
213- MicroAPI::Adds(halfReg2, halfReg2, static_cast<half>(-BOUNDARY), mask);
214- MicroAPI::Cast<int8_t, half, ASIN_CAST_TRAIT_FLOOR>(s8Reg, halfReg2, mask);
215- MicroAPI::Cast<half, int8_t, ASIN_CAST_TRAIT_NONE>(halfReg2, s8Reg, mask);
216- MicroAPI::Muls(halfReg2, halfReg2, static_cast<half>(NEG_ONE), mask);
217- MicroAPI::Mul(dstReg, dstReg, halfReg2, mask);
218- MicroAPI::Muls(halfReg2, halfReg2, static_cast<half>(NEG_ONE), mask);
219- MicroAPI::Adds(halfReg2, halfReg2, static_cast<half>(NUM_ONE), mask);
220- MicroAPI::Cast<float, half, ASIN_CAST_TRAIT_NONE>(floatReg2, halfReg2, mask);
221- MicroAPI::Mul(floatReg1, floatReg1, floatReg2, mask);
222- MicroAPI::Cast<float, half, ASIN_CAST_TRAIT_NONE>(floatReg2, dstReg, mask);
223- MicroAPI::Add(floatReg1, floatReg2, floatReg1, mask);
224- GetSign<half>(halfReg1, srcReg, mask);
225- MicroAPI::Cast<float, half, ASIN_CAST_TRAIT_NONE>(floatReg2, halfReg1, mask);
226- MicroAPI::Mul(floatReg1, floatReg1, floatReg2, mask);
227- if constexpr (convertToAcos) {
228- // Compute acos values according to formula: arccos(x) = PI*0.5 - arcsin(x).
229- MicroAPI::Adds(floatReg1, floatReg1, -HALF_PI, mask);
230- MicroAPI::Muls(floatReg1, floatReg1, NEG_ONE, mask);
231- }
232- MicroAPI::Cast<half, float, ASIN_CAST_TRAIT_RINT>(dstReg, floatReg1, mask);
233- MicroAPI::DataCopy<half, MicroAPI::StoreDist::DIST_PACK_B32>(dstUb + i * stride, dstReg, mask);
234- }
235-}
236-} // namespace Internal
237- 
238-template <typename T, bool convertToAcos = false>
239-__aicore__ inline void AsinCompute(const LocalTensor<T>& dst, const LocalTensor<T>& src, uint32_t calSize)
240-{
241- __ubuf__ T *dstUb = (__ubuf__ T *)dst.GetPhyAddr();
242- __ubuf__ T *srcUb = (__ubuf__ T *)src.GetPhyAddr();
243- 
244- // half dtype will be converted to float to improve precision;
245- constexpr uint16_t stride = GetVecLen() / sizeof(float);
246- uint16_t repeatTimes = CeilDivision(calSize, stride);
247- if constexpr (IsSameType<T, half>::value) {
248- Internal::AsinComputeVFF16<T, convertToAcos>(dstUb, srcUb, calSize, repeatTimes, stride);
249- } else if (IsSameType<T, float>::value) {
250- Internal::AsinComputeVFF32<T, convertToAcos>(dstUb, srcUb, calSize, repeatTimes, stride);
251- }
252-}
253- 
254-template <typename T, bool isReuseSource = false>
255-__aicore__ inline void AsinImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
256- const uint32_t calCount)
257-{
258- // Only for AI Vector Core.
259- if ASCEND_IS_AIC {
260- return;
261- }
262- static_assert(SupportType<T, half, float>(), "Asin only support half/float data type on current device!");
263- 
264- CheckTensorPosition(dstTensor, "dstTensor", "VECIN, VECOUT, VECCALC");
265- CheckTensorPosition(srcTensor, "srcTensor", "VECIN, VECOUT, VECCALC");
266- CheckCalCount(calCount, "calCount", srcTensor, "srcTensor", "Asin");
267- CheckCalCount(calCount, "calCount", dstTensor, "dstTensor", "Asin");
268- AsinCompute(dstTensor, srcTensor, calCount);
269-}
270- 
271-template <typename T, bool isReuseSource = false>
272-__aicore__ inline void AsinImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
273- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
274-{
275- // Only for AI Vector Core.
276- if ASCEND_IS_AIC {
277- return;
278- }
279- CheckTensorPosition(sharedTmpBuffer, "sharedTmpBuffer", "VECIN, VECOUT, VECCALC");
280- AsinImpl<T, isReuseSource>(dstTensor, srcTensor, calCount);
281-}
282-} // namespace AscendC
283- 
284-#endif // IMPL_MATH_ASIN_ASIN_L300_IMPL_H
Dimpl/adv_api/detail/math/atan/atan_l300_impl.h+0-416
@@ -1,416 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/* !
12- * \file atan_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_MATH_ATAN_ATAN_L300_IMPL_H
16-#define IMPL_MATH_ATAN_ATAN_L300_IMPL_H
17- 
18-#include "kernel_tensor.h"
19-#include "kernel_pop_stack_buffer.h"
20- 
21-namespace AscendC {
22-enum class AtanAlgo { TAYLOR_EXPANSION = 0, POLYNOMIAL_APPROXIMATION };
23- 
24-struct AtanConfig {
25- AtanAlgo algo = AtanAlgo::TAYLOR_EXPANSION;
26-};
27- 
28-constexpr AtanConfig defaultAtanConfig = { AtanAlgo::TAYLOR_EXPANSION };
29- 
30-namespace Internal {
31-constexpr float ATAN_FP16_MAX = 32768; // 2^15
32-constexpr float ATAN_FP16_MIN = 3.0517578125e-05; // 2^-15
33-constexpr float ATAN_FP32_MAX = 4611686018427387904; // 2^62
34-constexpr float ATAN_FP32_MIN = 2.168404344971009e-19; // 2^-62
35-constexpr uint16_t TAYLOR_COUNT_FOUR = 4; // x belongs to (0, tan(pi/8))
36-constexpr uint16_t TAYLOR_COUNT_SIX = 6; // x belongs to (tan(pi/8), tan(pi/4))
37-constexpr float MIN_INPUT_VALUE = -10000;
38-constexpr float MAX_INPUT_VALUE = 10000;
39-// Calculates the Sign of given values.
40-// Algorithm:
41-// FP16: sign(x) = 2**(15) * x /(2**(-15) + 2**(15) *|x|)
42-// FP32: sign(x) = 2**(62) * x /(2**(-62) + 2**(62) *|x|)
43-template <typename T>
44-__simd_callee__ inline void Sign(MicroAPI::RegTensor<T>& dstReg, MicroAPI::RegTensor<T>& srcReg,
45- MicroAPI::RegTensor<T>& denominator, MicroAPI::MaskReg preg)
46-{
47- constexpr float kFpMax = sizeof(T) == sizeof(float) ? ATAN_FP32_MAX : ATAN_FP16_MAX;
48- constexpr float kFpMin = sizeof(T) == sizeof(float) ? ATAN_FP32_MIN : ATAN_FP16_MIN;
49- MicroAPI::Muls(dstReg, srcReg, static_cast<T>(kFpMax), preg);
50- MicroAPI::Abs(denominator, dstReg, preg);
51- MicroAPI::Adds(denominator, denominator, static_cast<T>(kFpMin), preg);
52- MicroAPI::Div(dstReg, dstReg, denominator, preg);
53-}
54- 
55-// arctan(x) = x - x^3/3 + x^5/5 + ... + (-1)^k*x^(k*2+1)/( k*2+1)
56-// 1/(k*2+1)
57-__simd_callee__ inline void TaylorExpand(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg,
58- MicroAPI::RegTensor<float>& squareReg, const uint16_t expandLevel, MicroAPI::MaskReg preg)
59-{
60- // arctan(x) = x - x^3/3 + x^5/5 + ... + (-1)^k*x^(k*2+1)/( k*2+1)
61- // 1/(k*2+1)
62- constexpr float factorList[7] = {1, -0.3333333333333333, 0.2, -0.14285714285714285, 0.1111111111111111,
63- -0.09090909090909091, 0.07692307692307693};
64- uint16_t COUNT_SIX = expandLevel == TAYLOR_COUNT_SIX ? 1 : 0;
65- // The initial value of dstReg is assigned as the coefficient of the last item of expansion.
66- MicroAPI::Mul(squareReg, srcReg, srcReg, preg);
67- MicroAPI::Mul(dstReg, srcReg, srcReg, preg);
68- MicroAPI::Muls(dstReg, dstReg, factorList[expandLevel], preg);
69- for (uint16_t i = 0; i < COUNT_SIX; ++i) {
70- // dst*x^2+ the previois expand factor
71- MicroAPI::Adds(dstReg, dstReg, factorList[6], preg);
72- MicroAPI::Mul(dstReg, dstReg, squareReg, preg);
73- MicroAPI::Adds(dstReg, dstReg, factorList[5], preg);
74- }
75- // dst*x^2+ the previois expand factor
76- MicroAPI::Adds(dstReg, dstReg, factorList[4], preg);
77- MicroAPI::Mul(dstReg, dstReg, squareReg, preg);
78- MicroAPI::Adds(dstReg, dstReg, factorList[3], preg);
79- MicroAPI::Mul(dstReg, dstReg, squareReg, preg);
80- MicroAPI::Adds(dstReg, dstReg, factorList[2], preg);
81- MicroAPI::Mul(dstReg, dstReg, squareReg, preg);
82- MicroAPI::Adds(dstReg, dstReg, factorList[1], preg);
83- MicroAPI::Mul(dstReg, dstReg, squareReg, preg);
84- MicroAPI::Adds(dstReg, dstReg, factorList[0], preg);
85- MicroAPI::Mul(dstReg, dstReg, srcReg, preg);
86-}
87- 
88-// (x-y)/(1+xy)
89-__simd_callee__ inline void AtanTransform(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg,
90- MicroAPI::RegTensor<float>& tmpReg, const float transFactor, MicroAPI::MaskReg preg)
91-{
92- // x*y
93- MicroAPI::Muls(dstReg, srcReg, transFactor, preg);
94- // x*y + 1
95- MicroAPI::Adds(dstReg, dstReg, 1.0f, preg);
96- // x=x-y
97- MicroAPI::Adds(tmpReg, srcReg, -transFactor, preg);
98- // (x-y)/(1+xy)
99- MicroAPI::Div(dstReg, tmpReg, dstReg, preg);
100- MicroAPI::Abs(dstReg, dstReg, preg);
101-}
102- 
103-__simd_callee__ inline void AtanCompute(
104- MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& castReg, MicroAPI::MaskReg preg)
105-{
106- constexpr float piByFour = 0.78539816339744830961566084581988;
107- constexpr float piByEight = 0.39269908169872415480783042290994;
108- constexpr float tanPiByEight = 0.4142135623730950;
109- 
110- MicroAPI::RegTensor<float> clipReg;
111- MicroAPI::RegTensor<float> absReg;
112- MicroAPI::RegTensor<float> tmpReg;
113- MicroAPI::RegTensor<float> tmpReg2;
114- MicroAPI::RegTensor<float> squareReg;
115- // when x's value is too large the first caculator of TaylorExpand will be overflow. when epsilon is 0.0001,
116- // the approximate value of `tan(pi/2 - 0.0001)` is 10000
117- // Clip x to [MIN_INPUT_VALUE, MAX_INPUT_VALUE] in float
118- MicroAPI::Mins(clipReg, castReg, MAX_INPUT_VALUE, preg);
119- MicroAPI::Maxs(clipReg, clipReg, MIN_INPUT_VALUE, preg);
120- MicroAPI::Abs(absReg, clipReg, preg);
121- // 1. x in (0, tan(pi/8))
122- TaylorExpand(dstReg, absReg, squareReg, TAYLOR_COUNT_FOUR, preg);
123- // 2. x in (tan(pi/8), tan(pi/4)), atan(x) = pi/8 + atan((x-tan(pi/8)) / (1 + x*tan(pi/8)))
124- // normalize x to (0, tan(pi/8))
125- AtanTransform(tmpReg, absReg, tmpReg2, tanPiByEight, preg); // tan(pi/8)
126- TaylorExpand(tmpReg2, tmpReg, squareReg, TAYLOR_COUNT_FOUR, preg);
127- MicroAPI::Adds(tmpReg2, tmpReg2, piByEight, preg);
128- MicroAPI::Min(dstReg, dstReg, tmpReg2, preg);
129- // x in (tan(pi/4), +∞), atan(x) = pi/4 + atan((x-1)/(x+1))
130- // calculate |(x-1)/(x+1)|, normalize x to (0, tan(pi/4))
131- // find the minimum value between atan(|(x-1)/(x+1)|) calculate in (0, tan(pi/8)) and (tan(pi/8), tan(pi/4))
132- MicroAPI::Adds(tmpReg2, absReg, 1.0f, preg);
133- MicroAPI::Adds(tmpReg, absReg, -1.0f, preg);
134- MicroAPI::Div(tmpReg, tmpReg, tmpReg2, preg);
135- MicroAPI::Abs(tmpReg, tmpReg, preg); // take the absolute value
136- // 3. atan(|(x-1)/(x+1)|)
137- TaylorExpand(tmpReg2, tmpReg, squareReg, TAYLOR_COUNT_FOUR, preg);
138- // pi/4 + atan(|(x-1)/(x+1)|)
139- MicroAPI::Adds(tmpReg2, tmpReg2, piByFour, preg);
140- MicroAPI::Min(dstReg, dstReg, tmpReg2, preg);
141- // 4.reuse the transform result in step 3, and calculate (x-tan(pi/8)) / (1 + x*tan(pi/8))
142- AtanTransform(tmpReg2, tmpReg, squareReg, tanPiByEight, preg);
143- TaylorExpand(tmpReg, tmpReg2, squareReg, TAYLOR_COUNT_SIX, preg);
144- // pi/8 + pi/4 + atan((x-tan(pi/8)) / (1 + x*tan(pi/8)))
145- MicroAPI::Adds(tmpReg, tmpReg, piByEight, preg);
146- MicroAPI::Adds(tmpReg, tmpReg, piByFour, preg);
147- MicroAPI::Min(dstReg, dstReg, tmpReg, preg);
148- Sign(tmpReg, clipReg, tmpReg2, preg);
149- // dst = sign(x) * dst.
150- MicroAPI::Mul(dstReg, dstReg, tmpReg, preg);
151-}
152- 
153-template <typename T, bool isReuseSource = false>
154-__simd_vf__ inline void AtanTaylorVFImpl(
155- __ubuf__ T* dstUb, __ubuf__ T* srcUb, const uint32_t calCount)
156-{
157- uint16_t repeatTimes = CeilDivision(calCount, B32_DATA_NUM_PER_REPEAT);
158- 
159- uint32_t sreg = calCount;
160- MicroAPI::MaskReg preg;
161- MicroAPI::RegTensor<T> srcReg;
162- MicroAPI::RegTensor<float> castReg;
163- MicroAPI::RegTensor<float> dstReg;
164- 
165- for (uint16_t i = 0; i < repeatTimes; ++i) {
166- preg = MicroAPI::UpdateMask<float>(sreg);
167- if constexpr (sizeof(T) == sizeof(half)) {
168- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcReg, srcUb + i * B32_DATA_NUM_PER_REPEAT);
169- MicroAPI::Cast<float, T, castTraitB16ToB32>(castReg, srcReg, preg);
170- } else {
171- MicroAPI::DataCopy(castReg, srcUb + i * B32_DATA_NUM_PER_REPEAT);
172- }
173- AtanCompute(dstReg, castReg, preg);
174- if constexpr (sizeof(T) == sizeof(half)) {
175- MicroAPI::Cast<T, float, castTraitB32ToB16>(srcReg, dstReg, preg);
176- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(
177- dstUb + i * B32_DATA_NUM_PER_REPEAT, srcReg, preg);
178- } else {
179- MicroAPI::DataCopy(dstUb + i * B32_DATA_NUM_PER_REPEAT, dstReg, preg);
180- }
181- }
182-}
183- 
184-// when x < 0, Atan(x) = atan(-x)
185-// when x belongs to (0, tan(pi/8)), Atan(x) = atan(x)
186-// when x belongs to (tan(pi/8), tan(pi/4)), Atan(x) = pi/8 + atan((x- tan(pi/8)) / (1+ x*tan(pi/8)))
187-// when x belongs to (tan(pi/4), +∞), Atan(x) = pi/4 + atan((x-1)/(x+1))
188-template <typename T, bool isReuseSource = false>
189-__aicore__ inline void AtanTaylorImpl(
190- const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor, const uint32_t calCount)
191-{
192- __ubuf__ T* dstUb = (__ubuf__ T*)dstTensor.GetPhyAddr();
193- __ubuf__ T* srcUb = (__ubuf__ T*)srcTensor.GetPhyAddr();
194- AtanTaylorVFImpl<T, isReuseSource>(dstUb, srcUb, calCount);
195-}
196- 
197-template <typename T, bool isReuseSource = false>
198-__simd_vf__ inline void AtanPolynomialVFImpl(
199- __ubuf__ T* dstUb, __ubuf__ T* srcUb, const uint32_t calCount)
200-{
201- constexpr float a1 = -0.333329409;
202- constexpr float a2 = 0.199887753;
203- constexpr float a3 = -0.141718030;
204- constexpr float a4 = 0.105184801;
205- constexpr float a5 = -0.0725297481;
206- constexpr float a6 = 0.0398497507;
207- constexpr float a7 = -0.0143969795;
208- constexpr float a8 = 0.00245002890;
209- constexpr float b1 = 1.68325555;
210- constexpr float b2 = 0.933189452;
211- constexpr float floatOne = 1.0;
212- constexpr int32_t signBit = -2147483648;
213- constexpr uint16_t vlSize = static_cast<uint16_t>(GetVecLen() / sizeof(T));
214- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(calCount, vlSize));
215- 
216- uint32_t sreg = calCount;
217- MicroAPI::MaskReg preg0;
218- MicroAPI::MaskReg preg1;
219- MicroAPI::MaskReg fullMask = MicroAPI::CreateMask<T, MicroAPI::MaskPattern::ALL>();
220- MicroAPI::RegTensor<T> vreg0;
221- MicroAPI::RegTensor<T> vreg1;
222- MicroAPI::RegTensor<T> vreg2;
223- MicroAPI::RegTensor<T> vreg3;
224- MicroAPI::RegTensor<T> vreg4;
225- MicroAPI::RegTensor<T> vreg5;
226- MicroAPI::RegTensor<T> vreg6;
227- MicroAPI::RegTensor<T> vreg7;
228- MicroAPI::RegTensor<T> vreg8;
229- MicroAPI::RegTensor<T> vreg9;
230- MicroAPI::RegTensor<T> vreg10;
231- MicroAPI::RegTensor<T> vreg11;
232- MicroAPI::RegTensor<T> vreg12;
233- MicroAPI::RegTensor<T> vreg13;
234- MicroAPI::RegTensor<T> vreg14;
235- MicroAPI::RegTensor<int32_t> vreg15;
236- 
237- MicroAPI::Duplicate(vreg1, a1, fullMask);
238- MicroAPI::Duplicate(vreg2, a2, fullMask);
239- MicroAPI::Duplicate(vreg3, a3, fullMask);
240- MicroAPI::Duplicate(vreg4, a4, fullMask);
241- MicroAPI::Duplicate(vreg5, a5, fullMask);
242- MicroAPI::Duplicate(vreg6, a6, fullMask);
243- MicroAPI::Duplicate(vreg8, a8, fullMask);
244- MicroAPI::Duplicate(vreg10, floatOne, fullMask);
245- 
246- for (uint16_t i = 0; i < repeatTimes; ++i) {
247- preg0 = MicroAPI::UpdateMask<T>(sreg);
248- MicroAPI::DataCopy(vreg0, srcUb + i * vlSize);
249- // x_con = (if x_abs > 1.0 ? 1.0 / x_abs : x_abs)
250- MicroAPI::Abs(vreg9, vreg0, preg0);
251- MicroAPI::Div(vreg11, vreg10, vreg9, preg0);
252- MicroAPI::CompareScalar<T, CMPMODE::GE>(preg1, vreg9, floatOne, preg0);
253- MicroAPI::Select(vreg11, vreg11, vreg9, preg1);
254- 
255- // y = taylor_expansion(x_con)
256- MicroAPI::Mul(vreg12, vreg11, vreg11, preg0); // s_x = x_con*x_con
257- MicroAPI::Duplicate(vreg7, a7, preg0);
258- MicroAPI::MulAddDst(vreg7, vreg8, vreg12, preg0); // y = a8*s_x + a7
259- MicroAPI::FusedMulDstAdd(vreg7, vreg12, vreg6, preg0); // y = y*s_x + a6
260- MicroAPI::FusedMulDstAdd(vreg7, vreg12, vreg5, preg0); // y = y*s_x + a5
261- MicroAPI::FusedMulDstAdd(vreg7, vreg12, vreg4, preg0); // y = y*s_x + a4
262- MicroAPI::FusedMulDstAdd(vreg7, vreg12, vreg3, preg0); // y = y*s_x + a3
263- MicroAPI::FusedMulDstAdd(vreg7, vreg12, vreg2, preg0); // y = y*s_x + a2
264- MicroAPI::FusedMulDstAdd(vreg7, vreg12, vreg1, preg0); // y = y*s_x + a1
265- 
266- MicroAPI::Mul(vreg12, vreg7, vreg12, preg0); // tmp = (y*s_x)
267- MicroAPI::FusedMulDstAdd(vreg12, vreg11, vreg11, preg0); // y = (y*s_x) * x_con + x_con
268- 
269- MicroAPI::Duplicate(vreg11, b1, preg0);
270- MicroAPI::Duplicate(vreg14, b2, preg0);
271- MicroAPI::Neg(vreg13, vreg12, preg0); // -1.0*y
272- MicroAPI::MulAddDst(vreg13, vreg11, vreg14, preg0); // y_if = b1*b2 + (-1.0 * y) //vmula
273- MicroAPI::Select(vreg13, vreg13, vreg12, preg1); // y = (if x_abs > 1.0 ? y_if : y)
274- 
275- // x_s32 = f32_to_s32(float(x))
276- // x_s32_temp = x_s32 & 0x80000000
277- MicroAPI::Duplicate(vreg15, signBit, preg0);
278- MicroAPI::And(vreg15, (MicroAPI::RegTensor<int32_t>&)vreg0, vreg15, preg0);
279- // y_s32 = f32_to_s32(float(y))
280- // y_temp = y_s32 | x_s32_temp
281- MicroAPI::Or(vreg15, (MicroAPI::RegTensor<int32_t>&)vreg13, vreg15, preg0);
282- // y = s32_to_f32(y_temp)
283- MicroAPI::DataCopy(dstUb + i * vlSize, (MicroAPI::RegTensor<T>&)vreg15, preg0);
284- }
285-}
286- 
287-template <typename T, bool isReuseSource = false>
288-__aicore__ inline void AtanPolynomialImpl(
289- const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor, const uint32_t calCount)
290-{
291- __ubuf__ T* dstUb = (__ubuf__ T*)dstTensor.GetPhyAddr();
292- __ubuf__ T* srcUb = (__ubuf__ T*)srcTensor.GetPhyAddr();
293- 
294- AtanPolynomialVFImpl<T, isReuseSource>(dstUb, srcUb, calCount);
295-}
296-} // namespace Internal
297- 
298-template <typename T, bool isReuseSource, const AtanConfig& config>
299-__aicore__ inline void AtanImpl(
300- const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor, const uint32_t calCount)
301-{
302- CheckTensorPos<T>(dstTensor, Hardware::UB, "dstTensor", "VECIN / VECCALC / VECOUT", "Atan");
303- CheckTensorPos<T>(srcTensor, Hardware::UB, "srcTensor", "VECIN / VECCALC / VECOUT", "Atan");
304- ASCENDC_ASSERT((calCount <= srcTensor.GetSize()), {
305- KERNEL_LOG(KERNEL_ERROR, "calCount is %u, which should not be larger than srcTensor length %u", calCount,
306- srcTensor.GetSize());
307- });
308- ASCENDC_ASSERT((calCount <= dstTensor.GetSize()), {
309- KERNEL_LOG(KERNEL_ERROR, "calCount is %u, which should not be larger than dstTensor length %u", calCount,
310- dstTensor.GetSize());
311- });
312- 
313- if constexpr (config.algo == defaultAtanConfig.algo) {
314- static_assert(SupportType<T, half, float>(),
315- "Atan with TAYLOR_EXPANSION algorithm only support half/float data type on current device!");
316- Internal::AtanTaylorImpl(dstTensor, srcTensor, calCount);
317- } else {
318- static_assert(SupportType<T, float>(),
319- "Atan with POLYNOMIAL_APPROXIMATION algorithm only support float data type on current device!!");
320- Internal::AtanPolynomialImpl(dstTensor, srcTensor, calCount);
321- }
322-}
323- 
324-template <typename T, bool isReuseSource = false, const AtanConfig& config = defaultAtanConfig>
325-__aicore__ inline void AtanImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
326- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
327-{
328- // Only for AI Vector Core.
329- if ASCEND_IS_AIC {
330- return;
331- }
332- CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN / VECCALC / VECOUT", "Atan");
333- 
334- AtanImpl<T, isReuseSource, config>(dstTensor, srcTensor, calCount);
335-}
336- 
337-/*!
338- * \ingroup Atan
339- * \brief compute Atan elementwisely
340- * \tparam T: half/float
341- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
342- * this parameter is reserved, please use the default value.
343- * \param [out] dstTensor: output LocalTensor
344- * \param [in] srcTensor: input LocalTensor
345- * \param [in] sharedTmpBuffer: extra temporary shared space used for intermediate values among calculation process,
346- * whose required space size should refer to corresponding tiling API, which is defined at atan_tiling.h.
347- * Generally, the more space you allocate, the better performance you will achieve, and the performance
348- * reaches peak when buffer size is maximum(calculated by tiling function). Moreover, it is not guaranteed
349- * that the shared space will be cleared after usage, the data could be anything.
350- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
351- */
352-template <typename T, bool isReuseSource = false, const AtanConfig& config = defaultAtanConfig>
353-__aicore__ inline void Atan(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
354- const LocalTensor<uint8_t>& sharedTmpBuffer)
355-{
356- Atan<T, isReuseSource, config>(dstTensor, srcTensor, sharedTmpBuffer, srcTensor.GetSize());
357-}
358- 
359- /*!
360- * \ingroup Atan
361- * \brief compute Atan elementwisely
362- * \tparam T: half/float
363- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
364- * this parameter is reserved, please use the default value.
365- * \param [out] dstTensor: output LocalTensor
366- * \param [in] srcTensor: input LocalTensor
367- * \param [in] sharedTmpBuffer: extra temporary shared space used for intermediate values among calculation process,
368- * whose required space size should refer to corresponding tiling API, which is defined at atan_tiling.h.
369- * Generally, the more space you allocate, the better performance you will achieve, and the performance
370- * reaches peak when buffer size is maximum(calculated by tiling function). Moreover, it is not guaranteed
371- * that the shared space will be cleared after usage, the data could be anything.
372- * \param [in] calCount: the number of elements to be processed.
373- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
374- */
375-template <typename T, bool isReuseSource = false, const AtanConfig& config = defaultAtanConfig>
376-__aicore__ inline void Atan(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
377- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
378-{
379- AtanImpl<T, isReuseSource, config>(dstTensor, srcTensor, sharedTmpBuffer, calCount);
380-}
381- 
382- /*!
383- * \ingroup Atan
384- * \brief compute Atan elementwisely
385- * \tparam T: half/float
386- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
387- * this parameter is reserved, please use the default value.
388- * \param [out] dstTensor: output LocalTensor
389- * \param [in] srcTensor: input LocalTensor
390- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
391- */
392-template <typename T, bool isReuseSource = false, const AtanConfig& config = defaultAtanConfig>
393-__aicore__ inline void Atan(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor)
394-{
395- Atan<T, isReuseSource, config>(dstTensor, srcTensor, srcTensor.GetSize());
396-}
397- 
398- /*!
399- * \ingroup Atan
400- * \brief compute Atan elementwisely
401- * \tparam T: half/float
402- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
403- * this parameter is reserved, please use the default value.
404- * \param [out] dstTensor: output LocalTensor
405- * \param [in] srcTensor: input LocalTensor
406- * \param [in] calCount: the number of elements to be processed.
407- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
408- */
409-template <typename T, bool isReuseSource = false, const AtanConfig& config = defaultAtanConfig>
410-__aicore__ inline void Atan(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
411- const uint32_t calCount)
412-{
413- AtanImpl<T, isReuseSource, config>(dstTensor, srcTensor, calCount);
414-}
415-} // namespace AscendC
416-#endif // IMPL_MATH_ATAN_ATAN_l300_IMPL_H
Dimpl/adv_api/detail/math/axpy/axpy_l300_impl.h+0-120
@@ -1,120 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/*!
12- * \file axpy_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_MATH_AXPY_AXPY_L300_IMPL_H
16-#define IMPL_MATH_AXPY_AXPY_L300_IMPL_H
17-#include "kernel_tensor.h"
18-#include "../../common/check.h"
19-#ifdef ASCENDC_CPU_DEBUG
20-#include "../../api_check/kernel_check/math/axpy/axpy_check.h"
21-#endif // ASCENDC_CPU_DEBUG
22-#include "../../api_check/kernel_api_check.h"
23- 
24-namespace AscendC {
25-namespace AxpyAPI {
26-constexpr MicroAPI::CastTrait castTraitF162F32 = {
27- MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::UNKNOWN, MicroAPI::MaskMergeMode::ZEROING, RoundMode::UNKNOWN};
28-template<typename T, typename U>
29-__simd_vf__ inline void AxpyCompute(__ubuf__ T* dst, __ubuf__ U* src, U scalarValue, uint32_t calCount,
30- uint16_t repeatTimes, uint16_t oneRepSize, uint32_t mainBlockCount,
31- uint32_t tailCount, uint16_t offset, uint16_t singleMainBlockCtrl)
32-{
33- MicroAPI::MaskReg mask, maskTail;
34- MicroAPI::RegTensor<T> dstVreg;
35- MicroAPI::RegTensor<U> srcVreg;
36- mask = MicroAPI::UpdateMask<T>(mainBlockCount);
37- maskTail = MicroAPI::UpdateMask<T>(tailCount);
38- if constexpr (IsSameType<U, half>::value && IsSameType<T, float>::value) {
39- MicroAPI::RegTensor<float> tempSrcVreg;
40- for (uint16_t i = 0; i < repeatTimes; ++i) {
41- MicroAPI::DataCopy<U, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcVreg, src + i * oneRepSize);
42- MicroAPI::Cast<float, U, castTraitF162F32>(tempSrcVreg, srcVreg, mask);
43- MicroAPI::DataCopy(dstVreg, dst + i * oneRepSize);
44- MicroAPI::Axpy(dstVreg, tempSrcVreg, scalarValue, mask);
45- MicroAPI::DataCopy(dst + i * oneRepSize, dstVreg, mask);
46- // unroll
47- MicroAPI::DataCopy<U, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcVreg, src + i * oneRepSize + offset);
48- MicroAPI::Cast<float, U, castTraitF162F32>(tempSrcVreg, srcVreg, mask);
49- MicroAPI::DataCopy(dstVreg, dst + i * oneRepSize + offset);
50- MicroAPI::Axpy(dstVreg, tempSrcVreg, scalarValue, mask);
51- MicroAPI::DataCopy(dst + i * oneRepSize + offset, dstVreg, mask);
52- }
53- for (uint16_t j = 0; j < singleMainBlockCtrl; ++j) {
54- MicroAPI::DataCopy<U, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcVreg, src + repeatTimes * oneRepSize * 2);
55- MicroAPI::Cast<float, U, castTraitF162F32>(tempSrcVreg, srcVreg, mask);
56- MicroAPI::DataCopy(dstVreg, dst + repeatTimes * oneRepSize * 2);
57- MicroAPI::Axpy(dstVreg, tempSrcVreg, scalarValue, mask);
58- MicroAPI::DataCopy(dst + repeatTimes * oneRepSize * 2, dstVreg, mask);
59- }
60- MicroAPI::DataCopy<U, MicroAPI::LoadDist::DIST_UNPACK_B16>(
61- srcVreg, src + repeatTimes * oneRepSize * 2 + singleMainBlockCtrl * oneRepSize);
62- MicroAPI::Cast<float, U, castTraitF162F32>(tempSrcVreg, srcVreg, maskTail);
63- MicroAPI::DataCopy(dstVreg, dst + repeatTimes * oneRepSize * 2 + singleMainBlockCtrl * oneRepSize);
64- MicroAPI::Axpy(dstVreg, tempSrcVreg, scalarValue, maskTail);
65- MicroAPI::DataCopy(dst + repeatTimes * oneRepSize * 2 + singleMainBlockCtrl * oneRepSize, dstVreg, maskTail);
66- } else {
67- for (uint16_t i = 0; i < repeatTimes; ++i) {
68- MicroAPI::DataCopy(srcVreg, src + i * oneRepSize);
69- MicroAPI::DataCopy(dstVreg, dst + i * oneRepSize);
70- MicroAPI::Axpy(dstVreg, srcVreg, scalarValue, mask);
71- MicroAPI::DataCopy(dst + i * oneRepSize, dstVreg, mask);
72- // unroll
73- MicroAPI::DataCopy(srcVreg, src + i * oneRepSize + offset);
74- MicroAPI::DataCopy(dstVreg, dst + i * oneRepSize + offset);
75- MicroAPI::Axpy(dstVreg, srcVreg, scalarValue, mask);
76- MicroAPI::DataCopy(dst + i * oneRepSize + offset, dstVreg, mask);
77- }
78- for (uint16_t j = 0; j < singleMainBlockCtrl; ++j) {
79- MicroAPI::DataCopy(srcVreg, src + repeatTimes * oneRepSize * 2);
80- MicroAPI::DataCopy(dstVreg, dst + repeatTimes * oneRepSize * 2);
81- MicroAPI::Axpy(dstVreg, srcVreg, scalarValue, mask);
82- MicroAPI::DataCopy(dst + repeatTimes * oneRepSize * 2, dstVreg, mask);
83- }
84- MicroAPI::DataCopy(srcVreg, src + repeatTimes * oneRepSize * 2 + singleMainBlockCtrl * oneRepSize);
85- MicroAPI::DataCopy(dstVreg, dst + repeatTimes * oneRepSize * 2 + singleMainBlockCtrl * oneRepSize);
86- MicroAPI::Axpy(dstVreg, srcVreg, scalarValue, maskTail);
87- MicroAPI::DataCopy(dst + repeatTimes * oneRepSize * 2 + singleMainBlockCtrl * oneRepSize, dstVreg, maskTail);
88- }
89-}
90-}//namespace AxpyAPI
91-template <typename T, typename U, bool isReuseSource>
92-__aicore__ inline void AxpyImpl(const LocalTensor<T> &dstLocal, const LocalTensor<U> &srcLocal, const U scalarValue,
93- const LocalTensor<uint8_t> &sharedTmpBuffer, const uint32_t calCount)
94-{
95- CHECK_FUNC_HIGHLEVEL_API(Axpy, (T, U, isReuseSource), (dstLocal, srcLocal, scalarValue, sharedTmpBuffer, calCount));
96- CheckTensorPosition(sharedTmpBuffer, "sharedTmpBuffer", "VECIN, VECOUT, VECCALC");
97- CheckTensorPosition(dstLocal, "dstLocal", "VECIN, VECOUT, VECCALC");
98- CheckTensorPosition(srcLocal, "srcLocal", "VECIN, VECOUT, VECCALC");
99- CheckCalCount(calCount, "calCount", dstLocal, "dstLocal", "Axpy");
100- CheckCalCount(calCount, "calCount", srcLocal, "srcLocal", "Axpy");
101- static_assert(SupportType<T, half, float>(), "Axpy current dst data type is not supported on current device!");
102- static_assert(SupportType<U, half, float>(), "Axpy current src data type is not supported on current device!");
103- __ubuf__ T *dst = (__ubuf__ T *)dstLocal.GetPhyAddr();
104- __ubuf__ U *src = (__ubuf__ U *)srcLocal.GetPhyAddr();
105- constexpr uint16_t oneRepSize = GetVecLen() / sizeof(T);
106- const uint32_t mainBlockCount = oneRepSize;
107- uint32_t tailCount = calCount % oneRepSize;
108- uint16_t repeatTimes = calCount / oneRepSize;
109- if (tailCount == 0 && repeatTimes > 0) {
110- repeatTimes--;
111- tailCount += oneRepSize;
112- }
113- uint16_t repeatTimesUnRoll = repeatTimes / 2;
114- uint16_t singleMainBlockCtrl = repeatTimes % 2;
115- uint16_t offset = repeatTimesUnRoll * oneRepSize;
116- AxpyAPI::AxpyCompute<T, U>(dst, src, scalarValue, calCount, repeatTimesUnRoll, oneRepSize, mainBlockCount,
117- tailCount, offset, singleMainBlockCtrl);
118-}
119-} // namespace AscendC
120-#endif // IMPL_MATH_AXPY_AXPY_L300_IMPL_H
Dimpl/adv_api/detail/math/ceil/ceil_l300_impl.h+0-72
@@ -1,72 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/*!
12- * \file ceil_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_MATH_CEIL_CEIL_L300_IMPL_H
16-#define IMPL_MATH_CEIL_CEIL_L300_IMPL_H
17-#include "kernel_tensor.h"
18-#include "../../common/check.h"
19-namespace AscendC {
20-template <typename T, bool isReuseSource = false>
21-__simd_vf__ inline void CeilCompute(__ubuf__ T* dstUb, __ubuf__ T* srcUb, uint32_t calCount,
22- const uint16_t repeatTimes)
23-{
24- constexpr uint32_t repeatElm = GetVecLen() / sizeof(T);
25- MicroAPI::RegTensor<T> srcReg;
26- MicroAPI::RegTensor<T> dstReg;
27- MicroAPI::MaskReg maskReg;
28- for (uint16_t i = 0; i < repeatTimes; ++i) {
29- maskReg = MicroAPI::UpdateMask<T>(calCount);
30- MicroAPI::DataCopy<T>(srcReg, srcUb + i * repeatElm);
31- MicroAPI::Truncate<T, RoundMode::CAST_CEIL>(dstReg, srcReg, maskReg);
32- MicroAPI::DataCopy<T>(dstUb + i * repeatElm, dstReg, maskReg);
33- }
34-}
35- 
36-template <typename T, bool isReuseSource = false>
37-__aicore__ inline void CeilImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
38- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
39-{
40- if ASCEND_IS_AIC {
41- return;
42- }
43- 
44- CheckTensorPosition(sharedTmpBuffer, "sharedTmpBuffer", "VECIN, VECOUT, VECCALC");
45- 
46- CeilImpl<T, isReuseSource>(dstTensor, srcTensor, calCount);
47-}
48- 
49-template <typename T, bool isReuseSource = false>
50-__aicore__ inline void CeilImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
51- const uint32_t calCount)
52-{
53- if ASCEND_IS_AIC {
54- return;
55- }
56- 
57- static_assert(SupportType<T, float, half>(),
58- "Ceil only support half/float data type on current device");
59- CheckTensorPosition(dstTensor, "dstTensor", "VECIN, VECOUT, VECCALC");
60- CheckTensorPosition(srcTensor, "srcTensor", "VECIN, VECOUT, VECCALC");
61- CheckCalCount(calCount, "calCount", srcTensor, "srcTensor", "Round");
62- CheckCalCount(calCount, "calCount", dstTensor, "dstTensor", "Round");
63- 
64- __ubuf__ T *srcUb = (__ubuf__ T *)srcTensor.GetPhyAddr();
65- __ubuf__ T *dstUb = (__ubuf__ T *)dstTensor.GetPhyAddr();
66- constexpr uint32_t repeatElm = GetVecLen() / sizeof(T);
67- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(calCount, repeatElm));
68- 
69- CeilCompute<T, isReuseSource>(dstUb, srcUb, calCount, repeatTimes);
70-}
71-} // namespace AscendC
72-#endif // IMPL_MATH_CEIL_CEIL_L300_IMPL_H
Dimpl/adv_api/detail/math/cos/cos_l300_impl.h+0-306
@@ -1,306 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/*!
12- * \file cos_L300_impl.h
13- * \brief
14- */
15-#ifndef LIB_MATH_COS_L300_IMPL_H
16-#define LIB_MATH_COS_L300_IMPL_H
17- 
18-#include "kernel_tensor.h"
19-#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3003 || __NPU_ARCH__ == 3113)
20-#include "cos_common_utils.h"
21-#endif
22-namespace AscendC {
23-namespace Internal {
24-// define the number of x div pi
25-constexpr float COS_PI_FOR_X_TODIV = 0.3183098733425140380859375;
26-// define the PI for compute
27-constexpr float PI_0 = 3.140625;
28-constexpr float COS_KPI_FIRS_PI_MULS = 0.0009670257568359375;
29-constexpr float COS_KPI_TWI_PI_MULS = 6.2771141529083251953125e-7;
30-constexpr float COS_KPI_THIR_PI_MULS = 1.21644916362129151821136474609375e-10;
31-constexpr float COS_KPI_FOR_PI_MULS = -1.0290623200529979163359041220560e-13;
32-// define the number of down of pi_div
33-constexpr float COS_PI_DOWN = 1.57079637050628662109375;
34-// kpi_2
35-constexpr float COS_PI_RESDOWN_ADDS_NEG = -0.00000004371139000189375;
36-// define the number of cos compute
37-constexpr float COS_RES_MULIT_SCA = 2.604926501e-6;
38-constexpr float COS_RES_ADDICT_UP = -0.0001980894471;
39-constexpr float COS_2ADDS = 0.008333049340;
40-constexpr float COS_3ADDS = -0.1666665792;
41-constexpr float COS_POINT_FIVE = 0.5;
42-constexpr float COS_M4_SCA = 4.0;
43-constexpr float COS_K2_SCA = -2.0;
44-constexpr float SCALAR_ONE = 1.0;
45- 
46-__simd_callee__ inline void CosPolynomialApproximation(MicroAPI::RegTensor<float>& dstReg,
47- MicroAPI::RegTensor<float>& srcReg, MicroAPI::RegTensor<float>& x, MicroAPI::RegTensor<float>& round,
48- MicroAPI::RegTensor<float>& kpi, MicroAPI::MaskReg mask)
49-{
50- // k = round(x * invpi + 1/2)
51- MicroAPI::Muls(round, srcReg, COS_PI_FOR_X_TODIV, mask);
52- MicroAPI::Adds(round, round, COS_POINT_FIVE, mask);
53- // tie to even
54- MicroAPI::Truncate<float, RoundMode::CAST_RINT, MicroAPI::MaskMergeMode::ZEROING>(round, round, mask);
55- // x -= k * pi_0
56- MicroAPI::Muls(kpi, round, PI_0, mask);
57- MicroAPI::Sub(x, srcReg, kpi, mask);
58- 
59- // x -= k * pi_1
60- MicroAPI::Muls(kpi, round, COS_KPI_FIRS_PI_MULS, mask);
61- MicroAPI::Sub(x, x, kpi, mask);
62- 
63- // x = x + COS_PI_DOWN
64- MicroAPI::Adds(x, x, COS_PI_DOWN, mask);
65- 
66- // x -= k * pi_2
67- MicroAPI::Muls(kpi, round, COS_KPI_TWI_PI_MULS, mask);
68- MicroAPI::Sub(x, x, kpi, mask);
69- 
70- // x -= k * pi_3
71- MicroAPI::Muls(kpi, round, COS_KPI_THIR_PI_MULS, mask);
72- MicroAPI::Sub(x, x, kpi, mask);
73- 
74- // x -= k * pi_4
75- MicroAPI::Muls(kpi, round, COS_KPI_FOR_PI_MULS, mask);
76- MicroAPI::Sub(x, x, kpi, mask);
77- 
78- // x = x + COS_PI_RESDOWN_ADDS_NEG
79- MicroAPI::Adds(x, x, COS_PI_RESDOWN_ADDS_NEG, mask);
80- 
81- // x^2 = mul(input_x, input_x)
82- MicroAPI::Mul(kpi, x, x, mask);
83- // kover2
84- MicroAPI::Muls(dstReg, round, COS_POINT_FIVE, mask);
85- MicroAPI::Truncate<float, RoundMode::CAST_FLOOR, MicroAPI::MaskMergeMode::ZEROING>(dstReg, dstReg, mask);
86- 
87- // kover2floorm4
88- MicroAPI::Muls(dstReg, dstReg, COS_M4_SCA, mask);
89- // k2
90- MicroAPI::Muls(round, round, COS_K2_SCA, mask);
91- // sign
92- MicroAPI::Add(dstReg, dstReg, round, mask);
93- MicroAPI::Adds(dstReg, dstReg, SCALAR_ONE, mask);
94- 
95- // res_up = mul(x^2, 2.604926501e-6)
96- MicroAPI::Muls(round, kpi, COS_RES_MULIT_SCA, mask);
97- MicroAPI::Adds(round, round, COS_RES_ADDICT_UP, mask);
98- // res_up = mul(res_up, x^2)
99- MicroAPI::Mul(round, round, kpi, mask);
100- MicroAPI::Adds(round, round, COS_2ADDS, mask);
101- // res_up = mul(res_up, x^2)
102- MicroAPI::Mul(round, round, kpi, mask);
103- MicroAPI::Adds(round, round, COS_3ADDS, mask);
104- // res_up = mul(res_up, x^2)
105- MicroAPI::Mul(round, round, kpi, mask);
106- MicroAPI::Adds(round, round, SCALAR_ONE, mask);
107- // sin(x) = xP(x)
108- MicroAPI::Mul(round, round, x, mask);
109- MicroAPI::Mul(dstReg, round, dstReg, mask);
110- MicroAPI::Mins(dstReg, dstReg, SCALAR_ONE, mask);
111- MicroAPI::Maxs(dstReg, dstReg, -SCALAR_ONE, mask);
112-}
113- 
114-template <typename T>
115-__simd_vf__ inline void CosPolymonial(__ubuf__ T* dst, __ubuf__ T* src, uint32_t calCount, uint16_t repeat)
116-{
117- MicroAPI::RegTensor<T> x;
118- MicroAPI::RegTensor<float> xTmp;
119- MicroAPI::RegTensor<float> round;
120- MicroAPI::RegTensor<float> kpi;
121- MicroAPI::RegTensor<T> srcReg;
122- MicroAPI::RegTensor<float> srcTmp;
123- MicroAPI::RegTensor<T> dstReg;
124- MicroAPI::RegTensor<float> dstTmp;
125- constexpr uint32_t oneRepSize = GetVecLen() / sizeof(float);
126- MicroAPI::MaskReg mask;
127- MicroAPI::MaskReg maskAll = MicroAPI::CreateMask<uint8_t>();
128- for (uint16_t i = 0; i < repeat; i++) {
129- mask = MicroAPI::UpdateMask<float>(calCount);
130- if constexpr (std::is_same<T, half>::value) {
131- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcReg, src + i * oneRepSize);
132- MicroAPI::Cast<float, half, castTraitB16ToB32>(srcTmp, srcReg, mask);
133- CosPolynomialApproximation(dstTmp, srcTmp, xTmp, round, kpi, mask);
134- MicroAPI::Cast<half, float, castTraitB32ToB16>(dstReg, dstTmp, mask);
135- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(dst + i * oneRepSize, dstReg, mask);
136- } else {
137- MicroAPI::DataCopy(srcReg, src + i * oneRepSize);
138- CosPolynomialApproximation(dstReg, srcReg, xTmp, round, kpi, mask);
139- MicroAPI::DataCopy(dst + i * oneRepSize, dstReg, mask);
140- }
141- }
142-}
143-} // namespace Internal
144- 
145-template <typename T>
146-__aicore__ inline void CosPolynomialImpl(__ubuf__ T *dst, __ubuf__ T *src, uint32_t calCount)
147-{
148- constexpr uint32_t oneRepSize = GetVecLen() / sizeof(float);
149- uint16_t repeat = CeilDivision(calCount, oneRepSize);
150- Internal::CosPolymonial<T>(dst, src, calCount, repeat);
151-}
152- 
153-__aicore__ inline constexpr uint32_t GetCosTmpBufferLiveNode() {
154- constexpr uint32_t tmpBufferLiveNode = sizeof(float) * 2;
155- return tmpBufferLiveNode;
156-}
157- 
158-template<typename T>
159-__aicore__ inline uint32_t GetCosTmpBufferSize(const LocalTensor<uint8_t>& sharedTmpBuffer) {
160- uint32_t sharedTmpBufferSize = sharedTmpBuffer.GetSize() / GetCosTmpBufferLiveNode();
161- return AlignUp(sharedTmpBufferSize, GetDataBlockSizeInBytes()) / sizeof(T);
162-}
163- 
164-template<typename T, bool isReuseSource = false, const CosConfig &config = defaultCosConfig>
165-__aicore__ inline void CosImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
166- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
167-{
168- // Only for AI Vector Core.
169- if ASCEND_IS_AIC {
170- return;
171- }
172- 
173- static_assert(SupportType<T, half, float>(), "current data type is not supported on current device!");
174- CheckTensorPos<T>(dstTensor, Hardware::UB, "dstTensor", "VECIN / VECCALC / VECOUT", "Cos");
175- CheckTensorPos<T>(srcTensor, Hardware::UB, "srcTensor", "VECIN / VECCALC / VECOUT", "Cos");
176- CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN / VECCALC / VECOUT", "Cos");
177- ASCENDC_ASSERT((calCount <= srcTensor.GetSize()), {
178- KERNEL_LOG(KERNEL_ERROR, "calCount is %u, which should not be larger than srcTensor length %u", calCount,
179- srcTensor.GetSize());
180- });
181- ASCENDC_ASSERT((calCount <= dstTensor.GetSize()), {
182- KERNEL_LOG(KERNEL_ERROR, "calCount is %u, which should not be larger than dstTensor length %u", calCount,
183- dstTensor.GetSize());
184- });
185- 
186- if constexpr(config.algo == CosAlgo::POLYNOMIAL_APPROXIMATION) {
187- CosPolynomialImpl((__ubuf__ T*)dstTensor.GetPhyAddr(), (__ubuf__ T*)srcTensor.GetPhyAddr(), calCount);
188- } else if constexpr (config.algo == CosAlgo::RADIAN_REDUCTION) {
189- uint32_t sharedTmpBufferSize = GetCosTmpBufferSize<T>(sharedTmpBuffer);
190- uint32_t count = calCount;
191- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(calCount, sharedTmpBufferSize));
192- for (uint16_t i = 0; i < repeatTimes; i++) {
193- uint32_t remainCount = count - sharedTmpBufferSize * i;
194- uint32_t oneRepSize = remainCount < sharedTmpBufferSize ? remainCount : sharedTmpBufferSize;
195- CosRadianReductionImpl((__ubuf__ T*)dstTensor.GetPhyAddr() + i * sharedTmpBufferSize,
196- (__ubuf__ T*)srcTensor.GetPhyAddr() + i * sharedTmpBufferSize,
197- (__ubuf__ uint32_t*)sharedTmpBuffer.GetPhyAddr(), oneRepSize);
198- }
199- }
200-}
201- 
202-template <typename T, bool isReuseSource = false, const CosConfig &config = defaultCosConfig>
203-__aicore__ inline void CosImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
204- const uint32_t calCount)
205-{
206- // Only for AI Vector Core.
207- if ASCEND_IS_AIC {
208- return;
209- }
210- 
211- // Using the Stack Space to Allocate tmpBuffer
212- LocalTensor<uint8_t> sharedTmpBuffer;
213- bool ans = PopStackBuffer<uint8_t, TPosition::LCM>(sharedTmpBuffer);
214- ASCENDC_ASSERT((ans), { KERNEL_LOG(KERNEL_ERROR, "PopStackBuffer Error!"); });
215- CosImpl<T, isReuseSource, config>(dstTensor, srcTensor, sharedTmpBuffer, calCount);
216-}
217- 
218-__aicore__ inline void CosCastFullMask(
219- const LocalTensor<float>& dstTensor, const LocalTensor<float>& srcTensor, RoundMode castType)
220-{
221- uint64_t newMask = 64;
222- Cast<float, float, false>(dstTensor, srcTensor, castType, newMask, 1,
223- { 1, 1, DEFAULT_REPEAT_STRIDE, DEFAULT_REPEAT_STRIDE });
224- PipeBarrier<PIPE_V>();
225-}
226-/*!
227- * \ingroup Cos
228- * \brief compute Cos elementwisely
229- * \tparam T: half/float
230- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason, only support
231- * float input data type
232- * \param [out] dstTensor: output LocalTensor
233- * \param [in] srcTensor: input LocalTensor
234- * \param [in] sharedTmpBuffer: extra temporary shared space used for intermediate values among calculation process,
235- * whose required space size should refer to corresponding tiling API, which is defined at cos_tiling.h.
236- * Generally, the more space you allocate, the better performance you will achieve, and the performance
237- * reaches peak when buffer size is maximum(calculated by tiling function). Moreover, it is not guaranteed
238- * that the shared space will be cleared after usage, the data could be anything.
239- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
240- * Input data valid range should be [-65504, 65504]
241- */
242-template <typename T, bool isReuseSource = false, const CosConfig &config>
243-__aicore__ inline void Cos(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
244- const LocalTensor<uint8_t>& sharedTmpBuffer)
245-{
246- Cos<T, isReuseSource, config>(dstTensor, srcTensor, sharedTmpBuffer, srcTensor.GetSize());
247-}
248- 
249-/*!
250- * \ingroup Cos
251- * \brief compute Cos elementwisely
252- * \tparam T: half/float
253- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason
254- * \param [out] dstTensor: output LocalTensor
255- * \param [in] srcTensor: input LocalTensor
256- * \param [in] sharedTmpBuffer: extra temporary shared space used for intermediate values among calculation process,
257- * whose required space size should refer to corresponding tiling API, which is defined at cos_tiling.h.
258- * Generally, the more space you allocate, the better performance you will achieve, and the performance
259- * reaches peak when buffer size is maximum(calculated by tiling function). Moreover, it is not guaranteed
260- * that the shared space will be cleared after usage, the data could be anything.
261- * \param [in] calCount: the number of elements to be processed.
262- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
263- * Input data valid range should be [-65504, 65504]
264- */
265-template<typename T, bool isReuseSource = false, const CosConfig &config>
266-__aicore__ inline void Cos(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
267- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
268-{
269- CosImpl<T, isReuseSource, config>(dstTensor, srcTensor, sharedTmpBuffer, calCount);
270-}
271- 
272-/*!
273- * \ingroup Cos
274- * \brief compute Cos elementwisely
275- * \tparam T: half/float
276- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason
277- * \param [out] dstTensor: output LocalTensor
278- * \param [in] srcTensor: input LocalTensor
279- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
280- * Input data valid range should be [-65504, 65504]
281- */
282-template <typename T, bool isReuseSource = false, const CosConfig &config>
283-__aicore__ inline void Cos(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor)
284-{
285- Cos<T, isReuseSource, config>(dstTensor, srcTensor, srcTensor.GetSize());
286-}
287- 
288-/*!
289- * \ingroup Cos
290- * \brief compute Cos elementwisely
291- * \tparam T: half/float
292- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason
293- * \param [out] dstTensor: output LocalTensor
294- * \param [in] srcTensor: input LocalTensor
295- * \param [in] calCount: the number of elements to be processed.
296- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
297- * Input data valid range should be [-65504, 65504]
298- */
299-template<typename T, bool isReuseSource = false, const CosConfig &config>
300-__aicore__ inline void Cos(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
301- const uint32_t calCount)
302-{
303- CosImpl<T, isReuseSource, config>(dstTensor, srcTensor, calCount);
304-}
305-} // namespace AscendC
306-#endif // LIB_MATH_COS_L300_IMPL_H
Dimpl/adv_api/detail/math/cosh/cosh_l300_impl.h+0-98
@@ -1,98 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/*!
12- * \file cosh_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_MATH_COSH_COSH_L300_IMPL_H
16-#define IMPL_MATH_COSH_COSH_L300_IMPL_H
17- 
18-#include "kernel_tensor.h"
19-#include "../../common/check.h"
20- 
21-namespace AscendC {
22-namespace Internal {
23-// Computes cosh values based on input types.
24-// According formula: cosh(x) = (e^x + e^(-x))/2 = e^(x-ln2) + 0.25/(e^(x-ln2)).
25-template <typename T>
26-__simd_vf__ inline void CoshCompute(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint32_t calCount, uint16_t repeatTimes)
27-{
28- constexpr float scalarNegLnTwo = -0.6931472;
29- constexpr float scalarBrc = 0.25;
30- constexpr uint32_t vlSize = static_cast<uint32_t>(GetVecLen() / sizeof(float));
31- MicroAPI::MaskReg coshMask;
32- MicroAPI::RegTensor<float> brcReg;
33- MicroAPI::RegTensor<T> srcReg;
34- MicroAPI::RegTensor<float> castReg;
35- MicroAPI::RegTensor<float> computeReg0;
36- MicroAPI::RegTensor<float> computeReg1;
37- MicroAPI::RegTensor<float> resReg;
38- MicroAPI::RegTensor<T> dstReg;
39- MicroAPI::Duplicate(brcReg, scalarBrc);
40- for (uint16_t i = 0; i < repeatTimes; ++i) {
41- coshMask = MicroAPI::UpdateMask<float>(calCount);
42- if constexpr (SupportBytes<T, 2>()) {
43- MicroAPI::DataCopy<half, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcReg, srcUb + i * vlSize);
44- MicroAPI::Cast<float, half, castTraitB16ToB32>(castReg, srcReg, coshMask);
45- } else {
46- MicroAPI::DataCopy(castReg, srcUb + i * vlSize);
47- }
48- MicroAPI::Adds(castReg, castReg, scalarNegLnTwo, coshMask);
49- MicroAPI::Exp(computeReg0, castReg, coshMask);
50- MicroAPI::Div(computeReg1, brcReg, computeReg0, coshMask);
51- MicroAPI::Add(resReg, computeReg0, computeReg1, coshMask);
52- if constexpr (SupportBytes<T, 2>()) {
53- MicroAPI::Cast<half, float, castTraitB32ToB16>(dstReg, resReg, coshMask);
54- MicroAPI::DataCopy<half, MicroAPI::StoreDist::DIST_PACK_B32>(dstUb + i * vlSize, dstReg, coshMask);
55- } else {
56- MicroAPI::DataCopy(dstUb + i * vlSize, resReg, coshMask);
57- }
58- }
59-}
60-} // namespace Internal
61- 
62-template <typename T, bool isReuseSource = false>
63-__aicore__ inline void CoshImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
64- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
65-{
66- // Only for AI Vector Core.
67- if ASCEND_IS_AIC {
68- return;
69- }
70- 
71- CheckTensorPosition(sharedTmpBuffer, "sharedTmpBuffer", "VECIN, VECOUT, VECCALC");
72- CoshImpl<T, isReuseSource>(dstTensor, srcTensor, calCount);
73-}
74- 
75-template <typename T, bool isReuseSource = false>
76-__aicore__ inline void CoshImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
77- const uint32_t calCount)
78-{
79- // Only for AI Vector Core.
80- if ASCEND_IS_AIC {
81- return;
82- }
83- 
84- static_assert(SupportType<T, half, float>(), "Cosh only support half/float data type on current device!");
85- CheckTensorPosition(dstTensor, "dstTensor", "VECIN, VECOUT, VECCALC");
86- CheckTensorPosition(srcTensor, "srcTensor", "VECIN, VECOUT, VECCALC");
87- CheckCalCount(calCount, "calCount", srcTensor, "srcTensor", "Cosh");
88- CheckCalCount(calCount, "calCount", dstTensor, "dstTensor", "Cosh");
89- 
90- __ubuf__ T *dstUb = (__ubuf__ T *)dstTensor.GetPhyAddr();
91- __ubuf__ T *srcUb = (__ubuf__ T *)srcTensor.GetPhyAddr();
92- constexpr int32_t vlSize = static_cast<int32_t>(GetVecLen() / sizeof(float));
93- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(calCount, vlSize));
94- Internal::CoshCompute<T>(dstUb, srcUb, calCount, repeatTimes);
95-}
96-} // namespace AscendC
97- 
98-#endif // IMPL_MATH_COSH_COSH_C310_IMPL_H
Dimpl/adv_api/detail/math/cumsum/regbase/l300/cumsum_l300_impl.h+0-200
@@ -1,200 +0,0 @@
1-/**
2- * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3- * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4- * CANN Open Software License Agreement Version 2.0 (the "License").
5- * Please refer to the License for details. You may not use this file except in compliance with the License.
6- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7- * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8- * See LICENSE in the root of the software repository for the full text of the License.
9- */
10- 
11-/* !
12- * \file cumsum_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_MATH_CUMSUM_CUMSUM_IMPL_L300_H
16-#define IMPL_MATH_CUMSUM_CUMSUM_IMPL_L300_H
17- 
18-#include "kernel_tensor.h"
19-#include "include/adv_api/math/cumsum_utils.h"
20-#include "cumsum_l300_utils.h"
21- 
22-#ifdef ASCENDC_CPU_DEBUG
23-#include "../../../../api_check/kernel_check/math/cumsum/cumsum_check.h"
24-#endif // ASCENDC_CPU_DEBUG
25-#include "../../../../api_check/kernel_api_check.h"
26- 
27-namespace AscendC {
28-template <typename T>
29-__simd_vf__ inline void CumSumCopyLastRowVF(__ubuf__ T* dst, __ubuf__ T* src, uint32_t len)
30-{
31- constexpr uint16_t sregLower = static_cast<uint16_t>(GetVecLen() / sizeof(T));
32- uint32_t count = len;
33- uint16_t repeatTimes = CeilDivision(count, sregLower);
34- 
35- MicroAPI::RegTensor<T> srcReg;
36- MicroAPI::MaskReg preg;
37- for (uint16_t i = 0; i < repeatTimes; i++) {
38- preg = MicroAPI::UpdateMask<T>(count);
39- MicroAPI::DataCopy(srcReg, src + i * sregLower);
40- MicroAPI::DataCopy(dst + i * sregLower, srcReg, preg);
41- }
42-}
43- 
44-template <typename T>
45-__aicore__ inline void CumSumCopyLastRow(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor, uint32_t len)
46-{
47- __ubuf__ T* src = (__ubuf__ T*)srcTensor.GetPhyAddr();
48- __ubuf__ T* dst = (__ubuf__ T*)dstTensor.GetPhyAddr();
49- CumSumCopyLastRowVF<T>(dst, src, len);
50-}
51- 
52-template <typename T, const CumSumConfig& config>
53-__aicore__ inline void CumSumLastDim(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
54- LocalTensor<T> tempBuffer, const CumSumInfo& cumSumInfo)
55-{
56- ASCENDC_ASSERT(false, { KERNEL_LOG(KERNEL_ERROR, "current data type is not supported!"); });
57-}
58- 
59-template <typename T = float, const CumSumConfig& config>
60-__aicore__ inline void CumSumLastDim(const LocalTensor<float>& dstTensor, const LocalTensor<float>& srcTensor,
61- LocalTensor<float> tempBuffer, const CumSumInfo& cumSumInfo)
62-{
63- uint16_t alignOutter =
64- (cumSumInfo.outter + NCHW_CONV_ADDR_LIST_SIZE - 1) / NCHW_CONV_ADDR_LIST_SIZE * NCHW_CONV_ADDR_LIST_SIZE;
65- LocalTensor<float> tempBuffer2 = tempBuffer[alignOutter * cumSumInfo.inner];
66- Internal::Transpose5HDAB(tempBuffer, srcTensor, cumSumInfo);
67- if constexpr (config.algorithm == CumSumAlgorithm::CUMSUM_ALGORITHM_SKLANSKY) {
68- Internal::CumSumFirstDimSklansky(tempBuffer, cumSumInfo.inner, alignOutter);
69- } else {
70- Internal::CumSumFirstDimBasic(tempBuffer, cumSumInfo.inner, alignOutter);
71- }
72- Internal::Transpose5HDBA(tempBuffer2, tempBuffer, cumSumInfo);
73- Internal::CumSumCopyOut(dstTensor, tempBuffer2, cumSumInfo.outter, cumSumInfo.inner);
74-}
75- 
76-template <typename T = half, const CumSumConfig& config>
77-__aicore__ inline void CumSumLastDim(const LocalTensor<half>& dstTensor, const LocalTensor<half>& srcTensor,
78- LocalTensor<half> tempBuffer, const CumSumInfo& cumSumInfo)
79-{
80- uint16_t alignOutter =
81- (cumSumInfo.outter + NCHW_CONV_ADDR_LIST_SIZE - 1) / NCHW_CONV_ADDR_LIST_SIZE * NCHW_CONV_ADDR_LIST_SIZE;
82- LocalTensor<half> tempBuffer2 = tempBuffer[alignOutter * cumSumInfo.inner];
83- LocalTensor<float> floatTempBuffer = tempBuffer[alignOutter * cumSumInfo.inner].ReinterpretCast<float>();
84- Internal::Transpose5HDAB(tempBuffer, srcTensor, cumSumInfo);
85- Internal::CumSumCopyWithCast(floatTempBuffer, tempBuffer, cumSumInfo.inner, alignOutter);
86- if constexpr (config.algorithm == CumSumAlgorithm::CUMSUM_ALGORITHM_SKLANSKY) {
87- Internal::CumSumFirstDimSklansky(floatTempBuffer, cumSumInfo.inner, alignOutter);
88- } else {
89- Internal::CumSumFirstDimBasic(floatTempBuffer, cumSumInfo.inner, alignOutter);
90- }
91- Internal::CumSumCopyWithCast(tempBuffer, floatTempBuffer, cumSumInfo.inner, alignOutter);
92- 
93- if (cumSumInfo.outter % NCHW_CONV_ADDR_LIST_SIZE == 0 && cumSumInfo.inner % NCHW_CONV_ADDR_LIST_SIZE == 0) {
94- Internal::Transpose5HDBA(dstTensor, tempBuffer, cumSumInfo);
95- } else {
96- Internal::Transpose5HDBA(tempBuffer2, tempBuffer, cumSumInfo);
97- Internal::CumSumCopyOut(dstTensor, tempBuffer2, cumSumInfo.outter, cumSumInfo.inner);
98- }
99-}
100- 
101-template <typename T, const CumSumConfig& config>
102-__aicore__ inline void CumSumFirstDim(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
103- LocalTensor<uint8_t>& sharedTmpBuffer, const CumSumInfo& cumSumInfo)
104-{
105- ASCENDC_ASSERT(false, { KERNEL_LOG(KERNEL_ERROR, "current data type is not supported!"); });
106-}
107- 
108-template <typename T = float, const CumSumConfig& config>
109-__aicore__ inline void CumSumFirstDim(const LocalTensor<float>& dstTensor, const LocalTensor<float>& srcTensor,
110- LocalTensor<uint8_t>& sharedTmpBuffer, const CumSumInfo& cumSumInfo)
111-{
112- Internal::CumSumCopyOut(dstTensor, srcTensor, cumSumInfo.outter, cumSumInfo.inner);
113- if constexpr (config.algorithm == CumSumAlgorithm::CUMSUM_ALGORITHM_SKLANSKY) {
114- Internal::CumSumFirstDimSklansky(dstTensor, cumSumInfo.outter, cumSumInfo.inner);
115- } else {
116- // basic implementation
117- Internal::CumSumFirstDimBasic(dstTensor, cumSumInfo.outter, cumSumInfo.inner);
118- }
119-}
120- 
121-template <typename T = half, const CumSumConfig& config>
122-__aicore__ inline void CumSumFirstDim(const LocalTensor<half>& dstTensor, const LocalTensor<half>& srcTensor,
123- LocalTensor<uint8_t>& sharedTmpBuffer, const CumSumInfo& cumSumInfo)
124-{
125- const uint32_t minTmpBufferSize = cumSumInfo.outter * cumSumInfo.inner * sizeof(float);
126- const uint32_t tmpBufferSize = sharedTmpBuffer.GetSize();
127-#if ASCENDC_CPU_DEBUG
128- ASCENDC_ASSERT((tmpBufferSize >= minTmpBufferSize), {
129- KERNEL_LOG(KERNEL_ERROR,
130- "tmpBufferSize can't smaller than minTmpBufferSize, tmpBufferSize is %u, minTmpBufferSize is %u!",
131- tmpBufferSize, minTmpBufferSize);
132- });
133-#endif
134- LocalTensor<float> tmpBuffer = sharedTmpBuffer.ReinterpretCast<float>();
135- Internal::CumSumCopyWithCast(tmpBuffer, srcTensor, cumSumInfo.outter, cumSumInfo.inner);
136- if constexpr (config.algorithm == CumSumAlgorithm::CUMSUM_ALGORITHM_SKLANSKY) {
137- Internal::CumSumFirstDimSklansky(tmpBuffer, cumSumInfo.outter, cumSumInfo.inner);
138- } else {
139- // basic implementation
140- Internal::CumSumFirstDimBasic(tmpBuffer, cumSumInfo.outter, cumSumInfo.inner);
141- }
142- Internal::CumSumCopyWithCast(dstTensor, tmpBuffer, cumSumInfo.outter, cumSumInfo.inner);
143-}
144- 
145-template <typename T, const CumSumConfig& config>
146-__aicore__ inline void CumSumImpl(LocalTensor<T>& dstTensor, LocalTensor<T>& lastRowTensor,
147- const LocalTensor<T>& srcTensor, LocalTensor<uint8_t>& sharedTmpBuffer,
148- const CumSumInfo& cumSumInfo)
149-{
150- if ASCEND_IS_AIC {
151- return;
152- }
153- 
154- static_assert(SupportType<T, half, float>(), "current data type is not supported on current device!");
155- CHECK_FUNC_HIGHLEVEL_API(CumSum, (T, config), (dstTensor, lastRowTensor, srcTensor, sharedTmpBuffer, cumSumInfo));
156- 
157- if constexpr (config.isLastAxis) {
158- uint32_t minCastTempBufferSize = 0;
159- if constexpr (sizeof(T) == 2) { // 2 is for half
160- minCastTempBufferSize = cumSumInfo.inner * NCHW_CONV_ADDR_LIST_SIZE * sizeof(half);
161- }
162- const uint32_t minTmpBufferSize =
163- minCastTempBufferSize
164- + NCHW_CONV_ADDR_LIST_SIZE * cumSumInfo.inner * sizeof(T) * 2; // both transpose require a tempBuffer
165- const uint32_t tmpBufferSize = sharedTmpBuffer.GetSize();
166-#if ASCENDC_CPU_DEBUG
167- ASCENDC_ASSERT((tmpBufferSize >= minTmpBufferSize), {
168- KERNEL_LOG(
169- KERNEL_ERROR,
170- "tmpBufferSize can't smaller than minTmpBufferSize, tmpBufferSize is %u, minTmpBufferSize is %u!",
171- tmpBufferSize, minTmpBufferSize);
172- });
173-#endif
174- // loop over the outter and process at least 16 rows of data each time.
175- const uint32_t oneRepeateSize = tmpBufferSize / minTmpBufferSize * NCHW_CONV_ADDR_LIST_SIZE;
176- const uint32_t rangeM = cumSumInfo.outter / oneRepeateSize;
177- const uint32_t tailM = cumSumInfo.outter - oneRepeateSize * rangeM;
178- uint32_t dstLocalOffset = 0;
179- uint32_t srcLocalOffset = 0;
180- LocalTensor<T> tmpBuffer = sharedTmpBuffer.ReinterpretCast<T>();
181- for (uint32_t i = 0; i < rangeM; i++) {
182- CumSumLastDim<T, config>(dstTensor[dstLocalOffset], srcTensor[srcLocalOffset], tmpBuffer,
183- {oneRepeateSize, cumSumInfo.inner});
184- dstLocalOffset += cumSumInfo.inner * oneRepeateSize;
185- srcLocalOffset += cumSumInfo.inner * oneRepeateSize;
186- }
187- 
188- if (tailM != 0) {
189- CumSumLastDim<T, config>(dstTensor[dstLocalOffset], srcTensor[srcLocalOffset], tmpBuffer,
190- {tailM, cumSumInfo.inner});
191- }
192- } else {
193- CumSumFirstDim<T, config>(dstTensor, srcTensor, sharedTmpBuffer, cumSumInfo);
194- }
195- if constexpr (config.outputLastRow) {
196- CumSumCopyLastRow(lastRowTensor, dstTensor[(cumSumInfo.outter - 1) * cumSumInfo.inner], cumSumInfo.inner);
197- }
198-}
199-} // namespace AscendC
200-#endif // IMPL_MATH_CUMSUM_CUMSUM_IMPL_L300_H
Dimpl/adv_api/detail/math/cumsum/regbase/l300/cumsum_l300_utils.h+0-788
@@ -1,788 +0,0 @@
1-/**
2- * Copyright (c) 2025 Huawei Technologies Co., Ltd.
3- * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4- * CANN Open Software License Agreement Version 2.0 (the "License").
5- * Please refer to the License for details. You may not use this file except in compliance with the License.
6- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7- * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8- * See LICENSE in the root of the software repository for the full text of the License.
9- */
10- 
11-/* !
12- * \file cumsum_l300_utils.h
13- * \brief
14- */
15-#ifndef IMPL_MATH_CUMSUM_CUMSUM_L300_UTILS_H
16-#define IMPL_MATH_CUMSUM_CUMSUM_L300_UTILS_H
17- 
18-#include "kernel_tensor.h"
19- 
20-namespace AscendC {
21- 
22-namespace Internal {
23-template <typename T>
24-__simd_callee__ inline void LoadDataWithT(__ubuf__ T* src, MicroAPI::RegTensor<float>& dstReg,
25- MicroAPI::MaskReg& dstPreg, uint32_t srcOffset)
26-{
27- if constexpr (IsSameType<T, half>::value || IsSameType<T, bfloat16_t>::value) {
28- MicroAPI::RegTensor<T> srcOrigin;
29- DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcOrigin, src + srcOffset);
30- Cast<float, T, layoutZMrgZ>(dstReg, srcOrigin, dstPreg);
31- } else { // this branch: only support float
32- DataCopy(dstReg, src + srcOffset);
33- }
34-}
35- 
36-template <typename T>
37-__simd_callee__ inline void SaveDataWithT(__ubuf__ T* dst, MicroAPI::RegTensor<float>& srcReg,
38- MicroAPI::MaskReg& dstPreg, uint32_t dstOffset)
39-{
40- if constexpr (IsSameType<T, half>::value || IsSameType<T, bfloat16_t>::value) {
41- MicroAPI::RegTensor<T> regT;
42- Cast<T, float, LayoutZMrgZRndRSatNS>(regT, srcReg, dstPreg);
43- DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(dst + dstOffset, regT, dstPreg);
44- } else {
45- DataCopy(dst + dstOffset, srcReg, dstPreg);
46- }
47-}
48- 
49-// process by tempBuffer
50-// T: fp16-> U: fp32
51-// T: fp32-> U: fp16
52-template <typename U, typename T>
53-__simd_vf__ inline void CumSumCopyWithCastVF(__ubuf__ T* src, __ubuf__ U* dst, const uint16_t outter,
54- const uint16_t inner, uint16_t innerOneRepNum, uint16_t mainRepeatTime,
55- uint16_t tailRepeatTime, uint32_t tailCount, uint16_t halfOutter,
56- uint16_t tailOutter)
57-{
58- MicroAPI::RegTensor<float> srcReg;
59- MicroAPI::RegTensor<float> srcReg1;
60- MicroAPI::MaskReg fullMask = MicroAPI::CreateMask<float, MicroAPI::MaskPattern::ALL>();
61- MicroAPI::MaskReg tailMask = MicroAPI::UpdateMask<float>(tailCount);
62- for (uint16_t j = 0; j < halfOutter; ++j) {
63- for (uint16_t i = 0; i < mainRepeatTime; ++i) {
64- LoadDataWithT<T>(src, srcReg, fullMask, j * inner + i * innerOneRepNum);
65- SaveDataWithT<U>(dst, srcReg, fullMask, j * inner + i * innerOneRepNum);
66- LoadDataWithT<T>(src, srcReg1, fullMask, (j + halfOutter) * inner + i * innerOneRepNum);
67- SaveDataWithT<U>(dst, srcReg1, fullMask, (j + halfOutter) * inner + i * innerOneRepNum);
68- }
69- for (uint16_t i = 0; i < tailRepeatTime; ++i) {
70- LoadDataWithT<T>(src, srcReg, tailMask, j * inner + (i + mainRepeatTime) * innerOneRepNum);
71- SaveDataWithT<U>(dst, srcReg, tailMask, j * inner + (i + mainRepeatTime) * innerOneRepNum);
72- LoadDataWithT<T>(src, srcReg1, tailMask, (j + halfOutter) * inner + (i + mainRepeatTime) * innerOneRepNum);
73- SaveDataWithT<U>(dst, srcReg1, tailMask, (j + halfOutter) * inner + (i + mainRepeatTime) * innerOneRepNum);
74- }
75- }
76- for (uint16_t j = 0; j < tailOutter; ++j) {
77- for (uint16_t i = 0; i < mainRepeatTime; ++i) {
78- LoadDataWithT<T>(src, srcReg, fullMask, (j + halfOutter * 2) * inner + i * innerOneRepNum);
79- SaveDataWithT<U>(dst, srcReg, fullMask, (j + halfOutter * 2) * inner + i * innerOneRepNum);
80- }
81- for (uint16_t i = 0; i < tailRepeatTime; ++i) {
82- LoadDataWithT<T>(src, srcReg, tailMask,
83- (j + halfOutter * 2) * inner + (i + mainRepeatTime) * innerOneRepNum);
84- SaveDataWithT<U>(dst, srcReg, tailMask,
85- (j + halfOutter * 2) * inner + (i + mainRepeatTime) * innerOneRepNum);
86- }
87- }
88-}
89- 
90-template <typename U, typename T>
91-__simd_vf__ inline void CumSumCopyWithCast2VF(__ubuf__ T* src, __ubuf__ U* dst, const uint16_t outter,
92- const uint16_t inner, uint16_t innerOneRepNum, uint16_t mainRepeatTime,
93- uint16_t tailRepeatTime, uint32_t tailCount)
94-{
95- MicroAPI::RegTensor<float> srcReg;
96- MicroAPI::MaskReg fullMask = MicroAPI::CreateMask<float, MicroAPI::MaskPattern::ALL>();
97- MicroAPI::MaskReg tailMask = MicroAPI::UpdateMask<float>(tailCount);
98- for (uint16_t j = 0; j < outter; ++j) {
99- for (uint16_t i = 0; i < mainRepeatTime; ++i) {
100- LoadDataWithT<T>(src, srcReg, fullMask, j * inner + i * innerOneRepNum);
101- SaveDataWithT<U>(dst, srcReg, fullMask, j * inner + i * innerOneRepNum);
102- }
103- for (uint16_t i = 0; i < tailRepeatTime; ++i) {
104- LoadDataWithT<T>(src, srcReg, tailMask, j * inner + (i + mainRepeatTime) * innerOneRepNum);
105- SaveDataWithT<U>(dst, srcReg, tailMask, j * inner + (i + mainRepeatTime) * innerOneRepNum);
106- }
107- }
108-}
109- 
110-template <typename U, typename T>
111-__aicore__ inline void CumSumCopyWithCast(const LocalTensor<U>& dstTensor, const LocalTensor<T>& srcTensor,
112- const uint32_t outter, const uint32_t inner)
113-{
114- __ubuf__ T* src = (__ubuf__ T*)srcTensor.GetPhyAddr();
115- __ubuf__ U* dst = (__ubuf__ U*)dstTensor.GetPhyAddr();
116- constexpr uint16_t innerOneRepNum = (uint16_t)(GetVecLen() / sizeof(float));
117- uint16_t mainRepeatTime = inner / innerOneRepNum;
118- uint32_t tailCount = inner % innerOneRepNum;
119- uint16_t tailRepeatTime = tailCount > 0 ? 1 : 0;
120- uint16_t halfOutter = static_cast<uint16_t>(outter) / 2;
121- uint16_t tailOutter = static_cast<uint16_t>(outter) % 2;
122- if (outter > inner) {
123- CumSumCopyWithCastVF<U, T>(src, dst, static_cast<uint16_t>(outter), static_cast<uint16_t>(inner),
124- innerOneRepNum, mainRepeatTime, tailRepeatTime, tailCount, halfOutter, tailOutter);
125- } else {
126- CumSumCopyWithCast2VF<U, T>(src, dst, static_cast<uint16_t>(outter), static_cast<uint16_t>(inner),
127- innerOneRepNum, mainRepeatTime, tailRepeatTime, tailCount);
128- }
129-}
130- 
131-template <typename T>
132-__simd_vf__ inline void CumSumCopyOutWithBlockVF(__ubuf__ T* src, __ubuf__ T* dst, uint16_t outter, uint16_t inner,
133- uint16_t mainRepeatTime, uint16_t innerOneRepNum, uint32_t tailCount,
134- uint16_t tailRepeatTime)
135-{
136- MicroAPI::RegTensor<T> srcReg;
137- MicroAPI::MaskReg fullMask = MicroAPI::CreateMask<T, MicroAPI::MaskPattern::ALL>();
138- MicroAPI::MaskReg tailMask = MicroAPI::UpdateMask<T>(tailCount);
139- 
140- for (uint16_t i = 0; i < outter; ++i) {
141- for (uint16_t j = 0; j < mainRepeatTime; ++j) {
142- MicroAPI::DataCopy<T, MicroAPI::DataCopyMode::DATA_BLOCK_COPY>(srcReg, src + i * inner + j * innerOneRepNum,
143- 1, fullMask);
144- MicroAPI::DataCopy<T, MicroAPI::DataCopyMode::DATA_BLOCK_COPY>(dst + i * inner + j * innerOneRepNum, srcReg,
145- 1, fullMask);
146- }
147- for (uint16_t j = 0; j < tailRepeatTime; ++j) {
148- MicroAPI::DataCopy<T, MicroAPI::DataCopyMode::DATA_BLOCK_COPY>(
149- srcReg, src + i * inner + mainRepeatTime * innerOneRepNum, 1, tailMask);
150- MicroAPI::DataCopy<T, MicroAPI::DataCopyMode::DATA_BLOCK_COPY>(
151- dst + i * inner + mainRepeatTime * innerOneRepNum, srcReg, 1, tailMask);
152- }
153- }
154-}
155- 
156-template <typename T>
157-__simd_vf__ inline void CumSumCopyOutVF(__ubuf__ T* src, __ubuf__ T* dst, uint16_t outter, uint16_t inner,
158- uint16_t mainRepeatTime, uint16_t innerOneRepNum, uint32_t tailCount,
159- uint16_t tailRepeatTime, uint16_t halfOutter, uint16_t tailOutter)
160-{
161- MicroAPI::RegTensor<T> srcReg;
162- MicroAPI::RegTensor<T> srcReg1;
163- MicroAPI::MaskReg fullMask = MicroAPI::CreateMask<T, MicroAPI::MaskPattern::ALL>();
164- MicroAPI::MaskReg tailMask = MicroAPI::UpdateMask<T>(tailCount);
165- 
166- for (uint16_t j = 0; j < halfOutter; ++j) {
167- for (uint16_t i = 0; i < mainRepeatTime; ++i) {
168- MicroAPI::DataCopy(srcReg, src + (j * 2) * inner + i * innerOneRepNum);
169- MicroAPI::DataCopy(dst + (j * 2) * inner + i * innerOneRepNum, srcReg, fullMask);
170- MicroAPI::DataCopy(srcReg1, src + (j * 2 + 1) * inner + i * innerOneRepNum);
171- MicroAPI::DataCopy(dst + (j * 2 + 1) * inner + i * innerOneRepNum, srcReg1, fullMask);
172- }
173- for (uint16_t i = 0; i < tailRepeatTime; ++i) {
174- MicroAPI::DataCopy(srcReg, src + (j * 2) * inner + (i + mainRepeatTime) * innerOneRepNum);
175- MicroAPI::DataCopy(dst + (j * 2) * inner + (i + mainRepeatTime) * innerOneRepNum, srcReg, tailMask);
176- MicroAPI::DataCopy(srcReg1, src + (j * 2 + 1) * inner + (i + mainRepeatTime) * innerOneRepNum);
177- MicroAPI::DataCopy(dst + (j * 2 + 1) * inner + (i + mainRepeatTime) * innerOneRepNum, srcReg1, tailMask);
178- }
179- }
180- 
181- for (uint16_t j = 0; j < tailOutter; ++j) {
182- for (uint16_t i = 0; i < mainRepeatTime; ++i) {
183- MicroAPI::DataCopy(srcReg, src + (j + 2 * halfOutter) * inner + i * innerOneRepNum);
184- MicroAPI::DataCopy(dst + (j + 2 * halfOutter) * inner + i * innerOneRepNum, srcReg, fullMask);
185- }
186- for (uint16_t i = 0; i < tailRepeatTime; ++i) {
187- MicroAPI::DataCopy(srcReg, src + (j + 2 * halfOutter) * inner + (i + mainRepeatTime) * innerOneRepNum);
188- MicroAPI::DataCopy(dst + (j + 2 * halfOutter) * inner + (i + mainRepeatTime) * innerOneRepNum, srcReg,
189- tailMask);
190- }
191- }
192-}
193- 
194-template <typename T>
195-__aicore__ inline void CumSumCopyOut(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
196- const uint32_t outter, const uint32_t inner)
197-{
198- __ubuf__ T* src = (__ubuf__ T*)srcTensor.GetPhyAddr();
199- __ubuf__ T* dst = (__ubuf__ T*)dstTensor.GetPhyAddr();
200- constexpr uint16_t innerOneRepNum = GetVecLen() / sizeof(T);
201- constexpr uint16_t elePerBlock = ONE_BLK_SIZE / sizeof(T);
202- uint16_t mainRepeatTime = inner / innerOneRepNum;
203- uint32_t tailCount = inner % innerOneRepNum;
204- uint16_t tailRepeatTime = tailCount > 0 ? 1 : 0;
205- uint16_t halfOutter = outter / 2;
206- uint16_t tailOutter = outter % 2;
207- CumSumCopyOutVF<T>(src, dst, outter, inner, mainRepeatTime, innerOneRepNum, tailCount, tailRepeatTime, halfOutter,
208- tailOutter);
209-}
210- 
211-template <typename D, typename T, const MicroAPI::RegTrait& Trait, const uint16_t vlSize>
212-__aicore__ inline void TransposeCommonGather(__ubuf__ D* dstAddr, __ubuf__ T* srcAddr, uint32_t forLoop1,
213- uint32_t forLoop2, uint32_t srcStride1, uint32_t srcStride2)
214-{
215- ASCENDC_ASSERT(false, { KERNEL_LOG(KERNEL_ERROR, "current data type is not supported!"); });
216-}
217- 
218-// VF for TransposeCommonGather (float, float)
219-template <typename T, const MicroAPI::RegTrait& Trait, const uint16_t vlSize>
220-__simd_vf__ inline void TransposeCommonGatherVFFF(__ubuf__ float* dstAddr, __ubuf__ float* srcAddr, uint32_t forLoop1,
221- uint32_t forLoop2, uint32_t srcStride1, uint32_t srcStride2,
222- uint32_t tail, uint32_t count, uint16_t mainLoop, uint32_t dtypeSize,
223- uint32_t tailLoop)
224-{
225- MicroAPI::RegTensor<uint32_t, Trait> indexReg;
226- MicroAPI::RegTensor<T, Trait> srcReg;
227- MicroAPI::MaskReg fullMask = MicroAPI::CreateMask<uint16_t, MicroAPI::MaskPattern::ALL, Trait>();
228- MicroAPI::MaskReg indexFullMask = MicroAPI::CreateMask<float, MicroAPI::MaskPattern::ALL, Trait>();
229- MicroAPI::MaskReg mainMask = MicroAPI::CreateMask<float, MicroAPI::MaskPattern::ALL, Trait>();
230- MicroAPI::MaskReg tailMask = MicroAPI::UpdateMask<float, Trait>(count);
231- MicroAPI::UnalignReg ureg0;
232- Arange((MicroAPI::RegTensor<int32_t, Trait>&)indexReg, static_cast<int32_t>(0));
233- Muls(indexReg, indexReg, srcStride2, indexFullMask);
234- for (uint16_t j = 0; j < static_cast<uint16_t>(forLoop1); j++) {
235- uint64_t hoistDstAddr = (uint64_t)dstAddr + (uint64_t)(j * forLoop2 * dtypeSize);
236- for (uint16_t k = 0; k < static_cast<uint16_t>(mainLoop); k++) {
237- DataCopyGather(srcReg, srcAddr + j * srcStride1 + k * vlSize * srcStride2, indexReg, mainMask);
238- MicroAPI::DataCopyUnAlign(((__ubuf__ float*&)hoistDstAddr), srcReg, ureg0, vlSize);
239- }
240- for (uint16_t k = 0; k < static_cast<uint16_t>(tailLoop); k++) {
241- DataCopyGather(srcReg, srcAddr + j * srcStride1 + mainLoop * vlSize * srcStride2, indexReg, tailMask);
242- MicroAPI::DataCopyUnAlign(((__ubuf__ float*&)hoistDstAddr), srcReg, ureg0, tail);
243- }
244- MicroAPI::DataCopyUnAlignPost(((__ubuf__ float*&)hoistDstAddr), ureg0, 0);
245- }
246-}
247- 
248-template <typename D = float, typename T = float, const MicroAPI::RegTrait& Trait, const uint16_t vlSize>
249-__aicore__ inline void TransposeCommonGather(__ubuf__ float* dstAddr, __ubuf__ float* srcAddr, uint32_t forLoop1,
250- uint32_t forLoop2, uint32_t srcStride1, uint32_t srcStride2)
251-{
252- uint32_t tail = forLoop2 % vlSize;
253- uint32_t count = tail;
254- uint16_t mainLoop = forLoop2 / vlSize;
255- uint32_t dtypeSize = sizeof(float);
256- uint32_t tailLoop = tail > 0 ? 1 : 0;
257- TransposeCommonGatherVFFF<T, Trait, vlSize>(dstAddr, srcAddr, forLoop1, forLoop2, srcStride1, srcStride2, tail,
258- count, mainLoop, dtypeSize, tailLoop);
259-}
260- 
261-// VF for TransposeCommonGather (float, half)
262-template <const MicroAPI::RegTrait& Trait, const uint16_t vlSize>
263-__simd_vf__ inline void TransposeCommonGatherVFFH(__ubuf__ float* dstAddr, __ubuf__ half* srcAddr, uint32_t forLoop1,
264- uint32_t forLoop2, uint32_t srcStride1, uint32_t srcStride2,
265- uint32_t tail, uint32_t count, uint16_t mainLoop, uint32_t dtypeSize,
266- uint32_t tailLoop)
267-{
268- MicroAPI::RegTensor<uint16_t, Trait> indexReg;
269- MicroAPI::RegTensor<half, Trait> srcReg;
270- MicroAPI::RegTensor<float, Trait> vreg;
271- MicroAPI::RegTensor<uint16_t> zeroReg;
272- MicroAPI::RegTensor<half> castReg;
273- MicroAPI::RegTensor<uint16_t> tmpReg;
274- MicroAPI::MaskReg fullMask = MicroAPI::CreateMask<uint16_t, MicroAPI::MaskPattern::ALL, Trait>();
275- MicroAPI::MaskReg indexFullMask = MicroAPI::CreateMask<half, MicroAPI::MaskPattern::ALL, Trait>();
276- MicroAPI::MaskReg mainMask = MicroAPI::CreateMask<half, MicroAPI::MaskPattern::ALL, Trait>();
277- MicroAPI::MaskReg tailMask = MicroAPI::UpdateMask<half, Trait>(count);
278- MicroAPI::Duplicate(zeroReg, static_cast<uint16_t>(0), fullMask);
279- MicroAPI::UnalignReg ureg0;
280- Arange((MicroAPI::RegTensor<int16_t, Trait>&)indexReg, static_cast<int16_t>(0));
281- Muls(indexReg, indexReg, static_cast<uint16_t>(srcStride2), indexFullMask);
282- for (uint16_t j = 0; j < static_cast<uint16_t>(forLoop1); j++) {
283- uint64_t hoistDstAddr = (uint64_t)dstAddr + (uint64_t)(j * forLoop2 * dtypeSize);
284- for (uint16_t k = 0; k < static_cast<uint16_t>(mainLoop); k++) {
285- DataCopyGather(srcReg, srcAddr + j * srcStride1 + k * vlSize * srcStride2, indexReg, mainMask);
286- MicroAPI::Interleave((MicroAPI::RegTensor<uint16_t>&)castReg, (MicroAPI::RegTensor<uint16_t>&)tmpReg,
287- (MicroAPI::RegTensor<uint16_t>&)srcReg, (MicroAPI::RegTensor<uint16_t>&)zeroReg);
288- Cast<float, half, layoutZMrgZ>(vreg, castReg, mainMask);
289- MicroAPI::DataCopyUnAlign(((__ubuf__ float*&)hoistDstAddr), vreg, ureg0, vlSize);
290- }
291- for (uint16_t k = 0; k < static_cast<uint16_t>(tailLoop); k++) {
292- DataCopyGather(srcReg, srcAddr + j * srcStride1 + mainLoop * vlSize * srcStride2, indexReg, tailMask);
293- MicroAPI::Interleave((MicroAPI::RegTensor<uint16_t>&)castReg, (MicroAPI::RegTensor<uint16_t>&)tmpReg,
294- (MicroAPI::RegTensor<uint16_t>&)srcReg, (MicroAPI::RegTensor<uint16_t>&)zeroReg);
295- Cast<float, half, layoutZMrgZ>(vreg, castReg, mainMask);
296- MicroAPI::DataCopyUnAlign(((__ubuf__ float*&)hoistDstAddr), vreg, ureg0, tail);
297- }
298- MicroAPI::DataCopyUnAlignPost(((__ubuf__ float*&)hoistDstAddr), ureg0, 0);
299- }
300-}
301- 
302-template <typename D = float, typename T = half, const MicroAPI::RegTrait& Trait, const uint16_t vlSize>
303-__aicore__ inline void TransposeCommonGather(__ubuf__ float* dstAddr, __ubuf__ half* srcAddr, uint32_t forLoop1,
304- uint32_t forLoop2, uint32_t srcStride1, uint32_t srcStride2)
305-{
306- uint32_t tail = forLoop2 % vlSize;
307- uint32_t count = tail;
308- uint16_t mainLoop = forLoop2 / vlSize;
309- uint32_t dtypeSize = sizeof(float);
310- uint32_t tailLoop = tail > 0 ? 1 : 0;
311- TransposeCommonGatherVFFH<Trait, vlSize>(dstAddr, srcAddr, forLoop1, forLoop2, srcStride1, srcStride2, tail, count,
312- mainLoop, dtypeSize, tailLoop);
313-}
314- 
315-// VF for TransposeCommonGather (half, float)
316-template <const MicroAPI::RegTrait& Trait, const uint16_t vlSize>
317-__simd_vf__ inline void TransposeCommonGatherVHF(__ubuf__ half* dstAddr, __ubuf__ float* srcAddr, uint32_t forLoop1,
318- uint32_t forLoop2, uint32_t srcStride1, uint32_t srcStride2,
319- uint32_t tail, uint32_t count, uint16_t mainLoop, uint32_t dtypeSize,
320- uint32_t tailLoop)
321-{
322- MicroAPI::RegTensor<uint32_t, Trait> indexReg;
323- MicroAPI::RegTensor<float, Trait> srcReg;
324- MicroAPI::RegTensor<half, Trait> vreg;
325- MicroAPI::RegTensor<uint16_t> zeroReg;
326- MicroAPI::RegTensor<half> castReg;
327- MicroAPI::RegTensor<uint16_t> tmpReg;
328- MicroAPI::MaskReg fullMask = MicroAPI::CreateMask<uint16_t, MicroAPI::MaskPattern::ALL, Trait>();
329- MicroAPI::MaskReg indexFullMask = MicroAPI::CreateMask<float, MicroAPI::MaskPattern::ALL, Trait>();
330- MicroAPI::MaskReg mainMask = MicroAPI::CreateMask<float, MicroAPI::MaskPattern::ALL, Trait>();
331- MicroAPI::MaskReg tailMask = MicroAPI::UpdateMask<float, Trait>(count);
332- MicroAPI::Duplicate(zeroReg, static_cast<uint16_t>(0), fullMask);
333- MicroAPI::UnalignReg ureg0;
334- Arange((MicroAPI::RegTensor<int32_t, Trait>&)indexReg, static_cast<int32_t>(0));
335- Muls(indexReg, indexReg, srcStride2, indexFullMask);
336- for (uint16_t j = 0; j < static_cast<uint16_t>(forLoop1); j++) {
337- uint64_t hoistDstAddr = (uint64_t)dstAddr + (uint64_t)(j * forLoop2 * dtypeSize);
338- for (uint16_t k = 0; k < static_cast<uint16_t>(mainLoop); k++) {
339- DataCopyGather(srcReg, srcAddr + j * srcStride1 + k * vlSize * srcStride2, indexReg, mainMask);
340- Cast<half, float, LayoutZMrgZRndRSatNS>(vreg, srcReg, fullMask);
341- MicroAPI::DeInterleave((MicroAPI::RegTensor<uint16_t>&)castReg, (MicroAPI::RegTensor<uint16_t>&)tmpReg,
342- (MicroAPI::RegTensor<uint16_t>&)vreg, (MicroAPI::RegTensor<uint16_t>&)zeroReg);
343- MicroAPI::DataCopyUnAlign(((__ubuf__ half*&)hoistDstAddr), castReg, ureg0, vlSize);
344- }
345- for (uint16_t k = 0; k < static_cast<uint16_t>(tailLoop); k++) {
346- DataCopyGather(srcReg, srcAddr + j * srcStride1 + mainLoop * vlSize * srcStride2, indexReg, tailMask);
347- Cast<half, float, LayoutZMrgZRndRSatNS>(vreg, srcReg, fullMask);
348- MicroAPI::DeInterleave((MicroAPI::RegTensor<uint16_t>&)castReg, (MicroAPI::RegTensor<uint16_t>&)tmpReg,
349- (MicroAPI::RegTensor<uint16_t>&)vreg, (MicroAPI::RegTensor<uint16_t>&)zeroReg);
350- MicroAPI::DataCopyUnAlign(((__ubuf__ half*&)hoistDstAddr), castReg, ureg0, tail);
351- }
352- MicroAPI::DataCopyUnAlignPost(((__ubuf__ half*&)hoistDstAddr), ureg0, 0);
353- }
354-}
355- 
356-template <typename D = half, typename T = float, const MicroAPI::RegTrait& Trait, const uint16_t vlSize>
357-__aicore__ inline void TransposeCommonGather(__ubuf__ half* dstAddr, __ubuf__ float* srcAddr, uint32_t forLoop1,
358- uint32_t forLoop2, uint32_t srcStride1, uint32_t srcStride2)
359-{
360- uint32_t tail = forLoop2 % vlSize;
361- uint32_t count = tail;
362- uint16_t mainLoop = forLoop2 / vlSize;
363- uint32_t dtypeSize = sizeof(half);
364- uint32_t tailLoop = tail > 0 ? 1 : 0;
365- TransposeCommonGatherVHF<Trait, vlSize>(dstAddr, srcAddr, forLoop1, forLoop2, srcStride1, srcStride2, tail, count,
366- mainLoop, dtypeSize, tailLoop);
367-}
368- 
369-/*
370-scene: { shape:[A, B], format:"ND"} -->{ shape:[B, A], format:"ND"};
371-Src: T
372-Dst: D
373-1. need cast
374-TransposeAB [A, B] half => [B, A] float
375-TransposeAB [A, B] float => [B, A] half
376-2. no need cast
377-TransposeAB [A, B] float => [B, A] float
378-*/
379-template <typename D, typename T>
380-__aicore__ inline void TransposeAB(const LocalTensor<D>& dstTensor, const LocalTensor<T>& srcTensor, uint32_t outer,
381- uint32_t inner)
382-{
383- uint32_t srcStride1 = 1;
384- uint32_t srcStride2 = inner;
385- constexpr uint16_t vlSize = GetVecLen() / sizeof(float);
386- TransposeCommonGather<D, T, MicroAPI::RegTraitNumOne, vlSize>(
387- (__ubuf__ D*)dstTensor.GetPhyAddr(), (__ubuf__ T*)srcTensor.GetPhyAddr(), inner, outer, srcStride1, srcStride2);
388-}
389- 
390-__simd_vf__ inline void CumSumFirstDimSklanskyVF(__ubuf__ float* dst, uint32_t outer, uint32_t inner,
391- uint32_t currRound1, uint32_t currRound2, uint16_t indexRepeatTimes,
392- uint16_t jRepeatTimes, uint16_t repeatTimes, uint16_t sregLower)
393-{
394- MicroAPI::RegTensor<float> src0Reg;
395- MicroAPI::RegTensor<float> src1Reg;
396- MicroAPI::RegTensor<float> dstReg;
397- MicroAPI::MaskReg preg;
398- 
399- for (uint16_t index = 0; index < indexRepeatTimes; index++) {
400- // Position of the prefix sum in the previous round
401- uint32_t line0 = currRound1 - 1 + index * currRound2;
402- for (uint16_t j = 0; j < jRepeatTimes; j++) {
403- uint32_t line1 = line0 + j + 1;
404- uint32_t extent = 1;
405- if (line1 > outer - 1) {
406- extent = 0;
407- }
408- for (uint16_t k = 0; k < static_cast<uint16_t>(extent); k++) {
409- uint32_t count = inner;
410- for (uint16_t i = 0; i < repeatTimes; i++) {
411- preg = MicroAPI::UpdateMask<float>(count);
412- MicroAPI::DataCopy(src0Reg, dst + line0 * inner + i * sregLower);
413- MicroAPI::DataCopy(src1Reg, dst + line1 * inner + i * sregLower);
414- MicroAPI::Add(dstReg, src0Reg, src1Reg, preg);
415- MicroAPI::DataCopy(dst + line1 * inner + i * sregLower, dstReg, preg);
416- }
417- }
418- }
419- }
420-}
421- 
422-__aicore__ inline void CumSumFirstDimSklansky(const LocalTensor<float>& dstTensor, uint32_t outer, uint32_t inner)
423-{
424- constexpr uint32_t bound = 16;
425- constexpr uint32_t halfSize = 2;
426- uint32_t outerAlign = 0;
427- for (uint32_t i = 0; i < bound; i++) {
428- if (outer <= (1U << i)) {
429- outerAlign = (1U << i);
430- break;
431- }
432- }
433- uint32_t round = outerAlign / halfSize;
434- 
435- uint32_t currRound = 1;
436- __ubuf__ float* dst = (__ubuf__ float*)dstTensor.GetPhyAddr();
437- constexpr uint16_t sregLower = (uint32_t)(GetVecLen() / sizeof(float));
438- uint16_t repeatTimes = CeilDivision(inner, sregLower);
439- 
440- while (round >= 1) {
441- uint32_t currRound1 = 1 << (currRound - 1);
442- uint32_t currRound2 = 1 << currRound;
443- uint16_t indexRepeatTimes = static_cast<uint16_t>(outerAlign / currRound2);
444- uint16_t jRepeatTimes = static_cast<uint16_t>(currRound1);
445- CumSumFirstDimSklanskyVF(dst, outer, inner, currRound1, currRound2, indexRepeatTimes, jRepeatTimes, repeatTimes,
446- sregLower);
447- round = round / halfSize;
448- currRound += 1;
449- }
450-}
451- 
452-// simple implementation that cumulatively adds elements
453-// VF for CumSumFirstDimBasic
454-// simple implementation that cumulatively adds elements
455-// VF for CumSumFirstDimBasic
456-__simd_vf__ inline void CumSumFirstDimBasicVF(__ubuf__ float* dst, uint16_t outerRepeatTime, uint16_t inner,
457- uint16_t mainRepeatTime, uint16_t innerOneRepNum, uint16_t tailTime,
458- uint32_t tailCount, uint16_t halfMainRepeatTime,
459- uint16_t mainTailRepeatTime, uint16_t innerTailOffset1,
460- uint16_t innerTailOffset2)
461-{
462- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<float, MicroAPI::MaskPattern::ALL>();
463- MicroAPI::MaskReg pregTail = MicroAPI::UpdateMask<float>(tailCount);
464- MicroAPI::RegTensor<float> srcLeftReg;
465- MicroAPI::RegTensor<float> srcRightReg;
466- MicroAPI::RegTensor<float> dstLeftReg;
467- MicroAPI::RegTensor<float> dstRightReg;
468- 
469- for (uint16_t j = 0; j < halfMainRepeatTime; ++j) {
470- MicroAPI::DataCopy(dstLeftReg, dst + j * innerOneRepNum);
471- MicroAPI::DataCopy(dstRightReg, dst + (j + halfMainRepeatTime) * innerOneRepNum);
472- for (uint16_t i = 0; i < outerRepeatTime; ++i) {
473- MicroAPI::DataCopy(srcLeftReg, dst + (i + 1) * inner + j * innerOneRepNum);
474- MicroAPI::DataCopy(srcRightReg, dst + (i + 1) * inner + (j + halfMainRepeatTime) * innerOneRepNum);
475- MicroAPI::Add(dstLeftReg, srcLeftReg, dstLeftReg, pregFull);
476- MicroAPI::Add(dstRightReg, srcRightReg, dstRightReg, pregFull);
477- MicroAPI::DataCopy(dst + (i + 1) * inner + j * innerOneRepNum, dstLeftReg, pregFull);
478- MicroAPI::DataCopy(dst + (i + 1) * inner + (j + halfMainRepeatTime) * innerOneRepNum, dstRightReg,
479- pregFull);
480- }
481- }
482- 
483- MicroAPI::DataCopy(dstLeftReg, dst + innerTailOffset1);
484- MicroAPI::DataCopy(dstRightReg, dst + innerTailOffset2);
485- for (uint16_t i = 0; i < outerRepeatTime; ++i) {
486- for (uint16_t j = 0; j < mainTailRepeatTime; ++j) {
487- MicroAPI::DataCopy(srcLeftReg, dst + (i + 1) * inner + innerTailOffset1);
488- MicroAPI::Add(dstLeftReg, srcLeftReg, dstLeftReg, pregFull);
489- MicroAPI::DataCopy(dst + (i + 1) * inner + innerTailOffset1, dstLeftReg, pregFull);
490- }
491- for (uint16_t j = 0; j < tailTime; ++j) {
492- MicroAPI::DataCopy(srcRightReg, dst + (i + 1) * inner + innerTailOffset2);
493- MicroAPI::Add(dstRightReg, srcRightReg, dstRightReg, pregTail);
494- MicroAPI::DataCopy(dst + (i + 1) * inner + innerTailOffset2, dstRightReg, pregTail);
495- }
496- }
497-}
498- 
499-__simd_vf__ inline void CumSumFirstDimBasic2VF(__ubuf__ float* dst, uint16_t outerRepeatTime, uint16_t inner,
500- uint16_t mainRepeatTime, uint16_t innerOneRepNum, uint16_t tailTime,
501- uint32_t tailCount, uint16_t halfMainRepeatTime,
502- uint16_t mainTailRepeatTime, uint16_t innerTailOffset1,
503- uint16_t innerTailOffset2)
504-{
505- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<float, MicroAPI::MaskPattern::ALL>();
506- MicroAPI::MaskReg pregTail = MicroAPI::UpdateMask<float>(tailCount);
507- MicroAPI::RegTensor<float> srcLeftReg;
508- MicroAPI::RegTensor<float> dstLeftReg;
509- 
510- for (uint16_t j = 0; j < mainRepeatTime; ++j) {
511- MicroAPI::DataCopy(dstLeftReg, dst + j * innerOneRepNum);
512- for (uint16_t i = 0; i < outerRepeatTime; ++i) {
513- MicroAPI::DataCopy(srcLeftReg, dst + (i + 1) * inner + j * innerOneRepNum);
514- MicroAPI::Add(dstLeftReg, srcLeftReg, dstLeftReg, pregFull);
515- MicroAPI::DataCopy(dst + (i + 1) * inner + j * innerOneRepNum, dstLeftReg, pregFull);
516- }
517- }
518- 
519- MicroAPI::DataCopy(dstLeftReg, dst + innerTailOffset2);
520- for (uint16_t i = 0; i < outerRepeatTime; ++i) {
521- for (uint16_t j = 0; j < tailTime; ++j) {
522- MicroAPI::DataCopy(srcLeftReg, dst + (i + 1) * inner + innerTailOffset2);
523- MicroAPI::Add(dstLeftReg, srcLeftReg, dstLeftReg, pregTail);
524- MicroAPI::DataCopy(dst + (i + 1) * inner + innerTailOffset2, dstLeftReg, pregTail);
525- }
526- }
527-}
528- 
529-__aicore__ inline void CumSumFirstDimBasic(const LocalTensor<float>& dstTensor, uint32_t outer, uint32_t inner)
530-{
531- __ubuf__ float* dst = (__ubuf__ float*)dstTensor.GetPhyAddr();
532- constexpr uint16_t innerOneRepNum = (uint32_t)(GetVecLen() / sizeof(float));
533- uint16_t mainRepeatTime = 0;
534- if constexpr (innerOneRepNum > 0) {
535- mainRepeatTime = inner / innerOneRepNum;
536- }
537- uint16_t outterRepeatTime = static_cast<uint16_t>(outer - 1);
538- uint32_t tailCount = inner - mainRepeatTime * innerOneRepNum;
539- uint16_t halfMainRepeatTime = mainRepeatTime / 2;
540- uint16_t mainTailRepeatTime = mainRepeatTime % 2;
541- uint16_t tailRepeatTime = tailCount > 0 ? 1 : 0;
542- uint16_t castedInner = static_cast<uint16_t>(inner);
543- uint16_t innerTailOffset1 = halfMainRepeatTime * innerOneRepNum * 2;
544- uint16_t innerTailOffset2 = mainRepeatTime * innerOneRepNum;
545- CumSumFirstDimBasic2VF(dst, outterRepeatTime, castedInner, mainRepeatTime, innerOneRepNum, tailRepeatTime,
546- tailCount, halfMainRepeatTime, mainTailRepeatTime, innerTailOffset1, innerTailOffset2);
547-}
548- 
549-__aicore__ inline TransDataTo5HDParams ExtractTransDataParam(uint8_t repeatTimes, uint32_t inner, uint16_t alignOutter,
550- uint32_t oneBlockElementNum, uint16_t dstRepStride,
551- uint32_t srcRepStride)
552-{
553- repeatTimes = 0;
554- if (oneBlockElementNum > 0) {
555- repeatTimes = inner / oneBlockElementNum;
556- }
557- if (repeatTimes > 1) {
558- return TransDataTo5HDParams(false, false, repeatTimes, alignOutter, 1);
559- } else {
560- return TransDataTo5HDParams(false, false, repeatTimes, dstRepStride, srcRepStride);
561- }
562-}
563- 
564-template <typename T = float>
565-__aicore__ inline void Transpose5HDAB(const LocalTensor<float>& dstTensor, const LocalTensor<float>& srcTensor,
566- const CumSumInfo& cumSumInfo)
567-{
568- constexpr uint32_t oneBlockElementNum = ONE_BLK_SIZE / sizeof(float);
569- uint8_t repeatTimes = 1;
570- uint16_t dstRepStride = 0;
571- uint16_t srcRepStride = 0;
572- uint16_t alignOutter =
573- (cumSumInfo.outter + NCHW_CONV_ADDR_LIST_SIZE - 1) / NCHW_CONV_ADDR_LIST_SIZE * NCHW_CONV_ADDR_LIST_SIZE;
574- uint64_t transDataTo5HDDstLocalList[NCHW_CONV_ADDR_LIST_SIZE];
575- uint64_t transDataTo5HDSrcLocalList[NCHW_CONV_ADDR_LIST_SIZE];
576- if (cumSumInfo.outter == alignOutter && alignOutter > cumSumInfo.inner) {
577- repeatTimes = alignOutter / NCHW_CONV_ADDR_LIST_SIZE;
578- if (repeatTimes > 1) {
579- dstRepStride = 2;
580- srcRepStride = cumSumInfo.inner * 2;
581- }
582- TransDataTo5HDParams params(false, false, repeatTimes, dstRepStride, srcRepStride);
583- for (int32_t i = 0; i < cumSumInfo.inner / oneBlockElementNum; i++) {
584- for (int32_t n = 0; n < NCHW_CONV_ADDR_LIST_SIZE; n++) {
585- transDataTo5HDSrcLocalList[n] =
586- (uint64_t)srcTensor[i * oneBlockElementNum + n * cumSumInfo.inner].GetPhyAddr();
587- }
588- for (int32_t n = 0; n < NCHW_CONV_ADDR_LIST_SIZE / 2; n++) {
589- transDataTo5HDDstLocalList[n * 2] =
590- (uint64_t)dstTensor[(i * oneBlockElementNum + n) * alignOutter].GetPhyAddr();
591- transDataTo5HDDstLocalList[n * 2 + 1] =
592- (uint64_t)dstTensor[(i * oneBlockElementNum + n) * alignOutter + oneBlockElementNum].GetPhyAddr();
593- }
594- TransDataTo5HD<float>(transDataTo5HDDstLocalList, transDataTo5HDSrcLocalList, params);
595- }
596- } else {
597- TransDataTo5HDParams params = ExtractTransDataParam(repeatTimes, cumSumInfo.inner, alignOutter,
598- oneBlockElementNum, dstRepStride, srcRepStride);
599- for (int32_t i = 0; i < alignOutter / NCHW_CONV_ADDR_LIST_SIZE; i++) {
600- for (int32_t n = 0; n < NCHW_CONV_ADDR_LIST_SIZE; n++) {
601- transDataTo5HDSrcLocalList[n] =
602- (uint64_t)srcTensor[((i * NCHW_CONV_ADDR_LIST_SIZE
603- + n % (cumSumInfo.outter - i * NCHW_CONV_ADDR_LIST_SIZE))
604- * cumSumInfo.inner)]
605- .GetPhyAddr();
606- }
607- for (int32_t n = 0; n < NCHW_CONV_ADDR_LIST_SIZE / 2; n++) {
608- transDataTo5HDDstLocalList[n * 2] =
609- (uint64_t)dstTensor[i * NCHW_CONV_ADDR_LIST_SIZE + n * alignOutter].GetPhyAddr();
610- transDataTo5HDDstLocalList[n * 2 + 1] =
611- (uint64_t)dstTensor[i * NCHW_CONV_ADDR_LIST_SIZE + n * alignOutter + oneBlockElementNum]
612- .GetPhyAddr();
613- }
614- TransDataTo5HD<float>(transDataTo5HDDstLocalList, transDataTo5HDSrcLocalList, params);
615- }
616- }
617-}
618- 
619-template <typename T = float>
620-__aicore__ inline void Transpose5HDBA(const LocalTensor<float>& dstTensor, const LocalTensor<float>& srcTensor,
621- const CumSumInfo& cumSumInfo)
622-{
623- constexpr uint32_t oneBlockElementNum = ONE_BLK_SIZE / sizeof(float);
624- uint8_t repeatTimes = 1;
625- uint16_t dstRepStride = 0;
626- uint16_t srcRepStride = 0;
627- uint16_t alignOutter =
628- (cumSumInfo.outter + NCHW_CONV_ADDR_LIST_SIZE - 1) / NCHW_CONV_ADDR_LIST_SIZE * NCHW_CONV_ADDR_LIST_SIZE;
629- uint64_t transDataTo5HDDstLocalList[NCHW_CONV_ADDR_LIST_SIZE];
630- uint64_t transDataTo5HDSrcLocalList[NCHW_CONV_ADDR_LIST_SIZE];
631- if (alignOutter > cumSumInfo.inner) {
632- repeatTimes = alignOutter / NCHW_CONV_ADDR_LIST_SIZE;
633- if (repeatTimes > 1) {
634- dstRepStride = cumSumInfo.inner * 2;
635- srcRepStride = 2;
636- } else {
637- dstRepStride = 0;
638- srcRepStride = 0;
639- }
640- TransDataTo5HDParams paramsBack(false, false, repeatTimes, dstRepStride, srcRepStride);
641- for (int32_t i = 0; i < cumSumInfo.inner / oneBlockElementNum; i++) {
642- for (int32_t n = 0; n < NCHW_CONV_ADDR_LIST_SIZE / 2; n++) {
643- transDataTo5HDSrcLocalList[n] =
644- (uint64_t)srcTensor[i * oneBlockElementNum * alignOutter + n * alignOutter].GetPhyAddr();
645- transDataTo5HDSrcLocalList[n + NCHW_CONV_ADDR_LIST_SIZE / 2] =
646- (uint64_t)srcTensor[i * oneBlockElementNum * alignOutter + n * alignOutter + oneBlockElementNum]
647- .GetPhyAddr();
648- transDataTo5HDDstLocalList[n * 2] =
649- (uint64_t)dstTensor[i * oneBlockElementNum + n * cumSumInfo.inner].GetPhyAddr();
650- transDataTo5HDDstLocalList[n * 2 + 1] =
651- (uint64_t)dstTensor[i * oneBlockElementNum + (n + oneBlockElementNum) * cumSumInfo.inner]
652- .GetPhyAddr();
653- }
654- TransDataTo5HD<float>(transDataTo5HDDstLocalList, transDataTo5HDSrcLocalList, paramsBack);
655- }
656- } else {
657- repeatTimes = cumSumInfo.inner / oneBlockElementNum;
658- if (repeatTimes > 1) {
659- dstRepStride = alignOutter;
660- srcRepStride = 1;
661- } else {
662- dstRepStride = 0;
663- srcRepStride = 0;
664- }
665- TransDataTo5HDParams paramsBack(false, false, repeatTimes, srcRepStride, dstRepStride);
666- for (int32_t i = 0; i < alignOutter / NCHW_CONV_ADDR_LIST_SIZE; i++) {
667- for (int32_t n = 0; n < NCHW_CONV_ADDR_LIST_SIZE / 2; n++) {
668- transDataTo5HDSrcLocalList[n] =
669- (uint64_t)srcTensor[i * NCHW_CONV_ADDR_LIST_SIZE + n * alignOutter].GetPhyAddr();
670- transDataTo5HDSrcLocalList[n + NCHW_CONV_ADDR_LIST_SIZE / 2] =
671- (uint64_t)srcTensor[i * NCHW_CONV_ADDR_LIST_SIZE + n * alignOutter + oneBlockElementNum]
672- .GetPhyAddr();
673- }
674- for (int32_t n = 0; n < NCHW_CONV_ADDR_LIST_SIZE / 2; n++) {
675- transDataTo5HDDstLocalList[n * 2] =
676- (uint64_t)dstTensor[(i * NCHW_CONV_ADDR_LIST_SIZE + n) * cumSumInfo.inner].GetPhyAddr();
677- transDataTo5HDDstLocalList[n * 2 + 1] =
678- (uint64_t)dstTensor[(i * NCHW_CONV_ADDR_LIST_SIZE + (n + NCHW_CONV_ADDR_LIST_SIZE / 2))
679- * cumSumInfo.inner]
680- .GetPhyAddr();
681- }
682- TransDataTo5HD<float>(transDataTo5HDDstLocalList, transDataTo5HDSrcLocalList, paramsBack);
683- }
684- }
685-}
686- 
687-template <typename T = half>
688-__aicore__ inline void Transpose5HDAB(const LocalTensor<half>& dstTensor, const LocalTensor<half>& srcTensor,
689- const CumSumInfo& cumSumInfo)
690-{
691- constexpr uint32_t oneBlockElementNum = ONE_BLK_SIZE / sizeof(T);
692- uint16_t alignOutter =
693- (cumSumInfo.outter + NCHW_CONV_ADDR_LIST_SIZE - 1) / NCHW_CONV_ADDR_LIST_SIZE * NCHW_CONV_ADDR_LIST_SIZE;
694- uint64_t transDataTo5HDDstLocalList[NCHW_CONV_ADDR_LIST_SIZE];
695- uint64_t transDataTo5HDSrcLocalList[NCHW_CONV_ADDR_LIST_SIZE];
696- uint8_t repeatTimes = 1;
697- uint16_t dstRepStride = 0;
698- uint16_t srcRepStride = 0;
699- if (cumSumInfo.outter == alignOutter && alignOutter > cumSumInfo.inner) {
700- repeatTimes = alignOutter / NCHW_CONV_ADDR_LIST_SIZE;
701- if (repeatTimes > 1) {
702- dstRepStride = 1;
703- srcRepStride = cumSumInfo.inner;
704- }
705- TransDataTo5HDParams params(false, false, repeatTimes, dstRepStride, srcRepStride);
706- for (int32_t i = 0; i < cumSumInfo.inner / oneBlockElementNum; i++) {
707- for (int32_t n = 0; n < NCHW_CONV_ADDR_LIST_SIZE; n++) {
708- transDataTo5HDSrcLocalList[n] =
709- (uint64_t)srcTensor[i * oneBlockElementNum + n * cumSumInfo.inner].GetPhyAddr();
710- transDataTo5HDDstLocalList[n] =
711- (uint64_t)dstTensor[i * oneBlockElementNum * alignOutter + alignOutter * n].GetPhyAddr();
712- }
713- TransDataTo5HD<T>(transDataTo5HDDstLocalList, transDataTo5HDSrcLocalList, params);
714- }
715- } else {
716- TransDataTo5HDParams params = ExtractTransDataParam(repeatTimes, cumSumInfo.inner, alignOutter,
717- oneBlockElementNum, dstRepStride, srcRepStride);
718- for (int32_t i = 0; i < alignOutter / NCHW_CONV_ADDR_LIST_SIZE; i++) {
719- for (int32_t n = 0; n < NCHW_CONV_ADDR_LIST_SIZE; n++) {
720- transDataTo5HDSrcLocalList[n] =
721- (uint64_t)srcTensor[((i * NCHW_CONV_ADDR_LIST_SIZE
722- + n % (cumSumInfo.outter - i * NCHW_CONV_ADDR_LIST_SIZE))
723- * cumSumInfo.inner)]
724- .GetPhyAddr();
725- transDataTo5HDDstLocalList[n] =
726- (uint64_t)dstTensor[i * NCHW_CONV_ADDR_LIST_SIZE + alignOutter * n].GetPhyAddr();
727- }
728- TransDataTo5HD<T>(transDataTo5HDDstLocalList, transDataTo5HDSrcLocalList, params);
729- }
730- }
731-}
732- 
733-template <typename T = half>
734-__aicore__ inline void Transpose5HDBA(const LocalTensor<half>& dstTensor, const LocalTensor<half>& srcTensor,
735- const CumSumInfo& cumSumInfo)
736-{
737- constexpr uint32_t oneBlockElementNum = ONE_BLK_SIZE / sizeof(T);
738- uint16_t alignOutter =
739- (cumSumInfo.outter + NCHW_CONV_ADDR_LIST_SIZE - 1) / NCHW_CONV_ADDR_LIST_SIZE * NCHW_CONV_ADDR_LIST_SIZE;
740- uint64_t transDataTo5HDDstLocalList[NCHW_CONV_ADDR_LIST_SIZE];
741- uint64_t transDataTo5HDSrcLocalList[NCHW_CONV_ADDR_LIST_SIZE];
742- uint8_t repeatTimes = 1;
743- uint16_t dstRepStride = 0;
744- uint16_t srcRepStride = 0;
745- if (alignOutter > cumSumInfo.inner) {
746- repeatTimes = alignOutter / oneBlockElementNum;
747- if (repeatTimes > 1) {
748- dstRepStride = cumSumInfo.inner;
749- srcRepStride = 1;
750- } else {
751- dstRepStride = 0;
752- srcRepStride = 0;
753- }
754- TransDataTo5HDParams paramsBack(false, false, repeatTimes, dstRepStride, srcRepStride);
755- for (int32_t i = 0; i < cumSumInfo.inner / NCHW_CONV_ADDR_LIST_SIZE; i++) {
756- for (int32_t n = 0; n < NCHW_CONV_ADDR_LIST_SIZE; n++) {
757- transDataTo5HDSrcLocalList[n] =
758- (uint64_t)srcTensor[(i * NCHW_CONV_ADDR_LIST_SIZE + n) * alignOutter].GetPhyAddr();
759- transDataTo5HDDstLocalList[n] =
760- (uint64_t)dstTensor[i * NCHW_CONV_ADDR_LIST_SIZE + n * cumSumInfo.inner].GetPhyAddr();
761- }
762- TransDataTo5HD<T>(transDataTo5HDDstLocalList, transDataTo5HDSrcLocalList, paramsBack);
763- }
764- } else {
765- repeatTimes = cumSumInfo.inner / oneBlockElementNum;
766- if (repeatTimes > 1) {
767- srcRepStride = 1;
768- dstRepStride = alignOutter;
769- } else {
770- dstRepStride = 0;
771- srcRepStride = 0;
772- }
773- TransDataTo5HDParams paramsBack(false, false, repeatTimes, srcRepStride, dstRepStride);
774- for (int32_t i = 0; i < alignOutter / NCHW_CONV_ADDR_LIST_SIZE; i++) {
775- for (int32_t n = 0; n < NCHW_CONV_ADDR_LIST_SIZE; n++) {
776- transDataTo5HDSrcLocalList[n] =
777- (uint64_t)srcTensor[i * NCHW_CONV_ADDR_LIST_SIZE + alignOutter * n].GetPhyAddr();
778- transDataTo5HDDstLocalList[n] =
779- (uint64_t)dstTensor[(i * NCHW_CONV_ADDR_LIST_SIZE + n) * cumSumInfo.inner].GetPhyAddr();
780- }
781- TransDataTo5HD<T>(transDataTo5HDDstLocalList, transDataTo5HDSrcLocalList, paramsBack);
782- }
783- }
784-}
785-} // namespace Internal
786-} // namespace AscendC
787- 
788-#endif // IMPL_MATH_CUMSUM_CUMSUM_L300_UTILS_H
Dimpl/adv_api/detail/math/erf/erf_l300_impl.h+0-401
@@ -1,401 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/*!
12- * \file erf_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_MATH_ERF_ERF_L300_IMPL_H
16-#define IMPL_MATH_ERF_ERF_L300_IMPL_H
17- 
18-#include "kernel_tensor.h"
19-#include "kernel_pop_stack_buffer.h"
20-#include "include/adv_api/math/erf_utils.h"
21- 
22-namespace AscendC {
23-namespace ErfAPI {
24- 
25-constexpr MicroAPI::CastTrait castTraitF162F32 = {
26- MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::UNKNOWN, MicroAPI::MaskMergeMode::ZEROING, RoundMode::UNKNOWN};
27-constexpr MicroAPI::CastTrait castTraitF322F16 = {
28- MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::NO_SAT, MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};
29- 
30-constexpr uint32_t ERF_C0 = 0x3F8060FE;
31-constexpr uint32_t ERF_P1[] = {
32- 0x38EB4C3A, 0xBAAE005B, 0x3C09919F, 0xBD24D99A,
33- 0x3E235519, 0x3F69B4F9, 0x3F210A14
34-};
35-constexpr uint32_t ERF_P2[] = {
36- 0x38B1E96A, 0xBA574D20, 0x3BAAD5EA, 0xBCDC1BE7,
37- 0x3DE718AF, 0xBEC093AC, 0x3E0375D3
38-};
39- 
40-// Clip x to [-3.92, 3.92]
41-__simd_callee__ inline void ErfClip(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg, MicroAPI::MaskReg& mask)
42-{
43- constexpr float ERF_BOUNDARY_MAX = 3.92;
44- MicroAPI::Mins(dstReg, srcReg, ERF_BOUNDARY_MAX, mask);
45- MicroAPI::Maxs(dstReg, dstReg, -ERF_BOUNDARY_MAX, mask);
46-}
47- 
48-// P(x) = (((((0.053443748819x^2+0.75517016694e1)x^2+0.10162808918e3)x^2
49-// +0.13938061484e4)x^2+0.50637915060e4)x^2+0.29639384698e5)x
50-__simd_callee__ inline void ErfComputeP(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg, MicroAPI::MaskReg& mask)
51-{
52- constexpr float SCALAR_P0 = 0.29639384698e5;
53- constexpr float SCALAR_P1 = 0.50637915060e4;
54- constexpr float SCALAR_P2 = 0.13938061484e4;
55- constexpr float SCALAR_P3 = 0.10162808918e3;
56- constexpr float SCALAR_P4 = 0.75517016694e1;
57- constexpr float SCALAR_P5 = 0.053443748819;
58- 
59- MicroAPI::RegTensor<float> tmpReg;
60- MicroAPI::Mul(tmpReg, srcReg, srcReg, mask);
61- MicroAPI::Muls(dstReg, tmpReg, SCALAR_P5, mask);
62- MicroAPI::Adds(dstReg, dstReg, SCALAR_P4, mask);
63- MicroAPI::Mul(dstReg, dstReg, tmpReg, mask);
64- MicroAPI::Adds(dstReg, dstReg, SCALAR_P3, mask);
65- MicroAPI::Mul(dstReg, dstReg, tmpReg, mask);
66- MicroAPI::Adds(dstReg, dstReg, SCALAR_P2, mask);
67- MicroAPI::Mul(dstReg, dstReg, tmpReg, mask);
68- MicroAPI::Adds(dstReg, dstReg, SCALAR_P1, mask);
69- MicroAPI::Mul(dstReg, dstReg, tmpReg, mask);
70- MicroAPI::Adds(dstReg, dstReg, SCALAR_P0, mask);
71- MicroAPI::Mul(dstReg, dstReg, srcReg, mask);
72-}
73- 
74-// Q(x) = ((((x^2+0.31212858877e2)x^2+0.39856963806e3)x^2+0.30231248150e4)x^2+0.13243365831e5)x^2+0.26267224157e5
75-__simd_callee__ inline void ErfComputeQ(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg,
76- MicroAPI::MaskReg& mask)
77-{
78- constexpr float SCALAR_Q0 = 0.26267224157e5;
79- constexpr float SCALAR_Q1 = 0.13243365831e5;
80- constexpr float SCALAR_Q2 = 0.30231248150e4;
81- constexpr float SCALAR_Q3 = 0.39856963806e3;
82- constexpr float SCALAR_Q4 = 0.31212858877e2;
83- 
84- MicroAPI::RegTensor<float> tmpReg;
85- MicroAPI::Mul(tmpReg, srcReg, srcReg, mask);
86- MicroAPI::Adds(dstReg, tmpReg, SCALAR_Q4, mask);
87- MicroAPI::Mul(dstReg, dstReg, tmpReg, mask);
88- MicroAPI::Adds(dstReg, dstReg, SCALAR_Q3, mask);
89- MicroAPI::Mul(dstReg, dstReg, tmpReg, mask);
90- MicroAPI::Adds(dstReg, dstReg, SCALAR_Q2, mask);
91- MicroAPI::Mul(dstReg, dstReg, tmpReg, mask);
92- MicroAPI::Adds(dstReg, dstReg, SCALAR_Q1, mask);
93- MicroAPI::Mul(dstReg, dstReg, tmpReg, mask);
94- MicroAPI::Adds(dstReg, dstReg, SCALAR_Q0, mask);
95-}
96- 
97-__simd_callee__ inline void ErfPadeCompute(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg,
98- MicroAPI::MaskReg& mask)
99-{
100- // x = Clip(x), Erf(x) = P(x) / Q(x)
101- MicroAPI::RegTensor<float> tmpReg;
102- ErfClip(dstReg, srcReg, mask);
103- ErfComputeP(tmpReg, dstReg, mask);
104- ErfComputeQ(dstReg, dstReg, mask);
105- 
106- MicroAPI::Div(dstReg, tmpReg, dstReg, mask);
107-}
108- 
109-__simd_callee__ inline void FMaf(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg1,
110- MicroAPI::RegTensor<float>& srcReg2, MicroAPI::RegTensor<float>& srcReg3, MicroAPI::MaskReg& mask)
111-{
112- MicroAPI::RegTensor<float> tmpReg = srcReg1;
113- MicroAPI::FusedMulDstAdd(tmpReg, srcReg2, srcReg3, mask);
114- dstReg = tmpReg;
115-}
116- 
117-__simd_callee__ inline void ErfSpecialCaseCompute(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg,
118- MicroAPI::RegTensor<float>& tmpReg, MicroAPI::MaskReg& mask)
119-{
120- /*
121- * if (f5 < int32_as_float(0x3F8060FE)) {
122- * *y = f26;
123- * } else {
124- * float f23 = exp(f26 * log(2.0));
125- * float f25 = int32_as_float(0x3F800000) - f23;
126- * unsigned int r3 = float_as_int32(f4) & 0x80000000;
127- * unsigned int r4 = r3 | float_as_int32(f25);
128- * *y = int32_as_float(r4);
129- * }
130- */
131- constexpr uint32_t ERF_R0 = 0x3F8060FE;
132- constexpr uint32_t ERF_R1 = 0x3F800000;
133- constexpr uint32_t ERF_R2 = 0x80000000;
134- constexpr float LOG2_VAULE = 2.0f;
135- 
136- MicroAPI::RegTensor<float> tmpF5Reg, tmpF32Reg, tmpF32Reg1;
137- MicroAPI::RegTensor<uint32_t> tmpU32Reg;
138- MicroAPI::MaskReg cmpMask;
139- MicroAPI::Abs(tmpF5Reg, srcReg, mask);
140- MicroAPI::Duplicate(tmpU32Reg, ERF_R0, mask);
141- MicroAPI::Compare<float, CMPMODE::LT>(cmpMask, tmpF5Reg, (MicroAPI::RegTensor<float> &)tmpU32Reg, mask);
142- 
143- MicroAPI::Duplicate(tmpF32Reg, LOG2_VAULE, mask);
144- MicroAPI::Log(tmpF32Reg, tmpF32Reg, mask);
145- MicroAPI::Mul(tmpF32Reg, tmpReg, tmpF32Reg, mask);
146- MicroAPI::Exp(tmpF32Reg, tmpF32Reg, mask); // tmpF32Reg: f23
147- MicroAPI::Duplicate(tmpU32Reg, ERF_R1, mask);
148- MicroAPI::Sub(tmpF32Reg1, (MicroAPI::RegTensor<float> &)tmpU32Reg, tmpF32Reg, mask); //tmpF32Reg1: f25
149- 
150- MicroAPI::Duplicate(tmpU32Reg, ERF_R2, mask);
151- MicroAPI::And(tmpU32Reg, (MicroAPI::RegTensor<uint32_t> &)srcReg, tmpU32Reg, mask);
152- MicroAPI::Or(tmpU32Reg, tmpU32Reg, (MicroAPI::RegTensor<uint32_t> &)tmpF32Reg1, mask); // tmpU32Reg: r4
153-
154- MicroAPI::Select(dstReg, tmpReg, (MicroAPI::RegTensor<float> &)tmpU32Reg, cmpMask);
155-}
156- 
157-__simd_callee__ inline void ErfSubsectionCompute(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg,
158- MicroAPI::MaskReg& mask)
159-{
160- MicroAPI::RegTensor<float> tmpF5Reg, tmpF32Reg, tmpF32Reg1, tmpF32Reg2;
161- MicroAPI::RegTensor<uint32_t> tmpU32Reg, tmpU32Reg1;
162- MicroAPI::MaskReg cmpMask;
163- /*
164- * float f4 = x;
165- * float f5 = fabsf(x);
166- * bool p1 = f5 < int32_as_float(0x3F8060FE);
167- * bool p2 = f5 >= int32_as_float(0x3F8060FE);
168- */
169- MicroAPI::Abs(tmpF5Reg, srcReg, mask);
170- MicroAPI::Duplicate(tmpU32Reg, ERF_C0, mask);
171- MicroAPI::Compare<float, CMPMODE::GE>(cmpMask, tmpF5Reg, (MicroAPI::RegTensor<float> &)tmpU32Reg, mask);
172- /*
173- * float f6 = f4 * f4;
174- * float f7 = p2 ? f5 : f6;
175- * float f8 = p2 ? int32_as_float(0x38EB4C3A) : int32_as_float(0x28B1E96A);
176- * float f9 = p2 ? int32_as_float(0xBAAE005B) : int32_as_float(0xBA574D20);
177- * float f10 = fmaf(f8, f7, f9);
178- */
179- MicroAPI::Mul(tmpF32Reg1, srcReg, srcReg, mask);
180- MicroAPI::Select(tmpF32Reg, tmpF5Reg, tmpF32Reg1, cmpMask); // tmpF32Reg: f7
181- MicroAPI::Duplicate(tmpU32Reg, ERF_P1[0], mask);
182- MicroAPI::Duplicate(tmpU32Reg1, ERF_P2[0], mask);
183- MicroAPI::Select(tmpF32Reg1,
184- (MicroAPI::RegTensor<float> &)tmpU32Reg, (MicroAPI::RegTensor<float> &)tmpU32Reg1, cmpMask);
185- MicroAPI::Duplicate(tmpU32Reg, ERF_P1[1], mask);
186- MicroAPI::Duplicate(tmpU32Reg1, ERF_P2[1], mask);
187- MicroAPI::Select(tmpF32Reg2,
188- (MicroAPI::RegTensor<float> &)tmpU32Reg, (MicroAPI::RegTensor<float> &)tmpU32Reg1, cmpMask);
189- FMaf(tmpF32Reg1, tmpF32Reg1, tmpF32Reg, tmpF32Reg2, mask); // tmpF32Reg1: f10
190- /*
191- * float f11 = p2 ? int32_as_float(0x3C09919F) : int32_as_float(0x3BAAD5EA);
192- * float f12 = fmaf(f10, f7, f11);
193- */
194- MicroAPI::Duplicate(tmpU32Reg, ERF_P1[2], mask); // int32_as_float(0x3C09919F)
195- MicroAPI::Duplicate(tmpU32Reg1, ERF_P2[2], mask); // int32_as_float(0x3BAAD5EA)
196- MicroAPI::Select(tmpF32Reg2,
197- (MicroAPI::RegTensor<float> &)tmpU32Reg, (MicroAPI::RegTensor<float> &)tmpU32Reg1, cmpMask); // tmpF32Reg2: f11
198- FMaf(tmpF32Reg1, tmpF32Reg1, tmpF32Reg, tmpF32Reg2, mask); // tmpF32Reg1: f12
199- /*
200- * float f13 = p2 ? int32_as_float(0xBD24D99A) : int32_as_float(0xBCDC1BE7);
201- * float f14 = fmaf(f12, f7, f13);
202- */
203- MicroAPI::Duplicate(tmpU32Reg, ERF_P1[3], mask); // int32_as_float(0xBD24D99A)
204- MicroAPI::Duplicate(tmpU32Reg1, ERF_P2[3], mask); // int32_as_float(0xBCDC1BE7)
205- MicroAPI::Select(tmpF32Reg2,
206- (MicroAPI::RegTensor<float> &)tmpU32Reg, (MicroAPI::RegTensor<float> &)tmpU32Reg1, cmpMask); // tmpF32Reg2: f13
207- FMaf(tmpF32Reg1, tmpF32Reg1, tmpF32Reg, tmpF32Reg2, mask); // tmpF32Reg1: f14
208- /*
209- * float f15 = p2 ? int32_as_float(0x3E235519) : int32_as_float(0x3DE718AF);
210- * float f16 = fmaf(f14, f7, f15);
211- */
212- MicroAPI::Duplicate(tmpU32Reg, ERF_P1[4], mask); // int32_as_float(0x3E235519)
213- MicroAPI::Duplicate(tmpU32Reg1, ERF_P2[4], mask); // int32_as_float(0x3DE718AF)
214- MicroAPI::Select(tmpF32Reg2,
215- (MicroAPI::RegTensor<float> &)tmpU32Reg, (MicroAPI::RegTensor<float> &)tmpU32Reg1, cmpMask); // tmpF32Reg2: f13
216- FMaf(tmpF32Reg1, tmpF32Reg1, tmpF32Reg, tmpF32Reg2, mask); // tmpF32Reg1: f16
217- /*
218- * float f17 = p2 ? int32_as_float(0x3F69B4F9) : int32_as_float(0xBEC093AC);
219- * float f18 = fmaf(f16, f7, f17);
220- */
221- MicroAPI::Duplicate(tmpU32Reg, ERF_P1[5], mask); // int32_as_float(0x3F69B4F9)
222- MicroAPI::Duplicate(tmpU32Reg1, ERF_P2[5], mask); // int32_as_float(0xBEC093AC)
223- MicroAPI::Select(tmpF32Reg2,
224- (MicroAPI::RegTensor<float> &)tmpU32Reg, (MicroAPI::RegTensor<float> &)tmpU32Reg1, cmpMask); // tmpF32Reg2: f13
225- FMaf(tmpF32Reg1, tmpF32Reg1, tmpF32Reg, tmpF32Reg2, mask); // tmpF32Reg1: f18
226- /*
227- * float f19 = p2 ? int32_as_float(0x3F210A14) : int32_as_float(0x3E0375D3);
228- * float f20 = fmaf(f18, f7, f19);
229- */
230- MicroAPI::Duplicate(tmpU32Reg, ERF_P1[6], mask); // int32_as_float(0x3F210A14)
231- MicroAPI::Duplicate(tmpU32Reg1, ERF_P2[6], mask); // int32_as_float(0x3E0375D3)
232- MicroAPI::Select(tmpF32Reg2,
233- (MicroAPI::RegTensor<float> &)tmpU32Reg, (MicroAPI::RegTensor<float> &)tmpU32Reg1, cmpMask); // tmpF32Reg2: f19
234- FMaf(tmpF32Reg1, tmpF32Reg1, tmpF32Reg, tmpF32Reg2, mask); // tmpF32Reg1: f20
235- /*
236- * float f21 = -f5;
237- * float f22 = p2 ? f21 : f4
238- * float f26 = fmaf(f20, f22, f22);
239- */
240- MicroAPI::Neg(tmpF32Reg, tmpF5Reg, mask);
241- MicroAPI::Select(tmpF32Reg2, tmpF32Reg, srcReg, cmpMask);
242- FMaf(tmpF32Reg1, tmpF32Reg1, tmpF32Reg2, tmpF32Reg2, mask); // tmpF32Reg1: f26
243- ErfSpecialCaseCompute(dstReg, srcReg, tmpF32Reg1, mask);
244-}
245- 
246-template <typename T, bool isReuseSource = false, const ErfConfig &config = defaultErfConfig>
247-__simd_vf__ inline void ErfCoreImpl(__ubuf__ T* dstUb, __ubuf__ T* srcUb, uint32_t calCount, uint16_t repeatTimes)
248-{
249- MicroAPI::MaskReg mask;
250- MicroAPI::RegTensor<T> srcReg;
251- MicroAPI::RegTensor<float> castReg;
252- MicroAPI::RegTensor<float> tmpReg;
253- MicroAPI::RegTensor<float> dstReg;
254- 
255- for (uint16_t i = 0; i < repeatTimes; ++i) {
256- mask = MicroAPI::UpdateMask<float>(calCount);
257- if constexpr (sizeof(T) == sizeof(half)) {
258- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcReg, srcUb + i * B32_DATA_NUM_PER_REPEAT);
259- MicroAPI::Cast<float, T, castTraitF162F32>(castReg, srcReg, mask);
260- } else {
261- MicroAPI::DataCopy(castReg, srcUb + i * B32_DATA_NUM_PER_REPEAT);
262- }
263- if constexpr (config.algo == ErfAlgo::PADE_APPROXIMATION) {
264- ErfPadeCompute(dstReg, castReg, mask);
265- } else {
266- ErfSubsectionCompute(dstReg, castReg, mask);
267- }
268- if constexpr (sizeof(T) == sizeof(half)) {
269- MicroAPI::Cast<T, float, castTraitF322F16>(srcReg, dstReg, mask);
270- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(dstUb + i * B32_DATA_NUM_PER_REPEAT, srcReg, mask);
271- } else {
272- MicroAPI::DataCopy(dstUb + i * B32_DATA_NUM_PER_REPEAT, dstReg, mask);
273- }
274- }
275-}
276-} // namespace ErfAPI
277- 
278-template <typename T, bool isReuseSource = false, const ErfConfig &config = defaultErfConfig>
279-__aicore__ inline void ErfCheckParams(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
280- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
281-{
282- static_assert(SupportType<T, half, float>(), "current data type is not supported on current device!");
283- CheckTensorPos<T>(dstTensor, Hardware::UB, "dstTensor", "VECIN / VECCALC / VECOUT", "Erf");
284- CheckTensorPos<T>(srcTensor, Hardware::UB, "srcTensor", "VECIN / VECCALC / VECOUT", "Erf");
285- CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN / VECCALC / VECOUT", "Erf");
286- CheckCalCount(calCount, "calCount", srcTensor, "srcTensor", "Erf");
287- CheckCalCount(calCount, "calCount", dstTensor, "dstTensor", "Erf");
288-}
289- 
290-template <typename T, bool isReuseSource = false, const ErfConfig &config = defaultErfConfig>
291-__aicore__ inline void ErfImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
292- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
293-{
294- // Only for AI Vector Core.
295- if ASCEND_IS_AIC {
296- return;
297- }
298- 
299- ErfCheckParams<T, isReuseSource, config>(dstTensor, srcTensor, sharedTmpBuffer, calCount);
300- __ubuf__ T *dstUb = (__ubuf__ T *)dstTensor.GetPhyAddr();
301- __ubuf__ T *srcUb = (__ubuf__ T *)srcTensor.GetPhyAddr();
302- uint16_t repeatTimes = CeilDivision(calCount, B32_DATA_NUM_PER_REPEAT);
303- ErfAPI::ErfCoreImpl<T, isReuseSource, config>(dstUb, srcUb, calCount, repeatTimes);
304-}
305- 
306-template <typename T, bool isReuseSource = false, const ErfConfig &config = defaultErfConfig>
307-__aicore__ inline void ErfImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
308- const uint32_t calCount)
309-{
310- // Only for AI Vector Core.
311- if ASCEND_IS_AIC {
312- return;
313- }
314- 
315- // Using the Stack Space to Allocate tmpBuffer
316- LocalTensor<uint8_t> sharedTmpBuffer;
317- bool ans = PopStackBuffer<uint8_t, TPosition::LCM>(sharedTmpBuffer);
318- ASCENDC_ASSERT((ans), { KERNEL_LOG(KERNEL_ERROR, "PopStackBuffer Error!"); });
319- ErfImpl<T, isReuseSource, config>(dstTensor, srcTensor, sharedTmpBuffer, calCount);
320-}
321- 
322-/*!
323- * \ingroup Erf
324- * \brief compute Erf elementwisely
325- * \tparam T: half/float
326- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason
327- * \tparam defaultErfConfig: the config for Erf, choose different algo
328- * \param [out] dstTensor: output LocalTensor
329- * \param [in] srcTensor: input LocalTensor
330- * \param [in] sharedTmpBuffer: extra temporary shared space used for intermediate values among calculation process,
331- * whose required space size should refer to corresponding tiling API, which is defined at erf_tiling.h.
332- * Generally, the more space you allocate, the better performance you will achieve, and the performance
333- * reaches peak when buffer size is maximum(calculated by tiling function). Moreover, it is not guaranteed
334- * that the shared space will be cleared after usage, the data could be anything.
335- * \param [in] calCount: the number of elements to be processed.
336- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
337- */
338-template <typename T, bool isReuseSource = false, const ErfConfig& config = defaultErfConfig>
339-__aicore__ inline void Erf(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
340- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
341-{
342- ErfImpl<T, isReuseSource, config>(dstTensor, srcTensor, sharedTmpBuffer, calCount);
343-}
344- 
345-/*!
346- * \ingroup Erf
347- * \brief compute Erf elementwisely for whole source tensor
348- * \tparam T: half/float
349- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason
350- * \tparam defaultErfConfig: the config for Erf, choose different algo
351- * \param [out] dstTensor: output LocalTensor
352- * \param [in] srcTensor: input LocalTensor
353- * \param [in] sharedTmpBuffer: extra temporary shared space used for intermediate values among calculation process,
354- * whose required space size should refer to corresponding tiling API, which is defined at erf_tiling.h.
355- * Generally, the more space you allocate, the better performance you will achieve, and the performance
356- * reaches peak when buffer size is maximum(calculated by tiling function). Moreover, it is not guaranteed
357- * that the shared space will be cleared after usage, the data could be anything.
358- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
359- */
360-template <typename T, bool isReuseSource = false, const ErfConfig& config = defaultErfConfig>
361-__aicore__ inline void Erf(
362- const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor, const LocalTensor<uint8_t>& sharedTmpBuffer)
363-{
364- Erf<T, isReuseSource, config>(dstTensor, srcTensor, sharedTmpBuffer, srcTensor.GetSize());
365-}
366- 
367-/*!
368- * \ingroup Erf
369- * \brief compute Erf elementwisely
370- * \tparam T: half/float
371- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason
372- * \tparam defaultErfConfig: the config for Erf, choose different algo
373- * \param [out] dstTensor: output LocalTensor
374- * \param [in] srcTensor: input LocalTensor
375- * \param [in] calCount: the number of elements to be processed.
376- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
377- */
378-template <typename T, bool isReuseSource = false, const ErfConfig& config = defaultErfConfig>
379-__aicore__ inline void Erf(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor, const uint32_t calCount)
380-{
381- ErfImpl<T, isReuseSource, config>(dstTensor, srcTensor, calCount);
382-}
383- 
384-/*!
385- * \ingroup Erf
386- * \brief compute Erf elementwisely for whole source tensor
387- * \tparam T: half/float
388- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason
389- * \tparam defaultErfConfig: the config for Erf, choose different algo
390- * \param [out] dstTensor: output LocalTensor
391- * \param [in] srcTensor: input LocalTensor
392- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
393- */
394-template <typename T, bool isReuseSource = false, const ErfConfig& config = defaultErfConfig>
395-__aicore__ inline void Erf(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor)
396-{
397- Erf<T, isReuseSource, config>(dstTensor, srcTensor, srcTensor.GetSize());
398-}
399-} // namespace AscendC
400- 
401-#endif // IMPL_MATH_ERF_ERF_C310_IMPL_H
Dimpl/adv_api/detail/math/erfc/erfc_l300_impl.h+0-187
@@ -1,187 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/*!
12- * \file erfc_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_MATH_ERFC_ERFC_L300_IMPL_H
16-#define IMPL_MATH_ERFC_ERFC_L300_IMPL_H
17- 
18-#include "kernel_tensor.h"
19- 
20-namespace AscendC {
21-namespace ERFC {
22- 
23-constexpr MicroAPI::CastTrait castTraitF162F32 = {
24- MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::UNKNOWN, MicroAPI::MaskMergeMode::ZEROING, RoundMode::UNKNOWN};
25-constexpr MicroAPI::CastTrait castTraitF322F16 = {
26- MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::NO_SAT, MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};
27- 
28-__simd_callee__ inline void MulAdds(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg1,
29- MicroAPI::RegTensor<float>& srcReg2, const float scalarValue, MicroAPI::MaskReg& mask)
30-{
31- // dst = src1 * src2 + scalerValue
32- MicroAPI::RegTensor<float> tmpReg;
33- MicroAPI::Mul(tmpReg, srcReg1, srcReg2, mask);
34- MicroAPI::Adds(dstReg, tmpReg, scalarValue, mask);
35-}
36-
37-// compute Erfc with xa = |x| + fp32_min
38-__simd_callee__ inline void ErfcPreCompute(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg, MicroAPI::MaskReg mask)
39-{
40- constexpr float SCALAR_ERFC_FP32_MIN = 2.168404344971009e-19; // 2^-62
41- MicroAPI::RegTensor<float> tmpReg;
42- MicroAPI::Abs(tmpReg, srcReg, mask);
43- MicroAPI::Adds(dstReg, tmpReg, SCALAR_ERFC_FP32_MIN, mask);
44-}
45- 
46-// compute Erfc R(z) = ((((((((z*r0 + r1)*z + r2)*z + r3)*z + r4)*z + r5)*z + r6)*z + r7)*z + r8)
47-__simd_callee__ inline void ErfcComputeR(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg, MicroAPI::MaskReg mask)
48-{
49- // Specific value used for approximate calculation.
50- constexpr float R0 = 0.1735313680e-7;
51- constexpr float R1 = -0.9856738394e-6;
52- constexpr float R2 = 0.2517003236e-4;
53- constexpr float R3 = -0.3848015171e-3;
54- constexpr float R4 = 0.5681528564e0;
55- constexpr float R5 = 0.5245623129e1;
56- constexpr float R6 = 0.2107740710e2;
57- constexpr float R7 = 0.4212761755e2;
58- constexpr float R8 = 0.4380524149e2;
59- 
60- MicroAPI::RegTensor<float> rReg;
61- MicroAPI::Duplicate(rReg, R0, mask);
62- MulAdds(dstReg, srcReg, rReg, R1, mask);
63- MulAdds(dstReg, dstReg, srcReg, R2, mask);
64- MulAdds(dstReg, dstReg, srcReg, R3, mask);
65- MulAdds(dstReg, dstReg, srcReg, R4, mask);
66- MulAdds(dstReg, dstReg, srcReg, R5, mask);
67- MulAdds(dstReg, dstReg, srcReg, R6, mask);
68- MulAdds(dstReg, dstReg, srcReg, R7, mask);
69- MulAdds(dstReg, dstReg, srcReg, R8, mask);
70-}
71- 
72-// compute Erfc S(z) = (((((z + s1)*z + s2)*z + s3)*z + s4)*z + s5)
73-__simd_callee__ inline void ErfcComputeS(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg, MicroAPI::MaskReg mask)
74-{
75- // Specific value used for approximate calculation.
76- constexpr float S1 = 0.9349684299e1;
77- constexpr float S2 = 0.3756930664e2;
78- constexpr float S3 = 0.8058268949e2;
79- constexpr float S4 = 0.9155653738e2;
80- constexpr float S5 = 0.4380524152e2;
81- 
82- MicroAPI::Adds(dstReg, srcReg, S1, mask);
83- MulAdds(dstReg, dstReg, srcReg, S2, mask);
84- MulAdds(dstReg, dstReg, srcReg, S3, mask);
85- MulAdds(dstReg, dstReg, srcReg, S4, mask);
86- MulAdds(dstReg, dstReg, srcReg, S5, mask);
87-}
88- 
89-__simd_callee__ inline void ErfcClip(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg, MicroAPI::MaskReg mask)
90-{
91- constexpr float ERFC_BOUNDARY_MAX = 10.0f;
92- MicroAPI::Mins(dstReg, srcReg, ERFC_BOUNDARY_MAX, mask);
93- MicroAPI::Maxs(dstReg, dstReg, -ERFC_BOUNDARY_MAX, mask);
94-}
95- 
96-// Compute Erfc: exp(-xa^2) * (R(z) / S(z)) * xb + (1 - xb)
97-__simd_callee__ inline void ErfcPublicSteps(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg, MicroAPI::MaskReg mask)
98-{
99- constexpr float MIN_BOUNDARY = 10.0f;
100- // Compute xa = |x| + min_pf32, exp(-xa^2)
101- MicroAPI::RegTensor<float> xaReg, xbReg, zReg, rzReg, szReg;
102- MicroAPI::RegTensor<float> oneReg, tmpReg;
103- ErfcPreCompute(xaReg, srcReg, mask);
104- MicroAPI::Mul(tmpReg, xaReg, xaReg, mask);
105- MicroAPI::Neg(tmpReg, tmpReg, mask);
106- MicroAPI::Exp(dstReg, tmpReg, mask);
107- // Compute z = min(xa, 10), xb = x / xa, exp(-xa^2) * xb
108- MicroAPI::Mins(zReg, xaReg, MIN_BOUNDARY, mask);
109- MicroAPI::Div(xbReg, srcReg, xaReg, mask);
110- MicroAPI::Mul(dstReg, dstReg, xbReg, mask);
111- // Compute exp(-xa^2) * xb * (R(z) / S(z))
112- ErfcComputeR(rzReg, zReg, mask);
113- ErfcComputeS(szReg, zReg, mask);
114- MicroAPI::Mul(dstReg, dstReg, rzReg, mask);
115- MicroAPI::Div(dstReg, dstReg, szReg, mask);
116- 
117- MicroAPI::Duplicate(oneReg, 1.0f, mask);
118- MicroAPI::Sub(tmpReg, oneReg, xbReg, mask);
119- MicroAPI::Add(dstReg, dstReg, tmpReg, mask);
120-}
121- 
122-template<typename T>
123-__simd_vf__ inline void ErfcCoreImpl(__ubuf__ T* dstUb, __ubuf__ T* srcUb, uint32_t calCount, uint16_t repeatTimes)
124-{
125- MicroAPI::RegTensor<T> srcReg;
126- MicroAPI::RegTensor<float> castReg;
127- MicroAPI::RegTensor<float> tmpReg;
128- MicroAPI::RegTensor<float> dstReg;
129- 
130- for (uint16_t i = 0; i < repeatTimes; ++i) {
131- MicroAPI::MaskReg mask = MicroAPI::UpdateMask<float>(calCount);
132- if constexpr (sizeof(T) == sizeof(half)) {
133- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcReg, srcUb + i * B32_DATA_NUM_PER_REPEAT);
134- MicroAPI::Cast<float, T, castTraitF162F32>(castReg, srcReg, mask);
135- } else {
136- MicroAPI::DataCopy(castReg, srcUb + i * B32_DATA_NUM_PER_REPEAT);
137- }
138- ErfcClip(tmpReg, castReg, mask);
139- ErfcPublicSteps(dstReg, tmpReg, mask);
140- if constexpr (sizeof(T) == sizeof(half)) {
141- MicroAPI::Cast<T, float, castTraitF322F16>(srcReg, dstReg, mask);
142- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(dstUb + i * B32_DATA_NUM_PER_REPEAT, srcReg, mask);
143- } else {
144- MicroAPI::DataCopy(dstUb + i * B32_DATA_NUM_PER_REPEAT, dstReg, mask);
145- }
146- }
147-}
148- 
149-} // namespace ERFC
150- 
151-template <typename T, bool isReuseSource = false>
152-__aicore__ inline void ErfcImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
153- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
154-{
155- CheckTensorPos(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN / VECOUT / VECCALC", "Erfc");
156- ErfcImpl(dstTensor, srcTensor, calCount);
157-}
158- 
159-template <typename T, bool isReuseSource = false>
160-__aicore__ inline void ErfcImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
161- const uint32_t calCount)
162-{
163- // Only for AI Vector Core.
164- if ASCEND_IS_AIC {
165- return;
166- }
167- 
168- static_assert(SupportType<T, half, float>(), "current data type is not supported on current device!");
169- CheckTensorPos<T>(dstTensor, Hardware::UB, "dstTensor", "VECIN / VECCALC / VECOUT", "ERFC");
170- CheckTensorPos<T>(srcTensor, Hardware::UB, "srcTensor", "VECIN / VECCALC / VECOUT", "ERFC");
171- ASCENDC_ASSERT((calCount <= srcTensor.GetSize()), {
172- KERNEL_LOG(KERNEL_ERROR, "calCount is %u, which should not be larger than srcTensor length %u", calCount,
173- srcTensor.GetSize());
174- });
175- ASCENDC_ASSERT((calCount <= dstTensor.GetSize()), {
176- KERNEL_LOG(KERNEL_ERROR, "calCount is %u, which should not be larger than dstTensor length %u", calCount,
177- dstTensor.GetSize());
178- });
179- 
180- __ubuf__ T *dstUb = (__ubuf__ T *)dstTensor.GetPhyAddr();
181- __ubuf__ T *srcUb = (__ubuf__ T *)srcTensor.GetPhyAddr();
182- uint16_t repeatTimes = CeilDivision(calCount, B32_DATA_NUM_PER_REPEAT);
183- ERFC::ErfcCoreImpl<T>(dstUb, srcUb, calCount, repeatTimes);
184-}
185-} // namespace AscendC
186- 
187-#endif // IMPL_MATH_ERFC_ERFC_C310_IMPL_H
Dimpl/adv_api/detail/math/floor/floor_l300_impl.h+0-43
@@ -1,43 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/*!
12- * \file floor_l300_impl.h
13- * \brief
14- */
15-#ifndef DETAIL_MATH_FLOOR_FLOOR_L300_IMPL_H
16-#define DETAIL_MATH_FLOOR_FLOOR_L300_IMPL_H
17-#include "kernel_tensor.h"
18-#include "kernel_tiling/kernel_tiling.h"
19-#include "../../common/check.h"
20-
21-namespace AscendC {
22- 
23-template<typename T, bool isReuseSource = false>
24-__aicore__ inline void FloorImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
25- const uint32_t calCount)
26-{
27- CheckTensorPosition(srcTensor, "srcTensor", "VECIN, VECOUT, VECCALC");
28- CheckTensorPosition(dstTensor, "dstTensor", "VECIN, VECOUT, VECCALC");
29- CheckCalCount(calCount, "calCount", dstTensor, "dstTensor", "Floor");
30- CheckCalCount(calCount, "calCount", srcTensor, "srcTensor", "Floor");
31- Truncate<T, RoundMode::CAST_FLOOR>(dstTensor, srcTensor, calCount);
32-}
33- 
34-template <typename T, bool isReuseSource = false>
35-__aicore__ inline void FloorImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
36- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
37-{
38- CheckTensorPosition(sharedTmpBuffer, "sharedTmpBuffer", "VECIN, VECOUT, VECCALC");
39- FloorImpl<T, isReuseSource>(dstTensor, srcTensor, calCount);
40-}
41- 
42-} // namespace AscendC
43-#endif //DETAIL_MATH_FLOOR_FLOOR_L300_IMPL_H
Dimpl/adv_api/detail/math/log/log_l300_impl.h+0-82
@@ -1,82 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/*!
12- * \file log_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_MATH_LOG_LOG_L300_IMPL_H
16-#define IMPL_MATH_LOG_LOG_L300_IMPL_H
17-#include "kernel_tensor.h"
18-#include "kernel_pop_stack_buffer.h"
19-#include "kernel_tiling/kernel_tiling.h"
20- 
21-namespace AscendC {
22-template <typename T> __simd_vf__ inline void LogImpl(__ubuf__ T *dst, __ubuf__ T *src, const uint32_t calCount)
23-{
24- static_assert((std::is_same_v<T, half> || std::is_same_v<T, float>),
25- "current data type is not supported on current device!");
26- constexpr uint32_t sregLower = static_cast<uint32_t>(GetVecLen() / sizeof(T));
27- uint16_t repeatTimes = CeilDivision(calCount, sregLower);
28- MicroAPI::RegTensor<T> vreg0;
29- MicroAPI::RegTensor<T> vreg1;
30- uint32_t sreg = calCount;
31- MicroAPI::MaskReg preg;
32- for (uint16_t i = 0; i < repeatTimes; ++i) {
33- preg = MicroAPI::UpdateMask<T>(sreg);
34- MicroAPI::DataCopy(vreg0, src + i * sregLower);
35- MicroAPI::Log(vreg1, vreg0, preg);
36- MicroAPI::DataCopy(dst + i * sregLower, vreg1, preg);
37- }
38-}
39- 
40-template <typename T>
41-__simd_vf__ inline void LogXImpl(__ubuf__ T *dst, __ubuf__ T *src, const uint32_t calCount, const float LnXRec)
42-{
43- static_assert((std::is_same_v<T, half> || std::is_same_v<T, float>),
44- "current data type is not supported on current device!");
45- constexpr uint32_t sregLower = (uint32_t)(GetVecLen() / sizeof(float));
46- uint16_t repeatTimes = CeilDivision(calCount, sregLower);
47- static constexpr MicroAPI::CastTrait castTraitB16ToB32 = {
48- MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::UNKNOWN, MicroAPI::MaskMergeMode::ZEROING, RoundMode::UNKNOWN};
49- static constexpr MicroAPI::CastTrait castTraitB32ToB16 = {
50- MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::SAT, MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};
51- if constexpr (std::is_same_v<T, float>) {
52- MicroAPI::RegTensor<float> vreg0;
53- MicroAPI::RegTensor<float> vreg1;
54- uint32_t sreg = calCount;
55- MicroAPI::MaskReg mask;
56- for (uint16_t i = 0; i < repeatTimes; ++i) {
57- mask = MicroAPI::UpdateMask<T>(sreg);
58- MicroAPI::DataCopy(vreg0, src + i * sregLower);
59- MicroAPI::Log(vreg1, vreg0, mask);
60- MicroAPI::Muls(vreg1, vreg1, LnXRec, mask);
61- MicroAPI::DataCopy(dst + i * sregLower, vreg1, mask);
62- }
63- } else if constexpr (std::is_same_v<T, half>) {
64- MicroAPI::RegTensor<T> vreg0;
65- MicroAPI::RegTensor<T> dst0;
66- MicroAPI::RegTensor<T> dst1;
67- MicroAPI::RegTensor<float> vreg1;
68- uint32_t sreg = calCount;
69- MicroAPI::MaskReg mask;
70- for (uint16_t i = 0; i < repeatTimes; ++i) {
71- mask = MicroAPI::UpdateMask<float>(sreg);
72- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, src + i * sregLower);
73- MicroAPI::Cast<float, half, castTraitB16ToB32>(vreg1, vreg0, mask);
74- MicroAPI::Log(vreg1, vreg1, mask);
75- MicroAPI::Muls(vreg1, vreg1, LnXRec, mask);
76- MicroAPI::Cast<half, float, castTraitB32ToB16>(dst0, vreg1, mask);
77- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(dst + i * sregLower, dst0, mask);
78- }
79- }
80-}
81-} // namespace AscendC
82-#endif // IMPL_MATH_LOG_LOG_L300_IMPL_H
Dimpl/adv_api/detail/math/power/power_l300_impl.h+0-1484
@@ -1,1484 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/* !
12- * \file power_l300_impl.h
13- * \brief
14- */
15- 
16-#ifndef IMPL_MATH_POWER_POWER_L300_IMPL_H
17-#define IMPL_MATH_POWER_POWER_L300_IMPL_H
18-#include "kernel_tensor.h"
19-#include "kernel_tiling/kernel_tiling.h"
20- 
21-#include "power_common_utils.h"
22- 
23-namespace AscendC {
24-namespace PowerL300Impl {
25- 
26-constexpr MicroAPI::CastTrait castTraitF16F32 = { MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::NO_SAT,
27- MicroAPI::MaskMergeMode::ZEROING };
28-constexpr MicroAPI::CastTrait castTraitF32F16 = { MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::NO_SAT,
29- MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_RINT };
30-constexpr MicroAPI::CastTrait castTraitF32I32 = { MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::NO_SAT,
31- MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_ROUND };
32-constexpr MicroAPI::CastTrait castTraitI32F32 = { MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::NO_SAT,
33- MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_TRUNC };
34-constexpr MicroAPI::CastTrait castTraitI8I16 = { MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::UNKNOWN,
35- MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_ROUND };
36-constexpr MicroAPI::CastTrait castTraitI16I8 = { MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::NO_SAT,
37- MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_ROUND };
38- 
39- 
40-namespace PowF {
41-constexpr float LOG2_LOWEST_VALUE = 1.175494351e-38f;
42-constexpr float LOG2_LOWEST_VALUE_MULS = 8388608.0f;
43- 
44-constexpr float LOG2_REDUCE_COEFF1 = 0.70710678f;
45-constexpr int32_t LOG2_REDUCE_COEFF2 = 0xff800000;
46-constexpr float LOG2_REDUCE_FMAF_COEFF1 = 1.19209290e-7f;
47- 
48-constexpr float LOG2_BEST_FMAF_COEFF1 = 0.37282535f;
49-constexpr float LOG2_BEST_FMAF_COEFF2 = 0.4097556f;
50-constexpr float LOG2_BEST_FMAF_COEFF3 = 0.57711965f;
51-constexpr float LOG2_BEST_FMAF_COEFF4 = 0.96179646f;
52-constexpr float LOG2_BEST_FMAF_COEFF5 = 2.88539f;
53-constexpr float LOG2_BEST_FMAF_COEFF6 = 0.00000003851926f;
54- 
55-constexpr float EXPF_BEST_FMAF_COEFF1 = 0.000152392517f;
56-constexpr float EXPF_BEST_FMAF_COEFF2 = 0.00133913534f;
57-constexpr float EXPF_BEST_FMAF_COEFF3 = 0.00961883925f;
58-constexpr float EXPF_BEST_FMAF_COEFF4 = 0.0555035882f;
59-constexpr float EXPF_BEST_FMAF_COEFF5 = 0.240226448f;
60-constexpr float EXPF_BEST_FMAF_COEFF6 = 0.693147182f;
61- 
62- 
63-constexpr int32_t EXPF_INTERVAL_CMP = -2097152000;
64-constexpr int32_t EXPF_INTERVAL_CAST = 2130706432;
65- 
66-constexpr float EXP_OVFL_UNFL_F = 152.0f;
67-constexpr int32_t INF = 0x7f800000;
68-constexpr int32_t NEG_INF = 0xff800000;
69-constexpr int32_t I32_NAN = 0x7f7fffff;
70-constexpr int32_t F32_NAN = 0x7fc00000;
71- 
72-constexpr int32_t R10_COEFF = 0x7F800000;
73-constexpr int32_t R12_COEFF = 0x7FFFFFFF;
74- 
75- 
76-constexpr int16_t COMPARE_ZERO_OFFSET = 31;
77-constexpr int16_t SHITF_OFFSET = 23;
78-constexpr float F32_FRACTIONS = -23.0f;
79- 
80-struct PowerLogParams {
81- MicroAPI::RegTensor<float> zeroReg;
82- MicroAPI::RegTensor<float> oneReg;
83- MicroAPI::RegTensor<float> fractionReg;
84- MicroAPI::RegTensor<float> subReg;
85- MicroAPI::RegTensor<int32_t> intReg;
86- MicroAPI::RegTensor<float> rReg;
87- MicroAPI::RegTensor<float> addReg1;
88- MicroAPI::RegTensor<float> addReg2;
89-};
90- 
91-struct PowerExpParams {
92- MicroAPI::RegTensor<float> zeroReg;
93- MicroAPI::RegTensor<float> oneReg;
94- MicroAPI::RegTensor<float> expReg2;
95- MicroAPI::RegTensor<float> expReg3;
96- MicroAPI::RegTensor<float> expReg4;
97- MicroAPI::RegTensor<float> expReg5;
98- MicroAPI::RegTensor<float> expReg6;
99-};
100- 
101-__simd_callee__ inline void PowerLogParamsInit(PowerLogParams& params) {
102- MicroAPI::Duplicate(params.zeroReg, 0.0f);
103- MicroAPI::Duplicate(params.oneReg, 1.0f);
104- MicroAPI::Duplicate(params.fractionReg, F32_FRACTIONS);
105- MicroAPI::Duplicate(params.subReg, LOG2_REDUCE_COEFF1);
106- MicroAPI::Duplicate(params.intReg, LOG2_REDUCE_COEFF2);
107- MicroAPI::Duplicate(params.rReg, LOG2_BEST_FMAF_COEFF2);
108- MicroAPI::Duplicate(params.addReg1, LOG2_BEST_FMAF_COEFF3);
109- MicroAPI::Duplicate(params.addReg2, LOG2_BEST_FMAF_COEFF4);
110-}
111- 
112-__simd_callee__ inline void PowerExpParamsInit(PowerExpParams& params) {
113- MicroAPI::Duplicate(params.zeroReg, 0.0f);
114- MicroAPI::Duplicate(params.oneReg, 1.0f);
115- MicroAPI::Duplicate(params.expReg2, EXPF_BEST_FMAF_COEFF2);
116- MicroAPI::Duplicate(params.expReg3, EXPF_BEST_FMAF_COEFF3);
117- MicroAPI::Duplicate(params.expReg4, EXPF_BEST_FMAF_COEFF4);
118- MicroAPI::Duplicate(params.expReg5, EXPF_BEST_FMAF_COEFF5);
119- MicroAPI::Duplicate(params.expReg6, EXPF_BEST_FMAF_COEFF6);
120-}
121- 
122-__simd_callee__ inline void IsInfNum(MicroAPI::MaskReg &infMask, MicroAPI::RegTensor<float> &srcReg,
123- MicroAPI::RegTensor<int32_t>& tmpR12Reg, MicroAPI::MaskReg& mask)
124-{
125- MicroAPI::RegTensor<float> tmpFloatReg;
126- MicroAPI::And((MicroAPI::RegTensor<int32_t>&)tmpFloatReg, (MicroAPI::RegTensor<int32_t>&)srcReg, tmpR12Reg, mask);
127- MicroAPI::CompareScalar<int32_t, CMPMODE::EQ>(infMask, (MicroAPI::RegTensor<int32_t>&)tmpFloatReg, INF, mask);
128-}
129- 
130-__simd_callee__ inline void IsNanNum(MicroAPI::MaskReg &nanMask, MicroAPI::RegTensor<float> &srcReg, MicroAPI::MaskReg& mask)
131-{
132- MicroAPI::Compare<float, CMPMODE::NE>(nanMask, srcReg, srcReg, mask);
133-}
134- 
135-__simd_callee__ inline void RFloor(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg, MicroAPI::MaskReg& mask)
136-{
137- MicroAPI::Truncate<float, RoundMode::CAST_FLOOR, MicroAPI::MaskMergeMode::ZEROING>(dstReg, srcReg, mask);
138-}
139- 
140-__aicore__ inline void CompareNegZero(MicroAPI::MaskReg &filterMask, MicroAPI::RegTensor<float>& srcReg, MicroAPI::MaskReg& mask)
141-{
142- MicroAPI::RegTensor<uint32_t> tmpReg;
143- MicroAPI::ShiftRights(tmpReg, (MicroAPI::RegTensor<uint32_t>&)srcReg, COMPARE_ZERO_OFFSET, mask);
144-#if (defined(__NPU_ARCH__) && (__NPU_ARCH__ ==3003 || __NPU_ARCH__ ==3113))
145- MicroAPI::CompareScalar<uint32_t, CMPMODE::EQ>(filterMask, tmpReg, 0, mask);
146-#else
147- MicroAPI::CompareScalar<uint32_t, CMPMODE::EQ>(filterMask, tmpReg, 0.0f, mask);
148-#endif
149-}
150- 
151-__aicore__ inline void CopySignF(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg1,
152- MicroAPI::RegTensor<float>& srcReg2, MicroAPI::MaskReg& mask, MicroAPI::MaskReg& cmpMask1, MicroAPI::MaskReg &cmpMask2)
153-{
154- MicroAPI::RegTensor<float> tmpFloatReg, tmpFloatReg2;
155- MicroAPI::CompareScalar<float, CMPMODE::GE>(cmpMask1, srcReg2, 0.0f, mask);
156- CompareNegZero(cmpMask2, srcReg2, mask);
157- MicroAPI::MaskAnd(cmpMask1, cmpMask1, cmpMask2, mask);
158- MicroAPI::Abs(tmpFloatReg, srcReg1, mask);
159- MicroAPI::Neg(tmpFloatReg2, tmpFloatReg, mask);
160- MicroAPI::Select(dstReg, tmpFloatReg, tmpFloatReg2, cmpMask1);
161-}
162- 
163-__simd_callee__ inline void FMaf(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg1,
164- MicroAPI::RegTensor<float>& srcReg2, MicroAPI::RegTensor<float>& srcReg3, MicroAPI::MaskReg& mask)
165-{
166- MicroAPI::RegTensor<float> tmpReg = srcReg1;
167- MicroAPI::FusedMulDstAdd(tmpReg, srcReg2, srcReg3, mask);
168- dstReg = tmpReg;
169-}
170- 
171-__simd_callee__ inline void FMaf(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg1,
172- float scalarValue, MicroAPI::RegTensor<float>& srcReg2, MicroAPI::MaskReg& mask)
173-{
174- MicroAPI::RegTensor<float> tmpReg;
175- MicroAPI::Duplicate(tmpReg, scalarValue, mask);
176- FMaf(dstReg, srcReg1, tmpReg, srcReg2, mask);
177-}
178- 
179-__simd_callee__ inline void GetLogFExt1(MicroAPI::RegTensor<float>& tmpIReg, MicroAPI::RegTensor<float>& tmpMReg,
180- MicroAPI::RegTensor<float>& srcReg, PowerLogParams& params, MicroAPI::MaskReg& mask)
181-{
182- MicroAPI::RegTensor<float> tmpAReg, tmpFloatReg;
183- MicroAPI::RegTensor<int32_t> tmpEReg;
184- /* init varaiable a and i:
185- * if (a < 1.175494351e-38f){ // 0x1.0p-126
186- * a = a * 8388608.0f; // 0x1.0p+23
187- * i = -23.0f;
188- * }
189- */
190- MicroAPI::MaskReg cmpMask;
191- MicroAPI::CompareScalar<float, CMPMODE::LT>(cmpMask, srcReg, LOG2_LOWEST_VALUE, mask);
192- MicroAPI::Muls(tmpAReg, srcReg, LOG2_LOWEST_VALUE_MULS, mask);
193- MicroAPI::Select(tmpIReg, params.fractionReg, params.zeroReg, cmpMask);
194- // step 1: e = (__float_as_int (a) - __float_as_int (0.70710678f)) & 0xff800000;
195- tmpFloatReg = params.subReg;
196- MicroAPI::Sub(tmpEReg, (MicroAPI::RegTensor<int32_t> &)srcReg, (MicroAPI::RegTensor<int32_t> &)tmpFloatReg, mask);
197- MicroAPI::And(tmpEReg, tmpEReg, params.intReg, mask);
198- // step 2: m = __int_as_float (__float_as_int (a) - e);
199- MicroAPI::Sub((MicroAPI::RegTensor<int32_t> &)tmpMReg, (MicroAPI::RegTensor<int32_t> &)srcReg, tmpEReg, mask);
200- // step 3: i = fmaf ((float)e, 1.19209290e-7f, i);
201- MicroAPI::Cast<float, int32_t, castTraitF32I32>(tmpFloatReg, tmpEReg, mask);
202- MicroAPI::Axpy(tmpIReg, tmpFloatReg, LOG2_REDUCE_FMAF_COEFF1, mask);
203-}
204- 
205-__simd_callee__ inline void GetLogFExt2(MicroAPI::RegTensor<float>& logHigh, MicroAPI::RegTensor<float>& logLow,
206- MicroAPI::RegTensor<float>& tmpIReg, MicroAPI::RegTensor<float>& tmpMReg, PowerLogParams& params, MicroAPI::MaskReg& mask)
207-{
208- // step 4：p = m + 1.0f; m = m - 1.0f;
209- MicroAPI::RegTensor<float> tmpPReg, tmpFloatReg, tmpFloatReg2;
210- MicroAPI::Adds(tmpPReg, tmpMReg, 1.0f, mask);
211- MicroAPI::Adds(tmpMReg, tmpMReg, -1.0f, mask);
212- // step 5：r = 1.0f / p
213- MicroAPI::RegTensor<float> tmpRReg;
214- MicroAPI::Div(tmpRReg, params.oneReg, tmpPReg, mask);
215- // step 6：qhi = m * r;
216- MicroAPI::RegTensor<float> tmpQHIReg, tmpQLOReg;
217- MicroAPI::Mul(tmpQHIReg, tmpMReg, tmpRReg, mask);
218- // step 7：qhi1 = fmaf (qhi, -m, fmaf (qhi, -2.0f, m))
219- MicroAPI::Muls(tmpFloatReg, tmpQHIReg, -2.0f, mask);
220- MicroAPI::Add(tmpFloatReg, tmpFloatReg, tmpMReg, mask);
221- MicroAPI::Neg(tmpFloatReg2, tmpMReg, mask);
222- MicroAPI::Mul(tmpFloatReg2, tmpQHIReg, tmpFloatReg2, mask);
223- MicroAPI::Add(tmpFloatReg, tmpFloatReg2, tmpFloatReg, mask);
224- // step 8：qlo = r * qhi1
225- MicroAPI::Mul(tmpQLOReg, tmpRReg, tmpFloatReg, mask);
226- // step 9：s = qhi * qhi;
227- MicroAPI::RegTensor<float> tmpSReg;
228- MicroAPI::Mul(tmpSReg, tmpQHIReg, tmpQHIReg, mask);
229- /*
230- * step 10：
231- * r = 0.37282535f;
232- * r = fmaf (r, s, 0.4097556f)
233- * r = fmaf (r, s, 0.57711965f)
234- * r = fmaf (r, s, 0.96179646f)
235- */
236- tmpRReg = params.rReg;
237- MicroAPI::Axpy<float>(tmpRReg, tmpSReg, LOG2_BEST_FMAF_COEFF1, mask);
238- MicroAPI::FusedMulDstAdd<float>(tmpRReg, tmpSReg, params.addReg1, mask);
239- MicroAPI::FusedMulDstAdd<float>(tmpRReg, tmpSReg, params.addReg2, mask);
240- // step 11：r = r * s
241- MicroAPI::Mul(tmpRReg, tmpRReg, tmpSReg, mask);
242- /*
243- * step 12:
244- * first_hi = fmaf(2.88539f, qhi, i)
245- * first_lo = fmaf(2.88539f, qhi, i-first_hi)
246- */
247- MicroAPI::RegTensor<float> tmpFHIReg, tmpFLOReg;
248- MicroAPI::Muls(tmpFHIReg, tmpQHIReg, LOG2_BEST_FMAF_COEFF5, mask);
249- MicroAPI::Add(tmpFHIReg, tmpFHIReg, tmpIReg, mask);
250- MicroAPI::Sub(tmpFLOReg, tmpIReg, tmpFHIReg, mask);
251- MicroAPI::Axpy(tmpFLOReg, tmpQHIReg, LOG2_BEST_FMAF_COEFF5, mask);
252- /*
253- * step 13:
254- * GOOD:
255- * last_lo= fmaf(r,qhi,fmaf(2.88539f,qlo,first_lo));
256- * sum_lo=fmaf(0.00000003851926f,qhi,last_lo);
257- * BETTER:
258- * last_lo= fmaf(r,qhi,fmaf(2.88539f,qlo,r*qlo));
259- * sum_lo=fmaf(0.00000003851926f,qhi,last_lo)+first_lo;
260- * BEST:
261- * last_lo= fmaf(r,qhi,fmaf(2.88539f,qlo,3.0f*r*qlo));
262- * sum_lo=fmaf(0.00000003851926f,qhi,last_lo)+first_lo;
263- */
264- MicroAPI::RegTensor<float> tmpLLOReg, tmpSLOReg;
265- MicroAPI::Mul(tmpLLOReg, tmpRReg, tmpQLOReg, mask);
266- MicroAPI::Axpy(tmpLLOReg, tmpQLOReg, LOG2_BEST_FMAF_COEFF5, mask);
267- MicroAPI::MulAddDst(tmpLLOReg, tmpRReg, tmpQHIReg, mask);
268- MicroAPI::Axpy(tmpLLOReg, tmpQHIReg, LOG2_BEST_FMAF_COEFF6, mask);
269- MicroAPI::Add(tmpSLOReg, tmpLLOReg, tmpFLOReg, mask);
270- /*
271- * step 14:
272- * loghi = first_hi+sum_lo;
273- * loglo = (first_hi - *loghi) + sum_lo;
274- */
275- MicroAPI::Add(logHigh, tmpFHIReg, tmpSLOReg, mask);
276- MicroAPI::Sub(tmpFloatReg, tmpFHIReg, logHigh, mask);
277- MicroAPI::Add(logLow, tmpFloatReg, tmpSLOReg, mask);
278-}
279- 
280-__simd_callee__ inline void GetExpfUnchecked(MicroAPI::RegTensor<float>& dstReg,MicroAPI::RegTensor<float>& tmPHIReg,
281- MicroAPI::RegTensor<float>& tmPLOReg, PowerExpParams& params, MicroAPI::MaskReg& mask)
282-{
283- /*
284- * step 1:
285- * r = fmaf (0.000152392517f, plo, 0.00133913534f);
286- * r = fmaf (r, plo, 0.00961883925f);
287- * r = fmaf (r, plo, 0.0555035882f);
288- * r = fmaf (r, plo, 0.240226448f);
289- * r = fmaf (r, plo, 0.693147182f);
290- * r = fmaf (r, plo, 1.0f);
291- */
292- MicroAPI::RegTensor<float> tmpFloatReg;
293- MicroAPI::RegTensor<float> tmpRReg = params.expReg2;
294- MicroAPI::Axpy(tmpRReg, tmPLOReg, EXPF_BEST_FMAF_COEFF1, mask);
295- MicroAPI::FusedMulDstAdd(tmpRReg, tmPLOReg, params.expReg3, mask);
296- MicroAPI::FusedMulDstAdd(tmpRReg, tmPLOReg, params.expReg4, mask);
297- MicroAPI::FusedMulDstAdd(tmpRReg, tmPLOReg, params.expReg5, mask);
298- MicroAPI::FusedMulDstAdd(tmpRReg, tmPLOReg, params.expReg6, mask);
299- MicroAPI::FusedMulDstAdd(tmpRReg, tmPLOReg, params.oneReg, mask);
300- 
301- // step2: r1 = (phi>0.0f) ? 0 : -2097152000;
302- MicroAPI::RegTensor<float> tmpF1Reg, tmpF2Reg;
303- MicroAPI::RegTensor<int32_t> tmpR1Reg, tmpR2Reg;
304- MicroAPI::MaskReg cmpMask;
305- MicroAPI::CompareScalar<float,CMPMODE::LE>(cmpMask,tmPHIReg, 0, mask);
306- MicroAPI::Duplicate(tmpR1Reg, EXPF_INTERVAL_CMP, cmpMask);
307- // step3: r2 = r1 + 2130706432;
308- MicroAPI::Adds(tmpR2Reg, tmpR1Reg, EXPF_INTERVAL_CAST, mask);
309- // step4: f1 = r * s32_to_f32(r2);
310- MicroAPI::Mul(tmpF1Reg, tmpRReg, (MicroAPI::RegTensor<float>&)tmpR2Reg, mask);
311- /*
312- * step5: int32_t r5 = cvt_rzi(phi);
313- * if (phi < 0.0f) return (int32_t)ceilf(phi);
314- * else return (int32_t)floorf(phi);
315- */
316- MicroAPI::RegTensor<int32_t> tmpR5Reg;
317- MicroAPI::MaskReg condMask;
318- MicroAPI::Cast<int32_t, float, castTraitI32F32>(tmpR5Reg, tmPHIReg, mask);
319- // step6: f2 = s32_to_f32((r5 << 23) - r1);
320- MicroAPI::RegTensor<int32_t> tmpTReg;
321- MicroAPI::ShiftLefts(tmpR5Reg, tmpR5Reg, SHITF_OFFSET, mask);
322- MicroAPI::Sub(tmpTReg, tmpR5Reg, tmpR1Reg, mask);
323- tmpF2Reg = (MicroAPI::RegTensor<float>&)tmpTReg;
324- // step7: f1 * f2
325- MicroAPI::Mul(dstReg, tmpF1Reg, tmpF2Reg, mask);
326-}
327- 
328-__simd_callee__ inline void ComputeExpoOddInt(MicroAPI::MaskReg& oddMask, MicroAPI::RegTensor<float>& expReg, MicroAPI::MaskReg& mask)
329-{
330- // calculate exp is odd or not: expo_odd_int = fmaf (-2.0f, floorf (0.5f * b), b) == 1.0f;
331- MicroAPI::RegTensor<float> tmpFloatReg;
332- MicroAPI::Muls(tmpFloatReg, expReg, 0.5f, mask);
333- RFloor(tmpFloatReg, tmpFloatReg, mask);
334- FMaf(tmpFloatReg, tmpFloatReg, -2.0f, expReg, mask);
335- MicroAPI::CompareScalar<float, CMPMODE::EQ>(oddMask, tmpFloatReg, 1.0f, mask);
336-}
337- 
338- 
339-__simd_callee__ inline void ProcessSpecialCaseForPowF(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& baseReg,
340- MicroAPI::RegTensor<float>& expReg, MicroAPI::RegTensor<int32_t>& tmpR10Reg, MicroAPI::RegTensor<int32_t>& tmpR12Reg, MicroAPI::MaskReg& mask)
341-{
342- MicroAPI::RegTensor<float> tmpFloatReg, tmpFloatReg2;
343- MicroAPI::MaskReg cmpMask1, cmpMask2, curMask;
344- /*
345- * bool p3_b_eq_0 = (b==0.0f);
346- * bool p4_a_eq_1 = (a==1.0f);
347- * if (p3_b_eq_0 || p4_a_eq_1)
348- * return 1.0f;
349- * if (isnan(a) || isnan(b))
350- * return NAN;
351- */
352- MicroAPI::CompareScalar<float, CMPMODE::EQ>(cmpMask1, expReg, 0.0f, mask);
353- MicroAPI::CompareScalar<float, CMPMODE::EQ>(cmpMask2, baseReg, 1.0f, mask);
354- MicroAPI::MaskOr(cmpMask2, cmpMask1, cmpMask2, mask);
355- MicroAPI::Duplicate<float, MicroAPI::MaskMergeMode::MERGING>(dstReg, 1.0f, cmpMask2);
356- MicroAPI::MaskNot(curMask, cmpMask2, mask);
357- IsNanNum(cmpMask1, baseReg, mask);
358- IsNanNum(cmpMask2, expReg, mask);
359- MicroAPI::MaskOr(cmpMask2, cmpMask1, cmpMask2, curMask);
360- MicroAPI::Duplicate<int32_t, MicroAPI::MaskMergeMode::MERGING>((MicroAPI::RegTensor<int32_t>&)dstReg, F32_NAN, cmpMask2);
361- MicroAPI::MaskXor(curMask, cmpMask2, curMask, mask);
362- /*
363- * if (isinf(a) || (a==0.0f))
364- * int32_t r10=f32_to_s32(a);
365- * int32_t r11=r10^0x7F800000;
366- * bool p8_a_lower_0 = (b < 0.0f);
367- * int32_t r12 = (p8_a_lower_0 ? r11 : r10);
368- * int32_t r13= r12 & 0x7FFFFFFF;
369- * return s32_to_f32(p1_expo_odd_int?r12:r13);
370- *
371- */
372- IsInfNum(cmpMask1, baseReg, tmpR12Reg, curMask);
373- MicroAPI::CompareScalar<float, CMPMODE::EQ>(cmpMask2, baseReg, 0.0f, curMask);
374- MicroAPI::MaskOr(cmpMask1, cmpMask1, cmpMask2, mask);
375- MicroAPI::CompareScalar<float, CMPMODE::LT>(cmpMask2, expReg, 0.0f, cmpMask1);
376- MicroAPI::Xor((MicroAPI::RegTensor<int32_t>&)tmpFloatReg, (MicroAPI::RegTensor<int32_t>&)baseReg, tmpR10Reg, curMask);
377- MicroAPI::Select(tmpFloatReg, tmpFloatReg, baseReg, cmpMask2);
378- MicroAPI::And((MicroAPI::RegTensor<int32_t>&)tmpFloatReg2, (MicroAPI::RegTensor<int32_t>&)tmpFloatReg, tmpR12Reg, curMask);
379- ComputeExpoOddInt(cmpMask2, expReg, mask);
380- MicroAPI::Select(tmpFloatReg, tmpFloatReg, tmpFloatReg2, cmpMask2);
381- MicroAPI::Select(dstReg, tmpFloatReg, dstReg, cmpMask1);
382- MicroAPI::MaskXor(curMask, cmpMask1, curMask, mask);
383- /*
384- * if (a < 0.0f)
385- * float tmp_r=p1_expo_odd_int?(-r):r;
386- * r = (b != floorf(b)) ? NAN : tmp_r;
387- *
388- */
389- MicroAPI::Neg(tmpFloatReg, dstReg, curMask);
390- MicroAPI::Select(tmpFloatReg, tmpFloatReg, dstReg, cmpMask2);
391- RFloor(tmpFloatReg2, expReg, curMask);
392- MicroAPI::Compare<float, CMPMODE::NE>(cmpMask1, expReg, tmpFloatReg2, curMask);
393- MicroAPI::Duplicate<int32_t, MicroAPI::MaskMergeMode::MERGING>(
394- (MicroAPI::RegTensor<int32_t>&)tmpFloatReg,F32_NAN, cmpMask1);
395- MicroAPI::CompareScalar<float, CMPMODE::LT>(cmpMask2, baseReg, 0.0f, curMask);
396- MicroAPI::Select(dstReg, tmpFloatReg, dstReg, cmpMask2);
397- /*
398- * if ((a == -1.0f) && isinf(b))
399- * r = 1.0f;
400- *
401- */
402- MicroAPI::CompareScalar<int32_t, CMPMODE::EQ>(cmpMask1, (MicroAPI::RegTensor<int32_t>&)expReg, INF, curMask);
403- MicroAPI::CompareScalar<int32_t, CMPMODE::EQ>(cmpMask2, (MicroAPI::RegTensor<int32_t>&)expReg, NEG_INF, curMask);
404- MicroAPI::MaskOr(cmpMask1, cmpMask1, cmpMask2, mask);
405- MicroAPI::CompareScalar<float, CMPMODE::EQ>(cmpMask2, baseReg, -1.0f, cmpMask1);
406- MicroAPI::Duplicate<float, MicroAPI::MaskMergeMode::MERGING>(dstReg, 1.0f, cmpMask2);
407-}
408- 
409-__simd_callee__ inline void GetExpCore(MicroAPI::RegTensor<float>& dstReg,
410- MicroAPI::RegTensor<float>& tmpLHIReg, MicroAPI::RegTensor<float>& tmpLLOReg,
411- MicroAPI::RegTensor<float>& expReg, PowerExpParams& params, MicroAPI::MaskReg& mask)
412-{
413- MicroAPI::RegTensor<float> tmpTHIReg, tmPHIReg, tmPLOReg, tmpRReg;
414- // step 1: thi = lhi * b;
415- MicroAPI::Mul(tmpTHIReg, tmpLHIReg, expReg, mask);
416- // step 2: phi = roundf(thi);
417- MicroAPI::Truncate<float, RoundMode::CAST_ROUND>(tmPHIReg, tmpTHIReg, mask);
418- // step 3: plo = fmaf(lhi, b, -phi)+llo*b;
419- MicroAPI::RegTensor<float> tmpFloatReg, tmpFloatReg2;
420- MicroAPI::Neg(tmPLOReg, tmPHIReg, mask);
421- MicroAPI::MulAddDst(tmPLOReg, tmpLHIReg, expReg, mask);
422- MicroAPI::MulAddDst(tmPLOReg, tmpLLOReg, expReg, mask);
423- // step 4: my_expf__improved(phi, plo, &r);
424- GetExpfUnchecked(tmpRReg, tmPHIReg, tmPLOReg, params, mask);
425- /*
426- * step 5:
427- * tmp_r =(thi < 0.0f) ?0.0f : MY_INF_F;
428- * r = (fabsf(thi) > EXP_OVFL_UNFL_F) ? tmp_r : r;
429- */
430- MicroAPI::MaskReg cmpMask1, cmpMask2;
431- MicroAPI::CompareScalar<float, CMPMODE::GE>(cmpMask1, tmpTHIReg, 0.0f, mask);
432- // mode zeroing dup inf/zero reg.
433- MicroAPI::Duplicate((MicroAPI::RegTensor<int32_t>&)tmpFloatReg, INF, cmpMask1);
434- MicroAPI::Abs(tmpFloatReg2, tmpTHIReg, mask);
435- MicroAPI::CompareScalar<float, CMPMODE::GT>(cmpMask2, tmpFloatReg2, EXP_OVFL_UNFL_F, mask);
436- MicroAPI::Select(dstReg, tmpFloatReg, tmpRReg, cmpMask2);
437-}
438- 
439-template<typename T>
440-__simd_callee__ inline void LoadSrcData(MicroAPI::RegTensor<float>& srcReg, __ubuf__ T* src0, uint16_t index, MicroAPI::MaskReg& mask)
441-{
442- MicroAPI::RegTensor<T> srcTmpReg;
443- if constexpr (std::is_same<T, half>::value || std::is_same<T, bfloat16_t>::value) {
444- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcTmpReg, src0 + index * B32_DATA_NUM_PER_REPEAT);
445- MicroAPI::Cast<float, T, castTraitF16F32>(srcReg, srcTmpReg, mask);
446- } else {
447- MicroAPI::DataCopy(srcReg, src0 + index * B32_DATA_NUM_PER_REPEAT);
448- }
449-}
450- 
451-template<typename T>
452-__simd_callee__ inline void LoadSrcScalarData(MicroAPI::RegTensor<float>& srcReg, const T scalarValue)
453-{
454- MicroAPI::RegTensor<T> srcTmpReg;
455- MicroAPI::MaskReg fullMask = MicroAPI::CreateMask<T>();
456- MicroAPI::Duplicate(srcTmpReg, scalarValue, fullMask);
457- if constexpr (std::is_same<T, half>::value || std::is_same<T, bfloat16_t>::value) {
458- MicroAPI::Cast<float, T, castTraitF16F32>(srcReg, srcTmpReg, fullMask);
459- } else {
460- srcReg = srcTmpReg;
461- }
462-}
463- 
464-template<typename T>
465-__simd_callee__ inline void StoreDstData(__ubuf__ T* dst, MicroAPI::RegTensor<float>& dstReg, uint16_t index, MicroAPI::MaskReg& mask)
466-{
467- MicroAPI::RegTensor<T> dstTmpReg;
468- if constexpr (std::is_same<T, half>::value || std::is_same<T, bfloat16_t>::value) {
469- MicroAPI::Cast<T, float, castTraitF32F16>(dstTmpReg, dstReg, mask);
470- MicroAPI::Pack<uint16_t, uint32_t, MicroAPI::HighLowPart::LOWEST>(
471- (MicroAPI::RegTensor<uint16_t>&)dstTmpReg, (MicroAPI::RegTensor<uint32_t>&)dstTmpReg);
472- MicroAPI::MaskPack(mask, mask);
473- MicroAPI::DataCopy(dst + index * B32_DATA_NUM_PER_REPEAT, dstTmpReg, mask);
474- } else {
475- MicroAPI::DataCopy(dst + index * B32_DATA_NUM_PER_REPEAT, dstReg, mask);
476- }
477-}
478- 
479-template<typename T>
480-__simd_vf__ inline void ComputePowFBaseLog1Impl(__ubuf__ float* tmpLHIBuffer, __ubuf__ float* tmpLLOBuffer,
481- __ubuf__ T* src0, uint32_t calCount, uint16_t repeatTime)
482-{
483- MicroAPI::MaskReg mask;
484- MicroAPI::RegTensor<float> tmpBaseReg, tmpDstReg;
485- MicroAPI::RegTensor<float> tmpLHIReg, tmpLLOReg;
486- 
487- PowerLogParams params;
488- PowerLogParamsInit(params);
489- 
490- for(uint16_t i = 0; i < repeatTime; i++) {
491- mask = MicroAPI::UpdateMask<float>(calCount);
492- LoadSrcData(tmpBaseReg, src0, i, mask);
493- 
494- MicroAPI::Abs(tmpDstReg, tmpBaseReg, mask);
495- GetLogFExt1(tmpLHIReg, tmpLLOReg, tmpDstReg, params, mask);
496- 
497- MicroAPI::DataCopy(tmpLHIBuffer + i * B32_DATA_NUM_PER_REPEAT, tmpLHIReg, mask);
498- MicroAPI::DataCopy(tmpLLOBuffer + i * B32_DATA_NUM_PER_REPEAT, tmpLLOReg, mask);
499- }
500-}
501- 
502-template<typename T>
503-__simd_vf__ inline void ComputePowFBaseLog1Impl(__ubuf__ float* tmpLHIBuffer, __ubuf__ float* tmpLLOBuffer,
504- const T scalarValue, uint32_t calCount, uint16_t repeatTime)
505-{
506- MicroAPI::MaskReg mask;
507- MicroAPI::RegTensor<float> tmpBaseReg, tmpDstReg;
508- MicroAPI::RegTensor<float> tmpLHIReg, tmpLLOReg;
509- LoadSrcScalarData(tmpBaseReg, scalarValue);
510- 
511- PowerLogParams params;
512- PowerLogParamsInit(params);
513- 
514- for (uint16_t i = 0; i < repeatTime; i++) {
515- mask = MicroAPI::UpdateMask<float>(calCount);
516- MicroAPI::Abs(tmpDstReg, tmpBaseReg, mask);
517- GetLogFExt1(tmpLHIReg, tmpLLOReg, tmpDstReg, params, mask);
518- MicroAPI::DataCopy(tmpLHIBuffer + i * B32_DATA_NUM_PER_REPEAT, tmpLHIReg, mask);
519- MicroAPI::DataCopy(tmpLLOBuffer + i * B32_DATA_NUM_PER_REPEAT, tmpLLOReg, mask);
520- }
521-}
522- 
523-template<typename T>
524-__simd_vf__ inline void ComputePowFBaseLog2Impl(__ubuf__ float* tmpLHIBuffer, __ubuf__ float* tmpLLOBuffer,
525- uint32_t calCount, uint16_t repeatTime)
526-{
527- MicroAPI::MaskReg mask;
528- MicroAPI::RegTensor<float> tmpLHIReg, tmpLLOReg;
529- MicroAPI::RegTensor<float> tmpIReg, tmpMReg;
530- 
531- PowerLogParams params;
532- PowerLogParamsInit(params);
533- 
534- for(uint16_t i = 0; i < repeatTime; i++) {
535- mask = MicroAPI::UpdateMask<float>(calCount);
536- LoadSrcData(tmpIReg, tmpLHIBuffer, i, mask);
537- LoadSrcData(tmpMReg, tmpLLOBuffer, i, mask);
538- 
539- GetLogFExt2(tmpLHIReg, tmpLLOReg, tmpIReg, tmpMReg, params, mask);
540- 
541- MicroAPI::DataCopy(tmpLHIBuffer + i * B32_DATA_NUM_PER_REPEAT, tmpLHIReg, mask);
542- MicroAPI::DataCopy(tmpLLOBuffer + i * B32_DATA_NUM_PER_REPEAT, tmpLLOReg, mask);
543- }
544-}
545- 
546-template<typename T>
547-__simd_vf__ inline void ComputePowFExpImpl(__ubuf__ float* tmpExpBuffer, __ubuf__ float* tmpLogHighBuffer,
548- __ubuf__ float* tmpLogLowBuffer, __ubuf__ T* src1, uint32_t calCount, uint16_t repeatTime)
549-{
550- MicroAPI::MaskReg mask;
551- MicroAPI::RegTensor<float> tmpLHIReg, tmpLLOReg, tmpExpReg, tmpDstReg;
552- 
553- PowerExpParams params;
554- PowerExpParamsInit(params);
555- 
556- for(uint16_t i = 0; i < repeatTime; i++) {
557- mask = MicroAPI::UpdateMask<float>(calCount);
558- LoadSrcData(tmpLHIReg, tmpLogHighBuffer, i, mask);
559- LoadSrcData(tmpLLOReg, tmpLogLowBuffer, i, mask);
560- LoadSrcData(tmpExpReg, src1, i, mask);
561- GetExpCore(tmpDstReg, tmpLHIReg, tmpLLOReg, tmpExpReg, params, mask);
562- MicroAPI::DataCopy(tmpExpBuffer + i * B32_DATA_NUM_PER_REPEAT, tmpDstReg, mask);
563- }
564-}
565- 
566-template<typename T>
567-__simd_vf__ inline void ComputePowFExpImpl(__ubuf__ float* tmpExpBuffer, __ubuf__ float* tmpLogHighBuffer,
568- __ubuf__ float* tmpLogLowBuffer, const T scalarValue, uint32_t calCount, uint16_t repeatTime)
569-{
570- MicroAPI::MaskReg mask;
571- MicroAPI::RegTensor<float> tmpLHIReg, tmpLLOReg, tmpExpReg, tmpDstReg;
572- LoadSrcScalarData(tmpExpReg, scalarValue);
573- 
574- PowerExpParams params;
575- PowerExpParamsInit(params);
576- 
577- for(uint16_t i = 0; i < repeatTime; i++) {
578- mask = MicroAPI::UpdateMask<float>(calCount);
579- LoadSrcData(tmpLHIReg, tmpLogHighBuffer, i, mask);
580- LoadSrcData(tmpLLOReg, tmpLogLowBuffer, i, mask);
581- GetExpCore(tmpDstReg, tmpLHIReg, tmpLLOReg, tmpExpReg, params, mask);
582- MicroAPI::DataCopy(tmpExpBuffer + i * B32_DATA_NUM_PER_REPEAT, tmpDstReg, mask);
583- }
584-}
585- 
586-template<typename T>
587-__simd_vf__ inline void ComputePowFSpecialCaseImpl(__ubuf__ T* dst, __ubuf__ T* src0, __ubuf__ T* src1,
588- __ubuf__ float* tmpExpBuffer, uint32_t calCount, uint16_t repeatTime)
589-{
590- MicroAPI::MaskReg mask;
591- MicroAPI::RegTensor<float> tmpBaseReg, tmpExpReg, castDstReg;
592- MicroAPI::RegTensor<int32_t> tmpR10Reg, tmpR12Reg;
593- MicroAPI::Duplicate(tmpR10Reg, R10_COEFF);
594- MicroAPI::Duplicate(tmpR12Reg, R12_COEFF);
595- for(uint16_t i = 0; i < repeatTime; i++) {
596- mask = MicroAPI::UpdateMask<float>(calCount);
597- LoadSrcData(tmpBaseReg, src0, i, mask);
598- LoadSrcData(tmpExpReg, src1, i, mask);
599- MicroAPI::DataCopy(castDstReg, tmpExpBuffer + i * B32_DATA_NUM_PER_REPEAT);
600- ProcessSpecialCaseForPowF(castDstReg, tmpBaseReg, tmpExpReg, tmpR10Reg, tmpR12Reg, mask);
601- StoreDstData(dst, castDstReg, i, mask);
602- }
603-}
604- 
605-template<typename T>
606-__simd_vf__ inline void ComputePowFSpecialCaseImpl(__ubuf__ T* dst, __ubuf__ T* src0, const T scalarValue,
607- __ubuf__ float* tmpExpBuffer, uint32_t calCount, uint16_t repeatTime)
608-{
609- MicroAPI::MaskReg mask;
610- MicroAPI::RegTensor<float> tmpBaseReg, tmpExpReg, castDstReg;
611- MicroAPI::RegTensor<int32_t> tmpR10Reg, tmpR12Reg;
612- MicroAPI::Duplicate(tmpR10Reg, R10_COEFF);
613- MicroAPI::Duplicate(tmpR12Reg, R12_COEFF);
614- LoadSrcScalarData(tmpExpReg, scalarValue);
615- for(uint16_t i = 0; i < repeatTime; i++) {
616- mask = MicroAPI::UpdateMask<float>(calCount);
617- LoadSrcData(tmpBaseReg, src0, i, mask);
618- MicroAPI::DataCopy(castDstReg, tmpExpBuffer + i * B32_DATA_NUM_PER_REPEAT);
619- ProcessSpecialCaseForPowF(castDstReg, tmpBaseReg, tmpExpReg, tmpR10Reg, tmpR12Reg, mask);
620- StoreDstData(dst, castDstReg, i, mask);
621- }
622-}
623- 
624-template<typename T>
625-__simd_vf__ inline void ComputePowFSpecialCaseImpl(__ubuf__ T* dst, const T scalarValue, __ubuf__ T* src1,
626- __ubuf__ float* tmpExpBuffer, uint32_t calCount, uint16_t repeatTime)
627-{
628- MicroAPI::MaskReg mask;
629- MicroAPI::RegTensor<float> tmpBaseReg, tmpExpReg, castDstReg;
630- MicroAPI::RegTensor<int32_t> tmpR10Reg, tmpR12Reg;
631- MicroAPI::Duplicate(tmpR10Reg, R10_COEFF);
632- MicroAPI::Duplicate(tmpR12Reg, R12_COEFF);
633- LoadSrcScalarData(tmpBaseReg, scalarValue);
634- for(uint16_t i = 0; i < repeatTime; i++) {
635- mask = MicroAPI::UpdateMask<float>(calCount);
636- LoadSrcData(tmpExpReg, src1, i, mask);
637- MicroAPI::DataCopy(castDstReg, tmpExpBuffer + i * B32_DATA_NUM_PER_REPEAT);
638- ProcessSpecialCaseForPowF(castDstReg, tmpBaseReg, tmpExpReg, tmpR10Reg, tmpR12Reg, mask);
639- StoreDstData(dst, castDstReg, i, mask);
640- }
641-}
642- 
643-__aicore__ inline void InitTmpBuffer(__ubuf__ uint32_t*& tmpBuffer, __ubuf__ float*& tmpLogBuffer,
644- __ubuf__ float*& tmpExpBuffer, const uint32_t alignCount)
645-{
646- tmpLogBuffer = (__ubuf__ float *)tmpBuffer;
647- tmpExpBuffer = (__ubuf__ float *)((__ubuf__ uint8_t*)tmpLogBuffer + sizeof(float) * alignCount);
648-}
649- 
650-template<typename T>
651-__aicore__ inline void PowFComputeImpl(__ubuf__ T* dst, __ubuf__ T* src0, __ubuf__ T* src1,
652- __ubuf__ uint32_t* tmpBuffer, uint32_t calCount)
653-{
654- constexpr uint16_t eleCountPerVL = GetVecLen() / sizeof(float);
655- uint16_t repeatTime = DivCeil(calCount, eleCountPerVL);
656- __ubuf__ float* tmpLowBuffer;
657- __ubuf__ float* tmpHighBuffer;
658- 
659- uint32_t alignCount = (calCount + 31) / 32 * 32;
660- 
661- InitTmpBuffer(tmpBuffer, tmpHighBuffer, tmpLowBuffer, alignCount);
662- __ubuf__ float* tmpExpBuffer = tmpHighBuffer;
663- 
664- ComputePowFBaseLog1Impl<T>(tmpHighBuffer, tmpLowBuffer, src0, calCount, repeatTime);
665- ComputePowFBaseLog2Impl<T>(tmpHighBuffer, tmpLowBuffer, calCount, repeatTime);
666- ComputePowFExpImpl<T>(tmpExpBuffer, tmpHighBuffer, tmpLowBuffer, src1, calCount, repeatTime);
667- ComputePowFSpecialCaseImpl<T>(dst, src0, src1, tmpExpBuffer, calCount, repeatTime);
668-}
669- 
670-template<typename T>
671-__aicore__ inline void PowFComputeImpl(__ubuf__ T* dst, __ubuf__ T* src0, const T& scalarValue,
672- __ubuf__ uint32_t* tmpBuffer, uint32_t calCount)
673-{
674- constexpr uint16_t eleCountPerVL = GetVecLen() / sizeof(float);
675- uint16_t repeatTime = DivCeil(calCount, eleCountPerVL);
676- __ubuf__ float* tmpLowBuffer;
677- __ubuf__ float* tmpHighBuffer;
678- 
679- uint32_t alignCount = (calCount + 31) / 32 * 32;
680- 
681- InitTmpBuffer(tmpBuffer, tmpHighBuffer, tmpLowBuffer, alignCount);
682- 
683- __ubuf__ float* tmpExpBuffer = tmpHighBuffer;
684- 
685- ComputePowFBaseLog1Impl<T>(tmpHighBuffer, tmpLowBuffer, src0, calCount, repeatTime);
686- ComputePowFBaseLog2Impl<T>(tmpHighBuffer, tmpLowBuffer, calCount, repeatTime);
687- ComputePowFExpImpl<T>(tmpExpBuffer, tmpHighBuffer, tmpLowBuffer, scalarValue, calCount, repeatTime);
688- ComputePowFSpecialCaseImpl<T>(dst, src0, scalarValue, tmpExpBuffer, calCount, repeatTime);
689-}
690- 
691-template<typename T>
692-__aicore__ inline void PowFComputeImpl(__ubuf__ T* dst, const T& scalarValue, __ubuf__ T* src1,
693- __ubuf__ uint32_t* tmpBuffer, uint32_t calCount)
694-{
695- constexpr uint16_t eleCountPerVL = GetVecLen() / sizeof(float);
696- uint16_t repeatTime = DivCeil(calCount, eleCountPerVL);
697- __ubuf__ float* tmpLowBuffer;
698- __ubuf__ float* tmpHighBuffer;
699- 
700- uint32_t alignCount = (calCount + 31) / 32 * 32;
701- 
702- InitTmpBuffer(tmpBuffer, tmpHighBuffer, tmpLowBuffer, alignCount);
703- __ubuf__ float* tmpExpBuffer = tmpHighBuffer;
704- 
705- ComputePowFBaseLog1Impl<T>(tmpHighBuffer, tmpLowBuffer, scalarValue, calCount, repeatTime);
706- ComputePowFBaseLog2Impl<T>(tmpHighBuffer, tmpLowBuffer, calCount, repeatTime);
707- ComputePowFExpImpl<T>(tmpExpBuffer, tmpHighBuffer, tmpLowBuffer, src1, calCount, repeatTime);
708- ComputePowFSpecialCaseImpl<T>(dst, scalarValue, src1, tmpExpBuffer, calCount, repeatTime);
709-}
710- 
711-/*********** PowF Intrinsic Impl **********/
712-__aicore__ inline void GetPowFInstrinsicCore(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& baseReg,
713- MicroAPI::RegTensor<float>& expReg, MicroAPI::MaskReg& mask)
714-{
715- // Compute dst = exp(exp * ln(|base|))
716- MicroAPI::RegTensor<float> tmpReg;
717- MicroAPI::Abs(tmpReg, baseReg, mask);
718- MicroAPI::Ln(tmpReg, tmpReg, mask);
719- MicroAPI::Mul(dstReg, expReg, tmpReg, mask);
720- MicroAPI::Exp(dstReg, dstReg, mask);
721-}
722- 
723-template<typename T>
724-__aicore__ inline void PowFInstrinsicTensorTensorImpl(__ubuf__ T* dst, __ubuf__ T* src0, __ubuf__ T* src1,
725- uint32_t calCount, uint16_t repeatTime)
726-{
727- MicroAPI::MaskReg mask, tmpMask;
728- MicroAPI::RegTensor<float> tmpBaseReg, tmpExpReg, castDstReg;
729- MicroAPI::RegTensor<int32_t> tmpR10Reg, tmpR12Reg;
730- MicroAPI::Duplicate(tmpR10Reg, R10_COEFF);
731- MicroAPI::Duplicate(tmpR12Reg, R12_COEFF);
732- for(uint16_t i = 0; i < repeatTime; i++) {
733- mask = MicroAPI::UpdateMask<float>(calCount);
734- tmpMask = mask;
735- LoadSrcData(tmpBaseReg, src0, i, mask);
736- LoadSrcData(tmpExpReg, src1, i, mask);
737- GetPowFInstrinsicCore(castDstReg, tmpBaseReg, tmpExpReg, mask);
738- ProcessSpecialCaseForPowF(castDstReg, tmpBaseReg, tmpExpReg, tmpR10Reg, tmpR12Reg, tmpMask);
739- StoreDstData(dst, castDstReg, i, mask);
740- }
741-}
742- 
743-template<typename T>
744-__aicore__ inline void PowFInstrinsicTensorScalarImpl(__ubuf__ T* dst, __ubuf__ T* src0, const T& scalarValue,
745- uint32_t calCount, uint16_t repeatTime)
746-{
747- MicroAPI::MaskReg mask, tmpMask;
748- MicroAPI::RegTensor<float> tmpBaseReg, tmpExpReg, castDstReg;
749- MicroAPI::RegTensor<int32_t> tmpR10Reg, tmpR12Reg;
750- MicroAPI::Duplicate(tmpR10Reg, R10_COEFF);
751- MicroAPI::Duplicate(tmpR12Reg, R12_COEFF);
752- LoadSrcScalarData(tmpExpReg, scalarValue);
753- for(uint16_t i = 0; i < repeatTime; i++) {
754- mask = MicroAPI::UpdateMask<float>(calCount);
755- tmpMask = mask;
756- LoadSrcData(tmpBaseReg, src0, i, mask);
757- GetPowFInstrinsicCore(castDstReg, tmpBaseReg, tmpExpReg, mask);
758- ProcessSpecialCaseForPowF(castDstReg, tmpBaseReg, tmpExpReg, tmpR10Reg, tmpR12Reg,tmpMask);
759- StoreDstData(dst, castDstReg, i, mask);
760- }
761-}
762- 
763-template<typename T>
764-__aicore__ inline void PowFInstrinsicScalarTensorImpl(__ubuf__ T* dst, const T& scalarValue, __ubuf__ T* src1,
765- uint32_t calCount, uint16_t repeatTime)
766-{
767- MicroAPI::MaskReg mask, tmpMask;
768- MicroAPI::RegTensor<float> tmpBaseReg, tmpExpReg, castDstReg;
769- MicroAPI::RegTensor<int32_t> tmpR10Reg, tmpR12Reg;
770- MicroAPI::Duplicate(tmpR10Reg, R10_COEFF);
771- MicroAPI::Duplicate(tmpR12Reg, R12_COEFF);
772- LoadSrcScalarData(tmpBaseReg, scalarValue);
773- for(uint16_t i = 0; i < repeatTime; i++) {
774- mask = MicroAPI::UpdateMask<float>(calCount);
775- tmpMask = mask;
776- LoadSrcData(tmpExpReg, src1, i, mask);
777- GetPowFInstrinsicCore(castDstReg, tmpBaseReg, tmpExpReg, mask);
778- ProcessSpecialCaseForPowF(castDstReg, tmpBaseReg, tmpExpReg, tmpR10Reg, tmpR12Reg, tmpMask);
779- StoreDstData(dst, castDstReg, i, mask);
780- }
781-}
782- 
783-template<typename T>
784-__aicore__ inline void PowFInstrinsicImpl(__ubuf__ T* dst, __ubuf__ T* src0, __ubuf__ T* src1,
785- uint32_t calCount)
786-{
787- constexpr uint16_t eleCountPerVL = GetVecLen() / sizeof(float);
788- uint16_t repeatTimes = DivCeil(calCount, eleCountPerVL);
789- VF_CALL<PowFInstrinsicTensorTensorImpl<T>>(dst, src0, src1, calCount, repeatTimes);
790-}
791- 
792-template<typename T>
793-__aicore__ inline void PowFInstrinsicImpl(__ubuf__ T* dst, __ubuf__ T* src0, const T& scalarValue,
794- uint32_t calCount)
795-{
796- constexpr uint16_t eleCountPerVL = GetVecLen() / sizeof(float);
797- uint16_t repeatTimes = DivCeil(calCount, eleCountPerVL);
798- VF_CALL<PowFInstrinsicTensorScalarImpl<T>>(dst, src0, scalarValue, calCount, repeatTimes);
799-}
800- 
801-template<typename T>
802-__aicore__ inline void PowFInstrinsicImpl(__ubuf__ T* dst, const T& scalarValue, __ubuf__ T* src1,
803- uint32_t calCount)
804-{
805- constexpr uint16_t eleCountPerVL = GetVecLen() / sizeof(float);
806- uint16_t repeatTimes = DivCeil(calCount, eleCountPerVL);
807- VF_CALL<PowFInstrinsicScalarTensorImpl<T>>(dst, scalarValue, src1, calCount, repeatTimes);
808-}
809-} // namespace PowF
810- 
811-namespace PowI {
812-constexpr int16_t SHIFT_ONE_BIT = 1;
813-constexpr int16_t BITS_PER_BYTE = 8;
814- 
815-template<typename T>
816-__simd_callee__ inline void GetPowI(MicroAPI::RegTensor<T>& dstReg, MicroAPI::RegTensor<T>& baseReg,
817- MicroAPI::RegTensor<T>& expReg, MicroAPI::MaskReg& mask)
818-{
819- MicroAPI::RegTensor<T> tmpReg, tmpReg2;
820- MicroAPI::MaskReg expMask;
821- // step 1: result1 = result * a;
822- MicroAPI::Mul(tmpReg, dstReg, baseReg, mask);
823- MicroAPI::Duplicate(tmpReg2, 1, mask);
824- // step 2: mask = b & 1;
825- MicroAPI::And(tmpReg2, expReg, tmpReg2, mask);
826- // step 3: result = select(mask, result1, result);
827- MicroAPI::CompareScalar<T, CMPMODE::EQ>(expMask, tmpReg2, 1, mask);
828- MicroAPI::Select(dstReg, tmpReg, dstReg, expMask);
829- // step4: b /= 2;
830- MicroAPI::ShiftRights(expReg, expReg, SHIFT_ONE_BIT, mask);
831- // step5: a *= a;
832- MicroAPI::Mul(baseReg, baseReg, baseReg, mask);
833-}
834- 
835-template<typename T>
836-__simd_callee__ inline void ProcessSpecialCaseForPowI(MicroAPI::RegTensor<T>& dstReg, MicroAPI::RegTensor<T>& baseReg,
837- MicroAPI::RegTensor<T>& expReg, MicroAPI::MaskReg& mask)
838-{
839- MicroAPI::RegTensor<T> tmpRReg;
840- MicroAPI::MaskReg cmpMask1, cmpMask2, condMask;
841- /*
842- * special case 1:
843- * if (exp == 0) || (base == 1) {
844- * r = 1;
845- * }
846- */
847- MicroAPI::CompareScalar<T, CMPMODE::EQ>(cmpMask1, expReg, 0, mask);
848- MicroAPI::CompareScalar<T, CMPMODE::EQ>(cmpMask2, baseReg, 1, mask);
849- MicroAPI::MaskOr(condMask, cmpMask1, cmpMask2, mask);
850- MicroAPI::Duplicate(tmpRReg, 1, mask);
851- MicroAPI::Select(dstReg, tmpRReg, dstReg, condMask);
852- MicroAPI::MaskXor(mask, mask, condMask, mask);
853- 
854- if constexpr (SupportType<T, int8_t, int16_t, int32_t>()) {
855- /*
856- * special case 2:
857- * else if (base != -1 && exp < 0) {
858- * r = 0;
859- * }
860- */
861- MicroAPI::CompareScalar<T, CMPMODE::NE>(cmpMask1, baseReg, -1, mask);
862- MicroAPI::CompareScalar<T, CMPMODE::LT>(cmpMask2, expReg, 0, mask);
863- MicroAPI::MaskAnd(condMask, cmpMask1, cmpMask2, mask);
864- MicroAPI::Duplicate(tmpRReg, 0, mask);
865- MicroAPI::Select(dstReg, tmpRReg, dstReg, condMask);
866- }
867-}
868- 
869-template<typename T>
870-__simd_callee__ inline void GetPowICompute(MicroAPI::RegTensor<T>& dstReg, MicroAPI::RegTensor<T>& baseReg,
871- MicroAPI::RegTensor<T>& expReg, MicroAPI::MaskReg& mask, const uint16_t maxLoop)
872-{
873- MicroAPI::RegTensor<T> tmpBaseReg = baseReg;
874- MicroAPI::RegTensor<T> tmpExpReg = expReg;
875- MicroAPI::MaskReg tmpMask = mask;
876- for (uint16_t j = 0; j < maxLoop; j++) {
877- GetPowI(dstReg, tmpBaseReg, tmpExpReg, mask);
878- }
879- ProcessSpecialCaseForPowI(dstReg, baseReg, expReg, tmpMask);
880-}
881- 
882-template<typename T>
883-struct PowICastType {
884- using type = T;
885-};
886- 
887-template<>
888-struct PowICastType<int8_t> {
889- using type = int16_t;
890-};
891- 
892-template<>
893-struct PowICastType<uint8_t> {
894- using type = uint16_t;
895-};
896- 
897-template<typename T>
898-__aicore__ inline uint16_t CountLeadingZeros(T x) {
899- if (x == 0) return sizeof(T) * 8;
900- uint16_t count = 0;
901- T mask = static_cast<T>(1 << (sizeof(T) * 8 - 1));
902- while((x & mask) == 0) {
903- count ++;
904- x <<= 1;
905- }
906- return count;
907-}
908- 
909-template<typename T, typename ConvType>
910-__simd_callee__ inline void LoadSrcData(MicroAPI::RegTensor<ConvType>& dstReg, __ubuf__ T* src, uint16_t index, MicroAPI::MaskReg& mask)
911-{
912- constexpr uint16_t eleCountPerVL = GetVecLen() / sizeof(ConvType);
913- MicroAPI::RegTensor<T> srcTmpReg;
914- if constexpr (sizeof(T) == 1) {
915- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B8>(srcTmpReg, src + index * eleCountPerVL);
916- MicroAPI::Cast<ConvType, T, castTraitI8I16>(dstReg, srcTmpReg, mask);
917- } else {
918- MicroAPI::DataCopy(dstReg, src + index * eleCountPerVL);
919- }
920-}
921- 
922-template<typename T, typename ConvType>
923-__simd_callee__ inline void StoreDstData(__ubuf__ T* dst, MicroAPI::RegTensor<ConvType>& dstReg, uint16_t index, MicroAPI::MaskReg& mask)
924-{
925- constexpr uint16_t eleCountPerVL = GetVecLen() / sizeof(ConvType);
926- MicroAPI::RegTensor<T> dstTmpReg;
927- 
928- if constexpr (sizeof(T) == 1) {
929- MicroAPI::Pack<uint8_t, uint16_t, MicroAPI::HighLowPart::LOWEST>(
930- (MicroAPI::RegTensor<uint8_t>&)dstTmpReg, (MicroAPI::RegTensor<uint16_t>&)dstReg);
931- MicroAPI::MaskPack(mask, mask);
932- MicroAPI::DataCopy(dst + index * eleCountPerVL, dstTmpReg, mask);
933- } else {
934- MicroAPI::DataCopy(dst + index * eleCountPerVL, dstReg, mask);
935- }
936-}
937- 
938-template<typename T>
939-__aicore__ inline uint16_t GetMaxLoop(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor, uint32_t calCount)
940-{
941- if constexpr(sizeof(T) == 1) {
942- return sizeof(T) * BITS_PER_BYTE;
943- } else {
944- AscendC::ReduceMax(dstTensor, srcTensor, srcTensor, calCount);
945- event_t eventIdVToS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S));
946- SetFlag<HardEvent::V_S>(eventIdVToS);
947- WaitFlag<HardEvent::V_S>(eventIdVToS);
948- T maxNum = dstTensor.GetValue(0);
949- return sizeof(T) * BITS_PER_BYTE - CountLeadingZeros(maxNum);
950- }
951-}
952- 
953-template<typename T>
954-__simd_vf__ inline void PowIComputeImpl(__ubuf__ T* dst, __ubuf__ T* src0, __ubuf__ T* src1, uint32_t calCount, uint16_t maxLoop)
955-{
956- using ConvType = typename PowICastType<T>::type;
957- constexpr uint16_t eleCountPerVL = GetVecLen() / sizeof(ConvType);
958- uint16_t repeatTime = DivCeil(calCount, eleCountPerVL);
959- 
960- MicroAPI::RegTensor<ConvType> baseReg, expReg;
961- MicroAPI::RegTensor<ConvType> initRetReg, dstReg;
962- MicroAPI::MaskReg mask;
963- 
964- MicroAPI::MaskReg fullMask = MicroAPI::CreateMask<T>();
965- MicroAPI::Duplicate(initRetReg, 1, fullMask);
966- for (uint16_t i = 0; i < repeatTime; i++) {
967- mask = MicroAPI::UpdateMask<ConvType>(calCount);
968- 
969- LoadSrcData(baseReg, src0, i, mask);
970- LoadSrcData(expReg, src1, i, mask);
971- dstReg = initRetReg;
972- GetPowICompute(dstReg, baseReg, expReg, mask, maxLoop);
973- StoreDstData(dst, dstReg, i, mask);
974- }
975-}
976- 
977-template<typename T>
978-__simd_vf__ inline void PowIComputeImpl(__ubuf__ T* dst, __ubuf__ T* src0, const T scalarValue, uint32_t calCount, uint16_t maxLoop)
979-{
980- using ConvType = typename PowICastType<T>::type;
981- constexpr uint16_t eleCountPerVL = GetVecLen() / sizeof(ConvType);
982- uint16_t repeatTime = DivCeil(calCount, eleCountPerVL);
983- 
984- MicroAPI::RegTensor<ConvType> baseReg, expReg;
985- MicroAPI::RegTensor<ConvType> initRetReg, dstReg;
986- MicroAPI::MaskReg mask;
987- 
988- MicroAPI::MaskReg fullMask = MicroAPI::CreateMask<T>();
989- MicroAPI::Duplicate(initRetReg, 1, fullMask);
990- MicroAPI::Duplicate(expReg, scalarValue, fullMask);
991- 
992- for (uint16_t i = 0; i < repeatTime; i++) {
993- mask = MicroAPI::UpdateMask<ConvType>(calCount);
994- LoadSrcData(baseReg, src0, i, mask);
995- dstReg = initRetReg;
996- GetPowICompute(dstReg, baseReg, expReg, mask, maxLoop);
997- StoreDstData(dst, dstReg, i, mask);
998- }
999-}
1000- 
1001-template<typename T>
1002-__simd_vf__ inline void PowIComputeImpl(__ubuf__ T* dst, const T scalarValue, __ubuf__ T* src1, uint32_t calCount, uint16_t maxLoop)
1003-{
1004- using ConvType = typename PowICastType<T>::type;
1005- constexpr uint16_t eleCountPerVL = GetVecLen() / sizeof(ConvType);
1006- uint16_t repeatTime = DivCeil(calCount, eleCountPerVL);
1007- 
1008- MicroAPI::RegTensor<ConvType> baseReg, expReg;
1009- MicroAPI::RegTensor<ConvType> initRetReg, dstReg;
1010- MicroAPI::MaskReg mask;
1011- MicroAPI::MaskReg fullMask = MicroAPI::CreateMask<T>();
1012- MicroAPI::Duplicate(initRetReg, 1, fullMask);
1013- MicroAPI::Duplicate(baseReg, scalarValue, fullMask);
1014- 
1015- for (uint16_t i = 0; i < repeatTime; i++) {
1016- mask = MicroAPI::UpdateMask<ConvType>(calCount);
1017- LoadSrcData(expReg, src1, i, mask);
1018- dstReg = initRetReg;
1019- GetPowICompute(dstReg, baseReg, expReg, mask, maxLoop);
1020- StoreDstData(dst, dstReg, i, mask);
1021- }
1022-}
1023- 
1024-} // namespace PowI
1025-} // namespace PowerL300Impl
1026- 
1027-template <typename T, const PowerConfig& config = defaultPowerConfig>
1028-__aicore__ inline void PowCheckType()
1029-{
1030- if constexpr (config.algo == AscendC::PowerAlgo::DOUBLE_FLOAT_TECH) {
1031- static_assert(SupportType<T, half, float, bfloat16_t>(),
1032- "Type must be half/float/bfloat16 in double float tech algorithm."
1033- );
1034- }
1035- 
1036- if constexpr (config.algo == AscendC::PowerAlgo::INTRINSIC) {
1037- static_assert(SupportType<T, uint8_t, int8_t, uint16_t, int16_t, uint32_t, int32_t, half, float>(),
1038- "Type must be uint8_t/int8_t/uint16_t/int16_t/uint32_t/int32_t/half/float in intrinsic tech algorithm."
1039- );
1040- }
1041-}
1042- 
1043-template <typename T>
1044-__aicore__ inline constexpr bool IsFloatNum()
1045-{
1046- return SupportType<T, float, half, bfloat16_t>();
1047-}
1048- 
1049-template <typename T>
1050-__aicore__ inline constexpr bool IsIntegerNum()
1051-{
1052- return SupportType<T, uint8_t, int8_t, uint16_t, int16_t, uint32_t, int32_t>();
1053-}
1054- 
1055-__aicore__ inline constexpr uint32_t GetPowerTmpBufferLiveNode() {
1056- constexpr uint32_t tmpBufferLiveNode = sizeof(float) * 2;
1057- return tmpBufferLiveNode;
1058-}
1059- 
1060-template<typename T>
1061-__aicore__ inline uint32_t GetPowTmpBufferSize(const LocalTensor<uint8_t>& sharedTmpBuffer) {
1062- uint32_t sharedTmpBufferSize = sharedTmpBuffer.GetSize() / GetPowerTmpBufferLiveNode();
1063- return AlignUp(sharedTmpBufferSize, GetDataBlockSizeInBytes());
1064-}
1065- 
1066-// PowImpl(tensor, tensor) float/half input
1067-template<typename T, const PowerConfig& config>
1068-__aicore__ inline void PowImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& src0Tensor,
1069- const LocalTensor<T>& src1Tensor, const LocalTensor<uint8_t>& sharedTmpBuffer, uint32_t calCount)
1070-{
1071- __ubuf__ T* src0 = (__ubuf__ T *)src0Tensor.GetPhyAddr();
1072- __ubuf__ T* src1 = (__ubuf__ T *)src1Tensor.GetPhyAddr();
1073- __ubuf__ T* dst = (__ubuf__ T *)dstTensor.GetPhyAddr();
1074- 
1075- if constexpr (IsFloatNum<T>()) {
1076- if constexpr (config.algo == PowerAlgo::INTRINSIC) {
1077- PowerL300Impl::PowF::PowFInstrinsicImpl(dst, src0, src1, calCount);
1078- } else {
1079- __ubuf__ uint32_t* tmpBuffer = (__ubuf__ uint32_t *)sharedTmpBuffer.GetPhyAddr();
1080- uint32_t sharedTmpBufferSize = GetPowTmpBufferSize<T>(sharedTmpBuffer);
1081- uint32_t count = calCount;
1082- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(calCount, sharedTmpBufferSize));
1083- for (uint16_t i = 0; i < repeatTimes; i++) {
1084- uint32_t remainCount = count - sharedTmpBufferSize * i;
1085- uint32_t oneRepSize = remainCount < sharedTmpBufferSize ? remainCount : sharedTmpBufferSize;
1086- PowerL300Impl::PowF::PowFComputeImpl(dst + i * sharedTmpBufferSize, src0 + i * sharedTmpBufferSize,
1087- src1 + i * sharedTmpBufferSize, tmpBuffer, oneRepSize);
1088- }
1089- }
1090- } else if constexpr (IsIntegerNum<T>()) {
1091- uint16_t maxLoop = PowerL300Impl::PowI::GetMaxLoop(dstTensor, src1Tensor, calCount);
1092- event_t eventIdSToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V));
1093- SetFlag<HardEvent::S_V>(eventIdSToV);
1094- WaitFlag<HardEvent::S_V>(eventIdSToV);
1095- PowerL300Impl::PowI::PowIComputeImpl<T>(dst, src0, src1, calCount, maxLoop);
1096- }
1097-}
1098- 
1099-// PowImpl(tensor, scalar) float input
1100-template<typename T, const PowerConfig& config>
1101-__aicore__ inline void PowImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& src0Tensor,
1102- const T& scalarValue, const LocalTensor<uint8_t>& sharedTmpBuffer, uint32_t calCount)
1103-{
1104- __ubuf__ T *base = (__ubuf__ T *)src0Tensor.GetPhyAddr();
1105- __ubuf__ T *dst = (__ubuf__ T *)dstTensor.GetPhyAddr();
1106- 
1107- if constexpr (IsFloatNum<T>()) {
1108- if constexpr (config.algo == PowerAlgo::INTRINSIC) {
1109- PowerL300Impl::PowF::PowFInstrinsicImpl(dst, base, scalarValue, calCount);
1110- } else {
1111- __ubuf__ uint32_t* tmpBuffer = (__ubuf__ uint32_t *)sharedTmpBuffer.GetPhyAddr();
1112- uint32_t sharedTmpBufferSize = GetPowTmpBufferSize<T>(sharedTmpBuffer);
1113- uint32_t count = calCount;
1114- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(calCount, sharedTmpBufferSize));
1115- for (uint16_t i = 0; i < repeatTimes; i++) {
1116- uint32_t remainCount = count - sharedTmpBufferSize * i;
1117- uint32_t oneRepSize = remainCount < sharedTmpBufferSize ? remainCount : sharedTmpBufferSize;
1118- PowerL300Impl::PowF::PowFComputeImpl(
1119- dst + i * sharedTmpBufferSize, base + i * sharedTmpBufferSize, scalarValue, tmpBuffer, oneRepSize);
1120- }
1121- }
1122- } else if constexpr (IsIntegerNum<T>()) {
1123- uint16_t maxLoop = sizeof(T) * PowerL300Impl::PowI::BITS_PER_BYTE - PowerL300Impl::PowI::CountLeadingZeros(scalarValue);
1124- PowerL300Impl::PowI::PowIComputeImpl<T>(dst, base, scalarValue, calCount, maxLoop);
1125- }
1126-}
1127- 
1128-// PowImpl(scalar, tensor) float input
1129-template<typename T, const PowerConfig& config>
1130-__aicore__ inline void PowImpl(const LocalTensor<T>& dstTensor, const T& scalarValue,
1131- const LocalTensor<T>& src1Tensor, const LocalTensor<uint8_t>& sharedTmpBuffer, uint32_t calCount)
1132-{
1133- __ubuf__ T *exp = (__ubuf__ T *)src1Tensor.GetPhyAddr();
1134- __ubuf__ T *dst = (__ubuf__ T *)dstTensor.GetPhyAddr();
1135- 
1136- if constexpr (IsFloatNum<T>()) {
1137- if constexpr (config.algo == PowerAlgo::INTRINSIC) {
1138- PowerL300Impl::PowF::PowFInstrinsicImpl(dst, scalarValue, exp, calCount);
1139- } else {
1140- __ubuf__ uint32_t* tmpBuffer = (__ubuf__ uint32_t *)sharedTmpBuffer.GetPhyAddr();
1141- uint32_t sharedTmpBufferSize = GetPowTmpBufferSize<T>(sharedTmpBuffer);
1142- uint32_t count = calCount;
1143- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(calCount, sharedTmpBufferSize));
1144- for (uint16_t i = 0; i < repeatTimes; i++) {
1145- uint32_t remainCount = count - sharedTmpBufferSize * i;
1146- uint32_t oneRepSize = remainCount < sharedTmpBufferSize ? remainCount : sharedTmpBufferSize;
1147- PowerL300Impl::PowF::PowFComputeImpl(dst + i * sharedTmpBufferSize, scalarValue,
1148- exp + i * sharedTmpBufferSize, tmpBuffer, oneRepSize);
1149- }
1150- }
1151- } else if constexpr (IsIntegerNum<T>()) {
1152- uint16_t maxLoop = PowerL300Impl::PowI::GetMaxLoop(dstTensor, src1Tensor, calCount);
1153- event_t eventIdSToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V));
1154- SetFlag<HardEvent::S_V>(eventIdSToV);
1155- WaitFlag<HardEvent::S_V>(eventIdSToV);
1156- PowerL300Impl::PowI::PowIComputeImpl<T>(dst, scalarValue, exp, calCount, maxLoop);
1157- }
1158-}
1159- 
1160-template<typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1161-__aicore__ inline void PowerCommonImpl(const LocalTensor<T>& dstTensor, const T& scalarValue,
1162- const LocalTensor<T>& src1Tensor, const LocalTensor<uint8_t>& sharedTmpBuffer, uint32_t calCount)
1163-{
1164- if ASCEND_IS_AIC {
1165- return;
1166- }
1167- PowCheckType<T, config>();
1168- CheckTensorPos<T>(dstTensor, Hardware::UB, "dstTensor", "VECIN / VECOUT / VECCALC", "Power");
1169- CheckTensorPos<T>(src1Tensor, Hardware::UB, "src1Tensor", "VECIN / VECOUT / VECCALC", "Power");
1170- CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN / VECOUT / VECCALC", "Power");
1171- CheckCalCount(calCount, "calCount", src1Tensor, "src1Tensor", "Power");
1172- CheckCalCount(calCount, "calCount", dstTensor, "dstTensor", "Power");
1173- 
1174- PowImpl<T, config>(dstTensor, scalarValue, src1Tensor, sharedTmpBuffer, calCount);
1175-}
1176- 
1177-template<typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1178-__aicore__ inline void PowerCommonImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& src0Tensor,
1179- const T& scalarValue, const LocalTensor<uint8_t>& sharedTmpBuffer, uint32_t calCount)
1180-{
1181- if ASCEND_IS_AIC {
1182- return;
1183- }
1184- PowCheckType<T, config>();
1185- CheckTensorPos<T>(dstTensor, Hardware::UB, "dstTensor", "VECIN / VECOUT / VECCALC", "Power");
1186- CheckTensorPos<T>(src0Tensor, Hardware::UB, "src0Tensor", "VECIN / VECOUT / VECCALC", "Power");
1187- CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN / VECOUT / VECCALC", "Power");
1188- CheckCalCount(calCount, "calCount", src0Tensor, "src0Tensor", "Power");
1189- CheckCalCount(calCount, "calCount", dstTensor, "dstTensor", "Power");
1190- 
1191- PowImpl<T, config>(dstTensor, src0Tensor, scalarValue, sharedTmpBuffer, calCount);
1192-}
1193- 
1194-template<typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1195-__aicore__ inline void PowerCommonImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& src0Tensor,
1196- const LocalTensor<T>& src1Tensor, const LocalTensor<uint8_t>& sharedTmpBuffer, uint32_t calCount)
1197-{
1198- if ASCEND_IS_AIC {
1199- return;
1200- }
1201- PowCheckType<T, config>();
1202- CheckTensorPos<T>(dstTensor, Hardware::UB, "dstTensor", "VECIN / VECOUT / VECCALC", "Power");
1203- CheckTensorPos<T>(src0Tensor, Hardware::UB, "src0Tensor", "VECIN / VECOUT / VECCALC", "Power");
1204- CheckTensorPos<T>(src1Tensor, Hardware::UB, "src1Tensor", "VECIN / VECOUT / VECCALC", "Power");
1205- CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN / VECOUT / VECCALC", "Power");
1206- CheckCalCount(calCount, "calCount", src0Tensor, "src0Tensor", "Power");
1207- CheckCalCount(calCount, "calCount", src1Tensor, "src1Tensor", "Power");
1208- CheckCalCount(calCount, "calCount", dstTensor, "dstTensor", "Power");
1209- 
1210- PowImpl<T, config>(dstTensor, src0Tensor, src1Tensor, sharedTmpBuffer, calCount);
1211-}
1212- 
1213-template<typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1214-__aicore__ inline void PowerCommonImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& src0Tensor,
1215- const LocalTensor<T>& src1Tensor, uint32_t calCount)
1216-{
1217- LocalTensor<uint8_t> stackTensor;
1218- bool ans = PopStackBuffer<uint8_t, TPosition::LCM>(stackTensor);
1219- ASCENDC_ASSERT((ans),
1220- { KERNEL_LOG(KERNEL_ERROR, "PopStackBuffer Error!"); });
1221- PowerCommonImpl<T, isReuseSource, config>(dstTensor, src0Tensor, src1Tensor, stackTensor, calCount);
1222-}
1223- 
1224-template<typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1225-__aicore__ inline void PowerCommonImpl(const LocalTensor<T>& dstTensor, const T& src0Scalar,
1226- const LocalTensor<T>& src1Tensor, uint32_t calCount)
1227-{
1228- LocalTensor<uint8_t> stackTensor;
1229- bool ans = PopStackBuffer<uint8_t, TPosition::LCM>(stackTensor);
1230- ASCENDC_ASSERT((ans),
1231- { KERNEL_LOG(KERNEL_ERROR, "PopStackBuffer Error!"); });
1232- PowerCommonImpl<T, isReuseSource, config>(dstTensor, src0Scalar, src1Tensor, stackTensor, calCount);
1233-}
1234- 
1235-template<typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1236-__aicore__ inline void PowerCommonImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& src0Tensor,
1237- const T& src1Scalar, uint32_t calCount)
1238-{
1239- LocalTensor<uint8_t> stackTensor;
1240- bool ans = PopStackBuffer<uint8_t, TPosition::LCM>(stackTensor);
1241- ASCENDC_ASSERT((ans),
1242- { KERNEL_LOG(KERNEL_ERROR, "PopStackBuffer Error!"); });
1243- PowerCommonImpl<T, isReuseSource, config>(dstTensor, src0Tensor, src1Scalar, stackTensor, calCount);
1244-}
1245- 
1246-/*!
1247- * \ingroup Power
1248- * \brief compute Power elementwisely.
1249- * \tparam T: input dataType, support half/float
1250- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
1251- * not enabled currently.
1252- * \param [out] dstTensor: output LocalTensor
1253- * \param [in] src0Tensor: base LocalTensor
1254- * \param [in] src1Tensor: exponent LocalTensor
1255- * \param [in] sharedTmpBuffer：extra temporary shared space used for intermediate values among calculation process,
1256- * whose required space size should refer to corresponding tiling API, which is defined at
1257- * power_tiling.h. Generally, the more space you allocate, the better performance you will achieve,
1258- * and the performance reaches peak when buffer size is maximum(calculated by tiling function). Moreover, it
1259- * is not guaranteed that the shared space will be cleared after usage, the data could be anything.
1260- * \param [in] calCount: amount of output data to be calculated
1261- */
1262-template <typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1263-__aicore__ inline void Power(const LocalTensor<T>& dstTensor, const LocalTensor<T>& src0Tensor,
1264- const LocalTensor<T>& src1Tensor, const LocalTensor<uint8_t>& sharedTmpBuffer, uint32_t calCount)
1265-{
1266- PowerCommonImpl<T, isReuseSource, config>(dstTensor, src0Tensor, src1Tensor, sharedTmpBuffer, calCount);
1267-}
1268- 
1269-/*!
1270- * \ingroup Power
1271- * \brief compute Power elementwisely.
1272- * \tparam T: input dataType, support half/float
1273- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
1274- * not enabled currently.
1275- * \param [out] dstTensor: output LocalTensor
1276- * \param [in] src0Tensor: base LocalTensor
1277- * \param [in] src1Tensor: exponent LocalTensor
1278- * \param [in] calCount: amount of output data to be calculated
1279- */
1280-template <typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1281-__aicore__ inline void Power(const LocalTensor<T>& dstTensor, const LocalTensor<T>& src0Tensor,
1282- const LocalTensor<T>& src1Tensor, uint32_t calCount)
1283-{
1284- PowerCommonImpl<T, isReuseSource, config>(dstTensor, src0Tensor, src1Tensor, calCount);
1285-}
1286- 
1287-/*!
1288- * \ingroup Power
1289- * \brief compute Power elementwisely.
1290- * \tparam T: input dataType, support half/float
1291- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
1292- * not enabled currently.
1293- * \param [out] dstTensor: output LocalTensor
1294- * \param [in] src0Tensor: base LocalTensor
1295- * \param [in] src1Tensor: exponent LocalTensor
1296- * \param [in] sharedTmpBuffer：extra temporary shared space used for intermediate values among calculation process,
1297- * whose required space size should refer to corresponding tiling API, which is defined at
1298- * power_tiling.h. Generally, the more space you allocate, the better performance you will achieve,
1299- * and the performance reaches peak when buffer size is maximum(calculated by tiling function). Moreover, it
1300- * is not guaranteed that the shared space will be cleared after usage, the data could be anything.
1301- */
1302-template <typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1303-__aicore__ inline void Power(const LocalTensor<T>& dstTensor, const LocalTensor<T>& src0Tensor,
1304- const LocalTensor<T>& src1Tensor, const LocalTensor<uint8_t>& sharedTmpBuffer)
1305-{
1306- Power<T, isReuseSource, config>(dstTensor, src0Tensor, src1Tensor, sharedTmpBuffer, src0Tensor.GetSize());
1307-}
1308- 
1309-/*!
1310- * \ingroup Power
1311- * \brief compute Power elementwisely.
1312- * \tparam T: input dataType, support half/float
1313- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
1314- * not enabled currently.
1315- * \param [out] dstTensor: output LocalTensor
1316- * \param [in] src0Tensor: base LocalTensor
1317- * \param [in] src1Tensor: exponent LocalTensor
1318- */
1319-template <typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1320-__aicore__ inline void Power(
1321- const LocalTensor<T>& dstTensor, const LocalTensor<T>& src0Tensor, const LocalTensor<T>& src1Tensor)
1322-{
1323- Power<T, isReuseSource, config>(dstTensor, src0Tensor, src1Tensor, src0Tensor.GetSize());
1324-}
1325- 
1326-/*!
1327- * \ingroup Power
1328- * \brief compute Power elementwisely.
1329- * \tparam T: input dataType, support half/float
1330- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
1331- * not enabled currently.
1332- * \param [out] dstTensor: output LocalTensor
1333- * \param [in] src0Tensor: base LocalTensor
1334- * \param [in] src1Scalar: exponent Scalar
1335- * \param [in] sharedTmpBuffer：extra temporary shared space used for intermediate values among calculation process,
1336- * whose required space size should refer to corresponding tiling API, which is defined at
1337- * power_tiling.h. Generally, the more space you allocate, the better performance you will achieve,
1338- * and the performance reaches peak when buffer size is maximum(calculated by tiling function). Moreover, it
1339- * is not guaranteed that the shared space will be cleared after usage, the data could be anything.
1340- * \param [in] calCount: amount of output data to be calculated
1341- */
1342-template <typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1343-__aicore__ inline void Power(const LocalTensor<T>& dstTensor, const LocalTensor<T>& src0Tensor, const T& src1Scalar,
1344- const LocalTensor<uint8_t>& sharedTmpBuffer, uint32_t calCount)
1345-{
1346- PowerCommonImpl<T, isReuseSource, config>(dstTensor, src0Tensor, src1Scalar, sharedTmpBuffer, calCount);
1347-}
1348- 
1349-/*!
1350- * \ingroup Power
1351- * \brief compute Power elementwisely.
1352- * \tparam T: input dataType, support half/float
1353- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
1354- * not enabled currently.
1355- * \param [out] dstTensor: output LocalTensor
1356- * \param [in] src0Tensor: base LocalTensor
1357- * \param [in] src1Scalar: exponent Scalar
1358- * \param [in] calCount: amount of output data to be calculated
1359- */
1360-template <typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1361-__aicore__ inline void Power(
1362- const LocalTensor<T>& dstTensor, const LocalTensor<T>& src0Tensor, const T& src1Scalar, uint32_t calCount)
1363-{
1364- PowerCommonImpl<T, isReuseSource, config>(dstTensor, src0Tensor, src1Scalar, calCount);
1365-}
1366- 
1367-/*!
1368- * \ingroup Power
1369- * \brief compute Power elementwisely.
1370- * \tparam T: input dataType, support half/float
1371- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
1372- * not enabled currently.
1373- * \param [out] dstTensor: output LocalTensor
1374- * \param [in] src0Tensor: base LocalTensor
1375- * \param [in] src1Scalar: exponent Scalar
1376- * \param [in] sharedTmpBuffer：extra temporary shared space used for intermediate values among calculation process,
1377- * whose required space size should refer to corresponding tiling API, which is defined at
1378- * power_tiling.h. Generally, the more space you allocate, the better performance you will achieve,
1379- * and the performance reaches peak when buffer size is maximum(calculated by tiling function). Moreover, it
1380- * is not guaranteed that the shared space will be cleared after usage, the data could be anything.
1381- */
1382-template <typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1383-__aicore__ inline void Power(const LocalTensor<T>& dstTensor, const LocalTensor<T>& src0Tensor, const T& src1Scalar,
1384- const LocalTensor<uint8_t>& sharedTmpBuffer)
1385-{
1386- Power<T, isReuseSource, config>(dstTensor, src0Tensor, src1Scalar, sharedTmpBuffer, src0Tensor.GetSize());
1387-}
1388- 
1389-/*!
1390- * \ingroup Power
1391- * \brief compute Power elementwisely.
1392- * \tparam T: input dataType, support half/float
1393- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
1394- * not enabled currently.
1395- * \param [out] dstTensor: output LocalTensor
1396- * \param [in] src0Tensor: base LocalTensor
1397- * \param [in] src1Scalar: exponent Scalar
1398- */
1399-template <typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1400-__aicore__ inline void Power(const LocalTensor<T>& dstTensor, const LocalTensor<T>& src0Tensor, const T& src1Scalar)
1401-{
1402- Power<T, isReuseSource, config>(dstTensor, src0Tensor, src1Scalar, src0Tensor.GetSize());;
1403-}
1404- 
1405-/*!
1406- * \ingroup Power
1407- * \brief compute Power elementwisely.
1408- * \tparam T: input dataType, support half/float
1409- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
1410- * not enabled currently.
1411- * \param [out] dstTensor: output LocalTensor
1412- * \param [in] src0Scalar: base Scalar
1413- * \param [in] src1Tensor: exponent LocalTensor
1414- * \param [in] sharedTmpBuffer：extra temporary shared space used for intermediate values among calculation process,
1415- * whose required space size should refer to corresponding tiling API, which is defined at
1416- * power_tiling.h. Generally, the more space you allocate, the better performance you will achieve,
1417- * and the performance reaches peak when buffer size is maximum(calculated by tiling function). Moreover, it
1418- * is not guaranteed that the shared space will be cleared after usage, the data could be anything.
1419- * \param [in] calCount: amount of output data to be calculated
1420- */
1421-template <typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1422-__aicore__ inline void Power(const LocalTensor<T>& dstTensor, const T& src0Scalar, const LocalTensor<T>& src1Tensor,
1423- const LocalTensor<uint8_t>& sharedTmpBuffer, uint32_t calCount)
1424-{
1425- PowerCommonImpl<T, isReuseSource, config>(dstTensor, src0Scalar, src1Tensor, sharedTmpBuffer, calCount);
1426-}
1427- 
1428-/*!
1429- * \ingroup Power
1430- * \brief compute Power elementwisely.
1431- * \tparam T: input dataType, support half/float
1432- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
1433- * not enabled currently.
1434- * \param [out] dstTensor: output LocalTensor
1435- * \param [in] src0Scalar: input Scalar
1436- * \param [in] src1Tensor: exponent LocalTensor
1437- * \param [in] calCount: amount of output data to be calculated
1438- */
1439-template <typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1440-__aicore__ inline void Power(
1441- const LocalTensor<T>& dstTensor, const T& src0Scalar, const LocalTensor<T>& src1Tensor, uint32_t calCount)
1442-{
1443- PowerCommonImpl<T, isReuseSource, config>(dstTensor, src0Scalar, src1Tensor, calCount);
1444-}
1445- 
1446-/*!
1447- * \ingroup Power
1448- * \brief compute Power elementwisely.
1449- * \tparam T: input dataType, support half/float
1450- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
1451- * not enabled currently.
1452- * \param [out] dstTensor: output LocalTensor
1453- * \param [in] src0Scalar: base Scalar
1454- * \param [in] src1Tensor: exponent LocalTensor
1455- * \param [in] sharedTmpBuffer：extra temporary shared space used for intermediate values among calculation process,
1456- * whose required space size should refer to corresponding tiling API, which is defined at
1457- * power_tiling.h. Generally, the more space you allocate, the better performance you will achieve,
1458- * and the performance reaches peak when buffer size is maximum(calculated by tiling function). Moreover, it
1459- * is not guaranteed that the shared space will be cleared after usage, the data could be anything.
1460- */
1461-template <typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1462-__aicore__ inline void Power(const LocalTensor<T>& dstTensor, const T& src0Scalar, const LocalTensor<T>& src1Tensor,
1463- const LocalTensor<uint8_t>& sharedTmpBuffer)
1464-{
1465- PowerCommonImpl<T, isReuseSource, config>(dstTensor, src0Scalar, src1Tensor, sharedTmpBuffer, src1Tensor.GetSize());
1466-}
1467- 
1468-/*!
1469- * \ingroup Power
1470- * \brief compute Power elementwisely.
1471- * \tparam T: input dataType, support half/float
1472- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
1473- * not enabled currently.
1474- * \param [out] dstTensor: output LocalTensor
1475- * \param [in] src0Scalar: base Scalar
1476- * \param [in] src1Tensor: exponent LocalTensor
1477- */
1478-template <typename T, bool isReuseSource = false, const PowerConfig& config = defaultPowerConfig>
1479-__aicore__ inline void Power(const LocalTensor<T>& dstTensor, const T& src0Scalar, const LocalTensor<T>& src1Tensor)
1480-{
1481- PowerCommonImpl<T, isReuseSource, config>(dstTensor, src0Scalar, src1Tensor, src1Tensor.GetSize());
1482-}
1483-} //namesapce AscendC
1484-#endif
Dimpl/adv_api/detail/math/sign/sign_l300_impl.h+0-89
@@ -1,89 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/*!
12- * \file sign_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_MATH_SIGN_SIGN_L300_IMPL_H
16-#define IMPL_MATH_SIGN_SIGN_L300_IMPL_H
17- 
18-#include "kernel_tensor.h"
19-#include "../../common/check.h"
20- 
21-namespace AscendC {
22-namespace SignInternal {
23-template <typename T, typename RegT, const MicroAPI::RegTrait& trait = MicroAPI::RegTraitNumOne>
24-__simd_vf__ inline void SignCoreCompute(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint32_t calCount, uint16_t repeatTime, uint32_t vlSize)
25-{
26- MicroAPI::MaskReg signMask;
27- MicroAPI::MaskReg cmpMask0;
28- MicroAPI::MaskReg cmpMask1;
29- RegT brcZeroReg;
30- RegT brcOneReg;
31- RegT brcNegOneReg;
32- RegT srcReg;
33- RegT selReg0;
34- RegT selReg1;
35- MicroAPI::Duplicate(brcZeroReg, 0);
36- MicroAPI::Duplicate(brcOneReg, 1);
37- MicroAPI::Duplicate(brcNegOneReg, -1);
38- for (uint16_t i = 0; i < repeatTime; ++i) {
39- signMask = MicroAPI::UpdateMask<T, trait>(calCount);
40- MicroAPI::DataCopy(srcReg, srcUb + i * vlSize);
41- MicroAPI::CompareScalar<T, CMPMODE::LT>(cmpMask0, srcReg, 0, signMask);
42- MicroAPI::CompareScalar<T, CMPMODE::GT>(cmpMask1, srcReg, 0, signMask);
43- MicroAPI::Select(selReg0, brcNegOneReg, brcZeroReg, cmpMask0);
44- MicroAPI::Select(selReg1, brcOneReg, selReg0, cmpMask1);
45- MicroAPI::DataCopy(dstUb + i * vlSize, selReg1, signMask);
46- }
47-}
48-} // namespace SignInternal
49- 
50-template <typename T, bool isReuseSource = false>
51-__aicore__ inline void SignCompute(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
52- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
53-{
54- // Only for AI Vector Core.
55- if ASCEND_IS_AIC {
56- return;
57- }
58- 
59- CheckTensorPosition(sharedTmpBuffer, "sharedTmpBuffer", "VECIN, VECOUT, VECCALC");
60- SignCompute<T, isReuseSource>(dstTensor, srcTensor, calCount);
61-}
62- 
63-template <typename T, bool isReuseSource = false>
64-__aicore__ inline void SignCompute(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
65- const uint32_t calCount)
66-{
67- static_assert(SupportType<T, half, float, int64_t>(), "Sign only support half/float/int64_t data type on current device!");
68- CheckTensorPosition(dstTensor, "dstTensor", "VECIN, VECOUT, VECCALC");
69- CheckTensorPosition(srcTensor, "srcTensor", "VECIN, VECOUT, VECCALC");
70- CheckCalCount(calCount, "calCount", srcTensor, "srcTensor", "Sign");
71- CheckCalCount(calCount, "calCount", dstTensor, "dstTensor", "Sign");
72- constexpr uint32_t SIGN_B64_REPEAT_STRIDE = 2;
73- __ubuf__ T *dstUb = (__ubuf__ T *)dstTensor.GetPhyAddr();
74- __ubuf__ T *srcUb = (__ubuf__ T *)srcTensor.GetPhyAddr();
75- if constexpr (sizeof(T) == 8) {
76- using RegT = MicroAPI::RegTensor<T, MicroAPI::RegTraitNumTwo>;
77- constexpr int32_t vlSize = static_cast<int32_t>(GetVecLen() / sizeof(T) * SIGN_B64_REPEAT_STRIDE);
78- uint16_t repeatTime = static_cast<uint16_t>(CeilDivision(calCount, vlSize));
79- SignInternal::SignCoreCompute<T, RegT, MicroAPI::RegTraitNumTwo>(dstUb, srcUb, calCount, repeatTime, vlSize);
80- } else {
81- using RegT = MicroAPI::RegTensor<T>;
82- constexpr int32_t vlSize = static_cast<int32_t>(GetVecLen() / sizeof(T));
83- uint16_t repeatTime = static_cast<uint16_t>(CeilDivision(calCount, vlSize));
84- SignInternal::SignCoreCompute<T, RegT>(dstUb, srcUb, calCount, repeatTime, vlSize);
85- }
86-}
87-} // namespace AscendC
88- 
89-#endif // IMPL_MATH_SIGN_SIGN_L300_IMPL_H
Dimpl/adv_api/detail/math/sin/sin_l300_impl.h+0-298
@@ -1,298 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/* !
12- * \file sin_l300_impl.h
13- * \brief
14- */
15-#ifndef LIB_MATH_SIN_L300_IMPL_H
16-#define LIB_MATH_SIN_L300_IMPL_H
17-#include "kernel_tensor.h"
18-#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3003 || __NPU_ARCH__ == 3113)
19-#include "sin_common_utils.h"
20-#endif
21- 
22-namespace AscendC {
23-namespace MicroAPI {
24-namespace Sin {
25-const uint8_t SIN_FLOAT_NOREUSE_CALC_PROCEDURE = 3;
26-const uint8_t SIN_FLOAT_REUSE_CALC_PROCEDURE = 2;
27- 
28-// define the number of x div pi
29-constexpr float SIN_PI_FOR_X_TODIV = 0.3183098733425140380859375;
30-// define the PI for compute
31-constexpr float SIN_PI_V2 = 3.140625;
32-constexpr float SIN_KPI_FIRS_PI_MULS = 0.0009670257568359375;
33-constexpr float SIN_KPI_TWI_PI_MULS = 6.2771141529083251953125e-7;
34-constexpr float SIN_KPI_THIR_PI_MULS = 1.21644916362129151821136474609375e-10;
35-// define the number of sin compute
36-constexpr float SIN_RES_MULIT_SCA = 2.604926501e-6;
37-constexpr float SIN_RES_ADDICT_UP = -0.0001980894471;
38-constexpr float SIN_2ADDS = 0.008333049340;
39-constexpr float SIN_3ADDS = -0.1666665792;
40-constexpr float SIN_POINT_FIVE = 0.5;
41-constexpr float SIN_M4_SCA = 4.0;
42-constexpr float SIN_K2_SCA = -2.0;
43-constexpr float SIN_SCALAR_ONE = 1.0;
44-constexpr MicroAPI::CastTrait sinCastTraitF16F32 = { MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::UNKNOWN,
45- MicroAPI::MaskMergeMode::ZEROING };
46-constexpr MicroAPI::CastTrait sinCastTraitF32F16 = { MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::NO_SAT,
47- MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_RINT };
48- 
49-__simd_callee__ inline void SinPolynomialApproximation(MicroAPI::RegTensor<float> &dstReg,
50- MicroAPI::RegTensor<float> &srcReg, MicroAPI::RegTensor<float> &x, MicroAPI::RegTensor<float> &round,
51- MicroAPI::RegTensor<float> &kpi, MicroAPI::MaskReg mask)
52-{
53- // normalized x to [-π/2, π/2] using x = x-round(x/π)*π
54- // k = round(x * invpi)
55- MicroAPI::Muls(round, srcReg, SIN_PI_FOR_X_TODIV, mask);
56- MicroAPI::Truncate<float, RoundMode::CAST_RINT, MicroAPI::MaskMergeMode::ZEROING>(round, round, mask);
57- // x -= k * pi_0
58- MicroAPI::Muls(kpi, round, SIN_PI_V2, mask);
59- MicroAPI::Sub(x, srcReg, kpi, mask);
60- // x -= k * pi_1
61- MicroAPI::Muls(kpi, round, SIN_KPI_FIRS_PI_MULS, mask);
62- MicroAPI::Sub(x, x, kpi, mask);
63- // x -= k * pi_2
64- MicroAPI::Muls(kpi, round, SIN_KPI_TWI_PI_MULS, mask);
65- MicroAPI::Sub(x, x, kpi, mask);
66- // x -= k * pi_3
67- MicroAPI::Muls(kpi, round, SIN_KPI_THIR_PI_MULS, mask);
68- MicroAPI::Sub(x, x, kpi, mask);
69- 
70- /*
71- sin(x) = (-1)^k*sin(x0)
72- Finally, use sin(x) = xP(x) to calculate sin(x).
73- P(x) = (((x^2 * R0 + R1) * x^2 + R2) * x^2 + R3) * x^2 + 1.0
74- */
75- // x^2 = mul(input_x, input_x)
76- MicroAPI::Mul(kpi, x, x, mask);
77- // kover2
78- MicroAPI::Muls(dstReg, round, SIN_POINT_FIVE, mask);
79- MicroAPI::Truncate<float, RoundMode::CAST_FLOOR, MicroAPI::MaskMergeMode::ZEROING>(dstReg, dstReg, mask);
80- // kover2floorm4
81- MicroAPI::Muls(dstReg, dstReg, SIN_M4_SCA, mask);
82- // k2
83- MicroAPI::Muls(round, round, SIN_K2_SCA, mask);
84- // sign
85- MicroAPI::Add(dstReg, dstReg, round, mask);
86- MicroAPI::Adds(dstReg, dstReg, SIN_SCALAR_ONE, mask);
87- // res_up = mul(x^2, 2.604926501e-6)
88- MicroAPI::Muls(round, kpi, SIN_RES_MULIT_SCA, mask);
89- MicroAPI::Adds(round, round, SIN_RES_ADDICT_UP, mask);
90- // res_up = mul(res_up, x^2)
91- MicroAPI::Mul(round, round, kpi, mask);
92- MicroAPI::Adds(round, round, SIN_2ADDS, mask);
93- // res_up = mul(res_up, x^2)
94- MicroAPI::Mul(round, round, kpi, mask);
95- MicroAPI::Adds(round, round, SIN_3ADDS, mask);
96- // res_up = mul(res_up, x^2)
97- MicroAPI::Mul(round, round, kpi, mask);
98- MicroAPI::Adds(round, round, SIN_SCALAR_ONE, mask);
99- // res_up = mul(res_up, input_x)
100- MicroAPI::Mul(round, round, x, mask);
101- MicroAPI::Mul(dstReg, round, dstReg, mask);
102-}
103- 
104-template <typename T>
105-__simd_vf__ inline void SinPolynomial(__ubuf__ T *dst, __ubuf__ T *src, uint32_t calCount, uint16_t repeat)
106-{
107- MicroAPI::RegTensor<T> x;
108- MicroAPI::RegTensor<float> xTmp;
109- MicroAPI::RegTensor<float> round;
110- MicroAPI::RegTensor<float> kpi;
111- MicroAPI::RegTensor<T> srcReg;
112- MicroAPI::RegTensor<float> srcTmp;
113- MicroAPI::RegTensor<T> dstReg;
114- MicroAPI::RegTensor<float> dstTmp;
115- MicroAPI::MaskReg mask;
116- constexpr uint32_t oneRepSize = GetVecLen() / sizeof(float);
117- MicroAPI::MaskReg maskAll = MicroAPI::CreateMask<uint8_t>();
118- for (uint16_t i = 0; i < (uint16_t)repeat; i++) {
119- mask = MicroAPI::UpdateMask<float>(calCount);
120- if constexpr (std::is_same<T, half>::value) {
121- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcReg, src + i * oneRepSize);
122- MicroAPI::Cast<float, half, sinCastTraitF16F32>(srcTmp, srcReg, mask);
123- SinPolynomialApproximation(dstTmp, srcTmp, xTmp, round, kpi, mask);
124- MicroAPI::Cast<half, float, sinCastTraitF32F16>(dstReg, dstTmp, mask);
125- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(dst + i * oneRepSize, dstReg, mask);
126- } else {
127- MicroAPI::DataCopy(srcReg, src + i * oneRepSize);
128- SinPolynomialApproximation(dstReg, srcReg, xTmp, round, kpi, mask);
129- MicroAPI::DataCopy(dst + i * oneRepSize, dstReg, mask);
130- }
131- }
132-}
133- 
134-template <typename T> __aicore__ inline void SinPolynomialImpl(__ubuf__ T *dst, __ubuf__ T *src, uint32_t calCount)
135-{
136- constexpr uint32_t oneRepSize = GetVecLen() / sizeof(float);
137- uint16_t repeat = CeilDivision(calCount, oneRepSize);
138- SinPolynomial<T>(dst, src, calCount, repeat);
139-}
140-} // namespace MicroAPI
141-} // namespace Sin
142- 
143-__aicore__ inline constexpr uint32_t GetSinTmpBufferLiveNode() {
144- constexpr uint32_t tmpBufferLiveNode = sizeof(float) * 2;
145- return tmpBufferLiveNode;
146-}
147- 
148-template <typename T, bool isReuseSource = false, const SinConfig& config = defaultSinConfig>
149-__aicore__ inline void SinImpl(
150- const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor, const uint32_t calCount)
151-{
152- // Only for AI Vector Core.
153- if ASCEND_IS_AIC {
154- return;
155- }
156- 
157- // Using the Stack Space to Allocate tmpBuffer
158- LocalTensor<uint8_t> sharedTmpBuffer;
159- bool ans = PopStackBuffer<uint8_t, TPosition::LCM>(sharedTmpBuffer);
160- ASCENDC_ASSERT((ans), { KERNEL_LOG(KERNEL_ERROR, "PopStackBuffer Error!"); });
161- SinImpl<T, isReuseSource, config>(dstTensor, srcTensor, sharedTmpBuffer, calCount);
162-}
163- 
164-template<typename T>
165-__aicore__ inline uint32_t GetSinTmpBufferSize(const LocalTensor<uint8_t>& sharedTmpBuffer) {
166- uint32_t sharedTmpBufferSize = sharedTmpBuffer.GetSize() / GetSinTmpBufferLiveNode();
167- return AlignUp(sharedTmpBufferSize, GetDataBlockSizeInBytes()) / sizeof(T);
168-}
169- 
170-template <typename T, bool isReuseSource = false, const SinConfig &config = defaultSinConfig>
171-__aicore__ inline void SinImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
172- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
173-{
174- // Only for AI Vector Core.
175- if ASCEND_IS_AIC {
176- return;
177- }
178- 
179- static_assert(SupportType<T, half, float>(), "current data type is not supported on current device!");
180- CheckTensorPos<T>(dstTensor, Hardware::UB, "dstTensor", "VECIN / VECCALC / VECOUT", "Sin");
181- CheckTensorPos<T>(srcTensor, Hardware::UB, "srcTensor", "VECIN / VECCALC / VECOUT", "Sin");
182- CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN / VECCALC / VECOUT", "Sin");
183- ASCENDC_ASSERT((calCount <= srcTensor.GetSize()), {
184- KERNEL_LOG(KERNEL_ERROR, "calCount is %u, which should not be larger than srcTensor length %u", calCount,
185- srcTensor.GetSize());
186- });
187- ASCENDC_ASSERT((calCount <= dstTensor.GetSize()), {
188- KERNEL_LOG(KERNEL_ERROR, "calCount is %u, which should not be larger than dstTensor length %u", calCount,
189- dstTensor.GetSize());
190- });
191- 
192- if constexpr (config.algo == SinAlgo::POLYNOMIAL_APPROXIMATION) {
193- MicroAPI::Sin::SinPolynomialImpl((__ubuf__ T *)dstTensor.GetPhyAddr(),
194- (__ubuf__ T *)srcTensor.GetPhyAddr(), calCount);
195- } else if constexpr (config.algo == SinAlgo::RADIAN_REDUCTION) {
196- uint32_t sharedTmpBufferSize = GetSinTmpBufferSize<T>(sharedTmpBuffer);
197- uint32_t count = calCount;
198- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(calCount, sharedTmpBufferSize));
199- for (uint16_t i = 0; i < repeatTimes; i++) {
200- uint32_t remainCount = count - sharedTmpBufferSize * i;
201- uint32_t oneRepSize = remainCount < sharedTmpBufferSize ? remainCount : sharedTmpBufferSize;
202- SinRadianReductionImpl((__ubuf__ T*)dstTensor.GetPhyAddr() + i * sharedTmpBufferSize,
203- (__ubuf__ T*)srcTensor.GetPhyAddr() + i * sharedTmpBufferSize,
204- (__ubuf__ uint32_t*)sharedTmpBuffer.GetPhyAddr(), oneRepSize);
205- }
206- }
207-}
208- 
209-__aicore__ inline void SinCastFullMask(
210- const LocalTensor<float>& dstTensor, const LocalTensor<float>& srcTensor, RoundMode castType)
211-{
212- uint64_t newMask = 64;
213- Cast<float, float, false>(dstTensor, srcTensor, castType, newMask, 1,
214- { 1, 1, DEFAULT_REPEAT_STRIDE, DEFAULT_REPEAT_STRIDE });
215- PipeBarrier<PIPE_V>();
216-}
217- 
218- /*!
219- * \ingroup Sin
220- * \brief compute Sin elementwisely
221- * \tparam T: half/float
222- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason, only support
223- * float input data type
224- * \param [out] dstTensor: output LocalTensor
225- * \param [in] srcTensor: input LocalTensor
226- * \param [in] sharedTmpBuffer: extra temporary shared space used for intermediate values among calculation process,
227- * whose required space size should refer to corresponding tiling API, which is defined at sin_tiling.h.
228- * Generally, the more space you allocate, the better performance you will achieve, and the performance
229- * reaches peak when buffer size is maximum(calculated by tiling function). Moreover, it is not guaranteed
230- * that the shared space will be cleared after usage, the data could be anything.
231- * \param [in] calCount: the number of elements to be processed.
232- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
233- * Input data valid range should be [-65504, 65504]
234- */
235-template<typename T, bool isReuseSource = false, const SinConfig &config = defaultSinConfig>
236-__aicore__ inline void Sin(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
237- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
238-{
239- SinImpl<T, isReuseSource, config>(dstTensor, srcTensor, sharedTmpBuffer, calCount);
240-}
241- 
242- /*!
243- * \ingroup Sin
244- * \brief compute Sin elementwisely
245- * \tparam T: half/float
246- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason
247- * \param [out] dstTensor: output LocalTensor
248- * \param [in] srcTensor: input LocalTensor
249- * \param [in] sharedTmpBuffer: extra temporary shared space used for intermediate values among calculation process,
250- * whose required space size should refer to corresponding tiling API, which is defined at sin_tiling.h.
251- * Generally, the more space you allocate, the better performance you will achieve, and the performance
252- * reaches peak when buffer size is maximum(calculated by tiling function). Moreover, it is not guaranteed
253- * that the shared space will be cleared after usage, the data could be anything.
254- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
255- * Input data valid range should be [-65504, 65504]
256- */
257-template<typename T, bool isReuseSource = false, const SinConfig &config = defaultSinConfig>
258-__aicore__ inline void Sin(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
259- const LocalTensor<uint8_t>& sharedTmpBuffer)
260-{
261- Sin<T, isReuseSource, config>(dstTensor, srcTensor, sharedTmpBuffer, srcTensor.GetSize());
262-}
263- 
264- /*!
265- * \ingroup Sin
266- * \brief compute Sin elementwisely
267- * \tparam T: half/float
268- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason
269- * \param [out] dstTensor: output LocalTensor
270- * \param [in] srcTensor: input LocalTensor
271- * \param [in] calCount: the number of elements to be processed.
272- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
273- * Input data valid range should be [-65504, 65504]
274- */
275-template<typename T, bool isReuseSource = false, const SinConfig &config = defaultSinConfig>
276-__aicore__ inline void Sin(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
277- const uint32_t calCount)
278-{
279- SinImpl<T, isReuseSource, config>(dstTensor, srcTensor, calCount);
280-}
281- 
282- /*!
283- * \ingroup Sin
284- * \brief compute Sin elementwisely
285- * \tparam T: half/float
286- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason
287- * \param [out] dstTensor: output LocalTensor
288- * \param [in] srcTensor: input LocalTensor
289- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
290- * Input data valid range should be [-65504, 65504]
291- */
292-template<typename T, bool isReuseSource = false, const SinConfig &config = defaultSinConfig>
293-__aicore__ inline void Sin(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor)
294-{
295- Sin<T, isReuseSource, config>(dstTensor, srcTensor, srcTensor.GetSize());
296-}
297-}
298-#endif // LIB_MATH_SIN_L300_IMPL_H
Dimpl/adv_api/detail/math/sincos/sincos_l300_impl.h+0-760
@@ -1,760 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/* !
12- * \file sincos_l300_impl.h
13- * \brief
14- */
15-#ifndef LIB_MATH_SINCOS_SINCOS_L300_IMPL_H
16-#define LIB_MATH_SINCOS_SINCOS_L300_IMPL_H
17-#include "kernel_tensor.h"
18- 
19-namespace AscendC {
20-struct SinCosConfig {
21- bool isReuseSource;
22-};
23-constexpr SinCosConfig DEFAULT_SINCOS_CONFIG = { false };
24-namespace SinCosImpl {
25- 
26-constexpr MicroAPI::CastTrait castTraitF16F32 = { MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::UNKNOWN,
27- MicroAPI::MaskMergeMode::ZEROING, RoundMode::UNKNOWN };
28-constexpr MicroAPI::CastTrait castTraitF32F16 = { MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::NO_SAT,
29- MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_RINT };
30-constexpr MicroAPI::CastTrait castTraitI64F32 = { MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::NO_SAT,
31- MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_ROUND };
32-constexpr MicroAPI::CastTrait castTraitF32I64 = { MicroAPI::RegLayout::UNKNOWN, MicroAPI::SatMode::NO_SAT,
33- MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_ROUND };
34-constexpr MicroAPI::CastTrait castTraitI32F32 = { MicroAPI::RegLayout::UNKNOWN, MicroAPI::SatMode::NO_SAT,
35- MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_ROUND };
36- 
37-template <typename T, typename U>
38-__simd_callee__ inline void AndScalar(MicroAPI::RegTensor<T> &dstReg, MicroAPI::RegTensor<U> &srcReg,
39- T val, MicroAPI::MaskReg& mask)
40-{
41- MicroAPI::RegTensor<T> tmpReg;
42- MicroAPI::Duplicate(tmpReg, val, mask);
43- MicroAPI::And(dstReg, (MicroAPI::RegTensor<T>&)srcReg, tmpReg, mask);
44-}
45- 
46-__simd_callee__ inline void FMaf(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg,
47- float scalarValue, MicroAPI::MaskReg& mask)
48-{
49- // dst = dst * src + scalarValue
50- MicroAPI::RegTensor<float> tmpReg;
51- MicroAPI::Duplicate(tmpReg, scalarValue);
52- MicroAPI::FusedMulDstAdd(dstReg, srcReg, tmpReg, mask);
53-}
54- 
55-__simd_callee__ inline void FMaf(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg1,
56- MicroAPI::RegTensor<float>& srcReg2, MicroAPI::RegTensor<float>& srcReg3, MicroAPI::MaskReg& mask)
57-{
58- // dst = src1 * src2 + src3
59- MicroAPI::RegTensor<float> tmpReg = srcReg1;
60- MicroAPI::FusedMulDstAdd(tmpReg, srcReg2, srcReg3, mask);
61- dstReg = tmpReg;
62-}
63- 
64-__simd_callee__ inline void FMaf(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg1,
65- MicroAPI::RegTensor<float>& srcReg2, float scalarValue, MicroAPI::MaskReg& mask)
66-{
67- // dst = src1 * src2 + scalerValue
68- MicroAPI::RegTensor<float> tmpReg;
69- MicroAPI::Duplicate(tmpReg, scalarValue, mask);
70- FMaf(dstReg, srcReg1, srcReg2, tmpReg, mask);
71-}
72- 
73-__simd_callee__ inline void FMaf(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg1,
74- float scalarValue, MicroAPI::RegTensor<float>& srcReg2, MicroAPI::MaskReg& mask)
75-{
76- // dst = src1 * scalarValue + src2
77- MicroAPI::RegTensor<float> tmpReg;
78- MicroAPI::Duplicate(tmpReg, scalarValue, mask);
79- FMaf(dstReg, srcReg1, tmpReg, srcReg2, mask);
80-}
81- 
82-__simd_callee__ inline void FMaf(MicroAPI::RegTensor<float>& dstReg, MicroAPI::RegTensor<float>& srcReg1,
83- float scalarValue, float scalarValue2, MicroAPI::MaskReg& mask)
84-{
85- // dst = src1 * scalarValue + scalarValue2
86- MicroAPI::RegTensor<float> tmpReg, tmpReg2;
87- MicroAPI::Duplicate(tmpReg, scalarValue, mask);
88- MicroAPI::Duplicate(tmpReg2, scalarValue2, mask);
89- FMaf(dstReg, srcReg1, tmpReg, tmpReg2, mask);
90-}
91- 
92-__simd_callee__ inline void BitShiftCombine(MicroAPI::RegTensor<uint32_t> &dstReg, MicroAPI::RegTensor<uint32_t> &srcReg1,
93- MicroAPI::RegTensor<uint32_t> &srcReg2, MicroAPI::RegTensor<int32_t> &srcRegE, MicroAPI::MaskReg& mask)
94-{
95- // dst = (src1 << e) | (src2 >> (32 - e));
96- constexpr uint32_t BITSHIFTS = 32;
97- 
98- MicroAPI::RegTensor<uint32_t> tmpU32Reg1, tmpU32Reg2;
99- MicroAPI::ShiftLeft(tmpU32Reg1, srcReg1, (MicroAPI::RegTensor<int32_t>&)srcRegE, mask);
100- MicroAPI::Duplicate(tmpU32Reg2, BITSHIFTS, mask);
101- MicroAPI::Sub(tmpU32Reg2, tmpU32Reg2, (MicroAPI::RegTensor<uint32_t>&)srcRegE, mask);
102- MicroAPI::ShiftRight(tmpU32Reg2, srcReg2, (MicroAPI::RegTensor<int32_t>&)tmpU32Reg2, mask);
103- MicroAPI::Or(dstReg, tmpU32Reg1, tmpU32Reg2, mask);
104-}
105- 
106-__aicore__ inline void GenerateZeroVreg(MicroAPI::RegTensor<uint32_t>& zeroReg)
107-{
108- MicroAPI::MaskReg b32FullMask = MicroAPI::CreateMask<uint32_t, MicroAPI::MaskPattern::ALL>();
109- MicroAPI::Duplicate(zeroReg, 0, b32FullMask);
110-}
111- 
112-__simd_callee__ inline void ReinterpretedU32ToFloatAndCastToU32(MicroAPI::RegTensor<uint32_t> &dstReg,
113- MicroAPI::RegTensor<uint32_t> &srcReg, MicroAPI::MaskReg &mask)
114-{
115- // dst = (unsigned int) reinterpret_cast<float &>(src);
116- MicroAPI::RegTensor<float> tmpF32Reg;
117- MicroAPI::RegTensor<int64_t, MicroAPI::RegTraitNumTwo> tmpI64Reg;
118- 
119- tmpF32Reg = (MicroAPI::RegTensor<float>&)srcReg;
120- MicroAPI::Cast<int64_t, float, castTraitF32I64>(tmpI64Reg, tmpF32Reg, mask);
121- dstReg = (MicroAPI::RegTensor<uint32_t>&)tmpI64Reg.reg[0];
122-}
123- 
124-__simd_callee__ inline void TrigComputeP(MicroAPI::RegTensor<uint32_t> &regPHigh, MicroAPI::RegTensor<uint32_t> &regPLow,
125- MicroAPI::RegTensor<uint32_t> &regIa, MicroAPI::RegTensor<uint32_t> &regMid, MicroAPI::RegTensor<uint32_t> &regLo,
126- MicroAPI::RegTensor<uint32_t> &regHi, MicroAPI::MaskReg &mask)
127-{
128- MicroAPI::RegTensor<uint64_t, MicroAPI::RegTraitNumTwo> tmpU64Reg;
129- MicroAPI::RegTensor<uint32_t> tmpU32Reg, zeroReg;
130- MicroAPI::MaskReg carrypMask;
131- 
132- // step 12: p = (unsigned long long int)ia * lo;
133- MicroAPI::Mull((MicroAPI::RegTensor<uint32_t>&)regPLow,
134- (MicroAPI::RegTensor<uint32_t>&)regPHigh, regIa, regLo, mask);
135- 
136- // step 13: p = (unsigned long long int)ia * mid + (p >> 32);
137- MicroAPI::Mull((MicroAPI::RegTensor<uint32_t>&)tmpU64Reg.reg[0],
138- (MicroAPI::RegTensor<uint32_t>&)tmpU64Reg.reg[1], regIa, regMid, mask);
139- 
140- MicroAPI::AddCarryOut(carrypMask, (MicroAPI::RegTensor<uint32_t>&)regPLow,
141- (MicroAPI::RegTensor<uint32_t>&)tmpU64Reg.reg[0], (MicroAPI::RegTensor<uint32_t>&)regPHigh, mask);
142- MicroAPI::Duplicate(zeroReg, 0, mask);
143- MicroAPI::AddCarryOuts(carrypMask, (MicroAPI::RegTensor<uint32_t>&)regPHigh,
144- (MicroAPI::RegTensor<uint32_t>&)tmpU64Reg.reg[1], zeroReg, carrypMask, mask);
145- 
146- // step 14: p = ((unsigned long long int)(ia * hi) << 32) + p;
147- MicroAPI::Mul(tmpU32Reg, regIa, regHi, mask);
148- MicroAPI::AddCarryOut(carrypMask, (MicroAPI::RegTensor<uint32_t>&)regPHigh,
149- (MicroAPI::RegTensor<uint32_t>&)regPHigh, tmpU32Reg, mask);
150-}
151- 
152-__simd_callee__ inline void TrigComputeHLQ(MicroAPI::RegTensor<float> &regDh, MicroAPI::RegTensor<float> &regDl,
153- MicroAPI::RegTensor<int32_t> &regQ, MicroAPI::RegTensor<uint32_t> &regPHigh, MicroAPI::RegTensor<uint32_t> &regPLow,
154- MicroAPI::MaskReg &mask)
155-{
156- constexpr int16_t Q_SHIFT_BITS = 62;
157- constexpr int16_t B32_BITS = 32;
158- constexpr uint64_t P_AND_COEFF1 = 0x3fffffffffffffffULL;
159- constexpr uint64_t P_AND_COEFF2 = 0x2000000000000000ULL;
160- constexpr uint64_t P_SUBS_COEFF = 0x4000000000000000ULL;
161- constexpr float P_MULS = 2.0f;
162-#if (defined(__NPU_ARCH__) && (__NPU_ARCH__ ==3003 || __NPU_ARCH__ ==3113))
163- constexpr float B64_SHIFT_BITS = static_cast<float>(1ULL << 32);
164-#else
165- constexpr uint64_t B64_SHIFT_BITS = 1ULL << 32;
166-#endif
167- 
168- MicroAPI::RegTensor<int64_t, MicroAPI::RegTraitNumTwo> tmpI64Reg;
169- MicroAPI::RegTensor<int32_t> tmpRegQ;
170- MicroAPI::RegTensor<uint32_t> tmpU32Reg;
171- MicroAPI::MaskReg tmpMask;
172- 
173- // step 15: q = (int)(p >> 62);
174- MicroAPI::ShiftRights((MicroAPI::RegTensor<uint32_t>&)regQ,
175- (MicroAPI::RegTensor<uint32_t>&)regPHigh, (int16_t)(Q_SHIFT_BITS - B32_BITS), mask);
176- 
177- // step 16: p = p & 0x3fffffffffffffffULL;
178- MicroAPI::Duplicate(tmpU32Reg, P_AND_COEFF1 >> B32_BITS, mask);
179- MicroAPI::And((MicroAPI::RegTensor<uint32_t>&)regPHigh,
180- (MicroAPI::RegTensor<uint32_t>&)regPHigh, tmpU32Reg, mask);
181- 
182- /* step 17:
183- * if (p & 0x2000000000000000ULL) { // fraction >= 0.5
184- * p = p - 0x4000000000000000ULL; // fraction - 1.0
185- * q = q + 1;
186- * }
187- */
188- MicroAPI::Duplicate(tmpU32Reg, P_AND_COEFF2 >> B32_BITS, mask);
189- MicroAPI::And(tmpU32Reg, (MicroAPI::RegTensor<uint32_t>&)regPHigh, tmpU32Reg, mask);
190- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(tmpMask, tmpU32Reg, 0, mask);
191- MicroAPI::Duplicate(tmpU32Reg, P_SUBS_COEFF >> B32_BITS, mask);
192- MicroAPI::Sub(tmpU32Reg, (MicroAPI::RegTensor<uint32_t>&)regPHigh, tmpU32Reg, mask);
193- MicroAPI::Select((MicroAPI::RegTensor<uint32_t>&)regPHigh, tmpU32Reg,
194- (MicroAPI::RegTensor<uint32_t>&)regPHigh, tmpMask);
195- MicroAPI::Adds(tmpRegQ, regQ, 1, mask);
196- MicroAPI::Select(regQ, tmpRegQ, regQ, tmpMask);
197- 
198- /* compute remainder of x / (pi/2) */
199- // step 18: float d_h, d_l;
200- // step 19: long long int P = (long long int)p;
201- MicroAPI::RegTensor<float> tmpRegDH, tmpRegDL, tmpF32Reg;
202- MicroAPI::RegTensor<int32_t> tmpI32Reg, tmpRegPHigh, tmpRegPLow;
203- MicroAPI::Copy((MicroAPI::RegTensor<uint32_t>&)tmpRegPHigh, regPHigh);
204- /*
205- * d_h' = (float)P_high;
206- * d_l' = (float)(P_low >> 1) * 2;
207- */
208- MicroAPI::Cast<float, int32_t, castTraitI32F32>(tmpRegDH, tmpRegPHigh, mask);
209- MicroAPI::ShiftRights(tmpU32Reg, regPLow, (int16_t)1, mask);
210- MicroAPI::Cast<float, int32_t, castTraitI32F32>(tmpRegDL, (MicroAPI::RegTensor<int32_t>&)tmpU32Reg, mask);
211- MicroAPI::Muls(tmpRegDL, tmpRegDL, P_MULS, mask);
212- // next: d_l = (float)(P_high - (int)d_h') *(2**32) + (float)P_low
213- MicroAPI::Cast<int32_t, float, castTraitI32F32>(tmpI32Reg, tmpRegDH, mask);
214- MicroAPI::Sub(tmpI32Reg, tmpRegPHigh, tmpI32Reg, mask);
215- MicroAPI::Cast<float, int32_t, castTraitI32F32>(tmpF32Reg, tmpI32Reg, mask);
216- MicroAPI::Muls(tmpF32Reg, tmpF32Reg, B64_SHIFT_BITS, mask);
217- MicroAPI::Add(regDl, tmpF32Reg, tmpRegDL, mask);
218- // then: d_h = d_h' * (2**32)
219- MicroAPI::Muls(regDh, tmpRegDH, B64_SHIFT_BITS, mask);
220-}
221- 
222-__simd_callee__ inline void TrigRedSlowpathFComputeP(MicroAPI::RegTensor<uint32_t> &regPHigh, MicroAPI::RegTensor<uint32_t> &regPLow,
223- MicroAPI::RegTensor<float> &srcReg, MicroAPI::RegTensor<uint32_t>& oneOverPiFReg, MicroAPI::MaskReg& mask)
224-{
225- constexpr uint32_t TA_AND_COEFF = 0x007fffff;
226- constexpr uint32_t IA_ADD_COEFF = 0x4f000000;
227- constexpr int16_t TA_SHIFT_BITS = 23;
228- constexpr int32_t TA_SHIFT_AND_COEFF = 0x000000ff;
229- constexpr int16_t I_SHIFT_BITS = 5;
230- constexpr int32_t E_SUB_COEFF = 126;
231- constexpr int32_t E_AND_COEFF = 31;
232- constexpr uint32_t LO_SELECT = 1;
233- constexpr uint32_t TMP_SELECT = 2;
234- 
235- MicroAPI::RegTensor<uint32_t> regIa, regHi, regMid, regLo, regTmp, regI;
236- MicroAPI::RegTensor<int32_t> regE;
237- MicroAPI::RegTensor<int32_t> tmpI32Reg;
238- MicroAPI::RegTensor<uint32_t> tmpU32Reg;
239- 
240- // step 1: unsigned int ta = reinterpret_cast<unsigned int &>(a);
241- // ta can be obtained by (RegTensor<uint32_t>&)srcReg;
242- // step 2: ia = (ta&0x007fffff) + 0x4f000000;
243- AndScalar(regIa, (MicroAPI::RegTensor<uint32_t> &)srcReg, TA_AND_COEFF, mask);
244- MicroAPI::Adds(regIa, regIa, IA_ADD_COEFF, mask);
245- 
246- // step 3: ia = (unsigned int) reinterpret_cast<float &>(ia);
247- ReinterpretedU32ToFloatAndCastToU32(regIa, regIa, mask);
248- 
249- // step 4: e = ((ta >> 23) & 0x000000ff) - 127;
250- MicroAPI::ShiftRights(regE, (MicroAPI::RegTensor<int32_t> &)srcReg, TA_SHIFT_BITS, mask);
251- AndScalar(regE, regE, TA_SHIFT_AND_COEFF, mask);
252- MicroAPI::Adds(regE, regE, -E_SUB_COEFF, mask);
253- 
254- // step 5: i = (unsigned int)e >> 5;
255- MicroAPI::ShiftRights(regI, (MicroAPI::RegTensor<uint32_t>&)regE, I_SHIFT_BITS, mask);
256- // step 6: e = (unsigned int)e & 31;
257- MicroAPI::Duplicate(tmpI32Reg, E_AND_COEFF, mask);
258- MicroAPI::And(regE, (MicroAPI::RegTensor<int32_t>&)regE, tmpI32Reg, mask);
259- 
260- // step 7:hi = i ? one_over_pi_f [i-1] : 0;
261- MicroAPI::MaskReg tmpMask;
262- MicroAPI::RegTensor<uint32_t> tmpRegSelect;
263- MicroAPI::CompareScalar<uint32_t, CMPMODE::GT>(tmpMask, regI, 0, mask);
264- MicroAPI::Adds(tmpU32Reg, regI, -1, mask);
265- MicroAPI::Gather(tmpRegSelect, oneOverPiFReg, tmpU32Reg);
266- MicroAPI::Duplicate(regHi, 0, mask);
267- MicroAPI::Select(regHi, tmpRegSelect, regHi, tmpMask);
268- 
269- // step 8: mid = one_over_pi_f [i+0];
270- MicroAPI::Gather(regMid, oneOverPiFReg, regI);
271- // // step 9: lo = one_over_pi_f [i+1];
272- MicroAPI::Adds(tmpU32Reg, regI, LO_SELECT, mask);
273- MicroAPI::Gather(regLo, oneOverPiFReg, tmpU32Reg);
274- // step 10: tmp = one_over_pi_f [i+2];
275- MicroAPI::Adds(tmpU32Reg, regI, TMP_SELECT, mask);
276- MicroAPI::Gather(regTmp, oneOverPiFReg, tmpU32Reg);
277- 
278- /* step 11:
279- * if(e) {
280- * hi = (hi << e) | (mid >> (32 - e));
281- * mid = (mid << e) | (lo >> (32 - e));
282- * lo = (lo << e) | (tmp >> (32 - e));
283- * }
284- */
285- MicroAPI::CompareScalar<int32_t, CMPMODE::GT>(tmpMask, regE, 0, mask);
286- BitShiftCombine(tmpRegSelect, regHi, regMid, regE, mask);
287- MicroAPI::Select(regHi, tmpRegSelect, regHi, tmpMask);
288- BitShiftCombine(tmpRegSelect, regMid, regLo, regE, mask);
289- MicroAPI::Select(regMid, tmpRegSelect, regMid, tmpMask);
290- BitShiftCombine(tmpRegSelect, regLo, regTmp, regE, mask);
291- MicroAPI::Select(regLo, tmpRegSelect, regLo, tmpMask);
292- 
293- TrigComputeP(regPHigh, regPLow, regIa, regMid, regLo, regHi, mask);
294-}
295- 
296-__simd_callee__ inline void TrigRedSlowpathFComputeRI(MicroAPI::RegTensor<float> &dstRegR, MicroAPI::RegTensor<int32_t> &dstRegI,
297- MicroAPI::RegTensor<uint32_t> &regPHigh, MicroAPI::RegTensor<uint32_t> &regPLow, MicroAPI::RegTensor<float> &srcReg,
298- MicroAPI::MaskReg& mask)
299-{
300- constexpr float R_MUL_COEFF = 3.4061215800865545e-19;
301- 
302- MicroAPI::RegTensor<int32_t> regQ;
303- MicroAPI::RegTensor<float> regR, regDh, regDl;
304- MicroAPI::RegTensor<float> tmpF32Reg;
305- MicroAPI::RegTensor<int32_t> tmpRegQ;
306- MicroAPI::MaskReg tmpMask;
307- 
308- TrigComputeHLQ(regDh, regDl, regQ, regPHigh, regPLow, mask);
309-
310- // step 23: r = d_l*3.4061215800865545e-19;
311- MicroAPI::Muls(regR, regDl, R_MUL_COEFF, mask);
312- // step 24: r = r + d_h*3.4061215800865545e-19;
313- MicroAPI::Duplicate(tmpF32Reg, R_MUL_COEFF, mask);
314- MicroAPI::MulAddDst(regR, regDh, tmpF32Reg, mask);
315- 
316- /* step 25:
317- * if (a < 0.0f) {
318- * r = -r;
319- * q = -q;
320- * }
321- */
322- MicroAPI::CompareScalar<float, CMPMODE::LT>(tmpMask, srcReg, 0.0f, mask);
323- MicroAPI::Neg(tmpF32Reg, regR, mask);
324- MicroAPI::Select(regR, tmpF32Reg, regR, tmpMask);
325- MicroAPI::Neg(tmpRegQ, regQ, mask);
326- MicroAPI::Select(regQ, tmpRegQ, regQ, tmpMask);
327- 
328- // step 26: *quadrant = q;
329- dstRegR = regR;
330- dstRegI = regQ;
331-}
332- 
333-__simd_callee__ inline void SinfPoly(MicroAPI::RegTensor<float> &dstReg, MicroAPI::RegTensor<float> &srcRegA,
334- MicroAPI::RegTensor<float> &srcRegS, MicroAPI::MaskReg& mask)
335-{
336- constexpr float SIN_POLY_COEFF0 = 2.86567956e-6f;
337- constexpr float SIN_POLY_COEFF1 = -1.98559923e-4f;
338- constexpr float SIN_POLY_COEFF2 = 8.33338592e-3f;
339- constexpr float SIN_POLY_COEFF3 = -1.66666672e-1f;
340- constexpr float SIN_POLY_COEFF5 = 0.0f;
341- 
342- MicroAPI::RegTensor<float> tmpRegT;
343- // step 1: r = 2.86567956e-6f;
344- MicroAPI::Duplicate(dstReg, SIN_POLY_COEFF0, mask);
345- // step 2: r = r* s+ -1.98559923e-4f;
346- FMaf(dstReg, srcRegS, SIN_POLY_COEFF1, mask);
347- // step 3: r = r* s+ 8.33338592e-3f;
348- FMaf(dstReg, srcRegS, SIN_POLY_COEFF2, mask);
349- // step 4: r = r* s+ -1.66666672e-1f;
350- FMaf(dstReg, srcRegS, SIN_POLY_COEFF3, mask);
351- // step 5: t = a* s+ 0.0f;
352- FMaf(tmpRegT, srcRegA, srcRegS, SIN_POLY_COEFF5, mask);
353- // step 6: r = r* t+ a;
354- MicroAPI::FusedMulDstAdd(dstReg, tmpRegT, srcRegA, mask);
355-}
356- 
357-__simd_callee__ inline void CosfPoly(MicroAPI::RegTensor<float> &dstReg, MicroAPI::RegTensor<float> &srcRegS,
358- MicroAPI::MaskReg& mask)
359-{
360- constexpr float COS_POLY_COEFF0 = 2.44677067e-5f;
361- constexpr float COS_POLY_COEFF1 = -1.38877297e-3f;
362- constexpr float COS_POLY_COEFF2 = 4.16666567e-2f;
363- constexpr float COS_POLY_COEFF3 = -5.00000000e-1f;
364- constexpr float COS_POLY_COEFF4 = 1.00000000e+0f;
365- 
366- // step 1: r = 2.44677067e-5f;
367- MicroAPI::Duplicate(dstReg, COS_POLY_COEFF0, mask);
368- // step 2: r = r* s+ -1.38877297e-3f;
369- FMaf(dstReg, srcRegS, COS_POLY_COEFF1, mask);
370- // step 3: r = r* s+ 4.16666567e-2f;
371- FMaf(dstReg, srcRegS, COS_POLY_COEFF2, mask);
372- // step 4: r = r* s+ -5.00000000e-1f;
373- FMaf(dstReg, srcRegS, COS_POLY_COEFF3, mask);
374- // step 5: r = r* s+ 1.00000000e+0f;
375- FMaf(dstReg, srcRegS, COS_POLY_COEFF4, mask);
376-}
377- 
378-__simd_callee__ inline void TrigRedFPreporcessForHalf(MicroAPI::RegTensor<float> &regR, MicroAPI::RegTensor<int32_t> &regI,
379- MicroAPI::RegTensor<float> &srcRegA, MicroAPI::MaskReg& mask)
380-{
381- constexpr float J_MUL_COEFF = 0.636619747f;
382- constexpr float J_ADD_COEFF = 12582912.0f;
383- constexpr float J_MUL_COEFF1 = -1.57079601e+00f;
384- constexpr float J_MUL_COEFF2 = -3.13916473e-07f;
385- constexpr float J_MUL_COEFF3 = -5.39030253e-15f;
386- 
387- MicroAPI::RegTensor<float> regJ;
388- MicroAPI::RegTensor<float> tmpF32Reg;
389- MicroAPI::RegTensor<int32_t> tmpI32Reg;
390- 
391- // step 1: a = a * 0.0f + a; convert inf to NAN
392- MicroAPI::Duplicate(tmpF32Reg, 0.0f, mask);
393- MicroAPI::FusedMulDstAdd(srcRegA, tmpF32Reg, srcRegA, mask);
394- 
395- // step 2: j = a*0.636619747f + 12582912.0f;
396- FMaf(regJ, srcRegA, J_MUL_COEFF, J_ADD_COEFF, mask);
397- // step 3: i = reinterpret_cast<int&> (j);
398- regI = (MicroAPI::RegTensor<int32_t>&)regJ;
399- // step 4: j = j - 12582912.0f;
400- MicroAPI::Adds(regJ, regJ, -J_ADD_COEFF, mask);
401- // step 5: r = j* -1.57079601e+00f+ a; // -0x1.921fb0p+00 // pio2_high
402- FMaf(regR, regJ, J_MUL_COEFF1, srcRegA, mask);
403- // step 6: r = j* -3.13916473e-07f+ r; // -0x1.5110b4p-22 // pio2_mid
404- MicroAPI::Duplicate(tmpF32Reg, J_MUL_COEFF2, mask);
405- MicroAPI::MulAddDst(regR, regJ, tmpF32Reg, mask);
406- // step 7: r = j* -5.39030253e-15f+ r; // -0x1.846988p-48 // pio2_low
407- MicroAPI::Duplicate(tmpF32Reg, J_MUL_COEFF3, mask);
408- MicroAPI::MulAddDst(regR, regJ, tmpF32Reg, mask);
409-}
410- 
411-__simd_callee__ inline void TrigRedFComputeP(MicroAPI::MaskReg& tmpMask, MicroAPI::RegTensor<uint32_t> &regPHigh,
412- MicroAPI::RegTensor<uint32_t> &regPLow, MicroAPI::RegTensor<float> &srcRegA,
413- MicroAPI::RegTensor<uint32_t> &oneOverPiFReg, MicroAPI::MaskReg& mask)
414-{
415- constexpr float A_ABS_COEFF = 3.1415926535f*0.25f;
416- MicroAPI::RegTensor<float> tmpF32Reg;
417- MicroAPI::RegTensor<int32_t> tmpI32Reg;
418- /* step 8:
419- * if (std::abs(a) > 3.1415926535f*0.25f) {
420- * r = trig_red_slowpath_f (a, &i);
421- * }
422- */
423- MicroAPI::Abs(tmpF32Reg, srcRegA, mask);
424- MicroAPI::CompareScalar<float, CMPMODE::GT>(tmpMask, tmpF32Reg, A_ABS_COEFF, mask);
425- TrigRedSlowpathFComputeP(regPHigh, regPLow, srcRegA, oneOverPiFReg, mask);
426-}
427- 
428-__simd_callee__ inline void TrigRedFComputeRI(MicroAPI::MaskReg& tmpMask, MicroAPI::RegTensor<float> &dstRegR,
429- MicroAPI::RegTensor<int32_t> &dstRegI, MicroAPI::RegTensor<uint32_t> &regPHigh, MicroAPI::RegTensor<uint32_t> &regPLow,
430- MicroAPI::RegTensor<float> &srcRegA, MicroAPI::MaskReg& mask)
431-{
432- constexpr float A_ABS_COEFF = 3.1415926535f*0.25f;
433- MicroAPI::RegTensor<float> tmpF32Reg;
434- MicroAPI::RegTensor<int32_t> tmpI32Reg;
435- /* step 8:
436- * if (std::abs(a) > 3.1415926535f*0.25f) {
437- * r = trig_red_slowpath_f (a, &i);
438- * }
439- */
440- MicroAPI::Abs(tmpF32Reg, srcRegA, mask);
441- MicroAPI::CompareScalar<float, CMPMODE::GT>(tmpMask, tmpF32Reg, A_ABS_COEFF, mask);
442- TrigRedSlowpathFComputeRI(dstRegR, dstRegI, regPHigh, regPLow, srcRegA, mask);
443-}
444- 
445-/* Compute sine and cosine simultaneously, based on quadrant */
446-__simd_callee__ inline void SCFCore(MicroAPI::RegTensor<float> &dstRegSin, MicroAPI::RegTensor<float> &dstRegCos,
447- MicroAPI::RegTensor<int32_t> &regI, MicroAPI::RegTensor<float> &regR, MicroAPI::MaskReg& mask)
448-{
449- constexpr int32_t I_AND_CONDITION = 2;
450- 
451- // step 9: float c, s, t;
452- MicroAPI::RegTensor<float> regC, regS, regT;
453- MicroAPI::RegTensor<float> tmpF32Reg, tmpF32Reg1;
454- MicroAPI::RegTensor<int32_t> tmpI32Reg;
455- // step 10: s = r * r;
456- MicroAPI::Mul(regS, regR, regR, mask);
457- // step 11: c = cosf_poly (s);
458- CosfPoly(regC, regS, mask);
459- // step 12: s = sinf_poly (r, s);
460- SinfPoly(tmpF32Reg, regR, regS, mask);
461- regS = tmpF32Reg;
462- 
463- /* step 13:
464- * if (i & 2) {
465- * s = 0.0f - s; // don't change "sign" of NaNs or create negative zeros
466- * c = 0.0f - c; // don't change "sign" of NaNs or create negative zeros
467- * }
468- */
469- MicroAPI::MaskReg tmpMask;
470- MicroAPI::Duplicate(tmpI32Reg, I_AND_CONDITION, mask);
471- MicroAPI::And(tmpI32Reg, regI, tmpI32Reg, mask);
472- MicroAPI::CompareScalar<int32_t, CMPMODE::GT>(tmpMask, tmpI32Reg, 0, mask);
473- MicroAPI::Duplicate(tmpF32Reg1, 0.0f, mask);
474- MicroAPI::Sub(tmpF32Reg, tmpF32Reg1, regS, mask);
475- MicroAPI::Select(regS, tmpF32Reg, regS, tmpMask);
476- MicroAPI::Sub(tmpF32Reg, tmpF32Reg1, regC, mask);
477- MicroAPI::Select(regC, tmpF32Reg, regC, tmpMask);
478- 
479- /* step 14:
480- * if (i & 1) {
481- * t = 0.0f - s; // don't change "sign" of NaNs or create negative zeros
482- * s = c;
483- * c = t;
484- * }
485- */
486- MicroAPI::Duplicate(tmpI32Reg, 1, mask);
487- MicroAPI::And(tmpI32Reg, regI, tmpI32Reg, mask);
488- MicroAPI::CompareScalar<int32_t, CMPMODE::GT>(tmpMask, tmpI32Reg, 0, mask);
489- MicroAPI::Duplicate(tmpF32Reg, 0.0f, mask);
490- MicroAPI::Sub(tmpF32Reg, tmpF32Reg, regS, mask);
491- MicroAPI::Select(regT, tmpF32Reg, regT, tmpMask);
492- MicroAPI::Select(regS, regC, regS, tmpMask);
493- MicroAPI::Select(regC, regT, regC, tmpMask);
494- 
495- // step 15: *sp = s; //sp is the sin result
496- dstRegSin = regS;
497- // step 16: *cp = c; //cp is the cos result
498- dstRegCos = regC;
499-}
500- 
501-__aicore__ inline void InitializeFloatTempBuffer(__ubuf__ uint32_t *&tmpBuffer, __ubuf__ float *&tmpBufferR,
502- __ubuf__ int32_t *&tmpBufferI, const uint32_t alignCount)
503-{
504- constexpr uint32_t oneOverPiFAlignedLength = 8;
505- static unsigned int oneOverPiF[6] =
506- {
507- 0x28be60db, 0x9391054a, 0x7f09d5f4,
508- 0x7d4d3770, 0x36d8a566, 0x4f10e410
509- };
510- 
511- for (uint16_t i = 0; i < 6; ++i) {
512- tmpBuffer[i] = oneOverPiF[i];
513- }
514- 
515- tmpBufferR = (__ubuf__ float *)((__ubuf__ uint8_t *)tmpBuffer + sizeof(uint32_t) * oneOverPiFAlignedLength);
516- tmpBufferI = (__ubuf__ int32_t *)((__ubuf__ uint8_t *)tmpBufferR + sizeof(float) * alignCount);
517-}
518- 
519-__aicore__ inline void InitializeHalfTempBuffer(__ubuf__ uint32_t *&tmpBuffer, __ubuf__ float *&tmpBufferR,
520- __ubuf__ int32_t *&tmpBufferI, const uint32_t alignCount)
521-{
522- tmpBufferR = (__ubuf__ float *)((__ubuf__ uint8_t *)tmpBuffer);
523- tmpBufferI = (__ubuf__ int32_t *)((__ubuf__ uint8_t *)tmpBufferR + sizeof(float) * alignCount);
524-}
525- 
526-template <typename T>
527-__simd_vf__ inline void TrigRedFPreProcessImpl(__ubuf__ float *tmpBufferR, __ubuf__ int32_t *tmpBufferI,
528- __ubuf__ T *src, uint32_t calCount, uint16_t repeatTimes)
529-{
530- MicroAPI::RegTensor<T> srcReg;
531- MicroAPI::RegTensor<int32_t> regI;
532- MicroAPI::RegTensor<float> castReg, regR;
533- 
534- for (uint16_t i = 0; i < repeatTimes; i++) {
535- MicroAPI::MaskReg mask = MicroAPI::UpdateMask<float>(calCount);
536- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcReg, src + i * B32_DATA_NUM_PER_REPEAT);
537- MicroAPI::Cast<float, T, SinCosImpl::castTraitF16F32>(castReg, srcReg, mask);
538- TrigRedFPreporcessForHalf(regR, regI, castReg, mask);
539- 
540- MicroAPI::DataCopy(tmpBufferR + i * B32_DATA_NUM_PER_REPEAT, regR, mask);
541- MicroAPI::DataCopy(tmpBufferI + i * B32_DATA_NUM_PER_REPEAT, regI, mask);
542- }
543-}
544- 
545-template <typename T>
546-__simd_vf__ inline void TrigRedFComputePImpl(__ubuf__ uint32_t *tmpBufferRegPHigh, __ubuf__ uint32_t *tmpBufferRegPLow,
547- __ubuf__ T *src, __ubuf__ uint32_t *tmpBuffer, uint32_t calCount, uint16_t repeatTimes)
548-{
549- MicroAPI::RegTensor<T> srcReg;
550- MicroAPI::RegTensor<uint32_t> oneOverPiFReg, regPHigh, regPLow, regI;
551- MicroAPI::RegTensor<float> castReg, regR, tmpF32Reg;
552- MicroAPI::MaskReg selectMask;
553- 
554- // Load the array of one_over_pi_f
555- MicroAPI::DataCopy(oneOverPiFReg, tmpBuffer);
556- 
557- for (uint16_t i = 0; i < repeatTimes; i++) {
558- MicroAPI::MaskReg mask = MicroAPI::UpdateMask<float>(calCount);
559- MicroAPI::DataCopy(castReg, src + i * B32_DATA_NUM_PER_REPEAT);
560- // a = a * 0.0f + a
561- MicroAPI::Duplicate(tmpF32Reg, 0.0f, mask);
562- MicroAPI::FusedMulDstAdd(castReg, tmpF32Reg, castReg, mask);
563- // initilize q and r: *q = 0; r = a;
564- MicroAPI::Duplicate(regI, 0, mask);
565- regR = castReg;
566- // store the origin q and r in ub
567- MicroAPI::DataCopy((__ubuf__ float *)tmpBufferRegPHigh + i * B32_DATA_NUM_PER_REPEAT, regR, mask);
568- MicroAPI::DataCopy(tmpBufferRegPLow + i * B32_DATA_NUM_PER_REPEAT, regI, mask);
569- 
570- TrigRedFComputeP(selectMask, regPHigh, regPLow, castReg, oneOverPiFReg, mask);
571- 
572- MicroAPI::DataCopy(tmpBufferRegPHigh + i * B32_DATA_NUM_PER_REPEAT, regPHigh, selectMask);
573- MicroAPI::DataCopy(tmpBufferRegPLow + i * B32_DATA_NUM_PER_REPEAT, regPLow, selectMask);
574- }
575-}
576- 
577-template <typename T>
578-__simd_vf__ inline void TrigRedFComputeRIImpl(__ubuf__ uint32_t *tmpBufferRegPHigh, __ubuf__ uint32_t *tmpBufferRegPLow,
579- __ubuf__ T *src, __ubuf__ uint32_t *tmpBuffer, uint32_t calCount, uint16_t repeatTimes)
580-{
581- MicroAPI::RegTensor<T> srcReg;
582- MicroAPI::RegTensor<uint32_t> regPHigh, regPLow;
583- MicroAPI::RegTensor<int32_t> regI;
584- MicroAPI::RegTensor<float> castReg, regR, tmpF32Reg;
585- MicroAPI::MaskReg selectMask;
586- 
587- for (uint16_t i = 0; i < repeatTimes; i++) {
588- MicroAPI::MaskReg mask = MicroAPI::UpdateMask<float>(calCount);
589- MicroAPI::DataCopy(castReg, src + i * B32_DATA_NUM_PER_REPEAT);
590- MicroAPI::DataCopy(regPHigh, tmpBufferRegPHigh + i * B32_DATA_NUM_PER_REPEAT);
591- MicroAPI::DataCopy(regPLow, tmpBufferRegPLow + i * B32_DATA_NUM_PER_REPEAT);
592- // a = a * 0.0f + a
593- MicroAPI::Duplicate(tmpF32Reg, 0.0f, mask);
594- MicroAPI::FusedMulDstAdd(castReg, tmpF32Reg, castReg, mask);
595- 
596- TrigRedFComputeRI(selectMask, regR, regI, regPHigh, regPLow, castReg, mask);
597- 
598- MicroAPI::DataCopy((__ubuf__ float *)tmpBufferRegPHigh + i * B32_DATA_NUM_PER_REPEAT, regR, selectMask);
599- MicroAPI::DataCopy((__ubuf__ int32_t *)tmpBufferRegPLow + i * B32_DATA_NUM_PER_REPEAT, regI, selectMask);
600- }
601-}
602- 
603-template <typename T, int mode = 0>
604-__simd_vf__ inline void SCFCoreImpl(__ubuf__ T *dst, __ubuf__ float *tmpBufferR, __ubuf__ int32_t *tmpBufferI,
605- uint32_t calCount, uint16_t repeatTimes)
606-{
607- MicroAPI::RegTensor<T> srcReg;
608- MicroAPI::RegTensor<int32_t> regI;
609- MicroAPI::RegTensor<float> regR;
610- MicroAPI::RegTensor<float> dstRegCos, dstRegSin, dstReg;
611- 
612- for (uint16_t i = 0; i < repeatTimes; i++) {
613- MicroAPI::MaskReg mask = MicroAPI::UpdateMask<float>(calCount);
614- MicroAPI::DataCopy(regR, tmpBufferR + i * B32_DATA_NUM_PER_REPEAT);
615- MicroAPI::DataCopy(regI, tmpBufferI + i * B32_DATA_NUM_PER_REPEAT);
616- 
617- SCFCore(dstRegSin, dstRegCos, regI, regR, mask);
618- 
619- if constexpr (mode == 0) {
620- dstReg = dstRegSin;
621- } else {
622- dstReg = dstRegCos;
623- }
624- 
625- if constexpr (sizeof(T) == sizeof(half)) {
626- MicroAPI::Cast<T, float, SinCosImpl::castTraitF32F16>(srcReg, dstReg, mask);
627- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(dst + i * B32_DATA_NUM_PER_REPEAT, srcReg, mask);
628- } else {
629- MicroAPI::DataCopy(dst + i * B32_DATA_NUM_PER_REPEAT, dstReg, mask);
630- }
631- }
632-}
633- 
634-template <typename T>
635-__simd_vf__ inline void BSCFCoreImpl(__ubuf__ T *dstSin, __ubuf__ T *dstCos, __ubuf__ float *tmpBufferR, __ubuf__ int32_t *tmpBufferI,
636- uint32_t calCount, uint16_t repeatTimes)
637-{
638- MicroAPI::RegTensor<T> srcReg;
639- MicroAPI::RegTensor<int32_t> regI;
640- MicroAPI::RegTensor<float> regR;
641- MicroAPI::RegTensor<float> dstRegCos, dstRegSin;
642- 
643- for (uint16_t i = 0; i < repeatTimes; i++) {
644- MicroAPI::MaskReg mask = MicroAPI::UpdateMask<float>(calCount);
645- MicroAPI::DataCopy(regR, tmpBufferR + i * B32_DATA_NUM_PER_REPEAT);
646- MicroAPI::DataCopy(regI, tmpBufferI + i * B32_DATA_NUM_PER_REPEAT);
647- 
648- SCFCore(dstRegSin, dstRegCos, regI, regR, mask);
649- 
650- if constexpr (sizeof(T) == sizeof(half)) {
651- MicroAPI::Cast<T, float, SinCosImpl::castTraitF32F16>(srcReg, dstRegSin, mask);
652- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(dstSin + i * B32_DATA_NUM_PER_REPEAT, srcReg, mask);
653- MicroAPI::Cast<T, float, SinCosImpl::castTraitF32F16>(srcReg, dstRegCos, mask);
654- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(dstCos + i * B32_DATA_NUM_PER_REPEAT, srcReg, mask);
655- } else {
656- MicroAPI::DataCopy(dstSin + i * B32_DATA_NUM_PER_REPEAT, dstRegSin, mask);
657- MicroAPI::DataCopy(dstCos + i * B32_DATA_NUM_PER_REPEAT, dstRegCos, mask);
658- }
659- }
660-}
661-} // namespace SinCosImpl
662- 
663-template <typename T>
664-__aicore__ inline void SinRadianReductionImpl(__ubuf__ T *dst, __ubuf__ T *src, __ubuf__ uint32_t *tmpBuffer, uint32_t calCount)
665-{
666- static_assert((std::is_same_v<T, half> || std::is_same_v<T, float>),
667- "current data type is not supported on current device!");
668- constexpr uint32_t oneRepSize = GetVecLen() / sizeof(float);
669- uint16_t repeatTimes = CeilDivision(calCount, oneRepSize);
670- __ubuf__ float *tmpBufferR;
671- __ubuf__ int32_t *tmpBufferI;
672- uint32_t alignCount = (calCount + 31) / 32 * 32;
673- 
674- if constexpr (std::is_same_v<T, float>) {
675- SinCosImpl::InitializeFloatTempBuffer(tmpBuffer, tmpBufferR, tmpBufferI, alignCount);
676- SinCosImpl::TrigRedFComputePImpl<T>(
677- (__ubuf__ uint32_t *)tmpBufferR, (__ubuf__ uint32_t *)tmpBufferI, src, tmpBuffer, calCount, repeatTimes);
678- SinCosImpl::TrigRedFComputeRIImpl<T>(
679- (__ubuf__ uint32_t *)tmpBufferR, (__ubuf__ uint32_t *)tmpBufferI, src, tmpBuffer, calCount, repeatTimes);
680- } else if constexpr (std::is_same_v<T, half>) {
681- SinCosImpl::InitializeHalfTempBuffer(tmpBuffer, tmpBufferR, tmpBufferI, alignCount);
682- SinCosImpl::TrigRedFPreProcessImpl<T>(tmpBufferR, tmpBufferI, src, calCount, repeatTimes);
683- }
684- SinCosImpl::SCFCoreImpl<T, 0>(dst, tmpBufferR, tmpBufferI, calCount, repeatTimes);
685-}
686- 
687-template <typename T>
688-__aicore__ inline void CosRadianReductionImpl(__ubuf__ T *dst, __ubuf__ T *src, __ubuf__ uint32_t *tmpBuffer, uint32_t calCount)
689-{
690- static_assert((std::is_same_v<T, half> || std::is_same_v<T, float>),
691- "current data type is not supported on current device!");
692- constexpr uint32_t oneRepSize = GetVecLen() / sizeof(float);
693- uint16_t repeatTimes = CeilDivision(calCount, oneRepSize);
694- __ubuf__ float *tmpBufferR;
695- __ubuf__ int32_t *tmpBufferI;
696- uint32_t alignCount = (calCount + 31) / 32 * 32;
697- 
698- if constexpr (std::is_same_v<T, float>) {
699- SinCosImpl::InitializeFloatTempBuffer(tmpBuffer, tmpBufferR, tmpBufferI, alignCount);
700- SinCosImpl::TrigRedFComputePImpl<T>(
701- (__ubuf__ uint32_t *)tmpBufferR, (__ubuf__ uint32_t *)tmpBufferI, src, tmpBuffer, calCount, repeatTimes);
702- SinCosImpl::TrigRedFComputeRIImpl<T>(
703- (__ubuf__ uint32_t *)tmpBufferR, (__ubuf__ uint32_t *)tmpBufferI, src, tmpBuffer, calCount, repeatTimes);
704- } else if constexpr (std::is_same_v<T, half>) {
705- SinCosImpl::InitializeHalfTempBuffer(tmpBuffer, tmpBufferR, tmpBufferI, alignCount);
706- SinCosImpl::TrigRedFPreProcessImpl<T>(tmpBufferR, tmpBufferI, src, calCount, repeatTimes);
707- }
708- SinCosImpl::SCFCoreImpl<T, 1>(dst, tmpBufferR, tmpBufferI, calCount, repeatTimes);
709-}
710- 
711-template <const SinCosConfig& config, typename T>
712-__aicore__ inline void SinCosRadianReductionImpl(const LocalTensor<T>& dst0, const LocalTensor<T>& dst1,
713- const LocalTensor<T>& src, const LocalTensor<uint8_t>& sharedTmpBuffer, uint32_t calCount)
714-{
715- // Only for AI Vector Core.
716- if ASCEND_IS_AIC {
717- return;
718- }
719- CHECK_FUNC_HIGHLEVEL_API(SinCos, (T, config.isReuseSource), (dst0, dst1, src, sharedTmpBuffer, calCount));
720- __ubuf__ T* dstSinAddr = (__ubuf__ T*)dst0.GetPhyAddr();
721- __ubuf__ T* dstCosAddr = (__ubuf__ T*)dst1.GetPhyAddr();
722- __ubuf__ T* srcAddr = (__ubuf__ T*)src.GetPhyAddr();
723- __ubuf__ uint32_t* tmpBuffer = (__ubuf__ uint32_t*)sharedTmpBuffer.GetPhyAddr();
724- static_assert((std::is_same_v<T, half> || std::is_same_v<T, float>),
725- "current data type is not supported on current device!");
726- constexpr uint32_t oneRepSize = GetVecLen() / sizeof(float);
727- uint16_t repeatTimes = CeilDivision(calCount, oneRepSize);
728- __ubuf__ float *tmpBufferR;
729- __ubuf__ int32_t *tmpBufferI;
730- uint32_t alignCount = (calCount + 31) / 32 * 32;
731- 
732- if constexpr (std::is_same_v<T, float>) {
733- SinCosImpl::InitializeFloatTempBuffer(tmpBuffer, tmpBufferR, tmpBufferI, alignCount);
734- SinCosImpl::TrigRedFComputePImpl<T>(
735- (__ubuf__ uint32_t *)tmpBufferR, (__ubuf__ uint32_t *)tmpBufferI, srcAddr, tmpBuffer, calCount, repeatTimes);
736- SinCosImpl::TrigRedFComputeRIImpl<T>(
737- (__ubuf__ uint32_t *)tmpBufferR, (__ubuf__ uint32_t *)tmpBufferI, srcAddr, tmpBuffer, calCount, repeatTimes);
738- } else if constexpr (std::is_same_v<T, half>) {
739- SinCosImpl::InitializeHalfTempBuffer(tmpBuffer, tmpBufferR, tmpBufferI, alignCount);
740- SinCosImpl::TrigRedFPreProcessImpl<T>(tmpBufferR, tmpBufferI, srcAddr, calCount, repeatTimes);
741- }
742- SinCosImpl::BSCFCoreImpl<T>(dstSinAddr, dstCosAddr, tmpBufferR, tmpBufferI, calCount, repeatTimes);
743-}
744- 
745-template <const SinCosConfig& config, typename T>
746-__aicore__ inline void SinCosRadianReductionImpl(const LocalTensor<T>& dst0, const LocalTensor<T>& dst1,
747- const LocalTensor<T>& src, uint32_t calCount)
748-{
749- // Only for AI Vector Core.
750- if ASCEND_IS_AIC {
751- return;
752- }
753- LocalTensor<uint8_t> sharedTmpBuffer;
754- bool ans = PopStackBuffer<uint8_t, TPosition::LCM>(sharedTmpBuffer);
755- ASCENDC_ASSERT((ans), { KERNEL_LOG(KERNEL_ERROR, "PopStackBuffer Error!"); });
756- SinCosRadianReductionImpl<config, T>(dst0, dst1, src, sharedTmpBuffer, calCount);
757-}
758-} // namespace AscendC
759- 
760-#endif // LIB_MATH_SINCOS_SINCOS_C310_IMPL_H
Dimpl/adv_api/detail/math/sinh/sinh_l300_impl.h+0-102
@@ -1,102 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/*!
12- * \file sinh_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_MATH_SINH_SINH_L300_IMPL_H
16-#define IMPL_MATH_SINH_SINH_L300_IMPL_H
17- 
18-#include "kernel_tensor.h"
19-#include "../../common/check.h"
20- 
21-namespace AscendC {
22-namespace SinhInternal {
23-// Computes sinh values based on input types.
24-// According formula: sinh(x) = (e^x - e^(-x))/2 = e^(x-ln2) - 0.25/(e^(x-ln2)).
25-template <typename T>
26-__simd_vf__ inline void SinhCompute(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint32_t calCount, uint16_t repeatTimes)
27-{
28- constexpr float scalarNegLnTwo = -0.6931472;
29- constexpr float scalarBrc = 0.25;
30- constexpr uint32_t vlSize = static_cast<uint32_t>(GetVecLen() / sizeof(float));
31- static constexpr MicroAPI::CastTrait sinhCastTraitUpper = { MicroAPI::RegLayout::ZERO,
32- MicroAPI::SatMode::UNKNOWN, MicroAPI::MaskMergeMode::ZEROING, RoundMode::UNKNOWN };
33- static constexpr MicroAPI::CastTrait sinhCastTraitLower = { MicroAPI::RegLayout::ZERO,
34- MicroAPI::SatMode::NO_SAT, MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_RINT };
35- MicroAPI::MaskReg sinhMask;
36- MicroAPI::RegTensor<float> dupReg;
37- MicroAPI::RegTensor<T> srcReg;
38- MicroAPI::RegTensor<float> castUpperReg;
39- MicroAPI::RegTensor<float> computeReg0;
40- MicroAPI::RegTensor<float> computeReg1;
41- MicroAPI::RegTensor<float> resReg;
42- MicroAPI::RegTensor<T> dstReg;
43- MicroAPI::Duplicate(dupReg, scalarBrc);
44- for (uint16_t i = 0; i < repeatTimes; i++) {
45- sinhMask = MicroAPI::UpdateMask<float>(calCount);
46- if constexpr (SupportBytes<T, 2>()) {
47- MicroAPI::DataCopy<half, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcReg, srcUb + i * vlSize);
48- MicroAPI::Cast<float, half, sinhCastTraitUpper>(castUpperReg, srcReg, sinhMask);
49- } else {
50- MicroAPI::DataCopy(castUpperReg, srcUb + i * vlSize);
51- }
52- MicroAPI::Adds(castUpperReg, castUpperReg, scalarNegLnTwo, sinhMask);
53- MicroAPI::Exp(computeReg0, castUpperReg, sinhMask);
54- MicroAPI::Div(computeReg1, dupReg, computeReg0, sinhMask);
55- MicroAPI::Sub(resReg, computeReg0, computeReg1, sinhMask);
56- if constexpr (SupportBytes<T, 2>()) {
57- MicroAPI::Cast<half, float, sinhCastTraitLower>(dstReg, resReg, sinhMask);
58- MicroAPI::DataCopy<half, MicroAPI::StoreDist::DIST_PACK_B32>(dstUb + i * vlSize, dstReg, sinhMask);
59- } else {
60- MicroAPI::DataCopy(dstUb + i * vlSize, resReg, sinhMask);
61- }
62- }
63-}
64-} // namespace SinhInternal
65- 
66-template <typename T, bool isReuseSource = false>
67-__aicore__ inline void SinhImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
68- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
69-{
70- // Only for AI Vector Core.
71- if ASCEND_IS_AIC {
72- return;
73- }
74- 
75- CheckTensorPosition(sharedTmpBuffer, "sharedTmpBuffer", "VECIN, VECOUT, VECCALC");
76- SinhImpl<T, isReuseSource>(dstTensor, srcTensor, calCount);
77-}
78- 
79-template <typename T, bool isReuseSource = false>
80-__aicore__ inline void SinhImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
81- const uint32_t calCount)
82-{
83- // Only for AI Vector Core.
84- if ASCEND_IS_AIC {
85- return;
86- }
87- 
88- static_assert(SupportType<T, half, float>(), "Sinh only support half/float data type on current device!");
89- CheckTensorPosition(dstTensor, "dstTensor", "VECIN, VECOUT, VECCALC");
90- CheckTensorPosition(srcTensor, "srcTensor", "VECIN, VECOUT, VECCALC");
91- CheckCalCount(calCount, "calCount", srcTensor, "srcTensor", "Sinh");
92- CheckCalCount(calCount, "calCount", dstTensor, "dstTensor", "Sinh");
93- 
94- __ubuf__ T *dstUb = (__ubuf__ T *)dstTensor.GetPhyAddr();
95- __ubuf__ T *srcUb = (__ubuf__ T *)srcTensor.GetPhyAddr();
96- constexpr int32_t vlSize = static_cast<int32_t>(GetVecLen() / sizeof(float));
97- uint16_t repeatTimes = static_cast<uint16_t>(CeilDivision(calCount, vlSize));
98- SinhInternal::SinhCompute<T>(dstUb, srcUb, calCount, repeatTimes);
99-}
100-} // namespace AscendC
101- 
102-#endif // IMPL_MATH_SINH_SINH_C310_IMPL_H
Dimpl/adv_api/detail/math/tan/tan_l300_impl.h+0-235
@@ -1,235 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/* !
12- * \file tan_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_MATH_TAN_TAN_L300_IMPL_H
16-#define IMPL_MATH_TAN_TAN_L300_IMPL_H
17- 
18-#include "kernel_tensor.h"
19-#include "../../common/check.h"
20- 
21-namespace AscendC {
22-namespace TanInternal {
23-constexpr MicroAPI::CastTrait TAN_CAST_TRAIT_F162F32 = {
24- MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::UNKNOWN, MicroAPI::MaskMergeMode::ZEROING, RoundMode::UNKNOWN};
25-constexpr MicroAPI::CastTrait TAN_CAST_TRAIT_F322F16 = {
26- MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::SAT, MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};
27-// define the number of x div pi
28-constexpr float PI_FOR_X_TODIV = 0.3183098733425140380859375;
29-constexpr float KPI_FIRS_PI_MULS = 0.0009670257568359375;
30-// define the PI for compute
31-constexpr float PI_V2 = 3.140625;
32-// define the number of down of pi_div
33-constexpr float PI_DOWN = 1.57079637050628662109375;
34-constexpr float PI_DOWN_NEG = -1.57079637050628662109375;
35-// kpi_2
36-constexpr float KPI_TWI_PI_MULS = 6.2771141529083251953125e-7;
37-constexpr float PI_RESDOWN_ADDS = 0.00000004371139000189375;
38-constexpr float PI_RESDOWN_ADDS_NEG = -0.00000004371139000189375;
39-// kpi_3
40-constexpr float KPI_THIR_PI_MULS = 1.21644916362129151821136474609375e-10;
41-// kpi_4
42-constexpr float KPI_FOR_PI_MULS = -1.0291767438275201129727065563201904296875e-13;
43-// define the number of tan_compute
44-constexpr float TAN_RES_MULIT_SCA = 0.0698520831551998762793;
45-constexpr float TAN_RES_ADDICT_UP = -6.8711573651634203789;
46-constexpr float TAN_2ADDS = 61.20362572811089435388;
47-constexpr float TAN_3ADDS = -24.8048928861126769186219;
48- 
49-// normalized x to (-pi/2,pi/2) using x = x-round(x/π)*π
50-__simd_callee__ inline void TanRound(MicroAPI::RegTensor<float> &srcReg, MicroAPI::RegTensor<float> &tmpReg,
51- MicroAPI::RegTensor<float> &roundReg, MicroAPI::RegTensor<float> &resReg, MicroAPI::RegTensor<float> &downReg1,
52- MicroAPI::RegTensor<float> &downReg2, MicroAPI::MaskReg mask)
53-{
54- /*
55- k=round(x/π), x0=x-kπ, x0∈(-π/2, π/2)
56- π=π_0+π_1+π_2+π_3+π_4 achieve final precision compensation.
57- Final solution：
58- k = round(x * invpi)
59- x -= k * pi_0
60- x -= k * pi_1
61- down1 = x + pio2_high // pi/2 + x
62- down2 = x - pio2_high // x - pi/2
63- x -= k * pi_2
64- down1 -= k * pi_2
65- down2 -= k * pi_2
66- down1 -= down_adds
67- down2 += down_adds
68- x -= k * pi_3
69- down1 -= k * pi_3
70- down2 -= k * pi_3
71- x -= k * pi_4
72- down1 -= k * pi_4
73- down2 -= k * pi_4
74- */
75- 
76- // round_pi_div= round(x*0.3183098733425140380859375)
77- MicroAPI::Muls(roundReg, srcReg, PI_FOR_X_TODIV, mask);
78- // tie to even
79- MicroAPI::Truncate<float, RoundMode::CAST_RINT, MicroAPI::MaskMergeMode::ZEROING>(roundReg, roundReg, mask);
80- 
81- // kpi_0 = round_pi_div*3.140625
82- MicroAPI::Muls(tmpReg, roundReg, PI_V2, mask);
83- // input_x = (x-kpi_0)
84- MicroAPI::Sub(resReg, srcReg, tmpReg, mask);
85- 
86- // kpi_1 = muls(round_pi_div, 0.0009670257568359375)
87- MicroAPI::Muls(tmpReg, roundReg, KPI_FIRS_PI_MULS, mask);
88- // input_x = sub(input_x, kpi_1)
89- MicroAPI::Sub(resReg, resReg, tmpReg, mask);
90- // res_down1 = adds(input_x, 1.57079637050628662109375)
91- MicroAPI::Adds(downReg1, resReg, PI_DOWN, mask);
92- // res_down2 = adds(input_x, -1.57079637050628662109375)
93- MicroAPI::Adds(downReg2, resReg, PI_DOWN_NEG, mask);
94- 
95- // kpi_2 = muls(round_pi_div, 6.2771141529083251953125e-7)
96- MicroAPI::Muls(tmpReg, roundReg, KPI_TWI_PI_MULS, mask);
97- // input_x = sub(input_x, kpi_2)
98- MicroAPI::Sub(resReg, resReg, tmpReg, mask);
99- // res_down1 = sub(res_down1, kpi_2)
100- MicroAPI::Sub(downReg1, downReg1, tmpReg, mask);
101- // res_down2 = sub(res_down2, kpi_2)
102- MicroAPI::Sub(downReg2, downReg2, tmpReg, mask);
103- // res_down1 = adds(res_down1, -0.00000004371139000189375)
104- MicroAPI::Adds(downReg1, downReg1, PI_RESDOWN_ADDS_NEG, mask);
105- // res_down2 = adds(res_down2, 0.00000004371139000189375)
106- MicroAPI::Adds(downReg2, downReg2, PI_RESDOWN_ADDS, mask);
107- 
108- // kpi_3 = muls(round_pi_div, 1.21644916362129151821136474609375e-10)
109- MicroAPI::Muls(tmpReg, roundReg, KPI_THIR_PI_MULS, mask);
110- // input_x =sub(input_x, kpi_3)
111- MicroAPI::Sub(resReg, resReg, tmpReg, mask);
112- // res_down1 = sub(res_down1, kpi_3)
113- MicroAPI::Sub(downReg1, downReg1, tmpReg, mask);
114- // res_down2 = sub(res_down2, kpi_3)
115- MicroAPI::Sub(downReg2, downReg2, tmpReg, mask);
116- 
117- // kpi_4 = muls(round_pi_div, -1.0291767438275201129727065563201904296875e-13)
118- MicroAPI::Muls(tmpReg, roundReg, KPI_FOR_PI_MULS, mask);
119- // input_x =sub(input_x, kpi_4)
120- MicroAPI::Sub(resReg, resReg, tmpReg, mask);
121- // res_down1 = sub(res_down1, kpi_4)
122- MicroAPI::Sub(downReg1, downReg1, tmpReg, mask);
123- // res_down2 = sub(res_down2, kpi_4)
124- MicroAPI::Sub(downReg2, downReg2, tmpReg, mask);
125-}
126- 
127-__simd_callee__ inline void TanPolynomialApproximation(MicroAPI::RegTensor<float> &dstReg, MicroAPI::RegTensor<float> &tmpReg,
128- MicroAPI::RegTensor<float> &roundReg, MicroAPI::RegTensor<float> &resReg, MicroAPI::RegTensor<float> &downReg1,
129- MicroAPI::RegTensor<float> &downReg2, MicroAPI::MaskReg mask)
130-{
131- /*
132- tan(x) = xP(x) / ((π/2 - x)(π/2 + x)Q(x))
133- P(x) = (x^2 * R0 + R1) * x^2 + R2
134- Q(x) = x^2 * R3
135- R0 = 0.0698520831551998762793
136- R1 = -6.8711573651634203789
137- R2 = 61.20362572811089435388
138- R3 = -24.8048928861126769186219
139- */
140- 
141- // x^2 = mul(input_x, input_x)
142- MicroAPI::Mul(roundReg, resReg, resReg, mask);
143- // res_up = muls(x^2, 0.0698520831551998762793)
144- MicroAPI::Muls(tmpReg, roundReg, TAN_RES_MULIT_SCA, mask);
145- // res_up = adds(res_up, -6.8711573651634203789)
146- MicroAPI::Adds(tmpReg, tmpReg, TAN_RES_ADDICT_UP, mask);
147- // res_up = mul(res_up, x^2)
148- MicroAPI::Mul(tmpReg, tmpReg, roundReg, mask);
149- // res_up = adds(res_up, 61.20362572811089435388)
150- MicroAPI::Adds(tmpReg, tmpReg, TAN_2ADDS, mask);
151- // res_up = mul(res_up, input_x)
152- MicroAPI::Mul(tmpReg, tmpReg, resReg, mask);
153- // res_down = adds(x^2, -24.8048928861126769186219)
154- MicroAPI::Adds(roundReg, roundReg, TAN_3ADDS, mask);
155- // res_down = mul(res_down, res_down1)
156- MicroAPI::Mul(roundReg, roundReg, downReg1, mask);
157- // res_down = mul(res_down, res_down2)
158- MicroAPI::Mul(roundReg, roundReg, downReg2, mask);
159- // res = div(res_up, res_down)
160- MicroAPI::Div(dstReg, tmpReg, roundReg, mask);
161-}
162- 
163-template <typename T>
164-__simd_vf__ inline void TanCompute(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint32_t sreg, uint16_t repeatTimes)
165-{
166- constexpr uint32_t stride = GetVecLen() / sizeof(float);
167- 
168- MicroAPI::MaskReg mask;
169- MicroAPI::RegTensor<T> srcReg;
170- MicroAPI::RegTensor<float> castReg;
171- MicroAPI::RegTensor<float> roundReg;
172- MicroAPI::RegTensor<float> tmpReg;
173- MicroAPI::RegTensor<float> downReg1;
174- MicroAPI::RegTensor<float> downReg2;
175- MicroAPI::RegTensor<float> resReg;
176- MicroAPI::RegTensor<float> dstReg;
177- for (uint16_t i = 0; i < repeatTimes; ++i) {
178- mask = MicroAPI::UpdateMask<float>(sreg);
179- if constexpr (sizeof(T) == sizeof(half)) {
180- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcReg, srcUb + i * stride);
181- MicroAPI::Cast<float, T, TAN_CAST_TRAIT_F162F32>(castReg, srcReg, mask);
182- } else {
183- MicroAPI::DataCopy(castReg, srcUb + i * stride);
184- }
185- // the input is normalized to (-pi/2,pi/2)
186- TanRound(castReg, tmpReg, roundReg, resReg, downReg1, downReg2, mask);
187- TanPolynomialApproximation(dstReg, tmpReg, roundReg, resReg, downReg1, downReg2, mask);
188- 
189- if constexpr (sizeof(T) == sizeof(half)) {
190- MicroAPI::Cast<T, float, TAN_CAST_TRAIT_F322F16>(srcReg, dstReg, mask);
191- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(dstUb + i * stride, srcReg, mask);
192- } else {
193- MicroAPI::DataCopy(dstUb + i * stride, dstReg, mask);
194- }
195- }
196-}
197-} // namespace TanInternal
198- 
199-template <typename T, bool isReuseSource = false>
200-__aicore__ inline void TanImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
201- const uint32_t calCount)
202-{
203- // Only for AI Vector Core.
204- if ASCEND_IS_AIC {
205- return;
206- }
207- 
208- CheckTensorPosition(dstTensor, "dstTensor", "VECIN, VECOUT, VECCALC");
209- CheckTensorPosition(srcTensor, "srcTensor", "VECIN, VECOUT, VECCALC");
210- 
211- CheckCalCount(calCount, "calCount", srcTensor, "srcTensor", "Tan");
212- CheckCalCount(calCount, "calCount", dstTensor, "dstTensor", "Tan");
213- 
214- static_assert(SupportType<T, half, float>(), "current data type is not supported on current device!");
215- constexpr uint32_t stride = GetVecLen() / sizeof(float);
216- uint16_t repeatTimes = CeilDivision(calCount, stride);
217- 
218- __ubuf__ T *dstUb = (__ubuf__ T *)dstTensor.GetPhyAddr();
219- __ubuf__ T *srcUb = (__ubuf__ T *)srcTensor.GetPhyAddr();
220- 
221- TanInternal::TanCompute<T>(dstUb, srcUb, calCount, repeatTimes);
222-}
223- 
224-template <typename T, bool isReuseSource = false>
225-__aicore__ inline void TanImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
226- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
227-{
228- CheckTensorPosition(sharedTmpBuffer, "sharedTmpBuffer", "VECIN, VECOUT, VECCALC");
229- 
230- TanImpl(dstTensor, srcTensor, calCount);
231-}
232- 
233-} // namespace AscendC
234- 
235-#endif // IMPL_MATH_TAN_TAN_L300_IMPL_H
Dimpl/adv_api/detail/math/tanh/tanh_l300_impl.h+0-240
@@ -1,240 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/*!
12- * \file tanh_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_MATH_TANH_TANH_L300_IMPL_H
16-#define IMPL_MATH_TANH_TANH_L300_IMPL_H
17- 
18-#include "kernel_tensor.h"
19-#include "kernel_pop_stack_buffer.h"
20- 
21-namespace AscendC {
22-namespace TanhInternal {
23-constexpr float FP32_ZERO_015 = 0.0157296831;
24-constexpr float FP32_ZERO_NEG_052 = -0.0523029624;
25-constexpr float FP32_ZERO_133 = 0.133152977;
26-constexpr float FP32_ZERO_NEG_333 = -0.333327681;
27-constexpr float FP32_TWENTY = 20.0;
28-constexpr float FP32_TWO = 2.0;
29-constexpr float FP32_ZERO_55 = 0.55;
30-constexpr float FP32_MIN_EXP = -8.8;
31-constexpr float FP32_MAX_EXP = 8.8;
32- 
33-constexpr MicroAPI::CastTrait tanhCastTraitF162F32 = {
34- MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::UNKNOWN, MicroAPI::MaskMergeMode::ZEROING, RoundMode::UNKNOWN};
35-constexpr MicroAPI::CastTrait tanhCastTraitF322F16 = {
36- MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::NO_SAT, MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_RINT};
37-}
38- 
39-template <typename T>
40-__simd_vf__ inline void TanhIntrinsicImpl(__ubuf__ T *dstUb, __ubuf__ T *srcUb,
41- const uint32_t calCount, const uint16_t repeatTimes)
42-{
43- uint32_t sreg = calCount;
44- MicroAPI::MaskReg preg;
45- MicroAPI::RegTensor<T> srcReg;
46- MicroAPI::RegTensor<float> castReg;
47- MicroAPI::RegTensor<float> tmpReg;
48- MicroAPI::RegTensor<float> dstReg;
49- 
50- for (uint16_t i = 0; i < repeatTimes; ++i) {
51- preg = MicroAPI::UpdateMask<float>(sreg);
52- if constexpr (sizeof(T) == sizeof(half)) {
53- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcReg, srcUb + i * B32_DATA_NUM_PER_REPEAT);
54- MicroAPI::Cast<float, T, TanhInternal::tanhCastTraitF162F32>(castReg, srcReg, preg);
55- } else {
56- MicroAPI::DataCopy(castReg, srcUb + i * B32_DATA_NUM_PER_REPEAT);
57- }
58- MicroAPI::Mins(castReg, castReg, TanhInternal::FP32_MAX_EXP, preg);
59- MicroAPI::Maxs(castReg, castReg, TanhInternal::FP32_MIN_EXP, preg);
60- MicroAPI::Muls(tmpReg, castReg, TanhInternal::FP32_TWO, preg);
61- MicroAPI::Exp(castReg, tmpReg, preg);
62- 
63- MicroAPI::Adds(dstReg, castReg, -1.0f, preg);
64- MicroAPI::Adds(tmpReg, castReg, 1.0f, preg);
65- MicroAPI::Div(dstReg, dstReg, tmpReg, preg);
66- if constexpr (sizeof(T) == sizeof(half)) {
67- MicroAPI::Cast<T, float, TanhInternal::tanhCastTraitF322F16>(srcReg, dstReg, preg);
68- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(dstUb + i * B32_DATA_NUM_PER_REPEAT, srcReg, preg);
69- } else {
70- MicroAPI::DataCopy(dstUb + i * B32_DATA_NUM_PER_REPEAT, dstReg, preg);
71- }
72- }
73-}
74- 
75-template <typename T>
76-__simd_vf__ inline void TanhCompensationImpl(__ubuf__ T *dstUb, __ubuf__ T *srcUb,
77- const uint32_t calCount, const uint16_t repeatTimes)
78-{
79- uint32_t sreg = calCount;
80- MicroAPI::MaskReg preg, cmpMaskReg;
81- MicroAPI::RegTensor<T> srcReg;
82- MicroAPI::RegTensor<float> vregInput, vregInputAbs;
83- MicroAPI::RegTensor<float> vregInputSqr, vregInputMid;
84- MicroAPI::RegTensor<float> vregOutput;
85- MicroAPI::RegTensor<float> vregScalar1, vregScalar2;
86- 
87- MicroAPI::Duplicate(vregScalar1, TanhInternal::FP32_ZERO_133);
88- MicroAPI::Duplicate(vregScalar2, TanhInternal::FP32_ZERO_NEG_333);
89- for (uint16_t i = 0; i < repeatTimes; ++i) {
90- preg = MicroAPI::UpdateMask<float>(sreg);
91- if constexpr (sizeof(T) == sizeof(half)) {
92- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcReg, srcUb + i * B32_DATA_NUM_PER_REPEAT);
93- MicroAPI::Cast<float, T, TanhInternal::tanhCastTraitF162F32>(vregInput, srcReg, preg);
94- } else {
95- MicroAPI::DataCopy(vregInput, srcUb + i * B32_DATA_NUM_PER_REPEAT);
96- }
97- MicroAPI::Mul(vregInputSqr, vregInput, vregInput, preg);
98- MicroAPI::Muls(vregOutput, vregInputSqr, TanhInternal::FP32_ZERO_015, preg);
99- MicroAPI::Adds(vregOutput, vregOutput, TanhInternal::FP32_ZERO_NEG_052, preg);
100- MicroAPI::FusedMulDstAdd(vregOutput, vregInputSqr, vregScalar1, preg);
101- MicroAPI::FusedMulDstAdd(vregOutput, vregInputSqr, vregScalar2, preg);
102- MicroAPI::Mul(vregOutput, vregOutput, vregInputSqr, preg);
103- MicroAPI::FusedMulDstAdd(vregOutput, vregInput, vregInput, preg);
104- 
105- MicroAPI::Abs(vregInputAbs, vregInput, preg);
106- MicroAPI::Mins(vregInput, vregInput, TanhInternal::FP32_TWENTY, preg);
107- MicroAPI::Muls(vregInput, vregInput, TanhInternal::FP32_TWO, preg);
108- MicroAPI::Exp(vregInput, vregInput, preg);
109- MicroAPI::Adds(vregInputMid, vregInput, -1.0f, preg);
110- MicroAPI::Adds(vregInputSqr, vregInput, 1.0f, preg);
111- MicroAPI::Div(vregInputMid, vregInputMid, vregInputSqr, preg);
112- 
113- MicroAPI::CompareScalar<float, CMPMODE::LT>(cmpMaskReg, vregInputAbs, TanhInternal::FP32_ZERO_55, preg);
114- MicroAPI::Select(vregOutput, vregOutput, vregInputMid, cmpMaskReg);
115- 
116- if constexpr (sizeof(T) == sizeof(half)) {
117- MicroAPI::Cast<T, float, TanhInternal::tanhCastTraitF322F16>(srcReg, vregOutput, preg);
118- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(dstUb + i * B32_DATA_NUM_PER_REPEAT, srcReg, preg);
119- } else {
120- MicroAPI::DataCopy(dstUb + i * B32_DATA_NUM_PER_REPEAT, vregOutput, preg);
121- }
122- }
123-}
124- 
125-/*
126- * Formula is y= (e^(2x)-1)/(e^(2x)+1)
127- */
128-template <typename T, bool isReuseSource = false, const TanhConfig &config = DEFAULT_TANH_CONFIG>
129-__aicore__ inline void TanhImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
130- const uint32_t calCount)
131-{
132- static_assert(SupportType<T, half, float>(), "current data type is not supported on current device!");
133- CheckTensorPosition(dstTensor, "dstTensor", "VECIN, VECOUT, VECCALC");
134- CheckTensorPosition(srcTensor, "srcTensor", "VECIN, VECOUT, VECCALC");
135- 
136- CheckCalCount(calCount, "calCount", srcTensor, "srcTensor", "Tanh");
137- CheckCalCount(calCount, "calCount", dstTensor, "dstTensor", "Tanh");
138- 
139- // Only for AI Vector Core.
140- if ASCEND_IS_AIC {
141- return;
142- }
143- __ubuf__ T *dstUb = (__ubuf__ T *)dstTensor.GetPhyAddr();
144- __ubuf__ T *srcUb = (__ubuf__ T *)srcTensor.GetPhyAddr();
145- uint16_t repeatTimes = CeilDivision(calCount, B32_DATA_NUM_PER_REPEAT);
146- if constexpr (config.algo == TanhAlgo::INTRINSIC) {
147- TanhIntrinsicImpl<T>(dstUb, srcUb, calCount, repeatTimes);
148- } else {
149- TanhCompensationImpl<T>(dstUb, srcUb, calCount, repeatTimes);
150- }
151-}
152- 
153-template <typename T, bool isReuseSource = false, const TanhConfig &config = DEFAULT_TANH_CONFIG>
154-__aicore__ inline void TanhImpl(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
155- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
156-{
157- CheckTensorPosition(sharedTmpBuffer, "sharedTmpBuffer", "VECIN, VECOUT, VECCALC");
158- TanhImpl<T, isReuseSource, config>(dstTensor, srcTensor, calCount);
159-}
160- 
161-/*!
162- * \ingroup Tanh
163- * \brief compute Tanh elementwisely
164- * \tparam T: half/float
165- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
166- * this parameter is reserved, please use the default value.
167- * \param [out] dstTensor: output LocalTensor
168- * \param [in] srcTensor: input LocalTensor
169- * \param [in] sharedTmpBuffer: extra temporary shared space used for intermediate values among calculation process,
170- * whose required space size should refer to corresponding tiling API, which is defined at tanh_tiling.h.
171- * Generally, the more space you allocate, the better performance you will achieve, and the performance
172- * reaches peak when buffer size is maximum(calculated by tiling function). Moreover, it is not guaranteed
173- * that the shared space will be cleared after usage, the data could be anything.
174- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
175- */
176-template <typename T, bool isReuseSource = false, const TanhConfig& config = DEFAULT_TANH_CONFIG>
177-__aicore__ inline void Tanh(
178- const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor, const LocalTensor<uint8_t>& sharedTmpBuffer)
179-{
180- Tanh<T, isReuseSource, config>(dstTensor, srcTensor, sharedTmpBuffer, srcTensor.GetSize());
181-}
182- 
183-/*!
184- * \ingroup Tanh
185- * \brief compute Tanh elementwisely
186- * \tparam T: half/float
187- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
188- * this parameter is reserved, please use the default value.
189- * \param [out] dstTensor: output LocalTensor
190- * \param [in] srcTensor: input LocalTensor
191- * \param [in] sharedTmpBuffer: extra temporary shared space used for intermediate values among calculation process,
192- * whose required space size should refer to corresponding tiling API, which is defined at tanh_tiling.h.
193- * Generally, the more space you allocate, the better performance you will achieve, and the performance
194- * reaches peak when buffer size is maximum(calculated by tiling function). Moreover, it is not guaranteed
195- * that the shared space will be cleared after usage, the data could be anything.
196- * \param [in] calCount: the number of elements to be processed.
197- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
198- */
199-template <typename T, bool isReuseSource = false, const TanhConfig& config = DEFAULT_TANH_CONFIG>
200-__aicore__ inline void Tanh(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
201- const LocalTensor<uint8_t>& sharedTmpBuffer, const uint32_t calCount)
202-{
203- TanhImpl<T, isReuseSource, config>(dstTensor, srcTensor, sharedTmpBuffer, calCount);
204-}
205- 
206-/*!
207- * \ingroup Tanh
208- * \brief compute Tanh elementwisely
209- * \tparam T: half/float
210- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
211- * this parameter is reserved, please use the default value.
212- * \param [out] dstTensor: output LocalTensor
213- * \param [in] srcTensor: input LocalTensor
214- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
215- */
216-template <typename T, bool isReuseSource = false, const TanhConfig& config = DEFAULT_TANH_CONFIG>
217-__aicore__ inline void Tanh(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor)
218-{
219- Tanh<T, isReuseSource, config>(dstTensor, srcTensor, srcTensor.GetSize());
220-}
221- 
222-/*!
223- * \ingroup Tanh
224- * \brief compute Tanh elementwisely
225- * \tparam T: half/float
226- * \tparam isReuseSource: whether allows API to modify source data, usually for performance reason,
227- * this parameter is reserved, please use the default value.
228- * \param [out] dstTensor: output LocalTensor
229- * \param [in] srcTensor: input LocalTensor
230- * \param [in] calCount: the number of elements to be processed.
231- * \note src/dst Tensor must be 32B aligned, and it doesn't allow src/dst/sharedTmpBuffer tensor address overlap.
232- */
233-template <typename T, bool isReuseSource = false, const TanhConfig& config = DEFAULT_TANH_CONFIG>
234-__aicore__ inline void Tanh(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor, const uint32_t calCount)
235-{
236- TanhImpl<T, isReuseSource, config>(dstTensor, srcTensor, calCount);
237-}
238-} // namespace AscendC
239- 
240-#endif // IMPL_MATH_TANH_TANH_L300_IMPL_H
Dimpl/adv_api/detail/normalization/rmsnorm/rmsnorm_l300_impl.h+0-189
@@ -1,189 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-#ifndef IMPL_NORMALIZATION_RMSNORM_RMSNORM_L300_IMPL_H
12-#define IMPL_NORMALIZATION_RMSNORM_RMSNORM_L300_IMPL_H
13-#include "kernel_tensor.h"
14-#include "kernel_operator_intf.h"
15-#include "kernel_tiling/kernel_tiling.h"
16-#include "../../api_check/kernel_api_check.h"
17- 
18-namespace AscendC {
19-namespace RmsNormAPI {
20-constexpr int32_t oneRepSize = GetVecLen() / sizeof(float);
21- 
22-template <typename T>
23-__simd_callee__ inline void LoadDataWithT(
24- __ubuf__ T* src, MicroAPI::RegTensor<float>& dstReg, MicroAPI::MaskReg& mask, uint32_t offset)
25-{
26- if constexpr (IsSameType<T, half>::value) {
27- MicroAPI::RegTensor<T> srcOrigin;
28- DataCopy<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(srcOrigin, src + offset);
29- Cast<float, T, layoutZMrgZ>(dstReg, srcOrigin, mask);
30- } else {
31- DataCopy(dstReg, src + offset);
32- }
33-}
34- 
35-template <typename T>
36-__simd_callee__ inline void SaveDataWithT(
37- __ubuf__ T* dst, MicroAPI::RegTensor<float>& srcReg, MicroAPI::MaskReg& mask, uint32_t offset)
38-{
39- if constexpr (IsSameType<T, half>::value) {
40- MicroAPI::RegTensor<T> regT;
41- MicroAPI::Cast<T, float, LayoutZMrgZRndRSatNS>(regT, srcReg, mask);
42- MicroAPI::DataCopy<T, MicroAPI::StoreDist::DIST_PACK_B32>(dst + offset, regT, mask);
43- } else {
44- MicroAPI::DataCopy(dst + offset, srcReg, mask);
45- }
46-}
47- 
48-template <typename T>
49-__simd_callee__ inline void ComputeSum(__ubuf__ float* dstLocal, __ubuf__ T* srcLocal,
50- uint32_t bsLength, uint32_t hLength, uint32_t oriHLength)
51-{
52- uint16_t mainRepeatTime = static_cast<uint16_t>(oriHLength / oneRepSize);
53- uint32_t tailCount = oriHLength % oneRepSize;
54- uint16_t tailRepeatTime = static_cast<uint16_t>(CeilDivision(tailCount, oneRepSize));
55- MicroAPI::RegTensor<float> srcReg;
56- MicroAPI::RegTensor<float> dstReg;
57- MicroAPI::RegTensor<float> dstTailReg;
58- MicroAPI::MaskReg maskFull = MicroAPI::CreateMask<float, MicroAPI::MaskPattern::ALL>();
59- MicroAPI::MaskReg maskOne = MicroAPI::CreateMask<float, MicroAPI::MaskPattern::VL1>();
60- MicroAPI::MaskReg maskReg = MicroAPI::UpdateMask<float>(tailCount);
61- for (uint16_t bsIdx = 0; bsIdx < bsLength; bsIdx++) {
62- MicroAPI::Duplicate(dstReg, static_cast<float>(0), maskFull);
63- for (uint16_t i = 0; i < mainRepeatTime; i++) {
64- LoadDataWithT(srcLocal, srcReg, maskFull, bsIdx * hLength + i * oneRepSize);
65- // step 1: x²
66- MicroAPI::Mul(srcReg, srcReg, srcReg, maskFull);
67- // step 2: ∑x²
68- MicroAPI::Add(dstReg, dstReg, srcReg, maskFull);
69- }
70- for (uint16_t i = 0; i < tailRepeatTime; i++) {
71- LoadDataWithT(srcLocal, srcReg, maskReg, bsIdx * hLength + mainRepeatTime * oneRepSize);
72- // step 1: x²
73- MicroAPI::Mul(srcReg, srcReg, srcReg, maskReg);
74- // step 2: ∑x²
75- MicroAPI::Add(dstTailReg, dstReg, srcReg, maskReg);
76- MicroAPI::Select(dstReg, dstTailReg, dstReg, maskReg);
77- }
78- MicroAPI::ReduceSum(dstReg, dstReg, maskFull);
79- DataCopy<float, MicroAPI::StoreDist::DIST_FIRST_ELEMENT_B32>(dstLocal + bsIdx, dstReg, maskOne);
80- }
81-}
82-template <typename T>
83-__simd_callee__ inline void ComputeY(__ubuf__ T* dstLocal, __ubuf__ T* srcLocal, __ubuf__ T* gammaLocal, __ubuf__ float* tmpLocal,
84- uint32_t bsLength, uint32_t hLength, uint32_t oriHLength, const float epsilon, float reciprocalOfHLength)
85-{
86- constexpr float rsqrtExponent = -0.5;
87- MicroAPI::RegTensor<float> srcReg;
88- MicroAPI::RegTensor<float> src2Reg;
89- MicroAPI::RegTensor<float> gammaReg;
90- MicroAPI::RegTensor<float> dstReg;
91- MicroAPI::RegTensor<float> dstTailReg;
92- 
93- static constexpr MicroAPI::LnSpecificMode lnMode = {MicroAPI::MaskMergeMode::ZEROING, LnAlgo::INTRINSIC};
94- static constexpr MicroAPI::ExpSpecificMode expMode = {MicroAPI::MaskMergeMode::ZEROING, ExpAlgo::INTRINSIC};
95- MicroAPI::MaskReg maskFull = MicroAPI::CreateMask<float, MicroAPI::MaskPattern::ALL>();
96- MicroAPI::MaskReg maskOne = MicroAPI::CreateMask<float, MicroAPI::MaskPattern::VL1>();
97- MicroAPI::Duplicate(dstReg, static_cast<float>(0), maskFull);
98- uint16_t mainRepeatTime = static_cast<uint16_t>(oriHLength / oneRepSize);
99- uint32_t tailCount = oriHLength % oneRepSize;
100- uint16_t tailRepeatTime = static_cast<uint16_t>(CeilDivision(tailCount, oneRepSize));
101- MicroAPI::MaskReg maskReg = MicroAPI::UpdateMask<float>(tailCount);
102- for (uint16_t bsIdx = 0; bsIdx < bsLength; bsIdx++) {
103- for (uint16_t i = 0; i < mainRepeatTime; i++) {
104- LoadDataWithT(srcLocal, srcReg, maskFull, bsIdx * hLength + i * oneRepSize);
105- LoadDataWithT(gammaLocal, gammaReg, maskFull, i * oneRepSize);
106- DataCopy<float, MicroAPI::LoadDist::DIST_BRC_B32>(src2Reg, tmpLocal + bsIdx);
107- // step 3: rms = 1/n*∑
108- MicroAPI::Muls(src2Reg, src2Reg, reciprocalOfHLength, maskFull);
109- // step 4: rms + e
110- MicroAPI::Adds(src2Reg, src2Reg, epsilon, maskFull);
111- // step 5: rsqrt: ln + muls + exp
112- MicroAPI::Ln<float, &lnMode>(src2Reg, src2Reg, maskFull);
113- MicroAPI::Muls(src2Reg, src2Reg, rsqrtExponent, maskFull);
114- MicroAPI::Exp<float, &expMode>(src2Reg, src2Reg, maskFull);
115- // step 6: rms = xi * rsqrt
116- MicroAPI::Mul(src2Reg, srcReg, src2Reg, maskFull);
117- // step 7: rms = rms * gamma
118- MicroAPI::Mul(src2Reg, src2Reg, gammaReg, maskFull);
119- // save
120- SaveDataWithT(dstLocal, src2Reg, maskFull, bsIdx * hLength + i * oneRepSize);
121- }
122- for (uint16_t i = 0; i < tailRepeatTime; i++) {
123- LoadDataWithT(srcLocal, srcReg, maskReg, bsIdx * hLength + mainRepeatTime * oneRepSize);
124- LoadDataWithT(gammaLocal, gammaReg, maskReg, mainRepeatTime * oneRepSize);
125- DataCopy<float, MicroAPI::LoadDist::DIST_BRC_B32>(src2Reg, tmpLocal + bsIdx);
126- // step 3: rms = 1/n*∑
127- MicroAPI::Muls(src2Reg, src2Reg, reciprocalOfHLength, maskReg);
128- // step 4: rms + e
129- MicroAPI::Adds(src2Reg, src2Reg, epsilon, maskReg);
130- // step 5: rsqrt: ln + muls + exp
131- MicroAPI::Ln<float, &lnMode>(src2Reg, src2Reg, maskReg);
132- MicroAPI::Muls(src2Reg, src2Reg, rsqrtExponent, maskReg);
133- MicroAPI::Exp<float, &expMode>(src2Reg, src2Reg, maskReg);
134- // step 6: rms = xi * rsqrt
135- MicroAPI::Mul(src2Reg, srcReg, src2Reg, maskReg);
136- // step 7: rms = rms * gamma
137- MicroAPI::Mul(src2Reg, src2Reg, gammaReg, maskReg);
138- // save
139- SaveDataWithT(dstLocal, src2Reg, maskReg, bsIdx * hLength + mainRepeatTime * oneRepSize);
140- }
141- }
142-}
143- 
144-template <typename T>
145-__simd_vf__ inline void RmsNormImplVf(__ubuf__ T* dstLocal, __ubuf__ T* srcLocal,
146- __ubuf__ T* gammaLocal, __ubuf__ float* tmpLocal, const float epsilon, const RmsNormTiling tiling)
147-{
148- uint32_t bLength = tiling.bLength;
149- uint32_t sLength = tiling.sLength;
150- uint32_t hLength = tiling.hLength;
151- uint32_t oriHLength = tiling.originalHLength;
152- float reciprocalOfHLength = tiling.reciprocalOfHLength;
153- uint16_t loopRound = static_cast<uint16_t>(tiling.loopRound);
154- uint32_t mainBsLength = tiling.mainBsLength;
155- uint32_t mainBshLength = tiling.mainBshLength;
156- 
157- for (uint16_t i = 0; i < loopRound; i++) {
158- ComputeSum(tmpLocal, srcLocal + i * mainBshLength, mainBsLength, hLength, oriHLength);
159- MicroAPI::LocalMemBar<MicroAPI::MemType::VEC_STORE, MicroAPI::MemType::VEC_LOAD>();
160- ComputeY(dstLocal + i * mainBshLength, srcLocal + i * mainBshLength, gammaLocal, tmpLocal, mainBsLength, hLength, oriHLength, epsilon, reciprocalOfHLength);
161- }
162- uint32_t inputTailPos = tiling.inputTailPos;
163- uint32_t tailBsLength = tiling.tailBsLength;
164- uint16_t tailRound = static_cast<uint16_t>(CeilDivision(tailBsLength, mainBsLength));
165- for (uint16_t i = 0; i < tailRound; i++) {
166- ComputeSum(tmpLocal, srcLocal + inputTailPos, tailBsLength, hLength, oriHLength);
167- MicroAPI::LocalMemBar<MicroAPI::MemType::VEC_STORE, MicroAPI::MemType::VEC_LOAD>();
168- ComputeY(dstLocal + inputTailPos, srcLocal + inputTailPos, gammaLocal, tmpLocal, tailBsLength, hLength, oriHLength, epsilon, reciprocalOfHLength);
169- }
170-}
171- 
172-template <typename T, bool isBasicBlock = false>
173-__aicore__ inline void RmsNormImpl(const LocalTensor<T>& dstLocal, const LocalTensor<T>& srcLocal,
174- const LocalTensor<T>& gammaLocal, const LocalTensor<uint8_t>& sharedTmpBuffer, const T epsilon,
175- const RmsNormTiling& tiling)
176-{
177- if ASCEND_IS_AIC {
178- return;
179- }
180- static_assert(SupportType<T, half, float>(), "current data type is not supported on current device!");
181- CHECK_FUNC_HIGHLEVEL_API(RmsNorm, (T, isBasicBlock), (dstLocal, srcLocal, gammaLocal, sharedTmpBuffer, epsilon, tiling));
182- LocalTensor<float> tmpLocal = sharedTmpBuffer.ReinterpretCast<float>();
183- float eps = static_cast<float>(epsilon);
184- RmsNormImplVf<T>((__ubuf__ T*)dstLocal.GetPhyAddr(), (__ubuf__ T*)srcLocal.GetPhyAddr(),
185- (__ubuf__ T*)gammaLocal.GetPhyAddr(), (__ubuf__ float*)tmpLocal.GetPhyAddr(), eps, tiling);
186-}
187-} // namespace RmsNormAPI
188-} // namespace AscendC
189-#endif // IMPL_NORMALIZATION_RMSNORM_RMSNORM_L300_IMPL_H
Dimpl/adv_api/detail/pad/broadcast/broadcast_gather_l300_impl.h+0-485
@@ -1,485 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/* !
12- * \file broadcast_gather_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_PAD_BROADCAST_BROADCAST_GATHER_L300_IMPL_H
16-#define IMPL_PAD_BROADCAST_BROADCAST_GATHER_L300_IMPL_H
17- 
18-#include "kernel_tensor.h"
19-#include "kernel_operator_intf.h"
20- 
21-namespace AscendC {
22-template <typename T>
23-__simd_vf__ inline void VfGenIndex(__ubuf__ T *indexUb, uint32_t sizeI0, uint32_t sizeI1, uint32_t sizeI2, uint32_t strideI0,
24- uint32_t strideI1, uint32_t strideI2, uint32_t offset)
25-{
26- MicroAPI::RegTensor<T> v0;
27- MicroAPI::RegTensor<T> v1;
28- MicroAPI::RegTensor<T> v2;
29- 
30- MicroAPI::RegTensor<T> voffset;
31- MicroAPI::RegTensor<T> vr0;
32- 
33- MicroAPI::RegTensor<T> vd0;
34- MicroAPI::RegTensor<T> vd1;
35- MicroAPI::RegTensor<T> vd2;
36- 
37- MicroAPI::RegTensor<T> vi0;
38- MicroAPI::RegTensor<T> vi1;
39- MicroAPI::RegTensor<T> vi2;
40- 
41- MicroAPI::RegTensor<T> vs0;
42- MicroAPI::RegTensor<T> vs1;
43- MicroAPI::RegTensor<T> vs2;
44- 
45- MicroAPI::MaskReg p0;
46- 
47- p0 = MicroAPI::CreateMask<T>();
48- MicroAPI::Arange(v0, 0);
49- 
50- MicroAPI::Duplicate(v1, (T)sizeI2, p0);
51- MicroAPI::Div(vd0, v0, v1, p0);
52- MicroAPI::Mul(v2, vd0, v1, p0);
53- MicroAPI::Sub(vi2, v0, v2, p0);
54- 
55- MicroAPI::Duplicate(v1, (T)sizeI1, p0);
56- MicroAPI::Div(vd1, vd0, v1, p0);
57- MicroAPI::Mul(v2, vd1, v1, p0);
58- MicroAPI::Sub(vi1, vd0, v2, p0);
59- 
60- MicroAPI::Duplicate(v1, (T)sizeI0, p0);
61- MicroAPI::Div(vd2, vd1, v1, p0);
62- MicroAPI::Mul(v2, vd2, v1, p0);
63- MicroAPI::Sub(vi0, vd1, v2, p0);
64- 
65- MicroAPI::Duplicate(vs0, (T)strideI0, p0);
66- MicroAPI::Duplicate(vs1, (T)strideI1, p0);
67- MicroAPI::Duplicate(vs2, (T)strideI2, p0);
68- 
69- MicroAPI::Mul(vr0, vs2, vi2, p0);
70- MicroAPI::MulAddDst(vr0, vs1, vi1, p0);
71- MicroAPI::MulAddDst(vr0, vs0, vi0, p0);
72- 
73- MicroAPI::Duplicate(voffset, (T)offset, p0);
74- MicroAPI::Add(vr0, vr0, voffset, p0);
75- 
76- MicroAPI::DataCopy(indexUb, vr0, p0);
77-}
78- 
79-template <typename T>
80-__simd_vf__ inline void VfGenIndexForFourDim(__ubuf__ T *indexUb, uint32_t sizeI0, uint32_t sizeI1, uint32_t sizeI2, uint32_t sizeI3,
81- uint32_t strideI0, uint32_t strideI1, uint32_t strideI2, uint32_t strideI3, uint32_t offset)
82-{
83- MicroAPI::RegTensor<T> v0;
84- MicroAPI::RegTensor<T> v1;
85- MicroAPI::RegTensor<T> v2;
86- 
87- MicroAPI::RegTensor<T> voffset;
88- MicroAPI::RegTensor<T> vr0;
89- 
90- MicroAPI::RegTensor<T> vd0;
91- MicroAPI::RegTensor<T> vd1;
92- MicroAPI::RegTensor<T> vd2;
93- MicroAPI::RegTensor<T> vd3;
94- 
95- MicroAPI::RegTensor<T> vi0;
96- MicroAPI::RegTensor<T> vi1;
97- MicroAPI::RegTensor<T> vi2;
98- MicroAPI::RegTensor<T> vi3;
99- 
100- MicroAPI::RegTensor<T> vs0;
101- MicroAPI::RegTensor<T> vs1;
102- MicroAPI::RegTensor<T> vs2;
103- MicroAPI::RegTensor<T> vs3;
104- 
105- MicroAPI::MaskReg p0;
106- 
107- p0 = MicroAPI::CreateMask<T>();
108- MicroAPI::Arange(v0, 0);
109- 
110- MicroAPI::Duplicate(v1, (T)sizeI3, p0);
111- MicroAPI::Div(vd0, v0, v1, p0);
112- MicroAPI::Mul(v2, vd0, v1, p0);
113- MicroAPI::Sub(vi3, v0, v2, p0);
114- 
115- MicroAPI::Duplicate(v1, (T)sizeI2, p0);
116- MicroAPI::Div(vd1, vd0, v1, p0);
117- MicroAPI::Mul(v2, vd1, v1, p0);
118- MicroAPI::Sub(vi2, vd0, v2, p0);
119- 
120- MicroAPI::Duplicate(v1, (T)sizeI1, p0);
121- MicroAPI::Div(vd2, vd1, v1, p0);
122- MicroAPI::Mul(v2, vd2, v1, p0);
123- MicroAPI::Sub(vi1, vd1, v2, p0);
124- 
125- MicroAPI::Duplicate(v1, (T)sizeI0, p0);
126- MicroAPI::Div(vd3, vd2, v1, p0);
127- MicroAPI::Mul(v2, vd3, v1, p0);
128- MicroAPI::Sub(vi0, vd2, v2, p0);
129- 
130- MicroAPI::Duplicate(vs0, (T)strideI0, p0);
131- MicroAPI::Duplicate(vs1, (T)strideI1, p0);
132- MicroAPI::Duplicate(vs2, (T)strideI2, p0);
133- MicroAPI::Duplicate(vs3, (T)strideI3, p0);
134- 
135- MicroAPI::Mul(vr0, vs3, vi3, p0);
136- MicroAPI::MulAddDst(vr0, vs2, vi2, p0);
137- MicroAPI::MulAddDst(vr0, vs1, vi1, p0);
138- MicroAPI::MulAddDst(vr0, vs0, vi0, p0);
139- 
140- MicroAPI::Duplicate(voffset, (T)offset, p0);
141- MicroAPI::Add(vr0, vr0, voffset, p0);
142- 
143- MicroAPI::DataCopy(indexUb, vr0, p0);
144-}
145- 
146-template <typename T>
147-__simd_vf__ inline void VfGatherBrc(__ubuf__ T *dstUb, __ubuf__ T *srcUb, __ubuf__ T *indexUb, uint16_t size0, uint16_t size1,
148- uint16_t size2, uint16_t srcStride0, uint16_t srcStride1, uint16_t srcStride2, uint32_t main, uint32_t tail)
149-{
150- MicroAPI::UnalignReg u0;
151- MicroAPI::RegTensor<T> vindex0;
152- MicroAPI::RegTensor<T> vindex;
153- MicroAPI::RegTensor<T> vstride0;
154- MicroAPI::RegTensor<T> vstride1;
155- MicroAPI::RegTensor<T> vstride2;
156- MicroAPI::RegTensor<T> vbase0;
157- MicroAPI::RegTensor<T> vbase1;
158- MicroAPI::RegTensor<T> vbase2;
159- MicroAPI::RegTensor<T> voffset0;
160- MicroAPI::RegTensor<T> voffset1;
161- MicroAPI::RegTensor<T> voffset2;
162- 
163- MicroAPI::RegTensor<T> vd0;
164- MicroAPI::RegTensor<T> vd1;
165- 
166- MicroAPI::MaskReg pa;
167- MicroAPI::MaskReg p0;
168- MicroAPI::MaskReg p1;
169- pa = MicroAPI::CreateMask<T>();
170- uint32_t main1 = main;
171- uint32_t tail1 = tail;
172- p0 = MicroAPI::UpdateMask<T>(main);
173- p1 = MicroAPI::UpdateMask<T>(tail);
174- MicroAPI::Duplicate(vstride0, (T)srcStride0, pa);
175- MicroAPI::Duplicate(vstride1, (T)srcStride1, pa);
176- MicroAPI::Duplicate(vstride2, (T)srcStride2, pa);
177- MicroAPI::DataCopy(vindex0, indexUb);
178- for (uint16_t i0 = 0; i0 < size0; ++i0) {
179- MicroAPI::Muls(voffset0, vstride0, (T)i0, p0);
180- MicroAPI::Add(vbase0, voffset0, vindex0, p0);
181- for (uint16_t i1 = 0; i1 < size1; ++i1) {
182- MicroAPI::Muls(voffset1, vstride1, (T)i1, p0);
183- MicroAPI::Add(vbase1, vbase0, voffset1, p0);
184- for (uint16_t i2 = 0; i2 < size2; ++i2) {
185- MicroAPI::Muls(voffset2, vstride2, (T)i2, p0);
186- MicroAPI::Add(vindex, vbase1, voffset2, p0);
187- MicroAPI::DataCopyGather(vd0, srcUb, vindex, p0);
188- MicroAPI::DataCopyUnAlign(dstUb, vd0, u0, main1);
189- }
190- MicroAPI::Muls(voffset2, vstride2, (T)size2, p1);
191- MicroAPI::Add(vindex, vbase1, voffset2, p1);
192- MicroAPI::DataCopyGather(vd1, srcUb, vindex, p1);
193- MicroAPI::DataCopyUnAlign(dstUb, vd1, u0, tail1);
194- }
195- }
196- MicroAPI::DataCopyUnAlignPost(dstUb, u0, 0);
197-}
198- 
199-template <typename T>
200-__simd_vf__ inline void VfGatherBrcForFourDim(__ubuf__ T *dstUb, __ubuf__ T *srcUb, __ubuf__ T *indexUb, uint16_t size0, uint16_t size1,
201- uint16_t size2, uint16_t size3, uint16_t srcStride0, uint16_t srcStride1, uint16_t srcStride2, uint16_t srcStride3, uint32_t main,
202- uint32_t tail)
203-{
204- MicroAPI::UnalignReg u0;
205- MicroAPI::RegTensor<T> vindex0;
206- MicroAPI::RegTensor<T> vindex;
207- MicroAPI::RegTensor<T> vstride0;
208- MicroAPI::RegTensor<T> vstride1;
209- MicroAPI::RegTensor<T> vstride2;
210- MicroAPI::RegTensor<T> vstride3;
211- MicroAPI::RegTensor<T> vbase0;
212- MicroAPI::RegTensor<T> vbase1;
213- MicroAPI::RegTensor<T> vbase2;
214- MicroAPI::RegTensor<T> voffset0;
215- MicroAPI::RegTensor<T> voffset1;
216- MicroAPI::RegTensor<T> voffset2;
217- MicroAPI::RegTensor<T> voffset3;
218- 
219- MicroAPI::RegTensor<T> vd0;
220- MicroAPI::RegTensor<T> vd1;
221- 
222- MicroAPI::MaskReg pa;
223- MicroAPI::MaskReg p0;
224- MicroAPI::MaskReg p1;
225- pa = MicroAPI::CreateMask<T>();
226- uint32_t main1 = main;
227- uint32_t tail1 = tail;
228- p0 = MicroAPI::UpdateMask<T>(main);
229- p1 = MicroAPI::UpdateMask<T>(tail);
230- MicroAPI::Duplicate(vstride0, (T)srcStride0, pa);
231- MicroAPI::Duplicate(vstride1, (T)srcStride1, pa);
232- MicroAPI::Duplicate(vstride2, (T)srcStride2, pa);
233- MicroAPI::Duplicate(vstride3, (T)srcStride3, pa);
234- MicroAPI::DataCopy(vindex0, indexUb);
235- for (uint16_t i0 = 0; i0 < size0; ++i0) {
236- MicroAPI::Muls(voffset0, vstride0, (T)i0, p0);
237- MicroAPI::Add(vbase0, voffset0, vindex0, p0);
238- for (uint16_t i1 = 0; i1 < size1; ++i1) {
239- MicroAPI::Muls(voffset1, vstride1, (T)i1, p0);
240- MicroAPI::Add(vbase1, vbase0, voffset1, p0);
241- for (uint16_t i2 = 0; i2 < size2; ++i2) {
242- MicroAPI::Muls(voffset2, vstride2, (T)i2, p0);
243- MicroAPI::Add(vbase2, vbase1, voffset2, p0);
244- for (uint16_t i3 = 0; i3 < size3; ++i3) {
245- MicroAPI::Muls(voffset3, vstride3, (T)i3, p0);
246- MicroAPI::Add(vindex, vbase2, voffset3, p0);
247- MicroAPI::DataCopyGather(vd0, srcUb, vindex, p0);
248- MicroAPI::DataCopyUnAlign(dstUb, vd0, u0, main1);
249- }
250- MicroAPI::Muls(voffset3, vstride3, (T)size3, p1);
251- MicroAPI::Add(vindex, vbase2, voffset3, p1);
252- MicroAPI::DataCopyGather(vd1, srcUb, vindex, p1);
253- MicroAPI::DataCopyUnAlign(dstUb, vd1, u0, tail1);
254- }
255- }
256- }
257- MicroAPI::DataCopyUnAlignPost(dstUb, u0, 0);
258-}
259- 
260-template <typename T> __aicore__ inline void GenGatherIndex(__ubuf__ T *indexUb, uint32_t *size, uint32_t *srcStride)
261-{
262- constexpr uint32_t VF_LEN = GetVecLen() / sizeof(T);
263- uint32_t sizeI[3];
264- uint32_t srcStrideI[3];
265- srcStrideI[0] = srcStride[0];
266- srcStrideI[1] = srcStride[1];
267- srcStrideI[2] = srcStride[2];
268- 
269- if (size[2] * size[1] * size[0] < VF_LEN) {
270- sizeI[0] = size[0];
271- sizeI[1] = size[1];
272- sizeI[2] = size[2];
273- } else if (size[2] * size[1] < VF_LEN) {
274- sizeI[0] = VF_LEN / (size[2] * size[1]);
275- sizeI[1] = size[1];
276- sizeI[2] = size[2];
277- } else if (size[2] < VF_LEN) {
278- sizeI[0] = 1;
279- sizeI[1] = VF_LEN / size[2];
280- sizeI[2] = size[2];
281- } else {
282- sizeI[0] = 1;
283- sizeI[1] = 1;
284- sizeI[2] = VF_LEN;
285- }
286- 
287- VfGenIndex<T>(indexUb, sizeI[0], sizeI[1], sizeI[2], srcStrideI[0], srcStrideI[1],
288- srcStrideI[2], 0);
289-}
290- 
291-template <typename T> __aicore__ inline void GenGatherIndexForFourDim(__ubuf__ T *indexUb, uint32_t *size, uint32_t *srcStride)
292-{
293- constexpr uint32_t VF_LEN = GetVecLen() / sizeof(T);
294- uint32_t sizeI[4];
295- uint32_t srcStrideI[4];
296- srcStrideI[0] = srcStride[0];
297- srcStrideI[1] = srcStride[1];
298- srcStrideI[2] = srcStride[2];
299- srcStrideI[3] = srcStride[3];
300- 
301- if (size[3] * size[2] * size[1] * size[0] < VF_LEN) {
302- sizeI[0] = size[0];
303- sizeI[1] = size[1];
304- sizeI[2] = size[2];
305- sizeI[3] = size[3];
306- } else if (size[3] * size[2] * size[1] < VF_LEN) {
307- sizeI[0] = VF_LEN / (size[3] * size[2] * size[1]);
308- sizeI[1] = size[1];
309- sizeI[2] = size[2];
310- sizeI[3] = size[3];
311- } else if (size[3] * size[2] < VF_LEN) {
312- sizeI[0] = 1;
313- sizeI[1] = VF_LEN / (size[3] * size[2]);
314- sizeI[2] = size[2];
315- sizeI[3] = size[3];
316- } else if (size[3] < VF_LEN) {
317- sizeI[0] = 1;
318- sizeI[1] = 1;
319- sizeI[2] = VF_LEN / size[3];
320- sizeI[3] = size[3];
321- } else {
322- sizeI[0] = 1;
323- sizeI[1] = 1;
324- sizeI[2] = 1;
325- sizeI[3] = VF_LEN;
326- }
327- 
328- VfGenIndexForFourDim<T>(indexUb, sizeI[0], sizeI[1], sizeI[2], sizeI[3], srcStrideI[0], srcStrideI[1],
329- srcStrideI[2], srcStrideI[3], 0);
330-}
331- 
332-template <typename T>
333-__aicore__ inline void GatherWrapper(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint32_t *size, uint32_t *srcStride)
334-{
335- constexpr uint32_t VF_LEN = GetVecLen() / sizeof(T);
336- uint16_t sizeI[3];
337- uint16_t srcStrideI[3];
338- uint32_t main;
339- uint32_t tail;
340- uint32_t vlTile0;
341- uint32_t vlTile1;
342- uint32_t vlTile2;
343- 
344- if (size[2] * size[1] < VF_LEN) {
345- vlTile2 = size[2];
346- vlTile1 = size[1];
347- vlTile0 = VF_LEN / (vlTile2 * vlTile1);
348- sizeI[0] = 1;
349- sizeI[1] = 1;
350- sizeI[2] = static_cast<uint16_t>(size[0] / vlTile0);
351- srcStrideI[0] = 0;
352- srcStrideI[1] = 0;
353- srcStrideI[2] = static_cast<uint16_t>(srcStride[0] * vlTile0);
354- main = vlTile2 * vlTile1 * vlTile0;
355- tail = size[2] * size[1] * size[0] - sizeI[2] * main;
356- } else if (size[2] < VF_LEN) {
357- vlTile2 = size[2];
358- vlTile1 = VF_LEN / (vlTile2);
359- sizeI[0] = 1;
360- sizeI[1] = size[0];
361- sizeI[2] = size[1] / vlTile1;
362- srcStrideI[0] = 0;
363- srcStrideI[1] = static_cast<uint16_t>(srcStride[0]);
364- srcStrideI[2] = static_cast<uint16_t>(srcStride[1] * vlTile1);
365- main = vlTile2 * vlTile1;
366- tail = size[2] * size[1] - sizeI[2] * main;
367- } else {
368- vlTile2 = VF_LEN;
369- sizeI[0] = size[0];
370- sizeI[1] = size[1];
371- sizeI[2] = size[2] / vlTile2;
372- srcStrideI[0] = static_cast<uint16_t>(srcStride[0]);
373- srcStrideI[1] = static_cast<uint16_t>(srcStride[1]);
374- srcStrideI[2] = static_cast<uint16_t>(srcStride[2] * vlTile2);
375- main = vlTile2;
376- tail = size[2] - sizeI[2] * main;
377- }
378- constexpr uint32_t U16_MAX = 65536;
379- ASCENDC_ASSERT((sizeI[2] <= U16_MAX), { KERNEL_LOG(KERNEL_ERROR, "shape should less than uint16 max"); });
380- ASCENDC_ASSERT((sizeI[1] <= U16_MAX), { KERNEL_LOG(KERNEL_ERROR, "shape should less than uint16 max"); });
381- ASCENDC_ASSERT((sizeI[0] <= U16_MAX), { KERNEL_LOG(KERNEL_ERROR, "shape should less than uint16 max"); });
382- LocalTensor<T> indexUb;
383- PopStackBuffer<T, TPosition::LCM>(indexUb);
384- if constexpr (sizeof(T) == sizeof(uint32_t)) {
385- GenGatherIndex((__ubuf__ int32_t *)indexUb.GetPhyAddr(), size, srcStride);
386- VfGatherBrc<uint32_t>((__ubuf__ uint32_t *)dstUb, (__ubuf__ uint32_t *)srcUb,
387- (__ubuf__ uint32_t *)indexUb.GetPhyAddr(), sizeI[0], sizeI[1], sizeI[2], srcStrideI[0], srcStrideI[1],
388- srcStrideI[2], main, tail);
389- } else {
390- GenGatherIndex((__ubuf__ int16_t *)indexUb.GetPhyAddr(), size, srcStride);
391- VfGatherBrc<uint16_t>((__ubuf__ uint16_t *)dstUb, (__ubuf__ uint16_t *)srcUb,
392- (__ubuf__ uint16_t *)indexUb.GetPhyAddr(), sizeI[0], sizeI[1], sizeI[2], srcStrideI[0], srcStrideI[1],
393- srcStrideI[2], main, tail);
394- }
395-}
396- 
397-template <typename T>
398-__aicore__ inline void GatherWrapperForFourDim(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint32_t *size, uint32_t *srcStride)
399-{
400- constexpr uint32_t VF_LEN = GetVecLen() / sizeof(T);
401- uint16_t sizeI[4];
402- uint16_t srcStrideI[4];
403- uint32_t main;
404- uint32_t tail;
405- uint32_t vlTile0;
406- uint32_t vlTile1;
407- uint32_t vlTile2;
408- uint32_t vlTile3;
409- 
410- if (size[3] * size[2] * size[1] < VF_LEN) {
411- vlTile3 = size[3];
412- vlTile2 = size[2];
413- vlTile1 = size[1];
414- vlTile0 = VF_LEN / (vlTile3 * vlTile2 * vlTile1);
415- sizeI[0] = 1;
416- sizeI[1] = 1;
417- sizeI[2] = 1;
418- sizeI[3] = static_cast<uint16_t>(size[0] / vlTile0);
419- srcStrideI[0] = 0;
420- srcStrideI[1] = 0;
421- srcStrideI[2] = 0;
422- srcStrideI[3] = static_cast<uint16_t>(srcStride[0] * vlTile0);
423- main = vlTile3 * vlTile2 * vlTile1 * vlTile0;
424- tail = size[3] * size[2] * size[1] * size[0] - sizeI[3] * main;
425- } else if (size[3] * size[2] < VF_LEN) {
426- vlTile3 = size[3];
427- vlTile2 = size[2];
428- vlTile1 = VF_LEN / (vlTile2 * vlTile3);
429- sizeI[0] = 1;
430- sizeI[1] = 1;
431- sizeI[2] = size[0];
432- sizeI[3] = static_cast<uint16_t>(size[1] / vlTile1);
433- srcStrideI[0] = 0;
434- srcStrideI[1] = 0;
435- srcStrideI[2] = static_cast<uint16_t>(srcStride[0]);
436- srcStrideI[3] = static_cast<uint16_t>(srcStride[1] * vlTile1);
437- main = vlTile3 * vlTile2 * vlTile1;
438- tail = size[3] * size[2] * size[1] - sizeI[3] * main;
439- } else if (size[3] < VF_LEN) {
440- vlTile3 = size[3];
441- vlTile2 = VF_LEN / vlTile3;
442- sizeI[0] = 1;
443- sizeI[1] = size[0];;
444- sizeI[2] = size[1];
445- sizeI[3] = static_cast<uint16_t>(size[2] / vlTile2);
446- srcStrideI[0] = 0;
447- srcStrideI[1] = static_cast<uint16_t>(srcStride[0]);
448- srcStrideI[2] = static_cast<uint16_t>(srcStride[1]);
449- srcStrideI[3] = static_cast<uint16_t>(srcStride[2] * vlTile2);
450- main = vlTile3 * vlTile2;
451- tail = size[3] * size[2] - sizeI[3] * main;
452- } else {
453- vlTile3 = VF_LEN;
454- sizeI[0] = size[0];
455- sizeI[1] = size[1];
456- sizeI[2] = size[2];
457- sizeI[3] = static_cast<uint16_t>(size[3] / vlTile3);
458- srcStrideI[0] = static_cast<uint16_t>(srcStride[0]);
459- srcStrideI[1] = static_cast<uint16_t>(srcStride[1]);
460- srcStrideI[2] = static_cast<uint16_t>(srcStride[2]);
461- srcStrideI[3] = static_cast<uint16_t>(srcStride[3] * vlTile3);
462- main = vlTile3;
463- tail = size[3] - sizeI[3] * main;
464- }
465- constexpr uint32_t U16_MAX = 65536;
466- ASCENDC_ASSERT((sizeI[3] <= U16_MAX), { KERNEL_LOG(KERNEL_ERROR, "shape should less than uint16 max"); });
467- ASCENDC_ASSERT((sizeI[2] <= U16_MAX), { KERNEL_LOG(KERNEL_ERROR, "shape should less than uint16 max"); });
468- ASCENDC_ASSERT((sizeI[1] <= U16_MAX), { KERNEL_LOG(KERNEL_ERROR, "shape should less than uint16 max"); });
469- ASCENDC_ASSERT((sizeI[0] <= U16_MAX), { KERNEL_LOG(KERNEL_ERROR, "shape should less than uint16 max"); });
470- LocalTensor<T> indexUb;
471- PopStackBuffer<T, TPosition::LCM>(indexUb);
472- if constexpr (sizeof(T) == sizeof(uint32_t)) {
473- GenGatherIndexForFourDim((__ubuf__ int32_t *)indexUb.GetPhyAddr(), size, srcStride);
474- VfGatherBrcForFourDim<uint32_t>((__ubuf__ uint32_t *)dstUb, (__ubuf__ uint32_t *)srcUb,
475- (__ubuf__ uint32_t *)indexUb.GetPhyAddr(), sizeI[0], sizeI[1], sizeI[2], sizeI[3], srcStrideI[0],
476- srcStrideI[1], srcStrideI[2], srcStrideI[3], main, tail);
477- } else {
478- GenGatherIndexForFourDim((__ubuf__ int16_t *)indexUb.GetPhyAddr(), size, srcStride);
479- VfGatherBrcForFourDim<uint16_t>((__ubuf__ uint16_t *)dstUb, (__ubuf__ uint16_t *)srcUb,
480- (__ubuf__ uint16_t *)indexUb.GetPhyAddr(), sizeI[0], sizeI[1], sizeI[2], sizeI[3], srcStrideI[0],
481- srcStrideI[1], srcStrideI[2], srcStrideI[3], main, tail);
482- }
483-}
484-} // namespace AscendC
485-#endif // IMPL_PAD_BROADCAST_BROADCAST_GATHER_L300_IMPL_H
Dimpl/adv_api/detail/pad/broadcast/broadcast_l300_impl.h+0-1472
@@ -1,1472 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/* !
12- * \file broadcast_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_PAD_BROADCAST_BROADCAST_L300_IMPL_H
16-#define IMPL_PAD_BROADCAST_BROADCAST_L300_IMPL_H
17- 
18-#include "kernel_tensor.h"
19-#include "kernel_operator_intf.h"
20-#include "broadcast_gather_l300_impl.h"
21- 
22-namespace AscendC {
23-namespace BroadcastInternal {
24-template <typename T>
25-__simd_callee__ inline void E2bLoad(MicroAPI::RegTensor<T> &dstReg, __ubuf__ T *srcUb)
26-{
27- if constexpr (sizeof(T) == 2) {
28- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_E2B_B16>(dstReg, srcUb);
29- } else {
30- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_E2B_B32>(dstReg, srcUb);
31- }
32-}
33- 
34-template <typename T>
35-__simd_callee__ inline void BrcLoad(MicroAPI::RegTensor<T> &dstReg, __ubuf__ T *srcUb)
36-{
37- if constexpr (sizeof(T) == 2) {
38- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_BRC_B16>(dstReg, srcUb);
39- } else if constexpr (sizeof(T) == 4) {
40- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_BRC_B32>(dstReg, srcUb);
41- } else {
42- MicroAPI::DataCopy<T, MicroAPI::LoadDist::DIST_BRC_B8>(dstReg, srcUb);
43- }
44-}
45- 
46-template <typename T>
47-__simd_vf__ inline void BrcDuplicate(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint32_t dstSize)
48-{
49- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
50- uint16_t repeatTimes = CeilDivision(dstSize, VF_LEN);
51- uint32_t sreg = dstSize;
52- 
53- MicroAPI::MaskReg pregCnt;
54- MicroAPI::RegTensor<T> srcReg;
55- BrcLoad<T>(srcReg, srcUb);
56- for (uint16_t i = 0; i < repeatTimes; ++i) {
57- pregCnt = MicroAPI::UpdateMask<T>(sreg);
58- MicroAPI::DataCopy(dstUb + i * VF_LEN, srcReg, pregCnt);
59- }
60-}
61- 
62-template <typename T>
63-__simd_vf__ inline void GenLastGatherIndex(__ubuf__ T *indexUb, uint32_t size1, uint32_t offset)
64-{
65- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<T>();
66- MicroAPI::RegTensor<T> indexReg;
67- MicroAPI::RegTensor<T> tmpReg;
68- 
69- MicroAPI::Duplicate(indexReg, (T)size1, pregFull);
70- MicroAPI::Arange(tmpReg, (T)offset);
71- MicroAPI::Div(indexReg, tmpReg, indexReg, pregFull);
72- 
73- MicroAPI::DataCopy(indexUb, indexReg, pregFull);
74-}
75- 
76-template <typename T>
77-__simd_vf__ inline void GenNlastGatherIndex(__ubuf__ T *indexUb, uint32_t size1, uint32_t offset)
78-{
79- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<T>();
80- MicroAPI::RegTensor<T> indexReg;
81- MicroAPI::RegTensor<T> tmpReg;
82- MicroAPI::RegTensor<T> dstReg;
83- 
84- MicroAPI::Duplicate(indexReg, (T)size1, pregFull);
85- MicroAPI::Arange(tmpReg, (T)offset);
86- MicroAPI::Div(dstReg, tmpReg, indexReg, pregFull);
87- MicroAPI::Mul(dstReg, indexReg, dstReg, pregFull);
88- MicroAPI::Sub(indexReg, tmpReg, dstReg, pregFull);
89- 
90- MicroAPI::DataCopy(indexUb, indexReg, pregFull);
91-}
92- 
93-template <typename T, typename IndexT>
94-__simd_vf__ inline void BrcLastGatherOne(
95- __ubuf__ T *dstUb, __ubuf__ T *srcUb, __ubuf__ IndexT *indexUb, uint16_t size0, uint16_t size1)
96-{
97- constexpr uint32_t VF_LEN_HALF = GetVecLen() / 2 / sizeof(T);
98- uint32_t main = size0 * size1;
99- 
100- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<IndexT>();
101- MicroAPI::MaskReg pregCnt;
102- MicroAPI::RegTensor<T> srcReg;
103- MicroAPI::RegTensor<T> dummyReg;
104- MicroAPI::RegTensor<IndexT> srcReg1;
105- MicroAPI::RegTensor<IndexT> srcReg2;
106- MicroAPI::RegTensor<IndexT> indexReg1;
107- MicroAPI::RegTensor<IndexT> indexReg2;
108- 
109- MicroAPI::DataCopy(indexReg1, indexUb);
110- if constexpr (sizeof(T) == sizeof(uint8_t)) {
111- MicroAPI::DataCopy(indexReg2, indexUb + VF_LEN_HALF);
112- }
113- uint32_t sreg = main;
114- pregCnt = MicroAPI::UpdateMask<T>(sreg);
115- if constexpr (sizeof(T) == sizeof(uint8_t)) {
116- MicroAPI::DataCopyGather(srcReg1, srcUb, indexReg1, pregFull);
117- MicroAPI::DataCopyGather(srcReg2, srcUb, indexReg2, pregFull);
118- MicroAPI::DeInterleave(
119- srcReg, dummyReg, (MicroAPI::RegTensor<T> &)srcReg1, (MicroAPI::RegTensor<T> &)srcReg2);
120- } else {
121- MicroAPI::DataCopyGather(srcReg, srcUb, indexReg1, pregCnt);
122- }
123- MicroAPI::DataCopy(dstUb, srcReg, pregCnt);
124-}
125- 
126-template <typename T, typename IndexT>
127-__simd_vf__ inline void BrcLastGatherTwo(
128- __ubuf__ T *dstUb, __ubuf__ T *srcUb, __ubuf__ IndexT *indexUb, uint16_t size0, uint16_t size1)
129-{
130- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
131- constexpr uint32_t VF_LEN_HALF = GetVecLen() / 2 / sizeof(T);
132- uint16_t factor = VF_LEN / size1;
133- uint16_t repeatTimes = CeilDivision(size0, factor) - 1;
134- uint32_t main = factor * size1;
135- uint32_t mainBlock = main * repeatTimes;
136- uint32_t offset = factor * repeatTimes;
137- uint32_t tail = size0 * size1 - mainBlock;
138- 
139- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<IndexT>();
140- MicroAPI::RegTensor<T> srcReg;
141- MicroAPI::RegTensor<T> dummyReg;
142- MicroAPI::RegTensor<IndexT> indexReg1;
143- MicroAPI::RegTensor<IndexT> indexReg2;
144- MicroAPI::RegTensor<IndexT> factorReg;
145- MicroAPI::RegTensor<IndexT> srcReg1;
146- MicroAPI::RegTensor<IndexT> srcReg2;
147- MicroAPI::RegTensor<IndexT> dstReg;
148- MicroAPI::RegTensor<IndexT> tmpReg;
149- MicroAPI::UnalignReg ureg0;
150- 
151- MicroAPI::Duplicate(factorReg, (IndexT)factor, pregFull);
152- MicroAPI::DataCopy(indexReg1, indexUb);
153- if constexpr (sizeof(T) == sizeof(uint8_t)) {
154- MicroAPI::DataCopy(indexReg2, indexUb + VF_LEN_HALF);
155- }
156- for (uint16_t i = 0; i < repeatTimes; ++i) {
157- MicroAPI::Muls(tmpReg, factorReg, (IndexT)i, pregFull);
158- MicroAPI::Add(dstReg, tmpReg, indexReg1, pregFull);
159- if constexpr (sizeof(T) == sizeof(uint8_t)) {
160- MicroAPI::DataCopyGather(srcReg1, srcUb, dstReg, pregFull);
161- MicroAPI::Add(dstReg, tmpReg, indexReg2, pregFull);
162- MicroAPI::DataCopyGather(srcReg2, srcUb, dstReg, pregFull);
163- MicroAPI::DeInterleave(
164- srcReg, dummyReg, (MicroAPI::RegTensor<T> &)srcReg1, (MicroAPI::RegTensor<T> &)srcReg2);
165- } else {
166- MicroAPI::DataCopyGather(srcReg, srcUb, dstReg, pregFull);
167- }
168- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg0, main);
169- }
170- MicroAPI::Adds(dstReg, indexReg1, (IndexT)offset, pregFull);
171- if constexpr (sizeof(T) == sizeof(uint8_t)) {
172- MicroAPI::DataCopyGather(srcReg1, srcUb, dstReg, pregFull);
173- MicroAPI::Adds(dstReg, indexReg2, (IndexT)offset, pregFull);
174- MicroAPI::DataCopyGather(srcReg2, srcUb, dstReg, pregFull);
175- MicroAPI::DeInterleave(
176- srcReg, dummyReg, (MicroAPI::RegTensor<T> &)srcReg1, (MicroAPI::RegTensor<T> &)srcReg2);
177- } else {
178- MicroAPI::DataCopyGather(srcReg, srcUb, dstReg, pregFull);
179- }
180- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg0, tail);
181- MicroAPI::DataCopyUnAlignPost(dstUb, ureg0, 0);
182-}
183- 
184-template <typename T, typename IndexT>
185-__simd_vf__ inline void BrcNlastGatherOne(
186- __ubuf__ T *dstUb, __ubuf__ T *srcUb, __ubuf__ IndexT *indexUb, uint16_t size0, uint16_t size1)
187-{
188- constexpr uint32_t VF_LEN_HALF = GetVecLen() / 2 / sizeof(T);
189- uint32_t main = size0 * size1;
190- 
191- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<IndexT>();
192- MicroAPI::MaskReg pregCnt;
193- MicroAPI::RegTensor<IndexT> indexReg1;
194- MicroAPI::RegTensor<IndexT> indexReg2;
195- MicroAPI::RegTensor<IndexT> srcReg1;
196- MicroAPI::RegTensor<IndexT> srcReg2;
197- MicroAPI::RegTensor<T> srcReg;
198- MicroAPI::RegTensor<T> dummyReg;
199- MicroAPI::UnalignReg ureg0;
200- 
201- uint32_t sreg = main;
202- pregCnt = MicroAPI::UpdateMask<T>(sreg);
203- MicroAPI::DataCopy(indexReg1, indexUb);
204- if constexpr (sizeof(T) == sizeof(uint8_t)) {
205- MicroAPI::DataCopy(indexReg2, indexUb + VF_LEN_HALF);
206- MicroAPI::DataCopyGather(srcReg1, srcUb, indexReg1, pregFull);
207- MicroAPI::DataCopyGather(srcReg2, srcUb, indexReg2, pregFull);
208- MicroAPI::DeInterleave(
209- srcReg, dummyReg, (MicroAPI::RegTensor<T> &)srcReg1, (MicroAPI::RegTensor<T> &)srcReg2);
210- } else {
211- MicroAPI::DataCopyGather(srcReg, srcUb, indexReg1, pregCnt);
212- }
213- MicroAPI::DataCopy(dstUb, srcReg, pregCnt);
214-}
215- 
216-template <typename T, typename IndexT>
217-__simd_vf__ inline void BrcNlastGatherTwo(
218- __ubuf__ T *dstUb, __ubuf__ T *srcUb, __ubuf__ IndexT *indexUb, uint16_t size0, uint16_t size1)
219-{
220- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
221- constexpr uint32_t VF_LEN_HALF = GetVecLen() / 2 / sizeof(T);
222- uint16_t factor = VF_LEN / size1;
223- uint16_t repeatTimes = CeilDivision(size0, factor) - 1;
224- uint32_t main = factor * size1;
225- uint32_t mainBlock = main * repeatTimes;
226- uint32_t tail = size0 * size1 - mainBlock;
227- 
228- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<IndexT>();
229- MicroAPI::RegTensor<IndexT> indexReg1;
230- MicroAPI::RegTensor<IndexT> indexReg2;
231- MicroAPI::RegTensor<IndexT> srcReg1;
232- MicroAPI::RegTensor<IndexT> srcReg2;
233- MicroAPI::RegTensor<T> srcReg;
234- MicroAPI::RegTensor<T> dummyReg;
235- MicroAPI::UnalignReg ureg0;
236- 
237- MicroAPI::DataCopy(indexReg1, indexUb);
238- if constexpr (sizeof(T) == sizeof(uint8_t)) {
239- MicroAPI::DataCopy(indexReg2, indexUb + VF_LEN_HALF);
240- MicroAPI::DataCopyGather(srcReg1, srcUb, indexReg1, pregFull);
241- MicroAPI::DataCopyGather(srcReg2, srcUb, indexReg2, pregFull);
242- MicroAPI::DeInterleave(
243- srcReg, dummyReg, (MicroAPI::RegTensor<T> &)srcReg1, (MicroAPI::RegTensor<T> &)srcReg2);
244- } else {
245- MicroAPI::DataCopyGather(srcReg, srcUb, indexReg1, pregFull);
246- }
247- for (uint16_t i = 0; i < repeatTimes; ++i) {
248- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg0, main);
249- }
250- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg0, tail);
251- MicroAPI::DataCopyUnAlignPost(dstUb, ureg0, 0);
252-}
253- 
254-template <typename T>
255-__simd_vf__ inline void BrcLastE2B(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0, uint16_t size1)
256-{
257- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
258- uint16_t factor = VF_LEN / size1;
259- uint16_t repeatTimes = CeilDivision(size0, factor);
260- 
261- MicroAPI::MaskReg pregCnt;
262- MicroAPI::RegTensor<T> srcReg;
263- 
264- uint32_t sreg = size0 * size1;
265- for (uint16_t i = 0; i < repeatTimes; ++i) {
266- pregCnt = MicroAPI::UpdateMask<T>(sreg);
267- E2bLoad<T>(srcReg, srcUb + i * DEFAULT_BLK_NUM);
268- MicroAPI::DataCopy(dstUb + i * VF_LEN, srcReg, pregCnt);
269- }
270-}
271- 
272-template <typename T>
273-__simd_vf__ inline void BrcLastE2BLargerThanVL(
274- __ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0, uint16_t size1, uint16_t size2, uint16_t srcStride0)
275-{
276- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
277- uint16_t factor = VF_LEN / size2;
278- uint16_t repeatTimes = CeilDivision(size1, factor);
279- uint32_t preg = size1 * size2;
280- uint32_t sreg;
281- MicroAPI::MaskReg pregCnt;
282- MicroAPI::RegTensor<T> srcReg;
283- 
284- for (uint16_t i = 0; i < size0; ++i) {
285- sreg = preg;
286- for (uint16_t j = 0; j < repeatTimes; ++j) {
287- pregCnt = MicroAPI::UpdateMask<T>(sreg);
288- E2bLoad<T>(srcReg, srcUb + j * DEFAULT_BLK_NUM + i * srcStride0);
289- MicroAPI::DataCopy(dstUb + i * size1 * size2 + j * VF_LEN, srcReg, pregCnt);
290- }
291- }
292-}
293- 
294-template <typename T>
295-__simd_vf__ inline void BrcLastE2BLessThanVL(
296- __ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0, uint16_t size1, uint16_t size2, uint16_t srcStride0)
297-{
298- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
299- uint32_t preg = size1 * size2;
300- uint32_t sreg;
301- MicroAPI::MaskReg pregCnt;
302- MicroAPI::RegTensor<T> srcReg;
303- sreg = preg;
304- pregCnt = MicroAPI::UpdateMask<T>(sreg);
305- for (uint16_t i = 0; i < size0; ++i) {
306- E2bLoad<T>(srcReg, srcUb + i * srcStride0);
307- MicroAPI::DataCopy(dstUb + i * size1 * size2, srcReg, pregCnt);
308- }
309-}
310- 
311-template <typename T>
312-__simd_vf__ inline void BrcLastE2B(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0, uint16_t size1,
313- uint16_t size2, uint16_t size3, uint16_t srcStride0, uint16_t srcStride1)
314-{
315- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
316- uint16_t factor = VF_LEN / size3;
317- uint16_t repeatTimes = CeilDivision(size2, factor);
318- uint32_t preg = size2 * size3;
319- uint32_t sreg;
320- MicroAPI::MaskReg pregCnt;
321- MicroAPI::RegTensor<T> srcReg;
322- for (uint16_t i = 0; i < size0; ++i) {
323- for (uint16_t j = 0; j < size1; ++j) {
324- sreg = preg;
325- for (uint16_t k = 0; k < repeatTimes; ++k) {
326- pregCnt = MicroAPI::UpdateMask<T>(sreg);
327- E2bLoad<T>(srcReg, srcUb + i * srcStride0 + j * srcStride1 + k * DEFAULT_BLK_NUM);
328- MicroAPI::DataCopy(
329- dstUb + i * size1 * size2 * size3 + j * size2 * size3 + k * VF_LEN, srcReg, pregCnt);
330- }
331- }
332- }
333-}
334- 
335-template <typename T>
336-__simd_vf__ inline void BrcNlastGatherBOne(
337- __ubuf__ T *dstUb, __ubuf__ T *srcUb, __ubuf__ uint32_t *indexUb, uint16_t size0, uint16_t size1)
338-{
339- constexpr uint32_t oneBlockElementNum = GetDataBlockSizeInBytes() / sizeof(T);
340- uint32_t main = size0 * size1;
341- 
342- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<T>();
343- MicroAPI::MaskReg pregCnt;
344- MicroAPI::RegTensor<T> srcReg;
345- MicroAPI::RegTensor<uint32_t> indexReg;
346- 
347- MicroAPI::DataCopy(indexReg, indexUb);
348- MicroAPI::DataCopyGatherB(srcReg, srcUb, indexReg, pregFull);
349- uint32_t sreg = main;
350- pregCnt = MicroAPI::UpdateMask<T>(sreg);
351- MicroAPI::DataCopy(dstUb, srcReg, pregCnt);
352-}
353- 
354-template <typename T>
355-__simd_vf__ inline void BrcNlastGatherBTwo(
356- __ubuf__ T *dstUb, __ubuf__ T *srcUb, __ubuf__ uint32_t *indexUb, uint16_t size0, uint16_t size1)
357-{
358- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
359- constexpr uint32_t oneBlockElementNum = GetDataBlockSizeInBytes() / sizeof(T);
360- uint16_t factor = VF_LEN / size1;
361- uint16_t repeatTimes = CeilDivision(size0, factor) - 1;
362- uint32_t main = factor * size1;
363- uint32_t mainBlock = main * repeatTimes;
364- uint32_t tail = size0 * size1 - mainBlock;
365- 
366- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<T>();
367- MicroAPI::MaskReg pregCnt;
368- MicroAPI::RegTensor<T> srcReg;
369- MicroAPI::RegTensor<uint32_t> indexReg;
370- 
371- MicroAPI::DataCopy(indexReg, indexUb);
372- MicroAPI::DataCopyGatherB(srcReg, srcUb, indexReg, pregFull);
373- uint32_t sreg = main;
374- pregCnt = MicroAPI::UpdateMask<T>(sreg);
375- for (uint16_t i = 0; i < repeatTimes; ++i) {
376- MicroAPI::DataCopy(dstUb + i * main, srcReg, pregCnt);
377- }
378- sreg = tail;
379- pregCnt = MicroAPI::UpdateMask<T>(sreg);
380- MicroAPI::DataCopy(dstUb + mainBlock, srcReg, pregCnt);
381-}
382- 
383-template <typename T>
384-__simd_vf__ inline void BrcLastLessThanVLAligned(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0,
385- uint16_t size1, uint16_t size2, uint16_t srcStride0, uint16_t srcStride1)
386-{
387- MicroAPI::MaskReg pregCnt;
388- MicroAPI::RegTensor<T> srcReg;
389- 
390- uint32_t sreg = size2;
391- pregCnt = MicroAPI::UpdateMask<T>(sreg);
392- for (uint16_t i = 0; i < size1; ++i) {
393- for (uint16_t j = 0; j < size0; ++j) {
394- BrcLoad<T>(srcReg, srcUb + j * srcStride0 + i * srcStride1);
395- MicroAPI::DataCopy(dstUb + j * size1 * size2 + i * size2, srcReg, pregCnt);
396- }
397- }
398-}
399- 
400-template <typename T>
401-__simd_vf__ inline void BrcLastLessThanVLAligned(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0,
402- uint16_t size1, uint16_t size2, uint16_t size3, uint16_t srcStride0, uint16_t srcStride1, uint16_t srcStride2)
403-{
404- MicroAPI::MaskReg pregCnt;
405- MicroAPI::RegTensor<T> srcReg;
406- 
407- uint32_t sreg = size3;
408- pregCnt = MicroAPI::UpdateMask<T>(sreg);
409- for (uint16_t i = 0; i < size0; ++i) {
410- for (uint16_t j = 0; j < size2; ++j) {
411- for (uint16_t k = 0; k < size1; ++k) {
412- BrcLoad<T>(srcReg, srcUb + i * srcStride0 + j * srcStride2 + k * srcStride1);
413- MicroAPI::DataCopy(
414- dstUb + i * size1 * size2 * size3 + k * size2 * size3 + j * size3, srcReg, pregCnt);
415- }
416- }
417- }
418-}
419- 
420-template <typename T>
421-__simd_vf__ inline void BrcNlastLessThanVLAligned(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0,
422- uint16_t size1, uint16_t size2, uint16_t size3, uint16_t srcStride0, uint16_t srcStride1, uint16_t srcStride2)
423-{
424- MicroAPI::MaskReg pregCnt;
425- MicroAPI::RegTensor<T> srcReg;
426- 
427- uint32_t sreg = size3;
428- pregCnt = MicroAPI::UpdateMask<T>(sreg);
429- for (uint16_t i = 0; i < size1; ++i) {
430- for (uint16_t j = 0; j < size0; ++j) {
431- for (uint16_t k = 0; k < size2; ++k) {
432- MicroAPI::DataCopy(srcReg, srcUb + i * srcStride1 + j * srcStride0 + k * srcStride2);
433- MicroAPI::DataCopy(
434- dstUb + j * size1 * size2 * size3 + i * size2 * size3 + k * size3, srcReg, pregCnt);
435- }
436- }
437- }
438-}
439- 
440-template <typename T>
441-__simd_vf__ inline void BrcLastLessThanVLUnaligned(
442- __ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0, uint16_t size1)
443-{
444- MicroAPI::MaskReg pregCnt;
445- MicroAPI::RegTensor<T> srcReg;
446- MicroAPI::UnalignReg ureg0;
447- 
448- uint32_t sreg = size1;
449- pregCnt = MicroAPI::UpdateMask<T>(sreg);
450- for (uint16_t i = 0; i < size0; ++i) {
451- BrcLoad<T>(srcReg, srcUb + i);
452- MicroAPI::Duplicate(srcReg, srcReg, pregCnt);
453- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg0, size1);
454- }
455- MicroAPI::DataCopyUnAlignPost(dstUb, ureg0, 0);
456-}
457- 
458-template <typename T>
459-__simd_vf__ inline void BrcLastLessThanVLUnaligned(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0,
460- uint16_t size1, uint16_t size2, uint16_t srcStride0, uint16_t srcStride1)
461-{
462- MicroAPI::MaskReg pregCnt;
463- MicroAPI::RegTensor<T> srcReg;
464- MicroAPI::UnalignReg ureg0;
465- 
466- uint32_t sreg = size2;
467- pregCnt = MicroAPI::UpdateMask<T>(sreg);
468- for (uint16_t i = 0; i < size0; ++i) {
469- for (uint16_t j = 0; j < size1; ++j) {
470- BrcLoad<T>(srcReg, srcUb + j * srcStride1 + i * srcStride0);
471- MicroAPI::Duplicate(srcReg, srcReg, pregCnt);
472- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg0, size2);
473- }
474- }
475- MicroAPI::DataCopyUnAlignPost(dstUb, ureg0, 0);
476-}
477- 
478-template <typename T>
479-__simd_vf__ inline void BrcLastLessThanVLUnaligned(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0,
480- uint16_t size1, uint16_t size2, uint16_t size3, uint16_t srcStride0, uint16_t srcStride1, uint16_t srcStride2)
481-{
482- MicroAPI::MaskReg pregCnt;
483- MicroAPI::RegTensor<T> srcReg;
484- MicroAPI::UnalignReg ureg0;
485- 
486- uint32_t sreg = size3;
487- pregCnt = MicroAPI::UpdateMask<T>(sreg);
488- for (uint16_t i = 0; i < size0; ++i) {
489- for (uint16_t j = 0; j < size1; ++j) {
490- for (uint16_t k = 0; k < size2; ++k) {
491- BrcLoad<T>(srcReg, srcUb + i * srcStride0 + j * srcStride1 + k * srcStride2);
492- MicroAPI::Duplicate(srcReg, srcReg, pregCnt);
493- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg0, size3);
494- }
495- }
496- }
497- MicroAPI::DataCopyUnAlignPost(dstUb, ureg0, 0);
498-}
499- 
500-template <typename T>
501-__simd_vf__ inline void BrcNlastLessThanVLUnaligned(
502- __ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0, uint16_t size1)
503-{
504- MicroAPI::MaskReg pregCnt;
505- MicroAPI::RegTensor<T> srcReg;
506- MicroAPI::UnalignReg ureg0;
507- 
508- uint32_t sreg = size1;
509- pregCnt = MicroAPI::UpdateMask<T>(sreg);
510- MicroAPI::DataCopy(srcReg, srcUb);
511- for (uint16_t i = 0; i < size0; ++i) {
512- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg0, size1);
513- }
514- MicroAPI::DataCopyUnAlignPost(dstUb, ureg0, 0);
515-}
516- 
517-template <typename T>
518-__simd_vf__ inline void BrcNlastLessThanVLUnaligned(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0,
519- uint16_t size1, uint16_t size2, uint16_t srcStride0, uint16_t srcStride1)
520-{
521- MicroAPI::MaskReg pregCnt;
522- MicroAPI::RegTensor<T> srcReg;
523- MicroAPI::UnalignReg ureg0, ureg1;
524- 
525- uint32_t sreg = size2;
526- pregCnt = MicroAPI::UpdateMask<T>(sreg);
527- for (uint16_t i = 0; i < size0; ++i) {
528- for (uint16_t j = 0; j < size1; ++j) {
529- auto srcUbT = srcUb + i * srcStride0 + j * srcStride1;
530- MicroAPI::DataCopyUnAlignPre(ureg0, srcUbT);
531- MicroAPI::DataCopyUnAlign(srcReg, ureg0, srcUbT, size2);
532- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg1, size2);
533- }
534- }
535- MicroAPI::DataCopyUnAlignPost(dstUb, ureg1, 0);
536-}
537- 
538-template <typename T>
539-__simd_vf__ inline void BrcNlastLessThanVLUnaligned(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0,
540- uint16_t size1, uint16_t size2, uint16_t size3, uint16_t srcStride0, uint16_t srcStride1, uint16_t srcStride2)
541-{
542- MicroAPI::MaskReg pregCnt;
543- MicroAPI::RegTensor<T> srcReg;
544- MicroAPI::UnalignReg ureg0, ureg1;
545- 
546- uint32_t sreg = size3;
547- pregCnt = MicroAPI::UpdateMask<T>(sreg);
548- for (uint16_t i = 0; i < size0; ++i) {
549- for (uint16_t j = 0; j < size1; ++j) {
550- for (uint16_t k = 0; k < size2; ++k) {
551- __ubuf__ T *srcUbTmp = srcUb + i * srcStride0 + j * srcStride1 + k * srcStride2;
552- MicroAPI::DataCopyUnAlignPre(ureg0, srcUbTmp);
553- MicroAPI::DataCopyUnAlign(srcReg, ureg0, srcUbTmp, size3);
554- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg1, size3);
555- }
556- }
557- }
558- MicroAPI::DataCopyUnAlignPost(dstUb, ureg1, 0);
559-}
560- 
561-template <typename T>
562-__simd_vf__ inline void BrcLastLargerThanVLAligned(
563- __ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0, uint16_t size1)
564-{
565- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
566- uint16_t factor = CeilDivision(size1, VF_LEN);
567- 
568- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<T>();
569- MicroAPI::MaskReg pregCnt;
570- MicroAPI::RegTensor<T> srcReg;
571- 
572- for (uint16_t i = 0; i < size0; ++i) {
573- BrcLoad<T>(srcReg, srcUb + i);
574- uint32_t sreg = size1;
575- for (uint16_t j = 0; j < factor; ++j) {
576- pregCnt = MicroAPI::UpdateMask<T>(sreg);
577- MicroAPI::DataCopy(dstUb + i * size1 + j * VF_LEN, srcReg, pregCnt);
578- }
579- }
580-}
581- 
582-template <typename T>
583-__simd_vf__ inline void BrcLastLargerThanVLAligned(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0,
584- uint16_t size1, uint16_t size2, uint16_t srcStride0, uint16_t srcStride1)
585-{
586- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
587- uint16_t factor = CeilDivision(size2, VF_LEN);
588- 
589- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<T>();
590- MicroAPI::MaskReg pregCnt;
591- MicroAPI::RegTensor<T> srcReg;
592- 
593- for (uint16_t i = 0; i < size1; ++i) {
594- for (uint16_t j = 0; j < size0; ++j) {
595- BrcLoad<T>(srcReg, srcUb + i * srcStride1 + j * srcStride0);
596- uint32_t sreg = size2;
597- for (uint16_t k = 0; k < factor; ++k) {
598- pregCnt = MicroAPI::UpdateMask<T>(sreg);
599- MicroAPI::DataCopy(dstUb + j * size1 * size2 + i * size2 + k * VF_LEN, srcReg, pregCnt);
600- }
601- }
602- }
603-}
604- 
605-template <typename T>
606-__simd_vf__ inline void BrcLastLargerThanVLAligned(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0,
607- uint16_t size1, uint16_t size2, uint16_t size3, uint16_t srcStride0, uint16_t srcStride1, uint16_t srcStride2)
608-{
609- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
610- uint16_t factor = CeilDivision(size3, VF_LEN);
611- 
612- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<T>();
613- MicroAPI::MaskReg pregCnt;
614- MicroAPI::RegTensor<T> srcReg;
615- for (uint16_t i = 0; i < size0; ++i) {
616- for (uint16_t j = 0; j < size2; ++j) {
617- uint32_t sreg = size3;
618- for (uint16_t k = 0; k < factor; ++k) {
619- pregCnt = MicroAPI::UpdateMask<T>(sreg);
620- for (uint16_t t = 0; t < size1; ++t) {
621- BrcLoad<T>(srcReg, srcUb + i * srcStride0 + j * srcStride2 + t * srcStride1);
622- MicroAPI::DataCopy(
623- dstUb + i * size1 * size2 * size3 + t * size2 * size3 + j * size3 + k * VF_LEN,
624- srcReg,
625- pregCnt);
626- }
627- }
628- }
629- }
630-}
631- 
632-template <typename T>
633-__simd_vf__ inline void BrcNlastLargerThanVLAlignedWithBlock(
634- __ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0, uint16_t size1)
635-{
636- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
637- uint16_t factor = CeilDivision(size1, VF_LEN);
638- 
639- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<T>();
640- MicroAPI::MaskReg pregCnt;
641- MicroAPI::RegTensor<T> srcReg;
642- 
643- for (uint16_t i = 0; i < size0; ++i) {
644- uint32_t sreg = size1;
645- for (uint16_t j = 0; j < factor; ++j) {
646- pregCnt = MicroAPI::UpdateMask<T>(sreg);
647- MicroAPI::DataCopy(srcReg, srcUb + j * VF_LEN);
648- MicroAPI::DataCopy(dstUb + i * size1 + j * VF_LEN, srcReg, pregCnt);
649- }
650- }
651-}
652- 
653-template <typename T>
654-__simd_vf__ inline void BrcNlastLargerThanVLAlignedWithBlock(__ubuf__ T *dstUb, __ubuf__ T *srcUb,
655- uint16_t size0, uint16_t size1, uint16_t size2, uint16_t srcStride0, uint16_t srcStride1)
656-{
657- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
658- uint16_t factor = CeilDivision(size2, VF_LEN);
659- uint16_t jStride = srcStride1 == 0 ? 0 : VF_LEN;
660- 
661- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<T>();
662- MicroAPI::MaskReg pregCnt;
663- MicroAPI::RegTensor<T> srcReg;
664- 
665- for (uint16_t i = 0; i < size0; ++i) {
666- uint32_t sreg = size2;
667- for (uint16_t j = 0; j < factor; ++j) {
668- pregCnt = MicroAPI::UpdateMask<T>(sreg);
669- for (uint16_t k = 0; k < size1; ++k) {
670- MicroAPI::DataCopy(srcReg, srcUb + k * srcStride1 + i * srcStride0 + j * VF_LEN);
671- MicroAPI::DataCopy(dstUb + i * size1 * size2 + k * size2 + j * VF_LEN, srcReg, pregCnt);
672- }
673- }
674- }
675-}
676- 
677-template <typename T>
678-__simd_vf__ inline void BrcNlastLargerThanVLAlignedWithBlock(__ubuf__ T *dstUb, __ubuf__ T *srcUb,
679- uint16_t size0, uint16_t size1, uint16_t size2, uint16_t size3, uint16_t srcStride0, uint16_t srcStride1,
680- uint16_t srcStride2)
681-{
682- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
683- uint16_t factor = CeilDivision(size3, VF_LEN);
684- 
685- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<T>();
686- MicroAPI::MaskReg pregCnt;
687- MicroAPI::RegTensor<T> srcReg;
688- for (uint16_t i = 0; i < size1; ++i) {
689- uint32_t sreg = size3;
690- for (uint16_t j = 0; j < factor; ++j) {
691- pregCnt = MicroAPI::UpdateMask<T>(sreg);
692- for (uint16_t k = 0; k < size0; ++k) {
693- for (uint16_t t = 0; t < size2; ++t) {
694- MicroAPI::DataCopy(srcReg, srcUb + j * VF_LEN +
695- i * srcStride1 + k * srcStride0 + t * srcStride2);
696- MicroAPI::DataCopy(dstUb + k * size1 * size2 * size3 +
697- i * size2 * size3 + t * size3 + j * VF_LEN, srcReg, pregCnt);
698- }
699- }
700- }
701- }
702-}
703- 
704-template <typename T>
705-__simd_vf__ inline void BrcNlastLargerThanVLAlignedWithVL(
706- __ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0, uint16_t size1)
707-{
708- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
709- uint16_t factor = CeilDivision(size1, VF_LEN);
710- 
711- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<T>();
712- MicroAPI::RegTensor<T> srcReg;
713- 
714- for (uint16_t i = 0; i < factor; ++i) {
715- MicroAPI::DataCopy(srcReg, srcUb + i * VF_LEN);
716- for (uint16_t j = 0; j < size0; ++j) {
717- MicroAPI::DataCopy(dstUb + i * VF_LEN + j * size1, srcReg, pregFull);
718- }
719- }
720-}
721- 
722-template <typename T>
723-__simd_vf__ inline void BrcNlastLargerThanVLAlignedWithVL(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0,
724- uint16_t size1, uint16_t size2, uint16_t srcStride0, uint16_t srcStride1)
725-{
726- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
727- uint16_t factor = CeilDivision(size2, VF_LEN);
728- 
729- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<T>();
730- MicroAPI::RegTensor<T> srcReg;
731- 
732- for (uint16_t i = 0; i < size0; ++i) {
733- for (uint16_t j = 0; j < factor; ++j) {
734- for (uint16_t k = 0; k < size1; ++k) {
735- MicroAPI::DataCopy(srcReg, srcUb + i * srcStride0 + j * VF_LEN + k * srcStride1);
736- MicroAPI::DataCopy(dstUb + j * VF_LEN + k * size2 + i * size1 * size2, srcReg, pregFull);
737- }
738- }
739- }
740-}
741- 
742-template <typename T>
743-__simd_vf__ inline void BrcNlastLargerThanVLAlignedWithVL(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0,
744- uint16_t size1, uint16_t size2, uint16_t size3, uint16_t srcStride0, uint16_t srcStride1, uint16_t srcStride2)
745-{
746- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
747- uint16_t factor = CeilDivision(size3, VF_LEN);
748- 
749- MicroAPI::MaskReg pregFull = MicroAPI::CreateMask<T>();
750- MicroAPI::RegTensor<T> srcReg;
751- 
752- for (uint16_t i = 0; i < size1; ++i) {
753- for (uint16_t j = 0; j < factor; ++j) {
754- for (uint16_t k = 0; k < size0; ++k) {
755- for (uint16_t t = 0; t < size2; ++t) {
756- MicroAPI::DataCopy(srcReg, srcUb + i * srcStride1 +
757- j * VF_LEN + k * srcStride0 + t * srcStride2);
758- MicroAPI::DataCopy(dstUb + j * VF_LEN + t * size3 +
759- i * size2 * size3 + k * size1 * size2 * size3, srcReg, pregFull);
760- }
761- }
762- }
763- }
764-}
765- 
766-template <typename T>
767-__simd_vf__ inline void BrcLastLargerThanVLUnaligned(
768- __ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0, uint16_t size1)
769-{
770- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
771- uint16_t factor = size1 / VF_LEN;
772- uint32_t size1tail = size1 - factor * VF_LEN;
773- 
774- MicroAPI::MaskReg pregCnt;
775- MicroAPI::RegTensor<T> srcReg;
776- MicroAPI::UnalignReg ureg0;
777- 
778- uint32_t sreg = size1tail;
779- pregCnt = MicroAPI::UpdateMask<T>(sreg);
780- for (uint16_t i = 0; i < size0; ++i) {
781- BrcLoad<T>(srcReg, srcUb + i);
782- for (uint16_t j = 0; j < factor; ++j) {
783- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg0, VF_LEN);
784- }
785- MicroAPI::Duplicate(srcReg, srcReg, pregCnt);
786- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg0, size1tail);
787- }
788- MicroAPI::DataCopyUnAlignPost(dstUb, ureg0, 0);
789-}
790- 
791-template <typename T>
792-__simd_vf__ inline void BrcLastLargerThanVLUnaligned(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0,
793- uint16_t size1, uint16_t size2, uint16_t srcStride0, uint16_t srcStride1)
794-{
795- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
796- uint16_t factor = size2 / VF_LEN;
797- uint32_t size2tail = size2 - factor * VF_LEN;
798- 
799- MicroAPI::MaskReg pregCnt;
800- MicroAPI::RegTensor<T> srcReg;
801- MicroAPI::UnalignReg ureg0;
802- 
803- uint32_t sreg = size2tail;
804- pregCnt = MicroAPI::UpdateMask<T>(sreg);
805- for (uint16_t i = 0; i < size0; ++i) {
806- for (uint16_t j = 0; j < size1; ++j) {
807- BrcLoad<T>(srcReg, srcUb + j * srcStride1 + i * srcStride0);
808- for (uint16_t k = 0; k < factor; ++k) {
809- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg0, VF_LEN);
810- }
811- MicroAPI::Duplicate(srcReg, srcReg, pregCnt);
812- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg0, size2tail);
813- }
814- }
815- MicroAPI::DataCopyUnAlignPost(dstUb, ureg0, 0);
816-}
817- 
818-template <typename T>
819-__simd_vf__ inline void BrcLastLargerThanVLUnaligned(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0,
820- uint16_t size1, uint16_t size2, uint16_t size3, uint16_t srcStride0, uint16_t srcStride1, uint16_t srcStride2)
821-{
822- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
823- uint16_t factor = size3 / VF_LEN;
824- uint32_t size3tail = size3 - factor * VF_LEN;
825- 
826- MicroAPI::MaskReg pregCnt;
827- MicroAPI::RegTensor<T> srcReg;
828- MicroAPI::UnalignReg ureg0;
829- 
830- uint32_t sreg = size3tail;
831- pregCnt = MicroAPI::UpdateMask<T>(sreg);
832- for (uint16_t i = 0; i < size0; ++i) {
833- for (uint16_t j = 0; j < size1; ++j) {
834- for (uint16_t k = 0; k < size2; ++k) {
835- BrcLoad<T>(srcReg, srcUb + i * srcStride0 + j * srcStride1 + k * srcStride2);
836- for (uint16_t t = 0; t < factor; ++t) {
837- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg0, VF_LEN);
838- }
839- MicroAPI::Duplicate(srcReg, srcReg, pregCnt);
840- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg0, size3tail);
841- }
842- }
843- }
844- MicroAPI::DataCopyUnAlignPost(dstUb, ureg0, 0);
845-}
846- 
847-template <typename T>
848-__simd_vf__ inline void BrcNlastLargerThanVLUnaligned(
849- __ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0, uint16_t size1)
850-{
851- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
852- uint16_t factor = size1 / VF_LEN;
853- uint32_t size1tail = size1 - factor * VF_LEN;
854- 
855- MicroAPI::RegTensor<T> srcReg;
856- MicroAPI::RegTensor<T> tmpReg;
857- MicroAPI::UnalignReg ureg0;
858- 
859- uint32_t sreg = size1tail;
860- MicroAPI::DataCopy(tmpReg, srcUb + factor * VF_LEN);
861- for (uint16_t i = 0; i < size0; ++i) {
862- for (uint16_t j = 0; j < factor; ++j) {
863- MicroAPI::DataCopy(srcReg, srcUb + j * VF_LEN);
864- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg0, VF_LEN);
865- }
866- MicroAPI::DataCopyUnAlign(dstUb, tmpReg, ureg0, size1tail);
867- }
868- MicroAPI::DataCopyUnAlignPost(dstUb, ureg0, 0);
869-}
870- 
871-template <typename T>
872-__simd_vf__ inline void BrcNlastLargerThanVLUnaligned(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0,
873- uint16_t size1, uint16_t size2, uint16_t srcStride0, uint16_t srcStride1)
874-{
875- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
876- uint16_t factor = size2 / VF_LEN;
877- uint32_t size2tail = size2 - factor * VF_LEN;
878- 
879- MicroAPI::RegTensor<T> srcReg;
880- MicroAPI::UnalignReg ureg0, ureg1;
881- 
882- uint32_t sreg = size2tail;
883- for (uint16_t i = 0; i < size0; ++i) {
884- for (uint16_t j = 0; j < size1; ++j) {
885- __ubuf__ T *tmpSrcUb = srcUb + i * srcStride0 + j * srcStride1;
886- MicroAPI::DataCopyUnAlignPre(ureg0, tmpSrcUb);
887- for (uint16_t k = 0; k < factor; ++k) {
888- MicroAPI::DataCopyUnAlign(srcReg, ureg0, tmpSrcUb, VF_LEN);
889- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg1, VF_LEN);
890- }
891- MicroAPI::DataCopyUnAlign(srcReg, ureg0, tmpSrcUb, sreg);
892- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg1, sreg);
893- }
894- }
895- MicroAPI::DataCopyUnAlignPost(dstUb, ureg1, 0);
896-}
897- 
898-template <typename T>
899-__simd_vf__ inline void BrcNlastLargerThanVLUnaligned(__ubuf__ T *dstUb, __ubuf__ T *srcUb, uint16_t size0,
900- uint16_t size1, uint16_t size2, uint16_t size3, uint16_t srcStride0, uint16_t srcStride1, uint16_t srcStride2)
901-{
902- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
903- uint16_t factor = size3 / VF_LEN;
904- uint32_t size3tail = size3 - factor * VF_LEN;
905- 
906- MicroAPI::RegTensor<T> srcReg;
907- MicroAPI::UnalignReg ureg0, ureg1;
908- 
909- uint32_t sreg = size3tail;
910- for (uint16_t i = 0; i < size0; ++i) {
911- for (uint16_t j = 0; j < size1; ++j) {
912- for (uint16_t k = 0; k < size2; ++k) {
913- __ubuf__ T *tmpSrcUb = srcUb + i * srcStride0 + j * srcStride1 + k * srcStride2;
914- MicroAPI::DataCopyUnAlignPre(ureg0, tmpSrcUb);
915- for (uint16_t t = 0; t < factor; ++t) {
916- MicroAPI::DataCopyUnAlign(srcReg, ureg0, tmpSrcUb, VF_LEN);
917- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg1, VF_LEN);
918- }
919- MicroAPI::DataCopyUnAlign(srcReg, ureg0, tmpSrcUb, sreg);
920- MicroAPI::DataCopyUnAlign(dstUb, srcReg, ureg1, sreg);
921- }
922- }
923- }
924- MicroAPI::DataCopyUnAlignPost(dstUb, ureg1, 0);
925-}
926- 
927-template <typename T, int32_t constRank = -1>
928-__aicore__ inline bool BrcLastWrapperForTwoDim(
929- __ubuf__ T *dstUb, __ubuf__ T *srcUb, const uint32_t *dstShape)
930-{
931- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
932- constexpr uint32_t VF_LEN_HALF = GetVecLen() / 2 / sizeof(T);
933- constexpr uint32_t oneBlockElementNum = GetDataBlockSizeInBytes() / sizeof(T);
934- using GatherIndexType = typename ExtractSignedTypeBySize<sizeof(T)>::T;
935- using BrcIndexType = typename ExtractIndexTypeBySize<sizeof(T)>::T;
936- 
937- uint16_t sizeI[2];
938- sizeI[0] = static_cast<uint16_t>(dstShape[0]);
939- sizeI[1] = static_cast<uint16_t>(dstShape[1]);
940- 
941- if (sizeI[1] == oneBlockElementNum && sizeof(T) != sizeof(uint8_t)) {
942- BrcLastE2B(dstUb, srcUb, sizeI[0], sizeI[1]);
943- } else if (sizeI[1] < VF_LEN_HALF) {
944- LocalTensor<T> indexLocal;
945- PopStackBuffer<T, TPosition::LCM>(indexLocal);
946- __ubuf__ GatherIndexType *indexUb1 = (__ubuf__ GatherIndexType *)indexLocal.GetPhyAddr();
947- __ubuf__ GatherIndexType *indexUb2 = (__ubuf__ GatherIndexType *)indexLocal.GetPhyAddr(VF_LEN);
948- GenLastGatherIndex<GatherIndexType>(indexUb1, sizeI[1], 0);
949- if constexpr (sizeof(T) == sizeof(uint8_t)) {
950- GenLastGatherIndex<GatherIndexType>(indexUb2, sizeI[1], VF_LEN_HALF);
951- }
952- __ubuf__ BrcIndexType *indexUb = (__ubuf__ BrcIndexType *)indexLocal.GetPhyAddr();
953- if (sizeI[0] * sizeI[1] < VF_LEN) {
954- BrcLastGatherOne<T, BrcIndexType>(dstUb, srcUb, indexUb, sizeI[0], sizeI[1]);
955- } else if (sizeI[1] < VF_LEN) {
956- BrcLastGatherTwo<T, BrcIndexType>(dstUb, srcUb, indexUb, sizeI[0], sizeI[1]);
957- }
958- } else if (sizeI[1] <= VF_LEN) {
959- BrcLastLessThanVLUnaligned<T>(dstUb, srcUb, sizeI[0], sizeI[1]);
960- } else {
961- if (sizeI[1] % oneBlockElementNum == 0) {
962- BrcLastLargerThanVLAligned<T>(dstUb, srcUb, sizeI[0], sizeI[1]);
963- } else {
964- if constexpr (constRank == -1) {
965- return true;
966- } else {
967- BrcLastLargerThanVLUnaligned<T>(dstUb, srcUb, sizeI[0], sizeI[1]);
968- }
969- }
970- }
971- return false;
972-}
973- 
974-template <typename T, int32_t constRank = -1>
975-__aicore__ inline bool BrcLastWrapperForThreeDim(__ubuf__ T *dstUb, __ubuf__ T *srcUb,
976- const uint32_t *dstShape, const uint32_t *srcStride)
977-{
978- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
979- constexpr uint32_t VF_LEN_HALF = GetVecLen() / 2 / sizeof(T);
980- constexpr uint32_t oneBlockElementNum = GetDataBlockSizeInBytes() / sizeof(T);
981- uint16_t sizeI[3];
982- uint16_t stride[3];
983- sizeI[0] = static_cast<uint16_t>(dstShape[0]);
984- sizeI[1] = static_cast<uint16_t>(dstShape[1]);
985- sizeI[2] = static_cast<uint16_t>(dstShape[2]);
986- stride[0] = static_cast<uint16_t>(srcStride[0]);
987- stride[1] = static_cast<uint16_t>(srcStride[1]);
988- stride[2] = static_cast<uint16_t>(srcStride[2]);
989- 
990- if (sizeI[2] == oneBlockElementNum && sizeof(T) != sizeof(uint8_t) && sizeI[1] * sizeI[2] > VF_LEN_HALF &&
991- sizeI[1] % DEFAULT_BLK_NUM == 0 && stride[1] != 0) {
992- if (sizeI[1] * sizeI[2] > VF_LEN) {
993- BrcLastE2BLargerThanVL(dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], stride[0]);
994- } else {
995- BrcLastE2BLessThanVL(dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], stride[0]);
996- }
997- } else if (sizeI[2] < VF_LEN_HALF && sizeof(T) != sizeof(uint8_t)) {
998- uint32_t newDstShape[3] = {dstShape[0], dstShape[1], dstShape[2]};
999- uint32_t newSrcStride[3] = {srcStride[0], srcStride[1], srcStride[2]};
1000- GatherWrapper(dstUb, srcUb, newDstShape, newSrcStride);
1001- } else if (sizeI[2] <= VF_LEN) {
1002- if (sizeI[2] % oneBlockElementNum == 0) {
1003- BrcLastLessThanVLAligned<T>(dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], stride[0], stride[1]);
1004- } else {
1005- if constexpr (constRank == -1) {
1006- return true;
1007- } else {
1008- BrcLastLessThanVLUnaligned<T>(dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], stride[0], stride[1]);
1009- }
1010- }
1011- } else {
1012- if (sizeI[2] % oneBlockElementNum == 0) {
1013- BrcLastLargerThanVLAligned<T>(dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], stride[0], stride[1]);
1014- } else {
1015- if constexpr (constRank == -1) {
1016- return true;
1017- } else {
1018- BrcLastLargerThanVLUnaligned<T>(dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], stride[0], stride[1]);
1019- }
1020- }
1021- }
1022- return false;
1023-}
1024- 
1025-template <typename T, int32_t constRank = -1>
1026-__aicore__ inline bool BrcLastWrapperForFourDim(__ubuf__ T *dstUb, __ubuf__ T *srcUb,
1027- const uint32_t *dstShape, const uint32_t *srcStride)
1028-{
1029- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
1030- constexpr uint32_t VF_LEN_HALF = GetVecLen() / 2 / sizeof(T);
1031- constexpr uint32_t oneBlockElementNum = GetDataBlockSizeInBytes() / sizeof(T);
1032- uint16_t sizeI[4];
1033- uint16_t stride[4];
1034- sizeI[0] = static_cast<uint16_t>(dstShape[0]);
1035- sizeI[1] = static_cast<uint16_t>(dstShape[1]);
1036- sizeI[2] = static_cast<uint16_t>(dstShape[2]);
1037- sizeI[3] = static_cast<uint16_t>(dstShape[3]);
1038- stride[0] = static_cast<uint16_t>(srcStride[0]);
1039- stride[1] = static_cast<uint16_t>(srcStride[1]);
1040- stride[2] = static_cast<uint16_t>(srcStride[2]);
1041- stride[3] = static_cast<uint16_t>(srcStride[3]);
1042- 
1043- if (sizeI[3] == oneBlockElementNum && sizeof(T) != sizeof(uint8_t) &&
1044- stride[2] != 0 && sizeI[2] % DEFAULT_BLK_NUM == 0) {
1045- BrcLastE2B(dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], sizeI[3], stride[0], stride[1]);
1046- } else if (sizeI[3] < VF_LEN_HALF && sizeof(T) != sizeof(uint8_t)) {
1047- uint32_t newDstShape[4] = {dstShape[0], dstShape[1], dstShape[2], dstShape[3]};
1048- uint32_t newSrcStride[4] = {srcStride[0], srcStride[1], srcStride[2], srcStride[3]};
1049- GatherWrapperForFourDim(dstUb, srcUb, newDstShape, newSrcStride);
1050- } else if (sizeI[3] <= VF_LEN) {
1051- if (sizeI[3] % oneBlockElementNum == 0) {
1052- BrcLastLessThanVLAligned<T>(
1053- dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], sizeI[3], stride[0], stride[1], stride[2]);
1054- } else {
1055- if constexpr (constRank == -1) {
1056- return true;
1057- } else {
1058- BrcLastLessThanVLUnaligned<T>(
1059- dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], sizeI[3], stride[0], stride[1], stride[2]);
1060- }
1061- }
1062- } else {
1063- if (sizeI[3] % oneBlockElementNum == 0) {
1064- BrcLastLargerThanVLAligned<T>(
1065- dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], sizeI[3], stride[0], stride[1], stride[2]);
1066- } else {
1067- if constexpr (constRank == -1) {
1068- return true;
1069- } else {
1070- BrcLastLargerThanVLUnaligned<T>(
1071- dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], sizeI[3], stride[0], stride[1], stride[2]);
1072- }
1073- }
1074- }
1075- return false;
1076-}
1077- 
1078-template <typename T, int32_t constRank = -1>
1079-__aicore__ inline bool BrcNlastWrapperForTwoDim(
1080- __ubuf__ T *dstUb, __ubuf__ T *srcUb, const uint32_t *dstShape)
1081-{
1082- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
1083- constexpr uint32_t VF_LEN_HALF = GetVecLen() / 2 / sizeof(T);
1084- constexpr uint32_t oneBlockElementNum = GetDataBlockSizeInBytes() / sizeof(T);
1085- using GatherIndexType = typename ExtractSignedTypeBySize<sizeof(T)>::T;
1086- using BrcIndexType = typename ExtractIndexTypeBySize<sizeof(T)>::T;
1087- uint16_t sizeI[2];
1088- sizeI[0] = static_cast<uint16_t>(dstShape[0]);
1089- sizeI[1] = static_cast<uint16_t>(dstShape[1]);
1090- 
1091- if (sizeI[1] < VF_LEN_HALF) {
1092- LocalTensor<T> indexLocal;
1093- PopStackBuffer<T, TPosition::LCM>(indexLocal);
1094- if (sizeI[1] % oneBlockElementNum == 0) {
1095- event_t eventIdVToS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S));
1096- SetFlag<HardEvent::V_S>(eventIdVToS);
1097- WaitFlag<HardEvent::V_S>(eventIdVToS);
1098- __ubuf__ uint32_t *indexUb = (__ubuf__ uint32_t *)indexLocal.GetPhyAddr();
1099- if (sizeI[1] / oneBlockElementNum == 1) {
1100- indexUb[0] = 0;
1101- indexUb[1] = 0;
1102- indexUb[2] = 0;
1103- indexUb[3] = 0;
1104- indexUb[4] = 0;
1105- indexUb[5] = 0;
1106- indexUb[6] = 0;
1107- indexUb[7] = 0;
1108- } else if (sizeI[1] / oneBlockElementNum == 2) {
1109- indexUb[0] = 0;
1110- indexUb[1] = 32;
1111- indexUb[2] = 0;
1112- indexUb[3] = 32;
1113- indexUb[4] = 0;
1114- indexUb[5] = 32;
1115- indexUb[6] = 0;
1116- indexUb[7] = 32;
1117- } else {
1118- indexUb[0] = 0;
1119- indexUb[1] = 32;
1120- indexUb[2] = 64;
1121- indexUb[3] = 0;
1122- indexUb[4] = 32;
1123- indexUb[5] = 64;
1124- indexUb[6] = 0;
1125- indexUb[7] = 0;
1126- }
1127- event_t eventIdSToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V));
1128- SetFlag<HardEvent::S_V>(eventIdSToV);
1129- WaitFlag<HardEvent::S_V>(eventIdSToV);
1130- if (sizeI[0] * sizeI[1] < VF_LEN) {
1131- BrcNlastGatherBOne<T>(dstUb, srcUb, (__ubuf__ uint32_t *)indexUb, sizeI[0], sizeI[1]);
1132- } else if (sizeI[1] < VF_LEN) {
1133- BrcNlastGatherBTwo<T>(dstUb, srcUb, (__ubuf__ uint32_t *)indexUb, sizeI[0], sizeI[1]);
1134- }
1135- } else {
1136- __ubuf__ GatherIndexType *indexUb1 = (__ubuf__ GatherIndexType *)indexLocal.GetPhyAddr();
1137- __ubuf__ GatherIndexType *indexUb2 = (__ubuf__ GatherIndexType *)indexLocal.GetPhyAddr(VF_LEN);
1138- GenNlastGatherIndex<GatherIndexType>(indexUb1, sizeI[1], 0);
1139- if constexpr (sizeof(T) == sizeof(uint8_t)) {
1140- GenNlastGatherIndex<GatherIndexType>(indexUb2, sizeI[1], VF_LEN_HALF);
1141- }
1142- __ubuf__ BrcIndexType *indexUb = (__ubuf__ BrcIndexType *)indexLocal.GetPhyAddr();
1143- if (sizeI[0] * sizeI[1] < VF_LEN) {
1144- BrcNlastGatherOne<T, BrcIndexType>(dstUb, srcUb, indexUb, sizeI[0], sizeI[1]);
1145- } else if (sizeI[1] < VF_LEN) {
1146- BrcNlastGatherTwo<T, BrcIndexType>(dstUb, srcUb, indexUb, sizeI[0], sizeI[1]);
1147- }
1148- }
1149- } else if (sizeI[1] <= VF_LEN) {
1150- BrcNlastLessThanVLUnaligned<T>(dstUb, srcUb, sizeI[0], sizeI[1]);
1151- } else {
1152- if (sizeI[1] % oneBlockElementNum == 0) {
1153- if (sizeI[1] % VF_LEN == 0 && sizeI[0] > DEFAULT_BLK_NUM) {
1154- BrcNlastLargerThanVLAlignedWithVL<T>(dstUb, srcUb, sizeI[0], sizeI[1]);
1155- } else {
1156- BrcNlastLargerThanVLAlignedWithBlock<T>(dstUb, srcUb, sizeI[0], sizeI[1]);
1157- }
1158- } else {
1159- if constexpr (constRank == -1) {
1160- return true;
1161- } else {
1162- BrcNlastLargerThanVLUnaligned<T>(dstUb, srcUb, sizeI[0], sizeI[1]);
1163- }
1164- }
1165- }
1166- return false;
1167-}
1168- 
1169-template <typename T, int32_t constRank = -1>
1170-__aicore__ inline bool BrcNlastWrapperForThreeDim(__ubuf__ T *dstUb, __ubuf__ T *srcUb,
1171- const uint32_t *dstShape, const uint32_t *srcStride)
1172-{
1173- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
1174- constexpr uint32_t VF_LEN_HALF = GetVecLen() / 2 / sizeof(T);
1175- constexpr uint32_t oneBlockElementNum = GetDataBlockSizeInBytes() / sizeof(T);
1176- uint16_t sizeI[3];
1177- uint16_t stride[3];
1178- sizeI[0] = static_cast<uint16_t>(dstShape[0]);
1179- sizeI[1] = static_cast<uint16_t>(dstShape[1]);
1180- sizeI[2] = static_cast<uint16_t>(dstShape[2]);
1181- stride[0] = static_cast<uint16_t>(srcStride[0]);
1182- stride[1] = static_cast<uint16_t>(srcStride[1]);
1183- stride[2] = static_cast<uint16_t>(srcStride[2]);
1184- 
1185- if (sizeI[2] < VF_LEN_HALF && sizeof(T) != sizeof(uint8_t)) {
1186- uint32_t newDstShape[3] = {dstShape[0], dstShape[1], dstShape[2]};
1187- uint32_t newSrcStride[3] = {srcStride[0], srcStride[1], srcStride[2]};
1188- GatherWrapper(dstUb, srcUb, newDstShape, newSrcStride);
1189- } else if (sizeI[2] <= VF_LEN) {
1190- BrcNlastLessThanVLUnaligned<T>(dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], stride[0], stride[1]);
1191- } else {
1192- if (sizeI[2] % oneBlockElementNum == 0) {
1193- if (sizeI[2] % VF_LEN == 0) {
1194- BrcNlastLargerThanVLAlignedWithVL<T>(dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], stride[0], stride[1]);
1195- } else {
1196- BrcNlastLargerThanVLAlignedWithBlock<T>(
1197- dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], stride[0], stride[1]);
1198- }
1199- } else {
1200- if constexpr (constRank == -1) {
1201- return true;
1202- } else {
1203- BrcNlastLargerThanVLUnaligned<T>(dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], stride[0], stride[1]);
1204- }
1205- }
1206- }
1207- return false;
1208-}
1209- 
1210-template <typename T, int32_t constRank = -1>
1211-__aicore__ inline bool BrcNlastWrapperForFourDim(__ubuf__ T *dstUb, __ubuf__ T *srcUb,
1212- const uint32_t *dstShape, const uint32_t *srcStride)
1213-{
1214- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
1215- constexpr uint32_t VF_LEN_HALF = GetVecLen() / 2 / sizeof(T);
1216- constexpr uint32_t oneBlockElementNum = GetDataBlockSizeInBytes() / sizeof(T);
1217- uint16_t sizeI[4];
1218- uint16_t stride[4];
1219- sizeI[0] = static_cast<uint16_t>(dstShape[0]);
1220- sizeI[1] = static_cast<uint16_t>(dstShape[1]);
1221- sizeI[2] = static_cast<uint16_t>(dstShape[2]);
1222- sizeI[3] = static_cast<uint16_t>(dstShape[3]);
1223- stride[0] = static_cast<uint16_t>(srcStride[0]);
1224- stride[1] = static_cast<uint16_t>(srcStride[1]);
1225- stride[2] = static_cast<uint16_t>(srcStride[2]);
1226- stride[3] = static_cast<uint16_t>(srcStride[3]);
1227- 
1228- if (sizeI[3] < VF_LEN_HALF && sizeof(T) != sizeof(uint8_t)) {
1229- uint32_t newDstShape[4] = {dstShape[0], dstShape[1], dstShape[2], dstShape[3]};
1230- uint32_t newSrcStride[4] = {srcStride[0], srcStride[1], srcStride[2], srcStride[3]};
1231- GatherWrapperForFourDim(dstUb, srcUb, newDstShape, newSrcStride);
1232- } else if (sizeI[3] <= VF_LEN) {
1233- if (sizeI[3] % oneBlockElementNum == 0) {
1234- BrcNlastLessThanVLAligned<T>(
1235- dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], sizeI[3], stride[0], stride[1], stride[2]);
1236- } else {
1237- if constexpr (constRank == -1) {
1238- return true;
1239- } else {
1240- BrcNlastLessThanVLUnaligned<T>(
1241- dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], sizeI[3], stride[0], stride[1], stride[2]);
1242- }
1243- }
1244- } else {
1245- if (sizeI[3] % oneBlockElementNum == 0) {
1246- if (sizeI[3] % VF_LEN == 0) {
1247- BrcNlastLargerThanVLAlignedWithVL<T>(
1248- dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], sizeI[3], stride[0], stride[1], stride[2]);
1249- } else {
1250- BrcNlastLargerThanVLAlignedWithBlock<T>(
1251- dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], sizeI[3], stride[0], stride[1], stride[2]);
1252- }
1253- } else {
1254- if constexpr (constRank == -1) {
1255- return true;
1256- } else {
1257- BrcNlastLargerThanVLUnaligned<T>(
1258- dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], sizeI[3], stride[0], stride[1], stride[2]);
1259- }
1260- }
1261- }
1262- return false;
1263-}
1264- 
1265-template <typename T>
1266-__aicore__ inline void BrcNlastWrapperForMoreDim(__ubuf__ T *dstUb, __ubuf__ T *srcUb,
1267- const uint32_t *dstShape, const uint32_t *dstStride, const uint32_t *srcStride)
1268-{
1269- uint16_t sizeI[4];
1270- uint16_t stride[4];
1271- sizeI[0] = static_cast<uint16_t>(dstShape[1]);
1272- sizeI[1] = static_cast<uint16_t>(dstShape[2]);
1273- sizeI[2] = static_cast<uint16_t>(dstShape[3]);
1274- sizeI[3] = static_cast<uint16_t>(dstShape[4]);
1275- stride[0] = static_cast<uint16_t>(srcStride[1]);
1276- stride[1] = static_cast<uint16_t>(srcStride[2]);
1277- stride[2] = static_cast<uint16_t>(srcStride[3]);
1278- stride[3] = static_cast<uint16_t>(srcStride[4]);
1279- uint32_t totalDim = 9;
1280- 
1281- __ubuf__ T *srcUbTmp = srcUb;
1282- __ubuf__ T *dstUbTmp = dstUb;
1283- for (uint16_t p = 0; p < static_cast<uint16_t>(dstShape[0]); ++p) {
1284- dstUb = dstUbTmp + p * dstStride[0];
1285- srcUb = srcUbTmp + p * srcStride[0];
1286- uint32_t newDstShape[4] = {
1287- dstShape[1], dstShape[2], dstShape[3], dstShape[4]};
1288- uint32_t newSrcStride[4] = {
1289- srcStride[1], srcStride[2], srcStride[3], srcStride[4]};
1290- GatherWrapperForFourDim(dstUb, srcUb, newDstShape, newSrcStride);
1291- }
1292-}
1293- 
1294-template <typename T>
1295-__aicore__ inline void BrcNlastWrapperForMoreDimDynamicShape(__ubuf__ T *dstUb, __ubuf__ T *srcUb,
1296- const uint32_t dim, const uint32_t *dstShape, const uint32_t *dstStride, const uint32_t *srcStride)
1297-{
1298- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
1299- constexpr uint16_t VF_LEN_HALF = GetVecLen() / 2 / sizeof(T);
1300- constexpr uint32_t oneBlockElementNum = GetDataBlockSizeInBytes() / sizeof(T);
1301- uint16_t sizeI[4] = {1, 1, 1, 1};
1302- if(dim > 4) {
1303- sizeI[0] = dstShape[dim - 4];
1304- sizeI[1] = dstShape[dim - 3];
1305- sizeI[2] = dstShape[dim - 2];
1306- sizeI[3] = dstShape[dim - 1];
1307- } else {
1308- for (uint16_t i = 0; i < dim; ++i) {
1309- sizeI[4 - dim + i] = dstShape[i];
1310- }
1311- }
1312- uint32_t totalDim = 9;
1313- uint16_t loops[5] = {1, 1, 1, 1, 1};
1314- for (int16_t i = dim - 5, j = 4; i >= 0; --i, --j) {
1315- loops[j] = static_cast<uint16_t>(dstShape[i]);
1316- }
1317- uint16_t stride[4] = {0, 0, 0, 0};
1318- if (dim > 4) {
1319- stride[0] = srcStride[dim - 4];
1320- stride[1] = srcStride[dim - 3];
1321- stride[2] = srcStride[dim - 2];
1322- stride[3] = srcStride[dim - 1];
1323- } else {
1324- for (uint16_t i = 0; i < dim; ++i) {
1325- stride[4 - dim + i] = srcStride[i];
1326- }
1327- }
1328- __ubuf__ T *srcUbTmp = srcUb;
1329- __ubuf__ T *dstUbTmp = dstUb;
1330- for (uint16_t i = 0; i < loops[0]; ++i) {
1331- for (uint16_t j = 0; j < loops[1]; ++j) {
1332- for (uint16_t k = 0; k < loops[2]; ++k) {
1333- for (uint16_t t = 0; t < loops[3]; ++t) {
1334- for (uint16_t p = 0; p < loops[4]; ++p) {
1335- dstUb = dstUbTmp + p * dstStride[(dim - 5 + totalDim) % totalDim] +
1336- t * dstStride[(dim - 6 + totalDim) % totalDim] +
1337- k * dstStride[(dim - 7 + totalDim) % totalDim] +
1338- j * dstStride[(dim - 8 + totalDim) % totalDim] +
1339- i * dstStride[(dim - 9 + totalDim) % totalDim];
1340- srcUb = srcUbTmp + p * srcStride[(dim - 5 + totalDim) % totalDim] +
1341- t * srcStride[(dim - 6 + totalDim) % totalDim] +
1342- k * srcStride[(dim - 7 + totalDim) % totalDim] +
1343- j * srcStride[(dim - 8 + totalDim) % totalDim] +
1344- i * srcStride[(dim - 9 + totalDim) % totalDim];
1345- if (sizeI[3] < VF_LEN_HALF && sizeof(T) != sizeof(uint8_t)) {
1346- uint32_t newDstShape[4] = {
1347- dstShape[dim - 4], dstShape[dim - 3], dstShape[dim - 2], dstShape[dim - 1]};
1348- uint32_t newSrcStride[4] = {
1349- srcStride[dim - 4], srcStride[dim - 3], srcStride[dim - 2], srcStride[dim - 1]};
1350- GatherWrapperForFourDim(dstUb, srcUb, newDstShape, newSrcStride);
1351- } else if (sizeI[3] <= VF_LEN) {
1352- if (sizeI[3] % oneBlockElementNum == 0) {
1353- BrcNlastLessThanVLAligned<T>(dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2],
1354- sizeI[3], stride[0], stride[1], stride[2]);
1355- } else {
1356- BrcNlastLessThanVLUnaligned<T>(dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2],
1357- sizeI[3], stride[0], stride[1], stride[2]);
1358- }
1359- } else {
1360- if (sizeI[3] % oneBlockElementNum == 0) {
1361- BrcNlastLargerThanVLAlignedWithBlock<T>(dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2],
1362- sizeI[3], stride[0], stride[1], stride[2]);
1363- } else {
1364- BrcNlastLargerThanVLUnaligned<T>(dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2],
1365- sizeI[3], stride[0], stride[1], stride[2]);
1366- }
1367- }
1368- }
1369- }
1370- }
1371- }
1372- }
1373-}
1374- 
1375-template <typename T>
1376-__aicore__ inline void BrcLastWrapperForMoreDimDynamicShape(__ubuf__ T *dstUb, __ubuf__ T *srcUb,
1377- const uint32_t dim, const uint32_t *dstShape, const uint32_t *dstStride, const uint32_t *srcStride)
1378-{
1379- constexpr uint16_t VF_LEN = GetVecLen() / sizeof(T);
1380- constexpr uint32_t oneBlockElementNum = GetDataBlockSizeInBytes() / sizeof(T);
1381- uint16_t sizeI[4] = {1, 1, 1, 1};
1382- if(dim > 4) {
1383- sizeI[0] = dstShape[dim - 4];
1384- sizeI[1] = dstShape[dim - 3];
1385- sizeI[2] = dstShape[dim - 2];
1386- sizeI[3] = dstShape[dim - 1];
1387- } else {
1388- for (uint16_t i = 0; i < dim; ++i) {
1389- sizeI[4 - dim + i] = dstShape[i];
1390- }
1391- }
1392- uint32_t totalDim = 9;
1393- uint16_t loops[5] = {1, 1, 1, 1, 1};
1394- for (int16_t i = dim - 5, j = 4; i >= 0; --i, --j) {
1395- loops[j] = static_cast<uint16_t>(dstShape[i]);
1396- }
1397- uint16_t stride[4] = {0, 0, 0, 0};
1398- if (dim > 4) {
1399- stride[0] = srcStride[dim - 4];
1400- stride[1] = srcStride[dim - 3];
1401- stride[2] = srcStride[dim - 2];
1402- stride[3] = srcStride[dim - 1];
1403- } else {
1404- for (uint16_t i = 0; i < dim; ++i) {
1405- stride[4 - dim + i] = srcStride[i];
1406- }
1407- }
1408- __ubuf__ T *srcUbTmp = srcUb;
1409- __ubuf__ T *dstUbTmp = dstUb;
1410- for (uint16_t i = 0; i < loops[0]; ++i) {
1411- for (uint16_t j = 0; j < loops[1]; ++j) {
1412- for (uint16_t k = 0; k < loops[2]; ++k) {
1413- for (uint16_t t = 0; t < loops[3]; ++t) {
1414- for (uint16_t p = 0; p < loops[4]; ++p) {
1415- dstUb = dstUbTmp + p * dstStride[(dim - 5 + totalDim) % totalDim] +
1416- t * dstStride[(dim - 6 + totalDim) % totalDim] +
1417- k * dstStride[(dim - 7 + totalDim) % totalDim] +
1418- j * dstStride[(dim - 8 + totalDim) % totalDim] +
1419- i * dstStride[(dim - 9 + totalDim) % totalDim];
1420- srcUb = srcUbTmp + p * srcStride[(dim - 5 + totalDim) % totalDim] +
1421- t * srcStride[(dim - 6 + totalDim) % totalDim] +
1422- k * srcStride[(dim - 7 + totalDim) % totalDim] +
1423- j * srcStride[(dim - 8 + totalDim) % totalDim] +
1424- i * srcStride[(dim - 9 + totalDim) % totalDim];
1425- if (sizeI[3] <= VF_LEN) {
1426- BrcLastLessThanVLUnaligned<T>(
1427- dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], sizeI[3], stride[0], stride[1], stride[2]);
1428- } else {
1429- BrcLastLargerThanVLUnaligned<T>(
1430- dstUb, srcUb, sizeI[0], sizeI[1], sizeI[2], sizeI[3], stride[0], stride[1], stride[2]);
1431- }
1432- }
1433- }
1434- }
1435- }
1436- }
1437-}
1438-/*
1439- * @ingroup GetBroadcastTilingInfo
1440- * @brief get broadcast tiling information
1441- * @param [in] rank, the dimension of src and dst
1442- * @param [in] dstShape, the shape of dst tensor
1443- * @param [in] srcShape, the shape of src tensor
1444- * @param [in] srcInnerPad, if srcShape[rank-1] is aligned
1445- * @param [out] tiling, BroadcastTiling
1446- */
1447-template <typename T, int constRank = -1, uint32_t* constDstShape = nullptr, uint32_t* constSrcShape = nullptr>
1448-__aicore__ inline void GetBroadcastTilingInfo(
1449- uint32_t rank, const uint32_t* dstShape, const uint32_t* srcShape, bool srcInnerPad, BroadcastTiling& tiling)
1450-{
1451- GetBroadcastTilingInfoImpl<T, constRank, constDstShape, constSrcShape>(rank, dstShape, srcShape, srcInnerPad, tiling);
1452-}
1453- 
1454-/*
1455- * @ingroup Broadcast
1456- * @brief https://numpy.org.cn/user/basics/broadcasting.html
1457- * @param [out] dst, output LocalTensor
1458- * @param [in] src, input LocalTensor
1459- * @param [in] dstShape, the shape of dst tensor
1460- * @param [in] srcShape, the shape of src tensor
1461- * @param [in] tiling, broadcasttiling ptr
1462- */
1463-template <typename T, int constRank = -1, uint32_t* constDstShape = nullptr, uint32_t* constSrcShape = nullptr,
1464- bool constSrcInnerPad = false>
1465-__aicore__ inline void Broadcast(const LocalTensor<T>& dst, const LocalTensor<T>& src, const uint32_t* dstShape,
1466- const uint32_t* srcShape, BroadcastTiling* tiling)
1467-{
1468- BroadcastImpl<T, constRank, constDstShape, constSrcShape, constSrcInnerPad>(dst, src, dstShape, srcShape, tiling);
1469-}
1470-} // namespace BroadcastInternal
1471-} // namespace AscendC
1472-#endif
Dimpl/adv_api/detail/quantization/dequant/ascend_dequant_l300_impl.h+0-592
@@ -1,592 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/* !
12- * \file ascend_dequant_l300_impl.h
13- * \brief
14- */
15-#ifndef LIB_ASCEND_DEQUANT_ASCEND_DEQUANT_L300_IMPL_H
16-#define LIB_ASCEND_DEQUANT_ASCEND_DEQUANT_L300_IMPL_H
17-#include "kernel_tensor.h"
18-#include "kernel_tiling/kernel_tiling.h"
19-#include "ascend_dequant_common.h"
20- 
21-namespace AscendC {
22-constexpr uint32_t ASCENDC_DEQUANT_B32_VF_LEN = GetVecLen() / sizeof(uint32_t);
23-template <typename dstT, typename scaleT, DeQuantMode mode>
24-__simd_vf__ inline void DequantPerchannelVFImpl(__ubuf__ half* dstUb, __ubuf__ int32_t* srcUb,
25- __ubuf__ float* scaleUb, DequantParams params)
26-{
27- uint32_t rowNum = params.m;
28- uint32_t N = params.n;
29- uint32_t calCount = params.calCount;
30- uint32_t oneBlockNum = ONE_BLK_SIZE / sizeof(dstT);
31- uint32_t dstInner = CeilDivision(N, oneBlockNum) * oneBlockNum;
32- 
33- MicroAPI::MaskReg preg;
34- MicroAPI::RegTensor<int32_t> s32vreg;
35- MicroAPI::RegTensor<float> f32vreg;
36- MicroAPI::RegTensor<half> b16vreg;
37- MicroAPI::RegTensor<float> scaleB32Vreg0;
38- MicroAPI::RegTensor<float> scaleB32Vreg1;
39- 
40- uint32_t sregLower = ASCENDC_DEQUANT_B32_VF_LEN;
41- uint16_t repeat = static_cast<uint16_t>(CeilDivision(calCount, sregLower));
42- 
43- for (uint16_t i = 0; i < static_cast<uint16_t>(rowNum); ++i) {
44- uint32_t sreg = calCount;
45- for (uint16_t j = 0; j < repeat; ++j) {
46- preg = MicroAPI::UpdateMask<uint32_t>(sreg);
47- MicroAPI::DataCopy<int32_t, MicroAPI::LoadDist::DIST_NORM>(s32vreg, srcUb + i * N + j * sregLower);
48- MicroAPI::Cast<float, int32_t, MrgZRndA>(f32vreg, s32vreg, preg);
49- 
50- MicroAPI::DataCopy<float, MicroAPI::LoadDist::DIST_DINTLV_B32>(scaleB32Vreg0, scaleB32Vreg1,
51- scaleUb + 2 * j * sregLower); // only half of uint64_t is used
52- 
53- MicroAPI::Mul(f32vreg, f32vreg, scaleB32Vreg0, preg);
54- 
55- MicroAPI::Cast<dstT, float, LayoutZMrgZRndRSatS>(b16vreg, f32vreg, preg);
56- MicroAPI::DataCopy<dstT, MicroAPI::StoreDist::DIST_PACK_B32>(dstUb + i * dstInner + j * sregLower, b16vreg,
57- preg);
58- }
59- }
60-}
61-template <typename dstT, typename scaleT, DeQuantMode mode>
62-__aicore__ inline void DequantPerchannelImpl(const LocalTensor<half>& dstTensor, const LocalTensor<int32_t>& srcTensor,
63- const LocalTensor<uint64_t>& deqScale, DequantParams& params)
64-{
65- CheckTensorPosition(dstTensor, "dstTensor", "VECIN, VECOUT, VECCALC");
66- CheckTensorPosition(srcTensor, "srcTensor", "VECIN, VECOUT, VECCALC");
67- CheckTensorPosition(deqScale, "deqScale", "VECIN, VECOUT, VECCALC");
68- __ubuf__ half* dstUb = (__ubuf__ half*)dstTensor.GetPhyAddr();
69- __ubuf__ int32_t* srcUb = (__ubuf__ int32_t*)srcTensor.GetPhyAddr();
70- __ubuf__ float* scaleUb = reinterpret_cast<__ubuf__ float*>(deqScale.GetPhyAddr());
71- 
72- DequantPerchannelVFImpl<dstT, scaleT, mode>(dstUb, srcUb, scaleUb, params);
73-}
74- 
75-template <typename dstT, typename scaleT, DeQuantMode mode>
76-__simd_vf__ inline void DequantPerchannelVFImpl(__ubuf__ dstT* dstUb, __ubuf__ int32_t* srcUb,
77- __ubuf__ scaleT* scaleUb, DequantParams params)
78-{
79- uint32_t rowNum = params.m;
80- uint32_t N = params.n;
81- uint32_t calCount = params.calCount;
82- uint32_t oneBlockNum = ONE_BLK_SIZE / sizeof(dstT);
83- uint32_t dstInner = CeilDivision(N, oneBlockNum) * oneBlockNum;
84- 
85- MicroAPI::MaskReg preg;
86- MicroAPI::RegTensor<int32_t> s32vreg;
87- MicroAPI::RegTensor<float> f32vreg;
88- MicroAPI::RegTensor<dstT> b16vreg;
89- MicroAPI::RegTensor<scaleT> scaleVreg;
90- MicroAPI::RegTensor<float> scaleB32Vreg;
91- 
92- uint32_t sregLower = ASCENDC_DEQUANT_B32_VF_LEN;
93- uint16_t repeat = static_cast<uint16_t>(CeilDivision(calCount, sregLower));
94- 
95- for (uint16_t i = 0; i < static_cast<uint16_t>(rowNum); ++i) {
96- uint32_t sreg = calCount;
97- for (uint16_t j = 0; j < repeat; ++j) {
98- preg = MicroAPI::UpdateMask<uint32_t>(sreg);
99- MicroAPI::DataCopy<int32_t, MicroAPI::LoadDist::DIST_NORM>(s32vreg, srcUb + i * N + j * sregLower);
100- 
101- MicroAPI::Cast<float, int32_t, MrgZRndA>(f32vreg, s32vreg, preg);
102- if constexpr (SupportType<scaleT, bfloat16_t>()) {
103- MicroAPI::DataCopy<bfloat16_t, MicroAPI::LoadDist::DIST_UNPACK_B16>(scaleVreg,
104- scaleUb + j * sregLower);
105- MicroAPI::Cast<float, bfloat16_t, layoutZMrgZ>(scaleB32Vreg, scaleVreg, preg); // bf16->fp32
106- } else {
107- MicroAPI::DataCopy<float, MicroAPI::LoadDist::DIST_NORM>(scaleB32Vreg, scaleUb + j * sregLower);
108- }
109- 
110- MicroAPI::Mul(f32vreg, f32vreg, scaleB32Vreg, preg);
111- 
112- if constexpr (SupportType<dstT, bfloat16_t, half>()) {
113- MicroAPI::Cast<dstT, float, LayoutZMrgZRndRSatS>(b16vreg, f32vreg, preg);
114- MicroAPI::DataCopy<dstT, MicroAPI::StoreDist::DIST_PACK_B32>(dstUb + i * dstInner + j * sregLower, b16vreg,
115- preg);
116- } else { // out is fp32
117- MicroAPI::DataCopy<float, MicroAPI::StoreDist::DIST_NORM_B32>(dstUb + i * dstInner + j * sregLower,
118- f32vreg, preg);
119- }
120- }
121- }
122-}
123- 
124-template <typename dstT, typename scaleT, DeQuantMode mode>
125-__aicore__ inline void DequantPerchannelImpl(const LocalTensor<dstT>& dstTensor, const LocalTensor<int32_t>& srcTensor,
126- const LocalTensor<scaleT>& deqScale, DequantParams& params)
127-{
128- CheckTensorPosition(dstTensor, "dstTensor", "VECIN, VECOUT, VECCALC");
129- CheckTensorPosition(srcTensor, "srcTensor", "VECIN, VECOUT, VECCALC");
130- CheckTensorPosition(deqScale, "deqScale", "VECIN, VECOUT, VECCALC");
131- __ubuf__ dstT* dstUb = (__ubuf__ dstT*)dstTensor.GetPhyAddr();
132- __ubuf__ int32_t* srcUb = (__ubuf__ int32_t*)srcTensor.GetPhyAddr();
133- __ubuf__ scaleT* scaleUb = (__ubuf__ scaleT*)deqScale.GetPhyAddr();
134- 
135- DequantPerchannelVFImpl<dstT, scaleT, mode>(dstUb, srcUb, scaleUb,params);
136-}
137- 
138-template <typename dstT, typename scaleT, DeQuantMode mode>
139-__simd_vf__ inline void DequantPertensorVFImpl(__ubuf__ dstT* dstUb, __ubuf__ int32_t* srcUb,
140- const scaleT deqScale, DequantParams params)
141-{
142- uint32_t rowNum = params.m;
143- uint32_t N = params.n;
144- uint32_t calCount = params.calCount;
145- uint32_t oneBlockNum = ONE_BLK_SIZE / sizeof(dstT);
146- uint32_t dstInner = CeilDivision(N, oneBlockNum) * oneBlockNum;
147- 
148- MicroAPI::MaskReg preg;
149- MicroAPI::RegTensor<int32_t> s32vreg;
150- MicroAPI::RegTensor<float> f32vreg;
151- 
152- uint32_t sregLower = ASCENDC_DEQUANT_B32_VF_LEN;
153- uint16_t repeat = static_cast<uint16_t>(CeilDivision(calCount, sregLower));
154- 
155- for (uint16_t i = 0; i < static_cast<uint16_t>(rowNum); ++i) {
156- uint32_t sreg = calCount;
157- for (uint16_t j = 0; j < repeat; ++j) {
158- preg = MicroAPI::UpdateMask<uint32_t>(sreg);
159- MicroAPI::DataCopy<int32_t, MicroAPI::LoadDist::DIST_NORM>(s32vreg, srcUb + i * N + j * sregLower);
160- MicroAPI::Cast<float, int32_t, MrgZRndA>(f32vreg, s32vreg, preg);
161- MicroAPI::Muls(f32vreg, f32vreg, deqScale, preg);
162- // out is fp32
163- MicroAPI::DataCopy<float, MicroAPI::StoreDist::DIST_NORM_B32>(dstUb + i * dstInner + j * sregLower,
164- f32vreg, preg);
165- }
166- }
167-}
168- 
169-template <typename dstT, typename scaleT, DeQuantMode mode>
170-__aicore__ inline void DequantPertensorImpl(const LocalTensor<dstT>& dstTensor, const LocalTensor<int32_t>& srcTensor,
171- const scaleT deqScale, DequantParams& params)
172-{
173- CheckTensorPosition(dstTensor, "dstTensor", "VECIN, VECOUT, VECCALC");
174- CheckTensorPosition(srcTensor, "srcTensor", "VECIN, VECOUT, VECCALC");
175- __ubuf__ dstT* dstUb = (__ubuf__ dstT*)dstTensor.GetPhyAddr();
176- __ubuf__ int32_t* srcUb = (__ubuf__ int32_t*)srcTensor.GetPhyAddr();
177- DequantPertensorVFImpl<dstT, scaleT, mode>(dstUb, srcUb,deqScale, params);
178-}
179- 
180-template <typename scaleT>
181-__simd_callee__ inline void LoadPerTokenScale(__ubuf__ scaleT* addr, MicroAPI::RegTensor<scaleT>& vreg)
182-{
183- if constexpr (SupportType<scaleT, half, bfloat16_t>()) {
184- MicroAPI::DataCopy<scaleT, MicroAPI::LoadDist::DIST_BRC_B16>(vreg, addr);
185- } else {
186- MicroAPI::DataCopy<scaleT, MicroAPI::LoadDist::DIST_BRC_B32>(vreg, addr);
187- }
188-}
189- 
190-template <typename dstT>
191-__simd_callee__ inline void StoreRes(__ubuf__ dstT* dstAddr, MicroAPI::RegTensor<float>& vreg,
192- MicroAPI::MaskReg& preg)
193-{
194- if constexpr (SupportType<dstT, half, bfloat16_t>()) {
195- MicroAPI::RegTensor<dstT> tempVreg;
196- MicroAPI::Cast<dstT, float, LayoutZMrgZRndRSatS>(tempVreg, vreg, preg);
197- MicroAPI::DataCopy<dstT, MicroAPI::StoreDist::DIST_PACK_B32>(dstAddr, tempVreg, preg);
198- } else {
199- MicroAPI::DataCopy<dstT, MicroAPI::StoreDist::DIST_NORM_B32>(dstAddr, vreg, preg);
200- }
201-}
202- 
203-template <typename dstT, typename srcT, typename scaleT, const AscendDeQuantConfig& config>
204-__simd_vf__ inline void DeQuantPerTokenForS32VF(__ubuf__ dstT* dstUb, __ubuf__ srcT* srcUb,
205- __ubuf__ scaleT* scaleUb, const AscendDeQuantParam para)
206-{
207- uint16_t rowNum = para.calCount / para.n;
208- uint32_t vecLen = ASCENDC_QUANT_B32_VF_LEN;
209- uint16_t repeat = static_cast<uint16_t>(CeilDivision(para.n, vecLen));
210- 
211- MicroAPI::MaskReg preg;
212- MicroAPI::RegTensor<int32_t> srcVreg;
213- MicroAPI::RegTensor<float> f32Vreg;
214- MicroAPI::RegTensor<scaleT> scaleVreg;
215- MicroAPI::RegTensor<float> scaleF32Vreg;
216- for (uint16_t i = 0; i < static_cast<uint16_t>(rowNum); ++i) {
217- LoadPerTokenScale<scaleT>(scaleUb + i, scaleVreg);
218- uint32_t sreg = para.n;
219- for (uint16_t j = 0; j < repeat; ++j) {
220- preg = MicroAPI::UpdateMask<uint32_t>(sreg);
221- MicroAPI::DataCopy<int32_t, MicroAPI::LoadDist::DIST_NORM>(srcVreg, srcUb + i * para.n + j * vecLen);
222- MicroAPI::Cast<float, int32_t, MrgZRndA>(f32Vreg, srcVreg, preg);
223- if constexpr (SupportType<scaleT, half, bfloat16_t>()) {
224- MicroAPI::Cast<float, scaleT, layoutZMrgZ>(scaleF32Vreg, scaleVreg, preg);
225- MicroAPI::Mul<float, MicroAPI::MaskMergeMode::ZEROING>(f32Vreg, f32Vreg, scaleF32Vreg, preg);
226- } else {
227- MicroAPI::Mul<float, MicroAPI::MaskMergeMode::ZEROING>(f32Vreg, f32Vreg, scaleVreg, preg);
228- }
229- StoreRes<dstT>(dstUb + i * para.n + j * vecLen, f32Vreg, preg);
230- }
231- }
232-}
233- 
234-template <typename dstT, typename srcT, typename scaleT, const AscendDeQuantConfig& config>
235-__aicore__ inline void DeQuantPerTokenForS32(const LocalTensor<dstT>& dstTensor, const LocalTensor<srcT>& srcTensor,
236- const LocalTensor<scaleT>& scaleTensor, const LocalTensor<scaleT>& offsetTensor,
237- const AscendDeQuantParam& para)
238-{
239- __ubuf__ dstT* dstUb = (__ubuf__ dstT*)dstTensor.GetPhyAddr();
240- __ubuf__ srcT* srcUb = (__ubuf__ srcT*)srcTensor.GetPhyAddr();
241- __ubuf__ scaleT* scaleUb = (__ubuf__ scaleT*)scaleTensor.GetPhyAddr();
242- DeQuantPerTokenForS32VF<dstT, srcT, scaleT, config>(dstUb, srcUb, scaleUb, para);
243-}
244- 
245-template <typename dstT, typename srcT, typename scaleT, const AscendDeQuantConfig& config>
246-__simd_vf__ inline void DeQuantPerTokenForF32VF(__ubuf__ dstT* dstUb, __ubuf__ srcT* srcUb,
247- __ubuf__ scaleT* scaleUb, const AscendDeQuantParam para)
248-{
249- uint16_t rowNum = para.calCount / para.n;
250- uint32_t vecLen = ASCENDC_QUANT_B32_VF_LEN;
251- uint16_t repeat = static_cast<uint16_t>(CeilDivision(para.n, vecLen));
252- 
253- MicroAPI::MaskReg preg;
254- MicroAPI::RegTensor<float> srcVreg;
255- MicroAPI::RegTensor<float> f32Vreg;
256- MicroAPI::RegTensor<scaleT> scaleVreg;
257- MicroAPI::RegTensor<float> scaleF32Vreg;
258- for (uint16_t i = 0; i < static_cast<uint16_t>(rowNum); ++i) {
259- LoadPerTokenScale<scaleT>(scaleUb + i, scaleVreg);
260- uint32_t sreg = para.n;
261- for (uint16_t j = 0; j < repeat; ++j) {
262- preg = MicroAPI::UpdateMask<uint32_t>(sreg);
263- MicroAPI::DataCopy<float, MicroAPI::LoadDist::DIST_NORM>(srcVreg, srcUb + i * para.n + j * vecLen);
264- if constexpr (SupportType<scaleT, half, bfloat16_t>()) {
265- MicroAPI::Cast<float, scaleT, layoutZMrgZ>(scaleF32Vreg, scaleVreg, preg);
266- MicroAPI::Mul<float, MicroAPI::MaskMergeMode::ZEROING>(f32Vreg, srcVreg, scaleF32Vreg, preg);
267- } else {
268- MicroAPI::Mul<float, MicroAPI::MaskMergeMode::ZEROING>(f32Vreg, srcVreg, scaleVreg, preg);
269- }
270- StoreRes<dstT>(dstUb + i * para.n + j * vecLen, f32Vreg, preg);
271- }
272- }
273-}
274- 
275-template <typename dstT, typename srcT, typename scaleT, const AscendDeQuantConfig& config>
276-__aicore__ inline void DeQuantPerTokenForF32(const LocalTensor<dstT>& dstTensor, const LocalTensor<srcT>& srcTensor,
277- const LocalTensor<scaleT>& scaleTensor, const LocalTensor<scaleT>& offsetTensor,
278- const AscendDeQuantParam& para)
279-{
280- __ubuf__ dstT* dstUb = (__ubuf__ dstT*)dstTensor.GetPhyAddr();
281- __ubuf__ srcT* srcUb = (__ubuf__ srcT*)srcTensor.GetPhyAddr();
282- __ubuf__ scaleT* scaleUb = (__ubuf__ scaleT*)scaleTensor.GetPhyAddr();
283- DeQuantPerTokenForF32VF<dstT, srcT, scaleT, config>(dstUb, srcUb, scaleUb, para);
284-}
285- 
286-template <typename T>
287-__simd_callee__ inline void GetPerGroupScale(__ubuf__ T* scaleUb, const int32_t start, const AscendDeQuantParam& para,
288- const AscendDeQuantConfig& config, MicroAPI::RegTensor<T>& scaleReg)
289-{
290- // use vgather to get perGroup scale/offset
291- uint32_t groupSize = para.groupSize;
292- if constexpr (SupportType<T, half, bfloat16_t>()) {
293- MicroAPI::MaskReg preg = MicroAPI::CreateMask<uint16_t, MicroAPI::MaskPattern::ALL>();
294- MicroAPI::RegTensor<int16_t> vci_vreg;
295- MicroAPI::RegTensor<uint16_t> index_vreg;
296- MicroAPI::RegTensor<uint16_t> gsize_vreg;
297- MicroAPI::Duplicate(gsize_vreg, static_cast<uint16_t>(groupSize));
298- MicroAPI::Arange(vci_vreg, static_cast<int16_t>(start));
299- MicroAPI::Div(index_vreg, (MicroAPI::RegTensor<uint16_t> &)vci_vreg, gsize_vreg, preg);
300- MicroAPI::DataCopyGather(scaleReg, scaleUb, index_vreg, preg);
301- } else {
302- MicroAPI::MaskReg preg = MicroAPI::CreateMask<uint32_t, MicroAPI::MaskPattern::ALL>();
303- MicroAPI::RegTensor<int32_t> vci_vreg;
304- MicroAPI::RegTensor<uint32_t> index_vreg;
305- MicroAPI::RegTensor<uint32_t> gsize_vreg;
306- MicroAPI::Duplicate(gsize_vreg, static_cast<uint32_t>(groupSize));
307- MicroAPI::Arange(vci_vreg, static_cast<int32_t>(start));
308- MicroAPI::Div(index_vreg, (MicroAPI::RegTensor<uint32_t> &)vci_vreg, gsize_vreg, preg);
309- MicroAPI::DataCopyGather(scaleReg, scaleUb, index_vreg, preg);
310- }
311-}
312- 
313-template <typename dstT, typename srcT, typename scaleT, const AscendDeQuantConfig& config>
314-__simd_vf__ inline void DeQuantPerGroupForColS32VF(__ubuf__ dstT* dstUb, __ubuf__ srcT* srcUb,
315- __ubuf__ scaleT* scaleUb, const AscendDeQuantParam para)
316-{
317- uint16_t rowNum = para.calCount / para.n;
318- uint32_t vecLen = ASCENDC_QUANT_B32_VF_LEN;
319- uint16_t repeat = static_cast<uint16_t>(CeilDivision(para.n, vecLen));
320- uint32_t sreg = para.n;
321- uint16_t scaleK = static_cast<uint16_t>(CeilDivision(para.n, para.groupSize));
322- 
323- MicroAPI::MaskReg preg;
324- MicroAPI::RegTensor<int32_t> srcVreg;
325- MicroAPI::RegTensor<float> f32Vreg;
326- MicroAPI::RegTensor<scaleT> oriScaleVreg;
327- MicroAPI::RegTensor<scaleT> tempVreg;
328- MicroAPI::RegTensor<int32_t> offsetVreg;
329- MicroAPI::RegTensor<scaleT> scaleVreg;
330- MicroAPI::RegTensor<float> scaleF32Vreg;
331- MicroAPI::RegTensor<scaleT> zeroVreg;
332- if constexpr (SupportType<scaleT, half, bfloat16_t>()) {
333- MicroAPI::MaskReg b16FullPreg = MicroAPI::CreateMask<uint16_t, MicroAPI::MaskPattern::ALL>();
334- MicroAPI::Duplicate(zeroVreg, static_cast<scaleT>(0), b16FullPreg);
335- } else {
336- MicroAPI::MaskReg b32FullPreg = MicroAPI::CreateMask<uint32_t, MicroAPI::MaskPattern::ALL>();
337- MicroAPI::Duplicate(zeroVreg, static_cast<scaleT>(0), b32FullPreg);
338- }
339- for (uint16_t i = 0; i < static_cast<uint16_t>(rowNum); ++i) {
340- sreg = para.n;
341- for (uint16_t j = 0; j < repeat; ++j) {
342- preg = MicroAPI::UpdateMask<uint32_t>(sreg);
343- if constexpr (SupportType<scaleT, half, bfloat16_t>()) {
344- GetPerGroupScale<scaleT>(scaleUb + i * scaleK, j * vecLen, para, config, oriScaleVreg);
345- MicroAPI::Interleave(scaleVreg, tempVreg, oriScaleVreg, zeroVreg);
346- MicroAPI::Cast<float, scaleT, layoutZMrgZ>(scaleF32Vreg, scaleVreg, preg);
347- } else {
348- GetPerGroupScale<scaleT>(scaleUb + i * scaleK, j * vecLen, para, config, scaleF32Vreg);
349- }
350- MicroAPI::DataCopy<int32_t, MicroAPI::LoadDist::DIST_NORM>(srcVreg, srcUb + i * para.n + j * vecLen);
351- MicroAPI::Cast<float, int32_t, MrgZRndA>(f32Vreg, srcVreg, preg);
352- MicroAPI::Mul<float, MicroAPI::MaskMergeMode::ZEROING>(f32Vreg, f32Vreg, scaleF32Vreg, preg);
353- StoreRes<dstT>(dstUb + i * para.n + j * vecLen, f32Vreg, preg);
354- }
355- }
356-}
357- 
358-template <typename dstT, typename srcT, typename scaleT, const AscendDeQuantConfig& config>
359-__aicore__ inline void DeQuantPerGroupForColS32(const LocalTensor<dstT>& dstTensor, const LocalTensor<srcT>& srcTensor,
360- const LocalTensor<scaleT>& scaleTensor, const LocalTensor<scaleT>& offsetTensor,
361- const AscendDeQuantParam& para)
362-{
363- __ubuf__ dstT* dstUb = (__ubuf__ dstT*)dstTensor.GetPhyAddr();
364- __ubuf__ srcT* srcUb = (__ubuf__ srcT*)srcTensor.GetPhyAddr();
365- __ubuf__ scaleT* scaleUb = (__ubuf__ scaleT*)scaleTensor.GetPhyAddr();
366- DeQuantPerGroupForColS32VF<dstT, srcT, scaleT, config>(dstUb, srcUb, scaleUb, para);
367-}
368- 
369-template <typename dstT, typename srcT, typename scaleT, const AscendDeQuantConfig& config>
370-__simd_vf__ inline void DeQuantPerGroupForColF32VF(__ubuf__ dstT* dstUb, __ubuf__ srcT* srcUb,
371- __ubuf__ scaleT* scaleUb, const AscendDeQuantParam para)
372-{
373- uint16_t rowNum = para.calCount / para.n;
374- uint32_t vecLen = ASCENDC_QUANT_B32_VF_LEN;
375- uint16_t repeat = static_cast<uint16_t>(CeilDivision(para.n, vecLen));
376- uint16_t scaleK = static_cast<uint16_t>(CeilDivision(para.n, para.groupSize));
377- 
378- MicroAPI::MaskReg preg;
379- MicroAPI::RegTensor<float> srcVreg;
380- MicroAPI::RegTensor<float> f32Vreg;
381- MicroAPI::RegTensor<scaleT> oriScaleVreg;
382- MicroAPI::RegTensor<scaleT> tempVreg;
383- MicroAPI::RegTensor<float> offsetVreg;
384- MicroAPI::RegTensor<scaleT> scaleVreg;
385- MicroAPI::RegTensor<float> scaleF32Vreg;
386- MicroAPI::RegTensor<dstT> dstVreg;
387- MicroAPI::RegTensor<scaleT> zeroVreg;
388- if constexpr (SupportType<scaleT, half, bfloat16_t>()) {
389- MicroAPI::MaskReg b16FullPreg = MicroAPI::CreateMask<uint16_t, MicroAPI::MaskPattern::ALL>();
390- MicroAPI::Duplicate(zeroVreg, static_cast<scaleT>(0), b16FullPreg);
391- } else {
392- MicroAPI::MaskReg b32FullPreg = MicroAPI::CreateMask<uint32_t, MicroAPI::MaskPattern::ALL>();
393- MicroAPI::Duplicate(zeroVreg, static_cast<scaleT>(0), b32FullPreg);
394- }
395- for (uint16_t i = 0; i < static_cast<uint16_t>(rowNum); ++i) {
396- uint32_t sreg = para.n;
397- for (uint16_t j = 0; j < repeat; ++j) {
398- preg = MicroAPI::UpdateMask<uint32_t>(sreg);
399- if constexpr (SupportType<scaleT, half, bfloat16_t>()) {
400- GetPerGroupScale<scaleT>(scaleUb + i * scaleK, j * vecLen, para, config, oriScaleVreg);
401- MicroAPI::Interleave(scaleVreg, tempVreg, oriScaleVreg, zeroVreg);
402- MicroAPI::Cast<float, scaleT, layoutZMrgZ>(scaleF32Vreg, scaleVreg, preg);
403- } else {
404- GetPerGroupScale<scaleT>(scaleUb + i * scaleK, j * vecLen, para, config, scaleF32Vreg);
405- }
406- MicroAPI::DataCopy<float, MicroAPI::LoadDist::DIST_NORM>(srcVreg, srcUb + i * para.n + j * vecLen);
407- MicroAPI::Mul<float, MicroAPI::MaskMergeMode::ZEROING>(f32Vreg, srcVreg, scaleF32Vreg, preg);
408- StoreRes<dstT>(dstUb + i * para.n + j * vecLen, f32Vreg, preg);
409- }
410- }
411-}
412- 
413-template <typename dstT, typename srcT, typename scaleT, const AscendDeQuantConfig& config>
414-__aicore__ inline void DeQuantPerGroupForColF32(const LocalTensor<dstT>& dstTensor, const LocalTensor<srcT>& srcTensor,
415- const LocalTensor<scaleT>& scaleTensor, const LocalTensor<scaleT>& offsetTensor,
416- const AscendDeQuantParam& para)
417-{
418- __ubuf__ dstT* dstUb = (__ubuf__ dstT*)dstTensor.GetPhyAddr();
419- __ubuf__ srcT* srcUb = (__ubuf__ srcT*)srcTensor.GetPhyAddr();
420- __ubuf__ scaleT* scaleUb = (__ubuf__ scaleT*)scaleTensor.GetPhyAddr();
421- DeQuantPerGroupForColF32VF<dstT, srcT, scaleT, config>(dstUb, srcUb, scaleUb, para);
422-}
423- 
424-template <typename dstT, typename srcT, typename scaleT, const AscendDeQuantConfig& config>
425-__simd_callee__ inline void DeQuantPerGroupForRowTailBlock(__ubuf__ dstT* dstUb, __ubuf__ srcT* srcUb,
426- __ubuf__ scaleT* scaleUb, uint16_t repeat,
427- uint16_t tailRow, uint32_t n, uint32_t vecLen)
428-{
429- MicroAPI::MaskReg preg;
430- MicroAPI::RegTensor<scaleT> scaleVreg;
431- MicroAPI::RegTensor<float> f32ScaleVreg;
432- MicroAPI::RegTensor<srcT> srcVreg;
433- MicroAPI::RegTensor<float> f32Vreg;
434- MicroAPI::MaskReg b32FullPreg = MicroAPI::CreateMask<uint32_t, MicroAPI::MaskPattern::ALL>();
435- for (uint16_t i = 0; i < tailRow; ++i) {
436- uint32_t sreg = n;
437- for (uint16_t j = 0; j < repeat; ++j) {
438- if constexpr (SupportType<scaleT, half, bfloat16_t>()) {
439- MicroAPI::DataCopy<scaleT, MicroAPI::LoadDist::DIST_UNPACK_B16>(scaleVreg, scaleUb + j * vecLen);
440- MicroAPI::Cast<float, scaleT, layoutZMrgZ>(f32ScaleVreg, scaleVreg, b32FullPreg);
441- } else {
442- MicroAPI::DataCopy<scaleT, MicroAPI::LoadDist::DIST_NORM>(f32ScaleVreg, scaleUb + j * vecLen);
443- }
444- preg = MicroAPI::UpdateMask<uint32_t>(sreg);
445- MicroAPI::DataCopy<srcT, MicroAPI::LoadDist::DIST_NORM>(srcVreg, srcUb + i * n + j * vecLen);
446- if constexpr (SupportType<srcT, int32_t>()) {
447- MicroAPI::Cast<float, int32_t, MrgZRndA>(f32Vreg, srcVreg, preg);
448- MicroAPI::Mul<float, MicroAPI::MaskMergeMode::ZEROING>(f32Vreg, f32Vreg, f32ScaleVreg, preg);
449- } else {
450- MicroAPI::Mul<float, MicroAPI::MaskMergeMode::ZEROING>(f32Vreg, srcVreg, f32ScaleVreg, preg);
451- }
452- StoreRes<dstT>(dstUb + i * n + j * vecLen, f32Vreg, preg);
453- }
454- }
455-}
456- 
457-template <typename dstT, typename srcT, typename scaleT, const AscendDeQuantConfig& config>
458-__simd_vf__ inline void DeQuantPerGroupForRowVF(__ubuf__ dstT* dstUb, __ubuf__ srcT* srcUb,
459- __ubuf__ scaleT* scaleUb, const AscendDeQuantParam para, uint16_t rowNum, uint16_t tailRow)
460-{
461- uint16_t mainRowGroup = rowNum / para.groupSize;
462- uint32_t vecLen = ASCENDC_QUANT_B32_VF_LEN;
463- uint16_t repeat = static_cast<uint16_t>(CeilDivision(para.n, vecLen));
464- 
465- MicroAPI::MaskReg preg;
466- MicroAPI::RegTensor<scaleT> scaleVreg;
467- MicroAPI::RegTensor<float> f32ScaleVreg;
468- MicroAPI::RegTensor<srcT> srcVreg;
469- MicroAPI::RegTensor<float> f32Vreg;
470- MicroAPI::MaskReg b32FullPreg = MicroAPI::CreateMask<uint32_t, MicroAPI::MaskPattern::ALL>();
471- for (uint16_t i = 0; i < mainRowGroup; ++i) {
472- for (uint16_t j = 0; j < static_cast<uint16_t>(para.groupSize); ++j) {
473- uint32_t sreg = para.n;
474- for (uint16_t k = 0; k < repeat; ++k) {
475- if constexpr (SupportType<scaleT, half, bfloat16_t>()) {
476- MicroAPI::DataCopy<scaleT, MicroAPI::LoadDist::DIST_UNPACK_B16>(scaleVreg, scaleUb + i * para.n + k * vecLen);
477- MicroAPI::Cast<float, scaleT, layoutZMrgZ>(f32ScaleVreg, scaleVreg, b32FullPreg);
478- } else {
479- MicroAPI::DataCopy<scaleT, MicroAPI::LoadDist::DIST_NORM>(
480- f32ScaleVreg, scaleUb + i * para.n + k * vecLen);
481- }
482- preg = MicroAPI::UpdateMask<uint32_t>(sreg);
483- MicroAPI::DataCopy<srcT, MicroAPI::LoadDist::DIST_NORM>(
484- srcVreg, srcUb + (i * para.groupSize + j) * para.n + k * vecLen);
485- if constexpr (SupportType<srcT, int32_t>()) {
486- MicroAPI::Cast<float, int32_t, MrgZRndA>(f32Vreg, srcVreg, preg);
487- MicroAPI::Mul<float, MicroAPI::MaskMergeMode::ZEROING>(f32Vreg, f32Vreg, f32ScaleVreg, preg);
488- } else {
489- MicroAPI::Mul<float, MicroAPI::MaskMergeMode::ZEROING>(f32Vreg, srcVreg, f32ScaleVreg, preg);
490- }
491- StoreRes<dstT>(dstUb + (i * para.groupSize + j) * para.n + k * vecLen, f32Vreg, preg);
492- }
493- }
494- }
495- DeQuantPerGroupForRowTailBlock<dstT, srcT, scaleT, config>(
496- dstUb + mainRowGroup * para.groupSize * para.n, srcUb + mainRowGroup * para.groupSize * para.n,
497- scaleUb + mainRowGroup * para.n, repeat, tailRow, para.n, vecLen);
498-}
499- 
500-template <typename dstT, typename srcT, typename scaleT, const AscendDeQuantConfig& config>
501-__aicore__ inline void DeQuantPerGroupForRow(const LocalTensor<dstT>& dstTensor, const LocalTensor<srcT>& srcTensor,
502- const LocalTensor<scaleT>& scaleTensor, const LocalTensor<scaleT>& offsetTensor,
503- const AscendDeQuantParam& para)
504-{
505- __ubuf__ dstT* dstUb = (__ubuf__ dstT*)dstTensor.GetPhyAddr();
506- __ubuf__ srcT* srcUb = (__ubuf__ srcT*)srcTensor.GetPhyAddr();
507- __ubuf__ scaleT* scaleUb = (__ubuf__ scaleT*)scaleTensor.GetPhyAddr();
508- uint16_t rowNum = para.calCount / para.n;
509- uint16_t tailRow = rowNum % para.groupSize;
510- DeQuantPerGroupForRowVF<dstT, srcT, scaleT, config>(dstUb, srcUb, scaleUb, para, rowNum, tailRow);
511-}
512- 
513-template <typename dstT, typename srcT, typename scaleT, const AscendDeQuantConfig& config>
514-__aicore__ inline void AscendDeQuantPerToken(const LocalTensor<dstT>& dstTensor, const LocalTensor<srcT>& srcTensor,
515- const LocalTensor<uint8_t>& sharedTmpBuffer, const LocalTensor<scaleT>& scaleTensor,
516- const LocalTensor<scaleT>& offsetTensor, const AscendDeQuantParam& para)
517-{
518- if constexpr (SupportType<srcT, int32_t>()) {
519- DeQuantPerTokenForS32<dstT, srcT, scaleT, config>(dstTensor, srcTensor, scaleTensor, offsetTensor, para);
520- } else if constexpr (SupportType<srcT, float>()) {
521- DeQuantPerTokenForF32<dstT, srcT, scaleT, config>(dstTensor, srcTensor, scaleTensor, offsetTensor, para);
522- } else {
523- ASCENDC_ASSERT((false), { KERNEL_LOG(KERNEL_ERROR, "unsupport srcT for AscendDeQuant!"); });
524- }
525-}
526- 
527-template <typename dstT, typename srcT, typename scaleT, const AscendDeQuantConfig& config>
528-__aicore__ inline void AscendDeQuantPerGroupForCol(const LocalTensor<dstT>& dstTensor, const LocalTensor<srcT>& srcTensor,
529- const LocalTensor<uint8_t>& sharedTmpBuffer, const LocalTensor<scaleT>& scaleTensor,
530- const LocalTensor<scaleT>& offsetTensor, const AscendDeQuantParam& para)
531-{
532- if constexpr (SupportType<srcT, int32_t>()) {
533- DeQuantPerGroupForColS32<dstT, srcT, scaleT, config>(dstTensor, srcTensor, scaleTensor, offsetTensor, para);
534- } else if constexpr (SupportType<srcT, float>()) {
535- DeQuantPerGroupForColF32<dstT, srcT, scaleT, config>(dstTensor, srcTensor, scaleTensor, offsetTensor, para);
536- } else {
537- ASCENDC_ASSERT((false), { KERNEL_LOG(KERNEL_ERROR, "unsupport srcT for AscendDeQuant!"); });
538- }
539-}
540- 
541-template <typename dstT, typename srcT, typename scaleT, const AscendDeQuantConfig& config>
542-__aicore__ inline void AscendDeQuantPerGroupForRow(const LocalTensor<dstT>& dstTensor, const LocalTensor<srcT>& srcTensor,
543- const LocalTensor<uint8_t>& sharedTmpBuffer, const LocalTensor<scaleT>& scaleTensor,
544- const LocalTensor<scaleT>& offsetTensor, const AscendDeQuantParam& para)
545-{
546- if constexpr (SupportType<srcT, int32_t, float>()) {
547- DeQuantPerGroupForRow<dstT, srcT, scaleT, config>(dstTensor, srcTensor, scaleTensor, offsetTensor, para);
548- } else {
549- ASCENDC_ASSERT((false), { KERNEL_LOG(KERNEL_ERROR, "unsupport srcT for AscendDeQuant!"); });
550- }
551-}
552- 
553-template <typename dstT, typename srcT, typename scaleT, const AscendDeQuantConfig& config, const AscendDeQuantPolicy& policy>
554-__aicore__ inline void AscendDequantImpl(const LocalTensor<dstT>& dstTensor, const LocalTensor<srcT>& srcTensor,
555- const LocalTensor<uint8_t>& sharedTmpBuffer, const LocalTensor<scaleT>& scaleTensor,
556- const LocalTensor<scaleT>& offsetTensor, const AscendDeQuantParam& para)
557-{
558- if ASCEND_IS_AIC {
559- return;
560- }
561- CheckTensorPosition(dstTensor, "dstTensor", "VECIN, VECOUT, VECCALC");
562- CheckTensorPosition(srcTensor, "srcTensor", "VECIN, VECOUT, VECCALC");
563- CheckTensorPosition(scaleTensor, "scaleTensor", "VECIN, VECOUT, VECCALC");
564- CheckTensorPosition(offsetTensor, "offsetTensor", "VECIN, VECOUT, VECCALC");
565- static_assert(SupportType<srcT, int32_t, float>(),
566- "AscendDequant only support int32_t/float input dtype");
567- static_assert(SupportType<dstT, bfloat16_t, half, float>(),
568- "AscendDequant only support bfloat16_t/half/float output dtype");
569- static_assert(SupportType<scaleT, bfloat16_t, half, float>(),
570- "AscendDequant only support bfloat16_t/half/float scaleT dtype");
571- static_assert(((policy == AscendDeQuantPolicy::PER_TOKEN) || (policy == AscendDeQuantPolicy::PER_GROUP)),
572- "unsupported policy for AscendDequant in current device!");
573- ASCENDC_ASSERT((para.calCount <= srcTensor.GetSize() && para.calCount <= dstTensor.GetSize() && para.calCount >= 0), {
574- KERNEL_LOG(KERNEL_ERROR, "calCount is %u, which should be in [0, min(%u, %u)]",
575- para.calCount, srcTensor.GetSize(), dstTensor.GetSize());
576- });
577- if constexpr (policy == AscendDeQuantPolicy::PER_TOKEN) {
578- AscendDeQuantPerToken<dstT, srcT, scaleT, config>(dstTensor, srcTensor, sharedTmpBuffer, scaleTensor, offsetTensor, para);
579- } else if constexpr (policy == AscendDeQuantPolicy::PER_GROUP) {
580- static_assert(
581- ((config.kDim == 0) || (config.kDim == 1)), "AscendDequant PerGroup only support kDim is axis 0/1!");
582- ASCENDC_ASSERT((para.groupSize > 0 && para.groupSize % 32 == 0),
583- { KERNEL_LOG(KERNEL_ERROR, "groupSize must be an integer multiple of 32 and greater than 0 !"); });
584- if constexpr (config.kDim == 1) {
585- AscendDeQuantPerGroupForCol<dstT, srcT, scaleT, config>(dstTensor, srcTensor, sharedTmpBuffer, scaleTensor, offsetTensor, para);
586- } else {
587- AscendDeQuantPerGroupForRow<dstT, srcT, scaleT, config>(dstTensor, srcTensor, sharedTmpBuffer, scaleTensor, offsetTensor, para);
588- }
589- }
590-}
591-} // namespace AscendC
592-#endif // LIB_ASCEND_DEQUANT_ASCEND_DEQUANT_L300_IMPL_H
Dimpl/adv_api/detail/reduce/sum/sum_l300_impl.h+0-137
@@ -1,137 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/*!
12- * \file sum_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_REDUCE_SUM_L300_IMPL_H
16-#define IMPL_REDUCE_SUM_L300_IMPL_H
17- 
18-#include "kernel_tensor.h"
19-#include "kernel_operator_intf.h"
20-#include "include/adv_api/reduce/sum_utils.h"
21- 
22-namespace AscendC {
23-namespace SumInternal {
24-template <typename T>
25-__simd_vf__ inline void SumForOneRepeatTime(
26- __ubuf__ T* dstUb, __ubuf__ T* srcUb, const SumParams sumParams, uint32_t count, uint32_t offset)
27-{
28- uint32_t calCount;
29- MicroAPI::MaskReg mask;
30- MicroAPI::UnalignReg uregOut;
31- MicroAPI::RegTensor<T> srcReg, dstReg;
32- 
33- for (int i = 0; i < sumParams.outter; i++) {
34- calCount = count;
35- mask = MicroAPI::UpdateMask<T>(calCount);
36- MicroAPI::DataCopy(srcReg, srcUb + i * offset);
37- MicroAPI::ReduceSum(dstReg, srcReg, mask);
38- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(dstUb, dstReg, uregOut, 1);
39- }
40- MicroAPI::DataCopyUnAlignPost(dstUb, uregOut, 0);
41-}
42- 
43-template <typename T, bool isFirstRepeat>
44-__simd_vf__ inline void ReduceSumNextN(__ubuf__ T* dstUb, __ubuf__ T* srcUb, const SumParams sumParams,
45- uint32_t calCount, uint32_t repeatTimes, uint32_t offset)
46-{
47- uint32_t count;
48- MicroAPI::MaskReg mask;
49- MicroAPI::UnalignReg uregIn;
50- MicroAPI::RegTensor<T> srcReg, dstReg;
51- constexpr int32_t eleCountPerVL = GetVecLen() / sizeof(T);
52- for (uint16_t i = 0; i < sumParams.outter; i++) {
53- count = calCount;
54- auto dstTmpUb = dstUb + i * offset;
55- for (uint16_t j = 0; j < repeatTimes; j++) {
56- mask = MicroAPI::UpdateMask<T>(count);
57- if constexpr (isFirstRepeat) {
58- MicroAPI::DataCopy(srcReg, srcUb + i * sumParams.inner + j * eleCountPerVL);
59- } else {
60- MicroAPI::DataCopy(srcReg, srcUb + i * offset + j * eleCountPerVL);
61- }
62- MicroAPI::ReduceSum(dstReg, srcReg, mask);
63- MicroAPI::DataCopyUnAlign<T, MicroAPI::PostLiteral::POST_MODE_UPDATE>(dstTmpUb, dstReg, uregIn, 1);
64- }
65- MicroAPI::DataCopyUnAlignPost(dstTmpUb, uregIn, 0);
66- }
67- MicroAPI::LocalMemBar<MicroAPI::MemType::VEC_STORE, MicroAPI::MemType::VEC_LOAD>();
68-}
69-} // namespace SumInternal
70- 
71-template <typename T, int32_t reduceDim = -1, bool isReuseSource = false, bool isBasicBlock = false>
72-__aicore__ inline void SumCheckParams(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
73- const LocalTensor<uint8_t>& sharedTmpBuffer, const SumParams& sumParams)
74-{
75- static_assert(SupportType<T, half, float>(), "current data type is not supported on current device!");
76- CheckTensorPos<T>(dstTensor, Hardware::UB, "dstTensor", "VECIN / VECCALC / VECOUT", "Sum");
77- CheckTensorPos<T>(srcTensor, Hardware::UB, "srcTensor", "VECIN / VECCALC / VECOUT", "Sum");
78- CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN / VECCALC / VECOUT", "Sum");
79- constexpr uint32_t sumInnerAlignLen = 32;
80- ASCENDC_ASSERT((1 <= sumParams.n) && (sumParams.n <= sumParams.inner), {
81- KERNEL_LOG(KERNEL_ERROR, "The value of n must be greater than or equal to 1 and less than or equal to inner.");
82- });
83- ASCENDC_ASSERT((sumParams.inner * sizeof(T) % sumInnerAlignLen == 0),
84- { KERNEL_LOG(KERNEL_ERROR, "The value of inner * sizeof(T) must be an integer multiple of 32."); });
85-}
86- 
87-template <typename T, int32_t reduceDim = -1, bool isReuseSource = false, bool isBasicBlock = false>
88-__aicore__ inline void SumCompute(const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor,
89- const LocalTensor<uint8_t>& sharedTmpBuffer, const SumParams& sumParams)
90-{
91- // Only for AI Vector Core.
92- if ASCEND_IS_AIC {
93- return;
94- }
95- 
96- SumCheckParams<T, reduceDim, isReuseSource, isBasicBlock>(dstTensor, srcTensor, sharedTmpBuffer, sumParams);
97- __ubuf__ T* dstUb = (__ubuf__ T*)dstTensor.GetPhyAddr();
98- __ubuf__ T* srcUb = (__ubuf__ T*)srcTensor.GetPhyAddr();
99- __ubuf__ T* sharedTmpBufferUb = (__ubuf__ T*)sharedTmpBuffer.GetPhyAddr();
100- 
101- constexpr int32_t eleCountPerVL = GetVecLen() / sizeof(T);
102- uint16_t repeatTimes = CeilDivision(sumParams.n, eleCountPerVL);
103- uint32_t loopRepeatTimes;
104- uint32_t calCount = sumParams.n;
105- 
106- uint32_t totalCnt = 1;
107- uint32_t dataSize = repeatTimes;
108- uint32_t offset = AlignUp(CeilDivision(sumParams.inner, eleCountPerVL), 32);
109- while (dataSize > 1) {
110- ++totalCnt;
111- dataSize = CeilDivision(dataSize, eleCountPerVL);
112- }
113- 
114- if (repeatTimes == 1) {
115- SumInternal::SumForOneRepeatTime<T>(dstUb, srcUb, sumParams, sumParams.n, sumParams.inner);
116- return;
117- }
118- 
119- SumInternal::ReduceSumNextN<T, true>(sharedTmpBufferUb, srcUb, sumParams, calCount, repeatTimes, offset);
120- 
121- --totalCnt;
122- loopRepeatTimes = repeatTimes;
123- while (totalCnt != 0) {
124- calCount = loopRepeatTimes;
125- loopRepeatTimes = CeilDivision(loopRepeatTimes, eleCountPerVL);
126- if (totalCnt == 1) {
127- SumInternal::SumForOneRepeatTime<T>(dstUb, sharedTmpBufferUb, sumParams, calCount, offset);
128- } else {
129- SumInternal::ReduceSumNextN<T, false>(
130- sharedTmpBufferUb, sharedTmpBufferUb, sumParams, calCount, loopRepeatTimes, offset);
131- }
132- --totalCnt;
133- }
134-}
135-} // namespace AscendC
136- 
137-#endif // IMPL_REDUCE_SUM_C310_IMPL_H
Dimpl/adv_api/detail/select/selectwithbytesmask/selectwithbytesmask_l300_impl.h+0-159
@@ -1,159 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-#ifndef LIB_SELECT_SELECT_WITH_BYTES_MASK_L300_IMPL_H
12-#define LIB_SELECT_SELECT_WITH_BYTES_MASK_L300_IMPL_H
13-#include "kernel_tensor.h"
14-#include "kernel_utils.h"
15- 
16-namespace AscendC {
17-template <typename T, typename U, CMPMODE cmpMode>
18-__simd_callee__ inline void RegTensorToMaskReg(MicroAPI::RegTensor<U> &vMaskReg0, MicroAPI::RegTensor<U> &vMaskReg1,
19- MicroAPI::MaskReg &localMask0, MicroAPI::MaskReg &maskReg0)
20-{
21- MicroAPI::MaskReg maskReg1;
22- MicroAPI::MaskReg localMask1;
23- if constexpr (sizeof(U) == 1) {
24- MicroAPI::CompareScalar<uint8_t, cmpMode>(localMask0, (MicroAPI::RegTensor<uint8_t> &)vMaskReg0,
25- static_cast<uint8_t>(0), maskReg0);
26- } else if constexpr (sizeof(T) == 2 && sizeof(U) == 4) {
27- MicroAPI::MaskUnPack(maskReg1, maskReg0);
28- MicroAPI::CompareScalar<U, cmpMode>(localMask0, vMaskReg0, static_cast<U>(0), maskReg1);
29- MicroAPI::CompareScalar<U, cmpMode>(localMask1, vMaskReg1, static_cast<U>(0), maskReg1);
30- MicroAPI::MaskDeInterleave<T>(localMask0, localMask1, localMask0, localMask1);
31- } else {
32- MicroAPI::CompareScalar<U, cmpMode>(localMask0, vMaskReg0, static_cast<U>(0), maskReg0);
33- }
34-}
35- 
36-template <typename T, typename U, bool reverse = false>
37-__simd_vf__ inline void SelectWithBytesMaskPerAxisImpl(__ubuf__ T *dstUb, __ubuf__ T *src0Ub, T src1,
38- __ubuf__ U *maskUb, const uint32_t firstAxis, const uint32_t srcLastAxis, const uint32_t maskLastAxis)
39-{
40- MicroAPI::RegTensor<T> vSrcReg0;
41- MicroAPI::RegTensor<T> vSrcReg1;
42- MicroAPI::RegTensor<T> vDstReg;
43- MicroAPI::RegTensor<U> vMaskReg0;
44- MicroAPI::RegTensor<U> vMaskReg1;
45- MicroAPI::MaskReg maskReg0;
46- MicroAPI::MaskReg localMask0;
47- MicroAPI::Duplicate(vSrcReg1, src1);
48- uint32_t sreg;
49- uint32_t sregLower = static_cast<uint32_t>(GetVecLen() / sizeof(T));
50- uint16_t repeatTimes = static_cast<uint16_t>(DivCeil(srcLastAxis, sregLower));
51- for (uint16_t loopH = 0; loopH < static_cast<uint16_t>(firstAxis); ++loopH) {
52- sreg = srcLastAxis;
53- for (uint16_t i = 0; i < repeatTimes; ++i) {
54- maskReg0 = MicroAPI::UpdateMask<T>(sreg);
55- MicroAPI::DataCopy<T>(vSrcReg0, src0Ub + loopH * srcLastAxis + i * sregLower);
56- if constexpr (sizeof(T) == 2 && sizeof(U) == 1) {
57- MicroAPI::DataCopy<uint8_t, MicroAPI::LoadDist::DIST_UNPACK_B8>(
58- (MicroAPI::RegTensor<uint8_t> &)vMaskReg0,
59- (__ubuf__ uint8_t *)maskUb + loopH * maskLastAxis + i * sregLower);
60- } else if constexpr (sizeof(T) == 2 && sizeof(U) == 4) {
61- MicroAPI::DataCopy<U>(vMaskReg0, maskUb + loopH * maskLastAxis + i * sregLower);
62- MicroAPI::DataCopy<U>(vMaskReg1, maskUb + loopH * maskLastAxis + i * sregLower + sregLower / 2);
63- } else if constexpr (sizeof(T) == 4 && sizeof(U) == 1) {
64- MicroAPI::DataCopy<uint8_t, MicroAPI::LoadDist::DIST_UNPACK4_B8>(
65- (MicroAPI::RegTensor<uint8_t> &)vMaskReg0,
66- (__ubuf__ uint8_t *)maskUb + loopH * maskLastAxis + i * sregLower);
67- } else if constexpr (sizeof(T) == 4 && sizeof(U) == 2) {
68- MicroAPI::DataCopy<U, MicroAPI::LoadDist::DIST_UNPACK_B16>(vMaskReg0,
69- maskUb + loopH * maskLastAxis + i * sregLower);
70- } else if constexpr (sizeof(T) == sizeof(U)) {
71- MicroAPI::DataCopy<U>(vMaskReg0, maskUb + loopH * maskLastAxis + i * sregLower);
72- }
73- 
74- if constexpr (!reverse) {
75- RegTensorToMaskReg<T, U, CMPMODE::EQ>(vMaskReg0, vMaskReg1, localMask0, maskReg0);
76- } else {
77- RegTensorToMaskReg<T, U, CMPMODE::NE>(vMaskReg0, vMaskReg1, localMask0, maskReg0);
78- }
79- 
80- MicroAPI::Select(vDstReg, vSrcReg0, vSrcReg1, localMask0);
81- MicroAPI::DataCopy<T>(dstUb + loopH * srcLastAxis + i * sregLower, vDstReg, maskReg0);
82- }
83- }
84-}
85- 
86-template <typename T, typename U, bool reverse = false>
87-__aicore__ inline void SelectWithBytesMaskProcess(const LocalTensor<T>& dst, const LocalTensor<T>& src0, T src1,
88- const LocalTensor<U>& mask, const SelectWithBytesMaskShapeInfo& info)
89-{
90- __ubuf__ T *src0Ub = (__ubuf__ T *)src0.GetPhyAddr();
91- __ubuf__ T *dstUb = (__ubuf__ T *)dst.GetPhyAddr();
92- __ubuf__ U *maskUb = (__ubuf__ U *)mask.GetPhyAddr();
93- const uint32_t firstAxis = static_cast<uint32_t>(info.firstAxis);
94- const uint32_t srcLastAxis = static_cast<uint32_t>(info.srcLastAxis);
95- const uint32_t maskLastAxis = static_cast<uint32_t>(info.maskLastAxis);
96- SelectWithBytesMaskPerAxisImpl<T, U, reverse>(dstUb, src0Ub, src1, maskUb, firstAxis, srcLastAxis,
97- maskLastAxis);
98-}
99- 
100-// Selects Values from two sources and put into dst according to the mask values.
101-// True: Select scalar, False: select src.
102-template <typename T, typename U, bool isReuseMask, bool reverse = false>
103-__aicore__ inline __inout_pipe__(V) void SelectWithBytesMaskImpl(const LocalTensor<T>& dst, const LocalTensor<T>& src0,
104- T src1, const LocalTensor<U>& mask, const LocalTensor<uint8_t>& sharedTmpBuffer,
105- const SelectWithBytesMaskShapeInfo& info)
106-{
107- // Only for AI Vector Core.
108- if ASCEND_IS_AIC {
109- return;
110- }
111- static_assert(SupportType<T, float, half>(), "Select do not support this type on current device");
112- static_assert(SupportType<U, bool, uint8_t, int8_t, uint16_t, int16_t, uint32_t, int32_t>(),
113- "Select do not support this type on current device");
114- CheckTensorPos<T>(dst, Hardware::UB, "dst", "VECIN / VECCALC / VECOUT", "Select");
115- CheckTensorPos<T>(src0, Hardware::UB, "src", "VECIN / VECCALC / VECOUT", "Select");
116- CheckTensorPos<U>(mask, Hardware::UB, "mask", "VECIN / VECCALC / VECOUT", "Select");
117- CheckTensorPos<uint8_t>(sharedTmpBuffer, Hardware::UB, "sharedTmpBuffer", "VECIN / VECCALC / VECOUT",
118- "Select");
119- ASCENDC_ASSERT((info.srcLastAxis * sizeof(T) % ONE_BLK_SIZE == 0), {
120- KERNEL_LOG(KERNEL_ERROR, "srcLastAxis should be 32B aligned, current srcLastAxis is %u", info.srcLastAxis);
121- });
122- ASCENDC_ASSERT((info.maskLastAxis * sizeof(U) % ONE_BLK_SIZE == 0), {
123- KERNEL_LOG(KERNEL_ERROR, "maskLastAxis should be 32B aligned, current maskLastAxis is %u", info.maskLastAxis);
124- });
125- ASCENDC_ASSERT((info.maskLastAxis % BLOCK_CUBE == 0), {
126- KERNEL_LOG(KERNEL_ERROR, "maskLastAxis should be multiples of 16, current maskLastAxis is %u",
127- info.maskLastAxis);
128- });
129- 
130- const uint32_t firstAxis = info.firstAxis;
131- const uint32_t srcLastAxis = info.srcLastAxis;
132- const uint32_t maskLastAxis = info.maskLastAxis;
133- const uint32_t srcSize = src0.GetSize();
134- 
135- ASCENDC_ASSERT((srcSize == firstAxis * srcLastAxis),
136- { KERNEL_LOG(KERNEL_ERROR, "ShapeInfo must be match with src Tensor size."); });
137- ASCENDC_ASSERT((mask.GetSize() == firstAxis * maskLastAxis),
138- { KERNEL_LOG(KERNEL_ERROR, "ShapeInfo must be match with mask Tensor size."); });
139- ASCENDC_ASSERT((maskLastAxis >= srcLastAxis),
140- { KERNEL_LOG(KERNEL_ERROR, "maskLastAxis must be greater than or equal to srcLastAxis."); });
141- 
142- SelectWithBytesMaskProcess<T, U, reverse>(dst, src0, src1, mask, info);
143-}
144- 
145-template <typename T, typename U, bool isReuseMask = true>
146-__aicore__ inline void SelectWithBytesMask(const LocalTensor<T>& dst, const LocalTensor<T>& src0, T src1,
147- const LocalTensor<U>& mask, const LocalTensor<uint8_t>& sharedTmpBuffer, const SelectWithBytesMaskShapeInfo& info)
148-{
149- SelectWithBytesMaskImpl<T, U, isReuseMask, false>(dst, src0, src1, mask, sharedTmpBuffer, info);
150-}
151- 
152-template <typename T, typename U, bool isReuseMask = true>
153-__aicore__ inline void SelectWithBytesMask(const LocalTensor<T>& dst, T src0, const LocalTensor<T>& src1,
154- const LocalTensor<U>& mask, const LocalTensor<uint8_t>& sharedTmpBuffer, const SelectWithBytesMaskShapeInfo& info)
155-{
156- SelectWithBytesMaskImpl<T, U, isReuseMask, true>(dst, src1, src0, mask, sharedTmpBuffer, info);
157-}
158-} // namespace AscendC
159-#endif // LIB_SELECT_SELECT_WITH_BYTES_MASK_L300_IMPL_H
Dimpl/adv_api/detail/sort/topk/topk_l300_impl.h+0-1610
@@ -1,1610 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/*!
12- * \file topk_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_SORT_TOPK_TOPK_L300_IMPL_H
16-#define IMPL_SORT_TOPK_TOPK_L300_IMPL_H
17-#if defined(ASCENDC_CPU_DEBUG) && ASCENDC_CPU_DEBUG == 1
18-#include "kernel_log.h"
19-#endif
20- 
21-#include "kernel_tensor.h"
22-#include "kernel_pop_stack_buffer.h"
23-#include "topk_common_utils.h"
24- 
25-#include "../sort/sort_impl.h"
26-#include "../sort_common_utils.h"
27- 
28-namespace AscendC {
29-#if ASCENDC_CPU_DEBUG
30-template <typename T, bool isInitIndex = false, enum TopKMode topkMode = TopKMode::TOPK_NORMAL, const TopKConfig& config = defaultTopKConfig>
31-void TopkInputCheck(const int32_t k, const TopKInfo &topKInfo)
32-{
33- ASCENDC_ASSERT((1 <= k) && (k <= topKInfo.n), {
34- KERNEL_LOG(KERNEL_ERROR, "The value of k must be greater than or equal to 1 and less than or equal to inner.");
35- });
36- ASCENDC_ASSERT((1 <= topKInfo.n) && (topKInfo.n <= topKInfo.inner), {
37- KERNEL_LOG(KERNEL_ERROR, "The value of n must be greater than or equal to 1 and less than or equal to inner.");
38- });
39- ASCENDC_ASSERT((topKInfo.inner % TOPK_INNER_ALIGN_LEN == 0), {
40- KERNEL_LOG(KERNEL_ERROR, "The value of inner must be an integer multiple of 32.");
41- });
42- 
43- if constexpr (config.algo == TopKAlgo::MERGE_SORT) {
44- ASCENDC_ASSERT((std::is_same<T, half>::value) || (std::is_same<T, float>::value), {
45- KERNEL_LOG(KERNEL_ERROR, "Type must be half or float in merge sort algorithm.");
46- });
47- }
48- if constexpr (topkMode == TopKMode::TOPK_NORMAL) {
49- ASCENDC_ASSERT((topKInfo.inner <= TOPK_NORMAL_INNER_MAX_LEN), {
50- KERNEL_LOG(KERNEL_ERROR, "The maximum value supported by inner is 4096.");
51- });
52- }
53- if constexpr (topkMode == TopKMode::TOPK_NSMALL) {
54- ASCENDC_ASSERT((topKInfo.inner == TOPK_NSMALL_INNER_LEN), {
55- KERNEL_LOG(KERNEL_ERROR, "The value of inner must be an equal 32.");
56- });
57- }
58-}
59-#endif
60- 
61-template <typename T>
62-__aicore__ inline void GatherDstValAndDstIdx(const LocalTensor<T> &dstValueLocal,
63- const LocalTensor<int32_t> &dstIndexLocal, const LocalTensor<T> &tmpLocal, const TopkTiling &tilling,
64- const int32_t dstOffsetFourBytes, const int outterIdx)
65-{
66- uint64_t rsvdCnt = 0;
67- struct GatherMaskParams reducev2Params(DEFAULT_BLK_STRIDE, 1, DEFAULT_REPEAT_STRIDE, DEFAULT_BLK_STRIDE);
68- if constexpr (sizeof(T) == sizeof(float)) {
69- // Get Value, The index of the odd position is obtained for each repeat.
70- GatherMask<T>(dstValueLocal[dstOffsetFourBytes], tmpLocal[tilling.innerDataSize], REDUCEV2_MODE_ONE,
71- true, tilling.maskVreducev2FourBytes, reducev2Params, rsvdCnt);
72- } else {
73- int32_t dstOffsetTwoBytes = outterIdx * tilling.kAlignTwoBytes;
74- // Get Value. The first element is used for every four elements in each repeat.
75- GatherMask<T>(dstValueLocal[dstOffsetTwoBytes], tmpLocal[tilling.innerDataSize], REDUCEV2_MODE_THREE,
76- true, tilling.maskVreducev2TwoBytes, reducev2Params, rsvdCnt);
77- }
78- // Get Index, The index of the even position is obtained for each repeat.
79- LocalTensor<int32_t> tempBufferLocal = tmpLocal[tilling.innerDataSize].template ReinterpretCast<int32_t>();
80- GatherMask<int32_t>(dstIndexLocal[dstOffsetFourBytes], tempBufferLocal, REDUCEV2_MODE_TWO, true,
81- tilling.maskVreducev2FourBytes, reducev2Params, rsvdCnt);
82-}
83- 
84-template <typename T, bool isInitIndex>
85-__aicore__ inline void TmpLocalSort32(const LocalTensor<T> &srcLocal, const LocalTensor<int32_t> &srcIndexLocal,
86- const LocalTensor<T> &tmpLocal, const TopkTiling &tilling, const TopKInfo &topKInfo, const bool isLargest,
87- const int outterIdx)
88-{
89- int offset = outterIdx * topKInfo.inner;
90- LocalTensor<T> tmpBufferLocal = tmpLocal;
91- if constexpr (!isInitIndex) {
92- LocalTensor<uint32_t> tempBufferUint32 = tmpLocal[tilling.srcIndexOffset].template
93- ReinterpretCast<uint32_t>();
94- Sort<T, true>(tmpLocal[tilling.innerDataSize], srcLocal[offset], tempBufferUint32, tmpBufferLocal, tilling.sortRepeat);
95- } else {
96- LocalTensor<uint32_t> tempBufferUint32 = srcIndexLocal.template ReinterpretCast<uint32_t>();
97- Sort<T, true>(tmpLocal[tilling.innerDataSize], srcLocal[offset], tempBufferUint32, tmpBufferLocal, tilling.sortRepeat);
98- }
99-}
100- 
101-template <typename T, bool isInitIndex, bool isHasfinish>
102-__aicore__ inline void TopKCompute(const LocalTensor<T> &dstValueLocal, const LocalTensor<int32_t> &dstIndexLocal,
103- const LocalTensor<T> &srcLocal, const LocalTensor<int32_t> &srcIndexLocal, const LocalTensor<bool> &finishLocal,
104- const LocalTensor<T> &tmpLocal, const int32_t k, const TopkTiling &tilling, const TopKInfo &topKInfo,
105- const bool isLargest)
106-{
107- for (int j = 0; j < topKInfo.outter; ++j) {
108- int32_t dstOffsetFourBytes = j * tilling.kAlignFourBytes;
109- TmpLocalSort32<T, isInitIndex>(srcLocal, srcIndexLocal, tmpLocal, tilling, topKInfo, isLargest, j);
110- 
111- GatherDstValAndDstIdx(dstValueLocal, dstIndexLocal, tmpLocal, tilling, dstOffsetFourBytes, j);
112- 
113- if constexpr (isHasfinish) {
114- bool finishValue = finishLocal.GetValue(j);
115- auto eventID = GetTPipePtr()->FetchEventID(HardEvent::S_V);
116- SetFlag<HardEvent::S_V>(eventID);
117- WaitFlag<HardEvent::S_V>(eventID);
118- 
119- if (finishValue) {
120- Duplicate(dstIndexLocal[dstOffsetFourBytes], static_cast<int32_t>(topKInfo.n), k);
121- }
122- }
123- }
124-}
125- 
126-template <typename T>
127-__aicore__ inline void TopKNSmallGetTopKValue(const LocalTensor<T> &dstValueLocal,
128- const LocalTensor<int32_t> &dstIndexLocal, const LocalTensor<T> &tmpLocal, const int32_t k,
129- const TopkTiling &tilling, const TopKInfo &topKInfo)
130-{
131- uint64_t rsvdCnt = 0;
132- struct GatherMaskParams reducev2Params(DEFAULT_BLK_STRIDE, topKInfo.outter, DEFAULT_REPEAT_STRIDE, 0);
133- if constexpr (sizeof(T) == sizeof(float)) {
134- // Get Value, The index of the odd position is obtained for each repeat.
135- GatherMask<T>(dstValueLocal, tmpLocal, REDUCEV2_MODE_ONE,
136- true, 2 * k, reducev2Params, rsvdCnt);
137- } else {
138- // Get Value. The first element is used for every four elements in each repeat.
139- GatherMask<T>(dstValueLocal, tmpLocal, REDUCEV2_MODE_THREE,
140- true, 4 * k, reducev2Params, rsvdCnt);
141- }
142- // Get Index, The index of the even position is obtained for each repeat.
143- LocalTensor<int32_t> tempBufferLocal = tmpLocal.template ReinterpretCast<int32_t>();
144- GatherMask<int32_t>(dstIndexLocal, tempBufferLocal, REDUCEV2_MODE_TWO, true,
145- 2 * k, reducev2Params, rsvdCnt);
146-}
147- 
148-template <typename T, bool isInitIndex, bool isHasfinish>
149-__aicore__ inline void TopKNSmallCompute(const LocalTensor<T> &dstValueLocal, const LocalTensor<int32_t> &dstIndexLocal,
150- const LocalTensor<T> &srcLocal, const LocalTensor<int32_t> &srcIndexLocal, const LocalTensor<bool> &finishLocal,
151- const LocalTensor<T> &tmpLocal, const int32_t k, const TopkTiling &tilling, const TopKInfo &topKInfo,
152- const bool isLargest)
153-{
154- if constexpr (!isInitIndex) {
155- LocalTensor<uint32_t> tempBufferUint32 = tmpLocal[tilling.innerDataSize].template
156- ReinterpretCast<uint32_t>();
157- Sort32<T>(tmpLocal, srcLocal, tempBufferUint32, topKInfo.outter);
158- } else {
159- LocalTensor<uint32_t> tempBufferUint32 = srcIndexLocal.template ReinterpretCast<uint32_t>();
160- Sort32<T>(tmpLocal, srcLocal, tempBufferUint32, topKInfo.outter);
161- }
162- 
163- TopKNSmallGetTopKValue(dstValueLocal, dstIndexLocal, tmpLocal, k, tilling, topKInfo);
164-}
165- 
166-namespace MicroAPI {
167-namespace RadixSelectTopK {
168- 
169-constexpr uint32_t LOAD_NUMS_PER_ROUND = 256;
170-constexpr uint32_t BUCKET_BYTES = 512;
171- 
172-template <typename T>
173-__aicore__ inline constexpr bool IsFloatNum()
174-{
175- return SupportType<T, float, half, bfloat16_t>();
176-}
177- 
178-template <typename T>
179-__aicore__ inline constexpr bool NeedTwiddle()
180-{
181- return SupportType<T, int8_t, int16_t, int32_t, int64_t, float, half, bfloat16_t>();
182-}
183- 
184-__aicore__ inline constexpr bool NeedReverse(bool isLargest)
185-{
186- return !isLargest;
187-}
188- 
189-template <bool isLargest>
190-__aicore__ inline constexpr bool NeedReverse()
191-{
192- return !isLargest;
193-}
194- 
195-template <typename T>
196-__aicore__ inline constexpr bool NeedPreProcess(bool isLargest)
197-{
198- return NeedReverse(isLargest) || NeedTwiddle<T>();
199-}
200- 
201-template <typename T, bool isLargest>
202-__aicore__ inline constexpr bool NeedPreProcess()
203-{
204- return NeedReverse<isLargest>() || NeedTwiddle<T>();
205-}
206- 
207-template <typename T, typename U, bool isLargest>
208-__simd_vf__ inline void Preprocess(__ubuf__ T *src, __ubuf__ U *dst, uint32_t count)
209-{
210- uint16_t repeatTime = DivCeil(count, GetVecLen() / sizeof(T));
211- uint32_t currCount = count;
212- constexpr uint32_t postUpdateSize = GetVecLen() / sizeof(U);
213- for (uint16_t i = 0; i < repeatTime; i++) {
214- MaskReg maskReg = UpdateMask<U>(currCount);
215- 
216- RegTensor<U> local;
217- DataCopy<U, PostLiteral::POST_MODE_UPDATE>(local, (__ubuf__ U *&)src, postUpdateSize);
218- if constexpr (NeedTwiddle<T>()) {
219- AscendC::MicroAPI::Internal::TwiddleIn<T>(local, local, maskReg);
220- }
221- if constexpr (NeedReverse<isLargest>()) {
222- Not(local, local, maskReg);
223- }
224- DataCopy<U, PostLiteral::POST_MODE_UPDATE>((__ubuf__ U *&)dst, local, postUpdateSize, maskReg);
225- }
226-}
227- 
228-template <typename T, typename U, bool isLargest>
229-__simd_vf__ inline void PostProcess(__ubuf__ U *src, __ubuf__ U *dst, uint32_t count)
230-{
231- uint16_t repeatTime = DivCeil(count, GetVecLen() / sizeof(T));
232- constexpr uint32_t postUpdateSize = GetVecLen() / sizeof(U);
233- for (uint16_t i = 0; i < repeatTime; i++) {
234- MaskReg maskReg = UpdateMask<U>(count);
235- 
236- RegTensor<U> local;
237- DataCopy<U, PostLiteral::POST_MODE_UPDATE>(local, (__ubuf__ U *&)src, postUpdateSize);
238- if constexpr (NeedReverse<isLargest>()) {
239- Not(local, local, maskReg);
240- }
241- if constexpr (NeedTwiddle<T>()) {
242- AscendC::MicroAPI::Internal::TwiddleOut<T>(local, local, maskReg);
243- }
244- DataCopy<U, PostLiteral::POST_MODE_UPDATE>((__ubuf__ U *&)dst, local, postUpdateSize, maskReg);
245- }
246-}
247- 
248-template <typename T, typename U>
249-__simd_vf__ inline void Preprocess(__ubuf__ T *src, __ubuf__ U *dst, bool isLargest, uint32_t count)
250-{
251- uint16_t repeatTime = DivCeil(count, GetVecLen() / sizeof(T));
252- uint32_t currCount = count;
253- constexpr uint32_t postUpdateSize = GetVecLen() / sizeof(U);
254- for (uint16_t i = 0; i < repeatTime; i++) {
255- MaskReg maskReg = UpdateMask<U>(currCount);
256- 
257- RegTensor<U> local;
258- DataCopy<U, PostLiteral::POST_MODE_UPDATE>(local, (__ubuf__ U *&)src, postUpdateSize);
259- if constexpr (NeedTwiddle<T>()) {
260- AscendC::MicroAPI::Internal::TwiddleIn<T>(local, local, maskReg);
261- }
262- if (NeedReverse(isLargest)) {
263- Not(local, local, maskReg);
264- }
265- DataCopy<U, PostLiteral::POST_MODE_UPDATE>((__ubuf__ U *&)dst, local, postUpdateSize, maskReg);
266- }
267-}
268- 
269-template <typename T, typename U>
270-__simd_vf__ inline void PostProcess(__ubuf__ U *src, __ubuf__ U *dst, bool isLargest, uint32_t count)
271-{
272- uint16_t repeatTime = DivCeil(count, GetVecLen() / sizeof(T));
273- constexpr uint32_t postUpdateSize = GetVecLen() / sizeof(U);
274- for (uint16_t i = 0; i < repeatTime; i++) {
275- MaskReg maskReg = UpdateMask<U>(count);
276- 
277- RegTensor<U> local;
278- DataCopy<U, PostLiteral::POST_MODE_UPDATE>(local, (__ubuf__ U *&)src, postUpdateSize);
279- if (NeedReverse(isLargest)) {
280- Not(local, local, maskReg);
281- }
282- if constexpr (NeedTwiddle<T>()) {
283- AscendC::MicroAPI::Internal::TwiddleOut<T>(local, local, maskReg);
284- }
285- DataCopy<U, PostLiteral::POST_MODE_UPDATE>((__ubuf__ U *&)dst, local, postUpdateSize, maskReg);
286- }
287-}
288- 
289-template<typename T, bool isReuseSrc, bool isLargest>
290-__aicore__ inline void InitializeTempBuffer(const LocalTensor<T> &tempBuffer, const uint32_t alignCount, __ubuf__ T *&tmpSrcData,
291- __ubuf__ int32_t *&tmpSrcIndex, __ubuf__ uint16_t *&tmpHistData, __ubuf__ T *&realWorkData, __ubuf__ T *&sortTmpBuffer)
292-{
293- __ubuf__ uint8_t *tmp = (__ubuf__ uint8_t *)tempBuffer.GetPhyAddr();
294- tmpSrcData = (__ubuf__ T*)tmp;
295- 
296- if constexpr (sizeof(T) == 8) {
297- uint32_t srcOffset = 0;
298- if (alignCount < LOAD_NUMS_PER_ROUND) {
299- srcOffset += sizeof(T) * LOAD_NUMS_PER_ROUND;
300- } else {
301- srcOffset += sizeof(T) * alignCount;
302- }
303- 
304- tmpSrcIndex = (__ubuf__ int32_t*)((__ubuf__ uint8_t *)tmpSrcData + srcOffset);
305- } else {
306- tmpSrcIndex = (__ubuf__ int32_t*)((__ubuf__ uint8_t *)tmpSrcData + sizeof(T) * alignCount);
307- }
308- 
309- tmpHistData = (__ubuf__ uint16_t*)(tmpSrcIndex);
310- 
311- if constexpr (NeedPreProcess<T, isLargest>()) {
312- if constexpr (!isReuseSrc) {
313- uint32_t indexSpace = alignCount * sizeof(int32_t);
314- if (indexSpace < BUCKET_BYTES) {
315- indexSpace = BUCKET_BYTES;
316- }
317- realWorkData = (__ubuf__ T*)((__ubuf__ uint8_t*)tmpSrcIndex + indexSpace);
318- sortTmpBuffer = realWorkData;
319- } else {
320- sortTmpBuffer = (__ubuf__ T*)((__ubuf__ uint8_t*)tmpSrcIndex + sizeof(int32_t) * alignCount);
321- }
322- } else {
323- sortTmpBuffer = (__ubuf__ T*)((__ubuf__ uint8_t*)tmpSrcIndex + sizeof(int32_t) * alignCount);
324- }
325-}
326- 
327-template<typename T, bool isReuseSrc>
328-__aicore__ inline void InitializeTempBuffer(const LocalTensor<T> &tempBuffer, bool isLargest, const uint32_t alignCount,
329- __ubuf__ T *&tmpSrcData, __ubuf__ int32_t *&tmpSrcIndex, __ubuf__ uint16_t *&tmpHistData,
330- __ubuf__ T *&realWorkData, __ubuf__ T *&sortTmpBuffer)
331-{
332- __ubuf__ uint8_t *tmp = (__ubuf__ uint8_t *)tempBuffer.GetPhyAddr();
333- tmpSrcData = (__ubuf__ T*)tmp;
334- 
335- if constexpr (sizeof(T) == 8) {
336- uint32_t srcOffset = 0;
337- if (alignCount < LOAD_NUMS_PER_ROUND) {
338- srcOffset += sizeof(T) * LOAD_NUMS_PER_ROUND;
339- } else {
340- srcOffset += sizeof(T) * alignCount;
341- }
342- tmpSrcIndex = (__ubuf__ int32_t*)((__ubuf__ uint8_t *)tmpSrcData + srcOffset);
343- } else {
344- tmpSrcIndex = (__ubuf__ int32_t*)((__ubuf__ uint8_t *)tmpSrcData + sizeof(T) * alignCount);
345- }
346- 
347- tmpHistData = (__ubuf__ uint16_t*)(tmpSrcIndex);
348- 
349- if (NeedPreProcess<T>(isLargest)) {
350- if constexpr (!isReuseSrc) {
351- uint32_t indexSpace = alignCount * sizeof(int32_t);
352- if (indexSpace < BUCKET_BYTES) {
353- indexSpace = BUCKET_BYTES;
354- }
355- realWorkData = (__ubuf__ T*)((__ubuf__ uint8_t*)tmpSrcIndex + indexSpace);
356- sortTmpBuffer = realWorkData;
357- } else {
358- sortTmpBuffer = (__ubuf__ T*)((__ubuf__ uint8_t*)tmpSrcIndex + sizeof(int32_t) * alignCount);
359- }
360- } else {
361- sortTmpBuffer = (__ubuf__ T*)((__ubuf__ uint8_t*)tmpSrcIndex + sizeof(int32_t) * alignCount);
362- }
363-}
364- 
365-__simd_callee__ inline void GetLowerstByte(RegTensor<uint8_t>& dst, RegTensor<uint16_t>& src0, RegTensor<uint16_t>& src1)
366-{
367- RegTensor<uint8_t> tmpU8Reg;
368- DeInterleave(dst, tmpU8Reg, (RegTensor<uint8_t> &)src0, (RegTensor<uint8_t> &)src1);
369-}
370- 
371-__simd_callee__ inline void GetLowerstByte(RegTensor<uint8_t>& dst, RegTensor<uint32_t>& src0, RegTensor<uint32_t>& src1, RegTensor<uint32_t>& src2, RegTensor<uint32_t>& src3)
372-{
373- RegTensor<uint16_t> tmpU16Reg0, tmpU16Reg1, tmpU16Reg2;
374- DeInterleave(tmpU16Reg0, tmpU16Reg1, (RegTensor<uint16_t> &)src0, (RegTensor<uint16_t> &)src1);
375- DeInterleave(tmpU16Reg2, tmpU16Reg1, (RegTensor<uint16_t> &)src2, (RegTensor<uint16_t> &)src3);
376- 
377- GetLowerstByte(dst, tmpU16Reg0, tmpU16Reg2);
378-}
379- 
380-__simd_callee__ inline void TransToB8Mask(MaskReg& dst, MaskReg& u16Src0, MaskReg& u16Src1)
381-{
382- MaskReg tmpU8Mask;
383- MaskDeInterleave<uint8_t>(dst, tmpU8Mask, u16Src0, u16Src1);
384-}
385- 
386-__simd_callee__ inline void TransToB8Mask(MaskReg& dst, MaskReg& u32Src0, MaskReg& u32Src1, MaskReg& u32Src2, MaskReg& u32Src3)
387-{
388- MaskReg tmpU16LowPart0, tmpU16LowPart1, tmpU16LowPart2;
389- MaskDeInterleave<uint16_t>(tmpU16LowPart0, tmpU16LowPart1, u32Src0, u32Src1);
390- MaskDeInterleave<uint16_t>(tmpU16LowPart2, tmpU16LowPart1, u32Src2, u32Src3);
391- 
392- TransToB8Mask(dst, tmpU16LowPart0, tmpU16LowPart2);
393-}
394- 
395-__simd_callee__ inline void CollectGivenPosByte(RegTensor<uint8_t>& colWorkBits, __ubuf__ uint64_t *src, uint16_t byteNum, const int32_t srcOffset,
396-__ubuf__ uint64_t* tmpLocal)
397-{
398- uint32_t loadCount = GetVecLen() / sizeof(uint32_t);
399- uint32_t realCount = loadCount / 2;
400- 
401- int16_t offsets = static_cast<int16_t>((byteNum - 1) * 8);
402- __ubuf__ uint32_t* srcU32 = (__ubuf__ uint32_t*)src;
403- __ubuf__ uint32_t* tmpU32 = (__ubuf__ uint32_t*)tmpLocal;
404- 
405- MaskReg fullMask = CreateMask<uint32_t>();
406- MaskReg zeroMask = CreateMask<uint32_t, MaskPattern::ALLF>();
407- MaskReg halfMask = CreateMask<uint32_t, MaskPattern::H>();
408- 
409- constexpr uint16_t repeatTimes = sizeof(uint64_t);
410- MaskReg lowMask, highMask, tmpMask;
411- MaskInterleave<uint32_t>(lowMask, tmpMask, fullMask, zeroMask);
412- MaskInterleave<uint32_t>(highMask, tmpMask, zeroMask, fullMask);
413- 
414- int16_t highOffsets = offsets - 32;
415- if (offsets >= 32) {
416- for (uint16_t i = 0; i < repeatTimes; ++i) {
417- RegTensor<uint32_t> input, output;
418- DataCopy(input, srcU32 + srcOffset + i * loadCount);
419- GatherMask(output, input, highMask);
420- ShiftRights(output, output, highOffsets, halfMask);
421- 
422- DataCopy(tmpU32 + realCount * i, output, halfMask);
423- }
424- } else {
425- for (uint16_t i = 0; i < repeatTimes; ++i) {
426- RegTensor<uint32_t> input, output;
427- DataCopy(input, srcU32 + srcOffset + i * loadCount);
428- GatherMask(output, input, lowMask);
429- ShiftRights(output, output, offsets, halfMask);
430- 
431- DataCopy(tmpU32 + realCount * i, output, halfMask);
432- }
433- }
434- 
435- MicroAPI::LocalMemBar<MicroAPI::MemType::VEC_STORE, MicroAPI::MemType::VEC_LOAD>();
436- RegTensor<uint32_t> work0, work1, work2, work3;
437- DataCopy(work0, tmpU32);
438- DataCopy(work1, tmpU32 + loadCount);
439- DataCopy(work2, tmpU32 + loadCount * 2);
440- DataCopy(work3, tmpU32 + loadCount * 3);
441- 
442- GetLowerstByte(colWorkBits, work0, work1, work2, work3);
443-}
444- 
445-__simd_callee__ inline void CompareHighBytesBeforePos(MaskReg& filterMask, __ubuf__ uint64_t *&src, MaskReg& maskReg,
446- uint64_t value, uint16_t byteNum, int32_t srcOffset, __ubuf__ uint64_t* tmpLocal)
447-{
448- __ubuf__ uint32_t* srcU32 = (__ubuf__ uint32_t*)src;
449- __ubuf__ uint32_t* tmpU32 = (__ubuf__ uint32_t*)tmpLocal;
450-
451- uint32_t loadCount = GetVecLen() / sizeof(uint32_t);
452- uint32_t realCount = loadCount / 2;
453- int16_t maskOffsets = static_cast<int16_t>(byteNum * 8);
454- 
455- MaskReg fullMask = CreateMask<uint32_t>();
456- MaskReg zeroMask = CreateMask<uint32_t, MaskPattern::ALLF>();
457- MaskReg halfMask = CreateMask<uint32_t, MaskPattern::H>();
458- 
459- constexpr uint16_t repeatTimes = sizeof(uint64_t);
460- MaskReg lowMask, highMask, tmpMask;
461- MaskInterleave<uint32_t>(lowMask, tmpMask, fullMask, zeroMask);
462- MaskInterleave<uint32_t>(highMask, tmpMask, zeroMask, fullMask);
463- 
464- int16_t realMaskOffset = maskOffsets - 32;
465- int16_t reverseMaskOffset = 32 - maskOffsets;
466- for (uint16_t i = 0; i < repeatTimes; ++i) {
467- RegTensor<uint32_t> input, highOut, tHighOut, lowOut;
468- DataCopy(input, srcU32 + srcOffset + i * loadCount);
469- 
470- GatherMask(highOut, input, highMask);
471- 
472- if (maskOffsets > 32) {
473- ShiftRights(lowOut, highOut, realMaskOffset, halfMask);
474- Duplicate(highOut, 0, halfMask);
475- } else {
476- GatherMask(lowOut, input, lowMask);
477- ShiftRights(lowOut, lowOut, maskOffsets, halfMask);
478- ShiftLefts(tHighOut, highOut, reverseMaskOffset, halfMask);
479- Or(lowOut, lowOut, tHighOut, halfMask);
480- ShiftRights(highOut, highOut, maskOffsets, halfMask);
481- }
482- 
483- DataCopy(tmpU32 + realCount * i, highOut, halfMask);
484- DataCopy(tmpU32 + realCount * (i + repeatTimes), lowOut, halfMask);
485- }
486- MicroAPI::LocalMemBar<MicroAPI::MemType::VEC_STORE, MicroAPI::MemType::VEC_LOAD>();
487- RegTensor<uint32_t> highPart0, highPart1, highPart2, highPart3, lowPart0, lowPart1, lowPart2, lowPart3;
488- DataCopy(highPart0, tmpU32);
489- DataCopy(highPart1, tmpU32 + loadCount);
490- DataCopy(highPart2, tmpU32 + loadCount * 2);
491- DataCopy(highPart3, tmpU32 + loadCount * 3);
492- DataCopy(lowPart0, tmpU32 + loadCount * 4);
493- DataCopy(lowPart1, tmpU32 + loadCount * 5);
494- DataCopy(lowPart2, tmpU32 + loadCount * 6);
495- DataCopy(lowPart3, tmpU32 + loadCount * 7);
496- 
497- MaskReg highMask0, highMask1, highMask2, highMask3;
498- MaskReg lowMask0, lowMask1, lowMask2, lowMask3;
499- uint32_t highValue = value >> 32;
500- CompareScalar<uint32_t, CMPMODE::EQ>(highMask0, highPart0, highValue, fullMask);
501- CompareScalar<uint32_t, CMPMODE::EQ>(highMask1, highPart1, highValue, fullMask);
502- CompareScalar<uint32_t, CMPMODE::EQ>(highMask2, highPart2, highValue, fullMask);
503- CompareScalar<uint32_t, CMPMODE::EQ>(highMask3, highPart3, highValue, fullMask);
504- 
505- uint32_t lowValue = value & 0xffffffff;
506- CompareScalar<uint32_t, CMPMODE::EQ>(lowMask0, lowPart0, lowValue, fullMask);
507- CompareScalar<uint32_t, CMPMODE::EQ>(lowMask1, lowPart1, lowValue, fullMask);
508- CompareScalar<uint32_t, CMPMODE::EQ>(lowMask2, lowPart2, lowValue, fullMask);
509- CompareScalar<uint32_t, CMPMODE::EQ>(lowMask3, lowPart3, lowValue, fullMask);
510- 
511- MaskReg highRes, lowRes;
512- TransToB8Mask(lowRes, lowMask0, lowMask1, lowMask2, lowMask3);
513- TransToB8Mask(highRes, highMask0, highMask1, highMask2, highMask3);
514-
515- MaskReg res;
516- MaskAnd(res, highRes, lowRes, maskReg);
517- MaskAnd(filterMask, maskReg, res, maskReg);
518-}
519- 
520-__simd_callee__ inline void FilterDataAndGivenByteFromOri(
521- MaskReg& filterMask, RegTensor<uint8_t>& colWorkBits, __ubuf__ uint64_t *src, MaskReg& maskReg,
522- uint64_t value, uint16_t byteNum, int32_t srcOffset, __ubuf__ uint64_t* tmpLocal)
523-{
524- CollectGivenPosByte(colWorkBits, src, byteNum, srcOffset * 2, tmpLocal);
525- CompareHighBytesBeforePos(filterMask, src, maskReg, value, byteNum, srcOffset * 2, tmpLocal);
526-}
527- 
528-__simd_callee__ inline void FilterDataAndGivenByteFromOri(
529- MaskReg& filterMask, RegTensor<uint8_t>& colWorkBits, __ubuf__ uint32_t *src, MaskReg& maskReg,
530- uint32_t value, uint16_t byteNum, int32_t srcOffset, __ubuf__ uint32_t* tmpLocal)
531-{
532- constexpr uint32_t eleCountPerVL = GetVecLen() / sizeof(uint32_t);
533- MaskReg fullMask = CreateMask<uint32_t>();
534- int16_t byteOffsets = static_cast<int16_t>((byteNum - 1) * 8);
535- int16_t maskOffsets = static_cast<int16_t>(byteNum * 8);
536- 
537- RegTensor<uint32_t> input0, input1, input2, input3;
538- DataCopy(input0, src + srcOffset);
539- DataCopy(input1, src + srcOffset + eleCountPerVL);
540- DataCopy(input2, src + srcOffset + eleCountPerVL * 2);
541- DataCopy(input3, src + srcOffset + eleCountPerVL * 3);
542- 
543- RegTensor<uint32_t> tmpU32ByteReg0, tmpU32ByteReg1, tmpU32ByteReg2, tmpU32ByteReg3;
544- ShiftRights(tmpU32ByteReg0, input0, byteOffsets, fullMask);
545- ShiftRights(tmpU32ByteReg1, input1, byteOffsets, fullMask);
546- ShiftRights(tmpU32ByteReg2, input2, byteOffsets, fullMask);
547- ShiftRights(tmpU32ByteReg3, input3, byteOffsets, fullMask);
548- 
549- GetLowerstByte(colWorkBits, tmpU32ByteReg0, tmpU32ByteReg1, tmpU32ByteReg2, tmpU32ByteReg3);
550- 
551- RegTensor<uint32_t> tmpU32MaskReg0, tmpU32MaskReg1, tmpU32MaskReg2, tmpU32MaskReg3;
552- ShiftRights(tmpU32MaskReg0, input0, maskOffsets, fullMask);
553- ShiftRights(tmpU32MaskReg1, input1, maskOffsets, fullMask);
554- ShiftRights(tmpU32MaskReg2, input2, maskOffsets, fullMask);
555- ShiftRights(tmpU32MaskReg3, input3, maskOffsets, fullMask);
556- 
557- MaskReg mask0, mask1, mask2, mask3;
558- CompareScalar<uint32_t, CMPMODE::EQ>(mask0, tmpU32MaskReg0, value, fullMask);
559- CompareScalar<uint32_t, CMPMODE::EQ>(mask1, tmpU32MaskReg1, value, fullMask);
560- CompareScalar<uint32_t, CMPMODE::EQ>(mask2, tmpU32MaskReg2, value, fullMask);
561- CompareScalar<uint32_t, CMPMODE::EQ>(mask3, tmpU32MaskReg3, value, fullMask);
562- 
563- MaskReg res;
564- TransToB8Mask(res, mask0, mask1, mask2, mask3);
565- MaskAnd(filterMask, maskReg, res, maskReg);
566-}
567- 
568-__simd_callee__ inline void FilterDataAndGivenByteFromOri(
569- MaskReg& filterMask, RegTensor<uint8_t>& colWorkBits, __ubuf__ uint16_t *src, MaskReg& maskReg,
570- uint16_t value, uint16_t byteNum, int32_t srcOffset, __ubuf__ uint16_t* tmpLocal)
571-{
572- constexpr uint32_t eleCountPerVL = GetVecLen() / sizeof(uint16_t);
573- int16_t byteOffsets = static_cast<int16_t>((byteNum - 1) * 8);
574- int16_t maskOffsets = static_cast<int16_t>(byteNum * 8);
575- 
576- MaskReg fullMask = CreateMask<uint16_t>();
577- 
578- RegTensor<uint16_t> input0, input1;
579- DataCopy(input0, src + srcOffset);
580- DataCopy(input1, src + srcOffset + eleCountPerVL);
581- 
582- RegTensor<uint16_t> tmpBShift0, tmpBShift1;
583- ShiftRights(tmpBShift0, input0, byteOffsets, fullMask);
584- ShiftRights(tmpBShift1, input1, byteOffsets, fullMask);
585- 
586- GetLowerstByte(colWorkBits, tmpBShift0, tmpBShift1);
587- 
588- RegTensor<uint16_t> tmpMShift0, tmpMShift1;
589- ShiftRights(tmpMShift0, input0, maskOffsets, fullMask);
590- ShiftRights(tmpMShift1, input1, maskOffsets, fullMask);
591- 
592- MaskReg mask0, mask1;
593- CompareScalar<uint16_t, CMPMODE::EQ>(mask0, tmpMShift0, value, fullMask);
594- CompareScalar<uint16_t, CMPMODE::EQ>(mask1, tmpMShift1, value, fullMask);
595- 
596- MaskReg res;
597- TransToB8Mask(res, mask0, mask1);
598- MaskAnd(filterMask, maskReg, res, maskReg);
599-}
600- 
601-__simd_callee__ inline void FilterDataAndGivenByteFromOri(
602- MaskReg& filterMask, RegTensor<uint8_t>& colWorkBits, __ubuf__ uint8_t *src, MaskReg& maskReg,
603- uint8_t value, uint16_t byteNum, int32_t srcOffset, __ubuf__ uint8_t* tmpLocal)
604-{
605- DataCopy(colWorkBits, src + srcOffset);
606- MaskReg fullMask = CreateMask<uint8_t>();
607- MaskAnd(filterMask, maskReg, fullMask, maskReg);
608-}
609- 
610-template <typename T>
611-__simd_vf__ inline void GenerateAccumulateData(
612- __ubuf__ T *src, __ubuf__ uint16_t *hist, __ubuf__ T *tmpSrcData,
613- uint32_t count, T value, uint32_t byteNum)
614-{
615- using ConvType = typename AscendC::Internal::ExtractTypeBySize<sizeof(T)>::T;
616- auto unsignedValue = static_cast<ConvType>(value);
617- uint16_t repeatTimes = DivCeil(count, GetVecLen());
618- ConvType workingUnsignedValue = 0;
619- if (byteNum != sizeof(T)) {
620- workingUnsignedValue = unsignedValue >> (byteNum * 8);
621- }
622- 
623- RegTensor<uint16_t> acumHistLow, acumHistHigh;
624- MaskReg b16FullMask = CreateMask<uint16_t>();
625- 
626- Duplicate(acumHistLow, 0, b16FullMask);
627- Duplicate(acumHistHigh, 0, b16FullMask);
628- 
629- for (uint16_t i = 0; i < repeatTimes; ++i) {
630- MaskReg maskReg = UpdateMask<uint8_t>(count);
631- 
632- MaskReg filterMask;
633- RegTensor<uint8_t> colWorkBits;
634- FilterDataAndGivenByteFromOri(filterMask, colWorkBits, src, maskReg, workingUnsignedValue, byteNum, i * GetVecLen(), tmpSrcData);
635- 
636- Histograms<uint8_t, uint16_t, HistogramsBinType::BIN0, HistogramsType::ACCUMULATE>(acumHistLow, colWorkBits, filterMask);
637- Histograms<uint8_t, uint16_t, HistogramsBinType::BIN1, HistogramsType::ACCUMULATE>(acumHistHigh, colWorkBits, filterMask);
638- }
639- 
640- DataCopy((__ubuf__ uint16_t *&)hist, acumHistLow, b16FullMask);
641- DataCopy((__ubuf__ uint16_t *&)hist + GetVecLen() / sizeof(uint16_t), acumHistHigh, b16FullMask);
642-}
643- 
644-__simd_vf__ inline void GatherGreaterAndEqualKData(__ubuf__ uint64_t *src, __ubuf__ uint64_t *dst, const uint64_t value, uint32_t count)
645-{
646- count *= 2;
647- constexpr uint16_t eleCountPerVL = GetVecLen() / sizeof(uint32_t);
648- uint16_t repeatTimes = DivCeil(count, eleCountPerVL);
649- 
650- auto firstCount = count;
651- auto secondCount = count;
652- 
653- uint64_t lowValue = value & 0xffffffff;
654- uint64_t highValue = value >> 32;
655- 
656- __ubuf__ uint32_t* u32Src = (__ubuf__ uint32_t*)src;
657- __ubuf__ uint32_t* u32Dst = (__ubuf__ uint32_t*)dst;
658- 
659- ClearSpr<SpecialPurposeReg::AR>();
660- MaskReg fullMask = CreateMask<uint32_t>();
661- MaskReg zeroMask = CreateMask<uint32_t, MaskPattern::ALLF>();
662- MaskReg halfMask = CreateMask<uint32_t, MaskPattern::H>();
663-
664- MaskReg lowMask, highMask, tmpMask;
665- MaskInterleave<uint32_t>(lowMask, tmpMask, fullMask, zeroMask);
666- MaskInterleave<uint32_t>(highMask, tmpMask, zeroMask, fullMask);
667- 
668- UnalignReg unalignReg;
669- for (uint16_t i = 0; i < repeatTimes; ++i) {
670- MaskReg maskReg = UpdateMask<uint32_t>(firstCount);
671- 
672- RegTensor<uint32_t> in32Data;
673- DataCopy(in32Data, u32Src + i * eleCountPerVL);
674- 
675- RegTensor<uint32_t> highPart, lowPart;
676- GatherMask(lowPart, in32Data, lowMask);
677- GatherMask(highPart, in32Data, highMask);
678- 
679- MaskReg highGTMask, highEQMask, lowPartMask;
680- CompareScalar<uint32_t, CMPMODE::GT>(highGTMask, highPart, highValue, halfMask);
681- CompareScalar<uint32_t, CMPMODE::EQ>(highEQMask, highPart, highValue, halfMask);
682- CompareScalar<uint32_t, CMPMODE::GT>(lowPartMask, lowPart, lowValue, halfMask);
683- 
684- MaskReg cmpMask;
685- MaskAnd(cmpMask, highEQMask, lowPartMask, halfMask);
686- MaskOr(cmpMask, highGTMask, cmpMask, halfMask);
687- 
688- MaskReg cmpResMask0, cmpResMask1;
689- MaskInterleave<uint32_t>(cmpResMask0, cmpResMask1, cmpMask, cmpMask);
690- MaskAnd(cmpResMask0, cmpResMask0, maskReg, maskReg);
691- 
692- RegTensor<uint32_t> out32Data;
693- GatherMask<uint32_t, GatherMaskMode::STORE_REG>(out32Data, in32Data, cmpResMask0);
694- DataCopyUnAlign<uint32_t>(u32Dst, out32Data, unalignReg);
695- }
696- 
697- for (uint16_t i = 0; i < repeatTimes; ++i) {
698- MaskReg maskReg = UpdateMask<uint32_t>(secondCount);
699- 
700- RegTensor<uint32_t> in32Data;
701- DataCopy(in32Data, u32Src + i * eleCountPerVL);
702- 
703- RegTensor<uint32_t> highPart, lowPart;
704- GatherMask(lowPart, in32Data, lowMask);
705- GatherMask(highPart, in32Data, highMask);
706- 
707- MaskReg highEQMask, lowPartMask;
708- CompareScalar<uint32_t, CMPMODE::EQ>(highEQMask, highPart, highValue, halfMask);
709- CompareScalar<uint32_t, CMPMODE::EQ>(lowPartMask, lowPart, lowValue, halfMask);
710- 
711- MaskReg cmpMask;
712- MaskAnd(cmpMask, highEQMask, lowPartMask, halfMask);
713- 
714- MaskReg cmpResMask0, cmpResMask1;
715- MaskInterleave<uint32_t>(cmpResMask0, cmpResMask1, cmpMask, cmpMask);
716- MaskAnd(cmpResMask0, cmpResMask0, maskReg, maskReg);
717- 
718- RegTensor<uint32_t> out32Data;
719- GatherMask<uint32_t, GatherMaskMode::STORE_REG>(out32Data, in32Data, cmpResMask0);
720- DataCopyUnAlign<uint32_t>(u32Dst, out32Data, unalignReg);
721- }
722- 
723- DataCopyUnAlignPost(u32Dst, unalignReg);
724- ClearSpr<SpecialPurposeReg::AR>();
725-}
726- 
727-__simd_vf__ inline void GatherGreaterAndEqualKData(__ubuf__ uint32_t *src, __ubuf__ uint32_t *dst, const uint32_t value, uint32_t count)
728-{
729- constexpr uint16_t eleCountPerVL = GetVecLen() / sizeof(uint32_t);
730- uint16_t repeatTimes = DivCeil(count, eleCountPerVL);
731- 
732- auto firstCount = count;
733- auto secondCount = count;
734- 
735- ClearSpr<SpecialPurposeReg::AR>();
736- UnalignReg unalignReg;
737- for (uint16_t i = 0; i < repeatTimes; ++i) {
738- MaskReg maskReg = UpdateMask<uint32_t>(firstCount);
739- 
740- RegTensor<uint32_t> in32Data;
741- DataCopy(in32Data, src + i * eleCountPerVL);
742- 
743- MaskReg out32Mask;
744- CompareScalar<uint32_t, CMPMODE::GT>(out32Mask, in32Data, value, maskReg);
745- 
746- RegTensor<uint32_t> out32Data;
747- GatherMask<uint32_t, GatherMaskMode::STORE_REG>(out32Data, in32Data, out32Mask);
748- DataCopyUnAlign(dst, out32Data, unalignReg);
749- }
750- 
751- for (uint16_t i = 0; i < repeatTimes; ++i) {
752- MaskReg maskReg = UpdateMask<uint32_t>(secondCount);
753- 
754- RegTensor<uint32_t> in32Data;
755- DataCopy(in32Data, src + i * eleCountPerVL);
756- 
757- MaskReg out32Mask;
758- CompareScalar<uint32_t, CMPMODE::EQ>(out32Mask, in32Data, value, maskReg);
759- 
760- RegTensor<uint32_t> out32Data;
761- GatherMask<uint32_t, GatherMaskMode::STORE_REG>(out32Data, in32Data, out32Mask);
762- DataCopyUnAlign(dst, out32Data, unalignReg);
763- }
764- 
765- DataCopyUnAlignPost(dst, unalignReg);
766- ClearSpr<SpecialPurposeReg::AR>();
767-}
768- 
769-__simd_vf__ inline void GatherGreaterAndEqualKData(__ubuf__ uint16_t *src, __ubuf__ uint16_t *dst, const uint16_t value, uint32_t count)
770-{
771- constexpr uint16_t eleCountPerVL = GetVecLen() / sizeof(uint16_t);
772- uint16_t repeatTimes = DivCeil(count, eleCountPerVL);
773- 
774- auto firstCount = count;
775- auto secondCount = count;
776- 
777- ClearSpr<SpecialPurposeReg::AR>();
778- UnalignReg unalignReg0;
779- for (uint16_t i = 0; i < repeatTimes; ++i) {
780- MaskReg maskReg = UpdateMask<uint16_t>(firstCount);
781- 
782- RegTensor<uint16_t> in32Data;
783- DataCopy(in32Data, src + i * eleCountPerVL);
784- 
785- MaskReg out32Mask;
786- CompareScalar<uint16_t, CMPMODE::GT>(out32Mask, in32Data, value, maskReg);
787- 
788- RegTensor<uint16_t> out32Data;
789- GatherMask<uint16_t, GatherMaskMode::STORE_REG>(out32Data, in32Data, out32Mask);
790- DataCopyUnAlign(dst, out32Data, unalignReg0);
791- }
792- 
793- for (uint16_t i = 0; i < repeatTimes; ++i) {
794- MaskReg maskReg = UpdateMask<uint16_t>(secondCount);
795- 
796- RegTensor<uint16_t> in32Data;
797- DataCopy(in32Data, src + i * eleCountPerVL);
798- 
799- MaskReg out32Mask;
800- CompareScalar<uint16_t, CMPMODE::EQ>(out32Mask, in32Data, value, maskReg);
801- 
802- RegTensor<uint16_t> out32Data;
803- GatherMask<uint16_t, GatherMaskMode::STORE_REG>(out32Data, in32Data, out32Mask);
804- DataCopyUnAlign(dst, out32Data, unalignReg0);
805- }
806- 
807- DataCopyUnAlignPost(dst, unalignReg0);
808- ClearSpr<SpecialPurposeReg::AR>();
809-}
810- 
811-__simd_vf__ inline void GatherGreaterAndEqualKData(__ubuf__ uint8_t *src, __ubuf__ uint8_t *dst, const uint8_t value, uint32_t count)
812-{
813- constexpr uint16_t eleCountPerVL = GetVecLen() / sizeof(uint8_t);
814- uint16_t repeatTimes = DivCeil(count, eleCountPerVL);
815- 
816- auto firstCount = count;
817- auto secondCount = count;
818- 
819- ClearSpr<SpecialPurposeReg::AR>();
820- UnalignReg unalignReg;
821- for (uint16_t i = 0; i < repeatTimes; ++i) {
822- MaskReg fullMask = UpdateMask<uint8_t>(firstCount);
823- RegTensor<uint8_t> in8Data;
824- DataCopy(in8Data, src + i * eleCountPerVL);
825- 
826- MaskReg out8Mask;
827- CompareScalar<uint8_t, CMPMODE::GT>(out8Mask, in8Data, value, fullMask);
828- 
829- RegTensor<uint8_t> out8Data;
830- GatherMask<uint8_t, GatherMaskMode::STORE_REG>(out8Data, in8Data, out8Mask);
831- DataCopyUnAlign(dst, out8Data, unalignReg);
832- }
833- 
834- for (uint16_t i = 0; i < repeatTimes; ++i) {
835- MaskReg fullMask = UpdateMask<uint8_t>(secondCount);
836- RegTensor<uint8_t> in8Data;
837- DataCopy(in8Data, src + i * eleCountPerVL);
838- 
839- MaskReg out8Mask;
840- CompareScalar<uint8_t, CMPMODE::EQ>(out8Mask, in8Data, value, fullMask);
841- 
842- RegTensor<uint8_t> out8Data;
843- GatherMask<uint8_t, GatherMaskMode::STORE_REG>(out8Data, in8Data, out8Mask);
844- DataCopyUnAlign(dst, out8Data, unalignReg);
845- }
846- 
847- DataCopyUnAlignPost(dst, unalignReg);
848- ClearSpr<SpecialPurposeReg::AR>();
849-}
850- 
851- 
852-__simd_vf__ inline void GatherGreaterAndEqualKIndex(__ubuf__ uint64_t *src, __ubuf__ int32_t *inputIndex,
853- __ubuf__ int32_t *dstIndex, const uint64_t value, uint32_t count)
854-{
855- count *= 2;
856- 
857- constexpr uint16_t eleCountPerVL = GetVecLen() / sizeof(uint32_t);
858- uint16_t repeatTimes = DivCeil(count, eleCountPerVL);
859- constexpr uint16_t numsPerRound = eleCountPerVL / 2;
860- 
861- auto firstCount = count;
862- auto secondCount = count;
863- 
864- uint64_t lowValue = value & 0xffffffff;
865- uint64_t highValue = value >> 32;
866- 
867- __ubuf__ uint32_t* u32Src = (__ubuf__ uint32_t*)src;
868- 
869- MaskReg fullMask = CreateMask<uint32_t>();
870- MaskReg zeroMask = CreateMask<uint32_t, MaskPattern::ALLF>();
871- MaskReg halfMask = CreateMask<uint32_t, MaskPattern::H>();
872-
873- MaskReg lowMask, highMask, tmpMask;
874- MaskInterleave<uint32_t>(lowMask, tmpMask, fullMask, zeroMask);
875- MaskInterleave<uint32_t>(highMask, tmpMask, zeroMask, fullMask);
876- 
877- UnalignReg unalignReg;
878- for (uint16_t i = 0; i < repeatTimes; ++i) {
879- MaskReg maskReg = UpdateMask<uint32_t>(firstCount);
880- MaskReg indexMask, tmpMask;
881- MaskDeInterleave<uint32_t>(indexMask, tmpMask, maskReg, zeroMask);
882- 
883- RegTensor<uint32_t> in32Data;
884- DataCopy(in32Data, u32Src + i * eleCountPerVL);
885- 
886- RegTensor<uint32_t> highPart, lowPart;
887- GatherMask(lowPart, in32Data, lowMask);
888- GatherMask(highPart, in32Data, highMask);
889- 
890- MaskReg highGTMask, highEQMask, lowPartMask;
891- CompareScalar<uint32_t, CMPMODE::GT>(highGTMask, highPart, highValue, halfMask);
892- CompareScalar<uint32_t, CMPMODE::EQ>(highEQMask, highPart, highValue, halfMask);
893- CompareScalar<uint32_t, CMPMODE::GT>(lowPartMask, lowPart, lowValue, halfMask);
894- 
895- MaskReg cmpMask;
896- MaskAnd(cmpMask, highEQMask, lowPartMask, halfMask);
897- MaskOr(cmpMask, highGTMask, cmpMask, halfMask);
898- MaskAnd(cmpMask, cmpMask, indexMask, indexMask);
899- 
900- RegTensor<int32_t> index;
901- DataCopy(index, inputIndex + i * numsPerRound);
902- 
903- RegTensor<int32_t> outIndex;
904- GatherMask<int32_t, GatherMaskMode::STORE_REG>(outIndex, index, cmpMask);
905- DataCopyUnAlign<int32_t, PostLiteral::POST_MODE_UPDATE>(dstIndex, outIndex, unalignReg);
906- }
907- 
908- for (uint16_t i = 0; i < repeatTimes; ++i) {
909- MaskReg maskReg = UpdateMask<uint32_t>(secondCount);
910- MaskReg indexMask, tmpMask;
911- MaskDeInterleave<uint32_t>(indexMask, tmpMask, maskReg, zeroMask);
912- 
913- RegTensor<uint32_t> in32Data;
914- DataCopy(in32Data, u32Src + i * eleCountPerVL);
915- 
916- RegTensor<uint32_t> highPart, lowPart;
917- GatherMask(lowPart, in32Data, lowMask);
918- GatherMask(highPart, in32Data, highMask);
919- 
920- MaskReg highEQMask, lowPartMask;
921- CompareScalar<uint32_t, CMPMODE::EQ>(highEQMask, highPart, highValue, halfMask);
922- CompareScalar<uint32_t, CMPMODE::EQ>(lowPartMask, lowPart, lowValue, halfMask);
923- 
924- MaskReg cmpMask;
925- MaskAnd(cmpMask, highEQMask, lowPartMask, halfMask);
926- MaskAnd(cmpMask, cmpMask, indexMask, indexMask);
927- 
928- RegTensor<int32_t> index;
929- DataCopy(index, inputIndex + i * numsPerRound);
930- 
931- RegTensor<int32_t> outIndex;
932- GatherMask<int32_t, GatherMaskMode::STORE_REG>(outIndex, index, cmpMask);
933- DataCopyUnAlign<int32_t, PostLiteral::POST_MODE_UPDATE>(dstIndex, outIndex, unalignReg);
934- }
935- 
936- DataCopyUnAlignPost(dstIndex, unalignReg);
937- ClearSpr<SpecialPurposeReg::AR>();
938-}
939- 
940-__simd_vf__ inline void GatherGreaterAndEqualKIndex(__ubuf__ uint32_t *src, __ubuf__ int32_t *inputIndex,
941- __ubuf__ int32_t *dstIndex, const uint32_t value, uint32_t count)
942-{
943- constexpr uint32_t eleCountPerVL = GetVecLen() / sizeof(uint32_t);
944- uint16_t repeatTimes = DivCeil(count, eleCountPerVL);
945- 
946- auto firstCount = count;
947- auto secondCount = count;
948- 
949- UnalignReg unalignReg;
950- for (uint16_t i = 0; i < repeatTimes; ++i) {
951- MaskReg maskReg = UpdateMask<int32_t>(firstCount);
952- RegTensor<uint32_t> in32Data;
953- DataCopy(in32Data, src + i * eleCountPerVL);
954- 
955- MaskReg out32Mask;
956- CompareScalar<uint32_t, CMPMODE::GT>(out32Mask, in32Data, value, maskReg);
957- 
958- RegTensor<int32_t> index;
959- DataCopy(index, inputIndex + i * eleCountPerVL);
960- 
961- RegTensor<int32_t> outIndex;
962- GatherMask<int32_t, GatherMaskMode::STORE_REG>(outIndex, index, out32Mask);
963- DataCopyUnAlign<int32_t, PostLiteral::POST_MODE_UPDATE>(dstIndex, outIndex, unalignReg);
964- }
965- 
966- for (uint16_t i = 0; i < repeatTimes; ++i) {
967- MaskReg maskReg = UpdateMask<int32_t>(secondCount);
968- RegTensor<uint32_t> in32Data;
969- DataCopy(in32Data, src + i * eleCountPerVL);
970- 
971- MaskReg out32Mask;
972- CompareScalar<uint32_t, CMPMODE::EQ>(out32Mask, in32Data, value, maskReg);
973- 
974- RegTensor<int32_t> index;
975- DataCopy(index, inputIndex + i * eleCountPerVL);
976- 
977- RegTensor<int32_t> outIndex;
978- GatherMask<int32_t, GatherMaskMode::STORE_REG>(outIndex, index, out32Mask);
979- DataCopyUnAlign<int32_t, PostLiteral::POST_MODE_UPDATE>(dstIndex, outIndex, unalignReg);
980- }
981- 
982- DataCopyUnAlignPost(dstIndex, unalignReg);
983- ClearSpr<SpecialPurposeReg::AR>();
984-}
985- 
986-__simd_vf__ inline void GatherGreaterAndEqualKIndex(__ubuf__ uint16_t *src, __ubuf__ int32_t *inputIndex,
987- __ubuf__ int32_t *dstIndex, const uint16_t value, uint32_t count)
988-{
989- constexpr uint32_t u32EleCountPerVL = GetVecLen() / sizeof(int32_t);
990- constexpr uint32_t u16EleCountPerVL = GetVecLen() / sizeof(uint16_t);
991- uint16_t dataRepeatTimes = DivCeil(count, u16EleCountPerVL);
992- uint16_t indexRepeatTImes = DivCeil(count, u32EleCountPerVL);
993- 
994- auto firstCount = count;
995- auto secondCount = count;
996- 
997- UnalignReg unalignReg1;
998- for (uint16_t i = 0; i < dataRepeatTimes; ++i) {
999- MaskReg maskReg = UpdateMask<uint16_t>(firstCount);
1000- RegTensor<uint16_t> in16Data;
1001- DataCopy(in16Data, src + i * u16EleCountPerVL);
1002- 
1003- MaskReg out16Mask;
1004- CompareScalar<uint16_t, CMPMODE::GT>(out16Mask, in16Data, value, maskReg);
1005- 
1006- MaskReg zero16Mask = CreateMask<uint16_t, MaskPattern::ALLF>();
1007- MaskReg out32Mask0, out32Mask1;
1008- MaskInterleave<uint16_t>(out32Mask0, out32Mask1, out16Mask, zero16Mask);
1009- 
1010- RegTensor<int32_t> index0, index1;
1011- DataCopy(index0, inputIndex + i * u16EleCountPerVL);
1012- DataCopy(index1, inputIndex + i * u16EleCountPerVL + u32EleCountPerVL);
1013- 
1014- RegTensor<int32_t> outIndex0, outIndex1;
1015- GatherMask<int32_t, GatherMaskMode::STORE_REG>(outIndex0, index0, out32Mask0);
1016- DataCopyUnAlign<int32_t, PostLiteral::POST_MODE_UPDATE>(dstIndex, outIndex0, unalignReg1);
1017- GatherMask<int32_t, GatherMaskMode::STORE_REG>(outIndex1, index1, out32Mask1);
1018- DataCopyUnAlign<int32_t, PostLiteral::POST_MODE_UPDATE>(dstIndex, outIndex1, unalignReg1);
1019- }
1020- 
1021- for (uint16_t i = 0; i < dataRepeatTimes; ++i) {
1022- MaskReg maskReg = UpdateMask<uint16_t>(secondCount);
1023- RegTensor<uint16_t> in16Data;
1024- DataCopy(in16Data, src + i * u16EleCountPerVL);
1025- 
1026- MaskReg out16Mask;
1027- CompareScalar<uint16_t, CMPMODE::EQ>(out16Mask, in16Data, value, maskReg);
1028- 
1029- MaskReg zero16Mask = CreateMask<uint16_t, MaskPattern::ALLF>();
1030- MaskReg out32Mask0, out32Mask1;
1031- MaskInterleave<uint16_t>(out32Mask0, out32Mask1, out16Mask, zero16Mask);
1032- 
1033- RegTensor<int32_t> index0, index1;
1034- DataCopy(index0, inputIndex + i * u16EleCountPerVL );
1035- DataCopy(index1, inputIndex + i * u16EleCountPerVL + u32EleCountPerVL);
1036- 
1037- RegTensor<int32_t> outIndex0, outIndex1;
1038- GatherMask<int32_t, GatherMaskMode::STORE_REG>(outIndex0, index0, out32Mask0);
1039- DataCopyUnAlign<int32_t, PostLiteral::POST_MODE_UPDATE>(dstIndex, outIndex0, unalignReg1);
1040- GatherMask<int32_t, GatherMaskMode::STORE_REG>(outIndex1, index1, out32Mask1);
1041- DataCopyUnAlign<int32_t, PostLiteral::POST_MODE_UPDATE>(dstIndex, outIndex1, unalignReg1);
1042- }
1043- 
1044- DataCopyUnAlignPost(dstIndex, unalignReg1);
1045- ClearSpr<SpecialPurposeReg::AR>();
1046-}
1047- 
1048-__simd_vf__ inline void GatherGreaterAndEqualKIndex(__ubuf__ uint8_t *src, __ubuf__ int32_t *inputIndex,
1049- __ubuf__ int32_t *dstIndex, const uint8_t value, uint32_t count)
1050-{
1051- constexpr uint16_t u8EleCountPerVL = GetVecLen() / sizeof(uint8_t);
1052- constexpr uint32_t u32EleCountPerVL = GetVecLen() / sizeof(int32_t);
1053- 
1054- uint16_t dataRepeatTimes = DivCeil(count, u8EleCountPerVL);
1055- uint16_t indexRepeatTImes = DivCeil(count, u32EleCountPerVL);
1056- 
1057- auto firstCount = count;
1058- auto secondCount = count;
1059- 
1060- UnalignReg unalignReg;
1061- for (uint16_t i = 0; i < dataRepeatTimes; ++i) {
1062- MaskReg maskReg = UpdateMask<uint8_t>(firstCount);
1063- RegTensor<uint8_t> in8Data;
1064- DataCopy(in8Data, src + i * u8EleCountPerVL);
1065- 
1066- MaskReg out8Mask;
1067- CompareScalar<uint8_t, CMPMODE::GT>(out8Mask, in8Data, value, maskReg);
1068- 
1069- MaskReg zero8Mask = CreateMask<uint8_t, MaskPattern::ALLF>();
1070- MaskReg out16Mask0, out16Mask1;
1071- MaskInterleave<uint8_t>(out16Mask0, out16Mask1, out8Mask, zero8Mask);
1072- 
1073- MaskReg zero16Mask = CreateMask<uint16_t, MaskPattern::ALLF>();
1074- MaskReg out32Mask0, out32Mask1, out32Mask2, out32Mask3;
1075- MaskInterleave<uint16_t>(out32Mask0, out32Mask1, out16Mask0, zero16Mask);
1076- MaskInterleave<uint16_t>(out32Mask2, out32Mask3, out16Mask1, zero16Mask);
1077- 
1078- RegTensor<int32_t> index0, index1, index2, index3;
1079- DataCopy(index0, inputIndex + i * GetVecLen());
1080- DataCopy(index1, inputIndex + i * GetVecLen() + u32EleCountPerVL);
1081- DataCopy(index2, inputIndex + i * GetVecLen() + u32EleCountPerVL * 2);
1082- DataCopy(index3, inputIndex + i * GetVecLen() + u32EleCountPerVL * 3);
1083- 
1084- RegTensor<int32_t> outIndex0, outIndex1, outIndex2, outIndex3;
1085- GatherMask<int32_t, GatherMaskMode::STORE_REG>(outIndex0, index0, out32Mask0);
1086- DataCopyUnAlign<int32_t, PostLiteral::POST_MODE_UPDATE>(dstIndex, outIndex0, unalignReg);
1087- GatherMask<int32_t, GatherMaskMode::STORE_REG>(outIndex1, index1, out32Mask1);
1088- DataCopyUnAlign<int32_t, PostLiteral::POST_MODE_UPDATE>(dstIndex, outIndex1, unalignReg);
1089- GatherMask<int32_t, GatherMaskMode::STORE_REG>(outIndex2, index2, out32Mask2);
1090- DataCopyUnAlign<int32_t, PostLiteral::POST_MODE_UPDATE>(dstIndex, outIndex2, unalignReg);
1091- GatherMask<int32_t, GatherMaskMode::STORE_REG>(outIndex3, index3, out32Mask3);
1092- DataCopyUnAlign<int32_t, PostLiteral::POST_MODE_UPDATE>(dstIndex, outIndex3, unalignReg);
1093- }
1094- 
1095- for (uint16_t i = 0; i < dataRepeatTimes; ++i) {
1096- MaskReg maskReg = UpdateMask<uint8_t>(secondCount);
1097- RegTensor<uint8_t> in8Data;
1098- DataCopy(in8Data, src + i * u8EleCountPerVL);
1099- 
1100- MaskReg out8Mask;
1101- CompareScalar<uint8_t, CMPMODE::EQ>(out8Mask, in8Data, value, maskReg);
1102- 
1103- MaskReg zero8Mask = CreateMask<uint8_t, MaskPattern::ALLF>();
1104- MaskReg out16Mask0, out16Mask1;
1105- MaskInterleave<uint8_t>(out16Mask0, out16Mask1, out8Mask, zero8Mask);
1106- 
1107- MaskReg zero16Mask = CreateMask<uint16_t, MaskPattern::ALLF>();
1108- MaskReg out32Mask0, out32Mask1, out32Mask2, out32Mask3;
1109- MaskInterleave<uint16_t>(out32Mask0, out32Mask1, out16Mask0, zero16Mask);
1110- MaskInterleave<uint16_t>(out32Mask2, out32Mask3, out16Mask1, zero16Mask);
1111- 
1112- RegTensor<int32_t> index0, index1, index2, index3;
1113- DataCopy(index0, inputIndex + i * GetVecLen());
1114- DataCopy(index1, inputIndex + i * GetVecLen() + u32EleCountPerVL);
1115- DataCopy(index2, inputIndex + i * GetVecLen() + u32EleCountPerVL * 2);
1116- DataCopy(index3, inputIndex + i * GetVecLen() + u32EleCountPerVL * 3);
1117- 
1118- RegTensor<int32_t> outIndex0, outIndex1, outIndex2, outIndex3;
1119- GatherMask<int32_t, GatherMaskMode::STORE_REG>(outIndex0, index0, out32Mask0);
1120- DataCopyUnAlign<int32_t, PostLiteral::POST_MODE_UPDATE>(dstIndex, outIndex0, unalignReg);
1121- GatherMask<int32_t, GatherMaskMode::STORE_REG>(outIndex1, index1, out32Mask1);
1122- DataCopyUnAlign<int32_t, PostLiteral::POST_MODE_UPDATE>(dstIndex, outIndex1, unalignReg);
1123- GatherMask<int32_t, GatherMaskMode::STORE_REG>(outIndex2, index2, out32Mask2);
1124- DataCopyUnAlign<int32_t, PostLiteral::POST_MODE_UPDATE>(dstIndex, outIndex2, unalignReg);
1125- GatherMask<int32_t, GatherMaskMode::STORE_REG>(outIndex3, index3, out32Mask3);
1126- DataCopyUnAlign<int32_t, PostLiteral::POST_MODE_UPDATE>(dstIndex, outIndex3, unalignReg);
1127- }
1128- 
1129- DataCopyUnAlignPost(dstIndex, unalignReg);
1130- ClearSpr<SpecialPurposeReg::AR>();
1131-}
1132- 
1133-template <typename T>
1134-__simd_callee__ inline int32_t GetKPad(int32_t k)
1135-{
1136- constexpr int32_t alignBytes = 32;
1137- return AlignUp(k, alignBytes / sizeof(T));
1138-}
1139- 
1140-template <typename T>
1141-__simd_vf__ inline void SaveData(__ubuf__ T *dst, __ubuf__ int32_t *dstIndex, __ubuf__ T *src, __ubuf__ int32_t *srcIndex,
1142- const uint32_t count)
1143-{
1144- constexpr uint32_t dataCountPerTime = GetVecLen() / sizeof(T);
1145- uint16_t dataRepeatTimes = DivCeil(count, dataCountPerTime);
1146- constexpr uint32_t indexCountPerTime = GetVecLen() / sizeof(int32_t);
1147- uint16_t indexRepeatTimes = DivCeil(count, indexCountPerTime);
1148- 
1149- uint32_t dCount = count;
1150- uint32_t iCount = count;
1151- 
1152- for (uint16_t i = 0; i < dataRepeatTimes; ++i) {
1153- MaskReg maskReg = UpdateMask<T>(dCount);
1154- RegTensor<T> reg;
1155- DataCopy<T, PostLiteral::POST_MODE_UPDATE>(reg, (__ubuf__ T *&)src, dataCountPerTime);
1156- DataCopy<T, PostLiteral::POST_MODE_UPDATE>((__ubuf__ T *&)dst, reg, dataCountPerTime, maskReg);
1157- }
1158- 
1159- for (uint16_t i = 0; i < indexRepeatTimes; ++i) {
1160- MaskReg maskReg = UpdateMask<int32_t>(iCount);
1161- RegTensor<int32_t> reg;
1162- DataCopy<int32_t, PostLiteral::POST_MODE_UPDATE>(reg, (__ubuf__ int32_t *&)srcIndex, indexCountPerTime);
1163- DataCopy<int32_t, PostLiteral::POST_MODE_UPDATE>((__ubuf__ int32_t *&)dstIndex, reg, indexCountPerTime, maskReg);
1164- }
1165-}
1166- 
1167-template <typename T>
1168-__simd_vf__ inline void SaveDataUnAlignVF(__ubuf__ T *dst, __ubuf__ int32_t *dstIndex,
1169- __ubuf__ T *src, __ubuf__ int32_t *srcIndex, const TopKInfo topKInfo, const uint32_t k)
1170-{
1171- uint32_t dataMainCountPerTime = GetVecLen() / sizeof(T);
1172- uint32_t indexMainCountPerTime = GetVecLen() / sizeof(int32_t);
1173- 
1174- int32_t kPad = GetKPad<T>(k);
1175- int32_t kIndexPad = GetKPad<int32_t>(k);
1176- uint16_t dataMainRepeatTime = DivCeil(kPad, dataMainCountPerTime);
1177- uint16_t indexMainRepeatTime = DivCeil(kIndexPad, indexMainCountPerTime);
1178- 
1179- for (uint16_t j = 0; j < static_cast<uint16_t>(topKInfo.outter); ++j) {
1180- for (uint16_t i = 0; i < dataMainRepeatTime; i++) {
1181- UnalignReg ureg;
1182- RegTensor<T> reg;
1183- auto dstUBT = dst + j * k + i * kPad;
1184- DataCopy(reg, src + j * kPad + i * kPad);
1185- DataCopyUnAlign((__ubuf__ T *&)dstUBT, reg, ureg, k);
1186- DataCopyUnAlignPost(dstUBT, ureg, 0);
1187- }
1188- 
1189- for (uint16_t i = 0; i < indexMainRepeatTime; i++) {
1190- UnalignReg ureg;
1191- RegTensor<int32_t> reg;
1192- auto dstUBT = dstIndex + j * k + i * kIndexPad;
1193- DataCopy(reg, srcIndex + j * kIndexPad + i * kIndexPad);
1194- DataCopyUnAlign((__ubuf__ int32_t *&)dstUBT, reg, ureg, k);
1195- DataCopyUnAlignPost(dstUBT, ureg, 0);
1196- }
1197- }
1198-}
1199- 
1200-template <typename T>
1201-__aicore__ inline void SaveDataUnAlign(const LocalTensor<T> &dstValueLocal, const LocalTensor<int32_t> &dstIndexLocal,
1202- const LocalTensor<T> &srcLocal, const LocalTensor<int32_t> &srcIndexLocal, const TopKInfo &topKInfo, const uint32_t k)
1203-{
1204- __ubuf__ T *src = (__ubuf__ T *)srcLocal.GetPhyAddr();
1205- __ubuf__ int32_t *srcIndex = (__ubuf__ int32_t *)srcIndexLocal.GetPhyAddr();
1206- __ubuf__ T *dst = (__ubuf__ T *)dstValueLocal.GetPhyAddr();
1207- __ubuf__ int32_t *dstIndex = (__ubuf__ int32_t *)dstIndexLocal.GetPhyAddr();
1208- 
1209- SaveDataUnAlignVF<T>(dst, dstIndex, src, srcIndex, topKInfo, k);
1210-}
1211- 
1212-template <typename T, bool isInitIndex = false, bool isReuseSrc = false, const TopKConfig &config = defaultTopKConfig>
1213-__aicore__ inline void TopKRaidxSelect(const LocalTensor<T> &dstValueLocal, const LocalTensor<int32_t> &dstIndexLocal,
1214- const LocalTensor<T> &srcLocal, const LocalTensor<int32_t> &srcIndexLocal, const LocalTensor<T> &tempBuffer,
1215- const int32_t k, const TopkTiling &tilling, const TopKInfo &topKInfo, const bool isLargest)
1216-{
1217- using ConvType = typename AscendC::Internal::ExtractTypeBySize<sizeof(T)>::T;
1218- 
1219- constexpr bool isOrderFromTemplate = config.order != TopKOrder::UNSET;
1220- constexpr bool isLargestInTemplate = config.order == TopKOrder::LARGEST;
1221- 
1222- __ubuf__ ConvType *src = (__ubuf__ ConvType *)srcLocal.GetPhyAddr();
1223- __ubuf__ int32_t *srcIndex = (__ubuf__ int32_t *)srcIndexLocal.GetPhyAddr();
1224- __ubuf__ ConvType *dst = (__ubuf__ ConvType *)dstValueLocal.GetPhyAddr();
1225- __ubuf__ int32_t *dstIndex = (__ubuf__ int32_t *)dstIndexLocal.GetPhyAddr();
1226- __ubuf__ uint8_t *tmp = (__ubuf__ uint8_t *)tempBuffer.GetPhyAddr();
1227- 
1228- // temp data for storing values which are greater and equal than the topk value, same size with src
1229- __ubuf__ ConvType *tmpSrcData;
1230- // temp data for storing indexes related tmpSrcData, same size with srcIndex
1231- __ubuf__ int32_t *tmpSrcIndex;
1232- // temp data for storing accumulate data, 512B
1233- __ubuf__ uint16_t *tmpHistData;
1234- // temp data for storing work src values
1235- __ubuf__ ConvType *realWorkData = src;
1236- // temp data for sort
1237- __ubuf__ ConvType *sortTmpBuffer;
1238- 
1239- uint32_t count = topKInfo.inner;
1240- uint32_t realCount = topKInfo.n;
1241- 
1242- if constexpr (isOrderFromTemplate) {
1243- InitializeTempBuffer<T, isReuseSrc, isLargestInTemplate>(tempBuffer, count, (__ubuf__ T*&)tmpSrcData,
1244- tmpSrcIndex, tmpHistData, (__ubuf__ T*&)realWorkData, (__ubuf__ T*&)sortTmpBuffer);
1245- 
1246- if constexpr (NeedPreProcess<T, isLargestInTemplate>()) {
1247- Preprocess<T, ConvType, isLargestInTemplate>((__ubuf__ T*)src, realWorkData, count);
1248- }
1249- } else {
1250- InitializeTempBuffer<T, isReuseSrc>(tempBuffer, isLargest, count, (__ubuf__ T*&)tmpSrcData, tmpSrcIndex,
1251- tmpHistData, (__ubuf__ T*&)realWorkData, (__ubuf__ T*&)sortTmpBuffer);
1252- 
1253- if (NeedPreProcess<T>(isLargest)) {
1254- Preprocess<T, ConvType>((__ubuf__ T*)src, realWorkData, isLargest, count);
1255- }
1256- }
1257- 
1258- constexpr uint16_t typeBytes = sizeof(T);
1259- int32_t remainK = k;
1260- ConvType kthValue = 0;
1261- event_t eventVS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S));
1262- 
1263- for (uint16_t i = typeBytes; i > 0 && remainK > 0; --i) {
1264- GenerateAccumulateData<ConvType>(realWorkData, tmpHistData, tmpSrcData, realCount, kthValue, i);
1265- 
1266- SetFlag<HardEvent::V_S>(eventVS);
1267- WaitFlag<HardEvent::V_S>(eventVS);
1268-
1269- int32_t expValue = tmpHistData[255] - remainK;
1270- int16_t left = 0;
1271- int16_t right = 255;
1272- bool found = false;
1273- while (left <= right) {
1274- int16_t mid = left + (right - left) / 2;
1275- if (tmpHistData[mid] == expValue) {
1276- kthValue |= (static_cast<ConvType>(mid + 1) << ((i - 1) * 8));
1277- remainK = 0;
1278- found = true;
1279- break;
1280- } else if (tmpHistData[mid] > expValue) {
1281- right = mid - 1;
1282- } else {
1283- left = mid + 1;
1284- }
1285- }
1286- if (!found) {
1287- if (right >= 0){
1288- kthValue |= (static_cast<ConvType>(right + 1) << ((i - 1) * 8));
1289- remainK -= (tmpHistData[255] - tmpHistData[right + 1]);
1290- } else {
1291- remainK = tmpHistData[0] - expValue;
1292- }
1293- }
1294- }
1295- 
1296- GatherGreaterAndEqualKData(realWorkData, tmpSrcData, kthValue, realCount);
1297- GatherGreaterAndEqualKIndex(realWorkData, srcIndex, tmpSrcIndex, kthValue, realCount);
1298- 
1299- if constexpr (config.sorted) {
1300- static constexpr SortConfig sortConfig = {SortType::RADIX_SORT, true};
1301- 
1302- LocalTensor<ConvType> valueTensor = tempBuffer.template ReinterpretCast<ConvType>();
1303- LocalTensor<ConvType> sortDataSrc = valueTensor[(__ubuf__ T*)tmpSrcData - (__ubuf__ T*)tmp];
1304- 
1305- LocalTensor<int32_t> indexTensor = tempBuffer.template ReinterpretCast<int32_t>();
1306- LocalTensor<int32_t> sortIndexSrc = indexTensor[(__ubuf__ int32_t*)tmpSrcIndex - (__ubuf__ int32_t*)tmp];
1307- 
1308- LocalTensor<uint8_t> tmpTensor = tempBuffer.template ReinterpretCast<uint8_t>();
1309- LocalTensor<uint8_t> sortBufferTensor = tmpTensor[(__ubuf__ uint8_t*)sortTmpBuffer - (__ubuf__ uint8_t*)tmp];
1310- 
1311- LocalTensor<ConvType> dstValueTensor = dstValueLocal.template ReinterpretCast<ConvType>();
1312- LocalTensor<int32_t> dstIndexTensor = dstIndexLocal.template ReinterpretCast<int32_t>();
1313- Sort<ConvType, int32_t, false, sortConfig>(dstValueTensor, dstIndexTensor, sortDataSrc, sortIndexSrc, sortBufferTensor, static_cast<uint32_t>(k));
1314- } else {
1315- SaveData<T>((__ubuf__ T*)dst, dstIndex, (__ubuf__ T*)tmpSrcData, tmpSrcIndex, static_cast<uint32_t>(k));
1316- }
1317- 
1318- if constexpr (isOrderFromTemplate) {
1319- if constexpr (NeedPreProcess<T, isLargestInTemplate>()) {
1320- PostProcess<T, ConvType, isLargestInTemplate>(dst, dst, k);
1321- }
1322- } else {
1323- if (NeedPreProcess<T>(isLargest)) {
1324- PostProcess<T, ConvType>(dst, dst, isLargest, k);
1325- }
1326- }
1327-}
1328- 
1329-template <typename T, bool isInitIndex = false, bool isHasfinish = false, bool isReuseSrc = false, const TopKConfig &config = defaultTopKConfig>
1330-__aicore__ inline void TopKNormal(const LocalTensor<T> &dstValueLocal, const LocalTensor<int32_t> &dstIndexLocal,
1331- const LocalTensor<T> &srcLocal, const LocalTensor<int32_t> &srcIndexLocal, const LocalTensor<bool> &finishLocal,
1332- const LocalTensor<T> &tempBuffer, const int32_t k, const TopkTiling &tilling, const TopKInfo &topKInfo,
1333- const bool isLargest = true)
1334-{
1335- // if isInitIndex is false, The index of the input data needs to be generated here.
1336- LocalTensor<int32_t> realIndexSrc(srcIndexLocal);
1337- uint32_t initIndexTempBufferSize = 0;
1338- if constexpr (!isInitIndex) {
1339- LocalTensor<int32_t> indexSrcTmp = tempBuffer.template ReinterpretCast<int32_t>();
1340- CreateVecIndex(indexSrcTmp, static_cast<int32_t>(0), topKInfo.inner);
1341- realIndexSrc = indexSrcTmp;
1342- initIndexTempBufferSize += sizeof(int32_t) * topKInfo.inner / sizeof(T);
1343- }
1344- 
1345- int32_t kPad = GetKPad<T>(k);
1346- int32_t indexKPad = GetKPad<int32_t>(k);
1347- for (int32_t i = 0; i < topKInfo.outter; ++i) {
1348- TopKRaidxSelect<T, isInitIndex, isReuseSrc, config>(dstValueLocal[i * kPad], dstIndexLocal[i * indexKPad],
1349- srcLocal[i * topKInfo.inner], realIndexSrc, tempBuffer[initIndexTempBufferSize], k, tilling, topKInfo, isLargest);
1350- }
1351-}
1352- 
1353-template <typename T, bool isInitIndex = false, bool isHasfinish = false, bool isReuseSrc = false, const TopKConfig &config = defaultTopKConfig>
1354-__aicore__ inline void TopKNSmall(const LocalTensor<T> &dstValueLocal, const LocalTensor<int32_t> &dstIndexLocal,
1355- const LocalTensor<T> &srcLocal, const LocalTensor<int32_t> &srcIndexLocal, const LocalTensor<bool> &finishLocal,
1356- const LocalTensor<T> &tempBuffer, const int32_t k, const TopkTiling &tilling, const TopKInfo &topKInfo,
1357- const bool isLargest = true)
1358-{
1359- // if isInitIndex is false, The index of the input data needs to be generated here.
1360- LocalTensor<int32_t> realIndexSrc(srcIndexLocal);
1361- LocalTensor<T> tmpDstValueLocal(dstValueLocal);
1362- LocalTensor<int32_t> tmpDstIndexLocal(dstIndexLocal);
1363- 
1364- uint32_t initIndexTempBufferSize = 0;
1365- if constexpr (!isInitIndex) {
1366- LocalTensor<int32_t> indexLocalTmp = tempBuffer.template ReinterpretCast<int32_t>();
1367- CreateVecIndex(indexLocalTmp, static_cast<int32_t>(0), topKInfo.inner);
1368- if (topKInfo.outter > 1) {
1369- Copy(indexLocalTmp[topKInfo.inner], indexLocalTmp, topKInfo.inner, topKInfo.outter - 1, {1, 1, 4, 0});
1370- }
1371- realIndexSrc = indexLocalTmp;
1372- initIndexTempBufferSize += sizeof(int32_t) * (topKInfo.inner * topKInfo.outter) / sizeof(T);
1373- }
1374- 
1375- int32_t kPad = GetKPad<T>(k);
1376- int32_t indexKPad = GetKPad<int32_t>(k);
1377- event_t eventVS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S));
1378- for (int32_t i = 0; i < topKInfo.outter; ++i) {
1379- TopKRaidxSelect<T, isInitIndex, isReuseSrc, config>(tmpDstValueLocal[i * kPad], tmpDstIndexLocal[i * indexKPad],
1380- srcLocal[i * topKInfo.inner], realIndexSrc[i * topKInfo.inner], tempBuffer[initIndexTempBufferSize],
1381- k, tilling, topKInfo, isLargest);
1382- }
1383- SetFlag<HardEvent::V_S>(eventVS);
1384- WaitFlag<HardEvent::V_S>(eventVS);
1385- SaveDataUnAlign<T>(dstValueLocal, dstIndexLocal, tmpDstValueLocal, tmpDstIndexLocal, topKInfo, k);
1386-}
1387- 
1388-} // namespace RadixSelectTopK
1389-} // namespace MicroAPI
1390- 
1391-template <typename T, bool isInitIndex = false, bool isHasfinish = false, bool isReuseSrc = false>
1392-__aicore__ inline void TopKNormal(const LocalTensor<T> &dstValueLocal, const LocalTensor<int32_t> &dstIndexLocal,
1393- const LocalTensor<T> &srcLocal, const LocalTensor<int32_t> &srcIndexLocal, const LocalTensor<bool> &finishLocal,
1394- const LocalTensor<T> &tempBuffer, const int32_t k, const TopkTiling &tilling, const TopKInfo &topKInfo,
1395- const bool isLargest = true)
1396-{
1397- // if isInitIndex is false, The index of the input data needs to be generated here.
1398- if constexpr (!isInitIndex) {
1399- LocalTensor<int32_t> indexLocalTmp = tempBuffer[tilling.srcIndexOffset].template ReinterpretCast<int32_t>();
1400- CreateVecIndex(indexLocalTmp, static_cast<int32_t>(0), topKInfo.inner);
1401- }
1402- if (!isLargest) {
1403- Muls(srcLocal, srcLocal, T(-1), topKInfo.outter * topKInfo.inner);
1404- }
1405- 
1406- TopKCompute<T, isInitIndex, isHasfinish>(dstValueLocal, dstIndexLocal, srcLocal, srcIndexLocal,
1407- finishLocal, tempBuffer, k, tilling, topKInfo, isLargest);
1408- 
1409- if (!isLargest) {
1410- Muls(dstValueLocal, dstValueLocal, T(-1), tilling.maskOffset);
1411- Muls(srcLocal, srcLocal, T(-1), topKInfo.outter * topKInfo.inner);
1412- }
1413-}
1414- 
1415-template <typename T, bool isInitIndex = false, bool isHasfinish = false, bool isReuseSrc = false>
1416-__aicore__ inline void TopKNSmall(const LocalTensor<T> &dstValueLocal, const LocalTensor<int32_t> &dstIndexLocal,
1417- const LocalTensor<T> &srcLocal, const LocalTensor<int32_t> &srcIndexLocal, const LocalTensor<bool> &finishLocal,
1418- const LocalTensor<T> &tempBuffer, const int32_t k, const TopkTiling &tilling, const TopKInfo &topKInfo,
1419- const bool isLargest = true)
1420-{
1421- // if isInitIndex is false, The index of the input data needs to be generated here.
1422- if constexpr (!isInitIndex) {
1423- LocalTensor<int32_t> indexLocalTmp = tempBuffer[tilling.innerDataSize].template
1424- ReinterpretCast<int32_t>();
1425- CreateVecIndex(indexLocalTmp, static_cast<int32_t>(0), topKInfo.inner);
1426- if (topKInfo.outter > 1) {
1427- Copy(indexLocalTmp[topKInfo.inner], indexLocalTmp, topKInfo.inner, topKInfo.outter - 1, {1, 1, 4, 0});
1428- }
1429- }
1430- // if isLargest if false, sort Ascending
1431- if (!isLargest) {
1432- Muls(srcLocal, srcLocal, T(-1), topKInfo.outter * topKInfo.inner);
1433- }
1434- 
1435- TopKNSmallCompute<T, isInitIndex, isHasfinish>(dstValueLocal, dstIndexLocal, srcLocal, srcIndexLocal,
1436- finishLocal, tempBuffer, k, tilling, topKInfo, isLargest);
1437- 
1438- if (!isLargest) {
1439- Muls(dstValueLocal, dstValueLocal, T(-1), tilling.maskOffset);
1440- Muls(srcLocal, srcLocal, T(-1), topKInfo.outter * topKInfo.inner);
1441- }
1442-}
1443- 
1444-/*
1445- * @ingroup TopK
1446- * @brief Get the top k maximum or minimum values and their corresponding indices of the last dimension.
1447- * @tparam T: Data type to be sorted, half or float.
1448- * @tparam isInitIndex: Whether to transfer the index of the input data.
1449- If the value is true, srcIndexLocal is the index of the input data.
1450- If the value is false, the index is generated by the Topk API.
1451- * @tparam isHasfinish: The isHasfinish parameter is used to specify that the sorting of some rows is invalid.
1452- If the value is true, enable the function. If the value is false, disable the function.
1453- In normal mode, isHasfinish can be set to true or false.
1454- In small mode, isHasfinish can only be set to false.
1455- * @tparam isReuseSrc: Whether temporary variables can reuse the input memory.
1456- This parameter is reserved. Use the default value false.
1457- * @tparam topkMode: Normal mode or small mode,
1458- Small mode is recommended when the inner axis length is 32. Performance will be high.
1459- * @param [out] dstValueLocal: Used to store k sorted values.
1460- * @param [out] dstIndexLocal: Used to store indexes corresponding to sorted k values.
1461- * @param [in] srcLocal: Input data to hold values to be sorted.
1462- * @param [in] srcIndexLocal: The input data is used to store the index corresponding to the value of srcLocal.
1463- * @param [in] finishLocal: Used to specify that the sort of some rows is an invalid sort with shape of (outter, 1).
1464- * @param [in] tmpLocal: Temporary space for storing intermediate variables during internal calculation.
1465- * @param [in] k: Obtain the first k maximum or minimum values and their corresponding indexes.
1466- * @param [in] tilling: Tiling information required for TopK calculation.
1467- * @param [in] topKInfo: Shape information of srcLocal.
1468- * @param [in] isLargest: Descending or ascending order. The value true indicates descending order,
1469- and the value false indicates ascending order.
1470- */
1471-template <typename T, bool isInitIndex = false, bool isHasfinish = false, bool isReuseSrc = false,
1472- enum TopKMode topkMode = TopKMode::TOPK_NORMAL, const TopKConfig& config = defaultTopKConfig>
1473-__aicore__ inline void TopK(const LocalTensor<T> &dstValueLocal, const LocalTensor<int32_t> &dstIndexLocal,
1474- const LocalTensor<T> &srcLocal, const LocalTensor<int32_t> &srcIndexLocal, const LocalTensor<bool> &finishLocal,
1475- const LocalTensor<uint8_t> &tmpLocal, const int32_t k, const TopkTiling &tilling, const TopKInfo &topKInfo,
1476- const bool isLargest = true)
1477-{
1478- // Only for AI Vector Core.
1479- if ASCEND_IS_AIC {
1480- return;
1481- }
1482- 
1483-#if ASCENDC_CPU_DEBUG
1484- TopkInputCheck<T, isInitIndex, topkMode, config>(k, topKInfo);
1485-#endif
1486- LocalTensor<T> tempBuffer = tmpLocal.template ReinterpretCast<T>();
1487- 
1488- if constexpr (config.algo == TopKAlgo::RADIX_SELECT) {
1489- static_assert((SupportType<T, uint8_t, int8_t, uint16_t, int16_t, half, float, bfloat16_t, uint32_t, int32_t,
1490- uint64_t, int64_t>()),
1491- "Type must be uint8_t/int8_t/uint16_t/int16_t/half/float/bfloat16_t/uint32_t/int32_t/uint64_t/int64_t in "
1492- "topk radix select algorithm.");
1493- static_assert((!isHasfinish), "Topk radix select algorithm cannot support to set finish flag.");
1494- if constexpr (topkMode == TopKMode::TOPK_NORMAL) {
1495- MicroAPI::RadixSelectTopK::TopKNormal<T, isInitIndex, isHasfinish, isReuseSrc, config>(dstValueLocal,
1496- dstIndexLocal, srcLocal, srcIndexLocal, finishLocal, tempBuffer, k, tilling, topKInfo, isLargest);
1497- }
1498- 
1499- if constexpr (topkMode == TopKMode::TOPK_NSMALL) {
1500- MicroAPI::RadixSelectTopK::TopKNSmall<T, isInitIndex, isHasfinish, isReuseSrc, config>(dstValueLocal,
1501- dstIndexLocal, srcLocal, srcIndexLocal, finishLocal, tempBuffer, k, tilling, topKInfo, isLargest);
1502- }
1503- return;
1504- }
1505- 
1506- if constexpr (config.algo == TopKAlgo::MERGE_SORT) {
1507- static_assert((SupportType<T, half, float>()), "Type must be half/float in topk merge select algorithm.");
1508- if constexpr (topkMode == TopKMode::TOPK_NORMAL) {
1509- TopKNormal<T, isInitIndex, isHasfinish, isReuseSrc>(dstValueLocal, dstIndexLocal, srcLocal,
1510- srcIndexLocal, finishLocal, tempBuffer, k, tilling, topKInfo, isLargest);
1511- }
1512- if constexpr (topkMode == TopKMode::TOPK_NSMALL) {
1513- TopKNSmall<T, isInitIndex, isHasfinish, isReuseSrc>(dstValueLocal, dstIndexLocal, srcLocal,
1514- srcIndexLocal, finishLocal, tempBuffer, k, tilling, topKInfo, isLargest);
1515- }
1516- }
1517-}
1518- 
1519-/*
1520- * @ingroup TopK
1521- * @brief Get the top k maximum or minimum values and their corresponding indices of the last dimension.
1522- * @tparam T: Data type to be sorted, half or float.
1523- * @tparam isInitIndex: Whether to transfer the index of the input data.
1524- If the value is true, srcIndexLocal is the index of the input data.
1525- If the value is false, the index is generated by the Topk API.
1526- * @tparam isHasfinish: The isHasfinish parameter is used to specify that the sorting of some rows is invalid.
1527- If the value is true, enable the function. If the value is false, disable the function.
1528- In normal mode, isHasfinish can be set to true or false.
1529- In small mode, isHasfinish can only be set to false.
1530- * @tparam isReuseSrc: Whether temporary variables can reuse the input memory.
1531- This parameter is reserved. Use the default value false.
1532- * @tparam topkMode: Normal mode or small mode,
1533- Small mode is recommended when the inner axis length is 32. Performance will be high.
1534- * @param [out] dstValueLocal: Used to store k sorted values.
1535- * @param [out] dstIndexLocal: Used to store indexes corresponding to sorted k values.
1536- * @param [in] srcLocal: Input data to hold values to be sorted.
1537- * @param [in] srcIndexLocal: The input data is used to store the index corresponding to the value of srcLocal.
1538- * @param [in] finishLocal: Used to specify that the sort of some rows is an invalid sort with shape of (outter, 1).
1539- * @param [in] k: Obtain the first k maximum or minimum values and their corresponding indexes.
1540- * @param [in] tilling: Tiling information required for TopK calculation.
1541- * @param [in] topKInfo: Shape information of srcLocal.
1542- * @param [in] isLargest: Descending or ascending order. The value true indicates descending order,
1543- and the value false indicates ascending order.
1544- */
1545-template <typename T, bool isInitIndex = false, bool isHasfinish = false, bool isReuseSrc = false,
1546- enum TopKMode topkMode = TopKMode::TOPK_NORMAL, const TopKConfig& config = defaultTopKConfig>
1547-__aicore__ inline void TopK(const LocalTensor<T> &dstValueLocal, const LocalTensor<int32_t> &dstIndexLocal,
1548- const LocalTensor<T> &srcLocal, const LocalTensor<int32_t> &srcIndexLocal, const LocalTensor<bool> &finishLocal,
1549- const int32_t k, const TopkTiling &tilling, const TopKInfo &topKInfo, const bool isLargest = true)
1550-{
1551- // Only for AI Vector Core.
1552- if ASCEND_IS_AIC {
1553- return;
1554- }
1555- 
1556- LocalTensor<T> stackTensor;
1557- PopStackBuffer<T, TPosition::LCM>(stackTensor);
1558- 
1559- // float: tmpLocalSize = inner * 16 / sizeof(float) = inner * 4
1560- // half: tmpLocalSize = inner * 16 / sizeof(half) = inner * 8
1561-#if ASCENDC_CPU_DEBUG
1562- auto stackTensorSize = stackTensor.GetSize();
1563- bool ans = stackTensorSize >= tilling.tmpLocalSize;
1564- ASCENDC_ASSERT(ans, {
1565- KERNEL_LOG(KERNEL_ERROR,
1566- "The pop stack buffer is insufficient, topk api need %d, but only %d exists.",
1567- tilling.tmpLocalSize,
1568- stackTensorSize);
1569- });
1570- 
1571- TopkInputCheck<T, isInitIndex, topkMode, config>(k, topKInfo);
1572-#endif
1573- stackTensor.SetSize(tilling.tmpLocalSize);
1574- 
1575- if constexpr (config.algo == TopKAlgo::RADIX_SELECT) {
1576- static_assert((SupportType<T, uint8_t, int8_t, uint16_t, int16_t, half, float, bfloat16_t, uint32_t, int32_t,
1577- uint64_t, int64_t>()),
1578- "Type must be uint8_t/int8_t/uint16_t/int16_t/half/float/bfloat16_t/uint32_t/int32_t/uint64_t/int64_t in "
1579- "topk radix select algorithm.");
1580- static_assert((!isHasfinish), "Topk radix select algorithm cannot support to set finish flag.");
1581- if constexpr (topkMode == TopKMode::TOPK_NORMAL) {
1582- MicroAPI::RadixSelectTopK::TopKNormal<T, isInitIndex, isHasfinish, isReuseSrc, config>(dstValueLocal,
1583- dstIndexLocal, srcLocal, srcIndexLocal, finishLocal, stackTensor, k, tilling, topKInfo, isLargest);
1584- }
1585- 
1586- if constexpr (topkMode == TopKMode::TOPK_NSMALL) {
1587- MicroAPI::RadixSelectTopK::TopKNSmall<T, isInitIndex, isHasfinish, isReuseSrc, config>(dstValueLocal,
1588- dstIndexLocal, srcLocal, srcIndexLocal, finishLocal, stackTensor, k, tilling, topKInfo, isLargest);
1589- }
1590- return;
1591- }
1592- 
1593- if constexpr (config.algo == TopKAlgo::MERGE_SORT) {
1594- static_assert((SupportType<T, half, float>()), "Type must be half/float in topk merge select algorithm.");
1595- if constexpr (topkMode == TopKMode::TOPK_NORMAL) {
1596- TopKNormal<T, isInitIndex, isHasfinish, isReuseSrc>(
1597- dstValueLocal, dstIndexLocal, srcLocal, srcIndexLocal, finishLocal, stackTensor, k, tilling,
1598- topKInfo, isLargest);
1599- }
1600- if constexpr (topkMode == TopKMode::TOPK_NSMALL) {
1601- TopKNSmall<T, isInitIndex, isHasfinish, isReuseSrc>(
1602- dstValueLocal, dstIndexLocal, srcLocal, srcIndexLocal, finishLocal, stackTensor, k, tilling,
1603- topKInfo, isLargest);
1604- }
1605- }
1606-}
1607- 
1608-} // namespace AscendC
1609- 
1610-#endif // IMPL_SORT_TOPK_TOPK_C310_IMPL_H
Dimpl/adv_api/detail/transpose/confusion_transpose/confusion_transpose_l300_impl.h+0-555
@@ -1,555 +0,0 @@
1-/**
2-* Copyright (c) 2025 Huawei Technologies Co., Ltd.
3-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4-* CANN Open Software License Agreement Version 2.0 (the "License").
5-* Please refer to the License for details. You may not use this file except in compliance with the License.
6-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8-* See LICENSE in the root of the software repository for the full text of the License.
9-*/
10- 
11-/* !
12- * \file confusion_transpose_l300_impl.h
13- * \brief
14- */
15-#ifndef IMPL_TRANSPOSE_CONFUSION_TRANSPOSE_CONFUSION_TRANSPOSE_L300_IMPL_H
16-#define IMPL_TRANSPOSE_CONFUSION_TRANSPOSE_CONFUSION_TRANSPOSE_L300_IMPL_H
17- 
18-namespace AscendC {
19-template <typename T, typename X, typename U, const MicroAPI::RegTrait &Trait, const uint16_t vlSize>
20-__simd_vf__ inline void ConfusionTransposeCommonGatherVF(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint32_t forLoop0,
21- uint32_t forLoop1, uint32_t forLoop2, uint32_t srcStride0, uint32_t srcStride1, uint32_t srcStride2, uint32_t tail,
22- uint32_t count, uint16_t mainLoop, uint32_t dtypeSize, uint32_t tailLoop)
23-{
24- MicroAPI::RegTensor<U, Trait> indexReg;
25- MicroAPI::RegTensor<T, Trait> MainVreg;
26- MicroAPI::MaskReg indexFullMask = MicroAPI::CreateMask<U, MicroAPI::MaskPattern::ALL, Trait>();
27- MicroAPI::MaskReg mainMask = MicroAPI::CreateMask<T, MicroAPI::MaskPattern::ALL, Trait>();
28- MicroAPI::MaskReg tailMask = MicroAPI::UpdateMask<T, Trait>(count);
29- MicroAPI::UnalignReg ureg0;
30- Arange((MicroAPI::RegTensor<X, Trait> &)indexReg, static_cast<X>(0));
31- Muls(indexReg, indexReg, static_cast<U>(srcStride2), indexFullMask);
32- for (uint16_t i = 0; i < static_cast<uint16_t>(forLoop0); i++) {
33- for (uint16_t j = 0; j < static_cast<uint16_t>(forLoop1); j++) {
34- uint64_t hoistDstAddr = (uint64_t)dstAddr + (uint64_t)((i * forLoop1 + j) * forLoop2 * dtypeSize);
35- for (uint16_t k = 0; k < static_cast<uint16_t>(mainLoop); k++) {
36- DataCopyGather(MainVreg, srcAddr + i * srcStride0 + j * srcStride1 + k * vlSize * srcStride2,
37- indexReg, mainMask);
38- MicroAPI::DataCopyUnAlign(((__ubuf__ T *&)hoistDstAddr), MainVreg, ureg0, vlSize);
39- }
40- for (uint16_t k = 0; k < static_cast<uint16_t>(tailLoop); k++) {
41- DataCopyGather(MainVreg, srcAddr + i * srcStride0 + j * srcStride1 + mainLoop * vlSize * srcStride2,
42- indexReg, tailMask);
43- MicroAPI::DataCopyUnAlign(((__ubuf__ T *&)hoistDstAddr), MainVreg, ureg0, tail);
44- }
45- MicroAPI::DataCopyUnAlignPost(((__ubuf__ T *&)hoistDstAddr), ureg0, 0);
46- }
47- }
48-}
49- 
50-template <typename T, typename X, typename U, const MicroAPI::RegTrait &Trait, const uint16_t vlSize>
51-__aicore__ inline void ConfusionTransposeCommonGather(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint32_t forLoop0,
52- uint32_t forLoop1, uint32_t forLoop2, uint32_t srcStride0, uint32_t srcStride1, uint32_t srcStride2)
53-{
54- uint32_t tail = forLoop2 % vlSize;
55- uint32_t count = tail;
56- uint16_t mainLoop = forLoop2 / vlSize;
57- uint32_t dtypeSize = sizeof(T);
58- uint32_t tailLoop = tail > 0 ? 1 : 0;
59- ConfusionTransposeCommonGatherVF<T, X, U, Trait, vlSize>(dstAddr, srcAddr, forLoop0,
60- forLoop1, forLoop2, srcStride0, srcStride1, srcStride2, tail, count, mainLoop, dtypeSize, tailLoop);
61-}
62- 
63-template <typename T, typename X, typename U, const MicroAPI::RegTrait &Trait, const uint16_t vlSize>
64-__simd_vf__ inline void ConfusionTransposeCommonGatherB8VF(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint32_t forLoop0,
65- uint32_t forLoop1, uint32_t forLoop2, uint32_t srcStride0, uint32_t srcStride1, uint32_t srcStride2, uint32_t tail,
66- uint32_t count, uint16_t mainLoop, uint32_t dtypeSize, uint32_t tailLoop)
67-{
68- MicroAPI::RegTensor<U, Trait> indexReg;
69- MicroAPI::RegTensor<T, Trait> vreg0;
70- MicroAPI::RegTensor<T, Trait> vreg1;
71- MicroAPI::RegTensor<T, Trait> vreg2;
72- MicroAPI::RegTensor<T, Trait> vreg3;
73- MicroAPI::MaskReg indexFullMask = MicroAPI::CreateMask<U, MicroAPI::MaskPattern::ALL, Trait>();
74- MicroAPI::MaskReg mainMask = MicroAPI::CreateMask<T, MicroAPI::MaskPattern::ALL, Trait>();
75- MicroAPI::MaskReg tailMask0 = MicroAPI::UpdateMask<U, Trait>(count);
76- MicroAPI::MaskReg tailMask1 = MicroAPI::UpdateMask<U, Trait>(count);
77- MicroAPI::UnalignReg ureg0;
78- Arange((MicroAPI::RegTensor<X, Trait> &)indexReg, static_cast<X>(0));
79- Muls(indexReg, indexReg, static_cast<U>(srcStride2), indexFullMask);
80- for (uint16_t i = 0; i < static_cast<uint16_t>(forLoop0); i++) {
81- for (uint16_t j = 0; j < static_cast<uint16_t>(forLoop1); j++) {
82- uint64_t hoistDstAddr = (uint64_t)dstAddr + (uint64_t)((i * forLoop1 + j) * forLoop2 * dtypeSize);
83- for (uint16_t k = 0; k < static_cast<uint16_t>(mainLoop); k++) {
84- DataCopyGather((MicroAPI::RegTensor<U, Trait> &)vreg0,
85- srcAddr + i * srcStride0 + j * srcStride1 + k * vlSize * srcStride2, indexReg, mainMask);
86- DataCopyGather((MicroAPI::RegTensor<U, Trait> &)vreg1,
87- srcAddr + i * srcStride0 + j * srcStride1 + k * vlSize * srcStride2 + vlSize / 2 * srcStride2,
88- indexReg, mainMask);
89- DeInterleave(vreg2, vreg3, vreg0, vreg1);
90- MicroAPI::DataCopyUnAlign(((__ubuf__ T *&)hoistDstAddr), vreg2, ureg0, vlSize);
91- }
92- for (uint16_t k = 0; k < static_cast<uint16_t>(tailLoop); k++) {
93- DataCopyGather((MicroAPI::RegTensor<U, Trait> &)vreg0,
94- srcAddr + i * srcStride0 + j * srcStride1 + mainLoop * vlSize * srcStride2, indexReg,
95- tailMask0);
96- DataCopyGather((MicroAPI::RegTensor<U, Trait> &)vreg1,
97- srcAddr + i * srcStride0 + j * srcStride1 + mainLoop * vlSize * srcStride2 +
98- vlSize / 2 * srcStride2,
99- indexReg, tailMask1);
100- DeInterleave(vreg2, vreg3, vreg0, vreg1);
101- MicroAPI::DataCopyUnAlign(((__ubuf__ T *&)hoistDstAddr), vreg2, ureg0, tail);
102- }
103- MicroAPI::DataCopyUnAlignPost(((__ubuf__ T *&)hoistDstAddr), ureg0, 0);
104- }
105- }
106-}
107- 
108-template <typename T, typename X, typename U, const MicroAPI::RegTrait &Trait, const uint16_t vlSize>
109-__aicore__ inline void ConfusionTransposeCommonGatherB8(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint32_t forLoop0,
110- uint32_t forLoop1, uint32_t forLoop2, uint32_t srcStride0, uint32_t srcStride1, uint32_t srcStride2)
111-{
112- uint32_t tail = forLoop2 % vlSize;
113- uint32_t count = tail;
114- uint16_t mainLoop = forLoop2 / vlSize;
115- uint32_t dtypeSize = sizeof(T);
116- uint32_t tailLoop = tail > 0 ? 1 : 0;
117- ConfusionTransposeCommonGatherB8VF<T, X, U, Trait, vlSize>(dstAddr, srcAddr, forLoop0,
118- forLoop1, forLoop2, srcStride0, srcStride1, srcStride2, tail, count, mainLoop, dtypeSize, tailLoop);
119-}
120- 
121-template <typename T, typename X, typename U, const MicroAPI::RegTrait &Trait, const uint16_t vlSize>
122-__simd_vf__ inline void ConfusionTransposeComplexGatherVF(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint32_t forLoop0,
123- uint32_t forLoop1, uint32_t forLoop2, uint32_t srcStride0, uint32_t srcStride1, uint32_t srcStride2, uint32_t factor,
124- uint32_t mainSize, uint16_t mainLoop, uint32_t tail, uint32_t tailLoop, uint32_t mainCount, uint32_t tailCount,
125- uint32_t count, uint32_t dtypeSize)
126-{
127- MicroAPI::RegTensor<U, Trait> indexReg0;
128- MicroAPI::RegTensor<U, Trait> indexReg1;
129- MicroAPI::RegTensor<U, Trait> indexReg2;
130- MicroAPI::RegTensor<U, Trait> indexReg3;
131- MicroAPI::RegTensor<U, Trait> indexReg4;
132- MicroAPI::RegTensor<U, Trait> indexReg5;
133- MicroAPI::RegTensor<T, Trait> MainVreg;
134- MicroAPI::MaskReg indexFullMask = MicroAPI::CreateMask<U, MicroAPI::MaskPattern::ALL, Trait>();
135- MicroAPI::MaskReg mainMask = MicroAPI::UpdateMask<T, Trait>(mainCount);
136- MicroAPI::MaskReg tailMask = MicroAPI::UpdateMask<T, Trait>(tailCount);
137- MicroAPI::UnalignReg ureg0;
138- Arange((MicroAPI::RegTensor<X, Trait> &)indexReg0, static_cast<X>(0));
139- Duplicate(indexReg1, static_cast<U>(forLoop2));
140- Div(indexReg2, indexReg0, indexReg1, indexFullMask);
141- Muls(indexReg3, indexReg2, static_cast<U>(srcStride1), indexFullMask);
142- // k%c: k - k/c*c
143- Mul(indexReg4, indexReg2, indexReg1, indexFullMask);
144- Sub(indexReg4, indexReg0, indexReg4, indexFullMask);
145- Muls(indexReg4, indexReg4, static_cast<U>(srcStride2), indexFullMask);
146- Add(indexReg5, indexReg3, indexReg4, indexFullMask);
147- for (uint16_t i = 0; i < static_cast<uint16_t>(forLoop0); i++) {
148- uint64_t hoistDstAddr = (uint64_t)dstAddr + (uint64_t)(i * forLoop1 * forLoop2 * dtypeSize);
149- for (uint16_t j = 0; j < static_cast<uint16_t>(mainLoop); j++) {
150- DataCopyGather(MainVreg, srcAddr + i * srcStride0 + j * srcStride1 * factor, indexReg5, mainMask);
151- MicroAPI::DataCopyUnAlign(((__ubuf__ T *&)hoistDstAddr), MainVreg, ureg0, mainSize);
152- }
153- for (uint16_t k = 0; k < static_cast<uint16_t>(tailLoop); k++) {
154- DataCopyGather(MainVreg, srcAddr + i * srcStride0 + mainLoop * srcStride1 * factor, indexReg5,
155- tailMask);
156- MicroAPI::DataCopyUnAlign(((__ubuf__ T *&)hoistDstAddr), MainVreg, ureg0, tail);
157- }
158- MicroAPI::DataCopyUnAlignPost(((__ubuf__ T *&)hoistDstAddr), ureg0, 0);
159- }
160-}
161- 
162-template <typename T, typename X, typename U, const MicroAPI::RegTrait &Trait, const uint16_t vlSize>
163-__aicore__ inline void ConfusionTransposeComplexGather(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint32_t forLoop0,
164- uint32_t forLoop1, uint32_t forLoop2, uint32_t srcStride0, uint32_t srcStride1, uint32_t srcStride2)
165-{
166- uint32_t factor = vlSize / forLoop2;
167- uint32_t mainSize = factor * forLoop2;
168- uint16_t mainLoop = forLoop1 / factor;
169- uint32_t tail = forLoop1 % factor * forLoop2;
170- uint32_t tailLoop = tail > 0 ? 1 : 0;
171- uint32_t mainCount = mainSize;
172- uint32_t tailCount = tail;
173- 
174- uint32_t count = tail;
175- uint32_t dtypeSize = sizeof(T);
176- ConfusionTransposeComplexGatherVF<T, X, U, Trait, vlSize>(dstAddr, srcAddr, forLoop0, forLoop1, forLoop2,
177- srcStride0, srcStride1, srcStride2, factor, mainSize, mainLoop, tail, tailLoop, mainCount, tailCount,
178- count, dtypeSize);
179-}
180- 
181-template <typename T, typename X, typename U, const MicroAPI::RegTrait &Trait, const uint16_t vlSize>
182-__simd_vf__ inline void ConfusionTransposeComplexGatherB8VF(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint32_t forLoop0,
183- uint32_t forLoop1, uint32_t forLoop2, uint32_t srcStride0, uint32_t srcStride1, uint32_t srcStride2, uint32_t factor,
184- uint32_t mainSize, uint16_t mainLoop, uint32_t tail, uint32_t tailLoop, uint32_t mainCount, uint32_t tailCount,
185- uint32_t dtypeSize, uint32_t halfVlSize)
186-{
187- MicroAPI::RegTensor<U, Trait> indexReg0;
188- MicroAPI::RegTensor<U, Trait> indexReg1;
189- MicroAPI::RegTensor<U, Trait> indexReg2;
190- MicroAPI::RegTensor<U, Trait> indexReg3;
191- MicroAPI::RegTensor<U, Trait> indexReg4;
192- MicroAPI::RegTensor<U, Trait> indexReg5;
193- MicroAPI::RegTensor<U, Trait> indexReg6;
194- MicroAPI::RegTensor<U, Trait> indexReg7;
195- MicroAPI::RegTensor<U, Trait> indexReg8;
196- MicroAPI::RegTensor<U, Trait> indexReg9;
197- MicroAPI::RegTensor<U, Trait> indexReg10;
198- MicroAPI::RegTensor<T, Trait> vreg0;
199- MicroAPI::RegTensor<T, Trait> vreg1;
200- MicroAPI::RegTensor<T, Trait> vreg2;
201- MicroAPI::RegTensor<T, Trait> vreg3;
202- MicroAPI::MaskReg indexFullMask = MicroAPI::CreateMask<U, MicroAPI::MaskPattern::ALL, Trait>();
203- MicroAPI::MaskReg mainMask0 = MicroAPI::UpdateMask<U, Trait>(mainCount);
204- MicroAPI::MaskReg mainMask1 = MicroAPI::UpdateMask<U, Trait>(mainCount);
205- MicroAPI::MaskReg tailMask0 = MicroAPI::UpdateMask<U, Trait>(tailCount);
206- MicroAPI::MaskReg tailMask1 = MicroAPI::UpdateMask<U, Trait>(tailCount);
207- MicroAPI::UnalignReg ureg0;
208- Arange((MicroAPI::RegTensor<X, Trait> &)indexReg0, static_cast<X>(0));
209- Duplicate(indexReg1, static_cast<U>(forLoop2));
210- Div(indexReg2, indexReg0, indexReg1, indexFullMask);
211- Muls(indexReg3, indexReg2, static_cast<U>(srcStride1), indexFullMask);
212- // k%c: k - k/c*c
213- Mul(indexReg4, indexReg2, indexReg1, indexFullMask);
214- Sub(indexReg4, indexReg0, indexReg4, indexFullMask);
215- Muls(indexReg4, indexReg4, static_cast<U>(srcStride2), indexFullMask);
216- Add(indexReg5, indexReg3, indexReg4, indexFullMask);
217- 
218- // the other half vl index
219- Arange((MicroAPI::RegTensor<X, Trait> &)indexReg6, static_cast<X>(halfVlSize));
220- Div(indexReg7, indexReg6, indexReg1, indexFullMask);
221- Muls(indexReg8, indexReg7, static_cast<U>(srcStride1), indexFullMask);
222- // k%c: k - k/c*c
223- Mul(indexReg9, indexReg7, indexReg1, indexFullMask);
224- Sub(indexReg9, indexReg6, indexReg9, indexFullMask);
225- Muls(indexReg9, indexReg9, static_cast<U>(srcStride2), indexFullMask);
226- Add(indexReg10, indexReg8, indexReg9, indexFullMask);
227- for (uint16_t i = 0; i < static_cast<uint16_t>(forLoop0); i++) {
228- uint64_t hoistDstAddr = (uint64_t)dstAddr + (uint64_t)(i * forLoop1 * forLoop2 * dtypeSize);
229- for (uint16_t j = 0; j < static_cast<uint16_t>(mainLoop); j++) {
230- DataCopyGather((MicroAPI::RegTensor<U, Trait> &)vreg0,
231- srcAddr + i * srcStride0 + j * srcStride1 * factor, indexReg5, mainMask0);
232- DataCopyGather((MicroAPI::RegTensor<U, Trait> &)vreg1,
233- srcAddr + i * srcStride0 + j * srcStride1 * factor, indexReg10, mainMask1);
234- DeInterleave(vreg2, vreg3, vreg0, vreg1);
235- MicroAPI::DataCopyUnAlign(((__ubuf__ T *&)hoistDstAddr), vreg2, ureg0, mainSize);
236- }
237- for (uint16_t k = 0; k < static_cast<uint16_t>(tailLoop); k++) {
238- DataCopyGather((MicroAPI::RegTensor<U, Trait> &)vreg0,
239- srcAddr + i * srcStride0 + mainLoop * srcStride1 * factor, indexReg5, tailMask0);
240- DataCopyGather((MicroAPI::RegTensor<U, Trait> &)vreg1,
241- srcAddr + i * srcStride0 + mainLoop * srcStride1 * factor, indexReg10, tailMask1);
242- DeInterleave(vreg2, vreg3, vreg0, vreg1);
243- MicroAPI::DataCopyUnAlign(((__ubuf__ T *&)hoistDstAddr), vreg2, ureg0, tail);
244- }
245- MicroAPI::DataCopyUnAlignPost(((__ubuf__ T *&)hoistDstAddr), ureg0, 0);
246- }
247-}
248- 
249-template <typename T, typename X, typename U, const MicroAPI::RegTrait &Trait, const uint16_t vlSize>
250-__aicore__ inline void ConfusionTransposeComplexGatherB8(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint32_t forLoop0,
251- uint32_t forLoop1, uint32_t forLoop2, uint32_t srcStride0, uint32_t srcStride1, uint32_t srcStride2)
252-{
253- uint32_t factor = vlSize / forLoop2;
254- uint32_t mainSize = factor * forLoop2;
255- uint16_t mainLoop = forLoop1 / factor;
256- uint32_t tail = forLoop1 % factor * forLoop2;
257- uint32_t tailLoop = tail > 0 ? 1 : 0;
258- uint32_t mainCount = mainSize;
259- uint32_t tailCount = tail;
260- uint32_t dtypeSize = sizeof(T);
261- uint32_t halfVlSize = vlSize / 2;
262- 
263- ConfusionTransposeComplexGatherB8VF<T, X, U, Trait, vlSize>(dstAddr, srcAddr, forLoop0, forLoop1, forLoop2,
264- srcStride0, srcStride1, srcStride2, factor, mainSize, mainLoop, tail, tailLoop, mainCount, tailCount,
265- dtypeSize, halfVlSize);
266-}
267- 
268-template <typename T, const MicroAPI::RegTrait &Trait, const uint16_t vlSize>
269-__simd_vf__ inline void ConfusionTransposeCopySrcToDst(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint32_t totalCount)
270-{
271- uint32_t count = totalCount;
272- uint32_t forLoop = (totalCount + vlSize - 1) / vlSize;
273- MicroAPI::RegTensor<T, Trait> vreg0;
274- MicroAPI::MaskReg mainMask;
275- for (uint16_t i = 0; i < static_cast<uint16_t>(forLoop); i++) {
276- mainMask = MicroAPI::UpdateMask<T, Trait>(count);
277- DataCopy(vreg0, srcAddr + i * vlSize);
278- DataCopy(dstAddr + i * vlSize, vreg0, mainMask);
279- }
280-}
281- 
282-template <typename T, const MicroAPI::RegTrait &Trait, const uint16_t vlSize>
283-__simd_vf__ inline void ConfusionTransposeCommonDataCopyVF(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint32_t forLoop0,
284- uint32_t forLoop1, uint32_t forLoop2, uint32_t srcStride0, uint32_t srcStride1, uint32_t tail, uint32_t count,
285- uint16_t mainLoop, uint32_t dtypeSize, uint32_t tailLoop)
286-{
287- MicroAPI::RegTensor<T, Trait> MainVreg;
288- MicroAPI::UnalignReg DstUreg;
289- MicroAPI::UnalignReg SrcUreg;
290- for (uint16_t i = 0; i < static_cast<uint16_t>(forLoop0); i++) {
291- for (uint16_t j = 0; j < static_cast<uint16_t>(forLoop1); j++) {
292- uint64_t hoistSrcAddr = (uint64_t)srcAddr + (uint64_t)((i * srcStride0 + j * srcStride1) * dtypeSize);
293- uint64_t hoistDstAddr = (uint64_t)dstAddr + (uint64_t)((i * forLoop1 + j) * forLoop2 * dtypeSize);
294- for (uint16_t k = 0; k < static_cast<uint16_t>(mainLoop); k++) {
295- MicroAPI::DataCopyUnAlignPre(SrcUreg, ((__ubuf__ T *&)hoistSrcAddr));
296- MicroAPI::DataCopyUnAlign(MainVreg, SrcUreg, ((__ubuf__ T *&)hoistSrcAddr), vlSize);
297- MicroAPI::DataCopyUnAlign(((__ubuf__ T *&)hoistDstAddr), MainVreg, DstUreg, vlSize);
298- }
299- for (uint16_t k = 0; k < static_cast<uint16_t>(tailLoop); k++) {
300- MicroAPI::DataCopyUnAlignPre(SrcUreg, ((__ubuf__ T *&)hoistSrcAddr));
301- MicroAPI::DataCopyUnAlign(MainVreg, SrcUreg, ((__ubuf__ T *&)hoistSrcAddr), tail);
302- MicroAPI::DataCopyUnAlign(((__ubuf__ T *&)hoistDstAddr), MainVreg, DstUreg, tail);
303- }
304- MicroAPI::DataCopyUnAlignPost(((__ubuf__ T *&)hoistDstAddr), DstUreg, 0);
305- }
306- }
307-}
308- 
309-template <typename T, const MicroAPI::RegTrait &Trait, const uint16_t vlSize>
310-__aicore__ inline void ConfusionTransposeCommonDataCopy(__ubuf__ T *dstAddr, __ubuf__ T *srcAddr, uint32_t forLoop0,
311- uint32_t forLoop1, uint32_t forLoop2, uint32_t srcStride0, uint32_t srcStride1)
312-{
313- uint32_t tail = forLoop2 % vlSize;
314- uint32_t count = tail;
315- uint16_t mainLoop = forLoop2 / vlSize;
316- uint32_t dtypeSize = sizeof(T);
317- uint32_t tailLoop = tail > 0 ? 1 : 0;
318- ConfusionTransposeCommonDataCopyVF<T, Trait, vlSize>(dstAddr, srcAddr, forLoop0,
319- forLoop1, forLoop2, srcStride0, srcStride1, tail, count, mainLoop, dtypeSize, tailLoop);
320-}
321- 
322-/*
323-scene 13：{ shape:[H, W], format:"ND"} -->{ shape:[W, H], format:"ND"};
324- { shape:[N, H, W], format:"ND"} -->{ shape:[N, W, H], format:"ND"};
325-*/
326-template <typename T>
327-__aicore__ inline void ConfusionTranspose021(
328- const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor, ConfusionTranspose021Tiling& tiling)
329-{
330- constexpr uint16_t vlSize =
331- IsSameType<T, int64_t>::value ? GetVecLen() / sizeof(float) : GetVecLen() / sizeof(T);
332- if ((tiling.dim1 == 1) || (tiling.dim2 == 1)) {
333- uint32_t totalCount = tiling.dim0 * tiling.dim1 * tiling.dim2;
334- ConfusionTransposeCopySrcToDst<T, MicroAPI::RegTraitNumOne, vlSize>((__ubuf__ T *)dstTensor.GetPhyAddr(),
335- (__ubuf__ T *)srcTensor.GetPhyAddr(), totalCount);
336- } else {
337- uint32_t srcStride0 = tiling.dim1 * tiling.dim2;
338- uint32_t srcStride1 = 1;
339- uint32_t srcStride2 = tiling.dim2;
340- if (tiling.dim1 > vlSize / 2) {
341- if constexpr (SupportBytes<T, 4>()) {
342- ConfusionTransposeCommonGather<T, int32_t, uint32_t, MicroAPI::RegTraitNumOne, vlSize>(
343- (__ubuf__ T *)dstTensor.GetPhyAddr(), (__ubuf__ T *)srcTensor.GetPhyAddr(), tiling.dim0,
344- tiling.dim2, tiling.dim1, srcStride0, srcStride1, srcStride2);
345- } else if constexpr (SupportBytes<T, 2>()) {
346- ConfusionTransposeCommonGather<T, int16_t, uint16_t, MicroAPI::RegTraitNumOne, vlSize>(
347- (__ubuf__ T *)dstTensor.GetPhyAddr(), (__ubuf__ T *)srcTensor.GetPhyAddr(), tiling.dim0,
348- tiling.dim2, tiling.dim1, srcStride0, srcStride1, srcStride2);
349- } else if constexpr (SupportBytes<T, 1>()) {
350- ConfusionTransposeCommonGatherB8<T, int16_t, uint16_t, MicroAPI::RegTraitNumOne, vlSize>(
351- (__ubuf__ T *)dstTensor.GetPhyAddr(), (__ubuf__ T *)srcTensor.GetPhyAddr(), tiling.dim0,
352- tiling.dim2, tiling.dim1, srcStride0, srcStride1, srcStride2);
353- }
354- } else {
355- if constexpr (SupportBytes<T, 4>()) {
356- ConfusionTransposeComplexGather<T, int32_t, uint32_t, MicroAPI::RegTraitNumOne, vlSize>(
357- (__ubuf__ T *)dstTensor.GetPhyAddr(), (__ubuf__ T *)srcTensor.GetPhyAddr(), tiling.dim0,
358- tiling.dim2, tiling.dim1, srcStride0, srcStride1, srcStride2);
359- } else if constexpr (SupportBytes<T, 2>()) {
360- ConfusionTransposeComplexGather<T, int16_t, uint16_t, MicroAPI::RegTraitNumOne, vlSize>(
361- (__ubuf__ T *)dstTensor.GetPhyAddr(), (__ubuf__ T *)srcTensor.GetPhyAddr(), tiling.dim0,
362- tiling.dim2, tiling.dim1, srcStride0, srcStride1, srcStride2);
363- } else if constexpr (SupportBytes<T, 1>()) {
364- ConfusionTransposeComplexGatherB8<T, int16_t, uint16_t, MicroAPI::RegTraitNumOne, vlSize>(
365- (__ubuf__ T *)dstTensor.GetPhyAddr(), (__ubuf__ T *)srcTensor.GetPhyAddr(), tiling.dim0,
366- tiling.dim2, tiling.dim1, srcStride0, srcStride1, srcStride2);
367- }
368- }
369- }
370-}
371- 
372-/*
373-scene 14： { shape:[N, H, W], format:"ND"} -->{ shape:[H, N, W], format:"ND"};
374-*/
375-template <typename T>
376-__aicore__ inline void ConfusionTranspose102(
377- const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor, ConfusionTranspose102Tiling& tiling)
378-{
379- constexpr uint16_t vlSize =
380- IsSameType<T, int64_t>::value ? GetVecLen() / sizeof(float) : GetVecLen() / sizeof(T);
381- if ((tiling.dim0 == 1) || (tiling.dim1 == 1)) {
382- uint32_t totalCount = tiling.dim0 * tiling.dim1 * tiling.dim2;
383- ConfusionTransposeCopySrcToDst<T, MicroAPI::RegTraitNumOne, vlSize>((__ubuf__ T *)dstTensor.GetPhyAddr(),
384- (__ubuf__ T *)srcTensor.GetPhyAddr(), totalCount);
385- } else {
386- uint32_t srcStride0 = tiling.dim2;
387- uint32_t srcStride1 = tiling.dim1 * tiling.dim2;
388- uint32_t srcStride2 = 1;
389- if (tiling.dim2 > vlSize / 2) {
390- ConfusionTransposeCommonDataCopy<T, MicroAPI::RegTraitNumOne, vlSize>((__ubuf__ T *)dstTensor.GetPhyAddr(),
391- (__ubuf__ T *)srcTensor.GetPhyAddr(), tiling.dim1, tiling.dim0, tiling.dim2, srcStride0, srcStride1);
392- } else {
393- if constexpr (SupportBytes<T, 4>()) {
394- ConfusionTransposeComplexGather<T, int32_t, uint32_t, MicroAPI::RegTraitNumOne, vlSize>(
395- (__ubuf__ T *)dstTensor.GetPhyAddr(), (__ubuf__ T *)srcTensor.GetPhyAddr(), tiling.dim1,
396- tiling.dim0, tiling.dim2, srcStride0, srcStride1, srcStride2);
397- } else if constexpr (SupportBytes<T, 2>()) {
398- ConfusionTransposeComplexGather<T, int16_t, uint16_t, MicroAPI::RegTraitNumOne, vlSize>(
399- (__ubuf__ T *)dstTensor.GetPhyAddr(), (__ubuf__ T *)srcTensor.GetPhyAddr(), tiling.dim1,
400- tiling.dim0, tiling.dim2, srcStride0, srcStride1, srcStride2);
401- } else if constexpr (SupportBytes<T, 1>()) {
402- ConfusionTransposeComplexGatherB8<T, int16_t, uint16_t, MicroAPI::RegTraitNumOne, vlSize>(
403- (__ubuf__ T *)dstTensor.GetPhyAddr(), (__ubuf__ T *)srcTensor.GetPhyAddr(), tiling.dim1,
404- tiling.dim0, tiling.dim2, srcStride0, srcStride1, srcStride2);
405- }
406- }
407- }
408-}
409- 
410-/*
411-scene 15： { shape:[N, H, W], format:"ND"} -->{ shape:[W, H, N], format:"ND"};
412-*/
413-template <typename T>
414-__aicore__ inline void ConfusionTranspose210(
415- const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor, ConfusionTranspose210Tiling& tiling)
416-{
417- constexpr uint16_t vlSize =
418- IsSameType<T, int64_t>::value ? GetVecLen() / sizeof(float) : GetVecLen() / sizeof(T);
419- ConfusionTranspose021Tiling tiling021 = { 1, tiling.dim1, tiling.dim2 };
420- if (tiling.dim0 == 1 || tiling.dim1 == 1 || tiling.dim2 == 1) {
421- ConfusionTranspose021Tiling tiling021 = { 1, tiling.dim1, tiling.dim2 };
422- if (tiling.dim1 == 1) {
423- tiling021.dim1 = tiling.dim0;
424- } else if (tiling.dim2 == 1) {
425- tiling021.dim1 = tiling.dim0;
426- tiling021.dim2 = tiling.dim1;
427- }
428- ConfusionTranspose021(dstTensor, srcTensor, tiling021);
429- } else {
430- uint32_t srcStride0 = 1;
431- uint32_t srcStride1 = tiling.dim2;
432- uint32_t srcStride2 = tiling.dim1 * tiling.dim2;
433- if (tiling.dim0 > vlSize / 2) {
434- if constexpr (SupportBytes<T, 4>()) {
435- ConfusionTransposeCommonGather<T, int32_t, uint32_t, MicroAPI::RegTraitNumOne, vlSize>(
436- (__ubuf__ T *)dstTensor.GetPhyAddr(), (__ubuf__ T *)srcTensor.GetPhyAddr(), tiling.dim2,
437- tiling.dim1, tiling.dim0, srcStride0, srcStride1, srcStride2);
438- } else if constexpr (SupportBytes<T, 2>()) {
439- ConfusionTransposeCommonGather<T, int16_t, uint16_t, MicroAPI::RegTraitNumOne, vlSize>(
440- (__ubuf__ T *)dstTensor.GetPhyAddr(), (__ubuf__ T *)srcTensor.GetPhyAddr(), tiling.dim2,
441- tiling.dim1, tiling.dim0, srcStride0, srcStride1, srcStride2);
442- } else if constexpr (SupportBytes<T, 1>()) {
443- ConfusionTransposeCommonGatherB8<T, int16_t, uint16_t, MicroAPI::RegTraitNumOne, vlSize>(
444- (__ubuf__ T *)dstTensor.GetPhyAddr(), (__ubuf__ T *)srcTensor.GetPhyAddr(), tiling.dim2,
445- tiling.dim1, tiling.dim0, srcStride0, srcStride1, srcStride2);
446- }
447- } else {
448- if constexpr (SupportBytes<T, 4>()) {
449- ConfusionTransposeComplexGather<T, int32_t, uint32_t, MicroAPI::RegTraitNumOne, vlSize>(
450- (__ubuf__ T *)dstTensor.GetPhyAddr(), (__ubuf__ T *)srcTensor.GetPhyAddr(), tiling.dim2,
451- tiling.dim1, tiling.dim0, srcStride0, srcStride1, srcStride2);
452- } else if constexpr (SupportBytes<T, 2>()) {
453- ConfusionTransposeComplexGather<T, int16_t, uint16_t, MicroAPI::RegTraitNumOne, vlSize>(
454- (__ubuf__ T *)dstTensor.GetPhyAddr(), (__ubuf__ T *)srcTensor.GetPhyAddr(), tiling.dim2,
455- tiling.dim1, tiling.dim0, srcStride0, srcStride1, srcStride2);
456- } else if constexpr (SupportBytes<T, 1>()) {
457- ConfusionTransposeComplexGatherB8<T, int16_t, uint16_t, MicroAPI::RegTraitNumOne, vlSize>(
458- (__ubuf__ T *)dstTensor.GetPhyAddr(), (__ubuf__ T *)srcTensor.GetPhyAddr(), tiling.dim2,
459- tiling.dim1, tiling.dim0, srcStride0, srcStride1, srcStride2);
460- }
461- }
462- }
463-}
464- 
465-template <typename T>
466-__aicore__ inline void ConfusionTransposeND2NZWithInlv(
467- const LocalTensor<T>& dstTensor, const LocalTensor<T>& srcTensor, ConfusionTranspose210Tiling& tiling)
468-{
469- ConfusionTransposeND2NZWithInlvImpl((__ubuf__ T*) dstTensor.GetPhyAddr(), (__ubuf__ T*) srcTensor.GetPhyAddr(), tiling);
470-}
471- 
472-template <typename T>
473-__simd_vf__ inline void ConfusionTransposeND2NZWithInlvVFImpl(__ubuf__ T* dstAddr, __ubuf__ T* srcAddr, uint32_t regWidth,
474- uint32_t c0Size, uint32_t factor, uint32_t height, uint32_t width, uint32_t tailWidth, uint16_t repeatTimes)
475-{
476- AscendC::MicroAPI::RegTensor<T> vSrcReg0;
477- AscendC::MicroAPI::RegTensor<T> vSrcReg1;
478- AscendC::MicroAPI::RegTensor<T> vDstReg0;
479- AscendC::MicroAPI::RegTensor<T> vDstReg1;
480- AscendC::MicroAPI::MaskReg mask = AscendC::MicroAPI::CreateMask<T, MicroAPI::MaskPattern::ALL>();
481- uint32_t srcOffset = 0;
482- uint32_t dstOffset = 0;
483- const uint16_t heightLoopNum = height / factor;
484- for (uint16_t i = 0; i < repeatTimes; i++) {
485- srcOffset = i * regWidth;
486- dstOffset = i * regWidth * heightLoopNum * factor;
487- for (uint16_t j = 0; j < heightLoopNum; j++) {
488- uint64_t rowStrideAddr = uint64_t(srcAddr + srcOffset + factor * j * width);
489- uint64_t nerborStrideAddr = uint64_t(srcAddr + srcOffset + (factor * j + 1) * width);
490- 
491- AscendC::MicroAPI::DataCopy<T, MicroAPI::DataCopyMode::DATA_BLOCK_COPY, MicroAPI::PostLiteral::POST_MODE_NORMAL>(
492- vSrcReg0, (__ubuf__ T*&)rowStrideAddr, 1, 0, mask);
493- AscendC::MicroAPI::DataCopy<T, MicroAPI::DataCopyMode::DATA_BLOCK_COPY, MicroAPI::PostLiteral::POST_MODE_NORMAL>(
494- vSrcReg1, (__ubuf__ T*&)nerborStrideAddr, 1, 0, mask);
495- AscendC::MicroAPI::Interleave<T>(vDstReg0, vDstReg1, vSrcReg0, vSrcReg1);
496- 
497- uint64_t dstRowStrideAddr = uint64_t(dstAddr + dstOffset + j * c0Size);
498- AscendC::MicroAPI::DataCopy<T, MicroAPI::DataCopyMode::DATA_BLOCK_COPY, MicroAPI::PostLiteral::POST_MODE_NORMAL>(
499- (__ubuf__ T*&)dstRowStrideAddr, vDstReg0, heightLoopNum, 0, mask);
500- 
501- uint64_t dstNerborStrideAddr = uint64_t(dstAddr + dstOffset + regWidth * heightLoopNum + j * c0Size);
502- AscendC::MicroAPI::DataCopy<T, MicroAPI::DataCopyMode::DATA_BLOCK_COPY, MicroAPI::PostLiteral::POST_MODE_NORMAL>(
503- (__ubuf__ T*&)dstNerborStrideAddr, vDstReg1, heightLoopNum, 0, mask);
504- }
505- }
506- 
507- if (tailWidth) {
508- if (repeatTimes != 0) {
509- srcOffset += regWidth;
510- dstOffset += regWidth * heightLoopNum * factor;
511- }
512- const uint32_t regWidthHalf = 128;
513- for (uint16_t j = 0; j < heightLoopNum; j++) {
514- uint64_t rowStrideAddr = uint64_t(srcAddr + srcOffset + factor * j * width);
515- uint64_t nerborStrideAddr = uint64_t(srcAddr + srcOffset + (factor * j + 1) * width);
516- uint32_t mask_size = tailWidth * factor;
517- if (tailWidth > regWidthHalf) {
518- mask_size = regWidth;
519- }
520- AscendC::MicroAPI::MaskReg mask = AscendC::MicroAPI::UpdateMask<T>(mask_size);
521- AscendC::MicroAPI::DataCopy<T, MicroAPI::DataCopyMode::DATA_BLOCK_COPY, MicroAPI::PostLiteral::POST_MODE_NORMAL>(
522- vSrcReg0, (__ubuf__ T*&)rowStrideAddr, 1, 0, mask);
523- AscendC::MicroAPI::DataCopy<T, MicroAPI::DataCopyMode::DATA_BLOCK_COPY, MicroAPI::PostLiteral::POST_MODE_NORMAL>(
524- vSrcReg1, (__ubuf__ T*&)nerborStrideAddr, 1, 0, mask);
525- AscendC::MicroAPI::Interleave<T>(vDstReg0, vDstReg1, vSrcReg0, vSrcReg1);
526- 
527- uint64_t dstRowStrideAddr = uint64_t(dstAddr + dstOffset + j * c0Size);
528- AscendC::MicroAPI::DataCopy<T, MicroAPI::DataCopyMode::DATA_BLOCK_COPY, MicroAPI::PostLiteral::POST_MODE_NORMAL>(
529- (__ubuf__ T*&)dstRowStrideAddr, vDstReg0, heightLoopNum, 0, mask);
530- 
531- if (tailWidth > regWidthHalf) {
532- mask_size = tailWidth * factor - regWidth;
533- AscendC::MicroAPI::MaskReg mask1 = AscendC::MicroAPI::UpdateMask<T>(mask_size);
534- uint64_t dstNerborStrideAddr = uint64_t(dstAddr + dstOffset + regWidth * heightLoopNum + j * c0Size);
535- AscendC::MicroAPI::DataCopy<T, MicroAPI::DataCopyMode::DATA_BLOCK_COPY, MicroAPI::PostLiteral::POST_MODE_NORMAL>(
536- (__ubuf__ T*&)dstNerborStrideAddr, vDstReg1, heightLoopNum, 0, mask1);
537- }
538- }
539- }
540-}
541- 
542-template <typename T>
543-__aicore__ inline void ConfusionTransposeND2NZWithInlvImpl(__ubuf__ T* dstAddr, __ubuf__ T* srcAddr, ConfusionTranspose210Tiling& tiling)
544-{
545- constexpr uint32_t regWidth = 256; // vf reg length
546- constexpr uint32_t c0Size = 32; // factal size
547- constexpr uint32_t factor = 2; // row factor
548- uint32_t height = tiling.dim1;
549- uint32_t width = tiling.dim2;
550- uint32_t tailWidth = width % regWidth;
551- uint16_t repeatTimes = width / regWidth;
552- ConfusionTransposeND2NZWithInlvVFImpl<T>(dstAddr, srcAddr, regWidth, c0Size, factor, height, width, tailWidth, repeatTimes);
553-}
554-} // namespace AscendC
555-#endif // IMPL_TRANSPOSE_CONFUSION_TRANSPOSE_CONFUSION_TRANSPOSE_C310_IMPL_H
Mimpl/basic_api/kernel_scalar_convert.h+1-0
@@ -174,6 +174,7 @@ __aicore__ constexpr inline float Cast(const T& bVal)
174 return 0.0f;174 return 0.0f;
175}175}
176 176 
177+template <typename T>
177__aicore__ constexpr inline float ToFloat(const T& bVal)178__aicore__ constexpr inline float ToFloat(const T& bVal)
178{179{
179 ASCENDC_ASSERT((false), { KERNEL_LOG(KERNEL_ERROR, "unsupport ToFloat"); });180 ASCENDC_ASSERT((false), { KERNEL_LOG(KERNEL_ERROR, "unsupport ToFloat"); });