Star1729
1645
Code
Code
Issues489
Pull Requests549
Actions
Discussion
Wiki
Analysis
Star1729
1645
Merged
fix expint bug #4165
kangjiamingAdd on Jul 21
fix expint bug #4165
Merged
kangjiamingAdd on Jul 21
Merged fromkangjiaming/kangjiaming-ops-mathexpintintocann/ops-mathmaster
Total: 1 files changed+162-86
Mmath/expint/op_kernel/expint_kernel.h+162-86
@@ -26,6 +26,8 @@ namespace NsExpint {
26using namespace AscendC;26using namespace AscendC;
27 27 
28constexpr float EULER_GAMMA = 0.5772156649015329f;28constexpr float EULER_GAMMA = 0.5772156649015329f;
29+constexpr float FP16_MAX_VALUE = 65504.0f;
30+constexpr float BF16_MAX_VALUE = 3.38953138927157e+38f;
29 31 
30static constexpr float A1[6] = {32static constexpr float A1[6] = {
31 -5.350447357812542947283E0f, 2.185049168816613393830E2f, -4.176572384826693777058E3f,33 -5.350447357812542947283E0f, 2.185049168816613393830E2f, -4.176572384826693777058E3f,
@@ -148,7 +150,15 @@ class ExpintKernel {
148 static constexpr int64_t CMP_ALIGN = 64; // 256 bytes / 4 bytes per float150 static constexpr int64_t CMP_ALIGN = 64; // 256 bytes / 4 bytes per float
149 static constexpr int64_t MASK_ELEM_PER_FLOAT = 8; // sizeof(float) / sizeof(uint8_t)151 static constexpr int64_t MASK_ELEM_PER_FLOAT = 8; // sizeof(float) / sizeof(uint8_t)
150 static constexpr int64_t MASK_ALIGN = 32; // mask buffer alignment in bytes152 static constexpr int64_t MASK_ALIGN = 32; // mask buffer alignment in bytes
151- static constexpr float NEAR_ZERO_THRESHOLD = 1e-10f;153+ static constexpr float EXP_CLAMP = 88.0f; // exp(88) < FLT_MAX, exp(89) > FLT_MAX
154+ static constexpr float ONE = 1.0f; // multiplicative identity / reciprocal numerator
155+ static constexpr float ZERO = 0.0f; // additive identity / zero boundary
156+ static constexpr float INTERVAL_BOUND_2 = 2.0f; // Interval 1/2 boundary
157+ static constexpr float INTERVAL_BOUND_4 = 4.0f; // Interval 2/3 boundary
158+ static constexpr float INTERVAL_BOUND_8 = 8.0f; // Interval 3/4 boundary
159+ static constexpr float INTERVAL_BOUND_16 = 16.0f; // Interval 4/5 boundary
160+ static constexpr float INTERVAL_BOUND_32 = 32.0f; // Interval 5/6 boundary
161+ static constexpr float INTERVAL_BOUND_64 = 64.0f; // Interval 6/7 boundary
152 162 
153public:163public:
154 __aicore__ inline ExpintKernel() {}164 __aicore__ inline ExpintKernel() {}
@@ -158,13 +168,10 @@ public:
158 168 
159private:169private:
160 __aicore__ inline void CopyIn(int64_t progress, int64_t currentNum);170 __aicore__ inline void CopyIn(int64_t progress, int64_t currentNum);
161- __aicore__ inline void Compute(LocalTensor<float> xLocal, LocalTensor<float> yLocal, int64_t count);171+ __aicore__ inline void Compute(LocalTensor<float> xInput, LocalTensor<float> yLocal, int64_t count);
172+ template <size_t N>
162 __aicore__ inline void HornerEval(LocalTensor<float> acc, LocalTensor<float> scratch, LocalTensor<float> var,173 __aicore__ inline void HornerEval(LocalTensor<float> acc, LocalTensor<float> scratch, LocalTensor<float> var,
163- const float* coeffs, int degree, int64_t n);174+ const float (&coeffs)[N], int64_t n);
164- __aicore__ inline void ComputeRationalInterval(LocalTensor<float> xFp32, const float* A, int degA, const float* B,
165- int degB, float threshold, int64_t n);
166- __aicore__ inline void ComputeInterval1(LocalTensor<float> xFp32, int64_t n);
167- __aicore__ inline void ApplyBoundaries(LocalTensor<float> xFp32, int64_t n);
168 __aicore__ inline void CopyOut(int64_t progress, int64_t currentNum);175 __aicore__ inline void CopyOut(int64_t progress, int64_t currentNum);
169 __aicore__ inline void ProcessFp32(int64_t loopCount);176 __aicore__ inline void ProcessFp32(int64_t loopCount);
170 __aicore__ inline void ProcessFp16Bf16(int64_t loopCount);177 __aicore__ inline void ProcessFp16Bf16(int64_t loopCount);
@@ -178,10 +185,11 @@ private:
178 185 
179 TBuf<TPosition::VECCALC> tmpBuf1;186 TBuf<TPosition::VECCALC> tmpBuf1;
180 TBuf<TPosition::VECCALC> tmpBuf2;187 TBuf<TPosition::VECCALC> tmpBuf2;
181- TBuf<TPosition::VECCALC> resultBuf;
182 TBuf<TPosition::VECCALC> oldResultBuf;188 TBuf<TPosition::VECCALC> oldResultBuf;
183 TBuf<TPosition::VECCALC> scratchBuf;189 TBuf<TPosition::VECCALC> scratchBuf;
184 TBuf<TPosition::VECCALC> maskBuf;190 TBuf<TPosition::VECCALC> maskBuf;
191+ TBuf<TPosition::VECCALC> initBuf;
192+ TBuf<TPosition::VECCALC> xClampedBuf;
185 193 
186 TBuf<TPosition::VECCALC> castInBuf;194 TBuf<TPosition::VECCALC> castInBuf;
187 TBuf<TPosition::VECCALC> resultFp32Buf;195 TBuf<TPosition::VECCALC> resultFp32Buf;
@@ -208,11 +216,15 @@ __aicore__ inline void ExpintKernel<T>::Init(GM_ADDR x, GM_ADDR y, const ExpintT
208 216 
209 pipe.InitBuffer(tmpBuf1, alignedUbLength_ * sizeof(float));217 pipe.InitBuffer(tmpBuf1, alignedUbLength_ * sizeof(float));
210 pipe.InitBuffer(tmpBuf2, alignedUbLength_ * sizeof(float));218 pipe.InitBuffer(tmpBuf2, alignedUbLength_ * sizeof(float));
211- pipe.InitBuffer(resultBuf, alignedUbLength_ * sizeof(float));
212 pipe.InitBuffer(oldResultBuf, alignedUbLength_ * sizeof(float));219 pipe.InitBuffer(oldResultBuf, alignedUbLength_ * sizeof(float));
213 pipe.InitBuffer(scratchBuf, alignedUbLength_ * sizeof(float));220 pipe.InitBuffer(scratchBuf, alignedUbLength_ * sizeof(float));
221+ pipe.InitBuffer(xClampedBuf, alignedUbLength_ * sizeof(float));
214 pipe.InitBuffer(maskBuf, ((alignedUbLength_ / MASK_ELEM_PER_FLOAT + MASK_ALIGN - 1) / MASK_ALIGN) * MASK_ALIGN);222 pipe.InitBuffer(maskBuf, ((alignedUbLength_ / MASK_ELEM_PER_FLOAT + MASK_ALIGN - 1) / MASK_ALIGN) * MASK_ALIGN);
215 223 
224+ if constexpr (std::is_same_v<T, float>) {
225+ pipe.InitBuffer(initBuf, alignedUbLength_ * sizeof(float));
226+ }
227+ 
216 if constexpr (NEED_CAST) {228 if constexpr (NEED_CAST) {
217 pipe.InitBuffer(castInBuf, alignedUbLength_ * sizeof(float));229 pipe.InitBuffer(castInBuf, alignedUbLength_ * sizeof(float));
218 pipe.InitBuffer(resultFp32Buf, alignedUbLength_ * sizeof(float));230 pipe.InitBuffer(resultFp32Buf, alignedUbLength_ * sizeof(float));
@@ -246,75 +258,145 @@ __aicore__ inline void ExpintKernel<T>::CopyOut(int64_t progress, int64_t curren
246}258}
247 259 
248template <typename T>260template <typename T>
261+template <size_t N>
249__aicore__ inline void ExpintKernel<T>::HornerEval(LocalTensor<float> acc, LocalTensor<float> scratch,262__aicore__ inline void ExpintKernel<T>::HornerEval(LocalTensor<float> acc, LocalTensor<float> scratch,
250- LocalTensor<float> var, const float* coeffs, int degree, int64_t n)263+ LocalTensor<float> var, const float (&coeffs)[N], int64_t n)
251{264{
252 Duplicate(acc, coeffs[0], n);265 Duplicate(acc, coeffs[0], n);
253- for (int i = 1; i <= degree; i++) {266+ for (size_t i = 1; i < N; i++) {
254 Mul(scratch, acc, var, n);267 Mul(scratch, acc, var, n);
255 Adds(acc, scratch, coeffs[i], n);268 Adds(acc, scratch, coeffs[i], n);
256 }269 }
257}270}
258 271 
259template <typename T>272template <typename T>
260-__aicore__ inline void ExpintKernel<T>::ComputeRationalInterval(LocalTensor<float> xFp32, const float* A, int degA,273+__aicore__ inline void ExpintKernel<T>::Compute(LocalTensor<float> xInput, LocalTensor<float> yLocal, int64_t count)
261- const float* B, int degB, float threshold, int64_t n)
262{274{
263 LocalTensor<float> tmp1 = tmpBuf1.Get<float>();275 LocalTensor<float> tmp1 = tmpBuf1.Get<float>();
264 LocalTensor<float> tmp2 = tmpBuf2.Get<float>();276 LocalTensor<float> tmp2 = tmpBuf2.Get<float>();
265- LocalTensor<float> result = resultBuf.Get<float>();
266- LocalTensor<float> oldResult = oldResultBuf.Get<float>();
267 LocalTensor<float> scratch = scratchBuf.Get<float>();277 LocalTensor<float> scratch = scratchBuf.Get<float>();
278+ 
279+ int64_t n = ((count + CMP_ALIGN - 1) / CMP_ALIGN) * CMP_ALIGN;
280+ 
281+ // Use yLocal directly as result buffer (separate result TBuf has issues on Ascend950)
282+ LocalTensor<float>& result = yLocal;
283+ LocalTensor<float> oldResult = oldResultBuf.Get<float>();
268 LocalTensor<uint8_t> mask = maskBuf.Get<uint8_t>();284 LocalTensor<uint8_t> mask = maskBuf.Get<uint8_t>();
269 285 
270- DataCopy(oldResult, result, n);286+ // Clamp x to EXP_CLAMP to prevent exp(x) overflow in float32
271- Duplicate(tmp1, 1.0f, n);287+ LocalTensor<float> xFp32 = xClampedBuf.Get<float>();
272- Div(result, tmp1, xFp32, n);288+ Duplicate(tmp1, EXP_CLAMP, n);
273- HornerEval(tmp1, tmp2, result, A, degA, n);289+ Compare(mask, xInput, tmp1, CMPMODE::GT, n);
274- HornerEval(tmp2, scratch, result, B, degB, n);290+ Select(xFp32, mask, tmp1, xInput, SELMODE::VSEL_TENSOR_TENSOR_MODE, n);
291+ 
292+ // Interval 7 (x >= 64): asymptotic expansion
293+ Duplicate(tmp1, ONE, n);
294+ Div(result, tmp1, xFp32, n); // w = 1/x
295+ HornerEval(tmp1, tmp2, result, A7, n); // P7(w)
296+ HornerEval(tmp2, scratch, result, B7, n); // Q7(w)
297+ Div(scratch, tmp1, tmp2, n); // P7/Q7
298+ Exp(tmp2, xFp32, n); // e^x
299+ Div(tmp1, tmp2, xFp32, n); // e^x/x
300+ Mul(tmp2, result, scratch, n); // w * P7/Q7
301+ Adds(tmp2, tmp2, ONE, n); // w * P7/Q7 + 1
302+ Mul(result, tmp1, tmp2, n); // e^x/x * (w*P7/Q7 + 1)
303+ 
304+ // Intervals 6-2: rational approximation with Select merge
305+ // Interval 6 (32 <= x < 64)
306+ Adds(oldResult, result, ZERO, n);
307+ Duplicate(tmp1, ONE, n);
308+ Div(result, tmp1, xFp32, n); // w = 1/x
309+ HornerEval(tmp1, tmp2, result, A6, n);
310+ HornerEval(tmp2, scratch, result, B6, n);
275 Div(scratch, tmp1, tmp2, n);311 Div(scratch, tmp1, tmp2, n);
276 Exp(tmp2, xFp32, n);312 Exp(tmp2, xFp32, n);
277 Div(tmp1, tmp2, xFp32, n);313 Div(tmp1, tmp2, xFp32, n);
278 Mul(tmp2, result, scratch, n);314 Mul(tmp2, result, scratch, n);
279- Adds(tmp2, tmp2, 1.0f, n);315+ Adds(tmp2, tmp2, ONE, n);
280 Mul(result, tmp1, tmp2, n);316 Mul(result, tmp1, tmp2, n);
281- Duplicate(tmp1, threshold, n);317+ Duplicate(tmp1, INTERVAL_BOUND_64, n);
282 Compare(mask, xFp32, tmp1, CMPMODE::LT, n);318 Compare(mask, xFp32, tmp1, CMPMODE::LT, n);
283 Select(result, mask, result, oldResult, SELMODE::VSEL_TENSOR_TENSOR_MODE, n);319 Select(result, mask, result, oldResult, SELMODE::VSEL_TENSOR_TENSOR_MODE, n);
284-}
285 320 
286-template <typename T>321+ // Interval 5 (16 <= x < 32)
287-__aicore__ inline void ExpintKernel<T>::ComputeInterval1(LocalTensor<float> xFp32, int64_t n)322+ Adds(oldResult, result, ZERO, n);
288-{323+ Duplicate(tmp1, ONE, n);
289- LocalTensor<float> tmp1 = tmpBuf1.Get<float>();324+ Div(result, tmp1, xFp32, n);
290- LocalTensor<float> tmp2 = tmpBuf2.Get<float>();325+ HornerEval(tmp1, tmp2, result, A5, n);
291- LocalTensor<float> result = resultBuf.Get<float>();326+ HornerEval(tmp2, scratch, result, B5, n);
292- LocalTensor<float> oldResult = oldResultBuf.Get<float>();327+ Div(scratch, tmp1, tmp2, n);
293- LocalTensor<float> scratch = scratchBuf.Get<float>();328+ Exp(tmp2, xFp32, n);
294- LocalTensor<uint8_t> mask = maskBuf.Get<uint8_t>();329+ Div(tmp1, tmp2, xFp32, n);
330+ Mul(tmp2, result, scratch, n);
331+ Adds(tmp2, tmp2, ONE, n);
332+ Mul(result, tmp1, tmp2, n);
333+ Duplicate(tmp1, INTERVAL_BOUND_32, n);
334+ Compare(mask, xFp32, tmp1, CMPMODE::LT, n);
335+ Select(result, mask, result, oldResult, SELMODE::VSEL_TENSOR_TENSOR_MODE, n);
295 336 
296- HornerEval(tmp1, tmp2, xFp32, A1, 5, n);337+ // Interval 4 (8 <= x < 16)
297- DataCopy(scratch, result, n);338+ Adds(oldResult, result, ZERO, n);
298- HornerEval(tmp2, oldResult, xFp32, B1, 6, n);339+ Duplicate(tmp1, ONE, n);
340+ Div(result, tmp1, xFp32, n);
341+ HornerEval(tmp1, tmp2, result, A4, n);
342+ HornerEval(tmp2, scratch, result, B4, n);
343+ Div(scratch, tmp1, tmp2, n);
344+ Exp(tmp2, xFp32, n);
345+ Div(tmp1, tmp2, xFp32, n);
346+ Mul(tmp2, result, scratch, n);
347+ Adds(tmp2, tmp2, ONE, n);
348+ Mul(result, tmp1, tmp2, n);
349+ Duplicate(tmp1, INTERVAL_BOUND_16, n);
350+ Compare(mask, xFp32, tmp1, CMPMODE::LT, n);
351+ Select(result, mask, result, oldResult, SELMODE::VSEL_TENSOR_TENSOR_MODE, n);
352+ 
353+ // Interval 3 (4 <= x < 8)
354+ Adds(oldResult, result, ZERO, n);
355+ Duplicate(tmp1, ONE, n);
356+ Div(result, tmp1, xFp32, n);
357+ HornerEval(tmp1, tmp2, result, A3, n);
358+ HornerEval(tmp2, scratch, result, B3, n);
359+ Div(scratch, tmp1, tmp2, n);
360+ Exp(tmp2, xFp32, n);
361+ Div(tmp1, tmp2, xFp32, n);
362+ Mul(tmp2, result, scratch, n);
363+ Adds(tmp2, tmp2, ONE, n);
364+ Mul(result, tmp1, tmp2, n);
365+ Duplicate(tmp1, INTERVAL_BOUND_8, n);
366+ Compare(mask, xFp32, tmp1, CMPMODE::LT, n);
367+ Select(result, mask, result, oldResult, SELMODE::VSEL_TENSOR_TENSOR_MODE, n);
368+ 
369+ // Interval 2 (2 <= x < 4)
370+ Adds(oldResult, result, ZERO, n);
371+ Duplicate(tmp1, ONE, n);
372+ Div(result, tmp1, xFp32, n);
373+ HornerEval(tmp1, tmp2, result, A2, n);
374+ HornerEval(tmp2, scratch, result, B2, n);
375+ Div(scratch, tmp1, tmp2, n);
376+ Exp(tmp2, xFp32, n);
377+ Div(tmp1, tmp2, xFp32, n);
378+ Mul(tmp2, result, scratch, n);
379+ Adds(tmp2, tmp2, ONE, n);
380+ Mul(result, tmp1, tmp2, n);
381+ Duplicate(tmp1, INTERVAL_BOUND_4, n);
382+ Compare(mask, xFp32, tmp1, CMPMODE::LT, n);
383+ Select(result, mask, result, oldResult, SELMODE::VSEL_TENSOR_TENSOR_MODE, n);
384+ 
385+ // Interval 1 (0 < x < 2): Ei(x) = gamma + ln(x) + x * P1(x)/Q1(x)
386+ HornerEval(tmp1, tmp2, xFp32, A1, n);
387+ Adds(scratch, result, ZERO, n);
388+ HornerEval(tmp2, oldResult, xFp32, B1, n);
299 Div(oldResult, tmp1, tmp2, n);389 Div(oldResult, tmp1, tmp2, n);
300 Mul(tmp1, oldResult, xFp32, n);390 Mul(tmp1, oldResult, xFp32, n);
301 Adds(tmp1, tmp1, EULER_GAMMA, n);391 Adds(tmp1, tmp1, EULER_GAMMA, n);
302 Ln(tmp2, xFp32, n);392 Ln(tmp2, xFp32, n);
303 Add(oldResult, tmp1, tmp2, n);393 Add(oldResult, tmp1, tmp2, n);
304- Duplicate(tmp1, 2.0f, n);394+ Duplicate(tmp1, INTERVAL_BOUND_2, n);
305 Compare(mask, xFp32, tmp1, CMPMODE::LT, n);395 Compare(mask, xFp32, tmp1, CMPMODE::LT, n);
306 Select(result, mask, oldResult, scratch, SELMODE::VSEL_TENSOR_TENSOR_MODE, n);396 Select(result, mask, oldResult, scratch, SELMODE::VSEL_TENSOR_TENSOR_MODE, n);
307-}
308 397 
309-template <typename T>398+ // ApplyBoundaries
310-__aicore__ inline void ExpintKernel<T>::ApplyBoundaries(LocalTensor<float> xFp32, int64_t n)399+ Duplicate(tmp1, ZERO, n);
311-{
312- LocalTensor<float> tmp1 = tmpBuf1.Get<float>();
313- LocalTensor<float> result = resultBuf.Get<float>();
314- LocalTensor<float> scratch = scratchBuf.Get<float>();
315- LocalTensor<uint8_t> mask = maskBuf.Get<uint8_t>();
316- 
317- Duplicate(tmp1, 0.0f, n);
318 Compare(mask, xFp32, tmp1, CMPMODE::LT, n);400 Compare(mask, xFp32, tmp1, CMPMODE::LT, n);
319 Duplicate(tmp1, std::numeric_limits<float>::quiet_NaN(), n);401 Duplicate(tmp1, std::numeric_limits<float>::quiet_NaN(), n);
320 Select(result, mask, tmp1, result, SELMODE::VSEL_TENSOR_TENSOR_MODE, n);402 Select(result, mask, tmp1, result, SELMODE::VSEL_TENSOR_TENSOR_MODE, n);
@@ -323,57 +405,36 @@ __aicore__ inline void ExpintKernel<T>::ApplyBoundaries(LocalTensor<float> xFp32
323 Compare(mask, xFp32, tmp1, CMPMODE::EQ, n);405 Compare(mask, xFp32, tmp1, CMPMODE::EQ, n);
324 Select(result, mask, tmp1, result, SELMODE::VSEL_TENSOR_TENSOR_MODE, n);406 Select(result, mask, tmp1, result, SELMODE::VSEL_TENSOR_TENSOR_MODE, n);
325 407 
326- Duplicate(tmp1, NEAR_ZERO_THRESHOLD, n);408+ // x == 0 -> -inf (exact comparison)
327- Abs(scratch, xFp32, n);409+ Duplicate(tmp1, ZERO, n);
328- Compare(mask, scratch, tmp1, CMPMODE::LE, n);410+ Compare(mask, xFp32, tmp1, CMPMODE::EQ, n);
329 Duplicate(tmp1, -std::numeric_limits<float>::infinity(), n);411 Duplicate(tmp1, -std::numeric_limits<float>::infinity(), n);
330 Select(result, mask, tmp1, result, SELMODE::VSEL_TENSOR_TENSOR_MODE, n);412 Select(result, mask, tmp1, result, SELMODE::VSEL_TENSOR_TENSOR_MODE, n);
331}413}
332 414 
333-template <typename T>
334-__aicore__ inline void ExpintKernel<T>::Compute(LocalTensor<float> xFp32, LocalTensor<float> yLocal, int64_t count)
335-{
336- LocalTensor<float> tmp1 = tmpBuf1.Get<float>();
337- LocalTensor<float> tmp2 = tmpBuf2.Get<float>();
338- LocalTensor<float> result = resultBuf.Get<float>();
339- LocalTensor<float> scratch = scratchBuf.Get<float>();
340- 
341- int64_t n = ((count + CMP_ALIGN - 1) / CMP_ALIGN) * CMP_ALIGN;
342- 
343- Duplicate(tmp1, 1.0f, n);
344- Div(result, tmp1, xFp32, n);
345- HornerEval(tmp1, tmp2, result, A7, 8, n);
346- HornerEval(tmp2, scratch, result, B7, 9, n);
347- Div(scratch, tmp1, tmp2, n);
348- Exp(tmp2, xFp32, n);
349- Div(tmp1, tmp2, xFp32, n);
350- Mul(tmp2, result, scratch, n);
351- Adds(tmp2, tmp2, 1.0f, n);
352- Mul(result, tmp1, tmp2, n);
353- 
354- ComputeRationalInterval(xFp32, A6, 5, B6, 5, 64.0f, n);
355- ComputeRationalInterval(xFp32, A5, 7, B5, 8, 32.0f, n);
356- ComputeRationalInterval(xFp32, A4, 9, B4, 9, 16.0f, n);
357- ComputeRationalInterval(xFp32, A3, 7, B3, 8, 8.0f, n);
358- ComputeRationalInterval(xFp32, A2, 7, B2, 7, 4.0f, n);
359- 
360- ComputeInterval1(xFp32, n);
361- ApplyBoundaries(xFp32, n);
362- DataCopy(yLocal, result, count);
363-}
364- 
365template <typename T>415template <typename T>
366__aicore__ inline void ExpintKernel<T>::ProcessFp32(int64_t loopCount)416__aicore__ inline void ExpintKernel<T>::ProcessFp32(int64_t loopCount)
367{417{
368 for (int64_t i = 0; i < loopCount; i++) {418 for (int64_t i = 0; i < loopCount; i++) {
369 int64_t currentNum = (i == (loopCount - 1)) ? (blockLength_ - ubLength_ * i) : ubLength_;419 int64_t currentNum = (i == (loopCount - 1)) ? (blockLength_ - ubLength_ * i) : ubLength_;
370- CopyIn(i, currentNum);420+ int64_t alignedCount = ((currentNum + CMP_ALIGN - 1) / CMP_ALIGN) * CMP_ALIGN;
421+ 
422+ LocalTensor<float> xFp32 = initBuf.Get<float>();
423+ Duplicate(xFp32, ONE, alignedCount);
424+ SetFlag<HardEvent::V_MTE2>(0);
425+ WaitFlag<HardEvent::V_MTE2>(0);
426+ DataCopyParams copyParams;
427+ copyParams.blockCount = 1;
428+ copyParams.blockLen = currentNum * sizeof(float);
429+ copyParams.srcStride = 0;
430+ copyParams.dstStride = 0;
431+ DataCopyPad(xFp32, inputGM[i * ubLength_], copyParams, {false, 0, 0, 0});
432+ SetFlag<HardEvent::MTE2_V>(0);
433+ WaitFlag<HardEvent::MTE2_V>(0);
371 434 
372- LocalTensor<float> xLocal = inputQueue.template DeQue<float>();
373 LocalTensor<float> yLocal = outputQueue.template AllocTensor<float>();435 LocalTensor<float> yLocal = outputQueue.template AllocTensor<float>();
374- Compute(xLocal, yLocal, currentNum);436+ Compute(xFp32, yLocal, currentNum);
375 outputQueue.template EnQue<float>(yLocal);437 outputQueue.template EnQue<float>(yLocal);
376- inputQueue.FreeTensor(xLocal);
377 438 
378 CopyOut(i, currentNum);439 CopyOut(i, currentNum);
379 }440 }
@@ -390,12 +451,27 @@ __aicore__ inline void ExpintKernel<T>::ProcessFp16Bf16(int64_t loopCount)
390 451 
391 LocalTensor<float> xFp32 = castInBuf.Get<float>();452 LocalTensor<float> xFp32 = castInBuf.Get<float>();
392 int64_t alignedCount = ((currentNum + CMP_ALIGN - 1) / CMP_ALIGN) * CMP_ALIGN;453 int64_t alignedCount = ((currentNum + CMP_ALIGN - 1) / CMP_ALIGN) * CMP_ALIGN;
393- Duplicate(xFp32, 1.0f, alignedCount);454+ Duplicate(xFp32, ONE, alignedCount);
394 Cast<float, T>(xFp32, xInput, RoundMode::CAST_NONE, currentNum);455 Cast<float, T>(xFp32, xInput, RoundMode::CAST_NONE, currentNum);
395 456 
396 LocalTensor<float> resultFp32 = resultFp32Buf.Get<float>();457 LocalTensor<float> resultFp32 = resultFp32Buf.Get<float>();
397 Compute(xFp32, resultFp32, currentNum);458 Compute(xFp32, resultFp32, currentNum);
398 459 
460+ // Clamp float32 result to target dtype range to prevent Cast overflow to inf
461+ LocalTensor<float> clampVal = castInBuf.Get<float>();
462+ LocalTensor<uint8_t> clampMask = maskBuf.Get<uint8_t>();
463+ float maxVal;
464+ if constexpr (std::is_same_v<T, half>) {
465+ maxVal = FP16_MAX_VALUE;
466+ } else {
467+ maxVal = BF16_MAX_VALUE;
468+ }
469+ Duplicate(clampVal, maxVal, alignedCount);
470+ Compare(clampMask, resultFp32, clampVal, CMPMODE::GT, alignedCount);
471+ LocalTensor<float> clampTmp = oldResultBuf.Get<float>();
472+ Select(clampTmp, clampMask, clampVal, resultFp32, SELMODE::VSEL_TENSOR_TENSOR_MODE, alignedCount);
473+ Adds(resultFp32, clampTmp, ZERO, alignedCount);
474+ 
399 LocalTensor<T> yOutput = outputQueue.template AllocTensor<T>();475 LocalTensor<T> yOutput = outputQueue.template AllocTensor<T>();
400 Cast<T, float>(yOutput, resultFp32, RoundMode::CAST_ROUND, currentNum);476 Cast<T, float>(yOutput, resultFp32, RoundMode::CAST_ROUND, currentNum);
401 outputQueue.template EnQue<T>(yOutput);477 outputQueue.template EnQue<T>(yOutput);