已合并
MicroAPI namespace to Reg for arch35 kernels #9267
shichenyu创建于 18 天前
MicroAPI namespace to Reg for arch35 kernels #9267
已合并
共 216 个文件变更+11512-11962
| @@ -26,18 +26,18 @@ | |||
| 26 | namespace ClippedSwigluOp { | 26 | namespace ClippedSwigluOp { |
| 27 | 27 | ||
| 28 | using namespace AscendC; | 28 | using namespace AscendC; |
| 29 | -using namespace AscendC::MicroAPI; | 29 | +using namespace AscendC::Reg; |
| 30 | 30 | ||
| 31 | constexpr int64_t DB_BUFFER = 2; | 31 | constexpr int64_t DB_BUFFER = 2; |
| 32 | constexpr int64_t BLOCK_SIZE = Ops::Base::GetUbBlockSize(); // 32 | 32 | constexpr int64_t BLOCK_SIZE = Ops::Base::GetUbBlockSize(); // 32 |
| 33 | constexpr int64_t DIM_HALVE = 2; | 33 | constexpr int64_t DIM_HALVE = 2; |
| 34 | constexpr uint32_t VF_LEN_FP32 = Ops::Base::GetVRegSize() / sizeof(float); | 34 | constexpr uint32_t VF_LEN_FP32 = Ops::Base::GetVRegSize() / sizeof(float); |
| 35 | 35 | ||
| 36 | -static constexpr AscendC::MicroAPI::CastTrait CAST_BF16_FP16_TO_FP32 = { | 36 | +static constexpr AscendC::Reg::CastTrait CAST_BF16_FP16_TO_FP32 = { |
| 37 | - AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::UNKNOWN, AscendC::MicroAPI::MaskMergeMode::ZEROING, | 37 | + AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::UNKNOWN, AscendC::Reg::MaskMergeMode::ZEROING, |
| 38 | AscendC::RoundMode::UNKNOWN}; | 38 | AscendC::RoundMode::UNKNOWN}; |
| 39 | -constexpr static AscendC::MicroAPI::CastTrait CAST_FP32_TO_FP16_BF16 = { | 39 | +constexpr static AscendC::Reg::CastTrait CAST_FP32_TO_FP16_BF16 = { |
| 40 | - AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::NO_SAT, AscendC::MicroAPI::MaskMergeMode::ZEROING, | 40 | + AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::NO_SAT, AscendC::Reg::MaskMergeMode::ZEROING, |
| 41 | AscendC::RoundMode::CAST_RINT}; | 41 | AscendC::RoundMode::CAST_RINT}; |
| 42 | 42 | ||
| 43 | __aicore__ inline void ReduceAllVf(LocalTensor<int64_t>& reduceSumUb, LocalTensor<int64_t>& groupIndexUb, | 43 | __aicore__ inline void ReduceAllVf(LocalTensor<int64_t>& reduceSumUb, LocalTensor<int64_t>& groupIndexUb, |
| @@ -52,26 +52,26 @@ __aicore__ inline void ReduceAllVf(LocalTensor<int64_t>& reduceSumUb, LocalTenso | |||
| 52 | auto srcAddr1 = (__ubuf__ int64_t*)groupIndexUb[times * vfTidx].GetPhyAddr(); | 52 | auto srcAddr1 = (__ubuf__ int64_t*)groupIndexUb[times * vfTidx].GetPhyAddr(); |
| 53 | __VEC_SCOPE__ | 53 | __VEC_SCOPE__ |
| 54 | { | 54 | { |
| 55 | - AscendC::MicroAPI::RegTensor<int64_t> addReg; | 55 | + AscendC::Reg::RegTensor<int64_t> addReg; |
| 56 | - AscendC::MicroAPI::RegTensor<int64_t> reduceSumReg; | 56 | + AscendC::Reg::RegTensor<int64_t> reduceSumReg; |
| 57 | - AscendC::MicroAPI::RegTensor<int64_t> reduceSumTReg; | 57 | + AscendC::Reg::RegTensor<int64_t> reduceSumTReg; |
| 58 | - AscendC::MicroAPI::RegTensor<int64_t> srcReg; | 58 | + AscendC::Reg::RegTensor<int64_t> srcReg; |
| 59 | - AscendC::MicroAPI::Duplicate(addReg, 0); | 59 | + AscendC::Reg::Duplicate(addReg, 0); |
| 60 | - AscendC::MicroAPI::MaskReg mask = AscendC::MicroAPI::CreateMask<int64_t, MicroAPI::MaskPattern::ALL>(); | 60 | + AscendC::Reg::MaskReg mask = AscendC::Reg::CreateMask<int64_t, Reg::MaskPattern::ALL>(); |
| 61 | for (uint16_t i = 0; i < times; i++) { | 61 | for (uint16_t i = 0; i < times; i++) { |
| 62 | - AscendC::MicroAPI::AddrReg srcIdxOffset = AscendC::MicroAPI::CreateAddrReg<int64_t>(i, vfTidx); | 62 | + AscendC::Reg::AddrReg srcIdxOffset = AscendC::Reg::CreateAddrReg<int64_t>(i, vfTidx); |
| 63 | - AscendC::MicroAPI::LoadAlign(srcReg, srcAddr, srcIdxOffset); | 63 | + AscendC::Reg::LoadAlign(srcReg, srcAddr, srcIdxOffset); |
| 64 | - AscendC::MicroAPI::Add(addReg, addReg, srcReg, mask); | 64 | + AscendC::Reg::Add(addReg, addReg, srcReg, mask); |
| 65 | } | 65 | } |
| 66 | - AscendC::MicroAPI::Reduce<AscendC::Reg::ReduceType::SUM>(reduceSumReg, addReg, mask); | 66 | + AscendC::Reg::Reduce<AscendC::Reg::ReduceType::SUM>(reduceSumReg, addReg, mask); |
| 67 | for (uint16_t j = 0; j < tailTimes; j++) { | 67 | for (uint16_t j = 0; j < tailTimes; j++) { |
| 68 | - AscendC::MicroAPI::MaskReg maskT = AscendC::MicroAPI::UpdateMask<int64_t>(tailNum); | 68 | + AscendC::Reg::MaskReg maskT = AscendC::Reg::UpdateMask<int64_t>(tailNum); |
| 69 | - AscendC::MicroAPI::LoadAlign(srcReg, srcAddr1); | 69 | + AscendC::Reg::LoadAlign(srcReg, srcAddr1); |
| 70 | - AscendC::MicroAPI::Reduce<AscendC::Reg::ReduceType::SUM>(reduceSumTReg, srcReg, maskT); | 70 | + AscendC::Reg::Reduce<AscendC::Reg::ReduceType::SUM>(reduceSumTReg, srcReg, maskT); |
| 71 | - AscendC::MicroAPI::Add(reduceSumReg, reduceSumTReg, reduceSumReg, maskT); | 71 | + AscendC::Reg::Add(reduceSumReg, reduceSumTReg, reduceSumReg, maskT); |
| 72 | } | 72 | } |
| 73 | - AscendC::MicroAPI::MaskReg maskOne = AscendC::MicroAPI::CreateMask<int64_t, MicroAPI::MaskPattern::VL1>(); | 73 | + AscendC::Reg::MaskReg maskOne = AscendC::Reg::CreateMask<int64_t, Reg::MaskPattern::VL1>(); |
| 74 | - AscendC::MicroAPI::StoreAlign(dstAddr, reduceSumReg, maskOne); | 74 | + AscendC::Reg::StoreAlign(dstAddr, reduceSumReg, maskOne); |
| 75 | } | 75 | } |
| 76 | } | 76 | } |
| 77 | 77 | ||
| @@ -305,13 +305,13 @@ __aicore__ inline void ClippedSwigluKernel<T, isInterleaved, isGroup, clampMode> | |||
| 305 | uint16_t times = CeilDivision(size, vfLen); | 305 | uint16_t times = CeilDivision(size, vfLen); |
| 306 | __VEC_SCOPE__ | 306 | __VEC_SCOPE__ |
| 307 | { | 307 | { |
| 308 | - AscendC::MicroAPI::RegTensor<T> xReg; | 308 | + AscendC::Reg::RegTensor<T> xReg; |
| 309 | - AscendC::MicroAPI::MaskReg mask; | 309 | + AscendC::Reg::MaskReg mask; |
| 310 | for (uint16_t i = 0; i < times; i++) { | 310 | for (uint16_t i = 0; i < times; i++) { |
| 311 | - mask = MicroAPI::UpdateMask<T>(size); | 311 | + mask = Reg::UpdateMask<T>(size); |
| 312 | - AscendC::MicroAPI::AddrReg srcIdxOffset = AscendC::MicroAPI::CreateAddrReg<T>(i, vfLen); | 312 | + AscendC::Reg::AddrReg srcIdxOffset = AscendC::Reg::CreateAddrReg<T>(i, vfLen); |
| 313 | - AscendC::MicroAPI::LoadAlign(xReg, inAddr, srcIdxOffset); | 313 | + AscendC::Reg::LoadAlign(xReg, inAddr, srcIdxOffset); |
| 314 | - AscendC::MicroAPI::StoreAlign(outAddr, xReg, srcIdxOffset, mask); | 314 | + AscendC::Reg::StoreAlign(outAddr, xReg, srcIdxOffset, mask); |
| 315 | } | 315 | } |
| 316 | } | 316 | } |
| 317 | } | 317 | } |
| @@ -351,147 +351,146 @@ __aicore__ inline void ClippedSwigluKernel<T, isInterleaved, isGroup, clampMode> | |||
| 351 | __ubuf__ T* swigluUbAddrT = swigluUbAddr + dim1VfTimes * VF_LEN_FP32; | 351 | __ubuf__ T* swigluUbAddrT = swigluUbAddr + dim1VfTimes * VF_LEN_FP32; |
| 352 | __VEC_SCOPE__ | 352 | __VEC_SCOPE__ |
| 353 | { | 353 | { |
| 354 | - AscendC::MicroAPI::RegTensor<T> vregX1; | 354 | + AscendC::Reg::RegTensor<T> vregX1; |
| 355 | - AscendC::MicroAPI::RegTensor<T> vregX2; | 355 | + AscendC::Reg::RegTensor<T> vregX2; |
| 356 | - AscendC::MicroAPI::RegTensor<float> vregX1F; | 356 | + AscendC::Reg::RegTensor<float> vregX1F; |
| 357 | - AscendC::MicroAPI::RegTensor<float> vregX2F; | 357 | + AscendC::Reg::RegTensor<float> vregX2F; |
| 358 | 358 | ||
| 359 | - AscendC::MicroAPI::RegTensor<float> vregX1DeF; | 359 | + AscendC::Reg::RegTensor<float> vregX1DeF; |
| 360 | - AscendC::MicroAPI::RegTensor<float> vregX2DeF; | 360 | + AscendC::Reg::RegTensor<float> vregX2DeF; |
| 361 | - AscendC::MicroAPI::RegTensor<float> minsReg; | 361 | + AscendC::Reg::RegTensor<float> minsReg; |
| 362 | - AscendC::MicroAPI::RegTensor<float> mulsReg; | 362 | + AscendC::Reg::RegTensor<float> mulsReg; |
| 363 | - AscendC::MicroAPI::RegTensor<float> expReg; | 363 | + AscendC::Reg::RegTensor<float> expReg; |
| 364 | - AscendC::MicroAPI::RegTensor<float> addsReg; | 364 | + AscendC::Reg::RegTensor<float> addsReg; |
| 365 | - AscendC::MicroAPI::RegTensor<float> sigmoidReg; | 365 | + AscendC::Reg::RegTensor<float> sigmoidReg; |
| 366 | - AscendC::MicroAPI::RegTensor<float> outFReg; | 366 | + AscendC::Reg::RegTensor<float> outFReg; |
| 367 | - AscendC::MicroAPI::RegTensor<float> vregX1NegDeF; | 367 | + AscendC::Reg::RegTensor<float> vregX1NegDeF; |
| 368 | - AscendC::MicroAPI::RegTensor<T> outTReg; | 368 | + AscendC::Reg::RegTensor<T> outTReg; |
| 369 | - MicroAPI::MaskReg mask = MicroAPI::CreateMask<float, MicroAPI::MaskPattern::ALL>(); | 369 | + Reg::MaskReg mask = Reg::CreateMask<float, Reg::MaskPattern::ALL>(); |
| 370 | - MicroAPI::MaskReg maskT = MicroAPI::UpdateMask<float>(tail); | 370 | + Reg::MaskReg maskT = Reg::UpdateMask<float>(tail); |
| 371 | for (uint16_t dim0vfLoopIdx = 0; dim0vfLoopIdx < dim0VfTimes; dim0vfLoopIdx++) { | 371 | for (uint16_t dim0vfLoopIdx = 0; dim0vfLoopIdx < dim0VfTimes; dim0vfLoopIdx++) { |
| 372 | for (uint16_t dim1vfLoopIdx = 0; dim1vfLoopIdx < dim1VfTimes; dim1vfLoopIdx++) { | 372 | for (uint16_t dim1vfLoopIdx = 0; dim1vfLoopIdx < dim1VfTimes; dim1vfLoopIdx++) { |
| 373 | - AscendC::MicroAPI::AddrReg srcIdxOffset = AscendC::MicroAPI::CreateAddrReg<T>( | 373 | + AscendC::Reg::AddrReg srcIdxOffset = AscendC::Reg::CreateAddrReg<T>(dim0vfLoopIdx, alignDim1In, |
| 374 | - dim0vfLoopIdx, alignDim1In, dim1vfLoopIdx, vfLen); | 374 | + dim1vfLoopIdx, vfLen); |
| 375 | if constexpr (isInterleaved) { | 375 | if constexpr (isInterleaved) { |
| 376 | if constexpr (sizeof(T) == sizeof(half)) { | 376 | if constexpr (sizeof(T) == sizeof(half)) { |
| 377 | - AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX1, x1UbAddr, | 377 | + AscendC::Reg::LoadAlign<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregX1, x1UbAddr, |
| 378 | - srcIdxOffset); | 378 | + srcIdxOffset); |
| 379 | - AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX2, x2UbAddr, | 379 | + AscendC::Reg::LoadAlign<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregX2, x2UbAddr, |
| 380 | - srcIdxOffset); | 380 | + srcIdxOffset); |
| 381 | - AscendC::MicroAPI::Cast<float, T, CAST_BF16_FP16_TO_FP32>(vregX1F, vregX1, mask); | 381 | + AscendC::Reg::Cast<float, T, CAST_BF16_FP16_TO_FP32>(vregX1F, vregX1, mask); |
| 382 | - AscendC::MicroAPI::Cast<float, T, CAST_BF16_FP16_TO_FP32>(vregX2F, vregX2, mask); | 382 | + AscendC::Reg::Cast<float, T, CAST_BF16_FP16_TO_FP32>(vregX2F, vregX2, mask); |
| 383 | } else { | 383 | } else { |
| 384 | // float | 384 | // float |
| 385 | - AscendC::MicroAPI::LoadAlign((MicroAPI::RegTensor<T>&)vregX1F, x1UbAddr, srcIdxOffset); | 385 | + AscendC::Reg::LoadAlign((Reg::RegTensor<T>&)vregX1F, x1UbAddr, srcIdxOffset); |
| 386 | - AscendC::MicroAPI::LoadAlign((MicroAPI::RegTensor<T>&)vregX2F, x2UbAddr, srcIdxOffset); | 386 | + AscendC::Reg::LoadAlign((Reg::RegTensor<T>&)vregX2F, x2UbAddr, srcIdxOffset); |
| 387 | } | 387 | } |
| 388 | - AscendC::MicroAPI::DeInterleave(vregX1DeF, vregX2DeF, vregX1F, vregX2F); | 388 | + AscendC::Reg::DeInterleave(vregX1DeF, vregX2DeF, vregX1F, vregX2F); |
| 389 | } else { | 389 | } else { |
| 390 | if constexpr (sizeof(T) == sizeof(half)) { | 390 | if constexpr (sizeof(T) == sizeof(half)) { |
| 391 | - AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX1, x1UbAddr, | 391 | + AscendC::Reg::LoadAlign<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregX1, x1UbAddr, |
| 392 | - srcIdxOffset); | 392 | + srcIdxOffset); |
| 393 | - AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX2, x2UbAddr, | 393 | + AscendC::Reg::LoadAlign<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregX2, x2UbAddr, |
| 394 | - srcIdxOffset); | 394 | + srcIdxOffset); |
| 395 | - AscendC::MicroAPI::Cast<float, T, CAST_BF16_FP16_TO_FP32>(vregX1DeF, vregX1, mask); | 395 | + AscendC::Reg::Cast<float, T, CAST_BF16_FP16_TO_FP32>(vregX1DeF, vregX1, mask); |
| 396 | - AscendC::MicroAPI::Cast<float, T, CAST_BF16_FP16_TO_FP32>(vregX2DeF, vregX2, mask); | 396 | + AscendC::Reg::Cast<float, T, CAST_BF16_FP16_TO_FP32>(vregX2DeF, vregX2, mask); |
| 397 | } else { | 397 | } else { |
| 398 | // float | 398 | // float |
| 399 | - AscendC::MicroAPI::LoadAlign((MicroAPI::RegTensor<T>&)vregX1DeF, x1UbAddr, srcIdxOffset); | 399 | + AscendC::Reg::LoadAlign((Reg::RegTensor<T>&)vregX1DeF, x1UbAddr, srcIdxOffset); |
| 400 | - AscendC::MicroAPI::LoadAlign((MicroAPI::RegTensor<T>&)vregX2DeF, x2UbAddr, srcIdxOffset); | 400 | + AscendC::Reg::LoadAlign((Reg::RegTensor<T>&)vregX2DeF, x2UbAddr, srcIdxOffset); |
| 401 | } | 401 | } |
| 402 | } | 402 | } |
| 403 | if constexpr (clampMode == 0) { | 403 | if constexpr (clampMode == 0) { |
| 404 | - AscendC::MicroAPI::Mins(minsReg, vregX1DeF, clampLimit, mask); | 404 | + AscendC::Reg::Mins(minsReg, vregX1DeF, clampLimit, mask); |
| 405 | - AscendC::MicroAPI::Muls(mulsReg, minsReg, negAlpha, mask); | 405 | + AscendC::Reg::Muls(mulsReg, minsReg, negAlpha, mask); |
| 406 | - AscendC::MicroAPI::Exp(expReg, mulsReg, mask); | 406 | + AscendC::Reg::Exp(expReg, mulsReg, mask); |
| 407 | - AscendC::MicroAPI::Adds(addsReg, expReg, scalarOne, mask); | 407 | + AscendC::Reg::Adds(addsReg, expReg, scalarOne, mask); |
| 408 | - AscendC::MicroAPI::Div(sigmoidReg, minsReg, addsReg, mask); | 408 | + AscendC::Reg::Div(sigmoidReg, minsReg, addsReg, mask); |
| 409 | 409 | ||
| 410 | - AscendC::MicroAPI::Mins(vregX2DeF, vregX2DeF, clampLimit, mask); | 410 | + AscendC::Reg::Mins(vregX2DeF, vregX2DeF, clampLimit, mask); |
| 411 | - AscendC::MicroAPI::Maxs(vregX2DeF, vregX2DeF, negClampLimit, mask); | 411 | + AscendC::Reg::Maxs(vregX2DeF, vregX2DeF, negClampLimit, mask); |
| 412 | - AscendC::MicroAPI::Adds(vregX2DeF, vregX2DeF, gluBias, mask); | 412 | + AscendC::Reg::Adds(vregX2DeF, vregX2DeF, gluBias, mask); |
| 413 | 413 | ||
| 414 | - AscendC::MicroAPI::Mul(outFReg, sigmoidReg, vregX2DeF, mask); | 414 | + AscendC::Reg::Mul(outFReg, sigmoidReg, vregX2DeF, mask); |
| 415 | } else { | 415 | } else { |
| 416 | - AscendC::MicroAPI::Muls(vregX1NegDeF, vregX1DeF, negScalarOne, mask); | 416 | + AscendC::Reg::Muls(vregX1NegDeF, vregX1DeF, negScalarOne, mask); |
| 417 | - AscendC::MicroAPI::Exp(expReg, vregX1NegDeF, mask); | 417 | + AscendC::Reg::Exp(expReg, vregX1NegDeF, mask); |
| 418 | - AscendC::MicroAPI::Adds(addsReg, expReg, scalarOne, mask); | 418 | + AscendC::Reg::Adds(addsReg, expReg, scalarOne, mask); |
| 419 | - AscendC::MicroAPI::Div(sigmoidReg, vregX1DeF, addsReg, mask); | 419 | + AscendC::Reg::Div(sigmoidReg, vregX1DeF, addsReg, mask); |
| 420 | - AscendC::MicroAPI::Mins(minsReg, sigmoidReg, clampLimit, mask); | 420 | + AscendC::Reg::Mins(minsReg, sigmoidReg, clampLimit, mask); |
| 421 | 421 | ||
| 422 | - AscendC::MicroAPI::Mins(vregX2DeF, vregX2DeF, clampLimit, mask); | 422 | + AscendC::Reg::Mins(vregX2DeF, vregX2DeF, clampLimit, mask); |
| 423 | - AscendC::MicroAPI::Maxs(vregX2DeF, vregX2DeF, negClampLimit, mask); | 423 | + AscendC::Reg::Maxs(vregX2DeF, vregX2DeF, negClampLimit, mask); |
| 424 | - AscendC::MicroAPI::Mul(outFReg, minsReg, vregX2DeF, mask); | 424 | + AscendC::Reg::Mul(outFReg, minsReg, vregX2DeF, mask); |
| 425 | } | 425 | } |
| 426 | - AscendC::MicroAPI::AddrReg outOffset = AscendC::MicroAPI::CreateAddrReg<T>(dim0vfLoopIdx, alignDim1Out, | 426 | + AscendC::Reg::AddrReg outOffset = AscendC::Reg::CreateAddrReg<T>(dim0vfLoopIdx, alignDim1Out, |
| 427 | - dim1vfLoopIdx, VF_LEN_FP32); | 427 | + dim1vfLoopIdx, VF_LEN_FP32); |
| 428 | if constexpr (sizeof(T) == sizeof(half)) { | 428 | if constexpr (sizeof(T) == sizeof(half)) { |
| 429 | - AscendC::MicroAPI::Cast<T, float, CAST_FP32_TO_FP16_BF16>(outTReg, outFReg, mask); | 429 | + AscendC::Reg::Cast<T, float, CAST_FP32_TO_FP16_BF16>(outTReg, outFReg, mask); |
| 430 | - StoreAlign<T, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(swigluUbAddr, outTReg, outOffset, mask); | 430 | + StoreAlign<T, AscendC::Reg::StoreDist::DIST_PACK_B32>(swigluUbAddr, outTReg, outOffset, mask); |
| 431 | } else { | 431 | } else { |
| 432 | - StoreAlign(swigluUbAddr, (MicroAPI::RegTensor<T>&)outFReg, outOffset, mask); | 432 | + StoreAlign(swigluUbAddr, (Reg::RegTensor<T>&)outFReg, outOffset, mask); |
| 433 | } | 433 | } |
| 434 | } | 434 | } |
| 435 | - AscendC::MicroAPI::AddrReg srcIdxOffset1 = AscendC::MicroAPI::CreateAddrReg<T>(dim0vfLoopIdx, alignDim1In); | 435 | + AscendC::Reg::AddrReg srcIdxOffset1 = AscendC::Reg::CreateAddrReg<T>(dim0vfLoopIdx, alignDim1In); |
| 436 | - AscendC::MicroAPI::AddrReg outOffset1 = AscendC::MicroAPI::CreateAddrReg<T>(dim0vfLoopIdx, alignDim1Out); | 436 | + AscendC::Reg::AddrReg outOffset1 = AscendC::Reg::CreateAddrReg<T>(dim0vfLoopIdx, alignDim1Out); |
| 437 | for (uint16_t ti = 0; ti < tailTimes; ti++) { | 437 | for (uint16_t ti = 0; ti < tailTimes; ti++) { |
| 438 | if constexpr (isInterleaved) { | 438 | if constexpr (isInterleaved) { |
| 439 | if constexpr (sizeof(T) == sizeof(half)) { | 439 | if constexpr (sizeof(T) == sizeof(half)) { |
| 440 | - AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX1, x1UbAddrT, | 440 | + AscendC::Reg::LoadAlign<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregX1, x1UbAddrT, |
| 441 | - srcIdxOffset1); | 441 | + srcIdxOffset1); |
| 442 | - AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX2, x2UbAddrT, | 442 | + AscendC::Reg::LoadAlign<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregX2, x2UbAddrT, |
| 443 | - srcIdxOffset1); | 443 | + srcIdxOffset1); |
| 444 | - AscendC::MicroAPI::Cast<float, T, CAST_BF16_FP16_TO_FP32>(vregX1F, vregX1, mask); | 444 | + AscendC::Reg::Cast<float, T, CAST_BF16_FP16_TO_FP32>(vregX1F, vregX1, mask); |
| 445 | - AscendC::MicroAPI::Cast<float, T, CAST_BF16_FP16_TO_FP32>(vregX2F, vregX2, mask); | 445 | + AscendC::Reg::Cast<float, T, CAST_BF16_FP16_TO_FP32>(vregX2F, vregX2, mask); |
| 446 | } else { | 446 | } else { |
| 447 | // float | 447 | // float |
| 448 | - AscendC::MicroAPI::LoadAlign((MicroAPI::RegTensor<T>&)vregX1F, x1UbAddrT, srcIdxOffset1); | 448 | + AscendC::Reg::LoadAlign((Reg::RegTensor<T>&)vregX1F, x1UbAddrT, srcIdxOffset1); |
| 449 | - AscendC::MicroAPI::LoadAlign((MicroAPI::RegTensor<T>&)vregX2F, x2UbAddrT, srcIdxOffset1); | 449 | + AscendC::Reg::LoadAlign((Reg::RegTensor<T>&)vregX2F, x2UbAddrT, srcIdxOffset1); |
| 450 | } | 450 | } |
| 451 | - AscendC::MicroAPI::DeInterleave(vregX1DeF, vregX2DeF, vregX1F, vregX2F); | 451 | + AscendC::Reg::DeInterleave(vregX1DeF, vregX2DeF, vregX1F, vregX2F); |
| 452 | } else { | 452 | } else { |
| 453 | if constexpr (sizeof(T) == sizeof(half)) { | 453 | if constexpr (sizeof(T) == sizeof(half)) { |
| 454 | - AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX1, x1UbAddrT, | 454 | + AscendC::Reg::LoadAlign<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregX1, x1UbAddrT, |
| 455 | - srcIdxOffset1); | 455 | + srcIdxOffset1); |
| 456 | - AscendC::MicroAPI::LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX2, x2UbAddrT, | 456 | + AscendC::Reg::LoadAlign<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vregX2, x2UbAddrT, |
| 457 | - srcIdxOffset1); | 457 | + srcIdxOffset1); |
| 458 | - AscendC::MicroAPI::Cast<float, T, CAST_BF16_FP16_TO_FP32>(vregX1DeF, vregX1, mask); | 458 | + AscendC::Reg::Cast<float, T, CAST_BF16_FP16_TO_FP32>(vregX1DeF, vregX1, mask); |
| 459 | - AscendC::MicroAPI::Cast<float, T, CAST_BF16_FP16_TO_FP32>(vregX2DeF, vregX2, mask); | 459 | + AscendC::Reg::Cast<float, T, CAST_BF16_FP16_TO_FP32>(vregX2DeF, vregX2, mask); |
| 460 | } else { | 460 | } else { |
| 461 | // float | 461 | // float |
| 462 | - AscendC::MicroAPI::LoadAlign((MicroAPI::RegTensor<T>&)vregX1DeF, x1UbAddrT, srcIdxOffset1); | 462 | + AscendC::Reg::LoadAlign((Reg::RegTensor<T>&)vregX1DeF, x1UbAddrT, srcIdxOffset1); |
| 463 | - AscendC::MicroAPI::LoadAlign((MicroAPI::RegTensor<T>&)vregX2DeF, x2UbAddrT, srcIdxOffset1); | 463 | + AscendC::Reg::LoadAlign((Reg::RegTensor<T>&)vregX2DeF, x2UbAddrT, srcIdxOffset1); |
| 464 | } | 464 | } |
| 465 | } | 465 | } |
| 466 | if constexpr (clampMode == 0) { | 466 | if constexpr (clampMode == 0) { |
| 467 | - AscendC::MicroAPI::Mins(minsReg, vregX1DeF, clampLimit, maskT); | 467 | + AscendC::Reg::Mins(minsReg, vregX1DeF, clampLimit, maskT); |
| 468 | - AscendC::MicroAPI::Muls(mulsReg, minsReg, negAlpha, maskT); | 468 | + AscendC::Reg::Muls(mulsReg, minsReg, negAlpha, maskT); |
| 469 | - AscendC::MicroAPI::Exp(expReg, mulsReg, maskT); | 469 | + AscendC::Reg::Exp(expReg, mulsReg, maskT); |
| 470 | - AscendC::MicroAPI::Adds(addsReg, expReg, scalarOne, maskT); | 470 | + AscendC::Reg::Adds(addsReg, expReg, scalarOne, maskT); |
| 471 | - AscendC::MicroAPI::Div(sigmoidReg, minsReg, addsReg, maskT); | 471 | + AscendC::Reg::Div(sigmoidReg, minsReg, addsReg, maskT); |
| 472 | 472 | ||
| 473 | - AscendC::MicroAPI::Mins(vregX2DeF, vregX2DeF, clampLimit, maskT); | 473 | + AscendC::Reg::Mins(vregX2DeF, vregX2DeF, clampLimit, maskT); |
| 474 | - AscendC::MicroAPI::Maxs(vregX2DeF, vregX2DeF, negClampLimit, maskT); | 474 | + AscendC::Reg::Maxs(vregX2DeF, vregX2DeF, negClampLimit, maskT); |
| 475 | - AscendC::MicroAPI::Adds(vregX2DeF, vregX2DeF, gluBias, maskT); | 475 | + AscendC::Reg::Adds(vregX2DeF, vregX2DeF, gluBias, maskT); |
| 476 | 476 | ||
| 477 | - AscendC::MicroAPI::Mul(outFReg, sigmoidReg, vregX2DeF, maskT); | 477 | + AscendC::Reg::Mul(outFReg, sigmoidReg, vregX2DeF, maskT); |
| 478 | } else { | 478 | } else { |
| 479 | - AscendC::MicroAPI::Muls(vregX1NegDeF, vregX1DeF, negScalarOne, maskT); | 479 | + AscendC::Reg::Muls(vregX1NegDeF, vregX1DeF, negScalarOne, maskT); |
| 480 | - AscendC::MicroAPI::Exp(expReg, vregX1NegDeF, maskT); | 480 | + AscendC::Reg::Exp(expReg, vregX1NegDeF, maskT); |
| 481 | - AscendC::MicroAPI::Adds(addsReg, expReg, scalarOne, maskT); | 481 | + AscendC::Reg::Adds(addsReg, expReg, scalarOne, maskT); |
| 482 | - AscendC::MicroAPI::Div(sigmoidReg, vregX1DeF, addsReg, maskT); | 482 | + AscendC::Reg::Div(sigmoidReg, vregX1DeF, addsReg, maskT); |
| 483 | - AscendC::MicroAPI::Mins(minsReg, sigmoidReg, clampLimit, maskT); | 483 | + AscendC::Reg::Mins(minsReg, sigmoidReg, clampLimit, maskT); |
| 484 | 484 | ||
| 485 | - AscendC::MicroAPI::Mins(vregX2DeF, vregX2DeF, clampLimit, maskT); | 485 | + AscendC::Reg::Mins(vregX2DeF, vregX2DeF, clampLimit, maskT); |
| 486 | - AscendC::MicroAPI::Maxs(vregX2DeF, vregX2DeF, negClampLimit, maskT); | 486 | + AscendC::Reg::Maxs(vregX2DeF, vregX2DeF, negClampLimit, maskT); |
| 487 | - AscendC::MicroAPI::Mul(outFReg, minsReg, vregX2DeF, maskT); | 487 | + AscendC::Reg::Mul(outFReg, minsReg, vregX2DeF, maskT); |
| 488 | } | 488 | } |
| 489 | if constexpr (sizeof(T) == sizeof(half)) { | 489 | if constexpr (sizeof(T) == sizeof(half)) { |
| 490 | - AscendC::MicroAPI::Cast<T, float, CAST_FP32_TO_FP16_BF16>(outTReg, outFReg, maskT); | 490 | + AscendC::Reg::Cast<T, float, CAST_FP32_TO_FP16_BF16>(outTReg, outFReg, maskT); |
| 491 | - StoreAlign<T, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(swigluUbAddrT, outTReg, outOffset1, | 491 | + StoreAlign<T, AscendC::Reg::StoreDist::DIST_PACK_B32>(swigluUbAddrT, outTReg, outOffset1, maskT); |
| 492 | - maskT); | ||
| 493 | } else { | 492 | } else { |
| 494 | - StoreAlign(swigluUbAddrT, (MicroAPI::RegTensor<T>&)outFReg, outOffset1, maskT); | 493 | + StoreAlign(swigluUbAddrT, (Reg::RegTensor<T>&)outFReg, outOffset1, maskT); |
| 495 | } | 494 | } |
| 496 | } | 495 | } |
| 497 | } | 496 | } |
| @@ -13,7 +13,7 @@ | |||
| 13 | * \brief Regbase VF kernel for ClippedSwigluGrad (Ascend 950 / arch35) | 13 | * \brief Regbase VF kernel for ClippedSwigluGrad (Ascend 950 / arch35) |
| 14 | * | 14 | * |
| 15 | * 基于910B版反向逻辑,减少 UB 间搬运: | 15 | * 基于910B版反向逻辑,减少 UB 间搬运: |
| 16 | - * - 单个 __VEC_SCOPE__:MicroAPI RegTensor 加载 a/b/dy → 寄存器内 Compare/Select 做 clamp mask | 16 | + * - 单个 __VEC_SCOPE__:Reg RegTensor 加载 a/b/dy → 寄存器内 Compare/Select 做 clamp mask |
| 17 | * → 计算 da/db(寄存器内)→ 写回 vecBuf(交错) / dxFloatLocal(前后切分) | 17 | * → 计算 da/db(寄存器内)→ 写回 vecBuf(交错) / dxFloatLocal(前后切分) |
| 18 | * - interleaved 散开:scope 外用 LocalTensor 级 Interleave(向量化,无标量大循环;仅 <=7 元素 32B 尾标量补齐) | 18 | * - interleaved 散开:scope 外用 LocalTensor 级 Interleave(向量化,无标量大循环;仅 <=7 元素 32B 尾标量补齐) |
| 19 | * - 16-bit:写回后 Cast float→T(与910B一致) | 19 | * - 16-bit:写回后 Cast float→T(与910B一致) |
| @@ -37,8 +37,8 @@ constexpr int64_t SWI_FACTOR = 2; | |||
| 37 | constexpr int64_t ZERO_CHUNK_BYTES = 65535 / BLOCK_SIZE * BLOCK_SIZE; | 37 | constexpr int64_t ZERO_CHUNK_BYTES = 65535 / BLOCK_SIZE * BLOCK_SIZE; |
| 38 | constexpr uint32_t VF_LEN_FP32 = Ops::Base::GetVRegSize() / sizeof(float); | 38 | constexpr uint32_t VF_LEN_FP32 = Ops::Base::GetVRegSize() / sizeof(float); |
| 39 | 39 | ||
| 40 | -static constexpr MicroAPI::CastTrait CAST_BF16_FP16_TO_FP32 = {MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::UNKNOWN, | 40 | +static constexpr Reg::CastTrait CAST_BF16_FP16_TO_FP32 = {Reg::RegLayout::ZERO, Reg::SatMode::UNKNOWN, |
| 41 | - MicroAPI::MaskMergeMode::ZEROING, RoundMode::UNKNOWN}; | 41 | + Reg::MaskMergeMode::ZEROING, RoundMode::UNKNOWN}; |
| 42 | 42 | ||
| 43 | template <typename T, bool isInterleaved, bool isGroup> | 43 | template <typename T, bool isInterleaved, bool isGroup> |
| 44 | class ClippedSwigluGradArch35Kernel { | 44 | class ClippedSwigluGradArch35Kernel { |
| @@ -68,8 +68,8 @@ private: | |||
| 68 | __aicore__ inline void InitZeroBuffer(); | 68 | __aicore__ inline void InitZeroBuffer(); |
| 69 | __aicore__ inline void ZeroInvalidRows(); | 69 | __aicore__ inline void ZeroInvalidRows(); |
| 70 | __aicore__ inline int64_t AlignBytes(int64_t number) { return (number + BLOCK_SIZE - 1) / BLOCK_SIZE * BLOCK_SIZE; } | 70 | __aicore__ inline int64_t AlignBytes(int64_t number) { return (number + BLOCK_SIZE - 1) / BLOCK_SIZE * BLOCK_SIZE; } |
| 71 | - __aicore__ inline void LoadOneTensor(__local_mem__ void* input, MicroAPI::RegTensor<float>& dst, | 71 | + __aicore__ inline void LoadOneTensor(__local_mem__ void* input, Reg::RegTensor<float>& dst, Reg::MaskReg& preg, |
| 72 | - MicroAPI::MaskReg& preg, uint32_t offset); | 72 | + uint32_t offset); |
| 73 | 73 | ||
| 74 | private: | 74 | private: |
| 75 | GlobalTensor<T> xGm_; | 75 | GlobalTensor<T> xGm_; |
| @@ -386,19 +386,18 @@ __aicore__ inline void ClippedSwigluGradArch35Kernel<T, isInterleaved, isGroup>: | |||
| 386 | 386 | ||
| 387 | template <typename T, bool isInterleaved, bool isGroup> | 387 | template <typename T, bool isInterleaved, bool isGroup> |
| 388 | __aicore__ inline void ClippedSwigluGradArch35Kernel<T, isInterleaved, isGroup>::LoadOneTensor( | 388 | __aicore__ inline void ClippedSwigluGradArch35Kernel<T, isInterleaved, isGroup>::LoadOneTensor( |
| 389 | - __local_mem__ void* input, MicroAPI::RegTensor<float>& dst, MicroAPI::MaskReg& preg, uint32_t offset) | 389 | + __local_mem__ void* input, Reg::RegTensor<float>& dst, Reg::MaskReg& preg, uint32_t offset) |
| 390 | { | 390 | { |
| 391 | if constexpr (std::is_same_v<T, half>) { | 391 | if constexpr (std::is_same_v<T, half>) { |
| 392 | - MicroAPI::RegTensor<half> xFp16; | 392 | + Reg::RegTensor<half> xFp16; |
| 393 | - MicroAPI::LoadAlign<half, MicroAPI::LoadDist::DIST_UNPACK_B16>(xFp16, (__local_mem__ half*)input + offset); | 393 | + Reg::LoadAlign<half, Reg::LoadDist::DIST_UNPACK_B16>(xFp16, (__local_mem__ half*)input + offset); |
| 394 | Cast<float, half, CAST_BF16_FP16_TO_FP32>(dst, xFp16, preg); | 394 | Cast<float, half, CAST_BF16_FP16_TO_FP32>(dst, xFp16, preg); |
| 395 | } else if constexpr (std::is_same_v<T, bfloat16_t>) { | 395 | } else if constexpr (std::is_same_v<T, bfloat16_t>) { |
| 396 | - MicroAPI::RegTensor<bfloat16_t> xBf16; | 396 | + Reg::RegTensor<bfloat16_t> xBf16; |
| 397 | - MicroAPI::LoadAlign<bfloat16_t, MicroAPI::LoadDist::DIST_UNPACK_B16>(xBf16, | 397 | + Reg::LoadAlign<bfloat16_t, Reg::LoadDist::DIST_UNPACK_B16>(xBf16, (__local_mem__ bfloat16_t*)input + offset); |
| 398 | - (__local_mem__ bfloat16_t*)input + offset); | ||
| 399 | Cast<float, bfloat16_t, CAST_BF16_FP16_TO_FP32>(dst, xBf16, preg); | 398 | Cast<float, bfloat16_t, CAST_BF16_FP16_TO_FP32>(dst, xBf16, preg); |
| 400 | } else { | 399 | } else { |
| 401 | - MicroAPI::LoadAlign<float, MicroAPI::LoadDist::DIST_NORM>(dst, (__local_mem__ float*)input + offset); | 400 | + Reg::LoadAlign<float, Reg::LoadDist::DIST_NORM>(dst, (__local_mem__ float*)input + offset); |
| 402 | } | 401 | } |
| 403 | } | 402 | } |
| 404 | 403 | ||
| @@ -435,74 +434,72 @@ __aicore__ inline void ClippedSwigluGradArch35Kernel<T, isInterleaved, isGroup>: | |||
| 435 | // ---- VF: load a/b/dy, clamp-mask in reg, compute da/db ---- | 434 | // ---- VF: load a/b/dy, clamp-mask in reg, compute da/db ---- |
| 436 | __VEC_SCOPE__ | 435 | __VEC_SCOPE__ |
| 437 | { | 436 | { |
| 438 | - MicroAPI::RegTensor<float> vregX0; | 437 | + Reg::RegTensor<float> vregX0; |
| 439 | - MicroAPI::RegTensor<float> vregX1; | 438 | + Reg::RegTensor<float> vregX1; |
| 440 | - MicroAPI::RegTensor<float> vregX0DeF; | 439 | + Reg::RegTensor<float> vregX0DeF; |
| 441 | - MicroAPI::RegTensor<float> vregX1DeF; | 440 | + Reg::RegTensor<float> vregX1DeF; |
| 442 | - MicroAPI::RegTensor<float> vregDY; | 441 | + Reg::RegTensor<float> vregDY; |
| 443 | - MicroAPI::RegTensor<float> minsReg; | 442 | + Reg::RegTensor<float> minsReg; |
| 444 | - MicroAPI::RegTensor<float> mulsReg; | 443 | + Reg::RegTensor<float> mulsReg; |
| 445 | - MicroAPI::RegTensor<float> expReg; | 444 | + Reg::RegTensor<float> expReg; |
| 446 | - MicroAPI::RegTensor<float> addsReg; | 445 | + Reg::RegTensor<float> addsReg; |
| 447 | - MicroAPI::RegTensor<float> sigReg; | 446 | + Reg::RegTensor<float> sigReg; |
| 448 | - MicroAPI::RegTensor<float> tmpReg; | 447 | + Reg::RegTensor<float> tmpReg; |
| 449 | - MicroAPI::RegTensor<float> oneReg; | 448 | + Reg::RegTensor<float> oneReg; |
| 450 | - MicroAPI::RegTensor<float> daReg; | 449 | + Reg::RegTensor<float> daReg; |
| 451 | - MicroAPI::RegTensor<float> dbReg; | 450 | + Reg::RegTensor<float> dbReg; |
| 452 | - MicroAPI::RegTensor<float> limitReg; | 451 | + Reg::RegTensor<float> limitReg; |
| 453 | - MicroAPI::RegTensor<float> negLimitReg; | 452 | + Reg::RegTensor<float> negLimitReg; |
| 454 | - MicroAPI::RegTensor<float> zeroReg; | 453 | + Reg::RegTensor<float> zeroReg; |
| 455 | 454 | ||
| 456 | - MicroAPI::MaskReg maskAll = MicroAPI::CreateMask<float, MicroAPI::MaskPattern::ALL>(); | 455 | + Reg::MaskReg maskAll = Reg::CreateMask<float, Reg::MaskPattern::ALL>(); |
| 457 | - MicroAPI::MaskReg maskT = MicroAPI::UpdateMask<float>(tail); | 456 | + Reg::MaskReg maskT = Reg::UpdateMask<float>(tail); |
| 458 | - MicroAPI::MaskReg maskA; | 457 | + Reg::MaskReg maskA; |
| 459 | - MicroAPI::MaskReg maskB; | 458 | + Reg::MaskReg maskB; |
| 460 | - MicroAPI::MaskReg maskBn; | 459 | + Reg::MaskReg maskBn; |
| 461 | 460 | ||
| 462 | - MicroAPI::Duplicate(limitReg, clampLimit); | 461 | + Reg::Duplicate(limitReg, clampLimit); |
| 463 | - MicroAPI::Duplicate(negLimitReg, negClampLimit); | 462 | + Reg::Duplicate(negLimitReg, negClampLimit); |
| 464 | - MicroAPI::Duplicate(zeroReg, scalarZero); | 463 | + Reg::Duplicate(zeroReg, scalarZero); |
| 465 | 464 | ||
| 466 | for (uint16_t vfIdx = 0; vfIdx < dim1VfTimes + tailTimes; vfIdx++) { | 465 | for (uint16_t vfIdx = 0; vfIdx < dim1VfTimes + tailTimes; vfIdx++) { |
| 467 | uint32_t offset = vfIdx * static_cast<uint32_t>(VF_LEN_FP32); | 466 | uint32_t offset = vfIdx * static_cast<uint32_t>(VF_LEN_FP32); |
| 468 | - MicroAPI::MaskReg preg = (vfIdx < dim1VfTimes) ? maskAll : maskT; | 467 | + Reg::MaskReg preg = (vfIdx < dim1VfTimes) ? maskAll : maskT; |
| 469 | 468 | ||
| 470 | if constexpr (isInterleaved) { | 469 | if constexpr (isInterleaved) { |
| 471 | uint32_t vfLenT = VF_LEN_FP32 * SWI_FACTOR; | 470 | uint32_t vfLenT = VF_LEN_FP32 * SWI_FACTOR; |
| 472 | - MicroAPI::AddrReg srcIdxOffset = MicroAPI::CreateAddrReg<T>(vfIdx, vfLenT); | 471 | + Reg::AddrReg srcIdxOffset = Reg::CreateAddrReg<T>(vfIdx, vfLenT); |
| 473 | if constexpr (std::is_same_v<T, half>) { | 472 | if constexpr (std::is_same_v<T, half>) { |
| 474 | - MicroAPI::RegTensor<half> vregX0Raw; | 473 | + Reg::RegTensor<half> vregX0Raw; |
| 475 | - MicroAPI::RegTensor<half> vregX1Raw; | 474 | + Reg::RegTensor<half> vregX1Raw; |
| 476 | - MicroAPI::LoadAlign<half, MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX0Raw, xAddr, srcIdxOffset); | 475 | + Reg::LoadAlign<half, Reg::LoadDist::DIST_UNPACK_B16>(vregX0Raw, xAddr, srcIdxOffset); |
| 477 | - MicroAPI::LoadAlign<half, MicroAPI::LoadDist::DIST_UNPACK_B16>( | 476 | + Reg::LoadAlign<half, Reg::LoadDist::DIST_UNPACK_B16>( |
| 478 | vregX1Raw, xAddr + static_cast<uint32_t>(VF_LEN_FP32), srcIdxOffset); | 477 | vregX1Raw, xAddr + static_cast<uint32_t>(VF_LEN_FP32), srcIdxOffset); |
| 479 | - MicroAPI::Cast<float, half, CAST_BF16_FP16_TO_FP32>(vregX0, vregX0Raw, maskAll); | 478 | + Reg::Cast<float, half, CAST_BF16_FP16_TO_FP32>(vregX0, vregX0Raw, maskAll); |
| 480 | - MicroAPI::Cast<float, half, CAST_BF16_FP16_TO_FP32>(vregX1, vregX1Raw, maskAll); | 479 | + Reg::Cast<float, half, CAST_BF16_FP16_TO_FP32>(vregX1, vregX1Raw, maskAll); |
| 481 | } else if constexpr (std::is_same_v<T, bfloat16_t>) { | 480 | } else if constexpr (std::is_same_v<T, bfloat16_t>) { |
| 482 | - MicroAPI::RegTensor<bfloat16_t> vregX0Raw; | 481 | + Reg::RegTensor<bfloat16_t> vregX0Raw; |
| 483 | - MicroAPI::RegTensor<bfloat16_t> vregX1Raw; | 482 | + Reg::RegTensor<bfloat16_t> vregX1Raw; |
| 484 | - MicroAPI::LoadAlign<bfloat16_t, MicroAPI::LoadDist::DIST_UNPACK_B16>(vregX0Raw, xAddr, | 483 | + Reg::LoadAlign<bfloat16_t, Reg::LoadDist::DIST_UNPACK_B16>(vregX0Raw, xAddr, srcIdxOffset); |
| 485 | - srcIdxOffset); | 484 | + Reg::LoadAlign<bfloat16_t, Reg::LoadDist::DIST_UNPACK_B16>( |
| 486 | - MicroAPI::LoadAlign<bfloat16_t, MicroAPI::LoadDist::DIST_UNPACK_B16>( | ||
| 487 | vregX1Raw, xAddr + static_cast<uint32_t>(VF_LEN_FP32), srcIdxOffset); | 485 | vregX1Raw, xAddr + static_cast<uint32_t>(VF_LEN_FP32), srcIdxOffset); |
| 488 | - MicroAPI::Cast<float, bfloat16_t, CAST_BF16_FP16_TO_FP32>(vregX0, vregX0Raw, maskAll); | 486 | + Reg::Cast<float, bfloat16_t, CAST_BF16_FP16_TO_FP32>(vregX0, vregX0Raw, maskAll); |
| 489 | - MicroAPI::Cast<float, bfloat16_t, CAST_BF16_FP16_TO_FP32>(vregX1, vregX1Raw, maskAll); | 487 | + Reg::Cast<float, bfloat16_t, CAST_BF16_FP16_TO_FP32>(vregX1, vregX1Raw, maskAll); |
| 490 | } else { | 488 | } else { |
| 491 | - MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_NORM>((MicroAPI::RegTensor<T>&)vregX0, xAddr, | 489 | + Reg::LoadAlign<T, Reg::LoadDist::DIST_NORM>((Reg::RegTensor<T>&)vregX0, xAddr, srcIdxOffset); |
| 492 | - srcIdxOffset); | 490 | + Reg::LoadAlign<T, Reg::LoadDist::DIST_NORM>( |
| 493 | - MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_NORM>( | 491 | + (Reg::RegTensor<T>&)vregX1, xAddr + static_cast<uint32_t>(VF_LEN_FP32), srcIdxOffset); |
| 494 | - (MicroAPI::RegTensor<T>&)vregX1, xAddr + static_cast<uint32_t>(VF_LEN_FP32), srcIdxOffset); | ||
| 495 | } | 492 | } |
| 496 | - MicroAPI::DeInterleave(vregX0DeF, vregX1DeF, vregX0, vregX1); | 493 | + Reg::DeInterleave(vregX0DeF, vregX1DeF, vregX0, vregX1); |
| 497 | } else { | 494 | } else { |
| 498 | LoadOneTensor(x0Addr, vregX0DeF, preg, offset); | 495 | LoadOneTensor(x0Addr, vregX0DeF, preg, offset); |
| 499 | LoadOneTensor(x1Addr, vregX1DeF, preg, offset); | 496 | LoadOneTensor(x1Addr, vregX1DeF, preg, offset); |
| 500 | } | 497 | } |
| 501 | LoadOneTensor(dyAddr, vregDY, preg, offset); | 498 | LoadOneTensor(dyAddr, vregDY, preg, offset); |
| 502 | 499 | ||
| 503 | - MicroAPI::Compare<float, CMPMODE::LE>(maskA, vregX0DeF, limitReg, preg); | 500 | + Reg::Compare<float, CMPMODE::LE>(maskA, vregX0DeF, limitReg, preg); |
| 504 | - MicroAPI::Compare<float, CMPMODE::LE>(maskB, vregX1DeF, limitReg, preg); | 501 | + Reg::Compare<float, CMPMODE::LE>(maskB, vregX1DeF, limitReg, preg); |
| 505 | - MicroAPI::Compare<float, CMPMODE::GE>(maskBn, vregX1DeF, negLimitReg, preg); | 502 | + Reg::Compare<float, CMPMODE::GE>(maskBn, vregX1DeF, negLimitReg, preg); |
| 506 | 503 | ||
| 507 | Mins(minsReg, vregX0DeF, clampLimit, preg); | 504 | Mins(minsReg, vregX0DeF, clampLimit, preg); |
| 508 | 505 | ||
| @@ -529,16 +526,16 @@ __aicore__ inline void ClippedSwigluGradArch35Kernel<T, isInterleaved, isGroup>: | |||
| 529 | Mul(tmpReg, tmpReg, vregX1DeF, preg); | 526 | Mul(tmpReg, tmpReg, vregX1DeF, preg); |
| 530 | Mul(daReg, tmpReg, vregDY, preg); | 527 | Mul(daReg, tmpReg, vregDY, preg); |
| 531 | 528 | ||
| 532 | - MicroAPI::Select<float>(daReg, daReg, zeroReg, maskA); | 529 | + Reg::Select<float>(daReg, daReg, zeroReg, maskA); |
| 533 | - MicroAPI::Select<float>(dbReg, dbReg, zeroReg, maskB); | 530 | + Reg::Select<float>(dbReg, dbReg, zeroReg, maskB); |
| 534 | - MicroAPI::Select<float>(dbReg, dbReg, zeroReg, maskBn); | 531 | + Reg::Select<float>(dbReg, dbReg, zeroReg, maskBn); |
| 535 | 532 | ||
| 536 | if constexpr (isInterleaved) { | 533 | if constexpr (isInterleaved) { |
| 537 | - MicroAPI::StoreAlign<float, MicroAPI::StoreDist::DIST_NORM>(vecAddr + offset, daReg, preg); | 534 | + Reg::StoreAlign<float, Reg::StoreDist::DIST_NORM>(vecAddr + offset, daReg, preg); |
| 538 | - MicroAPI::StoreAlign<float, MicroAPI::StoreDist::DIST_NORM>(vecAddr + halfU32 + offset, dbReg, preg); | 535 | + Reg::StoreAlign<float, Reg::StoreDist::DIST_NORM>(vecAddr + halfU32 + offset, dbReg, preg); |
| 539 | } else { | 536 | } else { |
| 540 | - MicroAPI::StoreAlign<float, MicroAPI::StoreDist::DIST_NORM>(dxFAddr + offset, daReg, preg); | 537 | + Reg::StoreAlign<float, Reg::StoreDist::DIST_NORM>(dxFAddr + offset, daReg, preg); |
| 541 | - MicroAPI::StoreAlign<float, MicroAPI::StoreDist::DIST_NORM>(dxFAddr + halfU32 + offset, dbReg, preg); | 538 | + Reg::StoreAlign<float, Reg::StoreDist::DIST_NORM>(dxFAddr + halfU32 + offset, dbReg, preg); |
| 542 | } | 539 | } |
| 543 | } | 540 | } |
| 544 | } | 541 | } |
| @@ -58,10 +58,10 @@ private: | |||
| 58 | __aicore__ inline void CopyInX(const LocalTensor<T>& xInUb, const GlobalTensor<T>& xInGm, int64_t ubA, | 58 | __aicore__ inline void CopyInX(const LocalTensor<T>& xInUb, const GlobalTensor<T>& xInGm, int64_t ubA, |
| 59 | int64_t offset); | 59 | int64_t offset); |
| 60 | __aicore__ inline void CopyOutY(const LocalTensor<T>& yOutUb, int64_t ubA, int64_t offset); | 60 | __aicore__ inline void CopyOutY(const LocalTensor<T>& yOutUb, int64_t ubA, int64_t offset); |
| 61 | - __aicore__ inline void StoreTensorForDtypeTOut(__ubuf__ T* dst, AscendC::MicroAPI::RegTensor<float>& src, | 61 | + __aicore__ inline void StoreTensorForDtypeTOut(__ubuf__ T* dst, AscendC::Reg::RegTensor<float>& src, |
| 62 | - AscendC::MicroAPI::MaskReg& preg, uint32_t offset); | 62 | + AscendC::Reg::MaskReg& preg, uint32_t offset); |
| 63 | - __aicore__ inline void LoadTensorForDtypeTIn(__ubuf__ T* src, AscendC::MicroAPI::RegTensor<float>& dst, | 63 | + __aicore__ inline void LoadTensorForDtypeTIn(__ubuf__ T* src, AscendC::Reg::RegTensor<float>& dst, |
| 64 | - AscendC::MicroAPI::MaskReg& preg, uint32_t offset); | 64 | + AscendC::Reg::MaskReg& preg, uint32_t offset); |
| 65 | 65 | ||
| 66 | private: | 66 | private: |
| 67 | /* global memory address */ | 67 | /* global memory address */ |
| @@ -175,18 +175,17 @@ __aicore__ inline void ConfusionSoftmaxGradAR<T>::NormComputePostWithMul(const L | |||
| 175 | __VEC_SCOPE__ | 175 | __VEC_SCOPE__ |
| 176 | { | 176 | { |
| 177 | uint32_t count = static_cast<uint32_t>(rSize); | 177 | uint32_t count = static_cast<uint32_t>(rSize); |
| 178 | - AscendC::MicroAPI::RegTensor<float> reg0, reg1, reg2; | 178 | + AscendC::Reg::RegTensor<float> reg0, reg1, reg2; |
| 179 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 179 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 180 | - AscendC::MicroAPI::MaskReg | 180 | + AscendC::Reg::MaskReg pFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 181 | - pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 181 | + AscendC::Reg::MaskReg maskOri; |
| 182 | - AscendC::MicroAPI::MaskReg maskOri; | ||
| 183 | for (uint16_t i = 0; i < loopTimes; ++i) { | 182 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 184 | LoadTensorForDtypeTIn(x0, reg0, pMask, i * stride); | 183 | LoadTensorForDtypeTIn(x0, reg0, pMask, i * stride); |
| 185 | LoadTensorForDtypeTIn(x1, reg1, pMask, i * stride); | 184 | LoadTensorForDtypeTIn(x1, reg1, pMask, i * stride); |
| 186 | - AscendC::MicroAPI::Mul(reg2, reg0, reg1, pMask); | 185 | + AscendC::Reg::Mul(reg2, reg0, reg1, pMask); |
| 187 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg2, pMask); | 186 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg2, pMask); |
| 188 | Duplicate(reg2, reg2, pFull); | 187 | Duplicate(reg2, reg2, pFull); |
| 189 | - AscendC::MicroAPI::Sub(reg0, reg0, reg2, pMask); | 188 | + AscendC::Reg::Sub(reg0, reg0, reg2, pMask); |
| 190 | StoreTensorForDtypeTOut(dst, reg0, pMask, i * stride); | 189 | StoreTensorForDtypeTOut(dst, reg0, pMask, i * stride); |
| 191 | } | 190 | } |
| 192 | } | 191 | } |
| @@ -201,11 +200,10 @@ __aicore__ inline void ConfusionSoftmaxGradAR<T>::NormComputePostWithMul(const L | |||
| 201 | __VEC_SCOPE__ | 200 | __VEC_SCOPE__ |
| 202 | { | 201 | { |
| 203 | uint32_t count = static_cast<uint32_t>(rSize - VL_FP32); | 202 | uint32_t count = static_cast<uint32_t>(rSize - VL_FP32); |
| 204 | - AscendC::MicroAPI::RegTensor<float> reg0, reg1, reg0_1, reg1_1, reg2, reg2_1; | 203 | + AscendC::Reg::RegTensor<float> reg0, reg1, reg0_1, reg1_1, reg2, reg2_1; |
| 205 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 204 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 206 | - AscendC::MicroAPI::MaskReg | 205 | + AscendC::Reg::MaskReg pFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 207 | - pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 206 | + AscendC::Reg::MaskReg maskOri; |
| 208 | - AscendC::MicroAPI::MaskReg maskOri; | ||
| 209 | for (uint16_t i = 0; i < loopTimes; ++i) { | 207 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 210 | LoadTensorForDtypeTIn(x0, reg0, pFull, i * stride); | 208 | LoadTensorForDtypeTIn(x0, reg0, pFull, i * stride); |
| 211 | LoadTensorForDtypeTIn(x0_1, reg0_1, pMask, i * stride); | 209 | LoadTensorForDtypeTIn(x0_1, reg0_1, pMask, i * stride); |
| @@ -213,11 +211,11 @@ __aicore__ inline void ConfusionSoftmaxGradAR<T>::NormComputePostWithMul(const L | |||
| 213 | LoadTensorForDtypeTIn(x1, reg1, pFull, i * stride); | 211 | LoadTensorForDtypeTIn(x1, reg1, pFull, i * stride); |
| 214 | LoadTensorForDtypeTIn(x1_1, reg1_1, pMask, i * stride); | 212 | LoadTensorForDtypeTIn(x1_1, reg1_1, pMask, i * stride); |
| 215 | 213 | ||
| 216 | - AscendC::MicroAPI::Mul(reg2, reg0, reg1, pFull); | 214 | + AscendC::Reg::Mul(reg2, reg0, reg1, pFull); |
| 217 | - AscendC::MicroAPI::Mul(reg2_1, reg0_1, reg1_1, pMask); | 215 | + AscendC::Reg::Mul(reg2_1, reg0_1, reg1_1, pMask); |
| 218 | 216 | ||
| 219 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(reg2_1, reg2, reg2_1, pMask); | 217 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(reg2_1, reg2, reg2_1, pMask); |
| 220 | - Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(reg2, reg2_1, pMask); | 218 | + Move<float, AscendC::Reg::MaskMergeMode::MERGING>(reg2, reg2_1, pMask); |
| 221 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg2, pFull); | 219 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg2, pFull); |
| 222 | Duplicate(reg2, reg2, pFull); | 220 | Duplicate(reg2, reg2, pFull); |
| 223 | 221 | ||
| @@ -284,30 +282,30 @@ __aicore__ inline void ConfusionSoftmaxGradAR<T>::NormCompute( | |||
| 284 | 282 | ||
| 285 | __VEC_SCOPE__ | 283 | __VEC_SCOPE__ |
| 286 | { | 284 | { |
| 287 | - AscendC::MicroAPI::MaskReg pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 285 | + AscendC::Reg::MaskReg pFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 288 | - AscendC::MicroAPI::UnalignRegForStore UReg; | 286 | + AscendC::Reg::UnalignRegForStore UReg; |
| 289 | 287 | ||
| 290 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { | 288 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { |
| 291 | dst = (__ubuf__ float*)reduceSumTempTensor.GetPhyAddr() + i * outerLoopDstStride; | 289 | dst = (__ubuf__ float*)reduceSumTempTensor.GetPhyAddr() + i * outerLoopDstStride; |
| 292 | for (uint16_t j = 0; j < mainFoldLoopTimes; ++j) { | 290 | for (uint16_t j = 0; j < mainFoldLoopTimes; ++j) { |
| 293 | - AscendC::MicroAPI::RegTensor<float> reg0, reg1, reg0_1, reg1_1, reg2, reg2_1; | 291 | + AscendC::Reg::RegTensor<float> reg0, reg1, reg0_1, reg1_1, reg2, reg2_1; |
| 294 | LoadTensorForDtypeTIn(foldSrcX0A, reg0, pFull, i * outerLoopStride + j * innerLoopStride); | 292 | LoadTensorForDtypeTIn(foldSrcX0A, reg0, pFull, i * outerLoopStride + j * innerLoopStride); |
| 295 | LoadTensorForDtypeTIn(foldSrcX0B, reg1, pFull, i * outerLoopStride + j * innerLoopStride); | 293 | LoadTensorForDtypeTIn(foldSrcX0B, reg1, pFull, i * outerLoopStride + j * innerLoopStride); |
| 296 | 294 | ||
| 297 | LoadTensorForDtypeTIn(foldSrcX1A, reg0_1, pFull, i * outerLoopStride + j * innerLoopStride); | 295 | LoadTensorForDtypeTIn(foldSrcX1A, reg0_1, pFull, i * outerLoopStride + j * innerLoopStride); |
| 298 | LoadTensorForDtypeTIn(foldSrcX1B, reg1_1, pFull, i * outerLoopStride + j * innerLoopStride); | 296 | LoadTensorForDtypeTIn(foldSrcX1B, reg1_1, pFull, i * outerLoopStride + j * innerLoopStride); |
| 299 | 297 | ||
| 300 | - AscendC::MicroAPI::Mul(reg2, reg0, reg0_1, pFull); | 298 | + AscendC::Reg::Mul(reg2, reg0, reg0_1, pFull); |
| 301 | - AscendC::MicroAPI::Mul(reg2_1, reg1, reg1_1, pFull); | 299 | + AscendC::Reg::Mul(reg2_1, reg1, reg1_1, pFull); |
| 302 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(reg2, reg2, reg2_1, pFull); | 300 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(reg2, reg2, reg2_1, pFull); |
| 303 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg2, pFull); | 301 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg2, pFull); |
| 304 | - AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, reg2, UReg, 1); | 302 | + AscendC::Reg::StoreUnAlign((__ubuf__ float*&)dst, reg2, UReg, 1); |
| 305 | } | 303 | } |
| 306 | for (uint16_t j = 0; j < tailFoldLoopTimes; ++j) { | 304 | for (uint16_t j = 0; j < tailFoldLoopTimes; ++j) { |
| 307 | uint32_t count = static_cast<uint32_t>(tailFoldElemCount); | 305 | uint32_t count = static_cast<uint32_t>(tailFoldElemCount); |
| 308 | - AscendC::MicroAPI::RegTensor<float> reg0, reg1, reg0_1, reg1_1, reg2, reg2_1; | 306 | + AscendC::Reg::RegTensor<float> reg0, reg1, reg0_1, reg1_1, reg2, reg2_1; |
| 309 | 307 | ||
| 310 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 308 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 311 | 309 | ||
| 312 | LoadTensorForDtypeTIn(tailSrcX0A, reg0, pFull, i * outerLoopStride + j * innerLoopStride); | 310 | LoadTensorForDtypeTIn(tailSrcX0A, reg0, pFull, i * outerLoopStride + j * innerLoopStride); |
| 313 | LoadTensorForDtypeTIn(tailSrcX0B, reg1, pMask, i * outerLoopStride + j * innerLoopStride); | 311 | LoadTensorForDtypeTIn(tailSrcX0B, reg1, pMask, i * outerLoopStride + j * innerLoopStride); |
| @@ -315,24 +313,24 @@ __aicore__ inline void ConfusionSoftmaxGradAR<T>::NormCompute( | |||
| 315 | LoadTensorForDtypeTIn(tailSrcX1A, reg0_1, pFull, i * outerLoopStride + j * innerLoopStride); | 313 | LoadTensorForDtypeTIn(tailSrcX1A, reg0_1, pFull, i * outerLoopStride + j * innerLoopStride); |
| 316 | LoadTensorForDtypeTIn(tailSrcX1B, reg1_1, pMask, i * outerLoopStride + j * innerLoopStride); | 314 | LoadTensorForDtypeTIn(tailSrcX1B, reg1_1, pMask, i * outerLoopStride + j * innerLoopStride); |
| 317 | 315 | ||
| 318 | - AscendC::MicroAPI::Mul(reg2, reg0, reg0_1, pFull); | 316 | + AscendC::Reg::Mul(reg2, reg0, reg0_1, pFull); |
| 319 | - AscendC::MicroAPI::Mul(reg2_1, reg1, reg1_1, pMask); | 317 | + AscendC::Reg::Mul(reg2_1, reg1, reg1_1, pMask); |
| 320 | 318 | ||
| 321 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(reg2_1, reg2, reg2_1, pMask); | 319 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(reg2_1, reg2, reg2_1, pMask); |
| 322 | - Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(reg2, reg2_1, pMask); | 320 | + Move<float, AscendC::Reg::MaskMergeMode::MERGING>(reg2, reg2_1, pMask); |
| 323 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg2, pFull); | 321 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg2, pFull); |
| 324 | - AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, reg2, UReg, 1); | 322 | + AscendC::Reg::StoreUnAlign((__ubuf__ float*&)dst, reg2, UReg, 1); |
| 325 | } | 323 | } |
| 326 | for (uint16_t j = 0; j < unFoldLoopTimes; ++j) { | 324 | for (uint16_t j = 0; j < unFoldLoopTimes; ++j) { |
| 327 | - AscendC::MicroAPI::RegTensor<float> reg0, reg1, reg0_1; | 325 | + AscendC::Reg::RegTensor<float> reg0, reg1, reg0_1; |
| 328 | LoadTensorForDtypeTIn(unFoldX0, reg0, pFull, i * outerLoopStride + j * innerLoopStride); | 326 | LoadTensorForDtypeTIn(unFoldX0, reg0, pFull, i * outerLoopStride + j * innerLoopStride); |
| 329 | LoadTensorForDtypeTIn(unFoldX1, reg0_1, pFull, i * outerLoopStride + j * innerLoopStride); | 327 | LoadTensorForDtypeTIn(unFoldX1, reg0_1, pFull, i * outerLoopStride + j * innerLoopStride); |
| 330 | 328 | ||
| 331 | - AscendC::MicroAPI::Mul(reg1, reg0, reg0_1, pFull); | 329 | + AscendC::Reg::Mul(reg1, reg0, reg0_1, pFull); |
| 332 | Reduce<AscendC::Reg::ReduceType::SUM>(reg1, reg1, pFull); | 330 | Reduce<AscendC::Reg::ReduceType::SUM>(reg1, reg1, pFull); |
| 333 | - AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, reg1, UReg, 1); | 331 | + AscendC::Reg::StoreUnAlign((__ubuf__ float*&)dst, reg1, UReg, 1); |
| 334 | } | 332 | } |
| 335 | - AscendC::MicroAPI::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0); | 333 | + AscendC::Reg::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0); |
| 336 | } | 334 | } |
| 337 | } | 335 | } |
| 338 | NormComputePost(dstTensor, x0Tensor, x1Tensor, reduceSumTempTensor, aSize, foldPoint, outerLoopDstStride); | 336 | NormComputePost(dstTensor, x0Tensor, x1Tensor, reduceSumTempTensor, aSize, foldPoint, outerLoopDstStride); |
| @@ -367,21 +365,20 @@ __aicore__ inline void ConfusionSoftmaxGradAR<T>::NormComputePost( | |||
| 367 | __VEC_SCOPE__ | 365 | __VEC_SCOPE__ |
| 368 | { | 366 | { |
| 369 | uint32_t count = static_cast<uint32_t>(rSize); | 367 | uint32_t count = static_cast<uint32_t>(rSize); |
| 370 | - AscendC::MicroAPI::RegTensor<float> reg0, reg1, reg2; | 368 | + AscendC::Reg::RegTensor<float> reg0, reg1, reg2; |
| 371 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 369 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 372 | - AscendC::MicroAPI::MaskReg | 370 | + AscendC::Reg::MaskReg pFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 373 | - pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 371 | + AscendC::Reg::MaskReg maskOri; |
| 374 | - AscendC::MicroAPI::MaskReg maskOri; | ||
| 375 | for (uint16_t i = 0; i < loopTimes; ++i) { | 372 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 376 | LoadAlign(reg0, (__ubuf__ float*)sumTmp + i * stride); | 373 | LoadAlign(reg0, (__ubuf__ float*)sumTmp + i * stride); |
| 377 | Reduce<AscendC::Reg::ReduceType::SUM>(reg1, reg0, pMask); | 374 | Reduce<AscendC::Reg::ReduceType::SUM>(reg1, reg0, pMask); |
| 378 | Duplicate(reg2, reg1, pFull); | 375 | Duplicate(reg2, reg1, pFull); |
| 379 | uint32_t sreg0 = static_cast<uint32_t>(oriR); | 376 | uint32_t sreg0 = static_cast<uint32_t>(oriR); |
| 380 | for (uint16_t j = 0; j < rLoopCount; ++j) { | 377 | for (uint16_t j = 0; j < rLoopCount; ++j) { |
| 381 | - maskOri = AscendC::MicroAPI::UpdateMask<float>(sreg0); | 378 | + maskOri = AscendC::Reg::UpdateMask<float>(sreg0); |
| 382 | uint32_t addrPtr = j * VL_FP32 + i * oriRAligned; | 379 | uint32_t addrPtr = j * VL_FP32 + i * oriRAligned; |
| 383 | LoadTensorForDtypeTIn(x0, reg1, maskOri, addrPtr); | 380 | LoadTensorForDtypeTIn(x0, reg1, maskOri, addrPtr); |
| 384 | - AscendC::MicroAPI::Sub(reg1, reg1, reg2, maskOri); | 381 | + AscendC::Reg::Sub(reg1, reg1, reg2, maskOri); |
| 385 | StoreTensorForDtypeTOut(dst, reg1, maskOri, addrPtr); | 382 | StoreTensorForDtypeTOut(dst, reg1, maskOri, addrPtr); |
| 386 | } | 383 | } |
| 387 | } | 384 | } |
| @@ -395,25 +392,24 @@ __aicore__ inline void ConfusionSoftmaxGradAR<T>::NormComputePost( | |||
| 395 | __VEC_SCOPE__ | 392 | __VEC_SCOPE__ |
| 396 | { | 393 | { |
| 397 | uint32_t count = static_cast<uint32_t>(rSize - VL_FP32); | 394 | uint32_t count = static_cast<uint32_t>(rSize - VL_FP32); |
| 398 | - AscendC::MicroAPI::RegTensor<float> reg0, reg1, reg2; | 395 | + AscendC::Reg::RegTensor<float> reg0, reg1, reg2; |
| 399 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 396 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 400 | - AscendC::MicroAPI::MaskReg | 397 | + AscendC::Reg::MaskReg pFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 401 | - pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 398 | + AscendC::Reg::MaskReg maskOri; |
| 402 | - AscendC::MicroAPI::MaskReg maskOri; | ||
| 403 | for (uint16_t i = 0; i < loopTimes; ++i) { | 399 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 404 | LoadAlign(reg0, (__ubuf__ float*)sumTmpA + i * stride); | 400 | LoadAlign(reg0, (__ubuf__ float*)sumTmpA + i * stride); |
| 405 | LoadAlign(reg1, (__ubuf__ float*)sumTmpB + i * stride); | 401 | LoadAlign(reg1, (__ubuf__ float*)sumTmpB + i * stride); |
| 406 | 402 | ||
| 407 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(reg1, reg0, reg1, pMask); | 403 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(reg1, reg0, reg1, pMask); |
| 408 | - Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(reg0, reg1, pMask); | 404 | + Move<float, AscendC::Reg::MaskMergeMode::MERGING>(reg0, reg1, pMask); |
| 409 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg0, pFull); | 405 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg0, pFull); |
| 410 | Duplicate(reg2, reg2, pFull); | 406 | Duplicate(reg2, reg2, pFull); |
| 411 | uint32_t sreg0 = static_cast<uint32_t>(oriR); | 407 | uint32_t sreg0 = static_cast<uint32_t>(oriR); |
| 412 | for (uint16_t j = 0; j < rLoopCount; ++j) { | 408 | for (uint16_t j = 0; j < rLoopCount; ++j) { |
| 413 | - maskOri = AscendC::MicroAPI::UpdateMask<float>(sreg0); | 409 | + maskOri = AscendC::Reg::UpdateMask<float>(sreg0); |
| 414 | uint32_t addrPtr = j * VL_FP32 + i * oriRAligned; | 410 | uint32_t addrPtr = j * VL_FP32 + i * oriRAligned; |
| 415 | LoadTensorForDtypeTIn(x0, reg1, maskOri, addrPtr); | 411 | LoadTensorForDtypeTIn(x0, reg1, maskOri, addrPtr); |
| 416 | - AscendC::MicroAPI::Sub(reg1, reg1, reg2, maskOri); | 412 | + AscendC::Reg::Sub(reg1, reg1, reg2, maskOri); |
| 417 | StoreTensorForDtypeTOut(dst, reg1, maskOri, addrPtr); | 413 | StoreTensorForDtypeTOut(dst, reg1, maskOri, addrPtr); |
| 418 | } | 414 | } |
| 419 | } | 415 | } |
| @@ -423,15 +419,14 @@ __aicore__ inline void ConfusionSoftmaxGradAR<T>::NormComputePost( | |||
| 423 | 419 | ||
| 424 | template <typename T> | 420 | template <typename T> |
| 425 | __aicore__ inline void ConfusionSoftmaxGradAR<T>::LoadTensorForDtypeTIn(__ubuf__ T* src, | 421 | __aicore__ inline void ConfusionSoftmaxGradAR<T>::LoadTensorForDtypeTIn(__ubuf__ T* src, |
| 426 | - AscendC::MicroAPI::RegTensor<float>& dst, | 422 | + AscendC::Reg::RegTensor<float>& dst, |
| 427 | - AscendC::MicroAPI::MaskReg& preg, | 423 | + AscendC::Reg::MaskReg& preg, uint32_t offset) |
| 428 | - uint32_t offset) | ||
| 429 | { | 424 | { |
| 430 | if constexpr (IsSameType<T, float>::value) { | 425 | if constexpr (IsSameType<T, float>::value) { |
| 431 | - LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(dst, src + offset); | 426 | + LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(dst, src + offset); |
| 432 | } else { | 427 | } else { |
| 433 | - AscendC::MicroAPI::RegTensor<T> xFp16; | 428 | + AscendC::Reg::RegTensor<T> xFp16; |
| 434 | - LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(xFp16, src + offset); | 429 | + LoadAlign<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(xFp16, src + offset); |
| 435 | Cast<float, T, castTraitFp16ToFp32>(dst, xFp16, preg); | 430 | Cast<float, T, castTraitFp16ToFp32>(dst, xFp16, preg); |
| 436 | } | 431 | } |
| 437 | } | 432 | } |
| @@ -451,16 +446,15 @@ __aicore__ inline void ConfusionSoftmaxGradAR<T>::CopyInX(const LocalTensor<T>& | |||
| 451 | 446 | ||
| 452 | template <typename T> | 447 | template <typename T> |
| 453 | __aicore__ inline void ConfusionSoftmaxGradAR<T>::StoreTensorForDtypeTOut(__ubuf__ T* dst, | 448 | __aicore__ inline void ConfusionSoftmaxGradAR<T>::StoreTensorForDtypeTOut(__ubuf__ T* dst, |
| 454 | - AscendC::MicroAPI::RegTensor<float>& src, | 449 | + AscendC::Reg::RegTensor<float>& src, |
| 455 | - AscendC::MicroAPI::MaskReg& preg, | 450 | + AscendC::Reg::MaskReg& preg, uint32_t offset) |
| 456 | - uint32_t offset) | ||
| 457 | { | 451 | { |
| 458 | if constexpr (IsSameType<T, float>::value) { | 452 | if constexpr (IsSameType<T, float>::value) { |
| 459 | - StoreAlign<T, AscendC::MicroAPI::StoreDist::DIST_NORM>(dst + offset, src, preg); | 453 | + StoreAlign<T, AscendC::Reg::StoreDist::DIST_NORM>(dst + offset, src, preg); |
| 460 | } else { | 454 | } else { |
| 461 | - AscendC::MicroAPI::RegTensor<T> xFp16; | 455 | + AscendC::Reg::RegTensor<T> xFp16; |
| 462 | Cast<T, float, castTraitFp32ToFp16>(xFp16, src, preg); | 456 | Cast<T, float, castTraitFp32ToFp16>(xFp16, src, preg); |
| 463 | - StoreAlign<T, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(dst + offset, xFp16, preg); | 457 | + StoreAlign<T, AscendC::Reg::StoreDist::DIST_PACK_B32>(dst + offset, xFp16, preg); |
| 464 | } | 458 | } |
| 465 | } | 459 | } |
| 466 | 460 | ||
| @@ -42,11 +42,11 @@ private: | |||
| 42 | 42 | ||
| 43 | __aicore__ inline void MainBlockVF(__ubuf__ float* dst, uint32_t ubFactor); | 43 | __aicore__ inline void MainBlockVF(__ubuf__ float* dst, uint32_t ubFactor); |
| 44 | __aicore__ inline void FoldBlockVF(__ubuf__ float* dst, uint32_t ubFactor); | 44 | __aicore__ inline void FoldBlockVF(__ubuf__ float* dst, uint32_t ubFactor); |
| 45 | - __aicore__ inline void LoadTensorForDtypeT(__ubuf__ T* src, AscendC::MicroAPI::RegTensor<float>& dst, | 45 | + __aicore__ inline void LoadTensorForDtypeT(__ubuf__ T* src, AscendC::Reg::RegTensor<float>& dst, |
| 46 | - AscendC::MicroAPI::MaskReg& pregMask, uint32_t offset); | 46 | + AscendC::Reg::MaskReg& pregMask, uint32_t offset); |
| 47 | 47 | ||
| 48 | - __aicore__ inline void StoreTensorForDtypeTOut(__ubuf__ T* dst, AscendC::MicroAPI::RegTensor<float>& src, | 48 | + __aicore__ inline void StoreTensorForDtypeTOut(__ubuf__ T* dst, AscendC::Reg::RegTensor<float>& src, |
| 49 | - AscendC::MicroAPI::MaskReg& preg, uint32_t offset); | 49 | + AscendC::Reg::MaskReg& preg, uint32_t offset); |
| 50 | __aicore__ inline void CopyInX0(int64_t xGmOffset, uint32_t ubFactor); | 50 | __aicore__ inline void CopyInX0(int64_t xGmOffset, uint32_t ubFactor); |
| 51 | __aicore__ inline void CopyInX0X1(int64_t xGmOffset, uint32_t ubFactor); | 51 | __aicore__ inline void CopyInX0X1(int64_t xGmOffset, uint32_t ubFactor); |
| 52 | __aicore__ inline void CopyOutY(int64_t yGmOffset, int64_t ubFactor); | 52 | __aicore__ inline void CopyOutY(int64_t yGmOffset, int64_t ubFactor); |
| @@ -186,20 +186,20 @@ __aicore__ inline void ConfusionSoftmaxGradArRecompute<T>::MainBlockVF(__ubuf__ | |||
| 186 | 186 | ||
| 187 | __VEC_SCOPE__ | 187 | __VEC_SCOPE__ |
| 188 | { | 188 | { |
| 189 | - AscendC::MicroAPI::RegTensor<float> reg0, reg1; | 189 | + AscendC::Reg::RegTensor<float> reg0, reg1; |
| 190 | - AscendC::MicroAPI::MaskReg pregMask; | 190 | + AscendC::Reg::MaskReg pregMask; |
| 191 | 191 | ||
| 192 | uint32_t sreg = ubFactor; | 192 | uint32_t sreg = ubFactor; |
| 193 | uint16_t loopNum = CeilDivision(ubFactor, VL_FP32); | 193 | uint16_t loopNum = CeilDivision(ubFactor, VL_FP32); |
| 194 | for (uint16_t j = 0; j < loopNum; j++) { | 194 | for (uint16_t j = 0; j < loopNum; j++) { |
| 195 | - pregMask = AscendC::MicroAPI::UpdateMask<float>(sreg); | 195 | + pregMask = AscendC::Reg::UpdateMask<float>(sreg); |
| 196 | uint32_t offset = j * VL_FP32; | 196 | uint32_t offset = j * VL_FP32; |
| 197 | LoadTensorForDtypeT(x0Local, reg0, pregMask, offset); | 197 | LoadTensorForDtypeT(x0Local, reg0, pregMask, offset); |
| 198 | LoadTensorForDtypeT(x1Local, reg1, pregMask, offset); | 198 | LoadTensorForDtypeT(x1Local, reg1, pregMask, offset); |
| 199 | 199 | ||
| 200 | Mul(reg0, reg0, reg1, pregMask); | 200 | Mul(reg0, reg0, reg1, pregMask); |
| 201 | 201 | ||
| 202 | - AscendC::MicroAPI::StoreAlign(dst + offset, reg0, pregMask); | 202 | + AscendC::Reg::StoreAlign(dst + offset, reg0, pregMask); |
| 203 | } | 203 | } |
| 204 | } | 204 | } |
| 205 | 205 | ||
| @@ -218,15 +218,14 @@ __aicore__ inline void ConfusionSoftmaxGradArRecompute<T>::FoldBlockVF(__ubuf__ | |||
| 218 | 218 | ||
| 219 | __VEC_SCOPE__ | 219 | __VEC_SCOPE__ |
| 220 | { | 220 | { |
| 221 | - AscendC::MicroAPI::RegTensor<float> reg0, reg1; | 221 | + AscendC::Reg::RegTensor<float> reg0, reg1; |
| 222 | - AscendC::MicroAPI::MaskReg pregMask; | 222 | + AscendC::Reg::MaskReg pregMask; |
| 223 | - AscendC::MicroAPI::MaskReg | 223 | + AscendC::Reg::MaskReg maskFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 224 | - maskFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | ||
| 225 | 224 | ||
| 226 | uint16_t loopTimes = CeilDivision(ubFactor, VL_FP32); | 225 | uint16_t loopTimes = CeilDivision(ubFactor, VL_FP32); |
| 227 | uint32_t sreg = ubFactor; | 226 | uint32_t sreg = ubFactor; |
| 228 | for (uint16_t j = 0; j < loopTimes; j++) { | 227 | for (uint16_t j = 0; j < loopTimes; j++) { |
| 229 | - pregMask = AscendC::MicroAPI::UpdateMask<float>(sreg); | 228 | + pregMask = AscendC::Reg::UpdateMask<float>(sreg); |
| 230 | uint32_t offset = j * VL_FP32; | 229 | uint32_t offset = j * VL_FP32; |
| 231 | 230 | ||
| 232 | LoadTensorForDtypeT(x0Local, reg0, pregMask, offset); | 231 | LoadTensorForDtypeT(x0Local, reg0, pregMask, offset); |
| @@ -234,12 +233,12 @@ __aicore__ inline void ConfusionSoftmaxGradArRecompute<T>::FoldBlockVF(__ubuf__ | |||
| 234 | 233 | ||
| 235 | Mul(reg1, reg0, reg1, pregMask); | 234 | Mul(reg1, reg0, reg1, pregMask); |
| 236 | 235 | ||
| 237 | - AscendC::MicroAPI::LoadAlign(reg0, dst + offset); | 236 | + AscendC::Reg::LoadAlign(reg0, dst + offset); |
| 238 | 237 | ||
| 239 | - AscendC::MicroAPI::Add(reg1, reg0, reg1, pregMask); | 238 | + AscendC::Reg::Add(reg1, reg0, reg1, pregMask); |
| 240 | - AscendC::MicroAPI::Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(reg0, reg1, pregMask); | 239 | + AscendC::Reg::Move<float, AscendC::Reg::MaskMergeMode::MERGING>(reg0, reg1, pregMask); |
| 241 | 240 | ||
| 242 | - AscendC::MicroAPI::StoreAlign(dst + offset, reg0, maskFull); | 241 | + AscendC::Reg::StoreAlign(dst + offset, reg0, maskFull); |
| 243 | } | 242 | } |
| 244 | } | 243 | } |
| 245 | 244 | ||
| @@ -259,16 +258,16 @@ __aicore__ inline void ConfusionSoftmaxGradArRecompute<T>::CalcOutVF(uint32_t ub | |||
| 259 | 258 | ||
| 260 | __VEC_SCOPE__ | 259 | __VEC_SCOPE__ |
| 261 | { | 260 | { |
| 262 | - AscendC::MicroAPI::RegTensor<float> sumReg, x0Reg; | 261 | + AscendC::Reg::RegTensor<float> sumReg, x0Reg; |
| 263 | - AscendC::MicroAPI::MaskReg pregMask; | 262 | + AscendC::Reg::MaskReg pregMask; |
| 264 | 263 | ||
| 265 | uint32_t sreg = ubFactor; | 264 | uint32_t sreg = ubFactor; |
| 266 | uint16_t loopTimes = CeilDivision(ubFactor, VL_FP32); | 265 | uint16_t loopTimes = CeilDivision(ubFactor, VL_FP32); |
| 267 | 266 | ||
| 268 | - AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(sumReg, xSumLocal); | 267 | + AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(sumReg, xSumLocal); |
| 269 | 268 | ||
| 270 | for (uint16_t j = 0; j < loopTimes; j++) { | 269 | for (uint16_t j = 0; j < loopTimes; j++) { |
| 271 | - pregMask = AscendC::MicroAPI::UpdateMask<float>(sreg); | 270 | + pregMask = AscendC::Reg::UpdateMask<float>(sreg); |
| 272 | uint32_t offset = j * VL_FP32; | 271 | uint32_t offset = j * VL_FP32; |
| 273 | LoadTensorForDtypeT(x0Local, x0Reg, pregMask, offset); | 272 | LoadTensorForDtypeT(x0Local, x0Reg, pregMask, offset); |
| 274 | Sub(x0Reg, x0Reg, sumReg, pregMask); | 273 | Sub(x0Reg, x0Reg, sumReg, pregMask); |
| @@ -282,29 +281,31 @@ __aicore__ inline void ConfusionSoftmaxGradArRecompute<T>::CalcOutVF(uint32_t ub | |||
| 282 | 281 | ||
| 283 | template <typename T> | 282 | template <typename T> |
| 284 | __aicore__ inline void ConfusionSoftmaxGradArRecompute<T>::LoadTensorForDtypeT(__ubuf__ T* src, | 283 | __aicore__ inline void ConfusionSoftmaxGradArRecompute<T>::LoadTensorForDtypeT(__ubuf__ T* src, |
| 285 | - AscendC::MicroAPI::RegTensor<float>& dst, | 284 | + AscendC::Reg::RegTensor<float>& dst, |
| 286 | - AscendC::MicroAPI::MaskReg& pregMask, | 285 | + AscendC::Reg::MaskReg& pregMask, |
| 287 | uint32_t offset) | 286 | uint32_t offset) |
| 288 | { | 287 | { |
| 289 | if constexpr (IsSameType<T, float>::value) { | 288 | if constexpr (IsSameType<T, float>::value) { |
| 290 | - LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(dst, (__ubuf__ float*)src + offset); | 289 | + LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(dst, (__ubuf__ float*)src + offset); |
| 291 | } else { // fp16、bf16 | 290 | } else { // fp16、bf16 |
| 292 | - AscendC::MicroAPI::RegTensor<T> xFp16; | 291 | + AscendC::Reg::RegTensor<T> xFp16; |
| 293 | - LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(xFp16, ((__ubuf__ T*)src + offset)); | 292 | + LoadAlign<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(xFp16, ((__ubuf__ T*)src + offset)); |
| 294 | Cast<float, T, castTraitFp16ToFp32>(dst, xFp16, pregMask); | 293 | Cast<float, T, castTraitFp16ToFp32>(dst, xFp16, pregMask); |
| 295 | } | 294 | } |
| 296 | } | 295 | } |
| 297 | 296 | ||
| 298 | template <typename T> | 297 | template <typename T> |
| 299 | -__aicore__ inline void ConfusionSoftmaxGradArRecompute<T>::StoreTensorForDtypeTOut( | 298 | +__aicore__ inline void ConfusionSoftmaxGradArRecompute<T>::StoreTensorForDtypeTOut(__ubuf__ T* dst, |
| 300 | - __ubuf__ T* dst, AscendC::MicroAPI::RegTensor<float>& src, AscendC::MicroAPI::MaskReg& preg, uint32_t offset) | 299 | + AscendC::Reg::RegTensor<float>& src, |
| 300 | + AscendC::Reg::MaskReg& preg, | ||
| 301 | + uint32_t offset) | ||
| 301 | { | 302 | { |
| 302 | if constexpr (IsSameType<T, float>::value) { | 303 | if constexpr (IsSameType<T, float>::value) { |
| 303 | - StoreAlign<T, AscendC::MicroAPI::StoreDist::DIST_NORM>(dst + offset, src, preg); | 304 | + StoreAlign<T, AscendC::Reg::StoreDist::DIST_NORM>(dst + offset, src, preg); |
| 304 | } else { | 305 | } else { |
| 305 | - AscendC::MicroAPI::RegTensor<T> xFp16; | 306 | + AscendC::Reg::RegTensor<T> xFp16; |
| 306 | Cast<T, float, castTraitFp32ToFp16>(xFp16, src, preg); | 307 | Cast<T, float, castTraitFp32ToFp16>(xFp16, src, preg); |
| 307 | - StoreAlign<T, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(dst + offset, xFp16, preg); | 308 | + StoreAlign<T, AscendC::Reg::StoreDist::DIST_PACK_B32>(dst + offset, xFp16, preg); |
| 308 | } | 309 | } |
| 309 | } | 310 | } |
| 310 | 311 | ||
| @@ -20,13 +20,13 @@ | |||
| 20 | 20 | ||
| 21 | namespace ConfusionSoftmaxGradOps { | 21 | namespace ConfusionSoftmaxGradOps { |
| 22 | using namespace AscendC; | 22 | using namespace AscendC; |
| 23 | -using namespace AscendC::MicroAPI; | 23 | +using namespace AscendC::Reg; |
| 24 | 24 | ||
| 25 | -using AscendC::MicroAPI::LoadDist; | 25 | +using AscendC::Reg::LoadDist; |
| 26 | -using AscendC::MicroAPI::MaskMergeMode; | 26 | +using AscendC::Reg::MaskMergeMode; |
| 27 | -using AscendC::MicroAPI::MaskReg; | 27 | +using AscendC::Reg::MaskReg; |
| 28 | -using AscendC::MicroAPI::RegTensor; | 28 | +using AscendC::Reg::RegTensor; |
| 29 | -using AscendC::MicroAPI::StoreDist; | 29 | +using AscendC::Reg::StoreDist; |
| 30 | 30 | ||
| 31 | template <typename T> | 31 | template <typename T> |
| 32 | class ConfusionSoftmaxGradARSmallR { | 32 | class ConfusionSoftmaxGradARSmallR { |
| @@ -210,12 +210,11 @@ private: | |||
| 210 | Sub(x0Reg, x0Reg, sumReg, pregMask); | 210 | Sub(x0Reg, x0Reg, sumReg, pregMask); |
| 211 | 211 | ||
| 212 | if constexpr (xToFp32_) { | 212 | if constexpr (xToFp32_) { |
| 213 | - MicroAPI::StoreAlign(tmpAddrTy + xOffset, x0Reg, pregMask); | 213 | + Reg::StoreAlign(tmpAddrTy + xOffset, x0Reg, pregMask); |
| 214 | } else { // fp16、bf16 | 214 | } else { // fp16、bf16 |
| 215 | RegTensor<T> xFp16; | 215 | RegTensor<T> xFp16; |
| 216 | - MicroAPI::Cast<T, float, castTraitFp32ToFp16>(xFp16, x0Reg, pregMask); | 216 | + Reg::Cast<T, float, castTraitFp32ToFp16>(xFp16, x0Reg, pregMask); |
| 217 | - MicroAPI::StoreAlign<T, MicroAPI::StoreDist::DIST_PACK_B32>(tmpAddrTy + xOffset, xFp16, | 217 | + Reg::StoreAlign<T, Reg::StoreDist::DIST_PACK_B32>(tmpAddrTy + xOffset, xFp16, pregMask); |
| 218 | - pregMask); | ||
| 219 | } | 218 | } |
| 220 | } | 219 | } |
| 221 | } | 220 | } |
| @@ -25,17 +25,17 @@ namespace ConfusionSoftmaxGradOps { | |||
| 25 | using namespace AscendC; | 25 | using namespace AscendC; |
| 26 | using AscendC::Reg::LoadAlign; | 26 | using AscendC::Reg::LoadAlign; |
| 27 | 27 | ||
| 28 | -constexpr static AscendC::MicroAPI::CastTrait castTraitFp16ToFp32 = { | 28 | +constexpr static AscendC::Reg::CastTrait castTraitFp16ToFp32 = { |
| 29 | - AscendC::MicroAPI::RegLayout::ZERO, | 29 | + AscendC::Reg::RegLayout::ZERO, |
| 30 | - AscendC::MicroAPI::SatMode::UNKNOWN, | 30 | + AscendC::Reg::SatMode::UNKNOWN, |
| 31 | - AscendC::MicroAPI::MaskMergeMode::ZEROING, | 31 | + AscendC::Reg::MaskMergeMode::ZEROING, |
| 32 | AscendC::RoundMode::UNKNOWN, | 32 | AscendC::RoundMode::UNKNOWN, |
| 33 | }; | 33 | }; |
| 34 | 34 | ||
| 35 | -constexpr static AscendC::MicroAPI::CastTrait castTraitFp32ToFp16 = { | 35 | +constexpr static AscendC::Reg::CastTrait castTraitFp32ToFp16 = { |
| 36 | - AscendC::MicroAPI::RegLayout::ZERO, | 36 | + AscendC::Reg::RegLayout::ZERO, |
| 37 | - AscendC::MicroAPI::SatMode::NO_SAT, | 37 | + AscendC::Reg::SatMode::NO_SAT, |
| 38 | - AscendC::MicroAPI::MaskMergeMode::ZEROING, | 38 | + AscendC::Reg::MaskMergeMode::ZEROING, |
| 39 | AscendC::RoundMode::CAST_RINT, | 39 | AscendC::RoundMode::CAST_RINT, |
| 40 | }; | 40 | }; |
| 41 | 41 | ||
| @@ -93,11 +93,11 @@ __aicore__ inline int64_t ConfusionSoftmaxGradOpsBase::GetCacheID(const int64_t | |||
| 93 | 93 | ||
| 94 | template <uint32_t RSize, int32_t TailCount = -1, int32_t Index = 0, int32_t Depth = 1> | 94 | template <uint32_t RSize, int32_t TailCount = -1, int32_t Index = 0, int32_t Depth = 1> |
| 95 | struct NlastDichotomyAdd { | 95 | struct NlastDichotomyAdd { |
| 96 | - __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA, | 96 | + __aicore__ static inline void LoadAndAccumulate(AscendC::Reg::RegTensor<float>& acc, __ubuf__ float*& srcA, |
| 97 | - __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask, | 97 | + __ubuf__ float*& srcB, AscendC::Reg::MaskReg& pMask, |
| 98 | uint32_t stride) | 98 | uint32_t stride) |
| 99 | { | 99 | { |
| 100 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 100 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 101 | __ubuf__ float* srcAOffset = srcA + stride * CONST_TWO; | 101 | __ubuf__ float* srcAOffset = srcA + stride * CONST_TWO; |
| 102 | __ubuf__ float* srcBOffset = srcB + stride * CONST_TWO; | 102 | __ubuf__ float* srcBOffset = srcB + stride * CONST_TWO; |
| 103 | if constexpr (TailCount <= 0) { | 103 | if constexpr (TailCount <= 0) { |
| @@ -105,13 +105,13 @@ struct NlastDichotomyAdd { | |||
| 105 | stride * CONST_TWO); | 105 | stride * CONST_TWO); |
| 106 | NlastDichotomyAdd<RSize / CONST_TWO>::LoadAndAccumulate(bReg, srcB, srcBOffset, pMask, stride * CONST_TWO); | 106 | NlastDichotomyAdd<RSize / CONST_TWO>::LoadAndAccumulate(bReg, srcB, srcBOffset, pMask, stride * CONST_TWO); |
| 107 | } | 107 | } |
| 108 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 108 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 109 | } | 109 | } |
| 110 | - __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA, | 110 | + __aicore__ static inline void LoadAndAccumulate(AscendC::Reg::RegTensor<float>& acc, __ubuf__ float*& srcA, |
| 111 | - __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask, | 111 | + __ubuf__ float*& srcB, AscendC::Reg::MaskReg& pMask, |
| 112 | uint32_t stride, uint32_t offset) | 112 | uint32_t stride, uint32_t offset) |
| 113 | { | 113 | { |
| 114 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 114 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 115 | __ubuf__ float* srcAOffset = srcA + stride * CONST_TWO; | 115 | __ubuf__ float* srcAOffset = srcA + stride * CONST_TWO; |
| 116 | __ubuf__ float* srcBOffset = srcB + stride * CONST_TWO; | 116 | __ubuf__ float* srcBOffset = srcB + stride * CONST_TWO; |
| 117 | if constexpr (TailCount <= 0) { | 117 | if constexpr (TailCount <= 0) { |
| @@ -125,56 +125,56 @@ struct NlastDichotomyAdd { | |||
| 125 | NlastDichotomyAdd<RSize / CONST_TWO, TailCount, Index + Depth, Depth * CONST_TWO>::LoadAndAccumulate( | 125 | NlastDichotomyAdd<RSize / CONST_TWO, TailCount, Index + Depth, Depth * CONST_TWO>::LoadAndAccumulate( |
| 126 | bReg, srcB, srcBOffset, pMask, stride * CONST_TWO, offset); | 126 | bReg, srcB, srcBOffset, pMask, stride * CONST_TWO, offset); |
| 127 | } | 127 | } |
| 128 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 128 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 129 | } | 129 | } |
| 130 | }; | 130 | }; |
| 131 | 131 | ||
| 132 | template <int32_t TailCount, int32_t Index, int32_t Depth> | 132 | template <int32_t TailCount, int32_t Index, int32_t Depth> |
| 133 | struct NlastDichotomyAdd<CONST_TWO, TailCount, Index, Depth> { | 133 | struct NlastDichotomyAdd<CONST_TWO, TailCount, Index, Depth> { |
| 134 | - __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA, | 134 | + __aicore__ static inline void LoadAndAccumulate(AscendC::Reg::RegTensor<float>& acc, __ubuf__ float*& srcA, |
| 135 | - __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask, | 135 | + __ubuf__ float*& srcB, AscendC::Reg::MaskReg& pMask, |
| 136 | uint32_t stride) | 136 | uint32_t stride) |
| 137 | { | 137 | { |
| 138 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 138 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 139 | LoadAlign(aReg, (__ubuf__ float*)srcA); | 139 | LoadAlign(aReg, (__ubuf__ float*)srcA); |
| 140 | LoadAlign(bReg, (__ubuf__ float*)srcB); | 140 | LoadAlign(bReg, (__ubuf__ float*)srcB); |
| 141 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 141 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 142 | } | 142 | } |
| 143 | - __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA, | 143 | + __aicore__ static inline void LoadAndAccumulate(AscendC::Reg::RegTensor<float>& acc, __ubuf__ float*& srcA, |
| 144 | - __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask, | 144 | + __ubuf__ float*& srcB, AscendC::Reg::MaskReg& pMask, |
| 145 | uint32_t stride, uint32_t offset) | 145 | uint32_t stride, uint32_t offset) |
| 146 | { | 146 | { |
| 147 | if constexpr (TailCount <= 0) { | 147 | if constexpr (TailCount <= 0) { |
| 148 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg; | 148 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg; |
| 149 | LoadAlign(aReg, (__ubuf__ float*)srcA); | 149 | LoadAlign(aReg, (__ubuf__ float*)srcA); |
| 150 | LoadAlign(bReg, (__ubuf__ float*)srcA + offset); | 150 | LoadAlign(bReg, (__ubuf__ float*)srcA + offset); |
| 151 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); | 151 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); |
| 152 | LoadAlign(bReg, (__ubuf__ float*)srcB); | 152 | LoadAlign(bReg, (__ubuf__ float*)srcB); |
| 153 | LoadAlign(cReg, (__ubuf__ float*)srcB + offset); | 153 | LoadAlign(cReg, (__ubuf__ float*)srcB + offset); |
| 154 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask); | 154 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask); |
| 155 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 155 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 156 | } else { | 156 | } else { |
| 157 | if constexpr (Index + Depth < TailCount) { | 157 | if constexpr (Index + Depth < TailCount) { |
| 158 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg; | 158 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg; |
| 159 | LoadAlign(aReg, (__ubuf__ float*)srcA); | 159 | LoadAlign(aReg, (__ubuf__ float*)srcA); |
| 160 | LoadAlign(bReg, (__ubuf__ float*)srcA + offset); | 160 | LoadAlign(bReg, (__ubuf__ float*)srcA + offset); |
| 161 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); | 161 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); |
| 162 | LoadAlign(bReg, (__ubuf__ float*)srcB); | 162 | LoadAlign(bReg, (__ubuf__ float*)srcB); |
| 163 | LoadAlign(cReg, (__ubuf__ float*)srcB + offset); | 163 | LoadAlign(cReg, (__ubuf__ float*)srcB + offset); |
| 164 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask); | 164 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask); |
| 165 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 165 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 166 | } else if constexpr (Index < TailCount) { | 166 | } else if constexpr (Index < TailCount) { |
| 167 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 167 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 168 | LoadAlign(aReg, (__ubuf__ float*)srcA); | 168 | LoadAlign(aReg, (__ubuf__ float*)srcA); |
| 169 | LoadAlign(bReg, (__ubuf__ float*)srcA + offset); | 169 | LoadAlign(bReg, (__ubuf__ float*)srcA + offset); |
| 170 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); | 170 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); |
| 171 | LoadAlign(bReg, (__ubuf__ float*)srcB); | 171 | LoadAlign(bReg, (__ubuf__ float*)srcB); |
| 172 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 172 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 173 | } else { | 173 | } else { |
| 174 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 174 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 175 | LoadAlign(aReg, (__ubuf__ float*)srcA); | 175 | LoadAlign(aReg, (__ubuf__ float*)srcA); |
| 176 | LoadAlign(bReg, (__ubuf__ float*)srcB); | 176 | LoadAlign(bReg, (__ubuf__ float*)srcB); |
| 177 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 177 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 178 | } | 178 | } |
| 179 | } | 179 | } |
| 180 | } | 180 | } |
| @@ -182,34 +182,34 @@ struct NlastDichotomyAdd<CONST_TWO, TailCount, Index, Depth> { | |||
| 182 | 182 | ||
| 183 | template <> | 183 | template <> |
| 184 | struct NlastDichotomyAdd<CONST_TWO> { | 184 | struct NlastDichotomyAdd<CONST_TWO> { |
| 185 | - __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA, | 185 | + __aicore__ static inline void LoadAndAccumulate(AscendC::Reg::RegTensor<float>& acc, __ubuf__ float*& srcA, |
| 186 | - __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask, | 186 | + __ubuf__ float*& srcB, AscendC::Reg::MaskReg& pMask, |
| 187 | uint32_t stride) | 187 | uint32_t stride) |
| 188 | { | 188 | { |
| 189 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 189 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 190 | LoadAlign(aReg, (__ubuf__ float*)srcA); | 190 | LoadAlign(aReg, (__ubuf__ float*)srcA); |
| 191 | LoadAlign(bReg, (__ubuf__ float*)srcB); | 191 | LoadAlign(bReg, (__ubuf__ float*)srcB); |
| 192 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 192 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 193 | } | 193 | } |
| 194 | - __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA, | 194 | + __aicore__ static inline void LoadAndAccumulate(AscendC::Reg::RegTensor<float>& acc, __ubuf__ float*& srcA, |
| 195 | - __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask, | 195 | + __ubuf__ float*& srcB, AscendC::Reg::MaskReg& pMask, |
| 196 | uint32_t stride, uint32_t offset) | 196 | uint32_t stride, uint32_t offset) |
| 197 | { | 197 | { |
| 198 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg; | 198 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg; |
| 199 | LoadAlign(aReg, (__ubuf__ float*)srcA); | 199 | LoadAlign(aReg, (__ubuf__ float*)srcA); |
| 200 | LoadAlign(bReg, (__ubuf__ float*)srcA + offset); | 200 | LoadAlign(bReg, (__ubuf__ float*)srcA + offset); |
| 201 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); | 201 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); |
| 202 | LoadAlign(bReg, (__ubuf__ float*)srcB); | 202 | LoadAlign(bReg, (__ubuf__ float*)srcB); |
| 203 | LoadAlign(cReg, (__ubuf__ float*)srcB + offset); | 203 | LoadAlign(cReg, (__ubuf__ float*)srcB + offset); |
| 204 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask); | 204 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask); |
| 205 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 205 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 206 | } | 206 | } |
| 207 | }; | 207 | }; |
| 208 | 208 | ||
| 209 | template <> | 209 | template <> |
| 210 | struct NlastDichotomyAdd<1> { | 210 | struct NlastDichotomyAdd<1> { |
| 211 | - __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA, | 211 | + __aicore__ static inline void LoadAndAccumulate(AscendC::Reg::RegTensor<float>& acc, __ubuf__ float*& srcA, |
| 212 | - __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask, | 212 | + __ubuf__ float*& srcB, AscendC::Reg::MaskReg& pMask, |
| 213 | uint32_t stride) | 213 | uint32_t stride) |
| 214 | { | 214 | { |
| 215 | LoadAlign(acc, (__ubuf__ float*)srcA); | 215 | LoadAlign(acc, (__ubuf__ float*)srcA); |
| @@ -233,14 +233,14 @@ __aicore__ inline void ConfusionSoftmaxGradOpsBase::UpdateCache(const LocalTenso | |||
| 233 | __ubuf__ float* cah = (__ubuf__ float*)dstTensor.GetPhyAddr() + cacheID * stride; | 233 | __ubuf__ float* cah = (__ubuf__ float*)dstTensor.GetPhyAddr() + cacheID * stride; |
| 234 | __ubuf__ float* src = (__ubuf__ float*)srcTensor.GetPhyAddr(); | 234 | __ubuf__ float* src = (__ubuf__ float*)srcTensor.GetPhyAddr(); |
| 235 | uint32_t sreg = static_cast<uint32_t>(count); | 235 | uint32_t sreg = static_cast<uint32_t>(count); |
| 236 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 236 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 237 | - AscendC::MicroAPI::MaskReg pMask; | 237 | + AscendC::Reg::MaskReg pMask; |
| 238 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { | 238 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { |
| 239 | - pMask = AscendC::MicroAPI::UpdateMask<float>(sreg); | 239 | + pMask = AscendC::Reg::UpdateMask<float>(sreg); |
| 240 | LoadAlign(aReg, (__ubuf__ float*)src + i * outerLoopStride); | 240 | LoadAlign(aReg, (__ubuf__ float*)src + i * outerLoopStride); |
| 241 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { | 241 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { |
| 242 | LoadAlign(bReg, (__ubuf__ float*)dst + i * outerLoopStride + j * innerLoopStride); | 242 | LoadAlign(bReg, (__ubuf__ float*)dst + i * outerLoopStride + j * innerLoopStride); |
| 243 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); | 243 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); |
| 244 | } | 244 | } |
| 245 | StoreAlign((__ubuf__ float*)cah + i * outerLoopStride, aReg, pMask); | 245 | StoreAlign((__ubuf__ float*)cah + i * outerLoopStride, aReg, pMask); |
| 246 | } | 246 | } |
| @@ -266,18 +266,18 @@ __aicore__ inline void ConfusionSoftmaxGradOpsBase::Normalize(const LocalTensor< | |||
| 266 | __ubuf__ float* mean = (__ubuf__ float*)meanTensor.GetPhyAddr(); | 266 | __ubuf__ float* mean = (__ubuf__ float*)meanTensor.GetPhyAddr(); |
| 267 | __ubuf__ float* rstd = (__ubuf__ float*)rstdTensor.GetPhyAddr(); | 267 | __ubuf__ float* rstd = (__ubuf__ float*)rstdTensor.GetPhyAddr(); |
| 268 | uint32_t count; | 268 | uint32_t count; |
| 269 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg; | 269 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg; |
| 270 | - AscendC::MicroAPI::RegTensor<float> meanReg, rstdReg; | 270 | + AscendC::Reg::RegTensor<float> meanReg, rstdReg; |
| 271 | - AscendC::MicroAPI::MaskReg pMask; | 271 | + AscendC::Reg::MaskReg pMask; |
| 272 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { | 272 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { |
| 273 | count = static_cast<uint32_t>(colSize); | 273 | count = static_cast<uint32_t>(colSize); |
| 274 | - LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(meanReg, (__ubuf__ float*)mean + i); | 274 | + LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(meanReg, (__ubuf__ float*)mean + i); |
| 275 | - LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(rstdReg, (__ubuf__ float*)rstd + i); | 275 | + LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(rstdReg, (__ubuf__ float*)rstd + i); |
| 276 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { | 276 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { |
| 277 | - pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 277 | + pMask = AscendC::Reg::UpdateMask<float>(count); |
| 278 | LoadAlign(aReg, (__ubuf__ float*)src + i * outerLoopStride + j * innerLoopStride); | 278 | LoadAlign(aReg, (__ubuf__ float*)src + i * outerLoopStride + j * innerLoopStride); |
| 279 | - Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(bReg, aReg, meanReg, pMask); | 279 | + Sub<float, AscendC::Reg::MaskMergeMode::ZEROING>(bReg, aReg, meanReg, pMask); |
| 280 | - Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, bReg, rstdReg, pMask); | 280 | + Mul<float, AscendC::Reg::MaskMergeMode::ZEROING>(cReg, bReg, rstdReg, pMask); |
| 281 | StoreAlign((__ubuf__ float*)dst + i * outerLoopStride + j * innerLoopStride, cReg, pMask); | 281 | StoreAlign((__ubuf__ float*)dst + i * outerLoopStride + j * innerLoopStride, cReg, pMask); |
| 282 | } | 282 | } |
| 283 | } | 283 | } |
| @@ -23,12 +23,12 @@ namespace EluOp { | |||
| 23 | using namespace Ops::Base; | 23 | using namespace Ops::Base; |
| 24 | using namespace AscendC; | 24 | using namespace AscendC; |
| 25 | 25 | ||
| 26 | -constexpr static AscendC::MicroAPI::CastTrait castTrait0 = { | 26 | +constexpr static AscendC::Reg::CastTrait castTrait0 = {AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::UNKNOWN, |
| 27 | - AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::UNKNOWN, AscendC::MicroAPI::MaskMergeMode::ZEROING, | 27 | + AscendC::Reg::MaskMergeMode::ZEROING, |
| 28 | - AscendC::RoundMode::UNKNOWN}; | 28 | + AscendC::RoundMode::UNKNOWN}; |
| 29 | -constexpr static AscendC::MicroAPI::CastTrait castTrait1 = { | 29 | +constexpr static AscendC::Reg::CastTrait castTrait1 = {AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::NO_SAT, |
| 30 | - AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::NO_SAT, AscendC::MicroAPI::MaskMergeMode::ZEROING, | 30 | + AscendC::Reg::MaskMergeMode::ZEROING, |
| 31 | - AscendC::RoundMode::CAST_RINT}; | 31 | + AscendC::RoundMode::CAST_RINT}; |
| 32 | 32 | ||
| 33 | constexpr int ELU_ATTR_ALPHA_INDEX = 0; | 33 | constexpr int ELU_ATTR_ALPHA_INDEX = 0; |
| 34 | constexpr int ELU_ATTR_SCALE_INDEX = 1; | 34 | constexpr int ELU_ATTR_SCALE_INDEX = 1; |
| @@ -57,95 +57,95 @@ struct EluCustom : public Vec::ElemwiseQuaternaryOP<T, T, float, float, float> { | |||
| 57 | __ubuf__ T* srcAddr = (__ubuf__ T*)src.GetPhyAddr(); | 57 | __ubuf__ T* srcAddr = (__ubuf__ T*)src.GetPhyAddr(); |
| 58 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); | 58 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); |
| 59 | 59 | ||
| 60 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInput; | 60 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInput; |
| 61 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregInputFloat; | 61 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregInputFloat; |
| 62 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregNeg; | 62 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregNeg; |
| 63 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregOutputFloat; | 63 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregOutputFloat; |
| 64 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregOutput; | 64 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregOutput; |
| 65 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregExp; | 65 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregExp; |
| 66 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregPoly; | 66 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregPoly; |
| 67 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregAbsZ; | 67 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregAbsZ; |
| 68 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregT1; | 68 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregT1; |
| 69 | - MicroAPI::MaskReg mask, cmpMask, cmpExpm1; | 69 | + Reg::MaskReg mask, cmpMask, cmpExpm1; |
| 70 | if constexpr (std::is_same_v<T, float>) { | 70 | if constexpr (std::is_same_v<T, float>) { |
| 71 | __VEC_SCOPE__ | 71 | __VEC_SCOPE__ |
| 72 | { | 72 | { |
| 73 | for (uint16_t loopIdx = 0; loopIdx < static_cast<uint16_t>(loopNum); loopIdx++) { | 73 | for (uint16_t loopIdx = 0; loopIdx < static_cast<uint16_t>(loopNum); loopIdx++) { |
| 74 | - mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count); | 74 | + mask = Reg::UpdateMask<T, Reg::RegTraitNumOne>(count); |
| 75 | // OpCopyIn | 75 | // OpCopyIn |
| 76 | - MicroAPI::LoadAlign(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize)); | 76 | + Reg::LoadAlign(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize)); |
| 77 | - MicroAPI::Muls(vregNeg, vregInput, inputScale, mask); | 77 | + Reg::Muls(vregNeg, vregInput, inputScale, mask); |
| 78 | 78 | ||
| 79 | // Polynomial expm1(z) = z + z^2/2 + z^3/6 + z^4/24 + z^5/120 for |z| < threshold | 79 | // Polynomial expm1(z) = z + z^2/2 + z^3/6 + z^4/24 + z^5/120 for |z| < threshold |
| 80 | // Horner: T1 = T1 * z + coeff, then poly = T1 * z | 80 | // Horner: T1 = T1 * z + coeff, then poly = T1 * z |
| 81 | - MicroAPI::Muls(vregT1, vregNeg, coeff5, mask); | 81 | + Reg::Muls(vregT1, vregNeg, coeff5, mask); |
| 82 | - MicroAPI::Adds(vregT1, vregT1, coeff4, mask); | 82 | + Reg::Adds(vregT1, vregT1, coeff4, mask); |
| 83 | - MicroAPI::Mul(vregT1, vregT1, vregNeg, mask); | 83 | + Reg::Mul(vregT1, vregT1, vregNeg, mask); |
| 84 | - MicroAPI::Adds(vregT1, vregT1, coeff3, mask); | 84 | + Reg::Adds(vregT1, vregT1, coeff3, mask); |
| 85 | - MicroAPI::Mul(vregT1, vregT1, vregNeg, mask); | 85 | + Reg::Mul(vregT1, vregT1, vregNeg, mask); |
| 86 | - MicroAPI::Adds(vregT1, vregT1, coeff2, mask); | 86 | + Reg::Adds(vregT1, vregT1, coeff2, mask); |
| 87 | - MicroAPI::Mul(vregT1, vregT1, vregNeg, mask); | 87 | + Reg::Mul(vregT1, vregT1, vregNeg, mask); |
| 88 | - MicroAPI::Adds(vregT1, vregT1, coeff1, mask); | 88 | + Reg::Adds(vregT1, vregT1, coeff1, mask); |
| 89 | - MicroAPI::Mul(vregPoly, vregT1, vregNeg, mask); | 89 | + Reg::Mul(vregPoly, vregT1, vregNeg, mask); |
| 90 | 90 | ||
| 91 | // exp(z) - 1 for |z| >= threshold | 91 | // exp(z) - 1 for |z| >= threshold |
| 92 | - MicroAPI::Exp(vregExp, vregNeg, mask); | 92 | + Reg::Exp(vregExp, vregNeg, mask); |
| 93 | - MicroAPI::Adds(vregExp, vregExp, constNegOne, mask); | 93 | + Reg::Adds(vregExp, vregExp, constNegOne, mask); |
| 94 | 94 | ||
| 95 | // Select expm1 result based on |z| | 95 | // Select expm1 result based on |z| |
| 96 | - MicroAPI::Abs(vregAbsZ, vregNeg, mask); | 96 | + Reg::Abs(vregAbsZ, vregNeg, mask); |
| 97 | - MicroAPI::Compares<T, CMPMODE::LT>(cmpExpm1, vregAbsZ, expm1Threshold, mask); | 97 | + Reg::Compares<T, CMPMODE::LT>(cmpExpm1, vregAbsZ, expm1Threshold, mask); |
| 98 | - MicroAPI::Select<T>(vregNeg, vregPoly, vregExp, cmpExpm1); | 98 | + Reg::Select<T>(vregNeg, vregPoly, vregExp, cmpExpm1); |
| 99 | 99 | ||
| 100 | - MicroAPI::Muls(vregNeg, vregNeg, alpha, mask); | 100 | + Reg::Muls(vregNeg, vregNeg, alpha, mask); |
| 101 | 101 | ||
| 102 | - MicroAPI::Compares<T, CMPMODE::GT>(cmpMask, vregInput, (float)0.0, mask); | 102 | + Reg::Compares<T, CMPMODE::GT>(cmpMask, vregInput, (float)0.0, mask); |
| 103 | - MicroAPI::Select<T>(vregOutput, vregInput, vregNeg, cmpMask); | 103 | + Reg::Select<T>(vregOutput, vregInput, vregNeg, cmpMask); |
| 104 | - MicroAPI::Muls(vregOutput, vregOutput, scale, mask); | 104 | + Reg::Muls(vregOutput, vregOutput, scale, mask); |
| 105 | 105 | ||
| 106 | // OpCopyOut | 106 | // OpCopyOut |
| 107 | - MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); | 107 | + Reg::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); |
| 108 | } | 108 | } |
| 109 | } | 109 | } |
| 110 | } else { | 110 | } else { |
| 111 | __VEC_SCOPE__ | 111 | __VEC_SCOPE__ |
| 112 | { | 112 | { |
| 113 | for (uint16_t loopIdx = 0; loopIdx < static_cast<uint16_t>(loopNum); loopIdx++) { | 113 | for (uint16_t loopIdx = 0; loopIdx < static_cast<uint16_t>(loopNum); loopIdx++) { |
| 114 | - mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count); | 114 | + mask = Reg::UpdateMask<float, Reg::RegTraitNumOne>(count); |
| 115 | // OpCopyIn | 115 | // OpCopyIn |
| 116 | - MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>( | 116 | + Reg::LoadAlign<T, Reg::LoadDist::DIST_UNPACK_B16>(vregInput, |
| 117 | - vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize)); | 117 | + (__ubuf__ T*)(srcAddr + loopIdx * vlSize)); |
| 118 | - MicroAPI::Cast<float, T, castTrait0>(vregInputFloat, vregInput, mask); | 118 | + Reg::Cast<float, T, castTrait0>(vregInputFloat, vregInput, mask); |
| 119 | - MicroAPI::Muls(vregNeg, vregInputFloat, inputScale, mask); | 119 | + Reg::Muls(vregNeg, vregInputFloat, inputScale, mask); |
| 120 | 120 | ||
| 121 | // Polynomial expm1(z) = z + z^2/2 + z^3/6 + z^4/24 for |z| < threshold | 121 | // Polynomial expm1(z) = z + z^2/2 + z^3/6 + z^4/24 for |z| < threshold |
| 122 | // Horner: T1 = T1 * z + coeff, then poly = T1 * z | 122 | // Horner: T1 = T1 * z + coeff, then poly = T1 * z |
| 123 | - MicroAPI::Muls(vregT1, vregNeg, coeff4, mask); | 123 | + Reg::Muls(vregT1, vregNeg, coeff4, mask); |
| 124 | - MicroAPI::Adds(vregT1, vregT1, coeff3, mask); | 124 | + Reg::Adds(vregT1, vregT1, coeff3, mask); |
| 125 | - MicroAPI::Mul(vregT1, vregT1, vregNeg, mask); | 125 | + Reg::Mul(vregT1, vregT1, vregNeg, mask); |
| 126 | - MicroAPI::Adds(vregT1, vregT1, coeff2, mask); | 126 | + Reg::Adds(vregT1, vregT1, coeff2, mask); |
| 127 | - MicroAPI::Mul(vregT1, vregT1, vregNeg, mask); | 127 | + Reg::Mul(vregT1, vregT1, vregNeg, mask); |
| 128 | - MicroAPI::Adds(vregT1, vregT1, coeff1, mask); | 128 | + Reg::Adds(vregT1, vregT1, coeff1, mask); |
| 129 | - MicroAPI::Mul(vregPoly, vregT1, vregNeg, mask); | 129 | + Reg::Mul(vregPoly, vregT1, vregNeg, mask); |
| 130 | 130 | ||
| 131 | // exp(z) - 1 for |z| >= threshold | 131 | // exp(z) - 1 for |z| >= threshold |
| 132 | - MicroAPI::Exp(vregExp, vregNeg, mask); | 132 | + Reg::Exp(vregExp, vregNeg, mask); |
| 133 | - MicroAPI::Adds(vregExp, vregExp, constNegOne, mask); | 133 | + Reg::Adds(vregExp, vregExp, constNegOne, mask); |
| 134 | 134 | ||
| 135 | // Select expm1 result based on |z| | 135 | // Select expm1 result based on |z| |
| 136 | - MicroAPI::Abs(vregAbsZ, vregNeg, mask); | 136 | + Reg::Abs(vregAbsZ, vregNeg, mask); |
| 137 | - MicroAPI::Compares<float, CMPMODE::LT>(cmpExpm1, vregAbsZ, expm1Threshold, mask); | 137 | + Reg::Compares<float, CMPMODE::LT>(cmpExpm1, vregAbsZ, expm1Threshold, mask); |
| 138 | - MicroAPI::Select<float>(vregNeg, vregPoly, vregExp, cmpExpm1); | 138 | + Reg::Select<float>(vregNeg, vregPoly, vregExp, cmpExpm1); |
| 139 | 139 | ||
| 140 | - MicroAPI::Muls(vregNeg, vregNeg, alpha, mask); | 140 | + Reg::Muls(vregNeg, vregNeg, alpha, mask); |
| 141 | 141 | ||
| 142 | - MicroAPI::Compares<float, CMPMODE::GT>(cmpMask, vregInputFloat, (float)0.0, mask); | 142 | + Reg::Compares<float, CMPMODE::GT>(cmpMask, vregInputFloat, (float)0.0, mask); |
| 143 | - MicroAPI::Select<float>(vregOutputFloat, vregInputFloat, vregNeg, cmpMask); | 143 | + Reg::Select<float>(vregOutputFloat, vregInputFloat, vregNeg, cmpMask); |
| 144 | - MicroAPI::Muls(vregOutputFloat, vregOutputFloat, scale, mask); | 144 | + Reg::Muls(vregOutputFloat, vregOutputFloat, scale, mask); |
| 145 | - MicroAPI::Cast<T, float, castTrait1>(vregOutput, vregOutputFloat, mask); | 145 | + Reg::Cast<T, float, castTrait1>(vregOutput, vregOutputFloat, mask); |
| 146 | // OpCopyOut | 146 | // OpCopyOut |
| 147 | - MicroAPI::StoreAlign<T, MicroAPI::StoreDist::DIST_PACK_B32>( | 147 | + Reg::StoreAlign<T, Reg::StoreDist::DIST_PACK_B32>((__ubuf__ T*)(dstAddr + loopIdx * vlSize), |
| 148 | - (__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); | 148 | + vregOutput, mask); |
| 149 | } | 149 | } |
| 150 | } | 150 | } |
| 151 | } | 151 | } |
| @@ -34,23 +34,23 @@ struct FastGeluCustom : public Vec::ElemwiseUnaryOP<T, T> { | |||
| 34 | __ubuf__ T* srcAddr = (__ubuf__ T*)src.GetPhyAddr(); | 34 | __ubuf__ T* srcAddr = (__ubuf__ T*)src.GetPhyAddr(); |
| 35 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); | 35 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); |
| 36 | 36 | ||
| 37 | - AscendC::MicroAPI::RegTensor<T, AscendC::MicroAPI::RegTraitNumOne> x; | 37 | + AscendC::Reg::RegTensor<T, AscendC::Reg::RegTraitNumOne> x; |
| 38 | - AscendC::MicroAPI::RegTensor<T, AscendC::MicroAPI::RegTraitNumOne> denominator; | 38 | + AscendC::Reg::RegTensor<T, AscendC::Reg::RegTraitNumOne> denominator; |
| 39 | - AscendC::MicroAPI::RegTensor<T, AscendC::MicroAPI::RegTraitNumOne> result; | 39 | + AscendC::Reg::RegTensor<T, AscendC::Reg::RegTraitNumOne> result; |
| 40 | - static constexpr AscendC::MicroAPI::DivSpecificMode mode = {AscendC::MicroAPI::MaskMergeMode::ZEROING, | 40 | + static constexpr AscendC::Reg::DivSpecificMode mode = {AscendC::Reg::MaskMergeMode::ZEROING, |
| 41 | - highPrecisionDiv}; | 41 | + highPrecisionDiv}; |
| 42 | - AscendC::MicroAPI::MaskReg mask; | 42 | + AscendC::Reg::MaskReg mask; |
| 43 | 43 | ||
| 44 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 44 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 45 | - mask = AscendC::MicroAPI::UpdateMask<T, AscendC::MicroAPI::RegTraitNumOne>(count); | 45 | + mask = AscendC::Reg::UpdateMask<T, AscendC::Reg::RegTraitNumOne>(count); |
| 46 | - AscendC::MicroAPI::LoadAlign(x, (__ubuf__ T*)(srcAddr + loopIdx * vlSize)); | 46 | + AscendC::Reg::LoadAlign(x, (__ubuf__ T*)(srcAddr + loopIdx * vlSize)); |
| 47 | - AscendC::MicroAPI::Muls(denominator, x, value1, mask); | 47 | + AscendC::Reg::Muls(denominator, x, value1, mask); |
| 48 | - AscendC::MicroAPI::Exp(denominator, denominator, mask); | 48 | + AscendC::Reg::Exp(denominator, denominator, mask); |
| 49 | - AscendC::MicroAPI::Adds(denominator, denominator, value2, mask); | 49 | + AscendC::Reg::Adds(denominator, denominator, value2, mask); |
| 50 | // result = x / (Exp(-1.702 * x) + 1) | 50 | // result = x / (Exp(-1.702 * x) + 1) |
| 51 | - AscendC::MicroAPI::Div<T, &mode>(result, x, denominator, mask); | 51 | + AscendC::Reg::Div<T, &mode>(result, x, denominator, mask); |
| 52 | // OpCopyOut | 52 | // OpCopyOut |
| 53 | - AscendC::MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), result, mask); | 53 | + AscendC::Reg::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), result, mask); |
| 54 | } | 54 | } |
| 55 | } | 55 | } |
| 56 | 56 | ||
| @@ -37,37 +37,36 @@ struct FastGeluGradCustom : public Vec::ElemwiseBinaryOP<T, T, T> { | |||
| 37 | __ubuf__ T* src2Addr = (__ubuf__ T*)src2.GetPhyAddr(); | 37 | __ubuf__ T* src2Addr = (__ubuf__ T*)src2.GetPhyAddr(); |
| 38 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); | 38 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); |
| 39 | 39 | ||
| 40 | - AscendC::MicroAPI::RegTensor<T, AscendC::MicroAPI::RegTraitNumOne> dy; | 40 | + AscendC::Reg::RegTensor<T, AscendC::Reg::RegTraitNumOne> dy; |
| 41 | - AscendC::MicroAPI::RegTensor<T, AscendC::MicroAPI::RegTraitNumOne> x; | 41 | + AscendC::Reg::RegTensor<T, AscendC::Reg::RegTraitNumOne> x; |
| 42 | - AscendC::MicroAPI::RegTensor<T, AscendC::MicroAPI::RegTraitNumOne> constantOne; | 42 | + AscendC::Reg::RegTensor<T, AscendC::Reg::RegTraitNumOne> constantOne; |
| 43 | - AscendC::MicroAPI::RegTensor<T, AscendC::MicroAPI::RegTraitNumOne> value1MulsX; | 43 | + AscendC::Reg::RegTensor<T, AscendC::Reg::RegTraitNumOne> value1MulsX; |
| 44 | - AscendC::MicroAPI::RegTensor<T, AscendC::MicroAPI::RegTraitNumOne> temp1Reg; | 44 | + AscendC::Reg::RegTensor<T, AscendC::Reg::RegTraitNumOne> temp1Reg; |
| 45 | - AscendC::MicroAPI::RegTensor<T, AscendC::MicroAPI::RegTraitNumOne> temp2Reg; | 45 | + AscendC::Reg::RegTensor<T, AscendC::Reg::RegTraitNumOne> temp2Reg; |
| 46 | - AscendC::MicroAPI::RegTensor<T, AscendC::MicroAPI::RegTraitNumOne> divRes; | 46 | + AscendC::Reg::RegTensor<T, AscendC::Reg::RegTraitNumOne> divRes; |
| 47 | - static constexpr AscendC::MicroAPI::DivSpecificMode mode = {AscendC::MicroAPI::MaskMergeMode::ZEROING, | 47 | + static constexpr AscendC::Reg::DivSpecificMode mode = {AscendC::Reg::MaskMergeMode::ZEROING, true}; |
| 48 | - true}; | 48 | + AscendC::Reg::MaskReg mask; |
| 49 | - AscendC::MicroAPI::MaskReg mask; | 49 | + AscendC::Reg::Duplicate(constantOne, value3); |
| 50 | - AscendC::MicroAPI::Duplicate(constantOne, value3); | ||
| 51 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 50 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 52 | - mask = AscendC::MicroAPI::UpdateMask<T, AscendC::MicroAPI::RegTraitNumOne>(count); | 51 | + mask = AscendC::Reg::UpdateMask<T, AscendC::Reg::RegTraitNumOne>(count); |
| 53 | // OpCopyIn0 | 52 | // OpCopyIn0 |
| 54 | - AscendC::MicroAPI::LoadAlign(x, (__ubuf__ T*)(src2Addr + loopIdx * vlSize)); | 53 | + AscendC::Reg::LoadAlign(x, (__ubuf__ T*)(src2Addr + loopIdx * vlSize)); |
| 55 | // temp1Reg = e^(-1.702x) + 1 | 54 | // temp1Reg = e^(-1.702x) + 1 |
| 56 | - AscendC::MicroAPI::Muls(value1MulsX, x, value2, mask); | 55 | + AscendC::Reg::Muls(value1MulsX, x, value2, mask); |
| 57 | - AscendC::MicroAPI::Exp(temp1Reg, value1MulsX, mask); | 56 | + AscendC::Reg::Exp(temp1Reg, value1MulsX, mask); |
| 58 | - AscendC::MicroAPI::Adds(temp1Reg, temp1Reg, value3, mask); | 57 | + AscendC::Reg::Adds(temp1Reg, temp1Reg, value3, mask); |
| 59 | // temp2Reg = (1/(e^(-1.702x) + 1)) - 1 | 58 | // temp2Reg = (1/(e^(-1.702x) + 1)) - 1 |
| 60 | - AscendC::MicroAPI::Div<T, &mode>(divRes, constantOne, temp1Reg, mask); | 59 | + AscendC::Reg::Div<T, &mode>(divRes, constantOne, temp1Reg, mask); |
| 61 | - AscendC::MicroAPI::Adds(temp2Reg, divRes, value4, mask); | 60 | + AscendC::Reg::Adds(temp2Reg, divRes, value4, mask); |
| 62 | // divRes = (temp2Reg * -1.702x + 1) * 1/(e^(-1.702x) * dy | 61 | // divRes = (temp2Reg * -1.702x + 1) * 1/(e^(-1.702x) * dy |
| 63 | - AscendC::MicroAPI::Mul(temp2Reg, temp2Reg, value1MulsX, mask); | 62 | + AscendC::Reg::Mul(temp2Reg, temp2Reg, value1MulsX, mask); |
| 64 | - AscendC::MicroAPI::Adds(temp2Reg, temp2Reg, value3, mask); | 63 | + AscendC::Reg::Adds(temp2Reg, temp2Reg, value3, mask); |
| 65 | - AscendC::MicroAPI::Mul(divRes, temp2Reg, divRes, mask); | 64 | + AscendC::Reg::Mul(divRes, temp2Reg, divRes, mask); |
| 66 | - AscendC::MicroAPI::LoadAlign(dy, (__ubuf__ T*)(src1Addr + loopIdx * vlSize)); | 65 | + AscendC::Reg::LoadAlign(dy, (__ubuf__ T*)(src1Addr + loopIdx * vlSize)); |
| 67 | - AscendC::MicroAPI::Mul(divRes, dy, divRes, mask); | 66 | + AscendC::Reg::Mul(divRes, dy, divRes, mask); |
| 68 | 67 | ||
| 69 | // OpCopyOut | 68 | // OpCopyOut |
| 70 | - AscendC::MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), divRes, mask); | 69 | + AscendC::Reg::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), divRes, mask); |
| 71 | } | 70 | } |
| 72 | } | 71 | } |
| 73 | 72 | ||
| @@ -41,28 +41,28 @@ struct GeluCustom : public Vec::ElemwiseUnaryOP<T, T> { | |||
| 41 | __ubuf__ T* srcAddr = (__ubuf__ T*)src.GetPhyAddr(); | 41 | __ubuf__ T* srcAddr = (__ubuf__ T*)src.GetPhyAddr(); |
| 42 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); | 42 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); |
| 43 | 43 | ||
| 44 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInput; | 44 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInput; |
| 45 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputSqr; | 45 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputSqr; |
| 46 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputCub; | 46 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputCub; |
| 47 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregOutput; | 47 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregOutput; |
| 48 | - MicroAPI::MaskReg mask; | 48 | + Reg::MaskReg mask; |
| 49 | if constexpr (std::is_same_v<T, float>) { | 49 | if constexpr (std::is_same_v<T, float>) { |
| 50 | __VEC_SCOPE__ | 50 | __VEC_SCOPE__ |
| 51 | { | 51 | { |
| 52 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 52 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 53 | - mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count); | 53 | + mask = Reg::UpdateMask<T, Reg::RegTraitNumOne>(count); |
| 54 | // OpCopyIn | 54 | // OpCopyIn |
| 55 | - MicroAPI::LoadAlign(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize)); | 55 | + Reg::LoadAlign(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize)); |
| 56 | - MicroAPI::Mul(vregInputSqr, vregInput, vregInput, mask); | 56 | + Reg::Mul(vregInputSqr, vregInput, vregInput, mask); |
| 57 | - MicroAPI::Mul(vregInputCub, vregInputSqr, vregInput, mask); | 57 | + Reg::Mul(vregInputCub, vregInputSqr, vregInput, mask); |
| 58 | - MicroAPI::Axpy(vregInputCub, vregInput, TANH_APPROX_FACTOR, mask); | 58 | + Reg::Axpy(vregInputCub, vregInput, TANH_APPROX_FACTOR, mask); |
| 59 | - MicroAPI::Muls(vregInputCub, vregInputCub, NEG_SQRT_EIGHT_OVER_PI, mask); | 59 | + Reg::Muls(vregInputCub, vregInputCub, NEG_SQRT_EIGHT_OVER_PI, mask); |
| 60 | - MicroAPI::Exp(vregInputCub, vregInputCub, mask); | 60 | + Reg::Exp(vregInputCub, vregInputCub, mask); |
| 61 | - MicroAPI::Adds(vregInputCub, vregInputCub, (float)1.0, mask); | 61 | + Reg::Adds(vregInputCub, vregInputCub, (float)1.0, mask); |
| 62 | - MicroAPI::Div(vregOutput, vregInput, vregInputCub, mask); | 62 | + Reg::Div(vregOutput, vregInput, vregInputCub, mask); |
| 63 | 63 | ||
| 64 | // OpCopyOut | 64 | // OpCopyOut |
| 65 | - MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); | 65 | + Reg::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); |
| 66 | } | 66 | } |
| 67 | } | 67 | } |
| 68 | } | 68 | } |
| @@ -45,58 +45,58 @@ struct GeluGradCustom : public Vec::ElemwiseBinaryOP<T, T, T> { | |||
| 45 | __ubuf__ T* src1Addr = (__ubuf__ T*)src1.GetPhyAddr(); | 45 | __ubuf__ T* src1Addr = (__ubuf__ T*)src1.GetPhyAddr(); |
| 46 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); | 46 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); |
| 47 | 47 | ||
| 48 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputDy; | 48 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputDy; |
| 49 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputX; | 49 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputX; |
| 50 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputXSqr; | 50 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputXSqr; |
| 51 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputPX; | 51 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputPX; |
| 52 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputRes0; | 52 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputRes0; |
| 53 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputT; | 53 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputT; |
| 54 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputDiv; | 54 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputDiv; |
| 55 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputOne; | 55 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputOne; |
| 56 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputZero; | 56 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputZero; |
| 57 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputResp; | 57 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputResp; |
| 58 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregSelect; | 58 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregSelect; |
| 59 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregOutput; | 59 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregOutput; |
| 60 | - MicroAPI::MaskReg mask, cmpMask; | 60 | + Reg::MaskReg mask, cmpMask; |
| 61 | 61 | ||
| 62 | - static constexpr AscendC::MicroAPI::DivSpecificMode DIV_MODE = { | 62 | + static constexpr AscendC::Reg::DivSpecificMode DIV_MODE = { |
| 63 | - AscendC::MicroAPI::MaskMergeMode::ZEROING, | 63 | + AscendC::Reg::MaskMergeMode::ZEROING, |
| 64 | false, | 64 | false, |
| 65 | }; | 65 | }; |
| 66 | if constexpr (std::is_same_v<T, float>) { | 66 | if constexpr (std::is_same_v<T, float>) { |
| 67 | __VEC_SCOPE__ | 67 | __VEC_SCOPE__ |
| 68 | { | 68 | { |
| 69 | - MicroAPI::Duplicate(vregInputOne, (float)1.0); | 69 | + Reg::Duplicate(vregInputOne, (float)1.0); |
| 70 | - MicroAPI::Duplicate(vregInputZero, (float)0.0); | 70 | + Reg::Duplicate(vregInputZero, (float)0.0); |
| 71 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 71 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 72 | - mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count); | 72 | + mask = Reg::UpdateMask<T, Reg::RegTraitNumOne>(count); |
| 73 | - MicroAPI::Duplicate(vregInputPX, BETAN); | 73 | + Reg::Duplicate(vregInputPX, BETAN); |
| 74 | // OpCopyIn | 74 | // OpCopyIn |
| 75 | - MicroAPI::LoadAlign(vregInputDy, (__ubuf__ T*)(src0Addr + loopIdx * vlSize)); | 75 | + Reg::LoadAlign(vregInputDy, (__ubuf__ T*)(src0Addr + loopIdx * vlSize)); |
| 76 | - MicroAPI::LoadAlign(vregInputX, (__ubuf__ T*)(src1Addr + loopIdx * vlSize)); | 76 | + Reg::LoadAlign(vregInputX, (__ubuf__ T*)(src1Addr + loopIdx * vlSize)); |
| 77 | // compute | 77 | // compute |
| 78 | - MicroAPI::Mul(vregInputXSqr, vregInputX, vregInputX, mask); | 78 | + Reg::Mul(vregInputXSqr, vregInputX, vregInputX, mask); |
| 79 | - MicroAPI::Axpy(vregInputPX, vregInputXSqr, AN, mask); | 79 | + Reg::Axpy(vregInputPX, vregInputXSqr, AN, mask); |
| 80 | - MicroAPI::Mul(vregInputPX, vregInputPX, vregInputX, mask); | 80 | + Reg::Mul(vregInputPX, vregInputPX, vregInputX, mask); |
| 81 | - MicroAPI::Exp(vregInputPX, vregInputPX, mask); | 81 | + Reg::Exp(vregInputPX, vregInputPX, mask); |
| 82 | 82 | ||
| 83 | - MicroAPI::Duplicate(vregInputRes0, BETA); | 83 | + Reg::Duplicate(vregInputRes0, BETA); |
| 84 | - MicroAPI::Axpy(vregInputRes0, vregInputXSqr, A3, mask); | 84 | + Reg::Axpy(vregInputRes0, vregInputXSqr, A3, mask); |
| 85 | - MicroAPI::Mul(vregInputRes0, vregInputRes0, vregInputX, mask); | 85 | + Reg::Mul(vregInputRes0, vregInputRes0, vregInputX, mask); |
| 86 | 86 | ||
| 87 | - MicroAPI::Adds(vregInputT, vregInputPX, (float)1.0, mask); | 87 | + Reg::Adds(vregInputT, vregInputPX, (float)1.0, mask); |
| 88 | - MicroAPI::Div(vregInputDiv, vregInputOne, vregInputT, mask); | 88 | + Reg::Div(vregInputDiv, vregInputOne, vregInputT, mask); |
| 89 | 89 | ||
| 90 | - MicroAPI::Mul(vregInputResp, vregInputPX, vregInputDiv, mask); | 90 | + Reg::Mul(vregInputResp, vregInputPX, vregInputDiv, mask); |
| 91 | - MicroAPI::Mul(vregInputResp, vregInputResp, vregInputRes0, mask); | 91 | + Reg::Mul(vregInputResp, vregInputResp, vregInputRes0, mask); |
| 92 | - MicroAPI::Mul(vregInputResp, vregInputResp, vregInputDiv, mask); | 92 | + Reg::Mul(vregInputResp, vregInputResp, vregInputDiv, mask); |
| 93 | - MicroAPI::Compare<T, CMPMODE::EQ>(cmpMask, vregInputResp, vregInputResp, mask); | 93 | + Reg::Compare<T, CMPMODE::EQ>(cmpMask, vregInputResp, vregInputResp, mask); |
| 94 | - MicroAPI::Select<T>(vregSelect, vregInputResp, vregInputZero, cmpMask); | 94 | + Reg::Select<T>(vregSelect, vregInputResp, vregInputZero, cmpMask); |
| 95 | - MicroAPI::Add(vregInputResp, vregSelect, vregInputDiv, mask); | 95 | + Reg::Add(vregInputResp, vregSelect, vregInputDiv, mask); |
| 96 | - MicroAPI::Mul(vregOutput, vregInputDy, vregInputResp, mask); | 96 | + Reg::Mul(vregOutput, vregInputDy, vregInputResp, mask); |
| 97 | 97 | ||
| 98 | // OpCopyOut | 98 | // OpCopyOut |
| 99 | - MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); | 99 | + Reg::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); |
| 100 | } | 100 | } |
| 101 | } | 101 | } |
| 102 | } | 102 | } |
| @@ -56,41 +56,41 @@ struct GeluGradV2ErfPost : public Vec::ElemwiseTernaryOP<T, T, T, T> { | |||
| 56 | __ubuf__ T* src2Addr = (__ubuf__ T*)src2.GetPhyAddr(); | 56 | __ubuf__ T* src2Addr = (__ubuf__ T*)src2.GetPhyAddr(); |
| 57 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); | 57 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); |
| 58 | 58 | ||
| 59 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInput0; | 59 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInput0; |
| 60 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInput1; | 60 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInput1; |
| 61 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInput2; | 61 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInput2; |
| 62 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregCdfMuls; | 62 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregCdfMuls; |
| 63 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregCdfRes; | 63 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregCdfRes; |
| 64 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregPdfMul; | 64 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregPdfMul; |
| 65 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregPdfMuls; | 65 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregPdfMuls; |
| 66 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregPdfExp; | 66 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregPdfExp; |
| 67 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregPdfRes; | 67 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregPdfRes; |
| 68 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregMulRes; | 68 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregMulRes; |
| 69 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregAddRes; | 69 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregAddRes; |
| 70 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregOutput; | 70 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregOutput; |
| 71 | - MicroAPI::MaskReg mask; | 71 | + Reg::MaskReg mask; |
| 72 | if constexpr (std::is_same_v<T, float>) { | 72 | if constexpr (std::is_same_v<T, float>) { |
| 73 | __VEC_SCOPE__ | 73 | __VEC_SCOPE__ |
| 74 | { | 74 | { |
| 75 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 75 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 76 | - mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count); | 76 | + mask = Reg::UpdateMask<T, Reg::RegTraitNumOne>(count); |
| 77 | // OpCopyIn | 77 | // OpCopyIn |
| 78 | - MicroAPI::LoadAlign(vregInput0, (__ubuf__ T*)(src0Addr + loopIdx * vlSize)); | 78 | + Reg::LoadAlign(vregInput0, (__ubuf__ T*)(src0Addr + loopIdx * vlSize)); |
| 79 | - MicroAPI::LoadAlign(vregInput1, (__ubuf__ T*)(src1Addr + loopIdx * vlSize)); | 79 | + Reg::LoadAlign(vregInput1, (__ubuf__ T*)(src1Addr + loopIdx * vlSize)); |
| 80 | - MicroAPI::LoadAlign(vregInput2, (__ubuf__ T*)(src2Addr + loopIdx * vlSize)); | 80 | + Reg::LoadAlign(vregInput2, (__ubuf__ T*)(src2Addr + loopIdx * vlSize)); |
| 81 | - MicroAPI::Muls(vregCdfMuls, vregInput2, (float)0.5, mask); | 81 | + Reg::Muls(vregCdfMuls, vregInput2, (float)0.5, mask); |
| 82 | - MicroAPI::Adds(vregCdfRes, vregCdfMuls, (float)0.5, mask); | 82 | + Reg::Adds(vregCdfRes, vregCdfMuls, (float)0.5, mask); |
| 83 | 83 | ||
| 84 | - MicroAPI::Mul(vregPdfMul, vregInput1, vregInput1, mask); | 84 | + Reg::Mul(vregPdfMul, vregInput1, vregInput1, mask); |
| 85 | - MicroAPI::Muls(vregPdfMuls, vregPdfMul, (float)-0.5, mask); | 85 | + Reg::Muls(vregPdfMuls, vregPdfMul, (float)-0.5, mask); |
| 86 | - MicroAPI::Exp(vregPdfExp, vregPdfMuls, mask); | 86 | + Reg::Exp(vregPdfExp, vregPdfMuls, mask); |
| 87 | - MicroAPI::Muls(vregPdfRes, vregPdfExp, (float)0.3989422804, mask); // 1 / sqrt(2 * pi) | 87 | + Reg::Muls(vregPdfRes, vregPdfExp, (float)0.3989422804, mask); // 1 / sqrt(2 * pi) |
| 88 | 88 | ||
| 89 | - MicroAPI::Mul(vregMulRes, vregPdfRes, vregInput1, mask); | 89 | + Reg::Mul(vregMulRes, vregPdfRes, vregInput1, mask); |
| 90 | - MicroAPI::Add(vregAddRes, vregCdfRes, vregMulRes, mask); | 90 | + Reg::Add(vregAddRes, vregCdfRes, vregMulRes, mask); |
| 91 | - MicroAPI::Mul(vregOutput, vregAddRes, vregInput0, mask); | 91 | + Reg::Mul(vregOutput, vregAddRes, vregInput0, mask); |
| 92 | // OpCopyOut | 92 | // OpCopyOut |
| 93 | - MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); | 93 | + Reg::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); |
| 94 | } | 94 | } |
| 95 | } | 95 | } |
| 96 | } | 96 | } |
| @@ -112,54 +112,54 @@ struct GeluGradV2TanhCustom : public Vec::ElemwiseBinaryOP<T, T, T> { | |||
| 112 | __ubuf__ T* src1Addr = (__ubuf__ T*)src1.GetPhyAddr(); | 112 | __ubuf__ T* src1Addr = (__ubuf__ T*)src1.GetPhyAddr(); |
| 113 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); | 113 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); |
| 114 | 114 | ||
| 115 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputDy; | 115 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputDy; |
| 116 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputX; | 116 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputX; |
| 117 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputXSqr; | 117 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputXSqr; |
| 118 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputPX; | 118 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputPX; |
| 119 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputRes0; | 119 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputRes0; |
| 120 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputT; | 120 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputT; |
| 121 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputDiv; | 121 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputDiv; |
| 122 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputOne; | 122 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputOne; |
| 123 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputZero; | 123 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputZero; |
| 124 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputResp; | 124 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputResp; |
| 125 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregSelect; | 125 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregSelect; |
| 126 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregOutput; | 126 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregOutput; |
| 127 | - MicroAPI::MaskReg mask, cmpMask; | 127 | + Reg::MaskReg mask, cmpMask; |
| 128 | 128 | ||
| 129 | if constexpr (std::is_same_v<T, float>) { | 129 | if constexpr (std::is_same_v<T, float>) { |
| 130 | __VEC_SCOPE__ | 130 | __VEC_SCOPE__ |
| 131 | { | 131 | { |
| 132 | - MicroAPI::Duplicate(vregInputOne, (float)1.0); | 132 | + Reg::Duplicate(vregInputOne, (float)1.0); |
| 133 | - MicroAPI::Duplicate(vregInputZero, (float)0.0); | 133 | + Reg::Duplicate(vregInputZero, (float)0.0); |
| 134 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 134 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 135 | - mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count); | 135 | + mask = Reg::UpdateMask<T, Reg::RegTraitNumOne>(count); |
| 136 | - MicroAPI::Duplicate(vregInputPX, BETAN); | 136 | + Reg::Duplicate(vregInputPX, BETAN); |
| 137 | // OpCopyIn | 137 | // OpCopyIn |
| 138 | - MicroAPI::LoadAlign(vregInputDy, (__ubuf__ T*)(src0Addr + loopIdx * vlSize)); | 138 | + Reg::LoadAlign(vregInputDy, (__ubuf__ T*)(src0Addr + loopIdx * vlSize)); |
| 139 | - MicroAPI::LoadAlign(vregInputX, (__ubuf__ T*)(src1Addr + loopIdx * vlSize)); | 139 | + Reg::LoadAlign(vregInputX, (__ubuf__ T*)(src1Addr + loopIdx * vlSize)); |
| 140 | // compute | 140 | // compute |
| 141 | - MicroAPI::Mul(vregInputXSqr, vregInputX, vregInputX, mask); | 141 | + Reg::Mul(vregInputXSqr, vregInputX, vregInputX, mask); |
| 142 | - MicroAPI::Axpy(vregInputPX, vregInputXSqr, AN, mask); | 142 | + Reg::Axpy(vregInputPX, vregInputXSqr, AN, mask); |
| 143 | - MicroAPI::Mul(vregInputPX, vregInputPX, vregInputX, mask); | 143 | + Reg::Mul(vregInputPX, vregInputPX, vregInputX, mask); |
| 144 | - MicroAPI::Exp(vregInputPX, vregInputPX, mask); | 144 | + Reg::Exp(vregInputPX, vregInputPX, mask); |
| 145 | 145 | ||
| 146 | - MicroAPI::Duplicate(vregInputRes0, BETA); | 146 | + Reg::Duplicate(vregInputRes0, BETA); |
| 147 | - MicroAPI::Axpy(vregInputRes0, vregInputXSqr, A3, mask); | 147 | + Reg::Axpy(vregInputRes0, vregInputXSqr, A3, mask); |
| 148 | - MicroAPI::Mul(vregInputRes0, vregInputRes0, vregInputX, mask); | 148 | + Reg::Mul(vregInputRes0, vregInputRes0, vregInputX, mask); |
| 149 | 149 | ||
| 150 | - MicroAPI::Adds(vregInputT, vregInputPX, (float)1.0, mask); | 150 | + Reg::Adds(vregInputT, vregInputPX, (float)1.0, mask); |
| 151 | - MicroAPI::Div(vregInputDiv, vregInputOne, vregInputT, mask); | 151 | + Reg::Div(vregInputDiv, vregInputOne, vregInputT, mask); |
| 152 | 152 | ||
| 153 | - MicroAPI::Mul(vregInputResp, vregInputPX, vregInputDiv, mask); | 153 | + Reg::Mul(vregInputResp, vregInputPX, vregInputDiv, mask); |
| 154 | - MicroAPI::Mul(vregInputResp, vregInputResp, vregInputRes0, mask); | 154 | + Reg::Mul(vregInputResp, vregInputResp, vregInputRes0, mask); |
| 155 | - MicroAPI::Mul(vregInputResp, vregInputResp, vregInputDiv, mask); | 155 | + Reg::Mul(vregInputResp, vregInputResp, vregInputDiv, mask); |
| 156 | - MicroAPI::Compare<T, CMPMODE::EQ>(cmpMask, vregInputResp, vregInputResp, mask); | 156 | + Reg::Compare<T, CMPMODE::EQ>(cmpMask, vregInputResp, vregInputResp, mask); |
| 157 | - MicroAPI::Select<T>(vregSelect, vregInputResp, vregInputZero, cmpMask); | 157 | + Reg::Select<T>(vregSelect, vregInputResp, vregInputZero, cmpMask); |
| 158 | - MicroAPI::Add(vregInputResp, vregSelect, vregInputDiv, mask); | 158 | + Reg::Add(vregInputResp, vregSelect, vregInputDiv, mask); |
| 159 | - MicroAPI::Mul(vregOutput, vregInputDy, vregInputResp, mask); | 159 | + Reg::Mul(vregOutput, vregInputDy, vregInputResp, mask); |
| 160 | 160 | ||
| 161 | // OpCopyOut | 161 | // OpCopyOut |
| 162 | - MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); | 162 | + Reg::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); |
| 163 | } | 163 | } |
| 164 | } | 164 | } |
| 165 | } | 165 | } |
| @@ -286,63 +286,61 @@ __aicore__ inline void GeluDynamicQuant<T1, T2>::ComputeDynamicQuantRegbase(Loca | |||
| 286 | 286 | ||
| 287 | __VEC_SCOPE__ | 287 | __VEC_SCOPE__ |
| 288 | { | 288 | { |
| 289 | - AscendC::MicroAPI::RegTensor<float> vregInput; | 289 | + AscendC::Reg::RegTensor<float> vregInput; |
| 290 | - AscendC::MicroAPI::RegTensor<float> vregSmoothScale; | 290 | + AscendC::Reg::RegTensor<float> vregSmoothScale; |
| 291 | - AscendC::MicroAPI::RegTensor<float> vregAbs; | 291 | + AscendC::Reg::RegTensor<float> vregAbs; |
| 292 | - AscendC::MicroAPI::RegTensor<float> vregReduceMax; | 292 | + AscendC::Reg::RegTensor<float> vregReduceMax; |
| 293 | - AscendC::MicroAPI::RegTensor<float> vregOutScale; | 293 | + AscendC::Reg::RegTensor<float> vregOutScale; |
| 294 | - AscendC::MicroAPI::RegTensor<float> vregQuantRes; | 294 | + AscendC::Reg::RegTensor<float> vregQuantRes; |
| 295 | - AscendC::MicroAPI::RegTensor<float> vregMax; | 295 | + AscendC::Reg::RegTensor<float> vregMax; |
| 296 | - AscendC::MicroAPI::RegTensor<half> vregHalf; | 296 | + AscendC::Reg::RegTensor<half> vregHalf; |
| 297 | - AscendC::MicroAPI::RegTensor<dstType> vregY; | 297 | + AscendC::Reg::RegTensor<dstType> vregY; |
| 298 | - AscendC::MicroAPI::MaskReg preg0; | 298 | + AscendC::Reg::MaskReg preg0; |
| 299 | - AscendC::MicroAPI::MaskReg preg1 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 299 | + AscendC::Reg::MaskReg preg1 = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 300 | - AscendC::MicroAPI::MaskReg preg2; | 300 | + AscendC::Reg::MaskReg preg2; |
| 301 | 301 | ||
| 302 | for (uint16_t i = 0; i < rowCountLocal; i++) { | 302 | for (uint16_t i = 0; i < rowCountLocal; i++) { |
| 303 | - AscendC::MicroAPI::Duplicate(vregMax, 0.0); | 303 | + AscendC::Reg::Duplicate(vregMax, 0.0); |
| 304 | uint32_t sreg0 = endAxisActualAlignLen_; | 304 | uint32_t sreg0 = endAxisActualAlignLen_; |
| 305 | for (uint16_t j = 0; j < loopNum; j++) { | 305 | for (uint16_t j = 0; j < loopNum; j++) { |
| 306 | - preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0); | 306 | + preg0 = AscendC::Reg::UpdateMask<float>(sreg0); |
| 307 | - AscendC::MicroAPI::LoadAlign(vregInput, xAddr + i * endAxisActualAlignLen_ + j * vl); | 307 | + AscendC::Reg::LoadAlign(vregInput, xAddr + i * endAxisActualAlignLen_ + j * vl); |
| 308 | // compute smoothscale | 308 | // compute smoothscale |
| 309 | - AscendC::MicroAPI::LoadAlign(vregSmoothScale, smoothScaleAddr + j * vl); | 309 | + AscendC::Reg::LoadAlign(vregSmoothScale, smoothScaleAddr + j * vl); |
| 310 | - AscendC::MicroAPI::Mul(vregInput, vregInput, vregSmoothScale, preg0); | 310 | + AscendC::Reg::Mul(vregInput, vregInput, vregSmoothScale, preg0); |
| 311 | - AscendC::MicroAPI::Abs(vregAbs, vregInput, preg0); | 311 | + AscendC::Reg::Abs(vregAbs, vregInput, preg0); |
| 312 | - AscendC::MicroAPI::Max(vregMax, vregAbs, vregMax, preg1); | 312 | + AscendC::Reg::Max(vregMax, vregAbs, vregMax, preg1); |
| 313 | } | 313 | } |
| 314 | { | 314 | { |
| 315 | - AscendC::MicroAPI::Reduce<AscendC::MicroAPI::ReduceType::MAX>(vregReduceMax, vregMax, preg1); | 315 | + AscendC::Reg::Reduce<AscendC::Reg::ReduceType::MAX>(vregReduceMax, vregMax, preg1); |
| 316 | - AscendC::MicroAPI::Muls(vregReduceMax, vregReduceMax, maxValue_, preg1); | 316 | + AscendC::Reg::Muls(vregReduceMax, vregReduceMax, maxValue_, preg1); |
| 317 | - AscendC::MicroAPI::Duplicate(vregOutScale, vregReduceMax, preg1); | 317 | + AscendC::Reg::Duplicate(vregOutScale, vregReduceMax, preg1); |
| 318 | - AscendC::MicroAPI::StoreAlign(scaleOutAddr + i * FP32_BLOCK_NUM, vregOutScale, preg1); | 318 | + AscendC::Reg::StoreAlign(scaleOutAddr + i * FP32_BLOCK_NUM, vregOutScale, preg1); |
| 319 | } | 319 | } |
| 320 | uint32_t sreg1 = endAxisLen_; | 320 | uint32_t sreg1 = endAxisLen_; |
| 321 | - AscendC::MicroAPI::LocalMemBar<AscendC::MicroAPI::MemType::VEC_STORE, | 321 | + AscendC::Reg::LocalMemBar<AscendC::Reg::MemType::VEC_STORE, AscendC::Reg::MemType::VEC_LOAD>(); |
| 322 | - AscendC::MicroAPI::MemType::VEC_LOAD>(); | ||
| 323 | for (uint16_t j = 0; j < loopNum; j++) { | 322 | for (uint16_t j = 0; j < loopNum; j++) { |
| 324 | auto yOutAddr = yAddr + i * endAxisLenAlignTo8_ + j * vl; | 323 | auto yOutAddr = yAddr + i * endAxisLenAlignTo8_ + j * vl; |
| 325 | - preg2 = AscendC::MicroAPI::UpdateMask<float>(sreg1); | 324 | + preg2 = AscendC::Reg::UpdateMask<float>(sreg1); |
| 326 | - AscendC::MicroAPI::LoadAlign(vregInput, xAddr + i * endAxisActualAlignLen_ + j * vl); | 325 | + AscendC::Reg::LoadAlign(vregInput, xAddr + i * endAxisActualAlignLen_ + j * vl); |
| 327 | - AscendC::MicroAPI::LoadAlign(vregSmoothScale, smoothScaleAddr + j * vl); | 326 | + AscendC::Reg::LoadAlign(vregSmoothScale, smoothScaleAddr + j * vl); |
| 328 | - AscendC::MicroAPI::Mul(vregInput, vregInput, vregSmoothScale, preg2); | 327 | + AscendC::Reg::Mul(vregInput, vregInput, vregSmoothScale, preg2); |
| 329 | - AscendC::MicroAPI::Div(vregQuantRes, vregInput, vregOutScale, preg2); | 328 | + AscendC::Reg::Div(vregQuantRes, vregInput, vregOutScale, preg2); |
| 330 | 329 | ||
| 331 | if constexpr (IsSameType<dstType, int8_t>::value) { | 330 | if constexpr (IsSameType<dstType, int8_t>::value) { |
| 332 | - AscendC::MicroAPI::Cast<half, float, castTraitF32ToF16>(vregHalf, vregQuantRes, preg2); | 331 | + AscendC::Reg::Cast<half, float, castTraitF32ToF16>(vregHalf, vregQuantRes, preg2); |
| 333 | - AscendC::MicroAPI::Cast<dstType, half, castTraitF16ToI8Rint>(vregY, vregHalf, preg2); | 332 | + AscendC::Reg::Cast<dstType, half, castTraitF16ToI8Rint>(vregY, vregHalf, preg2); |
| 334 | } else if constexpr (IsSameType<dstType, fp8_e4m3fn_t>::value || | 333 | } else if constexpr (IsSameType<dstType, fp8_e4m3fn_t>::value || |
| 335 | IsSameType<dstType, fp8_e5m2_t>::value) { | 334 | IsSameType<dstType, fp8_e5m2_t>::value) { |
| 336 | - AscendC::MicroAPI::Cast<dstType, float, castTraitF32ToF8>(vregY, vregQuantRes, preg2); | 335 | + AscendC::Reg::Cast<dstType, float, castTraitF32ToF8>(vregY, vregQuantRes, preg2); |
| 337 | } else if constexpr (IsSameType<dstType, hifloat8_t>::value && | 336 | } else if constexpr (IsSameType<dstType, hifloat8_t>::value && |
| 338 | roundMode == AscendC::RoundMode::CAST_HYBRID) { | 337 | roundMode == AscendC::RoundMode::CAST_HYBRID) { |
| 339 | - AscendC::MicroAPI::Cast<dstType, float, castTraitF32ToH8Hybrid>(vregY, vregQuantRes, preg2); | 338 | + AscendC::Reg::Cast<dstType, float, castTraitF32ToH8Hybrid>(vregY, vregQuantRes, preg2); |
| 340 | } else if constexpr (IsSameType<dstType, hifloat8_t>::value && | 339 | } else if constexpr (IsSameType<dstType, hifloat8_t>::value && |
| 341 | roundMode == AscendC::RoundMode::CAST_ROUND) { | 340 | roundMode == AscendC::RoundMode::CAST_ROUND) { |
| 342 | - AscendC::MicroAPI::Cast<dstType, float, castTraitF32ToH8Round>(vregY, vregQuantRes, preg2); | 341 | + AscendC::Reg::Cast<dstType, float, castTraitF32ToH8Round>(vregY, vregQuantRes, preg2); |
| 343 | } | 342 | } |
| 344 | - AscendC::MicroAPI::StoreAlign<dstType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(yOutAddr, vregY, | 343 | + AscendC::Reg::StoreAlign<dstType, AscendC::Reg::StoreDist::DIST_PACK4_B32>(yOutAddr, vregY, preg2); |
| 345 | - preg2); | ||
| 346 | } | 344 | } |
| 347 | } | 345 | } |
| 348 | } | 346 | } |
| @@ -363,32 +363,32 @@ __aicore__ inline void GeluDynamicQuantWorkspace<T1, T2>::ComputeDynamicQuantReg | |||
| 363 | 363 | ||
| 364 | __VEC_SCOPE__ | 364 | __VEC_SCOPE__ |
| 365 | { | 365 | { |
| 366 | - AscendC::MicroAPI::RegTensor<float> geluResReg; | 366 | + AscendC::Reg::RegTensor<float> geluResReg; |
| 367 | - AscendC::MicroAPI::RegTensor<float> scaleReg; | 367 | + AscendC::Reg::RegTensor<float> scaleReg; |
| 368 | - AscendC::MicroAPI::RegTensor<float> absReg; | 368 | + AscendC::Reg::RegTensor<float> absReg; |
| 369 | - AscendC::MicroAPI::RegTensor<float> reduceMaxReg; | 369 | + AscendC::Reg::RegTensor<float> reduceMaxReg; |
| 370 | - AscendC::MicroAPI::RegTensor<float> vregMax; | 370 | + AscendC::Reg::RegTensor<float> vregMax; |
| 371 | - AscendC::MicroAPI::RegTensor<float> maxTemReg; | 371 | + AscendC::Reg::RegTensor<float> maxTemReg; |
| 372 | - AscendC::MicroAPI::MaskReg preg0; | 372 | + AscendC::Reg::MaskReg preg0; |
| 373 | - AscendC::MicroAPI::MaskReg preg1 = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 373 | + AscendC::Reg::MaskReg preg1 = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 374 | - AscendC::MicroAPI::UnalignRegForLoad ureg0; | 374 | + AscendC::Reg::UnalignRegForLoad ureg0; |
| 375 | 375 | ||
| 376 | - AscendC::MicroAPI::Duplicate(vregMax, 0.0); | 376 | + AscendC::Reg::Duplicate(vregMax, 0.0); |
| 377 | uint32_t sreg0 = calCount; | 377 | uint32_t sreg0 = calCount; |
| 378 | for (uint16_t i = 0; i < loopNum; i++) { | 378 | for (uint16_t i = 0; i < loopNum; i++) { |
| 379 | - preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0); | 379 | + preg0 = AscendC::Reg::UpdateMask<float>(sreg0); |
| 380 | - AscendC::MicroAPI::LoadAlign(geluResReg, xAddr + i * vl); | 380 | + AscendC::Reg::LoadAlign(geluResReg, xAddr + i * vl); |
| 381 | // compute smoothscale | 381 | // compute smoothscale |
| 382 | - AscendC::MicroAPI::LoadAlign(scaleReg, smoothScaleAddr + i * vl); | 382 | + AscendC::Reg::LoadAlign(scaleReg, smoothScaleAddr + i * vl); |
| 383 | - AscendC::MicroAPI::Mul(geluResReg, geluResReg, scaleReg, preg0); | 383 | + AscendC::Reg::Mul(geluResReg, geluResReg, scaleReg, preg0); |
| 384 | - AscendC::MicroAPI::StoreAlign(workspaceLocalAddr + i * vl, geluResReg, preg0); | 384 | + AscendC::Reg::StoreAlign(workspaceLocalAddr + i * vl, geluResReg, preg0); |
| 385 | - AscendC::MicroAPI::Abs(absReg, geluResReg, preg0); | 385 | + AscendC::Reg::Abs(absReg, geluResReg, preg0); |
| 386 | - AscendC::MicroAPI::Max(vregMax, absReg, vregMax, preg1); | 386 | + AscendC::Reg::Max(vregMax, absReg, vregMax, preg1); |
| 387 | } | 387 | } |
| 388 | - AscendC::MicroAPI::Reduce<AscendC::MicroAPI::ReduceType::MAX>(reduceMaxReg, vregMax, preg1); | 388 | + AscendC::Reg::Reduce<AscendC::Reg::ReduceType::MAX>(reduceMaxReg, vregMax, preg1); |
| 389 | - AscendC::MicroAPI::LoadAlign(maxTemReg, maxValueAddr); | 389 | + AscendC::Reg::LoadAlign(maxTemReg, maxValueAddr); |
| 390 | - AscendC::MicroAPI::Max(maxTemReg, reduceMaxReg, maxTemReg, preg1); | 390 | + AscendC::Reg::Max(maxTemReg, reduceMaxReg, maxTemReg, preg1); |
| 391 | - AscendC::MicroAPI::StoreAlign(maxValueAddr, maxTemReg, preg1); | 391 | + AscendC::Reg::StoreAlign(maxValueAddr, maxTemReg, preg1); |
| 392 | } | 392 | } |
| 393 | } | 393 | } |
| 394 | 394 | ||
| @@ -61,9 +61,9 @@ constexpr int8_t FLOAT_OVERFLOW_MODE_CTRL = 60; | |||
| 61 | constexpr int64_t FLOAT_OVERFLOW_MODE_SATURATE = 0; | 61 | constexpr int64_t FLOAT_OVERFLOW_MODE_SATURATE = 0; |
| 62 | 62 | ||
| 63 | 63 | ||
| 64 | -constexpr static AscendC::MicroAPI::CastTrait castTrait0 = { | 64 | +constexpr static AscendC::Reg::CastTrait castTrait0 = {AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::UNKNOWN, |
| 65 | - AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::UNKNOWN, AscendC::MicroAPI::MaskMergeMode::ZEROING, | 65 | + AscendC::Reg::MaskMergeMode::ZEROING, |
| 66 | - AscendC::RoundMode::UNKNOWN}; | 66 | + AscendC::RoundMode::UNKNOWN}; |
| 67 | 67 | ||
| 68 | class GeluQuantBase { | 68 | class GeluQuantBase { |
| 69 | public: | 69 | public: |
| @@ -110,23 +110,23 @@ public: | |||
| 110 | uint32_t dstType_; | 110 | uint32_t dstType_; |
| 111 | AscendC::RoundMode roundMode_; | 111 | AscendC::RoundMode roundMode_; |
| 112 | 112 | ||
| 113 | - constexpr static AscendC::MicroAPI::CastTrait castTraitF32ToF16 = { | 113 | + constexpr static AscendC::Reg::CastTrait castTraitF32ToF16 = { |
| 114 | - AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::NO_SAT, | 114 | + AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::NO_SAT, AscendC::Reg::MaskMergeMode::ZEROING, |
| 115 | - AscendC::MicroAPI::MaskMergeMode::ZEROING, AscendC::RoundMode::CAST_ODD}; | 115 | + AscendC::RoundMode::CAST_ODD}; |
| 116 | - constexpr static AscendC::MicroAPI::CastTrait castTraitF16ToI8Rint = { | 116 | + constexpr static AscendC::Reg::CastTrait castTraitF16ToI8Rint = { |
| 117 | - AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::NO_SAT, | 117 | + AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::NO_SAT, AscendC::Reg::MaskMergeMode::ZEROING, |
| 118 | - AscendC::MicroAPI::MaskMergeMode::ZEROING, AscendC::RoundMode::CAST_RINT}; | ||
| 119 | - constexpr static AscendC::MicroAPI::CastTrait castTraitF16ToI8Round = { | ||
| 120 | - AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::NO_SAT, | ||
| 121 | - AscendC::MicroAPI::MaskMergeMode::ZEROING, AscendC::RoundMode::CAST_ROUND}; | ||
| 122 | - constexpr static AscendC::MicroAPI::CastTrait castTraitF32ToF8 = { | ||
| 123 | - AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::SAT, AscendC::MicroAPI::MaskMergeMode::ZEROING, | ||
| 124 | AscendC::RoundMode::CAST_RINT}; | 118 | AscendC::RoundMode::CAST_RINT}; |
| 125 | - constexpr static AscendC::MicroAPI::CastTrait castTraitF32ToH8Round = { | 119 | + constexpr static AscendC::Reg::CastTrait castTraitF16ToI8Round = { |
| 126 | - AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::SAT, AscendC::MicroAPI::MaskMergeMode::ZEROING, | 120 | + AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::NO_SAT, AscendC::Reg::MaskMergeMode::ZEROING, |
| 127 | AscendC::RoundMode::CAST_ROUND}; | 121 | AscendC::RoundMode::CAST_ROUND}; |
| 128 | - constexpr static AscendC::MicroAPI::CastTrait castTraitF32ToH8Hybrid = { | 122 | + constexpr static AscendC::Reg::CastTrait castTraitF32ToF8 = { |
| 129 | - AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::SAT, AscendC::MicroAPI::MaskMergeMode::ZEROING, | 123 | + AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::SAT, AscendC::Reg::MaskMergeMode::ZEROING, |
| 124 | + AscendC::RoundMode::CAST_RINT}; | ||
| 125 | + constexpr static AscendC::Reg::CastTrait castTraitF32ToH8Round = { | ||
| 126 | + AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::SAT, AscendC::Reg::MaskMergeMode::ZEROING, | ||
| 127 | + AscendC::RoundMode::CAST_ROUND}; | ||
| 128 | + constexpr static AscendC::Reg::CastTrait castTraitF32ToH8Hybrid = { | ||
| 129 | + AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::SAT, AscendC::Reg::MaskMergeMode::ZEROING, | ||
| 130 | AscendC::RoundMode::CAST_HYBRID}; | 130 | AscendC::RoundMode::CAST_HYBRID}; |
| 131 | }; | 131 | }; |
| 132 | 132 | ||
| @@ -157,24 +157,24 @@ __aicore__ inline void GeluQuantBase::GeluV2ErfPost(LocalTensor<float>& dst, Loc | |||
| 157 | __ubuf__ float* src2Addr = (__ubuf__ float*)src2.GetPhyAddr(); | 157 | __ubuf__ float* src2Addr = (__ubuf__ float*)src2.GetPhyAddr(); |
| 158 | __ubuf__ float* dstAddr = (__ubuf__ float*)dst.GetPhyAddr(); | 158 | __ubuf__ float* dstAddr = (__ubuf__ float*)dst.GetPhyAddr(); |
| 159 | 159 | ||
| 160 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregInput1; | 160 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregInput1; |
| 161 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregInput2; | 161 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregInput2; |
| 162 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregInputAdds; | 162 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregInputAdds; |
| 163 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregInputMuls; | 163 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregInputMuls; |
| 164 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregOutput; | 164 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregOutput; |
| 165 | - MicroAPI::MaskReg mask; | 165 | + Reg::MaskReg mask; |
| 166 | __VEC_SCOPE__ | 166 | __VEC_SCOPE__ |
| 167 | { | 167 | { |
| 168 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 168 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 169 | - mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count); | 169 | + mask = Reg::UpdateMask<float, Reg::RegTraitNumOne>(count); |
| 170 | // OpCopyIn | 170 | // OpCopyIn |
| 171 | - MicroAPI::LoadAlign(vregInput1, (__ubuf__ float*)(src1Addr + loopIdx * vlSize)); | 171 | + Reg::LoadAlign(vregInput1, (__ubuf__ float*)(src1Addr + loopIdx * vlSize)); |
| 172 | - MicroAPI::LoadAlign(vregInput2, (__ubuf__ float*)(src2Addr + loopIdx * vlSize)); | 172 | + Reg::LoadAlign(vregInput2, (__ubuf__ float*)(src2Addr + loopIdx * vlSize)); |
| 173 | - MicroAPI::Adds(vregInputAdds, vregInput2, (float)1.0, mask); | 173 | + Reg::Adds(vregInputAdds, vregInput2, (float)1.0, mask); |
| 174 | - MicroAPI::Muls(vregInputMuls, vregInput1, (float)0.5, mask); | 174 | + Reg::Muls(vregInputMuls, vregInput1, (float)0.5, mask); |
| 175 | - MicroAPI::Mul(vregOutput, vregInputAdds, vregInputMuls, mask); | 175 | + Reg::Mul(vregOutput, vregInputAdds, vregInputMuls, mask); |
| 176 | // OpCopyOut | 176 | // OpCopyOut |
| 177 | - MicroAPI::StoreAlign((__ubuf__ float*)(dstAddr + loopIdx * vlSize), vregOutput, mask); | 177 | + Reg::StoreAlign((__ubuf__ float*)(dstAddr + loopIdx * vlSize), vregOutput, mask); |
| 178 | } | 178 | } |
| 179 | } | 179 | } |
| 180 | 180 | ||
| @@ -192,50 +192,50 @@ __aicore__ inline void GeluQuantBase::ComputeGeluTanh(const LocalTensor<T>& src, | |||
| 192 | __ubuf__ T* srcAddr = (__ubuf__ T*)src.GetPhyAddr(); | 192 | __ubuf__ T* srcAddr = (__ubuf__ T*)src.GetPhyAddr(); |
| 193 | __ubuf__ float* dstAddr = (__ubuf__ float*)dst.GetPhyAddr(); | 193 | __ubuf__ float* dstAddr = (__ubuf__ float*)dst.GetPhyAddr(); |
| 194 | 194 | ||
| 195 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregInput; | 195 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregInput; |
| 196 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregInputSqr; | 196 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregInputSqr; |
| 197 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregInputCub; | 197 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregInputCub; |
| 198 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregOutput; | 198 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregOutput; |
| 199 | - MicroAPI::MaskReg mask; | 199 | + Reg::MaskReg mask; |
| 200 | if constexpr (std::is_same_v<T, float>) { | 200 | if constexpr (std::is_same_v<T, float>) { |
| 201 | __VEC_SCOPE__ | 201 | __VEC_SCOPE__ |
| 202 | { | 202 | { |
| 203 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 203 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 204 | - mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count); | 204 | + mask = Reg::UpdateMask<float, Reg::RegTraitNumOne>(count); |
| 205 | // OpCopyIn | 205 | // OpCopyIn |
| 206 | - MicroAPI::LoadAlign(vregInput, (__ubuf__ float*)(srcAddr + loopIdx * vlSize)); | 206 | + Reg::LoadAlign(vregInput, (__ubuf__ float*)(srcAddr + loopIdx * vlSize)); |
| 207 | - MicroAPI::Mul(vregInputSqr, vregInput, vregInput, mask); | 207 | + Reg::Mul(vregInputSqr, vregInput, vregInput, mask); |
| 208 | - MicroAPI::Mul(vregInputCub, vregInputSqr, vregInput, mask); | 208 | + Reg::Mul(vregInputCub, vregInputSqr, vregInput, mask); |
| 209 | - MicroAPI::Axpy(vregInputCub, vregInput, TANH_APPROX_FACTOR, mask); | 209 | + Reg::Axpy(vregInputCub, vregInput, TANH_APPROX_FACTOR, mask); |
| 210 | - MicroAPI::Muls(vregInputCub, vregInputCub, NEG_SQRT_EIGHT_OVER_PI, mask); | 210 | + Reg::Muls(vregInputCub, vregInputCub, NEG_SQRT_EIGHT_OVER_PI, mask); |
| 211 | - MicroAPI::Exp(vregInputCub, vregInputCub, mask); | 211 | + Reg::Exp(vregInputCub, vregInputCub, mask); |
| 212 | - MicroAPI::Adds(vregInputCub, vregInputCub, 1.0f, mask); | 212 | + Reg::Adds(vregInputCub, vregInputCub, 1.0f, mask); |
| 213 | - MicroAPI::Div(vregOutput, vregInput, vregInputCub, mask); | 213 | + Reg::Div(vregOutput, vregInput, vregInputCub, mask); |
| 214 | 214 | ||
| 215 | // OpCopyOut | 215 | // OpCopyOut |
| 216 | - MicroAPI::StoreAlign((__ubuf__ float*)(dstAddr + loopIdx * vlSize), vregOutput, mask); | 216 | + Reg::StoreAlign((__ubuf__ float*)(dstAddr + loopIdx * vlSize), vregOutput, mask); |
| 217 | } | 217 | } |
| 218 | } | 218 | } |
| 219 | } else { | 219 | } else { |
| 220 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInput16; | 220 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInput16; |
| 221 | __VEC_SCOPE__ | 221 | __VEC_SCOPE__ |
| 222 | { | 222 | { |
| 223 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 223 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 224 | - mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count); | 224 | + mask = Reg::UpdateMask<float, Reg::RegTraitNumOne>(count); |
| 225 | // OpCopyIn | 225 | // OpCopyIn |
| 226 | - MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(vregInput16, | 226 | + Reg::LoadAlign<T, Reg::LoadDist::DIST_UNPACK_B16>(vregInput16, |
| 227 | - (__ubuf__ T*)(srcAddr + loopIdx * vlSize)); | 227 | + (__ubuf__ T*)(srcAddr + loopIdx * vlSize)); |
| 228 | - MicroAPI::Cast<float, T, castTrait0>(vregInput, vregInput16, mask); | 228 | + Reg::Cast<float, T, castTrait0>(vregInput, vregInput16, mask); |
| 229 | - MicroAPI::Mul(vregInputSqr, vregInput, vregInput, mask); | 229 | + Reg::Mul(vregInputSqr, vregInput, vregInput, mask); |
| 230 | - MicroAPI::Mul(vregInputCub, vregInputSqr, vregInput, mask); | 230 | + Reg::Mul(vregInputCub, vregInputSqr, vregInput, mask); |
| 231 | - MicroAPI::Axpy(vregInputCub, vregInput, TANH_APPROX_FACTOR, mask); | 231 | + Reg::Axpy(vregInputCub, vregInput, TANH_APPROX_FACTOR, mask); |
| 232 | - MicroAPI::Muls(vregInputCub, vregInputCub, NEG_SQRT_EIGHT_OVER_PI, mask); | 232 | + Reg::Muls(vregInputCub, vregInputCub, NEG_SQRT_EIGHT_OVER_PI, mask); |
| 233 | - MicroAPI::Exp(vregInputCub, vregInputCub, mask); | 233 | + Reg::Exp(vregInputCub, vregInputCub, mask); |
| 234 | - MicroAPI::Adds(vregInputCub, vregInputCub, 1.0f, mask); | 234 | + Reg::Adds(vregInputCub, vregInputCub, 1.0f, mask); |
| 235 | - MicroAPI::Div(vregOutput, vregInput, vregInputCub, mask); | 235 | + Reg::Div(vregOutput, vregInput, vregInputCub, mask); |
| 236 | 236 | ||
| 237 | // OpCopyOut | 237 | // OpCopyOut |
| 238 | - MicroAPI::StoreAlign((__ubuf__ float*)(dstAddr + loopIdx * vlSize), vregOutput, mask); | 238 | + Reg::StoreAlign((__ubuf__ float*)(dstAddr + loopIdx * vlSize), vregOutput, mask); |
| 239 | } | 239 | } |
| 240 | } | 240 | } |
| 241 | } | 241 | } |
| @@ -266,35 +266,34 @@ __aicore__ inline void GeluQuantBase::CastOutLocal(LocalTensor<float>& src, Loca | |||
| 266 | 266 | ||
| 267 | __VEC_SCOPE__ | 267 | __VEC_SCOPE__ |
| 268 | { | 268 | { |
| 269 | - AscendC::MicroAPI::RegTensor<float> vregInput; | 269 | + AscendC::Reg::RegTensor<float> vregInput; |
| 270 | - AscendC::MicroAPI::RegTensor<half> vregHalf; | 270 | + AscendC::Reg::RegTensor<half> vregHalf; |
| 271 | - AscendC::MicroAPI::RegTensor<dstType> vregY; | 271 | + AscendC::Reg::RegTensor<dstType> vregY; |
| 272 | - AscendC::MicroAPI::MaskReg preg0; | 272 | + AscendC::Reg::MaskReg preg0; |
| 273 | 273 | ||
| 274 | uint32_t sreg1 = calCount; | 274 | uint32_t sreg1 = calCount; |
| 275 | for (uint16_t i = 0; i < loopNum; i++) { | 275 | for (uint16_t i = 0; i < loopNum; i++) { |
| 276 | auto yOutAddr = yAddr + i * vl; | 276 | auto yOutAddr = yAddr + i * vl; |
| 277 | - preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg1); | 277 | + preg0 = AscendC::Reg::UpdateMask<float>(sreg1); |
| 278 | - AscendC::MicroAPI::LoadAlign(vregInput, xAddr + i * vl); | 278 | + AscendC::Reg::LoadAlign(vregInput, xAddr + i * vl); |
| 279 | 279 | ||
| 280 | if constexpr (IsSameType<dstType, int8_t>::value) { | 280 | if constexpr (IsSameType<dstType, int8_t>::value) { |
| 281 | - AscendC::MicroAPI::Cast<half, float, castTraitF32ToF16>(vregHalf, vregInput, preg0); | 281 | + AscendC::Reg::Cast<half, float, castTraitF32ToF16>(vregHalf, vregInput, preg0); |
| 282 | if constexpr (roundMode == AscendC::RoundMode::CAST_ROUND) { | 282 | if constexpr (roundMode == AscendC::RoundMode::CAST_ROUND) { |
| 283 | - AscendC::MicroAPI::Cast<dstType, half, castTraitF16ToI8Round>(vregY, vregHalf, preg0); | 283 | + AscendC::Reg::Cast<dstType, half, castTraitF16ToI8Round>(vregY, vregHalf, preg0); |
| 284 | } else if constexpr (roundMode == AscendC::RoundMode::CAST_RINT) { | 284 | } else if constexpr (roundMode == AscendC::RoundMode::CAST_RINT) { |
| 285 | - AscendC::MicroAPI::Cast<dstType, half, castTraitF16ToI8Rint>(vregY, vregHalf, preg0); | 285 | + AscendC::Reg::Cast<dstType, half, castTraitF16ToI8Rint>(vregY, vregHalf, preg0); |
| 286 | } | 286 | } |
| 287 | } else if constexpr (IsSameType<dstType, fp8_e4m3fn_t>::value || IsSameType<dstType, fp8_e5m2_t>::value) { | 287 | } else if constexpr (IsSameType<dstType, fp8_e4m3fn_t>::value || IsSameType<dstType, fp8_e5m2_t>::value) { |
| 288 | - AscendC::MicroAPI::Cast<dstType, float, castTraitF32ToF8>(vregY, vregInput, preg0); | 288 | + AscendC::Reg::Cast<dstType, float, castTraitF32ToF8>(vregY, vregInput, preg0); |
| 289 | } else if constexpr (IsSameType<dstType, hifloat8_t>::value && | 289 | } else if constexpr (IsSameType<dstType, hifloat8_t>::value && |
| 290 | roundMode == AscendC::RoundMode::CAST_HYBRID) { | 290 | roundMode == AscendC::RoundMode::CAST_HYBRID) { |
| 291 | - AscendC::MicroAPI::Cast<dstType, float, castTraitF32ToH8Hybrid>(vregY, vregInput, preg0); | 291 | + AscendC::Reg::Cast<dstType, float, castTraitF32ToH8Hybrid>(vregY, vregInput, preg0); |
| 292 | } else if constexpr (IsSameType<dstType, hifloat8_t>::value && | 292 | } else if constexpr (IsSameType<dstType, hifloat8_t>::value && |
| 293 | roundMode == AscendC::RoundMode::CAST_ROUND) { | 293 | roundMode == AscendC::RoundMode::CAST_ROUND) { |
| 294 | - AscendC::MicroAPI::Cast<dstType, float, castTraitF32ToH8Round>(vregY, vregInput, preg0); | 294 | + AscendC::Reg::Cast<dstType, float, castTraitF32ToH8Round>(vregY, vregInput, preg0); |
| 295 | } | 295 | } |
| 296 | - AscendC::MicroAPI::StoreAlign<dstType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(yOutAddr, vregY, | 296 | + AscendC::Reg::StoreAlign<dstType, AscendC::Reg::StoreDist::DIST_PACK4_B32>(yOutAddr, vregY, preg0); |
| 297 | - preg0); | ||
| 298 | } | 297 | } |
| 299 | } | 298 | } |
| 300 | } | 299 | } |
| @@ -305,20 +305,20 @@ __aicore__ inline void StaticQuantBlock<T1, T2>::ComputeScale(LocalTensor<float> | |||
| 305 | 305 | ||
| 306 | __VEC_SCOPE__ | 306 | __VEC_SCOPE__ |
| 307 | { | 307 | { |
| 308 | - AscendC::MicroAPI::RegTensor<float> vregGeluRes; | 308 | + AscendC::Reg::RegTensor<float> vregGeluRes; |
| 309 | - AscendC::MicroAPI::RegTensor<float> vregScale; | 309 | + AscendC::Reg::RegTensor<float> vregScale; |
| 310 | - AscendC::MicroAPI::RegTensor<float> vregTmpRes; | 310 | + AscendC::Reg::RegTensor<float> vregTmpRes; |
| 311 | - AscendC::MicroAPI::MaskReg preg0; | 311 | + AscendC::Reg::MaskReg preg0; |
| 312 | 312 | ||
| 313 | for (uint16_t i = 0; i < rowActualVF; i++) { | 313 | for (uint16_t i = 0; i < rowActualVF; i++) { |
| 314 | uint32_t sreg0 = colActual_; | 314 | uint32_t sreg0 = colActual_; |
| 315 | for (uint16_t j = 0; j < loopNum; j++) { | 315 | for (uint16_t j = 0; j < loopNum; j++) { |
| 316 | auto tmpOutAddr = geluResAddr + i * colActualAlignTo32_ + j * vl; | 316 | auto tmpOutAddr = geluResAddr + i * colActualAlignTo32_ + j * vl; |
| 317 | - preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0); | 317 | + preg0 = AscendC::Reg::UpdateMask<float>(sreg0); |
| 318 | - AscendC::MicroAPI::DataCopy(vregGeluRes, geluResAddr + i * colActualAlignTo32_ + j * vl); | 318 | + AscendC::Reg::DataCopy(vregGeluRes, geluResAddr + i * colActualAlignTo32_ + j * vl); |
| 319 | - AscendC::MicroAPI::DataCopy(vregScale, scaleAddr + j * vl); | 319 | + AscendC::Reg::DataCopy(vregScale, scaleAddr + j * vl); |
| 320 | - AscendC::MicroAPI::Mul(vregTmpRes, vregGeluRes, vregScale, preg0); | 320 | + AscendC::Reg::Mul(vregTmpRes, vregGeluRes, vregScale, preg0); |
| 321 | - AscendC::MicroAPI::DataCopy<float>(tmpOutAddr, vregTmpRes, preg0); | 321 | + AscendC::Reg::DataCopy<float>(tmpOutAddr, vregTmpRes, preg0); |
| 322 | } | 322 | } |
| 323 | } | 323 | } |
| 324 | } | 324 | } |
| @@ -341,20 +341,20 @@ __aicore__ inline void StaticQuantBlock<T1, T2>::ComputeOffset(LocalTensor<float | |||
| 341 | 341 | ||
| 342 | __VEC_SCOPE__ | 342 | __VEC_SCOPE__ |
| 343 | { | 343 | { |
| 344 | - AscendC::MicroAPI::RegTensor<float> vregGeluRes; | 344 | + AscendC::Reg::RegTensor<float> vregGeluRes; |
| 345 | - AscendC::MicroAPI::RegTensor<float> vregOffset; | 345 | + AscendC::Reg::RegTensor<float> vregOffset; |
| 346 | - AscendC::MicroAPI::RegTensor<float> vregTmpRes; | 346 | + AscendC::Reg::RegTensor<float> vregTmpRes; |
| 347 | - AscendC::MicroAPI::MaskReg preg0; | 347 | + AscendC::Reg::MaskReg preg0; |
| 348 | 348 | ||
| 349 | for (uint16_t i = 0; i < rowActualVF; i++) { | 349 | for (uint16_t i = 0; i < rowActualVF; i++) { |
| 350 | uint32_t sreg0 = colActual_; | 350 | uint32_t sreg0 = colActual_; |
| 351 | for (uint16_t j = 0; j < loopNum; j++) { | 351 | for (uint16_t j = 0; j < loopNum; j++) { |
| 352 | auto tmpOutAddr = geluResAddr + i * colActualAlignTo32_ + j * vl; | 352 | auto tmpOutAddr = geluResAddr + i * colActualAlignTo32_ + j * vl; |
| 353 | - preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0); | 353 | + preg0 = AscendC::Reg::UpdateMask<float>(sreg0); |
| 354 | - AscendC::MicroAPI::DataCopy(vregGeluRes, geluResAddr + i * colActualAlignTo32_ + j * vl); | 354 | + AscendC::Reg::DataCopy(vregGeluRes, geluResAddr + i * colActualAlignTo32_ + j * vl); |
| 355 | - AscendC::MicroAPI::DataCopy(vregOffset, offsetAddr + j * vl); | 355 | + AscendC::Reg::DataCopy(vregOffset, offsetAddr + j * vl); |
| 356 | - AscendC::MicroAPI::Add(vregTmpRes, vregGeluRes, vregOffset, preg0); | 356 | + AscendC::Reg::Add(vregTmpRes, vregGeluRes, vregOffset, preg0); |
| 357 | - AscendC::MicroAPI::DataCopy<float>(tmpOutAddr, vregTmpRes, preg0); | 357 | + AscendC::Reg::DataCopy<float>(tmpOutAddr, vregTmpRes, preg0); |
| 358 | } | 358 | } |
| 359 | } | 359 | } |
| 360 | } | 360 | } |
| @@ -378,37 +378,36 @@ __aicore__ inline void StaticQuantBlock<T1, T2>::ComputeCast(LocalTensor<float>& | |||
| 378 | 378 | ||
| 379 | __VEC_SCOPE__ | 379 | __VEC_SCOPE__ |
| 380 | { | 380 | { |
| 381 | - AscendC::MicroAPI::RegTensor<float> vregGeluRes; | 381 | + AscendC::Reg::RegTensor<float> vregGeluRes; |
| 382 | - AscendC::MicroAPI::RegTensor<half> vregHalf; | 382 | + AscendC::Reg::RegTensor<half> vregHalf; |
| 383 | - AscendC::MicroAPI::RegTensor<dstType> vregRes; | 383 | + AscendC::Reg::RegTensor<dstType> vregRes; |
| 384 | - AscendC::MicroAPI::MaskReg preg0; | 384 | + AscendC::Reg::MaskReg preg0; |
| 385 | 385 | ||
| 386 | for (uint16_t i = 0; i < rowActualVF; i++) { | 386 | for (uint16_t i = 0; i < rowActualVF; i++) { |
| 387 | uint32_t sreg0 = colActualAlignTo32_; | 387 | uint32_t sreg0 = colActualAlignTo32_; |
| 388 | for (uint16_t j = 0; j < loopNum; j++) { | 388 | for (uint16_t j = 0; j < loopNum; j++) { |
| 389 | auto yOutAddr = yAddr + i * colActualAlignTo8_ + j * vl; | 389 | auto yOutAddr = yAddr + i * colActualAlignTo8_ + j * vl; |
| 390 | - preg0 = AscendC::MicroAPI::UpdateMask<float>(sreg0); | 390 | + preg0 = AscendC::Reg::UpdateMask<float>(sreg0); |
| 391 | - AscendC::MicroAPI::DataCopy(vregGeluRes, geluResAddr + i * colActualAlignTo32_ + j * vl); | 391 | + AscendC::Reg::DataCopy(vregGeluRes, geluResAddr + i * colActualAlignTo32_ + j * vl); |
| 392 | 392 | ||
| 393 | if constexpr (IsSameType<dstType, int8_t>::value) { | 393 | if constexpr (IsSameType<dstType, int8_t>::value) { |
| 394 | - AscendC::MicroAPI::Cast<half, float, castTraitF32ToF16>(vregHalf, vregGeluRes, preg0); | 394 | + AscendC::Reg::Cast<half, float, castTraitF32ToF16>(vregHalf, vregGeluRes, preg0); |
| 395 | - AscendC::MicroAPI::Cast<dstType, half, castTraitF16ToI8Rint>(vregRes, vregHalf, preg0); | 395 | + AscendC::Reg::Cast<dstType, half, castTraitF16ToI8Rint>(vregRes, vregHalf, preg0); |
| 396 | } else if constexpr (IsSameType<dstType, fp8_e4m3fn_t>::value || | 396 | } else if constexpr (IsSameType<dstType, fp8_e4m3fn_t>::value || |
| 397 | IsSameType<dstType, fp8_e5m2_t>::value) { | 397 | IsSameType<dstType, fp8_e5m2_t>::value) { |
| 398 | - AscendC::MicroAPI::Cast<dstType, float, castTraitF32ToF8>(vregRes, vregGeluRes, preg0); | 398 | + AscendC::Reg::Cast<dstType, float, castTraitF32ToF8>(vregRes, vregGeluRes, preg0); |
| 399 | } else if constexpr (IsSameType<dstType, hifloat8_t>::value && | 399 | } else if constexpr (IsSameType<dstType, hifloat8_t>::value && |
| 400 | roundMode == AscendC::RoundMode::CAST_HYBRID) { | 400 | roundMode == AscendC::RoundMode::CAST_HYBRID) { |
| 401 | - AscendC::MicroAPI::Cast<dstType, float, castTraitF32ToH8Hybrid>(vregRes, vregGeluRes, preg0); | 401 | + AscendC::Reg::Cast<dstType, float, castTraitF32ToH8Hybrid>(vregRes, vregGeluRes, preg0); |
| 402 | } else if constexpr (IsSameType<dstType, hifloat8_t>::value && | 402 | } else if constexpr (IsSameType<dstType, hifloat8_t>::value && |
| 403 | roundMode == AscendC::RoundMode::CAST_ROUND) { | 403 | roundMode == AscendC::RoundMode::CAST_ROUND) { |
| 404 | - AscendC::MicroAPI::Cast<dstType, float, castTraitF32ToH8Round>(vregRes, vregGeluRes, preg0); | 404 | + AscendC::Reg::Cast<dstType, float, castTraitF32ToH8Round>(vregRes, vregGeluRes, preg0); |
| 405 | } | 405 | } |
| 406 | - AscendC::MicroAPI::DataCopy<dstType, AscendC::MicroAPI::StoreDist::DIST_PACK4_B32>(yOutAddr, vregRes, | 406 | + AscendC::Reg::DataCopy<dstType, AscendC::Reg::StoreDist::DIST_PACK4_B32>(yOutAddr, vregRes, preg0); |
| 407 | - preg0); | ||
| 408 | } | 407 | } |
| 409 | } | 408 | } |
| 410 | } | 409 | } |
| 411 | 410 | ||
| 412 | } | 411 | } |
| 413 | } // namespace GeluQuantALL | 412 | } // namespace GeluQuantALL |
| 414 | -#endif | 413 | +#endif |
| @@ -50,25 +50,25 @@ struct GeluV2ErfPost : public Vec::ElemwiseBinaryOP<T, T, T> { | |||
| 50 | __ubuf__ T* src2Addr = (__ubuf__ T*)src2.GetPhyAddr(); | 50 | __ubuf__ T* src2Addr = (__ubuf__ T*)src2.GetPhyAddr(); |
| 51 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); | 51 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); |
| 52 | 52 | ||
| 53 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInput1; | 53 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInput1; |
| 54 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInput2; | 54 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInput2; |
| 55 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputAdds; | 55 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputAdds; |
| 56 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputMuls; | 56 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputMuls; |
| 57 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregOutput; | 57 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregOutput; |
| 58 | - MicroAPI::MaskReg mask; | 58 | + Reg::MaskReg mask; |
| 59 | if constexpr (std::is_same_v<T, float>) { | 59 | if constexpr (std::is_same_v<T, float>) { |
| 60 | __VEC_SCOPE__ | 60 | __VEC_SCOPE__ |
| 61 | { | 61 | { |
| 62 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 62 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 63 | - mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count); | 63 | + mask = Reg::UpdateMask<T, Reg::RegTraitNumOne>(count); |
| 64 | // OpCopyIn | 64 | // OpCopyIn |
| 65 | - MicroAPI::LoadAlign(vregInput1, (__ubuf__ T*)(src1Addr + loopIdx * vlSize)); | 65 | + Reg::LoadAlign(vregInput1, (__ubuf__ T*)(src1Addr + loopIdx * vlSize)); |
| 66 | - MicroAPI::LoadAlign(vregInput2, (__ubuf__ T*)(src2Addr + loopIdx * vlSize)); | 66 | + Reg::LoadAlign(vregInput2, (__ubuf__ T*)(src2Addr + loopIdx * vlSize)); |
| 67 | - MicroAPI::Adds(vregInputAdds, vregInput2, (float)1.0, mask); | 67 | + Reg::Adds(vregInputAdds, vregInput2, (float)1.0, mask); |
| 68 | - MicroAPI::Muls(vregInputMuls, vregInput1, (float)0.5, mask); | 68 | + Reg::Muls(vregInputMuls, vregInput1, (float)0.5, mask); |
| 69 | - MicroAPI::Mul(vregOutput, vregInputAdds, vregInputMuls, mask); | 69 | + Reg::Mul(vregOutput, vregInputAdds, vregInputMuls, mask); |
| 70 | // OpCopyOut | 70 | // OpCopyOut |
| 71 | - MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); | 71 | + Reg::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); |
| 72 | } | 72 | } |
| 73 | } | 73 | } |
| 74 | } | 74 | } |
| @@ -88,28 +88,28 @@ struct GeluV2Tanh : public Vec::ElemwiseUnaryOP<T, T> { | |||
| 88 | __ubuf__ T* srcAddr = (__ubuf__ T*)src.GetPhyAddr(); | 88 | __ubuf__ T* srcAddr = (__ubuf__ T*)src.GetPhyAddr(); |
| 89 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); | 89 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); |
| 90 | 90 | ||
| 91 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInput; | 91 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInput; |
| 92 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputSqr; | 92 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputSqr; |
| 93 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputCub; | 93 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputCub; |
| 94 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregOutput; | 94 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregOutput; |
| 95 | - MicroAPI::MaskReg mask; | 95 | + Reg::MaskReg mask; |
| 96 | if constexpr (std::is_same_v<T, float>) { | 96 | if constexpr (std::is_same_v<T, float>) { |
| 97 | __VEC_SCOPE__ | 97 | __VEC_SCOPE__ |
| 98 | { | 98 | { |
| 99 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 99 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 100 | - mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count); | 100 | + mask = Reg::UpdateMask<T, Reg::RegTraitNumOne>(count); |
| 101 | // OpCopyIn | 101 | // OpCopyIn |
| 102 | - MicroAPI::LoadAlign(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize)); | 102 | + Reg::LoadAlign(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize)); |
| 103 | - MicroAPI::Mul(vregInputSqr, vregInput, vregInput, mask); | 103 | + Reg::Mul(vregInputSqr, vregInput, vregInput, mask); |
| 104 | - MicroAPI::Mul(vregInputCub, vregInputSqr, vregInput, mask); | 104 | + Reg::Mul(vregInputCub, vregInputSqr, vregInput, mask); |
| 105 | - MicroAPI::Axpy(vregInputCub, vregInput, TANH_APPROX_FACTOR, mask); | 105 | + Reg::Axpy(vregInputCub, vregInput, TANH_APPROX_FACTOR, mask); |
| 106 | - MicroAPI::Muls(vregInputCub, vregInputCub, NEG_SQRT_EIGHT_OVER_PI, mask); | 106 | + Reg::Muls(vregInputCub, vregInputCub, NEG_SQRT_EIGHT_OVER_PI, mask); |
| 107 | - MicroAPI::Exp(vregInputCub, vregInputCub, mask); | 107 | + Reg::Exp(vregInputCub, vregInputCub, mask); |
| 108 | - MicroAPI::Adds(vregInputCub, vregInputCub, (float)1.0, mask); | 108 | + Reg::Adds(vregInputCub, vregInputCub, (float)1.0, mask); |
| 109 | - MicroAPI::Div(vregOutput, vregInput, vregInputCub, mask); | 109 | + Reg::Div(vregOutput, vregInput, vregInputCub, mask); |
| 110 | 110 | ||
| 111 | // OpCopyOut | 111 | // OpCopyOut |
| 112 | - MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); | 112 | + Reg::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); |
| 113 | } | 113 | } |
| 114 | } | 114 | } |
| 115 | } | 115 | } |
| @@ -31,21 +31,21 @@ constexpr static int32_t BUFFER_NUM = 2; | |||
| 31 | constexpr static int64_t BUFFER_SIZE = 10 * 1024; | 31 | constexpr static int64_t BUFFER_SIZE = 10 * 1024; |
| 32 | constexpr static int32_t BLOCK_BYTES = 32; | 32 | constexpr static int32_t BLOCK_BYTES = 32; |
| 33 | 33 | ||
| 34 | -constexpr static AscendC::MicroAPI::CastTrait castTrait00 = { | 34 | +constexpr static AscendC::Reg::CastTrait castTrait00 = {AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::UNKNOWN, |
| 35 | - AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::UNKNOWN, AscendC::MicroAPI::MaskMergeMode::ZEROING, | 35 | + AscendC::Reg::MaskMergeMode::ZEROING, |
| 36 | - AscendC::RoundMode::UNKNOWN}; | 36 | + AscendC::RoundMode::UNKNOWN}; |
| 37 | 37 | ||
| 38 | -constexpr static AscendC::MicroAPI::CastTrait castTrait01 = { | 38 | +constexpr static AscendC::Reg::CastTrait castTrait01 = {AscendC::Reg::RegLayout::ONE, AscendC::Reg::SatMode::UNKNOWN, |
| 39 | - AscendC::MicroAPI::RegLayout::ONE, AscendC::MicroAPI::SatMode::UNKNOWN, AscendC::MicroAPI::MaskMergeMode::ZEROING, | 39 | + AscendC::Reg::MaskMergeMode::ZEROING, |
| 40 | - AscendC::RoundMode::UNKNOWN}; | 40 | + AscendC::RoundMode::UNKNOWN}; |
| 41 | 41 | ||
| 42 | -constexpr static AscendC::MicroAPI::CastTrait castTrait11 = { | 42 | +constexpr static AscendC::Reg::CastTrait castTrait11 = {AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::NO_SAT, |
| 43 | - AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::NO_SAT, AscendC::MicroAPI::MaskMergeMode::ZEROING, | 43 | + AscendC::Reg::MaskMergeMode::ZEROING, |
| 44 | - AscendC::RoundMode::CAST_RINT}; | 44 | + AscendC::RoundMode::CAST_RINT}; |
| 45 | 45 | ||
| 46 | -constexpr static AscendC::MicroAPI::CastTrait castTrait12 = { | 46 | +constexpr static AscendC::Reg::CastTrait castTrait12 = {AscendC::Reg::RegLayout::ONE, AscendC::Reg::SatMode::NO_SAT, |
| 47 | - AscendC::MicroAPI::RegLayout::ONE, AscendC::MicroAPI::SatMode::NO_SAT, AscendC::MicroAPI::MaskMergeMode::ZEROING, | 47 | + AscendC::Reg::MaskMergeMode::ZEROING, |
| 48 | - AscendC::RoundMode::CAST_RINT}; | 48 | + AscendC::RoundMode::CAST_RINT}; |
| 49 | 49 | ||
| 50 | template <typename T> | 50 | template <typename T> |
| 51 | __aicore__ inline void SetGlobalBufferForGlu(GlobalTensor<T>& xGm, GlobalTensor<T>& yGm, GM_ADDR x, GM_ADDR y) | 51 | __aicore__ inline void SetGlobalBufferForGlu(GlobalTensor<T>& xGm, GlobalTensor<T>& yGm, GM_ADDR x, GM_ADDR y) |
| @@ -57,71 +57,67 @@ __aicore__ inline void SetGlobalBufferForGlu(GlobalTensor<T>& xGm, GlobalTensor< | |||
| 57 | 57 | ||
| 58 | 58 | ||
| 59 | template <typename T> | 59 | template <typename T> |
| 60 | -__aicore__ inline void ComputeSigmoidAndMulCore(AscendC::MicroAPI::RegTensor<T>& vregA, | 60 | +__aicore__ inline void ComputeSigmoidAndMulCore(AscendC::Reg::RegTensor<T>& vregA, AscendC::Reg::RegTensor<T>& vregB, |
| 61 | - AscendC::MicroAPI::RegTensor<T>& vregB, | 61 | + AscendC::Reg::RegTensor<T>& vregOutput, __local_mem__ T* outLocalPtr, |
| 62 | - AscendC::MicroAPI::RegTensor<T>& vregOutput, | 62 | + uint16_t loopIdx, uint32_t vlSize, AscendC::Reg::MaskReg& preg0, |
| 63 | - __local_mem__ T* outLocalPtr, uint16_t loopIdx, uint32_t vlSize, | 63 | + AscendC::Reg::MaskReg& maskAll8, AscendC::Reg::RegTensor<float>& vreg0) |
| 64 | - AscendC::MicroAPI::MaskReg& preg0, AscendC::MicroAPI::MaskReg& maskAll8, | ||
| 65 | - AscendC::MicroAPI::RegTensor<float>& vreg0) | ||
| 66 | { | 64 | { |
| 67 | - AscendC::MicroAPI::RegTensor<float> vreg1; | 65 | + AscendC::Reg::RegTensor<float> vreg1; |
| 68 | - AscendC::MicroAPI::RegTensor<float> vreg2; | 66 | + AscendC::Reg::RegTensor<float> vreg2; |
| 69 | - AscendC::MicroAPI::RegTensor<float> vreg3; | 67 | + AscendC::Reg::RegTensor<float> vreg3; |
| 70 | - AscendC::MicroAPI::RegTensor<float> vreg4; | 68 | + AscendC::Reg::RegTensor<float> vreg4; |
| 71 | - AscendC::MicroAPI::RegTensor<float> vreg5; | 69 | + AscendC::Reg::RegTensor<float> vreg5; |
| 72 | - AscendC::MicroAPI::RegTensor<float> vreg6; | 70 | + AscendC::Reg::RegTensor<float> vreg6; |
| 73 | - AscendC::MicroAPI::RegTensor<float> vreg7; | 71 | + AscendC::Reg::RegTensor<float> vreg7; |
| 74 | - AscendC::MicroAPI::RegTensor<float> vreg8; | 72 | + AscendC::Reg::RegTensor<float> vreg8; |
| 75 | - AscendC::MicroAPI::RegTensor<float> vreg9; | 73 | + AscendC::Reg::RegTensor<float> vreg9; |
| 76 | - AscendC::MicroAPI::RegTensor<float> vreg10; | 74 | + AscendC::Reg::RegTensor<float> vreg10; |
| 77 | - AscendC::MicroAPI::RegTensor<float> vreg11; | 75 | + AscendC::Reg::RegTensor<float> vreg11; |
| 78 | - AscendC::MicroAPI::RegTensor<float> vreg12; | 76 | + AscendC::Reg::RegTensor<float> vreg12; |
| 79 | - AscendC::MicroAPI::RegTensor<float> vreg13; | 77 | + AscendC::Reg::RegTensor<float> vreg13; |
| 80 | - AscendC::MicroAPI::RegTensor<float> vreg14; | 78 | + AscendC::Reg::RegTensor<float> vreg14; |
| 81 | - AscendC::MicroAPI::RegTensor<T> vreg15; | 79 | + AscendC::Reg::RegTensor<T> vreg15; |
| 82 | - AscendC::MicroAPI::RegTensor<T> vreg16; | 80 | + AscendC::Reg::RegTensor<T> vreg16; |
| 83 | 81 | ||
| 84 | if constexpr (std::is_same_v<T, bfloat16_t> || std::is_same_v<T, half>) { | 82 | if constexpr (std::is_same_v<T, bfloat16_t> || std::is_same_v<T, half>) { |
| 85 | - AscendC::MicroAPI::Cast<float, T, castTrait00>(vreg5, vregA, maskAll8); | 83 | + AscendC::Reg::Cast<float, T, castTrait00>(vreg5, vregA, maskAll8); |
| 86 | - AscendC::MicroAPI::Cast<float, T, castTrait01>(vreg6, vregA, maskAll8); | 84 | + AscendC::Reg::Cast<float, T, castTrait01>(vreg6, vregA, maskAll8); |
| 87 | - AscendC::MicroAPI::Cast<float, T, castTrait00>(vreg8, vregB, maskAll8); | 85 | + AscendC::Reg::Cast<float, T, castTrait00>(vreg8, vregB, maskAll8); |
| 88 | - AscendC::MicroAPI::Cast<float, T, castTrait01>(vreg9, vregB, maskAll8); | 86 | + AscendC::Reg::Cast<float, T, castTrait01>(vreg9, vregB, maskAll8); |
| 89 | 87 | ||
| 90 | - AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 88 | + AscendC::Reg::Muls<float, float, AscendC::Reg::MaskMergeMode::ZEROING>(vreg1, vreg8, static_cast<float>(-1), |
| 91 | - vreg1, vreg8, static_cast<float>(-1), maskAll8); | 89 | + maskAll8); |
| 92 | - AscendC::MicroAPI::Exp<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg2, vreg1, maskAll8); | 90 | + AscendC::Reg::Exp<float, AscendC::Reg::MaskMergeMode::ZEROING>(vreg2, vreg1, maskAll8); |
| 93 | - AscendC::MicroAPI::Adds<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 91 | + AscendC::Reg::Adds<float, float, AscendC::Reg::MaskMergeMode::ZEROING>(vreg3, vreg2, static_cast<float>(1), |
| 94 | - vreg3, vreg2, static_cast<float>(1), maskAll8); | 92 | + maskAll8); |
| 95 | - AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg4, vreg0, vreg3, maskAll8); | 93 | + AscendC::Reg::Div<float, AscendC::Reg::MaskMergeMode::ZEROING>(vreg4, vreg0, vreg3, maskAll8); |
| 96 | 94 | ||
| 97 | - AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg7, vreg4, vreg5, maskAll8); | 95 | + AscendC::Reg::Mul<float, AscendC::Reg::MaskMergeMode::ZEROING>(vreg7, vreg4, vreg5, maskAll8); |
| 98 | 96 | ||
| 99 | - AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 97 | + AscendC::Reg::Muls<float, float, AscendC::Reg::MaskMergeMode::ZEROING>(vreg11, vreg9, static_cast<float>(-1), |
| 100 | - vreg11, vreg9, static_cast<float>(-1), maskAll8); | 98 | + maskAll8); |
| 101 | - AscendC::MicroAPI::Exp<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg12, vreg11, maskAll8); | 99 | + AscendC::Reg::Exp<float, AscendC::Reg::MaskMergeMode::ZEROING>(vreg12, vreg11, maskAll8); |
| 102 | - AscendC::MicroAPI::Adds<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 100 | + AscendC::Reg::Adds<float, float, AscendC::Reg::MaskMergeMode::ZEROING>(vreg13, vreg12, static_cast<float>(1), |
| 103 | - vreg13, vreg12, static_cast<float>(1), maskAll8); | 101 | + maskAll8); |
| 104 | - AscendC::MicroAPI::Div<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg14, vreg0, vreg13, maskAll8); | 102 | + AscendC::Reg::Div<float, AscendC::Reg::MaskMergeMode::ZEROING>(vreg14, vreg0, vreg13, maskAll8); |
| 105 | 103 | ||
| 106 | - AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg10, vreg14, vreg6, maskAll8); | 104 | + AscendC::Reg::Mul<float, AscendC::Reg::MaskMergeMode::ZEROING>(vreg10, vreg14, vreg6, maskAll8); |
| 107 | 105 | ||
| 108 | - AscendC::MicroAPI::Cast<T, float, castTrait11>(vreg15, vreg7, maskAll8); | 106 | + AscendC::Reg::Cast<T, float, castTrait11>(vreg15, vreg7, maskAll8); |
| 109 | - AscendC::MicroAPI::Cast<T, float, castTrait12>(vreg16, vreg10, maskAll8); | 107 | + AscendC::Reg::Cast<T, float, castTrait12>(vreg16, vreg10, maskAll8); |
| 110 | - AscendC::Reg::Or((MicroAPI::RegTensor<uint16_t>&)vregOutput, (MicroAPI::RegTensor<uint16_t>&)vreg15, | 108 | + AscendC::Reg::Or((Reg::RegTensor<uint16_t>&)vregOutput, (Reg::RegTensor<uint16_t>&)vreg15, |
| 111 | - (MicroAPI::RegTensor<uint16_t>&)vreg16, maskAll8); | 109 | + (Reg::RegTensor<uint16_t>&)vreg16, maskAll8); |
| 112 | 110 | ||
| 113 | - AscendC::MicroAPI::DataCopy(outLocalPtr + loopIdx * vlSize, vregOutput, preg0); | 111 | + AscendC::Reg::DataCopy(outLocalPtr + loopIdx * vlSize, vregOutput, preg0); |
| 114 | } else { | 112 | } else { |
| 115 | - AscendC::MicroAPI::Muls<T, T, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg1, vregB, static_cast<T>(-1), | 113 | + AscendC::Reg::Muls<T, T, AscendC::Reg::MaskMergeMode::ZEROING>(vreg1, vregB, static_cast<T>(-1), preg0); |
| 116 | - preg0); | 114 | + AscendC::Reg::Exp<T, AscendC::Reg::MaskMergeMode::ZEROING>(vreg2, vreg1, preg0); |
| 117 | - AscendC::MicroAPI::Exp<T, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg2, vreg1, preg0); | 115 | + AscendC::Reg::Adds<T, T, AscendC::Reg::MaskMergeMode::ZEROING>(vreg3, vreg2, static_cast<T>(1), preg0); |
| 118 | - AscendC::MicroAPI::Adds<T, T, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg3, vreg2, static_cast<T>(1), | 116 | + AscendC::Reg::Div<T, AscendC::Reg::MaskMergeMode::ZEROING>(vreg4, vreg0, vreg3, preg0); |
| 119 | - preg0); | ||
| 120 | - AscendC::MicroAPI::Div<T, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vreg4, vreg0, vreg3, preg0); | ||
| 121 | 117 | ||
| 122 | - AscendC::MicroAPI::Mul<T, AscendC::MicroAPI::MaskMergeMode::ZEROING>(vregOutput, vreg4, vregA, preg0); | 118 | + AscendC::Reg::Mul<T, AscendC::Reg::MaskMergeMode::ZEROING>(vregOutput, vreg4, vregA, preg0); |
| 123 | 119 | ||
| 124 | - AscendC::MicroAPI::DataCopy(outLocalPtr + loopIdx * vlSize, vregOutput, preg0); | 120 | + AscendC::Reg::DataCopy(outLocalPtr + loopIdx * vlSize, vregOutput, preg0); |
| 125 | } | 121 | } |
| 126 | } | 122 | } |
| 127 | 123 | ||
| @@ -138,22 +134,22 @@ __aicore__ inline void ComputeSigmoidAndMulImpl(__local_mem__ T* x1LocalPtr, __l | |||
| 138 | 134 | ||
| 139 | __VEC_SCOPE__ | 135 | __VEC_SCOPE__ |
| 140 | { | 136 | { |
| 141 | - AscendC::MicroAPI::RegTensor<float> vreg0; | 137 | + AscendC::Reg::RegTensor<float> vreg0; |
| 142 | - AscendC::MicroAPI::RegTensor<T> vregInput1; | 138 | + AscendC::Reg::RegTensor<T> vregInput1; |
| 143 | - AscendC::MicroAPI::RegTensor<T> vregInput2; | 139 | + AscendC::Reg::RegTensor<T> vregInput2; |
| 144 | - AscendC::MicroAPI::RegTensor<T> vregOutput; | 140 | + AscendC::Reg::RegTensor<T> vregOutput; |
| 145 | 141 | ||
| 146 | - AscendC::MicroAPI::MaskReg preg0; | 142 | + AscendC::Reg::MaskReg preg0; |
| 147 | uint32_t size = count; | 143 | uint32_t size = count; |
| 148 | - preg0 = AscendC::MicroAPI::CreateMask<T>(); | 144 | + preg0 = AscendC::Reg::CreateMask<T>(); |
| 149 | - AscendC::MicroAPI::Duplicate<float, AscendC::MicroAPI::MaskMergeMode::ZEROING, float>( | 145 | + AscendC::Reg::Duplicate<float, AscendC::Reg::MaskMergeMode::ZEROING, float>(vreg0, static_cast<float>(1), |
| 150 | - vreg0, static_cast<float>(1), preg0); | 146 | + preg0); |
| 151 | - AscendC::MicroAPI::MaskReg maskAll8 = MicroAPI::CreateMask<uint8_t, MicroAPI::MaskPattern::ALL>(); | 147 | + AscendC::Reg::MaskReg maskAll8 = Reg::CreateMask<uint8_t, Reg::MaskPattern::ALL>(); |
| 152 | 148 | ||
| 153 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 149 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 154 | - preg0 = AscendC::MicroAPI::UpdateMask<T>(size); | 150 | + preg0 = AscendC::Reg::UpdateMask<T>(size); |
| 155 | - AscendC::MicroAPI::DataCopy(vregInput1, (__ubuf__ T*)(x1LocalPtr + loopIdx * vlSize)); | 151 | + AscendC::Reg::DataCopy(vregInput1, (__ubuf__ T*)(x1LocalPtr + loopIdx * vlSize)); |
| 156 | - AscendC::MicroAPI::DataCopy(vregInput2, (__ubuf__ T*)(x2LocalPtr + loopIdx * vlSize)); | 152 | + AscendC::Reg::DataCopy(vregInput2, (__ubuf__ T*)(x2LocalPtr + loopIdx * vlSize)); |
| 157 | 153 | ||
| 158 | ComputeSigmoidAndMulCore<T>(vregInput1, vregInput2, vregOutput, outLocalPtr, loopIdx, vlSize, preg0, | 154 | ComputeSigmoidAndMulCore<T>(vregInput1, vregInput2, vregOutput, outLocalPtr, loopIdx, vlSize, preg0, |
| 159 | maskAll8, vreg0); | 155 | maskAll8, vreg0); |
| @@ -174,27 +170,27 @@ __aicore__ inline void ComputeSigmoidAndMulWithDeInterleave(__local_mem__ T* xLo | |||
| 174 | 170 | ||
| 175 | __VEC_SCOPE__ | 171 | __VEC_SCOPE__ |
| 176 | { | 172 | { |
| 177 | - AscendC::MicroAPI::RegTensor<float> vreg0; | 173 | + AscendC::Reg::RegTensor<float> vreg0; |
| 178 | - AscendC::MicroAPI::RegTensor<T> vregInput1; | 174 | + AscendC::Reg::RegTensor<T> vregInput1; |
| 179 | - AscendC::MicroAPI::RegTensor<T> vregInput2; | 175 | + AscendC::Reg::RegTensor<T> vregInput2; |
| 180 | - AscendC::MicroAPI::RegTensor<T> vregOutput; | 176 | + AscendC::Reg::RegTensor<T> vregOutput; |
| 181 | - AscendC::MicroAPI::RegTensor<T> vreg1; | 177 | + AscendC::Reg::RegTensor<T> vreg1; |
| 182 | - AscendC::MicroAPI::RegTensor<T> vreg2; | 178 | + AscendC::Reg::RegTensor<T> vreg2; |
| 183 | 179 | ||
| 184 | - AscendC::MicroAPI::MaskReg preg0; | 180 | + AscendC::Reg::MaskReg preg0; |
| 185 | uint32_t size = count; | 181 | uint32_t size = count; |
| 186 | - preg0 = AscendC::MicroAPI::CreateMask<T>(); | 182 | + preg0 = AscendC::Reg::CreateMask<T>(); |
| 187 | - AscendC::MicroAPI::Duplicate<float, AscendC::MicroAPI::MaskMergeMode::ZEROING, float>( | 183 | + AscendC::Reg::Duplicate<float, AscendC::Reg::MaskMergeMode::ZEROING, float>(vreg0, static_cast<float>(1), |
| 188 | - vreg0, static_cast<float>(1), preg0); | 184 | + preg0); |
| 189 | 185 | ||
| 190 | - MicroAPI::MaskReg maskAll8 = MicroAPI::CreateMask<uint8_t, MicroAPI::MaskPattern::ALL>(); | 186 | + Reg::MaskReg maskAll8 = Reg::CreateMask<uint8_t, Reg::MaskPattern::ALL>(); |
| 191 | 187 | ||
| 192 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 188 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 193 | - preg0 = AscendC::MicroAPI::UpdateMask<T>(size); | 189 | + preg0 = AscendC::Reg::UpdateMask<T>(size); |
| 194 | - AscendC::MicroAPI::DataCopy(vregInput1, (__ubuf__ T*)(xLocalPtr + loopIdx * 2 * vlSize)); | 190 | + AscendC::Reg::DataCopy(vregInput1, (__ubuf__ T*)(xLocalPtr + loopIdx * 2 * vlSize)); |
| 195 | - AscendC::MicroAPI::DataCopy(vregInput2, (__ubuf__ T*)(xLocalPtr + loopIdx * 2 * vlSize + vlSize)); | 191 | + AscendC::Reg::DataCopy(vregInput2, (__ubuf__ T*)(xLocalPtr + loopIdx * 2 * vlSize + vlSize)); |
| 196 | 192 | ||
| 197 | - MicroAPI::DeInterleave<T>(vreg1, vreg2, vregInput1, vregInput2); | 193 | + Reg::DeInterleave<T>(vreg1, vreg2, vregInput1, vregInput2); |
| 198 | 194 | ||
| 199 | ComputeSigmoidAndMulCore<T>(vreg1, vreg2, vregOutput, outLocalPtr, loopIdx, vlSize, preg0, maskAll8, vreg0); | 195 | ComputeSigmoidAndMulCore<T>(vreg1, vreg2, vregOutput, outLocalPtr, loopIdx, vlSize, preg0, maskAll8, vreg0); |
| 200 | } | 196 | } |
| @@ -206,4 +202,4 @@ __aicore__ inline void ComputeSigmoidAndMulWithDeInterleave(__local_mem__ T* xLo | |||
| 206 | } // namespace Common | 202 | } // namespace Common |
| 207 | } // namespace Glu | 203 | } // namespace Glu |
| 208 | 204 | ||
| 209 | -#endif // GLU_COMMON_ARCH35_H | 205 | +#endif // GLU_COMMON_ARCH35_H |
| @@ -4,7 +4,7 @@ | |||
| 4 | * CANN Open Software License Agreement Version 2.0 (the "License"). | 4 | * CANN Open Software License Agreement Version 2.0 (the "License"). |
| 5 | * Please refer to the License for details. You may not use this file except in compliance with the License. | 5 | * Please refer to the License for details. You may not use this file except in compliance with the License. |
| 6 | * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | 6 | * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, |
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| @@ -28,9 +28,8 @@ constexpr static int64_t BUFFER_SIZE = 4 * 1024; | |||
| 28 | constexpr static int32_t BLOCK_BYTES = 32; | 28 | constexpr static int32_t BLOCK_BYTES = 32; |
| 29 | 29 | ||
| 30 | template <typename T> | 30 | template <typename T> |
| 31 | -__aicore__ inline void SetGlobalBuffers( | 31 | +__aicore__ inline void SetGlobalBuffers(GlobalTensor<T>& gradOutGm, GlobalTensor<T>& selfGm, GlobalTensor<T>& outGm, |
| 32 | - GlobalTensor<T>& gradOutGm, GlobalTensor<T>& selfGm, GlobalTensor<T>& outGm, | 32 | + GM_ADDR gradOut, GM_ADDR self, GM_ADDR out) |
| 33 | - GM_ADDR gradOut, GM_ADDR self, GM_ADDR out) | ||
| 34 | { | 33 | { |
| 35 | gradOutGm.SetGlobalBuffer((__gm__ T*)gradOut); | 34 | gradOutGm.SetGlobalBuffer((__gm__ T*)gradOut); |
| 36 | selfGm.SetGlobalBuffer((__gm__ T*)self); | 35 | selfGm.SetGlobalBuffer((__gm__ T*)self); |
| @@ -40,148 +39,111 @@ __aicore__ inline void SetGlobalBuffers( | |||
| 40 | 39 | ||
| 41 | 40 | ||
| 42 | template <typename T> | 41 | template <typename T> |
| 43 | -__aicore__ inline void ComputeGluGradCore( | 42 | +__aicore__ inline void ComputeGluGradCore(AscendC::Reg::RegTensor<T>& vregA, AscendC::Reg::RegTensor<T>& vregB, |
| 44 | - AscendC::MicroAPI::RegTensor<T>& vregA, | 43 | + AscendC::Reg::RegTensor<T>& vregGrad, AscendC::Reg::RegTensor<T>& vregOutputA, |
| 45 | - AscendC::MicroAPI::RegTensor<T>& vregB, | 44 | + AscendC::Reg::RegTensor<T>& vregOutputB, __local_mem__ T* outALocalPtr, |
| 46 | - AscendC::MicroAPI::RegTensor<T>& vregGrad, | 45 | + __local_mem__ T* outBLocalPtr, uint16_t loopIdx, uint32_t vlSize, |
| 47 | - AscendC::MicroAPI::RegTensor<T>& vregOutputA, | 46 | + AscendC::Reg::MaskReg& preg0, AscendC::Reg::MaskReg& maskAll8, |
| 48 | - AscendC::MicroAPI::RegTensor<T>& vregOutputB, | 47 | + AscendC::Reg::RegTensor<float>& vregOne) |
| 49 | - __local_mem__ T* outALocalPtr, | ||
| 50 | - __local_mem__ T* outBLocalPtr, | ||
| 51 | - uint16_t loopIdx, | ||
| 52 | - uint32_t vlSize, | ||
| 53 | - AscendC::MicroAPI::MaskReg& preg0, | ||
| 54 | - AscendC::MicroAPI::MaskReg& maskAll8, | ||
| 55 | - AscendC::MicroAPI::RegTensor<float>& vregOne) | ||
| 56 | { | 48 | { |
| 57 | - AscendC::MicroAPI::RegTensor<float> vregSigmoidB; | 49 | + AscendC::Reg::RegTensor<float> vregSigmoidB; |
| 58 | - AscendC::MicroAPI::RegTensor<float> vregGradA; | 50 | + AscendC::Reg::RegTensor<float> vregGradA; |
| 59 | - AscendC::MicroAPI::RegTensor<float> vregTemp; | 51 | + AscendC::Reg::RegTensor<float> vregTemp; |
| 60 | - AscendC::MicroAPI::RegTensor<float> vregSub; | 52 | + AscendC::Reg::RegTensor<float> vregSub; |
| 61 | - AscendC::MicroAPI::RegTensor<float> vregGradB; | 53 | + AscendC::Reg::RegTensor<float> vregGradB; |
| 62 | 54 | ||
| 63 | - static constexpr AscendC::MicroAPI::DivSpecificMode highPrecisionDivMode = { | 55 | + static constexpr AscendC::Reg::DivSpecificMode highPrecisionDivMode = {AscendC::Reg::MaskMergeMode::ZEROING, true}; |
| 64 | - AscendC::MicroAPI::MaskMergeMode::ZEROING, true}; | ||
| 65 | 56 | ||
| 66 | if constexpr (std::is_same_v<T, bfloat16_t> || std::is_same_v<T, half>) { | 57 | if constexpr (std::is_same_v<T, bfloat16_t> || std::is_same_v<T, half>) { |
| 67 | - AscendC::MicroAPI::RegTensor<float> vregAF; | 58 | + AscendC::Reg::RegTensor<float> vregAF; |
| 68 | - AscendC::MicroAPI::RegTensor<float> vregBF; | 59 | + AscendC::Reg::RegTensor<float> vregBF; |
| 69 | - AscendC::MicroAPI::RegTensor<float> vregGradF; | 60 | + AscendC::Reg::RegTensor<float> vregGradF; |
| 70 | 61 | ||
| 71 | - constexpr static AscendC::MicroAPI::CastTrait castToFloatEven = { | 62 | + constexpr static AscendC::Reg::CastTrait castToFloatEven = { |
| 72 | - AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::NO_SAT, | 63 | + AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::NO_SAT, AscendC::Reg::MaskMergeMode::ZEROING, |
| 73 | - AscendC::MicroAPI::MaskMergeMode::ZEROING, RoundMode::UNKNOWN}; | 64 | + RoundMode::UNKNOWN}; |
| 74 | - constexpr static AscendC::MicroAPI::CastTrait castToFloatOdd = { | 65 | + constexpr static AscendC::Reg::CastTrait castToFloatOdd = { |
| 75 | - AscendC::MicroAPI::RegLayout::ONE, AscendC::MicroAPI::SatMode::NO_SAT, | 66 | + AscendC::Reg::RegLayout::ONE, AscendC::Reg::SatMode::NO_SAT, AscendC::Reg::MaskMergeMode::ZEROING, |
| 76 | - AscendC::MicroAPI::MaskMergeMode::ZEROING, RoundMode::UNKNOWN}; | 67 | + RoundMode::UNKNOWN}; |
| 77 | - constexpr static AscendC::MicroAPI::CastTrait castFromFloatEven = { | 68 | + constexpr static AscendC::Reg::CastTrait castFromFloatEven = { |
| 78 | - AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::NO_SAT, | 69 | + AscendC::Reg::RegLayout::ZERO, AscendC::Reg::SatMode::NO_SAT, AscendC::Reg::MaskMergeMode::ZEROING, |
| 79 | - AscendC::MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_RINT}; | 70 | + RoundMode::CAST_RINT}; |
| 80 | - constexpr static AscendC::MicroAPI::CastTrait castFromFloatOdd = { | 71 | + constexpr static AscendC::Reg::CastTrait castFromFloatOdd = { |
| 81 | - AscendC::MicroAPI::RegLayout::ONE, AscendC::MicroAPI::SatMode::NO_SAT, | 72 | + AscendC::Reg::RegLayout::ONE, AscendC::Reg::SatMode::NO_SAT, AscendC::Reg::MaskMergeMode::ZEROING, |
| 82 | - AscendC::MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_RINT}; | 73 | + RoundMode::CAST_RINT}; |
| 83 | 74 | ||
| 84 | - AscendC::MicroAPI::RegTensor<T> vregOutAT; | 75 | + AscendC::Reg::RegTensor<T> vregOutAT; |
| 85 | - AscendC::MicroAPI::RegTensor<T> vregOutBT; | 76 | + AscendC::Reg::RegTensor<T> vregOutBT; |
| 86 | 77 | ||
| 87 | - AscendC::MicroAPI::Cast<float, T, castToFloatEven>(vregAF, vregA, maskAll8); | 78 | + AscendC::Reg::Cast<float, T, castToFloatEven>(vregAF, vregA, maskAll8); |
| 88 | - AscendC::MicroAPI::Cast<float, T, castToFloatEven>(vregBF, vregB, maskAll8); | 79 | + AscendC::Reg::Cast<float, T, castToFloatEven>(vregBF, vregB, maskAll8); |
| 89 | - AscendC::MicroAPI::Cast<float, T, castToFloatEven>(vregGradF, vregGrad, maskAll8); | 80 | + AscendC::Reg::Cast<float, T, castToFloatEven>(vregGradF, vregGrad, maskAll8); |
| 90 | 81 | ||
| 91 | - AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 82 | + AscendC::Reg::Muls<float, float, AscendC::Reg::MaskMergeMode::ZEROING>(vregTemp, vregBF, static_cast<float>(-1), |
| 92 | - vregTemp, vregBF, static_cast<float>(-1), preg0); | 83 | + preg0); |
| 93 | - AscendC::MicroAPI::Exp<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 84 | + AscendC::Reg::Exp<float, AscendC::Reg::MaskMergeMode::ZEROING>(vregSub, vregTemp, preg0); |
| 94 | - vregSub, vregTemp, preg0); | 85 | + AscendC::Reg::Adds<float, float, AscendC::Reg::MaskMergeMode::ZEROING>(vregTemp, vregSub, static_cast<float>(1), |
| 95 | - AscendC::MicroAPI::Adds<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 86 | + preg0); |
| 96 | - vregTemp, vregSub, static_cast<float>(1), preg0); | 87 | + AscendC::Reg::Div<float, &highPrecisionDivMode>(vregSigmoidB, vregOne, vregTemp, preg0); |
| 97 | - AscendC::MicroAPI::Div<float, &highPrecisionDivMode>( | ||
| 98 | - vregSigmoidB, vregOne, vregTemp, preg0); | ||
| 99 | 88 | ||
| 100 | - AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 89 | + AscendC::Reg::Mul<float, AscendC::Reg::MaskMergeMode::ZEROING>(vregGradA, vregGradF, vregSigmoidB, preg0); |
| 101 | - vregGradA, vregGradF, vregSigmoidB, preg0); | ||
| 102 | 90 | ||
| 103 | - AscendC::MicroAPI::Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 91 | + AscendC::Reg::Sub<float, AscendC::Reg::MaskMergeMode::ZEROING>(vregTemp, vregOne, vregSigmoidB, preg0); |
| 104 | - vregTemp, vregOne, vregSigmoidB, preg0); | 92 | + AscendC::Reg::Mul<float, AscendC::Reg::MaskMergeMode::ZEROING>(vregTemp, vregTemp, vregSigmoidB, preg0); |
| 105 | - AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 93 | + AscendC::Reg::Mul<float, AscendC::Reg::MaskMergeMode::ZEROING>(vregTemp, vregTemp, vregAF, preg0); |
| 106 | - vregTemp, vregTemp, vregSigmoidB, preg0); | 94 | + AscendC::Reg::Mul<float, AscendC::Reg::MaskMergeMode::ZEROING>(vregGradB, vregTemp, vregGradF, preg0); |
| 107 | - AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | ||
| 108 | - vregTemp, vregTemp, vregAF, preg0); | ||
| 109 | - AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | ||
| 110 | - vregGradB, vregTemp, vregGradF, preg0); | ||
| 111 | 95 | ||
| 112 | - AscendC::MicroAPI::Cast<T, float, castFromFloatEven>(vregOutputA, vregGradA, maskAll8); | 96 | + AscendC::Reg::Cast<T, float, castFromFloatEven>(vregOutputA, vregGradA, maskAll8); |
| 113 | - AscendC::MicroAPI::Cast<T, float, castFromFloatEven>(vregOutputB, vregGradB, maskAll8); | 97 | + AscendC::Reg::Cast<T, float, castFromFloatEven>(vregOutputB, vregGradB, maskAll8); |
| 114 | 98 | ||
| 115 | - AscendC::MicroAPI::Cast<float, T, castToFloatOdd>(vregAF, vregA, maskAll8); | 99 | + AscendC::Reg::Cast<float, T, castToFloatOdd>(vregAF, vregA, maskAll8); |
| 116 | - AscendC::MicroAPI::Cast<float, T, castToFloatOdd>(vregBF, vregB, maskAll8); | 100 | + AscendC::Reg::Cast<float, T, castToFloatOdd>(vregBF, vregB, maskAll8); |
| 117 | - AscendC::MicroAPI::Cast<float, T, castToFloatOdd>(vregGradF, vregGrad, maskAll8); | 101 | + AscendC::Reg::Cast<float, T, castToFloatOdd>(vregGradF, vregGrad, maskAll8); |
| 118 | 102 | ||
| 119 | - AscendC::MicroAPI::Muls<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 103 | + AscendC::Reg::Muls<float, float, AscendC::Reg::MaskMergeMode::ZEROING>(vregTemp, vregBF, static_cast<float>(-1), |
| 120 | - vregTemp, vregBF, static_cast<float>(-1), preg0); | 104 | + preg0); |
| 121 | - AscendC::MicroAPI::Exp<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 105 | + AscendC::Reg::Exp<float, AscendC::Reg::MaskMergeMode::ZEROING>(vregSub, vregTemp, preg0); |
| 122 | - vregSub, vregTemp, preg0); | 106 | + AscendC::Reg::Adds<float, float, AscendC::Reg::MaskMergeMode::ZEROING>(vregTemp, vregSub, static_cast<float>(1), |
| 123 | - AscendC::MicroAPI::Adds<float, float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 107 | + preg0); |
| 124 | - vregTemp, vregSub, static_cast<float>(1), preg0); | 108 | + AscendC::Reg::Div<float, &highPrecisionDivMode>(vregSigmoidB, vregOne, vregTemp, preg0); |
| 125 | - AscendC::MicroAPI::Div<float, &highPrecisionDivMode>( | ||
| 126 | - vregSigmoidB, vregOne, vregTemp, preg0); | ||
| 127 | 109 | ||
| 128 | - AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 110 | + AscendC::Reg::Mul<float, AscendC::Reg::MaskMergeMode::ZEROING>(vregGradA, vregGradF, vregSigmoidB, preg0); |
| 129 | - vregGradA, vregGradF, vregSigmoidB, preg0); | ||
| 130 | 111 | ||
| 131 | - AscendC::MicroAPI::Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 112 | + AscendC::Reg::Sub<float, AscendC::Reg::MaskMergeMode::ZEROING>(vregTemp, vregOne, vregSigmoidB, preg0); |
| 132 | - vregTemp, vregOne, vregSigmoidB, preg0); | 113 | + AscendC::Reg::Mul<float, AscendC::Reg::MaskMergeMode::ZEROING>(vregTemp, vregTemp, vregSigmoidB, preg0); |
| 133 | - AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 114 | + AscendC::Reg::Mul<float, AscendC::Reg::MaskMergeMode::ZEROING>(vregTemp, vregTemp, vregAF, preg0); |
| 134 | - vregTemp, vregTemp, vregSigmoidB, preg0); | 115 | + AscendC::Reg::Mul<float, AscendC::Reg::MaskMergeMode::ZEROING>(vregGradB, vregTemp, vregGradF, preg0); |
| 135 | - AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | ||
| 136 | - vregTemp, vregTemp, vregAF, preg0); | ||
| 137 | - AscendC::MicroAPI::Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | ||
| 138 | - vregGradB, vregTemp, vregGradF, preg0); | ||
| 139 | 116 | ||
| 140 | - AscendC::MicroAPI::Cast<T, float, castFromFloatOdd>(vregOutAT, vregGradA, maskAll8); | 117 | + AscendC::Reg::Cast<T, float, castFromFloatOdd>(vregOutAT, vregGradA, maskAll8); |
| 141 | - AscendC::MicroAPI::Cast<T, float, castFromFloatOdd>(vregOutBT, vregGradB, maskAll8); | 118 | + AscendC::Reg::Cast<T, float, castFromFloatOdd>(vregOutBT, vregGradB, maskAll8); |
| 142 | 119 | ||
| 143 | - AscendC::MicroAPI::Add<T, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 120 | + AscendC::Reg::Add<T, AscendC::Reg::MaskMergeMode::ZEROING>(vregOutputA, vregOutputA, vregOutAT, preg0); |
| 144 | - vregOutputA, vregOutputA, vregOutAT, preg0); | 121 | + AscendC::Reg::Add<T, AscendC::Reg::MaskMergeMode::ZEROING>(vregOutputB, vregOutputB, vregOutBT, preg0); |
| 145 | - AscendC::MicroAPI::Add<T, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | ||
| 146 | - vregOutputB, vregOutputB, vregOutBT, preg0); | ||
| 147 | 122 | ||
| 148 | - AscendC::MicroAPI::DataCopy(outALocalPtr + loopIdx * vlSize, vregOutputA, preg0); | 123 | + AscendC::Reg::DataCopy(outALocalPtr + loopIdx * vlSize, vregOutputA, preg0); |
| 149 | - AscendC::MicroAPI::DataCopy(outBLocalPtr + loopIdx * vlSize, vregOutputB, preg0); | 124 | + AscendC::Reg::DataCopy(outBLocalPtr + loopIdx * vlSize, vregOutputB, preg0); |
| 150 | } else { | 125 | } else { |
| 151 | - AscendC::MicroAPI::Muls<T, T, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 126 | + AscendC::Reg::Muls<T, T, AscendC::Reg::MaskMergeMode::ZEROING>(vregTemp, vregB, static_cast<T>(-1), preg0); |
| 152 | - vregTemp, vregB, static_cast<T>(-1), preg0); | 127 | + AscendC::Reg::Exp<T, AscendC::Reg::MaskMergeMode::ZEROING>(vregSub, vregTemp, preg0); |
| 153 | - AscendC::MicroAPI::Exp<T, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 128 | + AscendC::Reg::Adds<T, T, AscendC::Reg::MaskMergeMode::ZEROING>(vregTemp, vregSub, static_cast<T>(1), preg0); |
| 154 | - vregSub, vregTemp, preg0); | 129 | + AscendC::Reg::Div<T, &highPrecisionDivMode>(vregSigmoidB, vregOne, vregTemp, preg0); |
| 155 | - AscendC::MicroAPI::Adds<T, T, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | ||
| 156 | - vregTemp, vregSub, static_cast<T>(1), preg0); | ||
| 157 | - AscendC::MicroAPI::Div<T, &highPrecisionDivMode>( | ||
| 158 | - vregSigmoidB, vregOne, vregTemp, preg0); | ||
| 159 | 130 | ||
| 160 | - AscendC::MicroAPI::Mul<T, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 131 | + AscendC::Reg::Mul<T, AscendC::Reg::MaskMergeMode::ZEROING>(vregGradA, vregGrad, vregSigmoidB, preg0); |
| 161 | - vregGradA, vregGrad, vregSigmoidB, preg0); | ||
| 162 | 132 | ||
| 163 | - AscendC::MicroAPI::Sub<T, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 133 | + AscendC::Reg::Sub<T, AscendC::Reg::MaskMergeMode::ZEROING>(vregTemp, vregOne, vregSigmoidB, preg0); |
| 164 | - vregTemp, vregOne, vregSigmoidB, preg0); | 134 | + AscendC::Reg::Mul<T, AscendC::Reg::MaskMergeMode::ZEROING>(vregTemp, vregTemp, vregSigmoidB, preg0); |
| 165 | - AscendC::MicroAPI::Mul<T, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | 135 | + AscendC::Reg::Mul<T, AscendC::Reg::MaskMergeMode::ZEROING>(vregTemp, vregTemp, vregA, preg0); |
| 166 | - vregTemp, vregTemp, vregSigmoidB, preg0); | 136 | + AscendC::Reg::Mul<T, AscendC::Reg::MaskMergeMode::ZEROING>(vregGradB, vregTemp, vregGrad, preg0); |
| 167 | - AscendC::MicroAPI::Mul<T, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | ||
| 168 | - vregTemp, vregTemp, vregA, preg0); | ||
| 169 | - AscendC::MicroAPI::Mul<T, AscendC::MicroAPI::MaskMergeMode::ZEROING>( | ||
| 170 | - vregGradB, vregTemp, vregGrad, preg0); | ||
| 171 | 137 | ||
| 172 | - AscendC::MicroAPI::DataCopy(outALocalPtr + loopIdx * vlSize, vregGradA, preg0); | 138 | + AscendC::Reg::DataCopy(outALocalPtr + loopIdx * vlSize, vregGradA, preg0); |
| 173 | - AscendC::MicroAPI::DataCopy(outBLocalPtr + loopIdx * vlSize, vregGradB, preg0); | 139 | + AscendC::Reg::DataCopy(outBLocalPtr + loopIdx * vlSize, vregGradB, preg0); |
| 174 | } | 140 | } |
| 175 | } | 141 | } |
| 176 | 142 | ||
| 177 | template <typename T> | 143 | template <typename T> |
| 178 | -__aicore__ inline void ComputeGluGradImpl( | 144 | +__aicore__ inline void ComputeGluGradImpl(__local_mem__ T* aLocalPtr, __local_mem__ T* bLocalPtr, |
| 179 | - __local_mem__ T* aLocalPtr, | 145 | + __local_mem__ T* gradLocalPtr, __local_mem__ T* outALocalPtr, |
| 180 | - __local_mem__ T* bLocalPtr, | 146 | + __local_mem__ T* outBLocalPtr, const int64_t& count) |
| 181 | - __local_mem__ T* gradLocalPtr, | ||
| 182 | - __local_mem__ T* outALocalPtr, | ||
| 183 | - __local_mem__ T* outBLocalPtr, | ||
| 184 | - const int64_t& count) | ||
| 185 | { | 147 | { |
| 186 | using namespace Ops::Base; | 148 | using namespace Ops::Base; |
| 187 | constexpr uint32_t VECTOR_LENGTH = GetVRegSize(); | 149 | constexpr uint32_t VECTOR_LENGTH = GetVRegSize(); |
| @@ -192,28 +154,28 @@ __aicore__ inline void ComputeGluGradImpl( | |||
| 192 | 154 | ||
| 193 | __VEC_SCOPE__ | 155 | __VEC_SCOPE__ |
| 194 | { | 156 | { |
| 195 | - AscendC::MicroAPI::RegTensor<float> vregOne; | 157 | + AscendC::Reg::RegTensor<float> vregOne; |
| 196 | - AscendC::MicroAPI::RegTensor<T> vregA; | 158 | + AscendC::Reg::RegTensor<T> vregA; |
| 197 | - AscendC::MicroAPI::RegTensor<T> vregB; | 159 | + AscendC::Reg::RegTensor<T> vregB; |
| 198 | - AscendC::MicroAPI::RegTensor<T> vregGrad; | 160 | + AscendC::Reg::RegTensor<T> vregGrad; |
| 199 | - AscendC::MicroAPI::RegTensor<T> vregOutputA; | 161 | + AscendC::Reg::RegTensor<T> vregOutputA; |
| 200 | - AscendC::MicroAPI::RegTensor<T> vregOutputB; | 162 | + AscendC::Reg::RegTensor<T> vregOutputB; |
| 201 | 163 | ||
| 202 | - AscendC::MicroAPI::MaskReg preg0; | 164 | + AscendC::Reg::MaskReg preg0; |
| 203 | uint32_t size = count; | 165 | uint32_t size = count; |
| 204 | - preg0 = AscendC::MicroAPI::CreateMask<T>(); | 166 | + preg0 = AscendC::Reg::CreateMask<T>(); |
| 205 | - AscendC::MicroAPI::Duplicate<float, AscendC::MicroAPI::MaskMergeMode::ZEROING, float>( | 167 | + AscendC::Reg::Duplicate<float, AscendC::Reg::MaskMergeMode::ZEROING, float>(vregOne, static_cast<float>(1), |
| 206 | - vregOne, static_cast<float>(1), preg0); | 168 | + preg0); |
| 207 | - AscendC::MicroAPI::MaskReg maskAll8 = MicroAPI::CreateMask<uint8_t, MicroAPI::MaskPattern::ALL>(); | 169 | + AscendC::Reg::MaskReg maskAll8 = Reg::CreateMask<uint8_t, Reg::MaskPattern::ALL>(); |
| 208 | 170 | ||
| 209 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 171 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 210 | - preg0 = AscendC::MicroAPI::UpdateMask<T>(size); | 172 | + preg0 = AscendC::Reg::UpdateMask<T>(size); |
| 211 | - AscendC::MicroAPI::DataCopy(vregA, (__ubuf__ T*)(aLocalPtr + loopIdx * vlSize)); | 173 | + AscendC::Reg::DataCopy(vregA, (__ubuf__ T*)(aLocalPtr + loopIdx * vlSize)); |
| 212 | - AscendC::MicroAPI::DataCopy(vregB, (__ubuf__ T*)(bLocalPtr + loopIdx * vlSize)); | 174 | + AscendC::Reg::DataCopy(vregB, (__ubuf__ T*)(bLocalPtr + loopIdx * vlSize)); |
| 213 | - AscendC::MicroAPI::DataCopy(vregGrad, (__ubuf__ T*)(gradLocalPtr + loopIdx * vlSize)); | 175 | + AscendC::Reg::DataCopy(vregGrad, (__ubuf__ T*)(gradLocalPtr + loopIdx * vlSize)); |
| 214 | 176 | ||
| 215 | - ComputeGluGradCore<T>(vregA, vregB, vregGrad, vregOutputA, vregOutputB, | 177 | + ComputeGluGradCore<T>(vregA, vregB, vregGrad, vregOutputA, vregOutputB, outALocalPtr, outBLocalPtr, loopIdx, |
| 216 | - outALocalPtr, outBLocalPtr, loopIdx, vlSize, preg0, maskAll8, vregOne); | 178 | + vlSize, preg0, maskAll8, vregOne); |
| 217 | } | 179 | } |
| 218 | } | 180 | } |
| 219 | } | 181 | } |
| @@ -224,4 +186,3 @@ __aicore__ inline void ComputeGluGradImpl( | |||
| 224 | } // namespace GluGrad | 186 | } // namespace GluGrad |
| 225 | 187 | ||
| 226 | 188 | ||
| 227 | - | ||
| @@ -10,7 +10,7 @@ | |||
| 10 | 10 | ||
| 11 | /*! | 11 | /*! |
| 12 | * \file leaky_relu_dag.h | 12 | * \file leaky_relu_dag.h |
| 13 | - * \brief LeakyReLU 算子 DAG 定义及 MicroAPI 自定义 Kernel 实现 | 13 | + * \brief LeakyReLU 算子 DAG 定义及 Reg 自定义 Kernel 实现 |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | 16 | ||
| @@ -32,24 +32,24 @@ struct LeakyReluCustom : public Vec::ElemwiseBinaryOP<T, T, float> { | |||
| 32 | __ubuf__ T* srcAddr = (__ubuf__ T*)src.GetPhyAddr(); | 32 | __ubuf__ T* srcAddr = (__ubuf__ T*)src.GetPhyAddr(); |
| 33 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); | 33 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); |
| 34 | 34 | ||
| 35 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInput; | 35 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInput; |
| 36 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregNegPart; | 36 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregNegPart; |
| 37 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregOutput; | 37 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregOutput; |
| 38 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregZero; | 38 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregZero; |
| 39 | - MicroAPI::MaskReg mask, cmpMask; | 39 | + Reg::MaskReg mask, cmpMask; |
| 40 | 40 | ||
| 41 | __VEC_SCOPE__ | 41 | __VEC_SCOPE__ |
| 42 | { | 42 | { |
| 43 | - MicroAPI::Duplicate(vregZero, (T)0.0); | 43 | + Reg::Duplicate(vregZero, (T)0.0); |
| 44 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 44 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 45 | - mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count); | 45 | + mask = Reg::UpdateMask<T, Reg::RegTraitNumOne>(count); |
| 46 | - MicroAPI::LoadAlign(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize)); | 46 | + Reg::LoadAlign(vregInput, (__ubuf__ T*)(srcAddr + loopIdx * vlSize)); |
| 47 | 47 | ||
| 48 | - MicroAPI::Muls(vregNegPart, vregInput, negativeSlope, mask); | 48 | + Reg::Muls(vregNegPart, vregInput, negativeSlope, mask); |
| 49 | - MicroAPI::Compare<T, CMPMODE::GT>(cmpMask, vregInput, vregZero, mask); | 49 | + Reg::Compare<T, CMPMODE::GT>(cmpMask, vregInput, vregZero, mask); |
| 50 | - MicroAPI::Select<T>(vregOutput, vregInput, vregNegPart, cmpMask); | 50 | + Reg::Select<T>(vregOutput, vregInput, vregNegPart, cmpMask); |
| 51 | 51 | ||
| 52 | - MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); | 52 | + Reg::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), vregOutput, mask); |
| 53 | } | 53 | } |
| 54 | } | 54 | } |
| 55 | 55 | ||
| @@ -44,45 +44,45 @@ struct LogSigmoidCustom : public Vec::ElemwiseUnaryOP<T, T> { | |||
| 44 | __VEC_SCOPE__ | 44 | __VEC_SCOPE__ |
| 45 | { | 45 | { |
| 46 | // init vars | 46 | // init vars |
| 47 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> x; | 47 | + Reg::RegTensor<T, Reg::RegTraitNumOne> x; |
| 48 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> zeroReg; | 48 | + Reg::RegTensor<T, Reg::RegTraitNumOne> zeroReg; |
| 49 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> xAbs; | 49 | + Reg::RegTensor<T, Reg::RegTraitNumOne> xAbs; |
| 50 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> xAbsNeg; | 50 | + Reg::RegTensor<T, Reg::RegTraitNumOne> xAbsNeg; |
| 51 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> expRes; | 51 | + Reg::RegTensor<T, Reg::RegTraitNumOne> expRes; |
| 52 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> expResPlusOne; | 52 | + Reg::RegTensor<T, Reg::RegTraitNumOne> expResPlusOne; |
| 53 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> expResPlusOneSubOne; | 53 | + Reg::RegTensor<T, Reg::RegTraitNumOne> expResPlusOneSubOne; |
| 54 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> logExpXPlus1; | 54 | + Reg::RegTensor<T, Reg::RegTraitNumOne> logExpXPlus1; |
| 55 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> divRes; | 55 | + Reg::RegTensor<T, Reg::RegTraitNumOne> divRes; |
| 56 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> mulRes; | 56 | + Reg::RegTensor<T, Reg::RegTraitNumOne> mulRes; |
| 57 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> minRes; | 57 | + Reg::RegTensor<T, Reg::RegTraitNumOne> minRes; |
| 58 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> selectRes; | 58 | + Reg::RegTensor<T, Reg::RegTraitNumOne> selectRes; |
| 59 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> ans; | 59 | + Reg::RegTensor<T, Reg::RegTraitNumOne> ans; |
| 60 | 60 | ||
| 61 | - MicroAPI::MaskReg mask; | 61 | + Reg::MaskReg mask; |
| 62 | - MicroAPI::MaskReg cmpLog1pPosMaskReg; | 62 | + Reg::MaskReg cmpLog1pPosMaskReg; |
| 63 | 63 | ||
| 64 | - MicroAPI::Duplicate(zeroReg, valZero); | 64 | + Reg::Duplicate(zeroReg, valZero); |
| 65 | 65 | ||
| 66 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 66 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 67 | // regCopyIn | 67 | // regCopyIn |
| 68 | - mask = MicroAPI::UpdateMask<T, MicroAPI::RegTraitNumOne>(count); | 68 | + mask = Reg::UpdateMask<T, Reg::RegTraitNumOne>(count); |
| 69 | - MicroAPI::LoadAlign(x, (__ubuf__ T*)(srcAddr + loopIdx * vlSize)); | 69 | + Reg::LoadAlign(x, (__ubuf__ T*)(srcAddr + loopIdx * vlSize)); |
| 70 | - MicroAPI::Min(minRes, x, zeroReg, mask); // x1 = min(x, 0) | 70 | + Reg::Min(minRes, x, zeroReg, mask); // x1 = min(x, 0) |
| 71 | - MicroAPI::Abs(xAbs, x, mask); // x2 = abs(x) | 71 | + Reg::Abs(xAbs, x, mask); // x2 = abs(x) |
| 72 | - MicroAPI::Muls(xAbsNeg, xAbs, valNegOne, mask); // x3 = -x2 | 72 | + Reg::Muls(xAbsNeg, xAbs, valNegOne, mask); // x3 = -x2 |
| 73 | - MicroAPI::Exp(expRes, xAbsNeg, mask); // x4 = e^x3 | 73 | + Reg::Exp(expRes, xAbsNeg, mask); // x4 = e^x3 |
| 74 | // log1p | 74 | // log1p |
| 75 | - MicroAPI::Adds(expResPlusOne, expRes, valPosOne, mask); // y1 = 1 + x4 | 75 | + Reg::Adds(expResPlusOne, expRes, valPosOne, mask); // y1 = 1 + x4 |
| 76 | - MicroAPI::Adds(expResPlusOneSubOne, expResPlusOne, valNegOne, mask); // y2 = y1 - 1 | 76 | + Reg::Adds(expResPlusOneSubOne, expResPlusOne, valNegOne, mask); // y2 = y1 - 1 |
| 77 | - MicroAPI::Div(divRes, expRes, expResPlusOneSubOne, mask); // y3 = x4 / y2 | 77 | + Reg::Div(divRes, expRes, expResPlusOneSubOne, mask); // y3 = x4 / y2 |
| 78 | - MicroAPI::Log(logExpXPlus1, expResPlusOne, mask); // y4 = log(y1) | 78 | + Reg::Log(logExpXPlus1, expResPlusOne, mask); // y4 = log(y1) |
| 79 | - MicroAPI::Mul(mulRes, logExpXPlus1, divRes, mask); // y5 = y4 * y3 | 79 | + Reg::Mul(mulRes, logExpXPlus1, divRes, mask); // y5 = y4 * y3 |
| 80 | - MicroAPI::Compares<T, CMPMODE::NE>(cmpLog1pPosMaskReg, expResPlusOne, valPosOne, mask); | 80 | + Reg::Compares<T, CMPMODE::NE>(cmpLog1pPosMaskReg, expResPlusOne, valPosOne, mask); |
| 81 | - MicroAPI::Select(selectRes, mulRes, expRes, cmpLog1pPosMaskReg); // z1 = select(x4, y5) | 81 | + Reg::Select(selectRes, mulRes, expRes, cmpLog1pPosMaskReg); // z1 = select(x4, y5) |
| 82 | - MicroAPI::Sub(ans, minRes, selectRes, mask); // z2 = x1 - z1 | 82 | + Reg::Sub(ans, minRes, selectRes, mask); // z2 = x1 - z1 |
| 83 | 83 | ||
| 84 | // regCopyOut | 84 | // regCopyOut |
| 85 | - MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), ans, mask); | 85 | + Reg::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), ans, mask); |
| 86 | } | 86 | } |
| 87 | } | 87 | } |
| 88 | 88 | ||
| @@ -28,13 +28,13 @@ | |||
| 28 | 28 | ||
| 29 | namespace LogSoftmaxGradOps { | 29 | namespace LogSoftmaxGradOps { |
| 30 | using namespace AscendC; | 30 | using namespace AscendC; |
| 31 | -using namespace AscendC::MicroAPI; | 31 | +using namespace AscendC::Reg; |
| 32 | 32 | ||
| 33 | -using AscendC::MicroAPI::LoadDist; | 33 | +using AscendC::Reg::LoadDist; |
| 34 | -using AscendC::MicroAPI::MaskMergeMode; | 34 | +using AscendC::Reg::MaskMergeMode; |
| 35 | -using AscendC::MicroAPI::MaskReg; | 35 | +using AscendC::Reg::MaskReg; |
| 36 | -using AscendC::MicroAPI::RegTensor; | 36 | +using AscendC::Reg::RegTensor; |
| 37 | -using AscendC::MicroAPI::StoreDist; | 37 | +using AscendC::Reg::StoreDist; |
| 38 | 38 | ||
| 39 | constexpr static uint32_t DOUBLE_BUFFER = 2; | 39 | constexpr static uint32_t DOUBLE_BUFFER = 2; |
| 40 | constexpr static uint32_t BLOCK_SIZE = 32; // 32B | 40 | constexpr static uint32_t BLOCK_SIZE = 32; // 32B |
| @@ -205,41 +205,41 @@ __aicore__ inline void LogSoftmaxGradAR<T>::NormCompute(const LocalTensor<T>& ds | |||
| 205 | 205 | ||
| 206 | __VEC_SCOPE__ | 206 | __VEC_SCOPE__ |
| 207 | { | 207 | { |
| 208 | - AscendC::MicroAPI::MaskReg pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 208 | + AscendC::Reg::MaskReg pFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 209 | - AscendC::MicroAPI::UnalignRegForStore UReg; | 209 | + AscendC::Reg::UnalignRegForStore UReg; |
| 210 | 210 | ||
| 211 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { | 211 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { |
| 212 | dst = (__ubuf__ float*)reduceSumTempTensor.GetPhyAddr() + i * outerLoopDstStride; | 212 | dst = (__ubuf__ float*)reduceSumTempTensor.GetPhyAddr() + i * outerLoopDstStride; |
| 213 | for (uint16_t j = 0; j < mainFoldLoopTimes; ++j) { | 213 | for (uint16_t j = 0; j < mainFoldLoopTimes; ++j) { |
| 214 | - AscendC::MicroAPI::RegTensor<float> reg0, reg1, reg2; | 214 | + AscendC::Reg::RegTensor<float> reg0, reg1, reg2; |
| 215 | LoadTensorForDtypeTIn(foldGradA, reg0, pFull, i * outerLoopStride + j * innerLoopStride); | 215 | LoadTensorForDtypeTIn(foldGradA, reg0, pFull, i * outerLoopStride + j * innerLoopStride); |
| 216 | LoadTensorForDtypeTIn(foldGradB, reg1, pFull, i * outerLoopStride + j * innerLoopStride); | 216 | LoadTensorForDtypeTIn(foldGradB, reg1, pFull, i * outerLoopStride + j * innerLoopStride); |
| 217 | 217 | ||
| 218 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(reg0, reg0, reg1, pFull); | 218 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(reg0, reg0, reg1, pFull); |
| 219 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg0, pFull); | 219 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg0, pFull); |
| 220 | - AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, reg2, UReg, 1); | 220 | + AscendC::Reg::StoreUnAlign((__ubuf__ float*&)dst, reg2, UReg, 1); |
| 221 | } | 221 | } |
| 222 | for (uint16_t j = 0; j < tailFoldLoopTimes; ++j) { | 222 | for (uint16_t j = 0; j < tailFoldLoopTimes; ++j) { |
| 223 | uint32_t count = static_cast<uint32_t>(tailFoldElemCount); | 223 | uint32_t count = static_cast<uint32_t>(tailFoldElemCount); |
| 224 | - AscendC::MicroAPI::RegTensor<float> reg0, reg1, reg2; | 224 | + AscendC::Reg::RegTensor<float> reg0, reg1, reg2; |
| 225 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 225 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 226 | 226 | ||
| 227 | LoadTensorForDtypeTIn(tailGradA, reg0, pFull, i * outerLoopStride + j * innerLoopStride); | 227 | LoadTensorForDtypeTIn(tailGradA, reg0, pFull, i * outerLoopStride + j * innerLoopStride); |
| 228 | LoadTensorForDtypeTIn(tailGradB, reg1, pMask, i * outerLoopStride + j * innerLoopStride); | 228 | LoadTensorForDtypeTIn(tailGradB, reg1, pMask, i * outerLoopStride + j * innerLoopStride); |
| 229 | 229 | ||
| 230 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(reg1, reg0, reg1, pMask); | 230 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(reg1, reg0, reg1, pMask); |
| 231 | - Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(reg0, reg1, pMask); | 231 | + Move<float, AscendC::Reg::MaskMergeMode::MERGING>(reg0, reg1, pMask); |
| 232 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg0, pFull); | 232 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg0, pFull); |
| 233 | - AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, reg2, UReg, 1); | 233 | + AscendC::Reg::StoreUnAlign((__ubuf__ float*&)dst, reg2, UReg, 1); |
| 234 | } | 234 | } |
| 235 | for (uint16_t j = 0; j < unFoldLoopTimes; ++j) { | 235 | for (uint16_t j = 0; j < unFoldLoopTimes; ++j) { |
| 236 | - AscendC::MicroAPI::RegTensor<float> reg0, reg1; | 236 | + AscendC::Reg::RegTensor<float> reg0, reg1; |
| 237 | LoadTensorForDtypeTIn(unFoldGrad, reg0, pFull, i * outerLoopStride + j * innerLoopStride); | 237 | LoadTensorForDtypeTIn(unFoldGrad, reg0, pFull, i * outerLoopStride + j * innerLoopStride); |
| 238 | 238 | ||
| 239 | Reduce<AscendC::Reg::ReduceType::SUM>(reg1, reg0, pFull); | 239 | Reduce<AscendC::Reg::ReduceType::SUM>(reg1, reg0, pFull); |
| 240 | - AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, reg1, UReg, 1); | 240 | + AscendC::Reg::StoreUnAlign((__ubuf__ float*&)dst, reg1, UReg, 1); |
| 241 | } | 241 | } |
| 242 | - AscendC::MicroAPI::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0); | 242 | + AscendC::Reg::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0); |
| 243 | } | 243 | } |
| 244 | } | 244 | } |
| 245 | NormComputePost(dstTensor, gradTensor, xTensor, reduceSumTempTensor, aSize, foldPoint, outerLoopDstStride); | 245 | NormComputePost(dstTensor, gradTensor, xTensor, reduceSumTempTensor, aSize, foldPoint, outerLoopDstStride); |
| @@ -272,11 +272,10 @@ __aicore__ inline void LogSoftmaxGradAR<T>::NormComputePostWithMul(const LocalTe | |||
| 272 | __VEC_SCOPE__ | 272 | __VEC_SCOPE__ |
| 273 | { | 273 | { |
| 274 | uint32_t count = static_cast<uint32_t>(rSize); | 274 | uint32_t count = static_cast<uint32_t>(rSize); |
| 275 | - AscendC::MicroAPI::RegTensor<float> reg0, reg1, reg2; | 275 | + AscendC::Reg::RegTensor<float> reg0, reg1, reg2; |
| 276 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 276 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 277 | - AscendC::MicroAPI::MaskReg | 277 | + AscendC::Reg::MaskReg pFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 278 | - pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 278 | + AscendC::Reg::MaskReg maskOri; |
| 279 | - AscendC::MicroAPI::MaskReg maskOri; | ||
| 280 | for (uint16_t i = 0; i < loopTimes; ++i) { | 279 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 281 | LoadTensorForDtypeTIn(grad, reg0, pMask, i * rAligned); | 280 | LoadTensorForDtypeTIn(grad, reg0, pMask, i * rAligned); |
| 282 | LoadTensorForDtypeTIn(x, reg1, pMask, i * rAligned); | 281 | LoadTensorForDtypeTIn(x, reg1, pMask, i * rAligned); |
| @@ -301,20 +300,19 @@ __aicore__ inline void LogSoftmaxGradAR<T>::NormComputePostWithMul(const LocalTe | |||
| 301 | __VEC_SCOPE__ | 300 | __VEC_SCOPE__ |
| 302 | { | 301 | { |
| 303 | uint32_t count = static_cast<uint32_t>(rSize - VL_FP32); | 302 | uint32_t count = static_cast<uint32_t>(rSize - VL_FP32); |
| 304 | - AscendC::MicroAPI::RegTensor<float> reg0, reg1, regExp, reg0_1, reg1_1, reg2, reg2_1, reg2_2; | 303 | + AscendC::Reg::RegTensor<float> reg0, reg1, regExp, reg0_1, reg1_1, reg2, reg2_1, reg2_2; |
| 305 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 304 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 306 | - AscendC::MicroAPI::MaskReg | 305 | + AscendC::Reg::MaskReg pFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 307 | - pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 306 | + AscendC::Reg::MaskReg maskOri; |
| 308 | - AscendC::MicroAPI::MaskReg maskOri; | ||
| 309 | for (uint16_t i = 0; i < loopTimes; ++i) { | 307 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 310 | LoadTensorForDtypeTIn(grad, reg0, pFull, i * rAligned); | 308 | LoadTensorForDtypeTIn(grad, reg0, pFull, i * rAligned); |
| 311 | LoadTensorForDtypeTIn(x, reg1, pFull, i * rAligned); | 309 | LoadTensorForDtypeTIn(x, reg1, pFull, i * rAligned); |
| 312 | LoadTensorForDtypeTIn(grad_1, reg0_1, pMask, i * rAligned); | 310 | LoadTensorForDtypeTIn(grad_1, reg0_1, pMask, i * rAligned); |
| 313 | LoadTensorForDtypeTIn(x_1, reg1_1, pMask, i * rAligned); | 311 | LoadTensorForDtypeTIn(x_1, reg1_1, pMask, i * rAligned); |
| 314 | 312 | ||
| 315 | - Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(reg2_1, reg0, pFull); | 313 | + Move<float, AscendC::Reg::MaskMergeMode::MERGING>(reg2_1, reg0, pFull); |
| 316 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(reg2_2, reg0, reg0_1, pMask); | 314 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(reg2_2, reg0, reg0_1, pMask); |
| 317 | - Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(reg0, reg2_2, pMask); | 315 | + Move<float, AscendC::Reg::MaskMergeMode::MERGING>(reg0, reg2_2, pMask); |
| 318 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg0, pFull); | 316 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg0, pFull); |
| 319 | Duplicate(reg2, reg2, pFull); | 317 | Duplicate(reg2, reg2, pFull); |
| 320 | 318 | ||
| @@ -361,11 +359,10 @@ __aicore__ inline void LogSoftmaxGradAR<T>::NormComputePost( | |||
| 361 | __VEC_SCOPE__ | 359 | __VEC_SCOPE__ |
| 362 | { | 360 | { |
| 363 | uint32_t count = static_cast<uint32_t>(rSize); | 361 | uint32_t count = static_cast<uint32_t>(rSize); |
| 364 | - AscendC::MicroAPI::RegTensor<float> reg0, reg1, reg2, regExp; | 362 | + AscendC::Reg::RegTensor<float> reg0, reg1, reg2, regExp; |
| 365 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 363 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 366 | - AscendC::MicroAPI::MaskReg | 364 | + AscendC::Reg::MaskReg pFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 367 | - pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 365 | + AscendC::Reg::MaskReg maskOri; |
| 368 | - AscendC::MicroAPI::MaskReg maskOri; | ||
| 369 | for (uint16_t i = 0; i < loopTimes; ++i) { | 366 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 370 | LoadAlign(reg0, (__ubuf__ float*)sumTmp + i * static_cast<uint32_t>(stride)); | 367 | LoadAlign(reg0, (__ubuf__ float*)sumTmp + i * static_cast<uint32_t>(stride)); |
| 371 | Reduce<AscendC::Reg::ReduceType::SUM>(reg1, reg0, pMask); | 368 | Reduce<AscendC::Reg::ReduceType::SUM>(reg1, reg0, pMask); |
| @@ -373,7 +370,7 @@ __aicore__ inline void LogSoftmaxGradAR<T>::NormComputePost( | |||
| 373 | 370 | ||
| 374 | uint32_t sreg0 = static_cast<uint32_t>(oriR); | 371 | uint32_t sreg0 = static_cast<uint32_t>(oriR); |
| 375 | for (uint16_t j = 0; j < rLoopCount; ++j) { | 372 | for (uint16_t j = 0; j < rLoopCount; ++j) { |
| 376 | - maskOri = AscendC::MicroAPI::UpdateMask<float>(sreg0); | 373 | + maskOri = AscendC::Reg::UpdateMask<float>(sreg0); |
| 377 | uint32_t offset = j * VL_FP32 + i * oriRAligned; | 374 | uint32_t offset = j * VL_FP32 + i * oriRAligned; |
| 378 | LoadTensorForDtypeTIn(grad, reg0, maskOri, offset); | 375 | LoadTensorForDtypeTIn(grad, reg0, maskOri, offset); |
| 379 | LoadTensorForDtypeTIn(x, reg1, maskOri, offset); | 376 | LoadTensorForDtypeTIn(x, reg1, maskOri, offset); |
| @@ -395,21 +392,20 @@ __aicore__ inline void LogSoftmaxGradAR<T>::NormComputePost( | |||
| 395 | __VEC_SCOPE__ | 392 | __VEC_SCOPE__ |
| 396 | { | 393 | { |
| 397 | uint32_t count = static_cast<uint32_t>(rSize - VL_FP32); | 394 | uint32_t count = static_cast<uint32_t>(rSize - VL_FP32); |
| 398 | - AscendC::MicroAPI::RegTensor<float> reg0, reg1, reg2, regExp; | 395 | + AscendC::Reg::RegTensor<float> reg0, reg1, reg2, regExp; |
| 399 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 396 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 400 | - AscendC::MicroAPI::MaskReg | 397 | + AscendC::Reg::MaskReg pFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 401 | - pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 398 | + AscendC::Reg::MaskReg maskOri; |
| 402 | - AscendC::MicroAPI::MaskReg maskOri; | ||
| 403 | for (uint16_t i = 0; i < loopTimes; ++i) { | 399 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 404 | LoadAlign(reg0, (__ubuf__ float*)sumTmpA + i * stride); | 400 | LoadAlign(reg0, (__ubuf__ float*)sumTmpA + i * stride); |
| 405 | LoadAlign(reg1, (__ubuf__ float*)sumTmpB + i * stride); | 401 | LoadAlign(reg1, (__ubuf__ float*)sumTmpB + i * stride); |
| 406 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(reg1, reg0, reg1, pMask); | 402 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(reg1, reg0, reg1, pMask); |
| 407 | - Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(reg0, reg1, pMask); | 403 | + Move<float, AscendC::Reg::MaskMergeMode::MERGING>(reg0, reg1, pMask); |
| 408 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg0, pFull); | 404 | Reduce<AscendC::Reg::ReduceType::SUM>(reg2, reg0, pFull); |
| 409 | Duplicate(reg2, reg2, pFull); | 405 | Duplicate(reg2, reg2, pFull); |
| 410 | uint32_t sreg0 = static_cast<uint32_t>(oriR); | 406 | uint32_t sreg0 = static_cast<uint32_t>(oriR); |
| 411 | for (uint16_t j = 0; j < rLoopCount; ++j) { | 407 | for (uint16_t j = 0; j < rLoopCount; ++j) { |
| 412 | - maskOri = AscendC::MicroAPI::UpdateMask<float>(sreg0); | 408 | + maskOri = AscendC::Reg::UpdateMask<float>(sreg0); |
| 413 | uint32_t offset = j * VL_FP32 + i * oriRAligned; | 409 | uint32_t offset = j * VL_FP32 + i * oriRAligned; |
| 414 | 410 | ||
| 415 | LoadTensorForDtypeTIn(grad, reg0, maskOri, offset); | 411 | LoadTensorForDtypeTIn(grad, reg0, maskOri, offset); |
| @@ -453,15 +449,15 @@ __aicore__ inline void LogSoftmaxGradAR<T>::CopyInX(const LocalTensor<T>& xInUb, | |||
| 453 | 449 | ||
| 454 | template <typename T> | 450 | template <typename T> |
| 455 | __aicore__ inline void LogSoftmaxGradAR<T>::StoreTensorForDtypeTOut(__ubuf__ T* dst, | 451 | __aicore__ inline void LogSoftmaxGradAR<T>::StoreTensorForDtypeTOut(__ubuf__ T* dst, |
| 456 | - AscendC::MicroAPI::RegTensor<float>& src, | 452 | + AscendC::Reg::RegTensor<float>& src, |
| 457 | - AscendC::MicroAPI::MaskReg& preg, uint32_t offset) | 453 | + AscendC::Reg::MaskReg& preg, uint32_t offset) |
| 458 | { | 454 | { |
| 459 | if constexpr (IsSameType<T, float>::value) { | 455 | if constexpr (IsSameType<T, float>::value) { |
| 460 | - StoreAlign<T, AscendC::MicroAPI::StoreDist::DIST_NORM>(dst + offset, src, preg); | 456 | + StoreAlign<T, AscendC::Reg::StoreDist::DIST_NORM>(dst + offset, src, preg); |
| 461 | } else { | 457 | } else { |
| 462 | - AscendC::MicroAPI::RegTensor<T> xFp16; | 458 | + AscendC::Reg::RegTensor<T> xFp16; |
| 463 | Cast<T, float, castTraitFp32ToFp16>(xFp16, src, preg); | 459 | Cast<T, float, castTraitFp32ToFp16>(xFp16, src, preg); |
| 464 | - StoreAlign<T, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(dst + offset, xFp16, preg); | 460 | + StoreAlign<T, AscendC::Reg::StoreDist::DIST_PACK_B32>(dst + offset, xFp16, preg); |
| 465 | } | 461 | } |
| 466 | } | 462 | } |
| 467 | 463 | ||
| @@ -242,41 +242,41 @@ __aicore__ inline void LogSoftmaxGradArRecompute<T>::CalculateOutVF(const LocalT | |||
| 242 | 242 | ||
| 243 | __VEC_SCOPE__ | 243 | __VEC_SCOPE__ |
| 244 | { | 244 | { |
| 245 | - MicroAPI::RegTensor<T> xRegFp16, gradRegFp16; | 245 | + Reg::RegTensor<T> xRegFp16, gradRegFp16; |
| 246 | - MicroAPI::RegTensor<float> sumReg, xRegFp32, gradRegFp32, expReg, vreg0, vreg1; | 246 | + Reg::RegTensor<float> sumReg, xRegFp32, gradRegFp32, expReg, vreg0, vreg1; |
| 247 | - MicroAPI::RegTensor<T> vreg2; | 247 | + Reg::RegTensor<T> vreg2; |
| 248 | - MicroAPI::MaskReg mask; | 248 | + Reg::MaskReg mask; |
| 249 | 249 | ||
| 250 | uint32_t width = ubFactor; | 250 | uint32_t width = ubFactor; |
| 251 | uint16_t repeatTimes = CeilDivision(ubFactor, VL_FP32); | 251 | uint16_t repeatTimes = CeilDivision(ubFactor, VL_FP32); |
| 252 | 252 | ||
| 253 | - MicroAPI::LoadAlign<float, MicroAPI::LoadDist::DIST_BRC_B32>(sumReg, gradSumPtr); | 253 | + Reg::LoadAlign<float, Reg::LoadDist::DIST_BRC_B32>(sumReg, gradSumPtr); |
| 254 | 254 | ||
| 255 | for (uint16_t j = 0; j < repeatTimes; j++) { | 255 | for (uint16_t j = 0; j < repeatTimes; j++) { |
| 256 | - mask = MicroAPI::UpdateMask<float>(width); | 256 | + mask = Reg::UpdateMask<float>(width); |
| 257 | auto gradAddr = gradPtr + j * VL_FP32; | 257 | auto gradAddr = gradPtr + j * VL_FP32; |
| 258 | auto xAddr = xPtr + j * VL_FP32; | 258 | auto xAddr = xPtr + j * VL_FP32; |
| 259 | auto yAddr = yPtr + j * VL_FP32; | 259 | auto yAddr = yPtr + j * VL_FP32; |
| 260 | 260 | ||
| 261 | if constexpr (xToFp32_) { | 261 | if constexpr (xToFp32_) { |
| 262 | - MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(xRegFp16, xAddr); | 262 | + Reg::LoadAlign<T, Reg::LoadDist::DIST_UNPACK_B16>(xRegFp16, xAddr); |
| 263 | - MicroAPI::Cast<float, T, castTraitFp16ToFp32>(xRegFp32, xRegFp16, mask); | 263 | + Reg::Cast<float, T, castTraitFp16ToFp32>(xRegFp32, xRegFp16, mask); |
| 264 | - MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(gradRegFp16, gradAddr); | 264 | + Reg::LoadAlign<T, Reg::LoadDist::DIST_UNPACK_B16>(gradRegFp16, gradAddr); |
| 265 | - MicroAPI::Cast<float, T, castTraitFp16ToFp32>(gradRegFp32, gradRegFp16, mask); | 265 | + Reg::Cast<float, T, castTraitFp16ToFp32>(gradRegFp32, gradRegFp16, mask); |
| 266 | } else { | 266 | } else { |
| 267 | - MicroAPI::LoadAlign(xRegFp32, xAddr); | 267 | + Reg::LoadAlign(xRegFp32, xAddr); |
| 268 | - MicroAPI::LoadAlign(gradRegFp32, gradAddr); | 268 | + Reg::LoadAlign(gradRegFp32, gradAddr); |
| 269 | } | 269 | } |
| 270 | 270 | ||
| 271 | - MicroAPI::Exp(expReg, xRegFp32, mask); | 271 | + Reg::Exp(expReg, xRegFp32, mask); |
| 272 | - MicroAPI::Mul(vreg0, expReg, sumReg, mask); | 272 | + Reg::Mul(vreg0, expReg, sumReg, mask); |
| 273 | - MicroAPI::Sub(vreg1, gradRegFp32, vreg0, mask); | 273 | + Reg::Sub(vreg1, gradRegFp32, vreg0, mask); |
| 274 | 274 | ||
| 275 | if constexpr (yToFp32_) { | 275 | if constexpr (yToFp32_) { |
| 276 | - MicroAPI::StoreAlign(yAddr, vreg1, mask); | 276 | + Reg::StoreAlign(yAddr, vreg1, mask); |
| 277 | } else { | 277 | } else { |
| 278 | - MicroAPI::Cast<T, float, castTraitFp32ToFp16>(vreg2, vreg1, mask); | 278 | + Reg::Cast<T, float, castTraitFp32ToFp16>(vreg2, vreg1, mask); |
| 279 | - MicroAPI::StoreAlign<T, MicroAPI::StoreDist::DIST_PACK_B32>(yAddr, vreg2, mask); | 279 | + Reg::StoreAlign<T, Reg::StoreDist::DIST_PACK_B32>(yAddr, vreg2, mask); |
| 280 | } | 280 | } |
| 281 | } | 281 | } |
| 282 | } | 282 | } |
| @@ -292,26 +292,26 @@ __aicore__ inline void LogSoftmaxGradArRecompute<T>::CastVF(const LocalTensor<fl | |||
| 292 | 292 | ||
| 293 | __VEC_SCOPE__ | 293 | __VEC_SCOPE__ |
| 294 | { | 294 | { |
| 295 | - MicroAPI::RegTensor<T> vreg0; | 295 | + Reg::RegTensor<T> vreg0; |
| 296 | - MicroAPI::RegTensor<float> vreg1, vreg2, vreg3; | 296 | + Reg::RegTensor<float> vreg1, vreg2, vreg3; |
| 297 | - MicroAPI::MaskReg mask; | 297 | + Reg::MaskReg mask; |
| 298 | 298 | ||
| 299 | uint32_t width = ubFactor; | 299 | uint32_t width = ubFactor; |
| 300 | uint16_t repeatTimes = CeilDivision(ubFactor, VL_FP32); | 300 | uint16_t repeatTimes = CeilDivision(ubFactor, VL_FP32); |
| 301 | 301 | ||
| 302 | for (uint16_t j = 0; j < repeatTimes; j++) { | 302 | for (uint16_t j = 0; j < repeatTimes; j++) { |
| 303 | - mask = MicroAPI::UpdateMask<float>(width); | 303 | + mask = Reg::UpdateMask<float>(width); |
| 304 | auto gradAddr = gradPtr + j * VL_FP32; | 304 | auto gradAddr = gradPtr + j * VL_FP32; |
| 305 | auto gradFp32Addr = gradFp32Ptr + j * VL_FP32; | 305 | auto gradFp32Addr = gradFp32Ptr + j * VL_FP32; |
| 306 | 306 | ||
| 307 | if constexpr (xToFp32_) { | 307 | if constexpr (xToFp32_) { |
| 308 | - MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, gradAddr); | 308 | + Reg::LoadAlign<T, Reg::LoadDist::DIST_UNPACK_B16>(vreg0, gradAddr); |
| 309 | - MicroAPI::Cast<float, T, castTraitFp16ToFp32>(vreg1, vreg0, mask); | 309 | + Reg::Cast<float, T, castTraitFp16ToFp32>(vreg1, vreg0, mask); |
| 310 | } else { | 310 | } else { |
| 311 | - MicroAPI::LoadAlign(vreg1, gradAddr); | 311 | + Reg::LoadAlign(vreg1, gradAddr); |
| 312 | } | 312 | } |
| 313 | 313 | ||
| 314 | - MicroAPI::StoreAlign(gradFp32Addr, vreg1, mask); | 314 | + Reg::StoreAlign(gradFp32Addr, vreg1, mask); |
| 315 | } | 315 | } |
| 316 | } | 316 | } |
| 317 | } | 317 | } |
| @@ -326,32 +326,32 @@ __aicore__ inline void LogSoftmaxGradArRecompute<T>::FoldBlockVF(const LocalTens | |||
| 326 | 326 | ||
| 327 | __VEC_SCOPE__ | 327 | __VEC_SCOPE__ |
| 328 | { | 328 | { |
| 329 | - MicroAPI::RegTensor<T> vreg0; | 329 | + Reg::RegTensor<T> vreg0; |
| 330 | - MicroAPI::RegTensor<float> vreg1, vreg2, vreg3; | 330 | + Reg::RegTensor<float> vreg1, vreg2, vreg3; |
| 331 | - MicroAPI::MaskReg mask; | 331 | + Reg::MaskReg mask; |
| 332 | - MicroAPI::MaskReg maskFull = MicroAPI::CreateMask<float, MicroAPI::MaskPattern::ALL>(); | 332 | + Reg::MaskReg maskFull = Reg::CreateMask<float, Reg::MaskPattern::ALL>(); |
| 333 | 333 | ||
| 334 | uint16_t foldTimes = CeilDivision(ubFactor, VL_FP32); | 334 | uint16_t foldTimes = CeilDivision(ubFactor, VL_FP32); |
| 335 | 335 | ||
| 336 | uint32_t width = ubFactor; | 336 | uint32_t width = ubFactor; |
| 337 | for (uint16_t j = 0; j < foldTimes; j++) { | 337 | for (uint16_t j = 0; j < foldTimes; j++) { |
| 338 | - mask = MicroAPI::UpdateMask<float>(width); | 338 | + mask = Reg::UpdateMask<float>(width); |
| 339 | auto grad1Addr = grad1Fp32Ptr + j * VL_FP32; | 339 | auto grad1Addr = grad1Fp32Ptr + j * VL_FP32; |
| 340 | auto grad2Addr = grad2Ptr + j * VL_FP32; | 340 | auto grad2Addr = grad2Ptr + j * VL_FP32; |
| 341 | 341 | ||
| 342 | if constexpr (xToFp32_) { | 342 | if constexpr (xToFp32_) { |
| 343 | - MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, grad2Addr); | 343 | + Reg::LoadAlign<T, Reg::LoadDist::DIST_UNPACK_B16>(vreg0, grad2Addr); |
| 344 | - MicroAPI::Cast<float, T, castTraitFp16ToFp32>(vreg2, vreg0, mask); | 344 | + Reg::Cast<float, T, castTraitFp16ToFp32>(vreg2, vreg0, mask); |
| 345 | } else { | 345 | } else { |
| 346 | - MicroAPI::LoadAlign(vreg2, grad2Addr); | 346 | + Reg::LoadAlign(vreg2, grad2Addr); |
| 347 | } | 347 | } |
| 348 | 348 | ||
| 349 | - MicroAPI::LoadAlign(vreg1, grad1Addr); | 349 | + Reg::LoadAlign(vreg1, grad1Addr); |
| 350 | 350 | ||
| 351 | - MicroAPI::Add(vreg3, vreg1, vreg2, mask); | 351 | + Reg::Add(vreg3, vreg1, vreg2, mask); |
| 352 | - MicroAPI::Move<float, MicroAPI::MaskMergeMode::MERGING>(vreg1, vreg3, mask); | 352 | + Reg::Move<float, Reg::MaskMergeMode::MERGING>(vreg1, vreg3, mask); |
| 353 | 353 | ||
| 354 | - MicroAPI::StoreAlign(grad1Addr, vreg1, maskFull); | 354 | + Reg::StoreAlign(grad1Addr, vreg1, maskFull); |
| 355 | } | 355 | } |
| 356 | } | 356 | } |
| 357 | } | 357 | } |
| @@ -375,16 +375,16 @@ __aicore__ inline void LogSoftmaxGradArRecompute<T>::UpdateCache(const LocalTens | |||
| 375 | __VEC_SCOPE__ | 375 | __VEC_SCOPE__ |
| 376 | { | 376 | { |
| 377 | uint32_t sreg = static_cast<uint32_t>(count); | 377 | uint32_t sreg = static_cast<uint32_t>(count); |
| 378 | - MicroAPI::RegTensor<float> aReg, bReg; | 378 | + Reg::RegTensor<float> aReg, bReg; |
| 379 | - MicroAPI::MaskReg pMask; | 379 | + Reg::MaskReg pMask; |
| 380 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { | 380 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { |
| 381 | - pMask = MicroAPI::UpdateMask<float>(sreg); | 381 | + pMask = Reg::UpdateMask<float>(sreg); |
| 382 | - MicroAPI::LoadAlign(aReg, (__ubuf__ float*)src + i * outerLoopStride); | 382 | + Reg::LoadAlign(aReg, (__ubuf__ float*)src + i * outerLoopStride); |
| 383 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { | 383 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { |
| 384 | - MicroAPI::LoadAlign(bReg, (__ubuf__ float*)dst + i * outerLoopStride + j * innerLoopStride); | 384 | + Reg::LoadAlign(bReg, (__ubuf__ float*)dst + i * outerLoopStride + j * innerLoopStride); |
| 385 | - MicroAPI::Add<float, MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); | 385 | + Reg::Add<float, Reg::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); |
| 386 | } | 386 | } |
| 387 | - MicroAPI::StoreAlign((__ubuf__ float*)cache + i * outerLoopStride, aReg, pMask); | 387 | + Reg::StoreAlign((__ubuf__ float*)cache + i * outerLoopStride, aReg, pMask); |
| 388 | } | 388 | } |
| 389 | } | 389 | } |
| 390 | } | 390 | } |
| @@ -28,13 +28,13 @@ | |||
| 28 | 28 | ||
| 29 | namespace LogSoftmaxGradOps { | 29 | namespace LogSoftmaxGradOps { |
| 30 | using namespace AscendC; | 30 | using namespace AscendC; |
| 31 | -using namespace AscendC::MicroAPI; | 31 | +using namespace AscendC::Reg; |
| 32 | 32 | ||
| 33 | -using AscendC::MicroAPI::LoadDist; | 33 | +using AscendC::Reg::LoadDist; |
| 34 | -using AscendC::MicroAPI::MaskMergeMode; | 34 | +using AscendC::Reg::MaskMergeMode; |
| 35 | -using AscendC::MicroAPI::MaskReg; | 35 | +using AscendC::Reg::MaskReg; |
| 36 | -using AscendC::MicroAPI::RegTensor; | 36 | +using AscendC::Reg::RegTensor; |
| 37 | -using AscendC::MicroAPI::StoreDist; | 37 | +using AscendC::Reg::StoreDist; |
| 38 | 38 | ||
| 39 | template <typename T> | 39 | template <typename T> |
| 40 | class LogSoftmaxGradARSmallR { | 40 | class LogSoftmaxGradARSmallR { |
| @@ -189,12 +189,11 @@ private: | |||
| 189 | Sub(gradReg, gradReg, xReg, pregMask); | 189 | Sub(gradReg, gradReg, xReg, pregMask); |
| 190 | 190 | ||
| 191 | if constexpr (xToFp32_) { | 191 | if constexpr (xToFp32_) { |
| 192 | - MicroAPI::StoreAlign(tmpAddrTy + xOffset, gradReg, pregMask); | 192 | + Reg::StoreAlign(tmpAddrTy + xOffset, gradReg, pregMask); |
| 193 | } else { // fp16、bf16 | 193 | } else { // fp16、bf16 |
| 194 | RegTensor<T> xFp16; | 194 | RegTensor<T> xFp16; |
| 195 | - MicroAPI::Cast<T, float, castTraitFp32ToFp16>(xFp16, gradReg, pregMask); | 195 | + Reg::Cast<T, float, castTraitFp32ToFp16>(xFp16, gradReg, pregMask); |
| 196 | - MicroAPI::StoreAlign<T, MicroAPI::StoreDist::DIST_PACK_B32>(tmpAddrTy + xOffset, xFp16, | 196 | + Reg::StoreAlign<T, Reg::StoreDist::DIST_PACK_B32>(tmpAddrTy + xOffset, xFp16, pregMask); |
| 197 | - pregMask); | ||
| 198 | } | 197 | } |
| 199 | } | 198 | } |
| 200 | } | 199 | } |
| @@ -28,13 +28,13 @@ | |||
| 28 | 28 | ||
| 29 | namespace LogSoftmaxGradOps { | 29 | namespace LogSoftmaxGradOps { |
| 30 | using namespace AscendC; | 30 | using namespace AscendC; |
| 31 | -using namespace AscendC::MicroAPI; | 31 | +using namespace AscendC::Reg; |
| 32 | 32 | ||
| 33 | -using AscendC::MicroAPI::LoadDist; | 33 | +using AscendC::Reg::LoadDist; |
| 34 | -using AscendC::MicroAPI::MaskMergeMode; | 34 | +using AscendC::Reg::MaskMergeMode; |
| 35 | -using AscendC::MicroAPI::MaskReg; | 35 | +using AscendC::Reg::MaskReg; |
| 36 | -using AscendC::MicroAPI::RegTensor; | 36 | +using AscendC::Reg::RegTensor; |
| 37 | -using AscendC::MicroAPI::StoreDist; | 37 | +using AscendC::Reg::StoreDist; |
| 38 | 38 | ||
| 39 | template <typename T> | 39 | template <typename T> |
| 40 | class LogSoftmaxGradARA { | 40 | class LogSoftmaxGradARA { |
| @@ -28,13 +28,13 @@ | |||
| 28 | 28 | ||
| 29 | namespace LogSoftmaxGradOps { | 29 | namespace LogSoftmaxGradOps { |
| 30 | using namespace AscendC; | 30 | using namespace AscendC; |
| 31 | -using namespace AscendC::MicroAPI; | 31 | +using namespace AscendC::Reg; |
| 32 | 32 | ||
| 33 | -using AscendC::MicroAPI::LoadDist; | 33 | +using AscendC::Reg::LoadDist; |
| 34 | -using AscendC::MicroAPI::MaskMergeMode; | 34 | +using AscendC::Reg::MaskMergeMode; |
| 35 | -using AscendC::MicroAPI::MaskReg; | 35 | +using AscendC::Reg::MaskReg; |
| 36 | -using AscendC::MicroAPI::RegTensor; | 36 | +using AscendC::Reg::RegTensor; |
| 37 | -using AscendC::MicroAPI::StoreDist; | 37 | +using AscendC::Reg::StoreDist; |
| 38 | 38 | ||
| 39 | template <typename T> | 39 | template <typename T> |
| 40 | class LogSoftmaxGradARARecompute : public LogSoftmaxGradOpsBase { | 40 | class LogSoftmaxGradARARecompute : public LogSoftmaxGradOpsBase { |
| @@ -33,17 +33,17 @@ using AscendC::Reg::Move; | |||
| 33 | using AscendC::Reg::Reduce; | 33 | using AscendC::Reg::Reduce; |
| 34 | using AscendC::Reg::StoreAlign; | 34 | using AscendC::Reg::StoreAlign; |
| 35 | 35 | ||
| 36 | -constexpr static AscendC::MicroAPI::CastTrait castTraitFp16ToFp32 = { | 36 | +constexpr static AscendC::Reg::CastTrait castTraitFp16ToFp32 = { |
| 37 | - AscendC::MicroAPI::RegLayout::ZERO, | 37 | + AscendC::Reg::RegLayout::ZERO, |
| 38 | - AscendC::MicroAPI::SatMode::UNKNOWN, | 38 | + AscendC::Reg::SatMode::UNKNOWN, |
| 39 | - AscendC::MicroAPI::MaskMergeMode::ZEROING, | 39 | + AscendC::Reg::MaskMergeMode::ZEROING, |
| 40 | AscendC::RoundMode::UNKNOWN, | 40 | AscendC::RoundMode::UNKNOWN, |
| 41 | }; | 41 | }; |
| 42 | 42 | ||
| 43 | -constexpr static AscendC::MicroAPI::CastTrait castTraitFp32ToFp16 = { | 43 | +constexpr static AscendC::Reg::CastTrait castTraitFp32ToFp16 = { |
| 44 | - AscendC::MicroAPI::RegLayout::ZERO, | 44 | + AscendC::Reg::RegLayout::ZERO, |
| 45 | - AscendC::MicroAPI::SatMode::NO_SAT, | 45 | + AscendC::Reg::SatMode::NO_SAT, |
| 46 | - AscendC::MicroAPI::MaskMergeMode::ZEROING, | 46 | + AscendC::Reg::MaskMergeMode::ZEROING, |
| 47 | AscendC::RoundMode::CAST_RINT, | 47 | AscendC::RoundMode::CAST_RINT, |
| 48 | }; | 48 | }; |
| 49 | 49 | ||
| @@ -187,14 +187,14 @@ __aicore__ inline void LogSoftmaxGradOpsBase::CastToFp32From(const LocalTensor<f | |||
| 187 | __ubuf__ float* dst = (__ubuf__ float*)dstTensor.GetPhyAddr(); | 187 | __ubuf__ float* dst = (__ubuf__ float*)dstTensor.GetPhyAddr(); |
| 188 | __ubuf__ T* src = (__ubuf__ T*)srcTensor.GetPhyAddr(); | 188 | __ubuf__ T* src = (__ubuf__ T*)srcTensor.GetPhyAddr(); |
| 189 | uint32_t count; | 189 | uint32_t count; |
| 190 | - AscendC::MicroAPI::RegTensor<float> fp32Reg; | 190 | + AscendC::Reg::RegTensor<float> fp32Reg; |
| 191 | - AscendC::MicroAPI::RegTensor<T> b16Reg; | 191 | + AscendC::Reg::RegTensor<T> b16Reg; |
| 192 | - AscendC::MicroAPI::MaskReg pMask; | 192 | + AscendC::Reg::MaskReg pMask; |
| 193 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { | 193 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { |
| 194 | count = static_cast<uint32_t>(colSize); | 194 | count = static_cast<uint32_t>(colSize); |
| 195 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { | 195 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { |
| 196 | - pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 196 | + pMask = AscendC::Reg::UpdateMask<float>(count); |
| 197 | - LoadAlign<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>( | 197 | + LoadAlign<T, AscendC::Reg::LoadDist::DIST_UNPACK_B16>( |
| 198 | b16Reg, (__ubuf__ T*)src + i * outerLoopSrcStride + j * innerLoopStride); | 198 | b16Reg, (__ubuf__ T*)src + i * outerLoopSrcStride + j * innerLoopStride); |
| 199 | Cast<float, T, castTraitFp16ToFp32>(fp32Reg, b16Reg, pMask); | 199 | Cast<float, T, castTraitFp16ToFp32>(fp32Reg, b16Reg, pMask); |
| 200 | StoreAlign((__ubuf__ float*)dst + i * outerLoopDstStride + j * innerLoopStride, fp32Reg, pMask); | 200 | StoreAlign((__ubuf__ float*)dst + i * outerLoopDstStride + j * innerLoopStride, fp32Reg, pMask); |
| @@ -230,16 +230,16 @@ __aicore__ inline void LogSoftmaxGradOpsBase::CastFromFp32To(const LocalTensor<T | |||
| 230 | __ubuf__ T* dst = (__ubuf__ T*)dstTensor.GetPhyAddr(); | 230 | __ubuf__ T* dst = (__ubuf__ T*)dstTensor.GetPhyAddr(); |
| 231 | __ubuf__ float* src = (__ubuf__ float*)srcTensor.GetPhyAddr(); | 231 | __ubuf__ float* src = (__ubuf__ float*)srcTensor.GetPhyAddr(); |
| 232 | uint32_t count; | 232 | uint32_t count; |
| 233 | - AscendC::MicroAPI::RegTensor<float> fp32Reg; | 233 | + AscendC::Reg::RegTensor<float> fp32Reg; |
| 234 | - AscendC::MicroAPI::RegTensor<T> b16Reg; | 234 | + AscendC::Reg::RegTensor<T> b16Reg; |
| 235 | - AscendC::MicroAPI::MaskReg pMask; | 235 | + AscendC::Reg::MaskReg pMask; |
| 236 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { | 236 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { |
| 237 | count = static_cast<uint32_t>(colSize); | 237 | count = static_cast<uint32_t>(colSize); |
| 238 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { | 238 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { |
| 239 | - pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 239 | + pMask = AscendC::Reg::UpdateMask<float>(count); |
| 240 | LoadAlign(fp32Reg, (__ubuf__ float*)src + i * outerLoopSrcStride + j * innerLoopStride); | 240 | LoadAlign(fp32Reg, (__ubuf__ float*)src + i * outerLoopSrcStride + j * innerLoopStride); |
| 241 | Cast<T, float, castTraitFp32ToFp16>(b16Reg, fp32Reg, pMask); | 241 | Cast<T, float, castTraitFp32ToFp16>(b16Reg, fp32Reg, pMask); |
| 242 | - StoreAlign<T, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>( | 242 | + StoreAlign<T, AscendC::Reg::StoreDist::DIST_PACK_B32>( |
| 243 | (__ubuf__ T*)dst + i * outerLoopDstStride + j * innerLoopStride, b16Reg, pMask); | 243 | (__ubuf__ T*)dst + i * outerLoopDstStride + j * innerLoopStride, b16Reg, pMask); |
| 244 | } | 244 | } |
| 245 | } | 245 | } |
| @@ -330,14 +330,14 @@ __aicore__ inline void LogSoftmaxGradOpsBase::VectorAdd(const LocalTensor<float> | |||
| 330 | __ubuf__ float* src0 = (__ubuf__ float*)src0Tensor.GetPhyAddr(); | 330 | __ubuf__ float* src0 = (__ubuf__ float*)src0Tensor.GetPhyAddr(); |
| 331 | __ubuf__ float* src1 = (__ubuf__ float*)src1Tensor.GetPhyAddr(); | 331 | __ubuf__ float* src1 = (__ubuf__ float*)src1Tensor.GetPhyAddr(); |
| 332 | uint32_t sreg = static_cast<uint32_t>(count); | 332 | uint32_t sreg = static_cast<uint32_t>(count); |
| 333 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg; | 333 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg; |
| 334 | - AscendC::MicroAPI::MaskReg pMask; | 334 | + AscendC::Reg::MaskReg pMask; |
| 335 | for (uint16_t i = 0; i < loopTimes; ++i) { | 335 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 336 | - pMask = AscendC::MicroAPI::UpdateMask<float>(sreg); | 336 | + pMask = AscendC::Reg::UpdateMask<float>(sreg); |
| 337 | LoadAlign(aReg, (__ubuf__ float*)src0 + i * VL_FP32); | 337 | LoadAlign(aReg, (__ubuf__ float*)src0 + i * VL_FP32); |
| 338 | LoadAlign(bReg, (__ubuf__ float*)src1 + i * VL_FP32); | 338 | LoadAlign(bReg, (__ubuf__ float*)src1 + i * VL_FP32); |
| 339 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); | 339 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); |
| 340 | - Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(aReg, cReg, pMask); | 340 | + Move<float, AscendC::Reg::MaskMergeMode::MERGING>(aReg, cReg, pMask); |
| 341 | StoreAlign((__ubuf__ float*)dst + i * VL_FP32, aReg, pMask); | 341 | StoreAlign((__ubuf__ float*)dst + i * VL_FP32, aReg, pMask); |
| 342 | } | 342 | } |
| 343 | } | 343 | } |
| @@ -360,15 +360,15 @@ __aicore__ inline void LogSoftmaxGradOpsBase::VectorAdd(const LocalTensor<float> | |||
| 360 | __ubuf__ float* src0 = (__ubuf__ float*)src0Tensor.GetPhyAddr(); | 360 | __ubuf__ float* src0 = (__ubuf__ float*)src0Tensor.GetPhyAddr(); |
| 361 | __ubuf__ float* src1 = (__ubuf__ float*)src1Tensor.GetPhyAddr(); | 361 | __ubuf__ float* src1 = (__ubuf__ float*)src1Tensor.GetPhyAddr(); |
| 362 | uint32_t count = nSize; | 362 | uint32_t count = nSize; |
| 363 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg; | 363 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg; |
| 364 | - AscendC::MicroAPI::MaskReg pMask; | 364 | + AscendC::Reg::MaskReg pMask; |
| 365 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { | 365 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { |
| 366 | - pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 366 | + pMask = AscendC::Reg::UpdateMask<float>(count); |
| 367 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { | 367 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { |
| 368 | LoadAlign(aReg, (__ubuf__ float*)src0 + i * outerLoopStride + j * innerLoopStride); | 368 | LoadAlign(aReg, (__ubuf__ float*)src0 + i * outerLoopStride + j * innerLoopStride); |
| 369 | LoadAlign(bReg, (__ubuf__ float*)src1 + i * outerLoopStride + j * innerLoopStride); | 369 | LoadAlign(bReg, (__ubuf__ float*)src1 + i * outerLoopStride + j * innerLoopStride); |
| 370 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); | 370 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); |
| 371 | - Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(aReg, cReg, pMask); | 371 | + Move<float, AscendC::Reg::MaskMergeMode::MERGING>(aReg, cReg, pMask); |
| 372 | StoreAlign((__ubuf__ float*)dst + i * outerLoopStride + j * innerLoopStride, aReg, pMask); | 372 | StoreAlign((__ubuf__ float*)dst + i * outerLoopStride + j * innerLoopStride, aReg, pMask); |
| 373 | } | 373 | } |
| 374 | } | 374 | } |
| @@ -391,14 +391,14 @@ __aicore__ inline void LogSoftmaxGradOpsBase::VectorMul(const LocalTensor<float> | |||
| 391 | __ubuf__ float* src0 = (__ubuf__ float*)src0Tensor.GetPhyAddr(); | 391 | __ubuf__ float* src0 = (__ubuf__ float*)src0Tensor.GetPhyAddr(); |
| 392 | __ubuf__ float* src1 = (__ubuf__ float*)src1Tensor.GetPhyAddr(); | 392 | __ubuf__ float* src1 = (__ubuf__ float*)src1Tensor.GetPhyAddr(); |
| 393 | uint32_t sreg = static_cast<uint32_t>(count); | 393 | uint32_t sreg = static_cast<uint32_t>(count); |
| 394 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg; | 394 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg; |
| 395 | - AscendC::MicroAPI::MaskReg pMask; | 395 | + AscendC::Reg::MaskReg pMask; |
| 396 | 396 | ||
| 397 | for (uint16_t i = 0; i < loopTimes; ++i) { | 397 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 398 | - pMask = AscendC::MicroAPI::UpdateMask<float>(sreg); | 398 | + pMask = AscendC::Reg::UpdateMask<float>(sreg); |
| 399 | LoadAlign(aReg, (__ubuf__ float*)src0 + i * VL_FP32); | 399 | LoadAlign(aReg, (__ubuf__ float*)src0 + i * VL_FP32); |
| 400 | LoadAlign(bReg, (__ubuf__ float*)src1 + i * VL_FP32); | 400 | LoadAlign(bReg, (__ubuf__ float*)src1 + i * VL_FP32); |
| 401 | - Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); | 401 | + Mul<float, AscendC::Reg::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); |
| 402 | StoreAlign((__ubuf__ float*)dst + i * VL_FP32, cReg, pMask); | 402 | StoreAlign((__ubuf__ float*)dst + i * VL_FP32, cReg, pMask); |
| 403 | } | 403 | } |
| 404 | } | 404 | } |
| @@ -427,14 +427,14 @@ __aicore__ inline void LogSoftmaxGradOpsBase::NlastBroadcastMul(const LocalTenso | |||
| 427 | __ubuf__ float* src0 = (__ubuf__ float*)src0Tensor.GetPhyAddr(); | 427 | __ubuf__ float* src0 = (__ubuf__ float*)src0Tensor.GetPhyAddr(); |
| 428 | __ubuf__ float* src1 = (__ubuf__ float*)src1Tensor.GetPhyAddr(); | 428 | __ubuf__ float* src1 = (__ubuf__ float*)src1Tensor.GetPhyAddr(); |
| 429 | uint32_t count = static_cast<uint32_t>(aSize); | 429 | uint32_t count = static_cast<uint32_t>(aSize); |
| 430 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg; | 430 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg; |
| 431 | - AscendC::MicroAPI::MaskReg pMask; | 431 | + AscendC::Reg::MaskReg pMask; |
| 432 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { | 432 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { |
| 433 | - pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 433 | + pMask = AscendC::Reg::UpdateMask<float>(count); |
| 434 | LoadAlign(bReg, (__ubuf__ float*)src1 + i * outerLoopStride); | 434 | LoadAlign(bReg, (__ubuf__ float*)src1 + i * outerLoopStride); |
| 435 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { | 435 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { |
| 436 | LoadAlign(aReg, (__ubuf__ float*)src0 + i * outerLoopStride + j * innerLoopStride); | 436 | LoadAlign(aReg, (__ubuf__ float*)src0 + i * outerLoopStride + j * innerLoopStride); |
| 437 | - Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); | 437 | + Mul<float, AscendC::Reg::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); |
| 438 | StoreAlign((__ubuf__ float*)dst + i * outerLoopStride + j * innerLoopStride, cReg, pMask); | 438 | StoreAlign((__ubuf__ float*)dst + i * outerLoopStride + j * innerLoopStride, cReg, pMask); |
| 439 | } | 439 | } |
| 440 | } | 440 | } |
| @@ -464,15 +464,15 @@ __aicore__ inline void LogSoftmaxGradOpsBase::LastReduceSumSmallR(const LocalTen | |||
| 464 | __ubuf__ float* dst = (__ubuf__ float*)dstTensor.GetPhyAddr(); | 464 | __ubuf__ float* dst = (__ubuf__ float*)dstTensor.GetPhyAddr(); |
| 465 | __ubuf__ float* src = (__ubuf__ float*)srcTensor.GetPhyAddr(); | 465 | __ubuf__ float* src = (__ubuf__ float*)srcTensor.GetPhyAddr(); |
| 466 | uint32_t count = static_cast<uint32_t>(rSize); | 466 | uint32_t count = static_cast<uint32_t>(rSize); |
| 467 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 467 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 468 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 468 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 469 | - AscendC::MicroAPI::UnalignRegForStore UReg; | 469 | + AscendC::Reg::UnalignRegForStore UReg; |
| 470 | for (uint16_t i = 0; i < loopTimes; ++i) { | 470 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 471 | LoadAlign(aReg, (__ubuf__ float*)src + i * stride); | 471 | LoadAlign(aReg, (__ubuf__ float*)src + i * stride); |
| 472 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pMask); | 472 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pMask); |
| 473 | - AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1); | 473 | + AscendC::Reg::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1); |
| 474 | } | 474 | } |
| 475 | - AscendC::MicroAPI::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0); | 475 | + AscendC::Reg::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0); |
| 476 | } | 476 | } |
| 477 | } else { | 477 | } else { |
| 478 | __VEC_SCOPE__ | 478 | __VEC_SCOPE__ |
| @@ -481,20 +481,19 @@ __aicore__ inline void LogSoftmaxGradOpsBase::LastReduceSumSmallR(const LocalTen | |||
| 481 | __ubuf__ float* src0 = (__ubuf__ float*)srcTensor.GetPhyAddr(); | 481 | __ubuf__ float* src0 = (__ubuf__ float*)srcTensor.GetPhyAddr(); |
| 482 | __ubuf__ float* src1 = (__ubuf__ float*)srcTensor.GetPhyAddr() + VL_FP32; | 482 | __ubuf__ float* src1 = (__ubuf__ float*)srcTensor.GetPhyAddr() + VL_FP32; |
| 483 | uint32_t count = static_cast<uint32_t>(rSize - VL_FP32); | 483 | uint32_t count = static_cast<uint32_t>(rSize - VL_FP32); |
| 484 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg; | 484 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg; |
| 485 | - AscendC::MicroAPI::UnalignRegForStore UReg; | 485 | + AscendC::Reg::UnalignRegForStore UReg; |
| 486 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 486 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 487 | - AscendC::MicroAPI::MaskReg | 487 | + AscendC::Reg::MaskReg pFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 488 | - pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | ||
| 489 | for (uint16_t i = 0; i < loopTimes; ++i) { | 488 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 490 | LoadAlign(aReg, (__ubuf__ float*)src0 + i * stride); | 489 | LoadAlign(aReg, (__ubuf__ float*)src0 + i * stride); |
| 491 | LoadAlign(bReg, (__ubuf__ float*)src1 + i * stride); | 490 | LoadAlign(bReg, (__ubuf__ float*)src1 + i * stride); |
| 492 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); | 491 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); |
| 493 | - Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(aReg, cReg, pMask); | 492 | + Move<float, AscendC::Reg::MaskMergeMode::MERGING>(aReg, cReg, pMask); |
| 494 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pFull); | 493 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pFull); |
| 495 | - AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1); | 494 | + AscendC::Reg::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1); |
| 496 | } | 495 | } |
| 497 | - AscendC::MicroAPI::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0); | 496 | + AscendC::Reg::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0); |
| 498 | } | 497 | } |
| 499 | } | 498 | } |
| 500 | } | 499 | } |
| @@ -546,37 +545,37 @@ __aicore__ inline void LogSoftmaxGradOpsBase::LastReduceSum(const LocalTensor<fl | |||
| 546 | __ubuf__ float* tailSrcA = (__ubuf__ float*)srcTensor.GetPhyAddr() + tailSrcAOffset; | 545 | __ubuf__ float* tailSrcA = (__ubuf__ float*)srcTensor.GetPhyAddr() + tailSrcAOffset; |
| 547 | __ubuf__ float* tailSrcB = (__ubuf__ float*)srcTensor.GetPhyAddr() + tailSrcBOffset; | 546 | __ubuf__ float* tailSrcB = (__ubuf__ float*)srcTensor.GetPhyAddr() + tailSrcBOffset; |
| 548 | __ubuf__ float* unFoldSrc = (__ubuf__ float*)srcTensor.GetPhyAddr() + unFoldSrcOffset; | 547 | __ubuf__ float* unFoldSrc = (__ubuf__ float*)srcTensor.GetPhyAddr() + unFoldSrcOffset; |
| 549 | - AscendC::MicroAPI::MaskReg pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 548 | + AscendC::Reg::MaskReg pFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 550 | - AscendC::MicroAPI::UnalignRegForStore UReg; | 549 | + AscendC::Reg::UnalignRegForStore UReg; |
| 551 | 550 | ||
| 552 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { | 551 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { |
| 553 | dst = (__ubuf__ float*)reduceSumTempTensor.GetPhyAddr() + i * outerLoopDstStride; | 552 | dst = (__ubuf__ float*)reduceSumTempTensor.GetPhyAddr() + i * outerLoopDstStride; |
| 554 | for (uint16_t j = 0; j < mainFoldLoopTimes; ++j) { | 553 | for (uint16_t j = 0; j < mainFoldLoopTimes; ++j) { |
| 555 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg, dReg; | 554 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg, dReg; |
| 556 | LoadAlign(aReg, (__ubuf__ float*)foldSrcA + i * outerLoopStride + j * innerLoopStride); | 555 | LoadAlign(aReg, (__ubuf__ float*)foldSrcA + i * outerLoopStride + j * innerLoopStride); |
| 557 | LoadAlign(bReg, (__ubuf__ float*)foldSrcB + i * outerLoopStride + j * innerLoopStride); | 556 | LoadAlign(bReg, (__ubuf__ float*)foldSrcB + i * outerLoopStride + j * innerLoopStride); |
| 558 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pFull); | 557 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pFull); |
| 559 | Reduce<AscendC::Reg::ReduceType::SUM>(dReg, cReg, pFull); | 558 | Reduce<AscendC::Reg::ReduceType::SUM>(dReg, cReg, pFull); |
| 560 | - AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, dReg, UReg, 1); | 559 | + AscendC::Reg::StoreUnAlign((__ubuf__ float*&)dst, dReg, UReg, 1); |
| 561 | } | 560 | } |
| 562 | for (uint16_t j = 0; j < tailFoldLoopTimes; ++j) { | 561 | for (uint16_t j = 0; j < tailFoldLoopTimes; ++j) { |
| 563 | uint32_t count = static_cast<uint32_t>(tailFoldElemCount); | 562 | uint32_t count = static_cast<uint32_t>(tailFoldElemCount); |
| 564 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg; | 563 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg; |
| 565 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 564 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 566 | LoadAlign(aReg, (__ubuf__ float*)tailSrcA + i * outerLoopStride + j * innerLoopStride); | 565 | LoadAlign(aReg, (__ubuf__ float*)tailSrcA + i * outerLoopStride + j * innerLoopStride); |
| 567 | LoadAlign(bReg, (__ubuf__ float*)tailSrcB + i * outerLoopStride + j * innerLoopStride); | 566 | LoadAlign(bReg, (__ubuf__ float*)tailSrcB + i * outerLoopStride + j * innerLoopStride); |
| 568 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); | 567 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); |
| 569 | - Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(aReg, cReg, pMask); | 568 | + Move<float, AscendC::Reg::MaskMergeMode::MERGING>(aReg, cReg, pMask); |
| 570 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pFull); | 569 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pFull); |
| 571 | - AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1); | 570 | + AscendC::Reg::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1); |
| 572 | } | 571 | } |
| 573 | for (uint16_t j = 0; j < unFoldLoopTimes; ++j) { | 572 | for (uint16_t j = 0; j < unFoldLoopTimes; ++j) { |
| 574 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 573 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 575 | LoadAlign(aReg, (__ubuf__ float*)unFoldSrc + i * outerLoopStride + j * innerLoopStride); | 574 | LoadAlign(aReg, (__ubuf__ float*)unFoldSrc + i * outerLoopStride + j * innerLoopStride); |
| 576 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pFull); | 575 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pFull); |
| 577 | - AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1); | 576 | + AscendC::Reg::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1); |
| 578 | } | 577 | } |
| 579 | - AscendC::MicroAPI::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0); | 578 | + AscendC::Reg::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0); |
| 580 | } | 579 | } |
| 581 | } | 580 | } |
| 582 | LastReduceSumSmallR(dstTensor, reduceSumTempTensor, aSize, foldPoint, outerLoopDstStride); | 581 | LastReduceSumSmallR(dstTensor, reduceSumTempTensor, aSize, foldPoint, outerLoopDstStride); |
| @@ -584,11 +583,11 @@ __aicore__ inline void LogSoftmaxGradOpsBase::LastReduceSum(const LocalTensor<fl | |||
| 584 | 583 | ||
| 585 | template <uint32_t RSize, int32_t TailCount = -1, int32_t Index = 0, int32_t Depth = 1> | 584 | template <uint32_t RSize, int32_t TailCount = -1, int32_t Index = 0, int32_t Depth = 1> |
| 586 | struct NlastDichotomyAdd { | 585 | struct NlastDichotomyAdd { |
| 587 | - __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA, | 586 | + __aicore__ static inline void LoadAndAccumulate(AscendC::Reg::RegTensor<float>& acc, __ubuf__ float*& srcA, |
| 588 | - __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask, | 587 | + __ubuf__ float*& srcB, AscendC::Reg::MaskReg& pMask, |
| 589 | uint32_t stride) | 588 | uint32_t stride) |
| 590 | { | 589 | { |
| 591 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 590 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 592 | __ubuf__ float* srcAOffset = srcA + stride * CONST_TWO; | 591 | __ubuf__ float* srcAOffset = srcA + stride * CONST_TWO; |
| 593 | __ubuf__ float* srcBOffset = srcB + stride * CONST_TWO; | 592 | __ubuf__ float* srcBOffset = srcB + stride * CONST_TWO; |
| 594 | if constexpr (TailCount <= 0) { | 593 | if constexpr (TailCount <= 0) { |
| @@ -596,13 +595,13 @@ struct NlastDichotomyAdd { | |||
| 596 | stride * CONST_TWO); | 595 | stride * CONST_TWO); |
| 597 | NlastDichotomyAdd<RSize / CONST_TWO>::LoadAndAccumulate(bReg, srcB, srcBOffset, pMask, stride * CONST_TWO); | 596 | NlastDichotomyAdd<RSize / CONST_TWO>::LoadAndAccumulate(bReg, srcB, srcBOffset, pMask, stride * CONST_TWO); |
| 598 | } | 597 | } |
| 599 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 598 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 600 | } | 599 | } |
| 601 | - __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA, | 600 | + __aicore__ static inline void LoadAndAccumulate(AscendC::Reg::RegTensor<float>& acc, __ubuf__ float*& srcA, |
| 602 | - __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask, | 601 | + __ubuf__ float*& srcB, AscendC::Reg::MaskReg& pMask, |
| 603 | uint32_t stride, uint32_t offset) | 602 | uint32_t stride, uint32_t offset) |
| 604 | { | 603 | { |
| 605 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 604 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 606 | __ubuf__ float* srcAOffset = srcA + stride * CONST_TWO; | 605 | __ubuf__ float* srcAOffset = srcA + stride * CONST_TWO; |
| 607 | __ubuf__ float* srcBOffset = srcB + stride * CONST_TWO; | 606 | __ubuf__ float* srcBOffset = srcB + stride * CONST_TWO; |
| 608 | if constexpr (TailCount <= 0) { | 607 | if constexpr (TailCount <= 0) { |
| @@ -616,56 +615,56 @@ struct NlastDichotomyAdd { | |||
| 616 | NlastDichotomyAdd<RSize / CONST_TWO, TailCount, Index + Depth, Depth * CONST_TWO>::LoadAndAccumulate( | 615 | NlastDichotomyAdd<RSize / CONST_TWO, TailCount, Index + Depth, Depth * CONST_TWO>::LoadAndAccumulate( |
| 617 | bReg, srcB, srcBOffset, pMask, stride * CONST_TWO, offset); | 616 | bReg, srcB, srcBOffset, pMask, stride * CONST_TWO, offset); |
| 618 | } | 617 | } |
| 619 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 618 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 620 | } | 619 | } |
| 621 | }; | 620 | }; |
| 622 | 621 | ||
| 623 | template <int32_t TailCount, int32_t Index, int32_t Depth> | 622 | template <int32_t TailCount, int32_t Index, int32_t Depth> |
| 624 | struct NlastDichotomyAdd<CONST_TWO, TailCount, Index, Depth> { | 623 | struct NlastDichotomyAdd<CONST_TWO, TailCount, Index, Depth> { |
| 625 | - __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA, | 624 | + __aicore__ static inline void LoadAndAccumulate(AscendC::Reg::RegTensor<float>& acc, __ubuf__ float*& srcA, |
| 626 | - __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask, | 625 | + __ubuf__ float*& srcB, AscendC::Reg::MaskReg& pMask, |
| 627 | uint32_t stride) | 626 | uint32_t stride) |
| 628 | { | 627 | { |
| 629 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 628 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 630 | LoadAlign(aReg, (__ubuf__ float*)srcA); | 629 | LoadAlign(aReg, (__ubuf__ float*)srcA); |
| 631 | LoadAlign(bReg, (__ubuf__ float*)srcB); | 630 | LoadAlign(bReg, (__ubuf__ float*)srcB); |
| 632 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 631 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 633 | } | 632 | } |
| 634 | - __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA, | 633 | + __aicore__ static inline void LoadAndAccumulate(AscendC::Reg::RegTensor<float>& acc, __ubuf__ float*& srcA, |
| 635 | - __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask, | 634 | + __ubuf__ float*& srcB, AscendC::Reg::MaskReg& pMask, |
| 636 | uint32_t stride, uint32_t offset) | 635 | uint32_t stride, uint32_t offset) |
| 637 | { | 636 | { |
| 638 | if constexpr (TailCount <= 0) { | 637 | if constexpr (TailCount <= 0) { |
| 639 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg; | 638 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg; |
| 640 | LoadAlign(aReg, (__ubuf__ float*)srcA); | 639 | LoadAlign(aReg, (__ubuf__ float*)srcA); |
| 641 | LoadAlign(bReg, (__ubuf__ float*)srcA + offset); | 640 | LoadAlign(bReg, (__ubuf__ float*)srcA + offset); |
| 642 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); | 641 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); |
| 643 | LoadAlign(bReg, (__ubuf__ float*)srcB); | 642 | LoadAlign(bReg, (__ubuf__ float*)srcB); |
| 644 | LoadAlign(cReg, (__ubuf__ float*)srcB + offset); | 643 | LoadAlign(cReg, (__ubuf__ float*)srcB + offset); |
| 645 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask); | 644 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask); |
| 646 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 645 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 647 | } else { | 646 | } else { |
| 648 | if constexpr (Index + Depth < TailCount) { | 647 | if constexpr (Index + Depth < TailCount) { |
| 649 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg; | 648 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg; |
| 650 | LoadAlign(aReg, (__ubuf__ float*)srcA); | 649 | LoadAlign(aReg, (__ubuf__ float*)srcA); |
| 651 | LoadAlign(bReg, (__ubuf__ float*)srcA + offset); | 650 | LoadAlign(bReg, (__ubuf__ float*)srcA + offset); |
| 652 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); | 651 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); |
| 653 | LoadAlign(bReg, (__ubuf__ float*)srcB); | 652 | LoadAlign(bReg, (__ubuf__ float*)srcB); |
| 654 | LoadAlign(cReg, (__ubuf__ float*)srcB + offset); | 653 | LoadAlign(cReg, (__ubuf__ float*)srcB + offset); |
| 655 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask); | 654 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask); |
| 656 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 655 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 657 | } else if constexpr (Index < TailCount) { | 656 | } else if constexpr (Index < TailCount) { |
| 658 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 657 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 659 | LoadAlign(aReg, (__ubuf__ float*)srcA); | 658 | LoadAlign(aReg, (__ubuf__ float*)srcA); |
| 660 | LoadAlign(bReg, (__ubuf__ float*)srcA + offset); | 659 | LoadAlign(bReg, (__ubuf__ float*)srcA + offset); |
| 661 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); | 660 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); |
| 662 | LoadAlign(bReg, (__ubuf__ float*)srcB); | 661 | LoadAlign(bReg, (__ubuf__ float*)srcB); |
| 663 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 662 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 664 | } else { | 663 | } else { |
| 665 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 664 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 666 | LoadAlign(aReg, (__ubuf__ float*)srcA); | 665 | LoadAlign(aReg, (__ubuf__ float*)srcA); |
| 667 | LoadAlign(bReg, (__ubuf__ float*)srcB); | 666 | LoadAlign(bReg, (__ubuf__ float*)srcB); |
| 668 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 667 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 669 | } | 668 | } |
| 670 | } | 669 | } |
| 671 | } | 670 | } |
| @@ -673,34 +672,34 @@ struct NlastDichotomyAdd<CONST_TWO, TailCount, Index, Depth> { | |||
| 673 | 672 | ||
| 674 | template <> | 673 | template <> |
| 675 | struct NlastDichotomyAdd<CONST_TWO> { | 674 | struct NlastDichotomyAdd<CONST_TWO> { |
| 676 | - __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA, | 675 | + __aicore__ static inline void LoadAndAccumulate(AscendC::Reg::RegTensor<float>& acc, __ubuf__ float*& srcA, |
| 677 | - __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask, | 676 | + __ubuf__ float*& srcB, AscendC::Reg::MaskReg& pMask, |
| 678 | uint32_t stride) | 677 | uint32_t stride) |
| 679 | { | 678 | { |
| 680 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 679 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 681 | LoadAlign(aReg, (__ubuf__ float*)srcA); | 680 | LoadAlign(aReg, (__ubuf__ float*)srcA); |
| 682 | LoadAlign(bReg, (__ubuf__ float*)srcB); | 681 | LoadAlign(bReg, (__ubuf__ float*)srcB); |
| 683 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 682 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 684 | } | 683 | } |
| 685 | - __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA, | 684 | + __aicore__ static inline void LoadAndAccumulate(AscendC::Reg::RegTensor<float>& acc, __ubuf__ float*& srcA, |
| 686 | - __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask, | 685 | + __ubuf__ float*& srcB, AscendC::Reg::MaskReg& pMask, |
| 687 | uint32_t stride, uint32_t offset) | 686 | uint32_t stride, uint32_t offset) |
| 688 | { | 687 | { |
| 689 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg; | 688 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg; |
| 690 | LoadAlign(aReg, (__ubuf__ float*)srcA); | 689 | LoadAlign(aReg, (__ubuf__ float*)srcA); |
| 691 | LoadAlign(bReg, (__ubuf__ float*)srcA + offset); | 690 | LoadAlign(bReg, (__ubuf__ float*)srcA + offset); |
| 692 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); | 691 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); |
| 693 | LoadAlign(bReg, (__ubuf__ float*)srcB); | 692 | LoadAlign(bReg, (__ubuf__ float*)srcB); |
| 694 | LoadAlign(cReg, (__ubuf__ float*)srcB + offset); | 693 | LoadAlign(cReg, (__ubuf__ float*)srcB + offset); |
| 695 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask); | 694 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(bReg, bReg, cReg, pMask); |
| 696 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); | 695 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(acc, aReg, bReg, pMask); |
| 697 | } | 696 | } |
| 698 | }; | 697 | }; |
| 699 | 698 | ||
| 700 | template <> | 699 | template <> |
| 701 | struct NlastDichotomyAdd<1> { | 700 | struct NlastDichotomyAdd<1> { |
| 702 | - __aicore__ static inline void LoadAndAccumulate(AscendC::MicroAPI::RegTensor<float>& acc, __ubuf__ float*& srcA, | 701 | + __aicore__ static inline void LoadAndAccumulate(AscendC::Reg::RegTensor<float>& acc, __ubuf__ float*& srcA, |
| 703 | - __ubuf__ float*& srcB, AscendC::MicroAPI::MaskReg& pMask, | 702 | + __ubuf__ float*& srcB, AscendC::Reg::MaskReg& pMask, |
| 704 | uint32_t stride) | 703 | uint32_t stride) |
| 705 | { | 704 | { |
| 706 | LoadAlign(acc, (__ubuf__ float*)srcA); | 705 | LoadAlign(acc, (__ubuf__ float*)srcA); |
| @@ -721,10 +720,10 @@ __aicore__ inline void LogSoftmaxGradOpsBase::NlastReduceSumSmallR(const LocalTe | |||
| 721 | __ubuf__ float* dst = (__ubuf__ float*)dstTensor.GetPhyAddr(); | 720 | __ubuf__ float* dst = (__ubuf__ float*)dstTensor.GetPhyAddr(); |
| 722 | __ubuf__ float* src = (__ubuf__ float*)srcTensor.GetPhyAddr(); | 721 | __ubuf__ float* src = (__ubuf__ float*)srcTensor.GetPhyAddr(); |
| 723 | uint32_t count = static_cast<uint32_t>(aSize); | 722 | uint32_t count = static_cast<uint32_t>(aSize); |
| 724 | - AscendC::MicroAPI::RegTensor<float> aReg; | 723 | + AscendC::Reg::RegTensor<float> aReg; |
| 725 | - AscendC::MicroAPI::MaskReg pMask; | 724 | + AscendC::Reg::MaskReg pMask; |
| 726 | for (uint16_t i = 0; i < loopTimes; ++i) { | 725 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 727 | - pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 726 | + pMask = AscendC::Reg::UpdateMask<float>(count); |
| 728 | LoadAlign(aReg, (__ubuf__ float*)src + i * VL_FP32); | 727 | LoadAlign(aReg, (__ubuf__ float*)src + i * VL_FP32); |
| 729 | StoreAlign((__ubuf__ float*)dst + i * VL_FP32, aReg, pMask); | 728 | StoreAlign((__ubuf__ float*)dst + i * VL_FP32, aReg, pMask); |
| 730 | } | 729 | } |
| @@ -736,12 +735,12 @@ __aicore__ inline void LogSoftmaxGradOpsBase::NlastReduceSumSmallR(const LocalTe | |||
| 736 | __ubuf__ float* srcA = (__ubuf__ float*)srcTensor.GetPhyAddr(); | 735 | __ubuf__ float* srcA = (__ubuf__ float*)srcTensor.GetPhyAddr(); |
| 737 | __ubuf__ float* srcB = (__ubuf__ float*)srcTensor.GetPhyAddr() + stride; | 736 | __ubuf__ float* srcB = (__ubuf__ float*)srcTensor.GetPhyAddr() + stride; |
| 738 | uint32_t count = static_cast<uint32_t>(aSize); | 737 | uint32_t count = static_cast<uint32_t>(aSize); |
| 739 | - AscendC::MicroAPI::RegTensor<float> aReg; | 738 | + AscendC::Reg::RegTensor<float> aReg; |
| 740 | - AscendC::MicroAPI::MaskReg pMask; | 739 | + AscendC::Reg::MaskReg pMask; |
| 741 | for (uint16_t i = 0; i < loopTimes; ++i) { | 740 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 742 | __ubuf__ float* curSrcA = srcA + i * VL_FP32; | 741 | __ubuf__ float* curSrcA = srcA + i * VL_FP32; |
| 743 | __ubuf__ float* curSrcB = srcB + i * VL_FP32; | 742 | __ubuf__ float* curSrcB = srcB + i * VL_FP32; |
| 744 | - pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 743 | + pMask = AscendC::Reg::UpdateMask<float>(count); |
| 745 | NlastDichotomyAdd<RSize>::LoadAndAccumulate(aReg, curSrcA, curSrcB, pMask, stride); | 744 | NlastDichotomyAdd<RSize>::LoadAndAccumulate(aReg, curSrcA, curSrcB, pMask, stride); |
| 746 | StoreAlign((__ubuf__ float*)dst + i * VL_FP32, aReg, pMask); | 745 | StoreAlign((__ubuf__ float*)dst + i * VL_FP32, aReg, pMask); |
| 747 | } | 746 | } |
| @@ -811,8 +810,8 @@ __aicore__ inline void LogSoftmaxGradOpsBase::NlastReduceSumLargeR(const LocalTe | |||
| 811 | __VEC_SCOPE__ | 810 | __VEC_SCOPE__ |
| 812 | { | 811 | { |
| 813 | uint32_t count = static_cast<uint32_t>(aSize); | 812 | uint32_t count = static_cast<uint32_t>(aSize); |
| 814 | - AscendC::MicroAPI::RegTensor<float> aReg; | 813 | + AscendC::Reg::RegTensor<float> aReg; |
| 815 | - AscendC::MicroAPI::MaskReg pMask; | 814 | + AscendC::Reg::MaskReg pMask; |
| 816 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { | 815 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { |
| 817 | pMask = plt_b32(count, POST_UPDATE); | 816 | pMask = plt_b32(count, POST_UPDATE); |
| 818 | for (uint16_t j = 0; j < mainFoldLoopTimes; ++j) { | 817 | for (uint16_t j = 0; j < mainFoldLoopTimes; ++j) { |
| @@ -908,14 +907,14 @@ __aicore__ inline void LogSoftmaxGradOpsBase::UpdateCache(const LocalTensor<floa | |||
| 908 | __ubuf__ float* cah = (__ubuf__ float*)dstTensor.GetPhyAddr() + cacheID * stride; | 907 | __ubuf__ float* cah = (__ubuf__ float*)dstTensor.GetPhyAddr() + cacheID * stride; |
| 909 | __ubuf__ float* src = (__ubuf__ float*)srcTensor.GetPhyAddr(); | 908 | __ubuf__ float* src = (__ubuf__ float*)srcTensor.GetPhyAddr(); |
| 910 | uint32_t sreg = static_cast<uint32_t>(count); | 909 | uint32_t sreg = static_cast<uint32_t>(count); |
| 911 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 910 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 912 | - AscendC::MicroAPI::MaskReg pMask; | 911 | + AscendC::Reg::MaskReg pMask; |
| 913 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { | 912 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { |
| 914 | - pMask = AscendC::MicroAPI::UpdateMask<float>(sreg); | 913 | + pMask = AscendC::Reg::UpdateMask<float>(sreg); |
| 915 | LoadAlign(aReg, (__ubuf__ float*)src + i * outerLoopStride); | 914 | LoadAlign(aReg, (__ubuf__ float*)src + i * outerLoopStride); |
| 916 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { | 915 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { |
| 917 | LoadAlign(bReg, (__ubuf__ float*)dst + i * outerLoopStride + j * innerLoopStride); | 916 | LoadAlign(bReg, (__ubuf__ float*)dst + i * outerLoopStride + j * innerLoopStride); |
| 918 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); | 917 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); |
| 919 | } | 918 | } |
| 920 | StoreAlign((__ubuf__ float*)cah + i * outerLoopStride, aReg, pMask); | 919 | StoreAlign((__ubuf__ float*)cah + i * outerLoopStride, aReg, pMask); |
| 921 | } | 920 | } |
| @@ -941,18 +940,18 @@ __aicore__ inline void LogSoftmaxGradOpsBase::Normalize(const LocalTensor<float> | |||
| 941 | __ubuf__ float* mean = (__ubuf__ float*)meanTensor.GetPhyAddr(); | 940 | __ubuf__ float* mean = (__ubuf__ float*)meanTensor.GetPhyAddr(); |
| 942 | __ubuf__ float* rstd = (__ubuf__ float*)rstdTensor.GetPhyAddr(); | 941 | __ubuf__ float* rstd = (__ubuf__ float*)rstdTensor.GetPhyAddr(); |
| 943 | uint32_t count; | 942 | uint32_t count; |
| 944 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg; | 943 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg; |
| 945 | - AscendC::MicroAPI::RegTensor<float> meanReg, rstdReg; | 944 | + AscendC::Reg::RegTensor<float> meanReg, rstdReg; |
| 946 | - AscendC::MicroAPI::MaskReg pMask; | 945 | + AscendC::Reg::MaskReg pMask; |
| 947 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { | 946 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { |
| 948 | count = static_cast<uint32_t>(colSize); | 947 | count = static_cast<uint32_t>(colSize); |
| 949 | - LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(meanReg, (__ubuf__ float*)mean + i); | 948 | + LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(meanReg, (__ubuf__ float*)mean + i); |
| 950 | - LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(rstdReg, (__ubuf__ float*)rstd + i); | 949 | + LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(rstdReg, (__ubuf__ float*)rstd + i); |
| 951 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { | 950 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { |
| 952 | - pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 951 | + pMask = AscendC::Reg::UpdateMask<float>(count); |
| 953 | LoadAlign(aReg, (__ubuf__ float*)src + i * outerLoopStride + j * innerLoopStride); | 952 | LoadAlign(aReg, (__ubuf__ float*)src + i * outerLoopStride + j * innerLoopStride); |
| 954 | - Sub<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(bReg, aReg, meanReg, pMask); | 953 | + Sub<float, AscendC::Reg::MaskMergeMode::ZEROING>(bReg, aReg, meanReg, pMask); |
| 955 | - Mul<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, bReg, rstdReg, pMask); | 954 | + Mul<float, AscendC::Reg::MaskMergeMode::ZEROING>(cReg, bReg, rstdReg, pMask); |
| 956 | StoreAlign((__ubuf__ float*)dst + i * outerLoopStride + j * innerLoopStride, cReg, pMask); | 955 | StoreAlign((__ubuf__ float*)dst + i * outerLoopStride + j * innerLoopStride, cReg, pMask); |
| 957 | } | 956 | } |
| 958 | } | 957 | } |
| @@ -63,10 +63,10 @@ private: | |||
| 63 | const int64_t rSize, const int64_t stride); | 63 | const int64_t rSize, const int64_t stride); |
| 64 | __aicore__ inline void CopyInX(const LocalTensor<T_in>& xInUb, int64_t ubA, int64_t offset); | 64 | __aicore__ inline void CopyInX(const LocalTensor<T_in>& xInUb, int64_t ubA, int64_t offset); |
| 65 | __aicore__ inline void CopyOutY(const LocalTensor<T_out>& yOutUb, int64_t ubA, int64_t offset); | 65 | __aicore__ inline void CopyOutY(const LocalTensor<T_out>& yOutUb, int64_t ubA, int64_t offset); |
| 66 | - __aicore__ inline void LoadTensorForDtypeTIn(__ubuf__ T_in* src, AscendC::MicroAPI::RegTensor<float>& dst, | 66 | + __aicore__ inline void LoadTensorForDtypeTIn(__ubuf__ T_in* src, AscendC::Reg::RegTensor<float>& dst, |
| 67 | - AscendC::MicroAPI::MaskReg& preg, uint32_t offset); | 67 | + AscendC::Reg::MaskReg& preg, uint32_t offset); |
| 68 | - __aicore__ inline void StoreTensorForDtypeTOut(__ubuf__ T_out* dst, AscendC::MicroAPI::RegTensor<float>& src, | 68 | + __aicore__ inline void StoreTensorForDtypeTOut(__ubuf__ T_out* dst, AscendC::Reg::RegTensor<float>& src, |
| 69 | - AscendC::MicroAPI::MaskReg& preg, uint32_t offset); | 69 | + AscendC::Reg::MaskReg& preg, uint32_t offset); |
| 70 | 70 | ||
| 71 | private: | 71 | private: |
| 72 | /* global memory address */ | 72 | /* global memory address */ |
| @@ -161,48 +161,48 @@ __aicore__ inline void LogSoftmaxV2AR<T_in, T_out>::FirstNormCompute(int64_t ubA | |||
| 161 | 161 | ||
| 162 | __VEC_SCOPE__ | 162 | __VEC_SCOPE__ |
| 163 | { | 163 | { |
| 164 | - AscendC::MicroAPI::RegTensor<float> vreg0; | 164 | + AscendC::Reg::RegTensor<float> vreg0; |
| 165 | - AscendC::MicroAPI::RegTensor<float> vreg1; | 165 | + AscendC::Reg::RegTensor<float> vreg1; |
| 166 | - AscendC::MicroAPI::RegTensor<float> vreg2; | 166 | + AscendC::Reg::RegTensor<float> vreg2; |
| 167 | - AscendC::MicroAPI::RegTensor<float> vreg3; | 167 | + AscendC::Reg::RegTensor<float> vreg3; |
| 168 | - AscendC::MicroAPI::RegTensor<float> vreg4; | 168 | + AscendC::Reg::RegTensor<float> vreg4; |
| 169 | - AscendC::MicroAPI::RegTensor<float> vreg5; | 169 | + AscendC::Reg::RegTensor<float> vreg5; |
| 170 | 170 | ||
| 171 | - AscendC::MicroAPI::MaskReg maskTail; | 171 | + AscendC::Reg::MaskReg maskTail; |
| 172 | - AscendC::MicroAPI::MaskReg maskSub; | 172 | + AscendC::Reg::MaskReg maskSub; |
| 173 | - AscendC::MicroAPI::MaskReg maskMax; | 173 | + AscendC::Reg::MaskReg maskMax; |
| 174 | - AscendC::MicroAPI::MaskReg maskAll; | 174 | + AscendC::Reg::MaskReg maskAll; |
| 175 | 175 | ||
| 176 | uint32_t sreg0 = tailUbBlockSize; | 176 | uint32_t sreg0 = tailUbBlockSize; |
| 177 | uint32_t sreg1 = VL_FP32; | 177 | uint32_t sreg1 = VL_FP32; |
| 178 | - maskTail = AscendC::MicroAPI::UpdateMask<float>(sreg0); | 178 | + maskTail = AscendC::Reg::UpdateMask<float>(sreg0); |
| 179 | - maskAll = AscendC::MicroAPI::UpdateMask<float>(sreg1); | 179 | + maskAll = AscendC::Reg::UpdateMask<float>(sreg1); |
| 180 | 180 | ||
| 181 | for (uint16_t k = 0; k < ubActualA; k++) { | 181 | for (uint16_t k = 0; k < ubActualA; k++) { |
| 182 | uint32_t tailAddrPtr = k * rAligned + VL_FP32 * (rLoopCount - 1); | 182 | uint32_t tailAddrPtr = k * rAligned + VL_FP32 * (rLoopCount - 1); |
| 183 | - AscendC::MicroAPI::Duplicate(vreg0, static_cast<float>(-INFINITY), maskAll); | 183 | + AscendC::Reg::Duplicate(vreg0, static_cast<float>(-INFINITY), maskAll); |
| 184 | LoadTensorForDtypeTIn(xInAddr, vreg1, maskTail, tailAddrPtr); | 184 | LoadTensorForDtypeTIn(xInAddr, vreg1, maskTail, tailAddrPtr); |
| 185 | - AscendC::MicroAPI::Max(vreg1, vreg0, vreg1, maskTail); | 185 | + AscendC::Reg::Max(vreg1, vreg0, vreg1, maskTail); |
| 186 | - AscendC::MicroAPI::Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(vreg0, vreg1, maskTail); | 186 | + AscendC::Reg::Move<float, AscendC::Reg::MaskMergeMode::MERGING>(vreg0, vreg1, maskTail); |
| 187 | 187 | ||
| 188 | uint32_t sreg2 = static_cast<uint32_t>(ubActualR - tailUbBlockSize); | 188 | uint32_t sreg2 = static_cast<uint32_t>(ubActualR - tailUbBlockSize); |
| 189 | for (uint16_t i = 0; i < rLoopCountTmp; i++) { | 189 | for (uint16_t i = 0; i < rLoopCountTmp; i++) { |
| 190 | - maskMax = AscendC::MicroAPI::UpdateMask<float>(sreg2); | 190 | + maskMax = AscendC::Reg::UpdateMask<float>(sreg2); |
| 191 | uint32_t addrPtr = i * VL_FP32 + k * rAligned; | 191 | uint32_t addrPtr = i * VL_FP32 + k * rAligned; |
| 192 | LoadTensorForDtypeTIn(xInAddr, vreg1, maskMax, addrPtr); | 192 | LoadTensorForDtypeTIn(xInAddr, vreg1, maskMax, addrPtr); |
| 193 | - AscendC::MicroAPI::Max(vreg0, vreg0, vreg1, maskMax); | 193 | + AscendC::Reg::Max(vreg0, vreg0, vreg1, maskMax); |
| 194 | } | 194 | } |
| 195 | - AscendC::MicroAPI::Reduce<AscendC::Reg::ReduceType::MAX>(vreg2, vreg0, maskAll); | 195 | + AscendC::Reg::Reduce<AscendC::Reg::ReduceType::MAX>(vreg2, vreg0, maskAll); |
| 196 | - AscendC::MicroAPI::Duplicate(vreg3, vreg2, maskAll); | 196 | + AscendC::Reg::Duplicate(vreg3, vreg2, maskAll); |
| 197 | 197 | ||
| 198 | // 求sub | 198 | // 求sub |
| 199 | uint32_t sreg3 = ubActualR; | 199 | uint32_t sreg3 = ubActualR; |
| 200 | for (uint16_t i = 0; i < rLoopCount; i++) { | 200 | for (uint16_t i = 0; i < rLoopCount; i++) { |
| 201 | - maskSub = AscendC::MicroAPI::UpdateMask<float>(sreg3); | 201 | + maskSub = AscendC::Reg::UpdateMask<float>(sreg3); |
| 202 | uint32_t addrPtr = i * VL_FP32 + k * rAligned; | 202 | uint32_t addrPtr = i * VL_FP32 + k * rAligned; |
| 203 | LoadTensorForDtypeTIn(xInAddr, vreg1, maskSub, addrPtr); | 203 | LoadTensorForDtypeTIn(xInAddr, vreg1, maskSub, addrPtr); |
| 204 | - AscendC::MicroAPI::Sub(vreg4, vreg1, vreg3, maskSub); | 204 | + AscendC::Reg::Sub(vreg4, vreg1, vreg3, maskSub); |
| 205 | - AscendC::MicroAPI::StoreAlign(((__ubuf__ float*)xTmpLocalAddr + addrPtr), vreg4, maskSub); | 205 | + AscendC::Reg::StoreAlign(((__ubuf__ float*)xTmpLocalAddr + addrPtr), vreg4, maskSub); |
| 206 | } | 206 | } |
| 207 | } | 207 | } |
| 208 | } | 208 | } |
| @@ -237,11 +237,10 @@ __aicore__ inline void LogSoftmaxV2AR<T_in, T_out>::SecondNormComputePost(const | |||
| 237 | __VEC_SCOPE__ | 237 | __VEC_SCOPE__ |
| 238 | { | 238 | { |
| 239 | uint32_t count = static_cast<uint32_t>(rSize); | 239 | uint32_t count = static_cast<uint32_t>(rSize); |
| 240 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg, dReg; | 240 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg, dReg; |
| 241 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 241 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 242 | - AscendC::MicroAPI::MaskReg | 242 | + AscendC::Reg::MaskReg pFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 243 | - pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 243 | + AscendC::Reg::MaskReg maskOri; |
| 244 | - AscendC::MicroAPI::MaskReg maskOri; | ||
| 245 | for (uint16_t i = 0; i < loopTimes; ++i) { | 244 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 246 | LoadAlign(aReg, (__ubuf__ float*)src + i * static_cast<uint32_t>(stride)); | 245 | LoadAlign(aReg, (__ubuf__ float*)src + i * static_cast<uint32_t>(stride)); |
| 247 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pMask); | 246 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pMask); |
| @@ -249,7 +248,7 @@ __aicore__ inline void LogSoftmaxV2AR<T_in, T_out>::SecondNormComputePost(const | |||
| 249 | Log(cReg, cReg, pFull); | 248 | Log(cReg, cReg, pFull); |
| 250 | uint32_t sreg0 = static_cast<uint32_t>(oriR); | 249 | uint32_t sreg0 = static_cast<uint32_t>(oriR); |
| 251 | for (uint16_t j = 0; j < rLoopCount; ++j) { | 250 | for (uint16_t j = 0; j < rLoopCount; ++j) { |
| 252 | - maskOri = AscendC::MicroAPI::UpdateMask<float>(sreg0); | 251 | + maskOri = AscendC::Reg::UpdateMask<float>(sreg0); |
| 253 | uint32_t addrPtr = j * VL_FP32 + i * oriRAligned; | 252 | uint32_t addrPtr = j * VL_FP32 + i * oriRAligned; |
| 254 | LoadAlign(dReg, ((__ubuf__ float*)oriSrc + addrPtr)); | 253 | LoadAlign(dReg, ((__ubuf__ float*)oriSrc + addrPtr)); |
| 255 | Sub(dReg, dReg, cReg, maskOri); | 254 | Sub(dReg, dReg, cReg, maskOri); |
| @@ -266,22 +265,21 @@ __aicore__ inline void LogSoftmaxV2AR<T_in, T_out>::SecondNormComputePost(const | |||
| 266 | __VEC_SCOPE__ | 265 | __VEC_SCOPE__ |
| 267 | { | 266 | { |
| 268 | uint32_t count = static_cast<uint32_t>(rSize - VL_FP32); | 267 | uint32_t count = static_cast<uint32_t>(rSize - VL_FP32); |
| 269 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg, dReg, eReg; | 268 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg, dReg, eReg; |
| 270 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 269 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 271 | - AscendC::MicroAPI::MaskReg | 270 | + AscendC::Reg::MaskReg pFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 272 | - pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 271 | + AscendC::Reg::MaskReg maskOri; |
| 273 | - AscendC::MicroAPI::MaskReg maskOri; | ||
| 274 | for (uint16_t i = 0; i < loopTimes; ++i) { | 272 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 275 | LoadAlign(aReg, (__ubuf__ float*)src0 + i * static_cast<uint32_t>(stride)); | 273 | LoadAlign(aReg, (__ubuf__ float*)src0 + i * static_cast<uint32_t>(stride)); |
| 276 | LoadAlign(bReg, (__ubuf__ float*)src1 + i * static_cast<uint32_t>(stride)); | 274 | LoadAlign(bReg, (__ubuf__ float*)src1 + i * static_cast<uint32_t>(stride)); |
| 277 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); | 275 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); |
| 278 | - Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(aReg, cReg, pMask); | 276 | + Move<float, AscendC::Reg::MaskMergeMode::MERGING>(aReg, cReg, pMask); |
| 279 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pFull); | 277 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pFull); |
| 280 | Duplicate(dReg, bReg, pFull); | 278 | Duplicate(dReg, bReg, pFull); |
| 281 | Log(dReg, dReg, pFull); | 279 | Log(dReg, dReg, pFull); |
| 282 | uint32_t sreg0 = static_cast<uint32_t>(oriR); | 280 | uint32_t sreg0 = static_cast<uint32_t>(oriR); |
| 283 | for (uint16_t j = 0; j < rLoopCount; ++j) { | 281 | for (uint16_t j = 0; j < rLoopCount; ++j) { |
| 284 | - maskOri = AscendC::MicroAPI::UpdateMask<float>(sreg0); | 282 | + maskOri = AscendC::Reg::UpdateMask<float>(sreg0); |
| 285 | uint32_t addrPtr = j * VL_FP32 + i * oriRAligned; | 283 | uint32_t addrPtr = j * VL_FP32 + i * oriRAligned; |
| 286 | LoadAlign(eReg, ((__ubuf__ float*)oriSrc + addrPtr)); | 284 | LoadAlign(eReg, ((__ubuf__ float*)oriSrc + addrPtr)); |
| 287 | Sub(eReg, eReg, dReg, maskOri); | 285 | Sub(eReg, eReg, dReg, maskOri); |
| @@ -319,11 +317,10 @@ __aicore__ inline void LogSoftmaxV2AR<T_in, T_out>::SecondNormComputeWithExpPost | |||
| 319 | __VEC_SCOPE__ | 317 | __VEC_SCOPE__ |
| 320 | { | 318 | { |
| 321 | uint32_t count = static_cast<uint32_t>(rSize); | 319 | uint32_t count = static_cast<uint32_t>(rSize); |
| 322 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg, dReg; | 320 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg, dReg; |
| 323 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 321 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 324 | - AscendC::MicroAPI::MaskReg | 322 | + AscendC::Reg::MaskReg pFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 325 | - pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 323 | + AscendC::Reg::MaskReg maskOri; |
| 326 | - AscendC::MicroAPI::MaskReg maskOri; | ||
| 327 | for (uint16_t i = 0; i < loopTimes; ++i) { | 324 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 328 | LoadAlign(aReg, (__ubuf__ float*)src + i * stride); | 325 | LoadAlign(aReg, (__ubuf__ float*)src + i * stride); |
| 329 | Exp(aReg, aReg, pMask); | 326 | Exp(aReg, aReg, pMask); |
| @@ -332,7 +329,7 @@ __aicore__ inline void LogSoftmaxV2AR<T_in, T_out>::SecondNormComputeWithExpPost | |||
| 332 | Log(cReg, cReg, pFull); | 329 | Log(cReg, cReg, pFull); |
| 333 | uint32_t sreg0 = static_cast<uint32_t>(oriR); | 330 | uint32_t sreg0 = static_cast<uint32_t>(oriR); |
| 334 | for (uint16_t j = 0; j < rLoopCount; ++j) { | 331 | for (uint16_t j = 0; j < rLoopCount; ++j) { |
| 335 | - maskOri = AscendC::MicroAPI::UpdateMask<float>(sreg0); | 332 | + maskOri = AscendC::Reg::UpdateMask<float>(sreg0); |
| 336 | uint32_t addrPtr = j * VL_FP32 + i * oriRAligned; | 333 | uint32_t addrPtr = j * VL_FP32 + i * oriRAligned; |
| 337 | LoadAlign(dReg, ((__ubuf__ float*)oriSrc + addrPtr)); | 334 | LoadAlign(dReg, ((__ubuf__ float*)oriSrc + addrPtr)); |
| 338 | Sub(dReg, dReg, cReg, maskOri); | 335 | Sub(dReg, dReg, cReg, maskOri); |
| @@ -348,24 +345,23 @@ __aicore__ inline void LogSoftmaxV2AR<T_in, T_out>::SecondNormComputeWithExpPost | |||
| 348 | __VEC_SCOPE__ | 345 | __VEC_SCOPE__ |
| 349 | { | 346 | { |
| 350 | uint32_t count = static_cast<uint32_t>(rSize - VL_FP32); | 347 | uint32_t count = static_cast<uint32_t>(rSize - VL_FP32); |
| 351 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg, dReg, eReg; | 348 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg, dReg, eReg; |
| 352 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 349 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 353 | - AscendC::MicroAPI::MaskReg | 350 | + AscendC::Reg::MaskReg pFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 354 | - pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 351 | + AscendC::Reg::MaskReg maskOri; |
| 355 | - AscendC::MicroAPI::MaskReg maskOri; | ||
| 356 | for (uint16_t i = 0; i < loopTimes; ++i) { | 352 | for (uint16_t i = 0; i < loopTimes; ++i) { |
| 357 | LoadAlign(aReg, (__ubuf__ float*)src0 + i * stride); | 353 | LoadAlign(aReg, (__ubuf__ float*)src0 + i * stride); |
| 358 | LoadAlign(bReg, (__ubuf__ float*)src1 + i * stride); | 354 | LoadAlign(bReg, (__ubuf__ float*)src1 + i * stride); |
| 359 | Exp(aReg, aReg, pFull); | 355 | Exp(aReg, aReg, pFull); |
| 360 | Exp(bReg, bReg, pMask); | 356 | Exp(bReg, bReg, pMask); |
| 361 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); | 357 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); |
| 362 | - Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(aReg, cReg, pMask); | 358 | + Move<float, AscendC::Reg::MaskMergeMode::MERGING>(aReg, cReg, pMask); |
| 363 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pFull); | 359 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pFull); |
| 364 | Duplicate(dReg, bReg, pFull); | 360 | Duplicate(dReg, bReg, pFull); |
| 365 | Log(dReg, dReg, pFull); | 361 | Log(dReg, dReg, pFull); |
| 366 | uint32_t sreg0 = static_cast<uint32_t>(oriR); | 362 | uint32_t sreg0 = static_cast<uint32_t>(oriR); |
| 367 | for (uint16_t j = 0; j < rLoopCount; ++j) { | 363 | for (uint16_t j = 0; j < rLoopCount; ++j) { |
| 368 | - maskOri = AscendC::MicroAPI::UpdateMask<float>(sreg0); | 364 | + maskOri = AscendC::Reg::UpdateMask<float>(sreg0); |
| 369 | uint32_t addrPtr = j * VL_FP32 + i * oriRAligned; | 365 | uint32_t addrPtr = j * VL_FP32 + i * oriRAligned; |
| 370 | LoadAlign(eReg, ((__ubuf__ float*)oriSrc + addrPtr)); | 366 | LoadAlign(eReg, ((__ubuf__ float*)oriSrc + addrPtr)); |
| 371 | Sub(eReg, eReg, dReg, maskOri); | 367 | Sub(eReg, eReg, dReg, maskOri); |
| @@ -424,42 +420,42 @@ __aicore__ inline void LogSoftmaxV2AR<T_in, T_out>::SecondNormCompute(const Loca | |||
| 424 | 420 | ||
| 425 | __VEC_SCOPE__ | 421 | __VEC_SCOPE__ |
| 426 | { | 422 | { |
| 427 | - AscendC::MicroAPI::MaskReg pFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 423 | + AscendC::Reg::MaskReg pFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 428 | - AscendC::MicroAPI::UnalignRegForStore UReg; | 424 | + AscendC::Reg::UnalignRegForStore UReg; |
| 429 | 425 | ||
| 430 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { | 426 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { |
| 431 | dst = (__ubuf__ float*)reduceSumTempTensor.GetPhyAddr() + i * outerLoopDstStride; | 427 | dst = (__ubuf__ float*)reduceSumTempTensor.GetPhyAddr() + i * outerLoopDstStride; |
| 432 | for (uint16_t j = 0; j < mainFoldLoopTimes; ++j) { | 428 | for (uint16_t j = 0; j < mainFoldLoopTimes; ++j) { |
| 433 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg, dReg; | 429 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg, dReg; |
| 434 | LoadAlign(aReg, (__ubuf__ float*)foldSrcA + i * outerLoopStride + j * innerLoopStride); | 430 | LoadAlign(aReg, (__ubuf__ float*)foldSrcA + i * outerLoopStride + j * innerLoopStride); |
| 435 | LoadAlign(bReg, (__ubuf__ float*)foldSrcB + i * outerLoopStride + j * innerLoopStride); | 431 | LoadAlign(bReg, (__ubuf__ float*)foldSrcB + i * outerLoopStride + j * innerLoopStride); |
| 436 | Exp(aReg, aReg, pFull); | 432 | Exp(aReg, aReg, pFull); |
| 437 | Exp(bReg, bReg, pFull); | 433 | Exp(bReg, bReg, pFull); |
| 438 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pFull); | 434 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pFull); |
| 439 | Reduce<AscendC::Reg::ReduceType::SUM>(dReg, cReg, pFull); | 435 | Reduce<AscendC::Reg::ReduceType::SUM>(dReg, cReg, pFull); |
| 440 | - AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, dReg, UReg, 1); | 436 | + AscendC::Reg::StoreUnAlign((__ubuf__ float*&)dst, dReg, UReg, 1); |
| 441 | } | 437 | } |
| 442 | for (uint16_t j = 0; j < tailFoldLoopTimes; ++j) { | 438 | for (uint16_t j = 0; j < tailFoldLoopTimes; ++j) { |
| 443 | uint32_t count = static_cast<uint32_t>(tailFoldElemCount); | 439 | uint32_t count = static_cast<uint32_t>(tailFoldElemCount); |
| 444 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg, cReg; | 440 | + AscendC::Reg::RegTensor<float> aReg, bReg, cReg; |
| 445 | - AscendC::MicroAPI::MaskReg pMask = AscendC::MicroAPI::UpdateMask<float>(count); | 441 | + AscendC::Reg::MaskReg pMask = AscendC::Reg::UpdateMask<float>(count); |
| 446 | LoadAlign(aReg, (__ubuf__ float*)tailSrcA + i * outerLoopStride + j * innerLoopStride); | 442 | LoadAlign(aReg, (__ubuf__ float*)tailSrcA + i * outerLoopStride + j * innerLoopStride); |
| 447 | LoadAlign(bReg, (__ubuf__ float*)tailSrcB + i * outerLoopStride + j * innerLoopStride); | 443 | LoadAlign(bReg, (__ubuf__ float*)tailSrcB + i * outerLoopStride + j * innerLoopStride); |
| 448 | Exp(aReg, aReg, pFull); | 444 | Exp(aReg, aReg, pFull); |
| 449 | Exp(bReg, bReg, pFull); | 445 | Exp(bReg, bReg, pFull); |
| 450 | - Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); | 446 | + Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(cReg, aReg, bReg, pMask); |
| 451 | - Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(aReg, cReg, pMask); | 447 | + Move<float, AscendC::Reg::MaskMergeMode::MERGING>(aReg, cReg, pMask); |
| 452 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pFull); | 448 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pFull); |
| 453 | - AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1); | 449 | + AscendC::Reg::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1); |
| 454 | } | 450 | } |
| 455 | for (uint16_t j = 0; j < unFoldLoopTimes; ++j) { | 451 | for (uint16_t j = 0; j < unFoldLoopTimes; ++j) { |
| 456 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 452 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 457 | LoadAlign(aReg, (__ubuf__ float*)unFoldSrc + i * outerLoopStride + j * innerLoopStride); | 453 | LoadAlign(aReg, (__ubuf__ float*)unFoldSrc + i * outerLoopStride + j * innerLoopStride); |
| 458 | Exp(aReg, aReg, pFull); | 454 | Exp(aReg, aReg, pFull); |
| 459 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pFull); | 455 | Reduce<AscendC::Reg::ReduceType::SUM>(bReg, aReg, pFull); |
| 460 | - AscendC::MicroAPI::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1); | 456 | + AscendC::Reg::StoreUnAlign((__ubuf__ float*&)dst, bReg, UReg, 1); |
| 461 | } | 457 | } |
| 462 | - AscendC::MicroAPI::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0); | 458 | + AscendC::Reg::StoreUnAlignPost((__ubuf__ float*&)dst, UReg, 0); |
| 463 | } | 459 | } |
| 464 | } | 460 | } |
| 465 | SecondNormComputePost(dstTensor, reduceSumTempTensor, srcTensor, aSize, foldPoint, outerLoopDstStride); | 461 | SecondNormComputePost(dstTensor, reduceSumTempTensor, srcTensor, aSize, foldPoint, outerLoopDstStride); |
| @@ -467,31 +463,30 @@ __aicore__ inline void LogSoftmaxV2AR<T_in, T_out>::SecondNormCompute(const Loca | |||
| 467 | 463 | ||
| 468 | template <typename T_in, typename T_out> | 464 | template <typename T_in, typename T_out> |
| 469 | __aicore__ inline void LogSoftmaxV2AR<T_in, T_out>::LoadTensorForDtypeTIn(__ubuf__ T_in* src, | 465 | __aicore__ inline void LogSoftmaxV2AR<T_in, T_out>::LoadTensorForDtypeTIn(__ubuf__ T_in* src, |
| 470 | - AscendC::MicroAPI::RegTensor<float>& dst, | 466 | + AscendC::Reg::RegTensor<float>& dst, |
| 471 | - AscendC::MicroAPI::MaskReg& preg, | 467 | + AscendC::Reg::MaskReg& preg, uint32_t offset) |
| 472 | - uint32_t offset) | ||
| 473 | { | 468 | { |
| 474 | if constexpr (IsSameType<T_in, float>::value) { | 469 | if constexpr (IsSameType<T_in, float>::value) { |
| 475 | - LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_NORM>(dst, src + offset); | 470 | + LoadAlign<float, AscendC::Reg::LoadDist::DIST_NORM>(dst, src + offset); |
| 476 | } else { | 471 | } else { |
| 477 | - AscendC::MicroAPI::RegTensor<T_in> xFp16; | 472 | + AscendC::Reg::RegTensor<T_in> xFp16; |
| 478 | - LoadAlign<T_in, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(xFp16, src + offset); | 473 | + LoadAlign<T_in, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(xFp16, src + offset); |
| 479 | Cast<float, T_in, castTraitFp16ToFp32>(dst, xFp16, preg); | 474 | Cast<float, T_in, castTraitFp16ToFp32>(dst, xFp16, preg); |
| 480 | } | 475 | } |
| 481 | } | 476 | } |
| 482 | 477 | ||
| 483 | template <typename T_in, typename T_out> | 478 | template <typename T_in, typename T_out> |
| 484 | __aicore__ inline void LogSoftmaxV2AR<T_in, T_out>::StoreTensorForDtypeTOut(__ubuf__ T_out* dst, | 479 | __aicore__ inline void LogSoftmaxV2AR<T_in, T_out>::StoreTensorForDtypeTOut(__ubuf__ T_out* dst, |
| 485 | - AscendC::MicroAPI::RegTensor<float>& src, | 480 | + AscendC::Reg::RegTensor<float>& src, |
| 486 | - AscendC::MicroAPI::MaskReg& preg, | 481 | + AscendC::Reg::MaskReg& preg, |
| 487 | uint32_t offset) | 482 | uint32_t offset) |
| 488 | { | 483 | { |
| 489 | if constexpr (IsSameType<T_out, float>::value) { | 484 | if constexpr (IsSameType<T_out, float>::value) { |
| 490 | - StoreAlign<T_out, AscendC::MicroAPI::StoreDist::DIST_NORM>(dst + offset, src, preg); | 485 | + StoreAlign<T_out, AscendC::Reg::StoreDist::DIST_NORM>(dst + offset, src, preg); |
| 491 | } else { | 486 | } else { |
| 492 | - AscendC::MicroAPI::RegTensor<T_out> xFp16; | 487 | + AscendC::Reg::RegTensor<T_out> xFp16; |
| 493 | Cast<T_out, float, castTraitFp32ToFp16>(xFp16, src, preg); | 488 | Cast<T_out, float, castTraitFp32ToFp16>(xFp16, src, preg); |
| 494 | - StoreAlign<T_out, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(dst + offset, xFp16, preg); | 489 | + StoreAlign<T_out, AscendC::Reg::StoreDist::DIST_PACK_B32>(dst + offset, xFp16, preg); |
| 495 | } | 490 | } |
| 496 | } | 491 | } |
| 497 | 492 | ||
| @@ -270,50 +270,50 @@ __aicore__ inline void LogSoftmaxV2ArRecompute<Tx, Ty>::CalculateMaxVF(__ubuf__ | |||
| 270 | { | 270 | { |
| 271 | __VEC_SCOPE__ | 271 | __VEC_SCOPE__ |
| 272 | { | 272 | { |
| 273 | - AscendC::MicroAPI::RegTensor<float> vreg1, vreg2, maxReg; | 273 | + AscendC::Reg::RegTensor<float> vreg1, vreg2, maxReg; |
| 274 | - AscendC::MicroAPI::RegTensor<Tx> vreg3; | 274 | + AscendC::Reg::RegTensor<Tx> vreg3; |
| 275 | - AscendC::MicroAPI::MaskReg maskTail, maskOne, maskFull; | 275 | + AscendC::Reg::MaskReg maskTail, maskOne, maskFull; |
| 276 | 276 | ||
| 277 | uint32_t constOne = 1; | 277 | uint32_t constOne = 1; |
| 278 | - maskFull = AscendC::MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 278 | + maskFull = AscendC::Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 279 | - maskOne = AscendC::MicroAPI::UpdateMask<float>(constOne); // 用于读写1个元素 | 279 | + maskOne = AscendC::Reg::UpdateMask<float>(constOne); // 用于读写1个元素 |
| 280 | 280 | ||
| 281 | uint16_t repeatTimes = CeilDivision(ubFactor, VL_FP32); | 281 | uint16_t repeatTimes = CeilDivision(ubFactor, VL_FP32); |
| 282 | uint16_t repeatTimesTmp = repeatTimes - 1; | 282 | uint16_t repeatTimesTmp = repeatTimes - 1; |
| 283 | 283 | ||
| 284 | // 尾块处理 | 284 | // 尾块处理 |
| 285 | uint32_t tail = static_cast<uint32_t>(ubFactor - VL_FP32 * (repeatTimes - 1)); | 285 | uint32_t tail = static_cast<uint32_t>(ubFactor - VL_FP32 * (repeatTimes - 1)); |
| 286 | - maskTail = AscendC::MicroAPI::UpdateMask<float>(tail); | 286 | + maskTail = AscendC::Reg::UpdateMask<float>(tail); |
| 287 | uint16_t j = repeatTimes - 1; | 287 | uint16_t j = repeatTimes - 1; |
| 288 | auto xAddr = xPtr + j * VL_FP32; | 288 | auto xAddr = xPtr + j * VL_FP32; |
| 289 | 289 | ||
| 290 | - AscendC::MicroAPI::Duplicate(maxReg, CONST_FP32_MIN); | 290 | + AscendC::Reg::Duplicate(maxReg, CONST_FP32_MIN); |
| 291 | 291 | ||
| 292 | if constexpr (xToFp32_) { | 292 | if constexpr (xToFp32_) { |
| 293 | - AscendC::MicroAPI::LoadAlign<Tx, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg3, xAddr); | 293 | + AscendC::Reg::LoadAlign<Tx, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vreg3, xAddr); |
| 294 | - AscendC::MicroAPI::Cast<float, Tx, castTraitFp16ToFp32>(vreg1, vreg3, maskTail); | 294 | + AscendC::Reg::Cast<float, Tx, castTraitFp16ToFp32>(vreg1, vreg3, maskTail); |
| 295 | } else { | 295 | } else { |
| 296 | - AscendC::MicroAPI::LoadAlign(vreg1, xAddr); | 296 | + AscendC::Reg::LoadAlign(vreg1, xAddr); |
| 297 | } | 297 | } |
| 298 | - AscendC::MicroAPI::Max(vreg1, maxReg, vreg1, maskTail); | 298 | + AscendC::Reg::Max(vreg1, maxReg, vreg1, maskTail); |
| 299 | - AscendC::MicroAPI::Move<float, AscendC::MicroAPI::MaskMergeMode::MERGING>(maxReg, vreg1, maskTail); | 299 | + AscendC::Reg::Move<float, AscendC::Reg::MaskMergeMode::MERGING>(maxReg, vreg1, maskTail); |
| 300 | 300 | ||
| 301 | // 整块处理 | 301 | // 整块处理 |
| 302 | for (uint16_t j = 0; j < repeatTimesTmp; j++) { | 302 | for (uint16_t j = 0; j < repeatTimesTmp; j++) { |
| 303 | auto xAddr = xPtr + j * VL_FP32; | 303 | auto xAddr = xPtr + j * VL_FP32; |
| 304 | if constexpr (xToFp32_) { | 304 | if constexpr (xToFp32_) { |
| 305 | - AscendC::MicroAPI::LoadAlign<Tx, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg3, xAddr); | 305 | + AscendC::Reg::LoadAlign<Tx, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vreg3, xAddr); |
| 306 | - AscendC::MicroAPI::Cast<float, Tx, castTraitFp16ToFp32>(vreg1, vreg3, maskFull); | 306 | + AscendC::Reg::Cast<float, Tx, castTraitFp16ToFp32>(vreg1, vreg3, maskFull); |
| 307 | } else { | 307 | } else { |
| 308 | - AscendC::MicroAPI::LoadAlign(vreg1, xAddr); | 308 | + AscendC::Reg::LoadAlign(vreg1, xAddr); |
| 309 | } | 309 | } |
| 310 | - AscendC::MicroAPI::Max(maxReg, maxReg, vreg1, maskFull); | 310 | + AscendC::Reg::Max(maxReg, maxReg, vreg1, maskFull); |
| 311 | } | 311 | } |
| 312 | - AscendC::MicroAPI::LoadAlign(vreg2, xMaxPtr); | 312 | + AscendC::Reg::LoadAlign(vreg2, xMaxPtr); |
| 313 | 313 | ||
| 314 | - AscendC::MicroAPI::Reduce<AscendC::Reg::ReduceType::MAX>(maxReg, maxReg, maskFull); | 314 | + AscendC::Reg::Reduce<AscendC::Reg::ReduceType::MAX>(maxReg, maxReg, maskFull); |
| 315 | - AscendC::MicroAPI::Max(maxReg, maxReg, vreg2, maskOne); | 315 | + AscendC::Reg::Max(maxReg, maxReg, vreg2, maskOne); |
| 316 | - AscendC::MicroAPI::StoreAlign(xMaxPtr, maxReg, maskOne); | 316 | + AscendC::Reg::StoreAlign(xMaxPtr, maxReg, maskOne); |
| 317 | } | 317 | } |
| 318 | } | 318 | } |
| 319 | 319 | ||
| @@ -325,39 +325,39 @@ __aicore__ inline void LogSoftmaxV2ArRecompute<Tx, Ty>::CalculateOutVF(__ubuf__ | |||
| 325 | { | 325 | { |
| 326 | __VEC_SCOPE__ | 326 | __VEC_SCOPE__ |
| 327 | { | 327 | { |
| 328 | - AscendC::MicroAPI::RegTensor<Tx> vreg0; | 328 | + AscendC::Reg::RegTensor<Tx> vreg0; |
| 329 | - AscendC::MicroAPI::RegTensor<float> sumReg, lnReg, maxReg, vreg1, vreg2, vreg3; | 329 | + AscendC::Reg::RegTensor<float> sumReg, lnReg, maxReg, vreg1, vreg2, vreg3; |
| 330 | - AscendC::MicroAPI::RegTensor<Ty> vreg4; | 330 | + AscendC::Reg::RegTensor<Ty> vreg4; |
| 331 | - AscendC::MicroAPI::MaskReg mask; | 331 | + AscendC::Reg::MaskReg mask; |
| 332 | - AscendC::MicroAPI::MaskReg maskAll = MicroAPI::CreateMask<float, AscendC::MicroAPI::MaskPattern::ALL>(); | 332 | + AscendC::Reg::MaskReg maskAll = Reg::CreateMask<float, AscendC::Reg::MaskPattern::ALL>(); |
| 333 | 333 | ||
| 334 | uint32_t width = ubFactor; | 334 | uint32_t width = ubFactor; |
| 335 | uint16_t repeatTimes = CeilDivision(ubFactor, VL_FP32); | 335 | uint16_t repeatTimes = CeilDivision(ubFactor, VL_FP32); |
| 336 | 336 | ||
| 337 | - AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(maxReg, xMaxPtr); | 337 | + AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(maxReg, xMaxPtr); |
| 338 | - AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(sumReg, xSumPtr); | 338 | + AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(sumReg, xSumPtr); |
| 339 | - AscendC::MicroAPI::Ln(lnReg, sumReg, maskAll); | 339 | + AscendC::Reg::Ln(lnReg, sumReg, maskAll); |
| 340 | 340 | ||
| 341 | for (uint16_t j = 0; j < repeatTimes; j++) { | 341 | for (uint16_t j = 0; j < repeatTimes; j++) { |
| 342 | - mask = AscendC::MicroAPI::UpdateMask<float>(width); | 342 | + mask = AscendC::Reg::UpdateMask<float>(width); |
| 343 | auto xAddr = xPtr + j * VL_FP32; | 343 | auto xAddr = xPtr + j * VL_FP32; |
| 344 | auto yAddr = yPtr + j * VL_FP32; | 344 | auto yAddr = yPtr + j * VL_FP32; |
| 345 | 345 | ||
| 346 | if constexpr (xToFp32_) { | 346 | if constexpr (xToFp32_) { |
| 347 | - AscendC::MicroAPI::LoadAlign<Tx, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, xAddr); | 347 | + AscendC::Reg::LoadAlign<Tx, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vreg0, xAddr); |
| 348 | - AscendC::MicroAPI::Cast<float, Tx, castTraitFp16ToFp32>(vreg1, vreg0, mask); | 348 | + AscendC::Reg::Cast<float, Tx, castTraitFp16ToFp32>(vreg1, vreg0, mask); |
| 349 | } else { | 349 | } else { |
| 350 | - AscendC::MicroAPI::LoadAlign(vreg1, xAddr); | 350 | + AscendC::Reg::LoadAlign(vreg1, xAddr); |
| 351 | } | 351 | } |
| 352 | 352 | ||
| 353 | - AscendC::MicroAPI::Sub(vreg2, vreg1, maxReg, mask); | 353 | + AscendC::Reg::Sub(vreg2, vreg1, maxReg, mask); |
| 354 | - AscendC::MicroAPI::Sub(vreg3, vreg2, lnReg, mask); | 354 | + AscendC::Reg::Sub(vreg3, vreg2, lnReg, mask); |
| 355 | 355 | ||
| 356 | if constexpr (yToFp32_) { | 356 | if constexpr (yToFp32_) { |
| 357 | - AscendC::MicroAPI::StoreAlign(yAddr, vreg3, mask); | 357 | + AscendC::Reg::StoreAlign(yAddr, vreg3, mask); |
| 358 | } else { | 358 | } else { |
| 359 | - AscendC::MicroAPI::Cast<Ty, float, castTraitFp32ToFp16>(vreg4, vreg3, mask); | 359 | + AscendC::Reg::Cast<Ty, float, castTraitFp32ToFp16>(vreg4, vreg3, mask); |
| 360 | - AscendC::MicroAPI::StoreAlign<Ty, AscendC::MicroAPI::StoreDist::DIST_PACK_B32>(yAddr, vreg4, mask); | 360 | + AscendC::Reg::StoreAlign<Ty, AscendC::Reg::StoreDist::DIST_PACK_B32>(yAddr, vreg4, mask); |
| 361 | } | 361 | } |
| 362 | } | 362 | } |
| 363 | } | 363 | } |
| @@ -371,30 +371,30 @@ __aicore__ inline void LogSoftmaxV2ArRecompute<Tx, Ty>::MainBlockCastSubExpVF(__ | |||
| 371 | { | 371 | { |
| 372 | __VEC_SCOPE__ | 372 | __VEC_SCOPE__ |
| 373 | { | 373 | { |
| 374 | - AscendC::MicroAPI::RegTensor<Tx> vreg0; | 374 | + AscendC::Reg::RegTensor<Tx> vreg0; |
| 375 | - AscendC::MicroAPI::RegTensor<float> vreg1, vreg2, vreg3, maxReg; | 375 | + AscendC::Reg::RegTensor<float> vreg1, vreg2, vreg3, maxReg; |
| 376 | - AscendC::MicroAPI::MaskReg mask; | 376 | + AscendC::Reg::MaskReg mask; |
| 377 | 377 | ||
| 378 | uint32_t width = ubFactor; | 378 | uint32_t width = ubFactor; |
| 379 | uint16_t repeatTimes = CeilDivision(ubFactor, VL_FP32); | 379 | uint16_t repeatTimes = CeilDivision(ubFactor, VL_FP32); |
| 380 | 380 | ||
| 381 | - AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(maxReg, xMaxPtr); | 381 | + AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(maxReg, xMaxPtr); |
| 382 | for (uint16_t j = 0; j < repeatTimes; j++) { | 382 | for (uint16_t j = 0; j < repeatTimes; j++) { |
| 383 | - mask = AscendC::MicroAPI::UpdateMask<float>(width); | 383 | + mask = AscendC::Reg::UpdateMask<float>(width); |
| 384 | auto xAddr = xPtr + j * VL_FP32; | 384 | auto xAddr = xPtr + j * VL_FP32; |
| 385 | auto xFp32Addr = xFp32Ptr + j * VL_FP32; | 385 | auto xFp32Addr = xFp32Ptr + j * VL_FP32; |
| 386 | 386 | ||
| 387 | if constexpr (xToFp32_) { | 387 | if constexpr (xToFp32_) { |
| 388 | - AscendC::MicroAPI::LoadAlign<Tx, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, xAddr); | 388 | + AscendC::Reg::LoadAlign<Tx, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vreg0, xAddr); |
| 389 | - AscendC::MicroAPI::Cast<float, Tx, castTraitFp16ToFp32>(vreg1, vreg0, mask); | 389 | + AscendC::Reg::Cast<float, Tx, castTraitFp16ToFp32>(vreg1, vreg0, mask); |
| 390 | } else { | 390 | } else { |
| 391 | - AscendC::MicroAPI::LoadAlign(vreg1, xAddr); | 391 | + AscendC::Reg::LoadAlign(vreg1, xAddr); |
| 392 | } | 392 | } |
| 393 | 393 | ||
| 394 | - AscendC::MicroAPI::Sub(vreg2, vreg1, maxReg, mask); | 394 | + AscendC::Reg::Sub(vreg2, vreg1, maxReg, mask); |
| 395 | - AscendC::MicroAPI::Exp(vreg3, vreg2, mask); | 395 | + AscendC::Reg::Exp(vreg3, vreg2, mask); |
| 396 | 396 | ||
| 397 | - AscendC::MicroAPI::StoreAlign(xFp32Addr, vreg3, mask); | 397 | + AscendC::Reg::StoreAlign(xFp32Addr, vreg3, mask); |
| 398 | } | 398 | } |
| 399 | } | 399 | } |
| 400 | } | 400 | } |
| @@ -407,33 +407,33 @@ __aicore__ inline void LogSoftmaxV2ArRecompute<Tx, Ty>::FoldBlockCastSubExpVF(__ | |||
| 407 | { | 407 | { |
| 408 | __VEC_SCOPE__ | 408 | __VEC_SCOPE__ |
| 409 | { | 409 | { |
| 410 | - AscendC::MicroAPI::RegTensor<Tx> vreg0; | 410 | + AscendC::Reg::RegTensor<Tx> vreg0; |
| 411 | - AscendC::MicroAPI::RegTensor<float> vreg1, vreg2, vreg3, maxReg, dstReg; | 411 | + AscendC::Reg::RegTensor<float> vreg1, vreg2, vreg3, maxReg, dstReg; |
| 412 | - AscendC::MicroAPI::MaskReg mask; | 412 | + AscendC::Reg::MaskReg mask; |
| 413 | 413 | ||
| 414 | uint32_t width = ubFactor; | 414 | uint32_t width = ubFactor; |
| 415 | uint16_t repeatTimes = CeilDivision(ubFactor, VL_FP32); | 415 | uint16_t repeatTimes = CeilDivision(ubFactor, VL_FP32); |
| 416 | 416 | ||
| 417 | - AscendC::MicroAPI::LoadAlign<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(maxReg, xMaxPtr); | 417 | + AscendC::Reg::LoadAlign<float, AscendC::Reg::LoadDist::DIST_BRC_B32>(maxReg, xMaxPtr); |
| 418 | for (uint16_t j = 0; j < repeatTimes; j++) { | 418 | for (uint16_t j = 0; j < repeatTimes; j++) { |
| 419 | - mask = AscendC::MicroAPI::UpdateMask<float>(width); | 419 | + mask = AscendC::Reg::UpdateMask<float>(width); |
| 420 | auto xAddr = xPtr + j * VL_FP32; | 420 | auto xAddr = xPtr + j * VL_FP32; |
| 421 | auto dstAddr = dstPtr + j * VL_FP32; | 421 | auto dstAddr = dstPtr + j * VL_FP32; |
| 422 | 422 | ||
| 423 | if constexpr (xToFp32_) { | 423 | if constexpr (xToFp32_) { |
| 424 | - AscendC::MicroAPI::LoadAlign<Tx, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(vreg0, xAddr); | 424 | + AscendC::Reg::LoadAlign<Tx, AscendC::Reg::LoadDist::DIST_UNPACK_B16>(vreg0, xAddr); |
| 425 | - AscendC::MicroAPI::Cast<float, Tx, castTraitFp16ToFp32>(vreg1, vreg0, mask); | 425 | + AscendC::Reg::Cast<float, Tx, castTraitFp16ToFp32>(vreg1, vreg0, mask); |
| 426 | } else { | 426 | } else { |
| 427 | - AscendC::MicroAPI::LoadAlign(vreg1, xAddr); | 427 | + AscendC::Reg::LoadAlign(vreg1, xAddr); |
| 428 | } | 428 | } |
| 429 | 429 | ||
| 430 | - AscendC::MicroAPI::Sub(vreg2, vreg1, maxReg, mask); | 430 | + AscendC::Reg::Sub(vreg2, vreg1, maxReg, mask); |
| 431 | - AscendC::MicroAPI::Exp(vreg3, vreg2, mask); | 431 | + AscendC::Reg::Exp(vreg3, vreg2, mask); |
| 432 | 432 | ||
| 433 | - AscendC::MicroAPI::LoadAlign(dstReg, dstAddr); | 433 | + AscendC::Reg::LoadAlign(dstReg, dstAddr); |
| 434 | - AscendC::MicroAPI::Add(dstReg, dstReg, vreg3, mask); | 434 | + AscendC::Reg::Add(dstReg, dstReg, vreg3, mask); |
| 435 | 435 | ||
| 436 | - AscendC::MicroAPI::StoreAlign(dstAddr, dstReg, mask); | 436 | + AscendC::Reg::StoreAlign(dstAddr, dstReg, mask); |
| 437 | } | 437 | } |
| 438 | } | 438 | } |
| 439 | } | 439 | } |
| @@ -462,16 +462,16 @@ __aicore__ inline void LogSoftmaxV2ArRecompute<Tx, Ty>::UpdateCache(const LocalT | |||
| 462 | __VEC_SCOPE__ | 462 | __VEC_SCOPE__ |
| 463 | { | 463 | { |
| 464 | uint32_t sreg = static_cast<uint32_t>(count); | 464 | uint32_t sreg = static_cast<uint32_t>(count); |
| 465 | - AscendC::MicroAPI::RegTensor<float> aReg, bReg; | 465 | + AscendC::Reg::RegTensor<float> aReg, bReg; |
| 466 | - AscendC::MicroAPI::MaskReg pMask; | 466 | + AscendC::Reg::MaskReg pMask; |
| 467 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { | 467 | for (uint16_t i = 0; i < outerLoopTimes; ++i) { |
| 468 | - pMask = AscendC::MicroAPI::UpdateMask<float>(sreg); | 468 | + pMask = AscendC::Reg::UpdateMask<float>(sreg); |
| 469 | - AscendC::MicroAPI::LoadAlign(aReg, (__ubuf__ float*)src + i * outerLoopStride); | 469 | + AscendC::Reg::LoadAlign(aReg, (__ubuf__ float*)src + i * outerLoopStride); |
| 470 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { | 470 | for (uint16_t j = 0; j < innerLoopTimes; ++j) { |
| 471 | - AscendC::MicroAPI::LoadAlign(bReg, (__ubuf__ float*)dst + i * outerLoopStride + j * innerLoopStride); | 471 | + AscendC::Reg::LoadAlign(bReg, (__ubuf__ float*)dst + i * outerLoopStride + j * innerLoopStride); |
| 472 | - AscendC::MicroAPI::Add<float, AscendC::MicroAPI::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); | 472 | + AscendC::Reg::Add<float, AscendC::Reg::MaskMergeMode::ZEROING>(aReg, aReg, bReg, pMask); |
| 473 | } | 473 | } |
| 474 | - AscendC::MicroAPI::StoreAlign((__ubuf__ float*)cache + i * outerLoopStride, aReg, pMask); | 474 | + AscendC::Reg::StoreAlign((__ubuf__ float*)cache + i * outerLoopStride, aReg, pMask); |
| 475 | } | 475 | } |
| 476 | } | 476 | } |
| 477 | } | 477 | } |
| @@ -30,11 +30,11 @@ namespace LogSoftmaxV2Ops { | |||
| 30 | using namespace AscendC; | 30 | using namespace AscendC; |
| 31 | using namespace SoftmaxV2Ops; | 31 | using namespace SoftmaxV2Ops; |
| 32 | 32 | ||
| 33 | -using AscendC::MicroAPI::LoadDist; | 33 | +using AscendC::Reg::LoadDist; |
| 34 | -using AscendC::MicroAPI::MaskMergeMode; | 34 | +using AscendC::Reg::MaskMergeMode; |
| 35 | -using AscendC::MicroAPI::MaskReg; | 35 | +using AscendC::Reg::MaskReg; |
| 36 | -using AscendC::MicroAPI::RegTensor; | 36 | +using AscendC::Reg::RegTensor; |
| 37 | -using AscendC::MicroAPI::StoreDist; | 37 | +using AscendC::Reg::StoreDist; |
| 38 | 38 | ||
| 39 | template <typename Tx, typename Ty> | 39 | template <typename Tx, typename Ty> |
| 40 | class LogSoftmaxV2ArSmallR : public SoftmaxV2OpsBase { | 40 | class LogSoftmaxV2ArSmallR : public SoftmaxV2OpsBase { |
| @@ -119,29 +119,29 @@ private: | |||
| 119 | 119 | ||
| 120 | __VEC_SCOPE__ | 120 | __VEC_SCOPE__ |
| 121 | { | 121 | { |
| 122 | - MicroAPI::RegTensor<Tx> reg0; | 122 | + Reg::RegTensor<Tx> reg0; |
| 123 | - MicroAPI::RegTensor<float> reg1, reg2, reg3, maxReg; | 123 | + Reg::RegTensor<float> reg1, reg2, reg3, maxReg; |
| 124 | - MicroAPI::MaskReg mask; | 124 | + Reg::MaskReg mask; |
| 125 | uint32_t width = curTileA0Len; | 125 | uint32_t width = curTileA0Len; |
| 126 | uint32_t tileA0LenLocal = tl_->tileA0Len; | 126 | uint32_t tileA0LenLocal = tl_->tileA0Len; |
| 127 | 127 | ||
| 128 | for (uint16_t j = 0; j < aLoopTimes; j++) { | 128 | for (uint16_t j = 0; j < aLoopTimes; j++) { |
| 129 | - mask = MicroAPI::UpdateMask<uint32_t>(width); | 129 | + mask = Reg::UpdateMask<uint32_t>(width); |
| 130 | - MicroAPI::Duplicate<float>(maxReg, static_cast<float>(-INFINITY)); | 130 | + Reg::Duplicate<float>(maxReg, static_cast<float>(-INFINITY)); |
| 131 | 131 | ||
| 132 | for (uint16_t i = 0; i < rLoopTimes; i++) { | 132 | for (uint16_t i = 0; i < rLoopTimes; i++) { |
| 133 | uint32_t offset = j * VL_FP32 + i * tileA0LenLocal; | 133 | uint32_t offset = j * VL_FP32 + i * tileA0LenLocal; |
| 134 | LoadTensorForDtypeT(xAddr, reg1, mask, offset); | 134 | LoadTensorForDtypeT(xAddr, reg1, mask, offset); |
| 135 | - MicroAPI::Max(maxReg, maxReg, reg1, mask); | 135 | + Reg::Max(maxReg, maxReg, reg1, mask); |
| 136 | } | 136 | } |
| 137 | 137 | ||
| 138 | for (uint16_t i = 0; i < rLoopTimes; i++) { | 138 | for (uint16_t i = 0; i < rLoopTimes; i++) { |
| 139 | uint32_t offset = j * VL_FP32 + i * tileA0LenLocal; | 139 | uint32_t offset = j * VL_FP32 + i * tileA0LenLocal; |
| 140 | LoadTensorForDtypeT(xAddr, reg2, mask, offset); | 140 | LoadTensorForDtypeT(xAddr, reg2, mask, offset); |
| 141 | - MicroAPI::Sub(reg2, reg2, maxReg, mask); | 141 | + Reg::Sub(reg2, reg2, maxReg, mask); |
| 142 | - MicroAPI::StoreAlign(tmpAddr2 + offset, reg2, mask); | 142 | + Reg::StoreAlign(tmpAddr2 + offset, reg2, mask); |
| 143 | - MicroAPI::Exp(reg2, reg2, mask); | 143 | + Reg::Exp(reg2, reg2, mask); |
| 144 | - MicroAPI::StoreAlign(tmpAddr + offset, reg2, mask); | 144 | + Reg::StoreAlign(tmpAddr + offset, reg2, mask); |
| 145 | } | 145 | } |
| 146 | } | 146 | } |
| 147 | } | 147 | } |
| @@ -167,31 +167,30 @@ private: | |||
| 167 | 167 | ||
| 168 | __VEC_SCOPE__ | 168 | __VEC_SCOPE__ |
| 169 | { | 169 | { |
| 170 | - MicroAPI::RegTensor<float> reg1; | 170 | + Reg::RegTensor<float> reg1; |
| 171 | - MicroAPI::RegTensor<float> sumReg, lnReg; | 171 | + Reg::RegTensor<float> sumReg, lnReg; |
| 172 | - MicroAPI::MaskReg mask; | 172 | + Reg::MaskReg mask; |
| 173 | 173 | ||
| 174 | uint32_t sreg = curTileA0Len; | 174 | uint32_t sreg = curTileA0Len; |
| 175 | uint32_t tileA0LenLocal = tl_->tileA0Len; | 175 | uint32_t tileA0LenLocal = tl_->tileA0Len; |
| 176 | 176 | ||
| 177 | for (uint16_t j = 0; j < aLoopTimes; j++) { // 列 | 177 | for (uint16_t j = 0; j < aLoopTimes; j++) { // 列 |
| 178 | - mask = MicroAPI::UpdateMask<float>(sreg); | 178 | + mask = Reg::UpdateMask<float>(sreg); |
| 179 | - MicroAPI::LoadAlign<float, MicroAPI::LoadDist::DIST_NORM>(sumReg, | 179 | + Reg::LoadAlign<float, Reg::LoadDist::DIST_NORM>(sumReg, (__ubuf__ float*)sumAddr + j * VL_FP32); |
| 180 | - (__ubuf__ float*)sumAddr + j * VL_FP32); | 180 | + Reg::Ln(lnReg, sumReg, mask); |
| 181 | - MicroAPI::Ln(lnReg, sumReg, mask); | ||
| 182 | 181 | ||
| 183 | for (uint16_t i = 0; i < rLoopTimes; i++) { // 行 | 182 | for (uint16_t i = 0; i < rLoopTimes; i++) { // 行 |
| 184 | uint32_t offset = j * VL_FP32 + i * tileA0LenLocal; | 183 | uint32_t offset = j * VL_FP32 + i * tileA0LenLocal; |
| 185 | 184 | ||
| 186 | - MicroAPI::LoadAlign(reg1, tmpAddr2 + offset); | 185 | + Reg::LoadAlign(reg1, tmpAddr2 + offset); |
| 187 | - MicroAPI::Sub(reg1, reg1, lnReg, mask); | 186 | + Reg::Sub(reg1, reg1, lnReg, mask); |
| 188 | 187 | ||
| 189 | if constexpr (yToFp32_) { | 188 | if constexpr (yToFp32_) { |
| 190 | - MicroAPI::StoreAlign(tmpAddrTy + offset, reg1, mask); | 189 | + Reg::StoreAlign(tmpAddrTy + offset, reg1, mask); |
| 191 | } else { // fp16、bf16 | 190 | } else { // fp16、bf16 |
| 192 | - MicroAPI::RegTensor<Ty> xFp16; | 191 | + Reg::RegTensor<Ty> xFp16; |
| 193 | - MicroAPI::Cast<Ty, float, castTraitFp32ToFp16>(xFp16, reg1, mask); | 192 | + Reg::Cast<Ty, float, castTraitFp32ToFp16>(xFp16, reg1, mask); |
| 194 | - MicroAPI::StoreAlign<Ty, MicroAPI::StoreDist::DIST_PACK_B32>(tmpAddrTy + offset, xFp16, mask); | 193 | + Reg::StoreAlign<Ty, Reg::StoreDist::DIST_PACK_B32>(tmpAddrTy + offset, xFp16, mask); |
| 195 | } | 194 | } |
| 196 | } | 195 | } |
| 197 | } | 196 | } |
| @@ -267,11 +266,11 @@ private: | |||
| 267 | uint32_t offset) | 266 | uint32_t offset) |
| 268 | { | 267 | { |
| 269 | if constexpr (xToFp32_) { | 268 | if constexpr (xToFp32_) { |
| 270 | - MicroAPI::RegTensor<Tx> xFp16; | 269 | + Reg::RegTensor<Tx> xFp16; |
| 271 | - MicroAPI::LoadAlign<Tx, MicroAPI::LoadDist::DIST_UNPACK_B16>(xFp16, ((__ubuf__ Tx*)src + offset)); | 270 | + Reg::LoadAlign<Tx, Reg::LoadDist::DIST_UNPACK_B16>(xFp16, ((__ubuf__ Tx*)src + offset)); |
| 272 | - MicroAPI::Cast<float, Tx, castTraitFp16ToFp32>(dst, xFp16, preg); | 271 | + Reg::Cast<float, Tx, castTraitFp16ToFp32>(dst, xFp16, preg); |
| 273 | } else { | 272 | } else { |
| 274 | - MicroAPI::LoadAlign<float, MicroAPI::LoadDist::DIST_NORM>(dst, (__ubuf__ float*)src + offset); | 273 | + Reg::LoadAlign<float, Reg::LoadDist::DIST_NORM>(dst, (__ubuf__ float*)src + offset); |
| 275 | } | 274 | } |
| 276 | } | 275 | } |
| 277 | 276 | ||
| @@ -28,14 +28,14 @@ | |||
| 28 | 28 | ||
| 29 | namespace LogSoftmaxV2Ops { | 29 | namespace LogSoftmaxV2Ops { |
| 30 | using namespace AscendC; | 30 | using namespace AscendC; |
| 31 | -using namespace AscendC::MicroAPI; | 31 | +using namespace AscendC::Reg; |
| 32 | using namespace SoftmaxV2Ops; | 32 | using namespace SoftmaxV2Ops; |
| 33 | 33 | ||
| 34 | -using AscendC::MicroAPI::LoadDist; | 34 | +using AscendC::Reg::LoadDist; |
| 35 | -using AscendC::MicroAPI::MaskMergeMode; | 35 | +using AscendC::Reg::MaskMergeMode; |
| 36 | -using AscendC::MicroAPI::MaskReg; | 36 | +using AscendC::Reg::MaskReg; |
| 37 | -using AscendC::MicroAPI::RegTensor; | 37 | +using AscendC::Reg::RegTensor; |
| 38 | -using AscendC::MicroAPI::StoreDist; | 38 | +using AscendC::Reg::StoreDist; |
| 39 | 39 | ||
| 40 | constexpr int64_t SCALE_COEF_TWO = 2; | 40 | constexpr int64_t SCALE_COEF_TWO = 2; |
| 41 | constexpr int64_t SCALE_COEF_FOUR = 4; | 41 | constexpr int64_t SCALE_COEF_FOUR = 4; |
| @@ -24,14 +24,14 @@ | |||
| 24 | 24 | ||
| 25 | namespace LogSoftmaxV2Ops { | 25 | namespace LogSoftmaxV2Ops { |
| 26 | using namespace AscendC; | 26 | using namespace AscendC; |
| 27 | -using namespace AscendC::MicroAPI; | 27 | +using namespace AscendC::Reg; |
| 28 | using namespace SoftmaxV2Ops; | 28 | using namespace SoftmaxV2Ops; |
| 29 | 29 | ||
| 30 | -using AscendC::MicroAPI::LoadDist; | 30 | +using AscendC::Reg::LoadDist; |
| 31 | -using AscendC::MicroAPI::MaskMergeMode; | 31 | +using AscendC::Reg::MaskMergeMode; |
| 32 | -using AscendC::MicroAPI::MaskReg; | 32 | +using AscendC::Reg::MaskReg; |
| 33 | -using AscendC::MicroAPI::RegTensor; | 33 | +using AscendC::Reg::RegTensor; |
| 34 | -using AscendC::MicroAPI::StoreDist; | 34 | +using AscendC::Reg::StoreDist; |
| 35 | 35 | ||
| 36 | template <typename T1, typename T2> | 36 | template <typename T1, typename T2> |
| 37 | class LogSoftmaxV2ARARecompute : public SoftmaxV2OpsBase { | 37 | class LogSoftmaxV2ARARecompute : public SoftmaxV2OpsBase { |
| @@ -23,10 +23,10 @@ namespace LogSigmoidGradOp { | |||
| 23 | using namespace AscendC; | 23 | using namespace AscendC; |
| 24 | using namespace Ops::Base; | 24 | using namespace Ops::Base; |
| 25 | 25 | ||
| 26 | -constexpr static MicroAPI::CastTrait castTrait0 = {MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::UNKNOWN, | 26 | +constexpr static Reg::CastTrait castTrait0 = {Reg::RegLayout::ZERO, Reg::SatMode::UNKNOWN, Reg::MaskMergeMode::ZEROING, |
| 27 | - MicroAPI::MaskMergeMode::ZEROING, RoundMode::UNKNOWN}; | 27 | + RoundMode::UNKNOWN}; |
| 28 | -constexpr static MicroAPI::CastTrait castTrait1 = {MicroAPI::RegLayout::ZERO, MicroAPI::SatMode::NO_SAT, | 28 | +constexpr static Reg::CastTrait castTrait1 = {Reg::RegLayout::ZERO, Reg::SatMode::NO_SAT, Reg::MaskMergeMode::ZEROING, |
| 29 | - MicroAPI::MaskMergeMode::ZEROING, RoundMode::CAST_RINT}; | 29 | + RoundMode::CAST_RINT}; |
| 30 | 30 | ||
| 31 | 31 | ||
| 32 | template <class T> | 32 | template <class T> |
| @@ -44,76 +44,76 @@ struct LogSigmoidGradCustom : public Vec::ElemwiseBinaryOP<T, T, T> { | |||
| 44 | __ubuf__ T* src1Addr = (__ubuf__ T*)src1.GetPhyAddr(); | 44 | __ubuf__ T* src1Addr = (__ubuf__ T*)src1.GetPhyAddr(); |
| 45 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); | 45 | __ubuf__ T* dstAddr = (__ubuf__ T*)dst.GetPhyAddr(); |
| 46 | 46 | ||
| 47 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> DataOne; | 47 | + Reg::RegTensor<float, Reg::RegTraitNumOne> DataOne; |
| 48 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> DataZero; | 48 | + Reg::RegTensor<float, Reg::RegTraitNumOne> DataZero; |
| 49 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregInputGradOut; | 49 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregInputGradOut; |
| 50 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregInputSelf; | 50 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregInputSelf; |
| 51 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> SelfAbs; | 51 | + Reg::RegTensor<float, Reg::RegTraitNumOne> SelfAbs; |
| 52 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> SelfAbsNeg; | 52 | + Reg::RegTensor<float, Reg::RegTraitNumOne> SelfAbsNeg; |
| 53 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> SelfAbsNegExp; | 53 | + Reg::RegTensor<float, Reg::RegTraitNumOne> SelfAbsNegExp; |
| 54 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> SelfAbsNegExpAdd; | 54 | + Reg::RegTensor<float, Reg::RegTraitNumOne> SelfAbsNegExpAdd; |
| 55 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> vregSelect; | 55 | + Reg::RegTensor<float, Reg::RegTraitNumOne> vregSelect; |
| 56 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> Answer; | 56 | + Reg::RegTensor<float, Reg::RegTraitNumOne> Answer; |
| 57 | - MicroAPI::RegTensor<float, MicroAPI::RegTraitNumOne> LastAnswer; | 57 | + Reg::RegTensor<float, Reg::RegTraitNumOne> LastAnswer; |
| 58 | - MicroAPI::MaskReg mask, cmpMask; | 58 | + Reg::MaskReg mask, cmpMask; |
| 59 | 59 | ||
| 60 | if constexpr (std::is_same_v<T, float>) { | 60 | if constexpr (std::is_same_v<T, float>) { |
| 61 | __VEC_SCOPE__ | 61 | __VEC_SCOPE__ |
| 62 | { | 62 | { |
| 63 | - MicroAPI::Duplicate(DataOne, (float)1.0); | 63 | + Reg::Duplicate(DataOne, (float)1.0); |
| 64 | - MicroAPI::Duplicate(DataZero, (float)0.0); | 64 | + Reg::Duplicate(DataZero, (float)0.0); |
| 65 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 65 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 66 | - mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count); | 66 | + mask = Reg::UpdateMask<float, Reg::RegTraitNumOne>(count); |
| 67 | // OpCopyIn | 67 | // OpCopyIn |
| 68 | - MicroAPI::LoadAlign(vregInputGradOut, (__ubuf__ T*)(src0Addr + loopIdx * vlSize)); | 68 | + Reg::LoadAlign(vregInputGradOut, (__ubuf__ T*)(src0Addr + loopIdx * vlSize)); |
| 69 | - MicroAPI::LoadAlign(vregInputSelf, (__ubuf__ T*)(src1Addr + loopIdx * vlSize)); | 69 | + Reg::LoadAlign(vregInputSelf, (__ubuf__ T*)(src1Addr + loopIdx * vlSize)); |
| 70 | // compute | 70 | // compute |
| 71 | - MicroAPI::Abs(SelfAbs, vregInputSelf, mask); | 71 | + Reg::Abs(SelfAbs, vregInputSelf, mask); |
| 72 | - MicroAPI::Muls(SelfAbsNeg, SelfAbs, (float)-1.0, mask); | 72 | + Reg::Muls(SelfAbsNeg, SelfAbs, (float)-1.0, mask); |
| 73 | - MicroAPI::Exp(SelfAbsNegExp, SelfAbsNeg, mask); | 73 | + Reg::Exp(SelfAbsNegExp, SelfAbsNeg, mask); |
| 74 | - MicroAPI::Adds(SelfAbsNegExpAdd, SelfAbsNegExp, (float)1.0, mask); | 74 | + Reg::Adds(SelfAbsNegExpAdd, SelfAbsNegExp, (float)1.0, mask); |
| 75 | 75 | ||
| 76 | - MicroAPI::Compare<float, CMPMODE::LT>(cmpMask, vregInputSelf, DataZero, mask); | 76 | + Reg::Compare<float, CMPMODE::LT>(cmpMask, vregInputSelf, DataZero, mask); |
| 77 | - MicroAPI::Select(vregSelect, DataOne, SelfAbsNegExp, cmpMask); | 77 | + Reg::Select(vregSelect, DataOne, SelfAbsNegExp, cmpMask); |
| 78 | - MicroAPI::Div(Answer, vregSelect, SelfAbsNegExpAdd, mask); | 78 | + Reg::Div(Answer, vregSelect, SelfAbsNegExpAdd, mask); |
| 79 | - MicroAPI::Mul(LastAnswer, Answer, vregInputGradOut, mask); | 79 | + Reg::Mul(LastAnswer, Answer, vregInputGradOut, mask); |
| 80 | 80 | ||
| 81 | // OpCopyOut | 81 | // OpCopyOut |
| 82 | - MicroAPI::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), LastAnswer, mask); | 82 | + Reg::StoreAlign((__ubuf__ T*)(dstAddr + loopIdx * vlSize), LastAnswer, mask); |
| 83 | } | 83 | } |
| 84 | } | 84 | } |
| 85 | } else { | 85 | } else { |
| 86 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputGradOutT; | 86 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputGradOutT; |
| 87 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> vregInputSelfT; | 87 | + Reg::RegTensor<T, Reg::RegTraitNumOne> vregInputSelfT; |
| 88 | - MicroAPI::RegTensor<T, MicroAPI::RegTraitNumOne> LastAnswerT; | 88 | + Reg::RegTensor<T, Reg::RegTraitNumOne> LastAnswerT; |
| 89 | __VEC_SCOPE__ | 89 | __VEC_SCOPE__ |
| 90 | { | 90 | { |
| 91 | - MicroAPI::Duplicate(DataOne, (float)1.0); | 91 | + Reg::Duplicate(DataOne, (float)1.0); |
| 92 | - MicroAPI::Duplicate(DataZero, (float)0.0); | 92 | + Reg::Duplicate(DataZero, (float)0.0); |
| 93 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { | 93 | for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) { |
| 94 | - mask = MicroAPI::UpdateMask<float, MicroAPI::RegTraitNumOne>(count); | 94 | + mask = Reg::UpdateMask<float, Reg::RegTraitNumOne>(count); |
| 95 | // OpCopyIn | 95 | // OpCopyIn |
| 96 | - MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>( | 96 | + Reg::LoadAlign<T, Reg::LoadDist::DIST_UNPACK_B16>(vregInputGradOutT, |
| 97 | - vregInputGradOutT, (__ubuf__ T*)(src0Addr + loopIdx * vlSize)); | 97 | + (__ubuf__ T*)(src0Addr + loopIdx * vlSize)); |
| 98 | - MicroAPI::LoadAlign<T, MicroAPI::LoadDist::DIST_UNPACK_B16>( | 98 | + Reg::LoadAlign<T, Reg::LoadDist::DIST_UNPACK_B16>(vregInputSelfT, |
| 99 | - vregInputSelfT, (__ubuf__ T*)(src1Addr + loopIdx * vlSize)); | 99 | + (__ubuf__ T*)(src1Addr + loopIdx * vlSize)); |
| 100 | - MicroAPI::Cast<float, T, castTrait0>(vregInputGradOut, vregInputGradOutT, mask); | 100 | + Reg::Cast<float, T, castTrait0>(vregInputGradOut, vregInputGradOutT, mask); |
| 101 | - MicroAPI::Cast<float, T, castTrait0>(vregInputSelf, vregInputSelfT, mask); | 101 | + Reg::Cast<float, T, castTrait0>(vregInputSelf, vregInputSelfT, mask); |
| 102 | // compute | 102 | // compute |
| 103 | - MicroAPI::Abs(SelfAbs, vregInputSelf, mask); | 103 | + Reg::Abs(SelfAbs, vregInputSelf, mask); |
| 104 | - MicroAPI::Muls(SelfAbsNeg, SelfAbs, (float)-1.0, mask); | 104 | + Reg::Muls(SelfAbsNeg, SelfAbs, (float)-1.0, mask); |
| 105 | - MicroAPI::Exp(SelfAbsNegExp, SelfAbsNeg, mask); | 105 | + Reg::Exp(SelfAbsNegExp, SelfAbsNeg, mask); |
| 106 | - MicroAPI::Adds(SelfAbsNegExpAdd, SelfAbsNegExp, (float)1.0, mask); | 106 | + Reg::Adds(SelfAbsNegExpAdd, SelfAbsNegExp, (float)1.0, mask); |
| 107 | 107 | ||
| 108 | - MicroAPI::Compare<float, CMPMODE::LT>(cmpMask, vregInputSelf, DataZero, mask); | 108 | + Reg::Compare<float, CMPMODE::LT>(cmpMask, vregInputSelf, DataZero, mask); |
| 109 | - MicroAPI::Select(vregSelect, DataOne, SelfAbsNegExp, cmpMask); | 109 | + Reg::Select(vregSelect, DataOne, SelfAbsNegExp, cmpMask); |
| 110 | - MicroAPI::Div(Answer, vregSelect, SelfAbsNegExpAdd, mask); | 110 | + Reg::Div(Answer, vregSelect, SelfAbsNegExpAdd, mask); |
| 111 | - MicroAPI::Mul(LastAnswer, Answer, vregInputGradOut, mask); | 111 | + Reg::Mul(LastAnswer, Answer, vregInputGradOut, mask); |
| 112 | 112 | ||
| 113 | - MicroAPI::Cast<T, float, castTrait1>(LastAnswerT, LastAnswer, mask); | 113 | + Reg::Cast<T, float, castTrait1>(LastAnswerT, LastAnswer, mask); |
| 114 | // OpCopyOut | 114 | // OpCopyOut |
| 115 | - MicroAPI::StoreAlign<T, MicroAPI::StoreDist::DIST_PACK_B32>( | 115 | + Reg::StoreAlign<T, Reg::StoreDist::DIST_PACK_B32>((__ubuf__ T*)(dstAddr + loopIdx * vlSize), |
| 116 | - (__ubuf__ T*)(dstAddr + loopIdx * vlSize), LastAnswerT, mask); | 116 | + LastAnswerT, mask); |
| 117 | } | 117 | } |
| 118 | } | 118 | } |
| 119 | } | 119 | } |
| @@ -135,4 +135,4 @@ struct LogSigmoidGradDag { | |||
| 135 | using OpDag = DAGSch<Outputs, void, MemCfg>; | 135 | using OpDag = DAGSch<Outputs, void, MemCfg>; |
| 136 | }; | 136 | }; |
| 137 | } // namespace LogSigmoidGradOp | 137 | } // namespace LogSigmoidGradOp |
| 138 | -#endif // LOGSIGMOID_GRAD_DAG_H | 138 | +#endif // LOGSIGMOID_GRAD_DAG_H |