Star1732
1645
Code
Code
Issues495
Pull Requests551
Actions
Discussion
Wiki
Analysis
Star1732
1645
Merged
算子中存在使用内部的asc api接口，需要切换为对外的api接口 #4954
hw-zhangpanpanAdd on Aug 22
算子中存在使用内部的asc api接口，需要切换为对外的api接口 #4954
Merged
hw-zhangpanpanAdd on Aug 22
Merged fromDeleted :masterintocann/ops-mathmaster
Total: 40 files changed+1051-1117
Mconversion/as_strided/op_kernel/arch35/as_strided.h+36-38
@@ -39,8 +39,7 @@ constexpr size_t NDDMA_INDEX4 = 4;
39template <typename T>39template <typename T>
40class KernelAsStrided {40class KernelAsStrided {
41public:41public:
42- __aicore__ inline KernelAsStrided()42+ __aicore__ inline KernelAsStrided() {}
43- {}
44 43 
45 template <typename U>44 template <typename U>
46 __aicore__ inline void CopyArray(const U* src, U* dst, int64_t size)45 __aicore__ inline void CopyArray(const U* src, U* dst, int64_t size)
@@ -50,8 +49,8 @@ public:
50 }49 }
51 }50 }
52 51 
53- __aicore__ inline void Init(52+ __aicore__ inline void Init(GM_ADDR input, GM_ADDR outShape, GM_ADDR outStride, GM_ADDR output,
54- GM_ADDR input, GM_ADDR outShape, GM_ADDR outStride, GM_ADDR output, AsStridedTilingData tilingData)53+ AsStridedTilingData tilingData)
55 {54 {
56 blockNum_ = tilingData.blockNum;55 blockNum_ = tilingData.blockNum;
57 loopsTailCore_ = tilingData.loopsTailCore;56 loopsTailCore_ = tilingData.loopsTailCore;
@@ -75,8 +74,8 @@ public:
75 CopyArray(tilingData.nddmaDstStride, nddmaDstStride_, TILING_NDDMA_LEN);74 CopyArray(tilingData.nddmaDstStride, nddmaDstStride_, TILING_NDDMA_LEN);
76 75 
77 blockOffset_ = loopsPerCore_ * ubFactor_;76 blockOffset_ = loopsPerCore_ * ubFactor_;
78- tileOffset_ =77+ tileOffset_ = ubFactorTail_ == 0 ? ubFactor_ * outerAxisFactor_ :
79- ubFactorTail_ == 0 ? ubFactor_ * outerAxisFactor_ : ubFactor_ * (outerAxisFactor_ - 1) + ubFactorTail_;78+ ubFactor_ * (outerAxisFactor_ - 1) + ubFactorTail_;
80 79 
81 inputGm_.SetGlobalBuffer((__gm__ T*)input + storageOffset_);80 inputGm_.SetGlobalBuffer((__gm__ T*)input + storageOffset_);
82 outputGm_.SetGlobalBuffer((__gm__ T*)output);81 outputGm_.SetGlobalBuffer((__gm__ T*)output);
@@ -118,36 +117,35 @@ public:
118 },117 },
119 0};118 0};
120 119 
121- dmaTailParam_ = {120+ dmaTailParam_ = {{
122- {121+ {// src stride
123- {// src stride122+ static_cast<uint64_t>(nddmaSrcStride_[NDDMA_INDEX4]),
124- static_cast<uint64_t>(nddmaSrcStride_[NDDMA_INDEX4]),123+ static_cast<uint64_t>(nddmaSrcStride_[NDDMA_INDEX3]),
125- static_cast<uint64_t>(nddmaSrcStride_[NDDMA_INDEX3]),124+ static_cast<uint64_t>(nddmaSrcStride_[NDDMA_INDEX2]),
126- static_cast<uint64_t>(nddmaSrcStride_[NDDMA_INDEX2]),125+ static_cast<uint64_t>(nddmaSrcStride_[NDDMA_INDEX1]),
127- static_cast<uint64_t>(nddmaSrcStride_[NDDMA_INDEX1]),126+ static_cast<uint64_t>(nddmaSrcStride_[NDDMA_INDEX0])},
128- static_cast<uint64_t>(nddmaSrcStride_[NDDMA_INDEX0])},127+ {// dst stride
129- {// dst stride128+ static_cast<uint32_t>(nddmaDstStride_[NDDMA_INDEX4]),
130- static_cast<uint32_t>(nddmaDstStride_[NDDMA_INDEX4]),129+ static_cast<uint32_t>(nddmaDstStride_[NDDMA_INDEX3]),
131- static_cast<uint32_t>(nddmaDstStride_[NDDMA_INDEX3]),130+ static_cast<uint32_t>(nddmaDstStride_[NDDMA_INDEX2]),
132- static_cast<uint32_t>(nddmaDstStride_[NDDMA_INDEX2]),131+ static_cast<uint32_t>(nddmaDstStride_[NDDMA_INDEX1]),
133- static_cast<uint32_t>(nddmaDstStride_[NDDMA_INDEX1]),132+ static_cast<uint32_t>(nddmaDstStride_[NDDMA_INDEX0])},
134- static_cast<uint32_t>(nddmaDstStride_[NDDMA_INDEX0])},133+ {// loop size
135- {// loop size134+ static_cast<uint32_t>(nddmaTailLoop_[NDDMA_INDEX4]),
136- static_cast<uint32_t>(nddmaTailLoop_[NDDMA_INDEX4]),135+ static_cast<uint32_t>(nddmaTailLoop_[NDDMA_INDEX3]),
137- static_cast<uint32_t>(nddmaTailLoop_[NDDMA_INDEX3]),136+ static_cast<uint32_t>(nddmaTailLoop_[NDDMA_INDEX2]),
138- static_cast<uint32_t>(nddmaTailLoop_[NDDMA_INDEX2]),137+ static_cast<uint32_t>(nddmaTailLoop_[NDDMA_INDEX1]),
139- static_cast<uint32_t>(nddmaTailLoop_[NDDMA_INDEX1]),138+ static_cast<uint32_t>(nddmaTailLoop_[NDDMA_INDEX0])},
140- static_cast<uint32_t>(nddmaTailLoop_[NDDMA_INDEX0])},139+ {ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8}, // left pad
141- {ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8}, // left pad140+ {ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8} // right pad
142- {ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8} // right pad141+ },
143- },142+ 0};
144- 0};
145 143 
146- copyParams_ = {144+ copyParams_ = {static_cast<uint16_t>(1), static_cast<uint32_t>(ubFactor_ * sizeof(T)), static_cast<uint32_t>(0),
147- static_cast<uint16_t>(1), static_cast<uint32_t>(ubFactor_ * sizeof(T)), static_cast<uint32_t>(0), 0, 0};145+ 0, 0};
148 146 
149- copyParamsTail_ = {147+ copyParamsTail_ = {static_cast<uint16_t>(1), static_cast<uint32_t>(ubFactorTail_ * sizeof(T)),
150- static_cast<uint16_t>(1), static_cast<uint32_t>(ubFactorTail_ * sizeof(T)), static_cast<uint32_t>(0), 0, 0};148+ static_cast<uint32_t>(0), 0, 0};
151 }149 }
152 150 
153 __aicore__ inline void Process()151 __aicore__ inline void Process()
@@ -158,7 +156,7 @@ public:
158 156 
159 __aicore__ inline void CopyProcess(int64_t srcOffset, int64_t dstOffset, uint32_t totalIdx)157 __aicore__ inline void CopyProcess(int64_t srcOffset, int64_t dstOffset, uint32_t totalIdx)
160 {158 {
161- static constexpr MultiCopyConfig Config = {false, 0, 0, false};159+ static constexpr NdDmaConfig Config = {false, 0, 0, false};
162 if (innerAxisFactorTail_ == 0) {160 if (innerAxisFactorTail_ == 0) {
163 LocalTensor<T> srcLocal = inQueue_.AllocTensor<T>();161 LocalTensor<T> srcLocal = inQueue_.AllocTensor<T>();
164 DataCopy<T, NDDMA_DIM, Config>(srcLocal, inputGm_[srcOffset], dmaParam_);162 DataCopy<T, NDDMA_DIM, Config>(srcLocal, inputGm_[srcOffset], dmaParam_);
@@ -244,10 +242,10 @@ private:
244 uint32_t nddmaTailLoop_[TILING_NDDMA_LEN] = {1, 1, 1, 1, 1};242 uint32_t nddmaTailLoop_[TILING_NDDMA_LEN] = {1, 1, 1, 1, 1};
245 uint32_t nddmaDstStride_[TILING_NDDMA_LEN] = {1, 1, 1, 1, 1};243 uint32_t nddmaDstStride_[TILING_NDDMA_LEN] = {1, 1, 1, 1, 1};
246 uint64_t nddmaSrcStride_[TILING_NDDMA_LEN] = {0};244 uint64_t nddmaSrcStride_[TILING_NDDMA_LEN] = {0};
247- MultiCopyParams<T, NDDMA_DIM> dmaParam_;245+ NdDmaParams<T, NDDMA_DIM> dmaParam_;
248- MultiCopyParams<T, NDDMA_DIM> dmaTailParam_;246+ NdDmaParams<T, NDDMA_DIM> dmaTailParam_;
249};247};
250 248 
251} // namespace AsStrided249} // namespace AsStrided
252 250 
253-#endif251+#endif
Mconversion/as_strided/op_kernel/arch35/as_strided_dual_cut.h+10-12
@@ -31,11 +31,10 @@ using namespace AscendC;
31template <typename T, int TILING_KEY>31template <typename T, int TILING_KEY>
32class KernelAsStridedDualCut {32class KernelAsStridedDualCut {
33public:33public:
34- __aicore__ inline KernelAsStridedDualCut()34+ __aicore__ inline KernelAsStridedDualCut() {}
35- {}
36 35 
37- __aicore__ inline void Init(36+ __aicore__ inline void Init(GM_ADDR input, GM_ADDR outShape, GM_ADDR outStride, GM_ADDR output,
38- GM_ADDR input, GM_ADDR outShape, GM_ADDR outStride, GM_ADDR output, const AsStridedTilingData* tiling)37+ const AsStridedTilingData* tiling)
39 {38 {
40 storageOffset_ = 0;39 storageOffset_ = 0;
41 40 
@@ -110,8 +109,8 @@ public:
110 }109 }
111 }110 }
112 111 
113- __aicore__ inline void MoveOutWithLoop(112+ __aicore__ inline void MoveOutWithLoop(LocalTensor<T>& outLocal, int64_t outAddr, LoopModeParams& loopMode,
114- LocalTensor<T>& outLocal, int64_t outAddr, LoopModeParams& loopMode, DataCopyExtParams& copyOutParamV2)113+ DataCopyExtParams& copyOutParamV2)
115 {114 {
116 loopMode.loop2Size = dmaParam_.loopInfo.loopSize[NDDMA_INDEX3];115 loopMode.loop2Size = dmaParam_.loopInfo.loopSize[NDDMA_INDEX3];
117 loopMode.loop2SrcStride = ubDstStride_[1] * sizeof(T);116 loopMode.loop2SrcStride = ubDstStride_[1] * sizeof(T);
@@ -128,10 +127,9 @@ public:
128 // add changes127 // add changes
129 SetLoopModePara(loopMode, DataCopyMVType::UB_TO_OUT);128 SetLoopModePara(loopMode, DataCopyMVType::UB_TO_OUT);
130 for (int aIdx = 0; aIdx < dmaParam_.loopInfo.loopSize[NDDMA_INDEX4]; ++aIdx) {129 for (int aIdx = 0; aIdx < dmaParam_.loopInfo.loopSize[NDDMA_INDEX4]; ++aIdx) {
131- DataCopyExtParams copyParams{130+ DataCopyExtParams copyParams{copyOutParamV2.blockCount, copyOutParamV2.blockLen,
132- copyOutParamV2.blockCount, copyOutParamV2.blockLen,131+ (copyOutParamV2.srcStride - copyOutParamV2.blockLen) / 32,
133- (copyOutParamV2.srcStride - copyOutParamV2.blockLen) / 32,132+ copyOutParamV2.dstStride - copyOutParamV2.blockLen, 0};
134- copyOutParamV2.dstStride - copyOutParamV2.blockLen, 0};
135 DataCopyPad(outputGm_[outAddr + aIdx * ubOutStride_[0]], outLocal[aIdx * ubDstStride_[0]], copyParams);133 DataCopyPad(outputGm_[outAddr + aIdx * ubOutStride_[0]], outLocal[aIdx * ubDstStride_[0]], copyParams);
136 }134 }
137 ResetLoopModePara(DataCopyMVType::UB_TO_OUT);135 ResetLoopModePara(DataCopyMVType::UB_TO_OUT);
@@ -275,7 +273,7 @@ private:
275 GlobalTensor<T> inputGm_, outputGm_;273 GlobalTensor<T> inputGm_, outputGm_;
276 TQueBind<TPosition::VECIN, TPosition::VECOUT, BUFFER_NUM> inOutQueue;274 TQueBind<TPosition::VECIN, TPosition::VECOUT, BUFFER_NUM> inOutQueue;
277 275 
278- MultiCopyParams<T, NDDMA_DIM> dmaParam_;276+ NdDmaParams<T, NDDMA_DIM> dmaParam_;
279 277 
280 uint32_t ubShape_[TILING_NDDMA_LEN] = {0};278 uint32_t ubShape_[TILING_NDDMA_LEN] = {0};
281 uint64_t ubInStride_[TILING_NDDMA_LEN] = {0};279 uint64_t ubInStride_[TILING_NDDMA_LEN] = {0};
@@ -306,4 +304,4 @@ private:
306 304 
307} // namespace AsStrided305} // namespace AsStrided
308 306 
309-#endif307+#endif
Mconversion/as_strided/op_kernel/arch35/as_strided_gather.h+35-34
@@ -296,11 +296,11 @@ template <typename T>
296__aicore__ inline void KernelAsStridedGather<T>::AsGather(int64_t idxOffset)296__aicore__ inline void KernelAsStridedGather<T>::AsGather(int64_t idxOffset)
297{297{
298 // index298 // index
299- __local_mem__ IdxType_* indexAddr = (__local_mem__ IdxType_*)indexLocal_.GetPhyAddr();299+ __ubuf__ IdxType_* indexAddr = (__ubuf__ IdxType_*)indexLocal_.GetPhyAddr();
300 // xUb300 // xUb
301- __local_mem__ T* xUbAddr = (__local_mem__ T*)ubFactorLocal_.GetPhyAddr();301+ __ubuf__ T* xUbAddr = (__ubuf__ T*)ubFactorLocal_.GetPhyAddr();
302 // yUb302 // yUb
303- __local_mem__ T* yUbAddr = (__local_mem__ T*)yUbFactorLocal_.GetPhyAddr();303+ __ubuf__ T* yUbAddr = (__ubuf__ T*)yUbFactorLocal_.GetPhyAddr();
304 304 
305 if constexpr (sizeof(T) == 1) {305 if constexpr (sizeof(T) == 1) {
306 uint16_t vfLen = VF_LENGTH / sizeof(CastType_);306 uint16_t vfLen = VF_LENGTH / sizeof(CastType_);
@@ -318,20 +318,21 @@ __aicore__ inline void KernelAsStridedGather<T>::AsGather(int64_t idxOffset)
318 Reg::RegTensor<IdxType_> vd0;318 Reg::RegTensor<IdxType_> vd0;
319 Reg::RegTensor<T> dstReg;319 Reg::RegTensor<T> dstReg;
320 for (uint16_t idx_reg = 0; idx_reg < mainLoopsCnt; ++idx_reg) {320 for (uint16_t idx_reg = 0; idx_reg < mainLoopsCnt; ++idx_reg) {
321- Reg::DataCopy(indexReg, indexAddr + idx_reg * xUbMain_copy);321+ Reg::LoadAlign(indexReg, indexAddr + idx_reg * xUbMain_copy);
322 Reg::Adds(vd0, indexReg, idxOffset, p0);322 Reg::Adds(vd0, indexReg, idxOffset, p0);
323- Reg::DataCopyGather((Reg::RegTensor<CastType_>&)dstReg, xUbAddr, vd0, p0);323+ Reg::Gather((Reg::RegTensor<CastType_>&)dstReg, xUbAddr, vd0, p0);
324- __local_mem__ CastType_* yUbAddrB16 = reinterpret_cast<__local_mem__ CastType_*>(324+ __ubuf__ CastType_* yUbAddrB16 = reinterpret_cast<__ubuf__ CastType_*>(yUbAddr +
325- yUbAddr + idx_reg * xUbMain_copy);325+ idx_reg * xUbMain_copy);
326- Reg::DataCopy<CastType_, Reg::StoreDist::DIST_PACK_B16>(yUbAddrB16, (Reg::RegTensor<CastType_>&)dstReg,326+ Reg::StoreAlign<CastType_, Reg::StoreDist::DIST_PACK_B16>(yUbAddrB16,
327- p0);327+ (Reg::RegTensor<CastType_>&)dstReg, p0);
328 }328 }
329- Reg::DataCopy(indexReg, indexAddr + mainLoopsCnt2 * xUbMain_copy);329+ Reg::LoadAlign(indexReg, indexAddr + mainLoopsCnt2 * xUbMain_copy);
330 Reg::Adds(vd0, indexReg, idxOffset, p1);330 Reg::Adds(vd0, indexReg, idxOffset, p1);
331- Reg::DataCopyGather((Reg::RegTensor<CastType_>&)dstReg, xUbAddr, vd0, p1);331+ Reg::Gather((Reg::RegTensor<CastType_>&)dstReg, xUbAddr, vd0, p1);
332- __local_mem__ CastType_* yUbAddrB16 = reinterpret_cast<__local_mem__ CastType_*>(332+ __ubuf__ CastType_* yUbAddrB16 = reinterpret_cast<__ubuf__ CastType_*>(yUbAddr +
333- yUbAddr + mainLoopsCnt2 * xUbMain_copy);333+ mainLoopsCnt2 * xUbMain_copy);
334- Reg::DataCopy<CastType_, Reg::StoreDist::DIST_PACK_B16>(yUbAddrB16, (Reg::RegTensor<CastType_>&)dstReg, p1);334+ Reg::StoreAlign<CastType_, Reg::StoreDist::DIST_PACK_B16>(yUbAddrB16, (Reg::RegTensor<CastType_>&)dstReg,
335+ p1);
335 }336 }
336 } else {337 } else {
337 uint16_t vfLen = VF_LENGTH / sizeof(CastType_);338 uint16_t vfLen = VF_LENGTH / sizeof(CastType_);
@@ -352,15 +353,15 @@ __aicore__ inline void KernelAsStridedGather<T>::AsGather(int64_t idxOffset)
352 Reg::RegTensor<T> dstReg;353 Reg::RegTensor<T> dstReg;
353 Reg::AddrReg aReg0;354 Reg::AddrReg aReg0;
354 for (uint16_t idx_reg = 0; idx_reg < mainLoopsCnt; ++idx_reg) {355 for (uint16_t idx_reg = 0; idx_reg < mainLoopsCnt; ++idx_reg) {
355- Reg::DataCopy(indexReg, indexAddr + idx_reg * xUbMain_copy);356+ Reg::LoadAlign(indexReg, indexAddr + idx_reg * xUbMain_copy);
356 Reg::Adds(vd0, indexReg, idxOffset, p2);357 Reg::Adds(vd0, indexReg, idxOffset, p2);
357- Reg::DataCopyGather(dstReg, xUbAddr, vd0, p0);358+ Reg::Gather(dstReg, xUbAddr, vd0, p0);
358- Reg::DataCopy(yUbAddr + idx_reg * xUbMain_copy, dstReg, p0);359+ Reg::StoreAlign(yUbAddr + idx_reg * xUbMain_copy, dstReg, p0);
359 }360 }
360- Reg::DataCopy(indexReg, indexAddr + mainLoopsCnt2 * xUbMain_copy);361+ Reg::LoadAlign(indexReg, indexAddr + mainLoopsCnt2 * xUbMain_copy);
361 Reg::Adds(vd0, indexReg, idxOffset, p2);362 Reg::Adds(vd0, indexReg, idxOffset, p2);
362- Reg::DataCopyGather(dstReg, xUbAddr, vd0, p1);363+ Reg::Gather(dstReg, xUbAddr, vd0, p1);
363- Reg::DataCopy(yUbAddr + mainLoopsCnt2 * xUbMain_copy, dstReg, p1);364+ Reg::StoreAlign(yUbAddr + mainLoopsCnt2 * xUbMain_copy, dstReg, p1);
364 }365 }
365 }366 }
366 outQue_.EnQue<T>(yUbFactorLocal_);367 outQue_.EnQue<T>(yUbFactorLocal_);
@@ -395,7 +396,7 @@ __aicore__ inline void KernelAsStridedGather<T>::AsComputeIdx()
395template <typename T>396template <typename T>
396__aicore__ inline void KernelAsStridedGather<T>::GenDim3Index()397__aicore__ inline void KernelAsStridedGather<T>::GenDim3Index()
397{398{
398- __local_mem__ IdxType_* idxAddr = (__local_mem__ IdxType_*)indexLocal_.GetPhyAddr();399+ __ubuf__ IdxType_* idxAddr = (__ubuf__ IdxType_*)indexLocal_.GetPhyAddr();
399 uint16_t vfLen = VF_LENGTH / sizeof(IdxType_);400 uint16_t vfLen = VF_LENGTH / sizeof(IdxType_);
400 401 
401 uint32_t lastDimInOffset = strideArr_[dimNum_ - 1];402 uint32_t lastDimInOffset = strideArr_[dimNum_ - 1];
@@ -429,9 +430,9 @@ __aicore__ inline void KernelAsStridedGather<T>::GenDim3Index()
429 Reg::Arange(tmp, 0);430 Reg::Arange(tmp, 0);
430 idxReg = (RegTensor<IdxType_>&)tmp;431 idxReg = (RegTensor<IdxType_>&)tmp;
431 Reg::Duplicate(dim0Reg, lastDimSize);432 Reg::Duplicate(dim0Reg, lastDimSize);
432- Reg::Copy(dim2Reg, dim0Reg); // backup a433+ Reg::Move(dim2Reg, dim0Reg); // backup a
433 Reg::Div(tmpReg, idxReg, dim0Reg, mask);434 Reg::Div(tmpReg, idxReg, dim0Reg, mask);
434- Reg::Copy(dim1Reg, tmpReg); // backup VL / a435+ Reg::Move(dim1Reg, tmpReg); // backup VL / a
435 Reg::Mul(tmpReg, tmpReg, dim0Reg, mask);436 Reg::Mul(tmpReg, tmpReg, dim0Reg, mask);
436 Reg::Sub(dim0Reg, idxReg, tmpReg, mask);437 Reg::Sub(dim0Reg, idxReg, tmpReg, mask);
437 // vec_b: VL / a % b438 // vec_b: VL / a % b
@@ -448,7 +449,7 @@ __aicore__ inline void KernelAsStridedGather<T>::GenDim3Index()
448 Reg::Muls(dstReg, dim2Reg, last3rdDimInOffset, mask);449 Reg::Muls(dstReg, dim2Reg, last3rdDimInOffset, mask);
449 Reg::Add(dstReg, dstReg, tmpReg, mask);450 Reg::Add(dstReg, dstReg, tmpReg, mask);
450 Reg::Add(dstReg, dstReg, tmp1Reg, mask);451 Reg::Add(dstReg, dstReg, tmp1Reg, mask);
451- Reg::DataCopy(idxAddr, dstReg, mask);452+ Reg::StoreAlign(idxAddr, dstReg, mask);
452 453 
453 MaskReg lpMask;454 MaskReg lpMask;
454 MaskReg selMask;455 MaskReg selMask;
@@ -464,7 +465,7 @@ __aicore__ inline void KernelAsStridedGather<T>::GenDim3Index()
464 * vec_a = vec_a - cmp_a * a465 * vec_a = vec_a - cmp_a * a
465 */466 */
466 Reg::Adds(dim0Reg, dim0Reg, lastDimInc, lpMask);467 Reg::Adds(dim0Reg, dim0Reg, lastDimInc, lpMask);
467- Reg::CompareScalar<IdxType_, CMPMODE::GE>(selMask, dim0Reg, lastDimSize, lpMask);468+ Reg::Compares<IdxType_, CMPMODE::GE>(selMask, dim0Reg, lastDimSize, lpMask);
468 Reg::Select(cmpReg, oneReg, zeroReg, selMask);469 Reg::Select(cmpReg, oneReg, zeroReg, selMask);
469 Reg::Muls(tmpReg, cmpReg, lastDimSize, lpMask);470 Reg::Muls(tmpReg, cmpReg, lastDimSize, lpMask);
470 Reg::Sub(dim0Reg, dim0Reg, tmpReg, lpMask);471 Reg::Sub(dim0Reg, dim0Reg, tmpReg, lpMask);
@@ -474,7 +475,7 @@ __aicore__ inline void KernelAsStridedGather<T>::GenDim3Index()
474 */475 */
475 Reg::Adds(cmpReg, cmpReg, last2ndDimInc, lpMask);476 Reg::Adds(cmpReg, cmpReg, last2ndDimInc, lpMask);
476 Reg::Add(dim1Reg, dim1Reg, cmpReg, lpMask);477 Reg::Add(dim1Reg, dim1Reg, cmpReg, lpMask);
477- Reg::CompareScalar<IdxType_, CMPMODE::GE>(selMask, dim1Reg, last2ndDimSize, lpMask);478+ Reg::Compares<IdxType_, CMPMODE::GE>(selMask, dim1Reg, last2ndDimSize, lpMask);
478 Reg::Select(cmpReg, oneReg, zeroReg, selMask);479 Reg::Select(cmpReg, oneReg, zeroReg, selMask);
479 Reg::Muls(tmpReg, cmpReg, last2ndDimSize, lpMask);480 Reg::Muls(tmpReg, cmpReg, last2ndDimSize, lpMask);
480 Reg::Sub(dim1Reg, dim1Reg, tmpReg, lpMask);481 Reg::Sub(dim1Reg, dim1Reg, tmpReg, lpMask);
@@ -487,7 +488,7 @@ __aicore__ inline void KernelAsStridedGather<T>::GenDim3Index()
487 Reg::Muls(dstReg, dim2Reg, last3rdDimInOffset, lpMask);488 Reg::Muls(dstReg, dim2Reg, last3rdDimInOffset, lpMask);
488 Reg::Add(dstReg, dstReg, tmpReg, lpMask);489 Reg::Add(dstReg, dstReg, tmpReg, lpMask);
489 Reg::Add(dstReg, dstReg, tmp1Reg, lpMask);490 Reg::Add(dstReg, dstReg, tmp1Reg, lpMask);
490- Reg::DataCopy(idxAddr + (lpIdx + 1) * vfLen, dstReg, lpMask);491+ Reg::StoreAlign(idxAddr + (lpIdx + 1) * vfLen, dstReg, lpMask);
491 }492 }
492 }493 }
493}494}
@@ -495,7 +496,7 @@ __aicore__ inline void KernelAsStridedGather<T>::GenDim3Index()
495template <typename T>496template <typename T>
496__aicore__ inline void KernelAsStridedGather<T>::GenDim2Index()497__aicore__ inline void KernelAsStridedGather<T>::GenDim2Index()
497{498{
498- __local_mem__ IdxType_* idxAddr = (__local_mem__ IdxType_*)indexLocal_.GetPhyAddr();499+ __ubuf__ IdxType_* idxAddr = (__ubuf__ IdxType_*)indexLocal_.GetPhyAddr();
499 uint16_t vfLen = VF_LENGTH / sizeof(IdxType_);500 uint16_t vfLen = VF_LENGTH / sizeof(IdxType_);
500 uint32_t lastDimInOffset = strideArr_[dimNum_ - 1];501 uint32_t lastDimInOffset = strideArr_[dimNum_ - 1];
501 uint32_t last2ndDimInOffset = strideArr_[dimNum_ - 2];502 uint32_t last2ndDimInOffset = strideArr_[dimNum_ - 2];
@@ -525,14 +526,14 @@ __aicore__ inline void KernelAsStridedGather<T>::GenDim2Index()
525 idxReg = (RegTensor<IdxType_>&)tmp;526 idxReg = (RegTensor<IdxType_>&)tmp;
526 Reg::Duplicate(dim0Reg, lastDimSize);527 Reg::Duplicate(dim0Reg, lastDimSize);
527 Reg::Div(tmpReg, idxReg, dim0Reg, mask);528 Reg::Div(tmpReg, idxReg, dim0Reg, mask);
528- Reg::Copy(dim1Reg, tmpReg); // vec_b: VL / a529+ Reg::Move(dim1Reg, tmpReg); // vec_b: VL / a
529 Reg::Mul(tmpReg, tmpReg, dim0Reg, mask);530 Reg::Mul(tmpReg, tmpReg, dim0Reg, mask);
530 Reg::Sub(dim0Reg, idxReg, tmpReg, mask);531 Reg::Sub(dim0Reg, idxReg, tmpReg, mask);
531 // index: vec_a * a_in_offset + vec_b * b_in_offset532 // index: vec_a * a_in_offset + vec_b * b_in_offset
532 Reg::Muls(tmpReg, dim0Reg, lastDimInOffset, mask);533 Reg::Muls(tmpReg, dim0Reg, lastDimInOffset, mask);
533 Reg::Muls(dstReg, dim1Reg, last2ndDimInOffset, mask);534 Reg::Muls(dstReg, dim1Reg, last2ndDimInOffset, mask);
534 Reg::Add(dstReg, dstReg, tmpReg, mask);535 Reg::Add(dstReg, dstReg, tmpReg, mask);
535- Reg::DataCopy(idxAddr, dstReg, mask);536+ Reg::StoreAlign(idxAddr, dstReg, mask);
536 537 
537 MaskReg lpMask;538 MaskReg lpMask;
538 MaskReg selMask;539 MaskReg selMask;
@@ -548,7 +549,7 @@ __aicore__ inline void KernelAsStridedGather<T>::GenDim2Index()
548 * vec_a = vec_a - cmp_a * a549 * vec_a = vec_a - cmp_a * a
549 */550 */
550 Reg::Adds(dim0Reg, dim0Reg, lastDimInc, lpMask);551 Reg::Adds(dim0Reg, dim0Reg, lastDimInc, lpMask);
551- Reg::CompareScalar<IdxType_, CMPMODE::GE>(selMask, dim0Reg, lastDimSize, lpMask);552+ Reg::Compares<IdxType_, CMPMODE::GE>(selMask, dim0Reg, lastDimSize, lpMask);
552 Reg::Select(cmpReg, oneReg, zeroReg, selMask);553 Reg::Select(cmpReg, oneReg, zeroReg, selMask);
553 Reg::Muls(tmpReg, cmpReg, lastDimSize, lpMask);554 Reg::Muls(tmpReg, cmpReg, lastDimSize, lpMask);
554 Reg::Sub(dim0Reg, dim0Reg, tmpReg, lpMask);555 Reg::Sub(dim0Reg, dim0Reg, tmpReg, lpMask);
@@ -559,7 +560,7 @@ __aicore__ inline void KernelAsStridedGather<T>::GenDim2Index()
559 Reg::Muls(tmpReg, dim0Reg, lastDimInOffset, lpMask);560 Reg::Muls(tmpReg, dim0Reg, lastDimInOffset, lpMask);
560 Reg::Muls(dstReg, dim1Reg, last2ndDimInOffset, lpMask);561 Reg::Muls(dstReg, dim1Reg, last2ndDimInOffset, lpMask);
561 Reg::Add(dstReg, dstReg, tmpReg, lpMask);562 Reg::Add(dstReg, dstReg, tmpReg, lpMask);
562- Reg::DataCopy(idxAddr + (lpIdx + 1) * vfLen, dstReg, lpMask);563+ Reg::StoreAlign(idxAddr + (lpIdx + 1) * vfLen, dstReg, lpMask);
563 }564 }
564 }565 }
565}566}
@@ -567,7 +568,7 @@ __aicore__ inline void KernelAsStridedGather<T>::GenDim2Index()
567template <typename T>568template <typename T>
568__aicore__ inline void KernelAsStridedGather<T>::GenDim1Index()569__aicore__ inline void KernelAsStridedGather<T>::GenDim1Index()
569{570{
570- __local_mem__ IdxType_* idxAddr = (__local_mem__ IdxType_*)indexLocal_.GetPhyAddr();571+ __ubuf__ IdxType_* idxAddr = (__ubuf__ IdxType_*)indexLocal_.GetPhyAddr();
571 uint16_t vfLen = VF_LENGTH / sizeof(IdxType_);572 uint16_t vfLen = VF_LENGTH / sizeof(IdxType_);
572 uint32_t lastDimInOffset = strideArr_[dimNum_ - 1];573 uint32_t lastDimInOffset = strideArr_[dimNum_ - 1];
573 uint16_t loopsCnt = CeilDiv(blockUbFactor_, vfLen);574 uint16_t loopsCnt = CeilDiv(blockUbFactor_, vfLen);
@@ -583,7 +584,7 @@ __aicore__ inline void KernelAsStridedGather<T>::GenDim1Index()
583 for (uint16_t lpIdx = 0; lpIdx < loopsCnt; ++lpIdx) {584 for (uint16_t lpIdx = 0; lpIdx < loopsCnt; ++lpIdx) {
584 mask = UpdateMask<IdxType_>(tempBUF);585 mask = UpdateMask<IdxType_>(tempBUF);
585 Reg::Muls(dstReg, srcReg, lastDimInOffset, mask);586 Reg::Muls(dstReg, srcReg, lastDimInOffset, mask);
586- Reg::DataCopy(idxAddr + lpIdx * vfLen, dstReg, mask);587+ Reg::StoreAlign(idxAddr + lpIdx * vfLen, dstReg, mask);
587 Reg::Adds(srcReg, srcReg, vfLen, mask);588 Reg::Adds(srcReg, srcReg, vfLen, mask);
588 }589 }
589 }590 }
Mconversion/as_strided/op_kernel/arch35/as_strided_zero_stride.h+4-4
@@ -122,8 +122,8 @@ __aicore__ inline void StridedIsZero<T>::CopyIn()
122template <typename T>122template <typename T>
123__aicore__ inline void StridedIsZero<T>::Compute()123__aicore__ inline void StridedIsZero<T>::Compute()
124{124{
125- __local_mem__ T* tempInTensorUbAddr = (__local_mem__ T*)tempInTensor_.GetPhyAddr();125+ __ubuf__ T* tempInTensorUbAddr = (__ubuf__ T*)tempInTensor_.GetPhyAddr();
126- __local_mem__ T* xTensorUbAddr = (__local_mem__ T*)xTensor_.GetPhyAddr();126+ __ubuf__ T* xTensorUbAddr = (__ubuf__ T*)xTensor_.GetPhyAddr();
127 127 
128 uint16_t vfLen = Ops::Base::GetVRegSize() / sizeof(T);128 uint16_t vfLen = Ops::Base::GetVRegSize() / sizeof(T);
129 uint32_t dupNum = static_cast<uint32_t>(dupNum_);129 uint32_t dupNum = static_cast<uint32_t>(dupNum_);
@@ -138,12 +138,12 @@ __aicore__ inline void StridedIsZero<T>::Compute()
138 Reg::MaskReg dupMask = Reg::CreateMask<T, Reg::MaskPattern::ALL>();138 Reg::MaskReg dupMask = Reg::CreateMask<T, Reg::MaskPattern::ALL>();
139 Reg::MaskReg lenMask;139 Reg::MaskReg lenMask;
140 140 
141- Reg::DataCopy(tempInRegTensor, tempInTensorUbAddr);141+ Reg::LoadAlign(tempInRegTensor, tempInTensorUbAddr);
142 Reg::Duplicate<T, Reg::HighLowPart::LOWEST, Reg::MaskMergeMode::ZEROING>(tempXTensor, tempInRegTensor, dupMask);142 Reg::Duplicate<T, Reg::HighLowPart::LOWEST, Reg::MaskMergeMode::ZEROING>(tempXTensor, tempInRegTensor, dupMask);
143 143 
144 for (uint16_t loop = 0; loop < loopsCnt; loop++) {144 for (uint16_t loop = 0; loop < loopsCnt; loop++) {
145 lenMask = Reg::UpdateMask<T>(dupNum);145 lenMask = Reg::UpdateMask<T>(dupNum);
146- Reg::DataCopy(xTensorUbAddr + loop * vfLen, tempXTensor, lenMask);146+ Reg::StoreAlign(xTensorUbAddr + loop * vfLen, tempXTensor, lenMask);
147 }147 }
148 }148 }
149}149}
Mconversion/concat/op_kernel/arch35/concat_base.h+22-22
@@ -85,19 +85,19 @@ __aicore__ inline void Copy(const LocalTensor<T>& dstLocal, const LocalTensor<T>
85 85 
86 __VEC_SCOPE__86 __VEC_SCOPE__
87 {87 {
88- AscendC::Reg::UnalignReg u0;88+ AscendC::Reg::UnalignRegForLoad u0;
89- AscendC::Reg::UnalignReg uReg;89+ AscendC::Reg::UnalignRegForStore uReg;
90 AscendC::Reg::RegTensor<T> vd0;90 AscendC::Reg::RegTensor<T> vd0;
91- AscendC::Reg::DataCopyUnAlignPre(u0, srcAddr);91+ AscendC::Reg::LoadUnAlignPre(u0, srcAddr);
92 for (uint16_t i = 0; i < size0; i++) {92 for (uint16_t i = 0; i < size0; i++) {
93 auto curDstAddr = dstAddr + i * rowStride;93 auto curDstAddr = dstAddr + i * rowStride;
94 for (uint16_t j = 0; j < size1; j++) {94 for (uint16_t j = 0; j < size1; j++) {
95- AscendC::Reg::DataCopyUnAlign(vd0, u0, srcAddr, main);95+ AscendC::Reg::LoadUnAlign(vd0, u0, srcAddr, main);
96- AscendC::Reg::DataCopyUnAlign(curDstAddr, vd0, uReg, main);96+ AscendC::Reg::StoreUnAlign(curDstAddr, vd0, uReg, main);
97 }97 }
98- AscendC::Reg::DataCopyUnAlign(vd0, u0, srcAddr, tail);98+ AscendC::Reg::LoadUnAlign(vd0, u0, srcAddr, tail);
99- AscendC::Reg::DataCopyUnAlign(curDstAddr, vd0, uReg, tail);99+ AscendC::Reg::StoreUnAlign(curDstAddr, vd0, uReg, tail);
100- AscendC::Reg::DataCopyUnAlignPost(curDstAddr, uReg, 0);100+ AscendC::Reg::StoreUnAlignPost(curDstAddr, uReg, 0);
101 }101 }
102 }102 }
103}103}
@@ -119,7 +119,7 @@ __aicore__ inline void ScatterConcat(const LocalTensor<T>& dstLocal, const Local
119 119 
120 __VEC_SCOPE__120 __VEC_SCOPE__
121 {121 {
122- AscendC::Reg::UnalignReg u0;122+ AscendC::Reg::UnalignRegForLoad u0;
123 AscendC::Reg::RegTensor<T> gather0;123 AscendC::Reg::RegTensor<T> gather0;
124 AscendC::Reg::RegTensor<T> gather1;124 AscendC::Reg::RegTensor<T> gather1;
125 AscendC::Reg::RegTensor<T> gather2;125 AscendC::Reg::RegTensor<T> gather2;
@@ -134,45 +134,45 @@ __aicore__ inline void ScatterConcat(const LocalTensor<T>& dstLocal, const Local
134 auto tailVf = tail;134 auto tailVf = tail;
135 AscendC::Reg::MaskReg p0 = AscendC::Reg::UpdateMask<U>(allVf);135 AscendC::Reg::MaskReg p0 = AscendC::Reg::UpdateMask<U>(allVf);
136 AscendC::Reg::MaskReg p1 = AscendC::Reg::UpdateMask<U>(tailVf);136 AscendC::Reg::MaskReg p1 = AscendC::Reg::UpdateMask<U>(tailVf);
137- AscendC::Reg::DataCopyUnAlignPre(u0, srcAddr);137+ AscendC::Reg::LoadUnAlignPre(u0, srcAddr);
138 for (uint16_t i = 0; i < size0; i++) {138 for (uint16_t i = 0; i < size0; i++) {
139 auto curDstAddr = dstAddr + i * rowStride;139 auto curDstAddr = dstAddr + i * rowStride;
140 if constexpr (ScatterNum > DIGIT_THREE) {140 if constexpr (ScatterNum > DIGIT_THREE) {
141- AscendC::Reg::DataCopyUnAlign(gather3, u0, srcAddr, main);141+ AscendC::Reg::LoadUnAlign(gather3, u0, srcAddr, main);
142 if constexpr (sizeof(T) == 1) {142 if constexpr (sizeof(T) == 1) {
143 AscendC::Reg::UnPack((AscendC::Reg::RegTensor<uint16_t>&)dst, gather3);143 AscendC::Reg::UnPack((AscendC::Reg::RegTensor<uint16_t>&)dst, gather3);
144- AscendC::Reg::DataCopyScatter(curDstAddr, dst, index, p0);144+ AscendC::Reg::Scatter(curDstAddr, dst, index, p0);
145 } else {145 } else {
146- AscendC::Reg::DataCopyScatter(curDstAddr, gather3, index, p0);146+ AscendC::Reg::Scatter(curDstAddr, gather3, index, p0);
147 }147 }
148 curDstAddr = curDstAddr + main;148 curDstAddr = curDstAddr + main;
149 }149 }
150 if constexpr (ScatterNum > DIGIT_TWO) {150 if constexpr (ScatterNum > DIGIT_TWO) {
151- AscendC::Reg::DataCopyUnAlign(gather2, u0, srcAddr, main);151+ AscendC::Reg::LoadUnAlign(gather2, u0, srcAddr, main);
152 if constexpr (sizeof(T) == 1) {152 if constexpr (sizeof(T) == 1) {
153 AscendC::Reg::UnPack((AscendC::Reg::RegTensor<uint16_t>&)dst, gather2);153 AscendC::Reg::UnPack((AscendC::Reg::RegTensor<uint16_t>&)dst, gather2);
154- AscendC::Reg::DataCopyScatter(curDstAddr, dst, index, p0);154+ AscendC::Reg::Scatter(curDstAddr, dst, index, p0);
155 } else {155 } else {
156- AscendC::Reg::DataCopyScatter(curDstAddr, gather2, index, p0);156+ AscendC::Reg::Scatter(curDstAddr, gather2, index, p0);
157 }157 }
158 curDstAddr = curDstAddr + main;158 curDstAddr = curDstAddr + main;
159 }159 }
160 if constexpr (ScatterNum > 1) {160 if constexpr (ScatterNum > 1) {
161- AscendC::Reg::DataCopyUnAlign(gather1, u0, srcAddr, main);161+ AscendC::Reg::LoadUnAlign(gather1, u0, srcAddr, main);
162 if constexpr (sizeof(T) == 1) {162 if constexpr (sizeof(T) == 1) {
163 AscendC::Reg::UnPack((AscendC::Reg::RegTensor<uint16_t>&)dst, gather1);163 AscendC::Reg::UnPack((AscendC::Reg::RegTensor<uint16_t>&)dst, gather1);
164- AscendC::Reg::DataCopyScatter(curDstAddr, dst, index, p0);164+ AscendC::Reg::Scatter(curDstAddr, dst, index, p0);
165 } else {165 } else {
166- AscendC::Reg::DataCopyScatter(curDstAddr, gather1, index, p0);166+ AscendC::Reg::Scatter(curDstAddr, gather1, index, p0);
167 }167 }
168 curDstAddr = curDstAddr + main;168 curDstAddr = curDstAddr + main;
169 }169 }
170- AscendC::Reg::DataCopyUnAlign(gather0, u0, srcAddr, tail);170+ AscendC::Reg::LoadUnAlign(gather0, u0, srcAddr, tail);
171 if constexpr (sizeof(T) == 1) {171 if constexpr (sizeof(T) == 1) {
172 AscendC::Reg::UnPack((AscendC::Reg::RegTensor<uint16_t>&)dst, gather0);172 AscendC::Reg::UnPack((AscendC::Reg::RegTensor<uint16_t>&)dst, gather0);
173- AscendC::Reg::DataCopyScatter(curDstAddr, dst, index, p1);173+ AscendC::Reg::Scatter(curDstAddr, dst, index, p1);
174 } else {174 } else {
175- AscendC::Reg::DataCopyScatter(curDstAddr, gather0, index, p1);175+ AscendC::Reg::Scatter(curDstAddr, gather0, index, p1);
176 }176 }
177 }177 }
178 }178 }
Mconversion/concat/op_kernel/arch35/one_axis_concat_no_align_diff_shape.h+10-10
@@ -315,7 +315,7 @@ __aicore__ inline void OneAxisConcatNoAlignDiffShape<T, U, TILINGDATA>::ScatterS
315 AscendC::Reg::RegTensor<T> dst1;315 AscendC::Reg::RegTensor<T> dst1;
316 AscendC::Reg::RegTensor<U> vd0;316 AscendC::Reg::RegTensor<U> vd0;
317 AscendC::Reg::RegTensor<U> vd1;317 AscendC::Reg::RegTensor<U> vd1;
318- AscendC::Reg::UnalignReg u0;318+ AscendC::Reg::UnalignRegForLoad u0;
319 319 
320 uint32_t num = (uint32_t)(regFactorDim0 * curLoopHandleCurTensorCols);320 uint32_t num = (uint32_t)(regFactorDim0 * curLoopHandleCurTensorCols);
321 uint32_t tailNum = (uint32_t)(tailRegFactorDim0 * curLoopHandleCurTensorCols);321 uint32_t tailNum = (uint32_t)(tailRegFactorDim0 * curLoopHandleCurTensorCols);
@@ -324,25 +324,25 @@ __aicore__ inline void OneAxisConcatNoAlignDiffShape<T, U, TILINGDATA>::ScatterS
324 AscendC::Reg::MaskReg p0 = AscendC::Reg::UpdateMask<U>(num);324 AscendC::Reg::MaskReg p0 = AscendC::Reg::UpdateMask<U>(num);
325 AscendC::Reg::MaskReg p1 = AscendC::Reg::UpdateMask<U>(tailNum);325 AscendC::Reg::MaskReg p1 = AscendC::Reg::UpdateMask<U>(tailNum);
326 326 
327- AscendC::Reg::DataCopy(vd0, indexAddr);327+ AscendC::Reg::LoadAlign(vd0, indexAddr);
328- AscendC::Reg::DataCopyUnAlignPre(u0, srcAddr);328+ AscendC::Reg::LoadUnAlignPre(u0, srcAddr);
329 for (uint16_t i = 0; i < size0; i++) {329 for (uint16_t i = 0; i < size0; i++) {
330- AscendC::Reg::DataCopyUnAlign(vd2, u0, srcAddr, pnum);330+ AscendC::Reg::LoadUnAlign(vd2, u0, srcAddr, pnum);
331 AscendC::Reg::Adds(vd1, vd0, (U)(i * curLoopHandleCols * regFactorDim0), p0);331 AscendC::Reg::Adds(vd1, vd0, (U)(i * curLoopHandleCols * regFactorDim0), p0);
332 if constexpr (sizeof(T) == 1) {332 if constexpr (sizeof(T) == 1) {
333 AscendC::Reg::Interleave(dst0, dst1, vd2, tmp);333 AscendC::Reg::Interleave(dst0, dst1, vd2, tmp);
334- AscendC::Reg::DataCopyScatter(dstAddr, dst0, vd1, p0);334+ AscendC::Reg::Scatter(dstAddr, dst0, vd1, p0);
335 } else {335 } else {
336- AscendC::Reg::DataCopyScatter(dstAddr, vd2, vd1, p0);336+ AscendC::Reg::Scatter(dstAddr, vd2, vd1, p0);
337 }337 }
338 }338 }
339- AscendC::Reg::DataCopyUnAlign(vd2, u0, srcAddr, tailPnum);339+ AscendC::Reg::LoadUnAlign(vd2, u0, srcAddr, tailPnum);
340 AscendC::Reg::Adds(vd1, vd0, (U)(size0 * curLoopHandleCols * regFactorDim0), p1);340 AscendC::Reg::Adds(vd1, vd0, (U)(size0 * curLoopHandleCols * regFactorDim0), p1);
341 if constexpr (sizeof(T) == 1) {341 if constexpr (sizeof(T) == 1) {
342 AscendC::Reg::Interleave(dst0, dst1, vd2, tmp);342 AscendC::Reg::Interleave(dst0, dst1, vd2, tmp);
343- AscendC::Reg::DataCopyScatter(dstAddr, dst0, vd1, p1);343+ AscendC::Reg::Scatter(dstAddr, dst0, vd1, p1);
344 } else {344 } else {
345- AscendC::Reg::DataCopyScatter(dstAddr, vd2, vd1, p1);345+ AscendC::Reg::Scatter(dstAddr, vd2, vd1, p1);
346 }346 }
347 }347 }
348}348}
@@ -437,7 +437,7 @@ __aicore__ inline void OneAxisConcatNoAlignDiffShape<T, U, TILINGDATA>::GenScatt
437 AscendC::Reg::Muls(vd0, vd2, curLoopHandleCols, p0);437 AscendC::Reg::Muls(vd0, vd2, curLoopHandleCols, p0);
438 AscendC::Reg::Add(vd3, vd0, vd7, p0);438 AscendC::Reg::Add(vd3, vd0, vd7, p0);
439 AscendC::Reg::Adds(vd10, vd3, curTensorStartCols, p0);439 AscendC::Reg::Adds(vd10, vd3, curTensorStartCols, p0);
440- AscendC::Reg::DataCopy(dstAddr, vd10, p0);440+ AscendC::Reg::StoreAlign(dstAddr, vd10, p0);
441 }441 }
442}442}
443 443 
Mconversion/concat/op_kernel/arch35/one_axis_concat_no_align_same_shape_copy.h+8-8
@@ -251,20 +251,20 @@ __aicore__ inline void OneAxisConcatNoAlignCopy<T, TILINGDATA>::ComputeNoSplitDi
251 {251 {
252 AscendC::Reg::RegTensor<T> vd0;252 AscendC::Reg::RegTensor<T> vd0;
253 AscendC::Reg::RegTensor<T> vd1;253 AscendC::Reg::RegTensor<T> vd1;
254- AscendC::Reg::UnalignReg u0;254+ AscendC::Reg::UnalignRegForLoad u0;
255- AscendC::Reg::UnalignReg u1;255+ AscendC::Reg::UnalignRegForStore u1;
256 for (uint16_t i = 0; i < size0; i++) {256 for (uint16_t i = 0; i < size0; i++) {
257 for (uint16_t j = 0; j < inputNum; j++) {257 for (uint16_t j = 0; j < inputNum; j++) {
258 curSrcAddr = srcAddr + j * tensorStride + i * inputCols;258 curSrcAddr = srcAddr + j * tensorStride + i * inputCols;
259 curDstAddr = dstAddr + i * dstColStride + j * inputCols;259 curDstAddr = dstAddr + i * dstColStride + j * inputCols;
260- AscendC::Reg::DataCopyUnAlignPre(u0, curSrcAddr);260+ AscendC::Reg::LoadUnAlignPre(u0, curSrcAddr);
261 for (uint16_t k = 0; k < size1; k++) {261 for (uint16_t k = 0; k < size1; k++) {
262- AscendC::Reg::DataCopyUnAlign(vd0, u0, curSrcAddr, main);262+ AscendC::Reg::LoadUnAlign(vd0, u0, curSrcAddr, main);
263- AscendC::Reg::DataCopyUnAlign(curDstAddr, vd0, u1, main);263+ AscendC::Reg::StoreUnAlign(curDstAddr, vd0, u1, main);
264 }264 }
265- AscendC::Reg::DataCopyUnAlign(vd0, u0, curSrcAddr, tail);265+ AscendC::Reg::LoadUnAlign(vd0, u0, curSrcAddr, tail);
266- AscendC::Reg::DataCopyUnAlign(curDstAddr, vd0, u1, tail);266+ AscendC::Reg::StoreUnAlign(curDstAddr, vd0, u1, tail);
267- AscendC::Reg::DataCopyUnAlignPost(curDstAddr, u1, 0);267+ AscendC::Reg::StoreUnAlignPost(curDstAddr, u1, 0);
268 }268 }
269 }269 }
270 }270 }
Mconversion/concat/op_kernel/arch35/one_axis_concat_no_align_same_shape_gather.h+35-35
@@ -330,7 +330,7 @@ __aicore__ inline void OneAxisConcatNoAlignGather<T, U, TILINGDATA>::GenGatherIn
330 330 
331 AscendC::Reg::Add(vd9, vd1, vd5, p0);331 AscendC::Reg::Add(vd9, vd1, vd5, p0);
332 AscendC::Reg::Add(vd10, vd8, vd9, p0);332 AscendC::Reg::Add(vd10, vd8, vd9, p0);
333- AscendC::Reg::DataCopy(dstAddr, vd10, p0);333+ AscendC::Reg::StoreAlign(dstAddr, vd10, p0);
334 }334 }
335}335}
336 336 
@@ -368,7 +368,7 @@ __aicore__ inline void OneAxisConcatNoAlignGather<T, U, TILINGDATA>::ComputeNoSp
368 AscendC::Reg::RegTensor<U> vd0;368 AscendC::Reg::RegTensor<U> vd0;
369 AscendC::Reg::RegTensor<U> vd1;369 AscendC::Reg::RegTensor<U> vd1;
370 AscendC::Reg::RegTensor<T> vd2;370 AscendC::Reg::RegTensor<T> vd2;
371- AscendC::Reg::UnalignReg u0;371+ AscendC::Reg::UnalignRegForStore u0;
372 372 
373 uint32_t num = static_cast<uint32_t>(regFactorDim0 * cols);373 uint32_t num = static_cast<uint32_t>(regFactorDim0 * cols);
374 uint32_t tailNum = static_cast<uint32_t>(tailRegFactorDim0 * cols);374 uint32_t tailNum = static_cast<uint32_t>(tailRegFactorDim0 * cols);
@@ -377,17 +377,17 @@ __aicore__ inline void OneAxisConcatNoAlignGather<T, U, TILINGDATA>::ComputeNoSp
377 AscendC::Reg::MaskReg p0 = AscendC::Reg::UpdateMask<U>(num);377 AscendC::Reg::MaskReg p0 = AscendC::Reg::UpdateMask<U>(num);
378 AscendC::Reg::MaskReg p1 = AscendC::Reg::UpdateMask<U>(tailNum);378 AscendC::Reg::MaskReg p1 = AscendC::Reg::UpdateMask<U>(tailNum);
379 379 
380- AscendC::Reg::DataCopy(vd0, indexAddr);380+ AscendC::Reg::LoadAlign(vd0, indexAddr);
381 for (uint16_t i = 0; i < size0; i++) {381 for (uint16_t i = 0; i < size0; i++) {
382 AscendC::Reg::Adds(vd1, vd0, (U)(i * stride), p0);382 AscendC::Reg::Adds(vd1, vd0, (U)(i * stride), p0);
383- AscendC::Reg::DataCopyGather(vd2, srcAddr, vd1, p0);383+ AscendC::Reg::Gather(vd2, srcAddr, vd1, p0);
384- AscendC::Reg::DataCopyUnAlign(dstAddr, vd2, u0, main);384+ AscendC::Reg::StoreUnAlign(dstAddr, vd2, u0, main);
385- AscendC::Reg::DataCopyUnAlignPost(dstAddr, u0, 0);385+ AscendC::Reg::StoreUnAlignPost(dstAddr, u0, 0);
386 }386 }
387 AscendC::Reg::Adds(vd1, vd0, (U)(size0 * stride), p1);387 AscendC::Reg::Adds(vd1, vd0, (U)(size0 * stride), p1);
388- AscendC::Reg::DataCopyGather(vd2, srcAddr, vd1, p1);388+ AscendC::Reg::Gather(vd2, srcAddr, vd1, p1);
389- AscendC::Reg::DataCopyUnAlign(dstAddr, vd2, u0, tail);389+ AscendC::Reg::StoreUnAlign(dstAddr, vd2, u0, tail);
390- AscendC::Reg::DataCopyUnAlignPost(dstAddr, u0, 0);390+ AscendC::Reg::StoreUnAlignPost(dstAddr, u0, 0);
391 }391 }
392 392 
393 inQueue_.FreeTensor(srcLocal);393 inQueue_.FreeTensor(srcLocal);
@@ -419,7 +419,7 @@ __aicore__ inline void OneAxisConcatNoAlignGather<T, U, TILINGDATA>::ComputeNoSp
419 AscendC::Reg::RegTensor<uint16_t> vd4;419 AscendC::Reg::RegTensor<uint16_t> vd4;
420 AscendC::Reg::RegTensor<uint8_t> vd5;420 AscendC::Reg::RegTensor<uint8_t> vd5;
421 421 
422- AscendC::Reg::UnalignReg u0;422+ AscendC::Reg::UnalignRegForStore u0;
423 423 
424 uint32_t num = static_cast<uint32_t>(regFactorDim0 * cols);424 uint32_t num = static_cast<uint32_t>(regFactorDim0 * cols);
425 uint32_t tailNum = static_cast<uint32_t>(tailRegFactorDim0 * cols);425 uint32_t tailNum = static_cast<uint32_t>(tailRegFactorDim0 * cols);
@@ -428,19 +428,19 @@ __aicore__ inline void OneAxisConcatNoAlignGather<T, U, TILINGDATA>::ComputeNoSp
428 AscendC::Reg::MaskReg p0 = AscendC::Reg::UpdateMask<U>(num);428 AscendC::Reg::MaskReg p0 = AscendC::Reg::UpdateMask<U>(num);
429 AscendC::Reg::MaskReg p1 = AscendC::Reg::UpdateMask<U>(tailNum);429 AscendC::Reg::MaskReg p1 = AscendC::Reg::UpdateMask<U>(tailNum);
430 430 
431- AscendC::Reg::DataCopy(vd0, indexAddr);431+ AscendC::Reg::LoadAlign(vd0, indexAddr);
432 for (uint16_t i = 0; i < size0; i++) {432 for (uint16_t i = 0; i < size0; i++) {
433 AscendC::Reg::Adds(vd1, vd0, (U)(i * stride), p0);433 AscendC::Reg::Adds(vd1, vd0, (U)(i * stride), p0);
434- AscendC::Reg::DataCopyGather(vd2, srcAddr, vd1, p0);434+ AscendC::Reg::Gather(vd2, srcAddr, vd1, p0);
435 AscendC::Reg::Pack(vd3, vd2);435 AscendC::Reg::Pack(vd3, vd2);
436- AscendC::Reg::DataCopyUnAlign(dstAddr, vd3, u0, main);436+ AscendC::Reg::StoreUnAlign(dstAddr, vd3, u0, main);
437- AscendC::Reg::DataCopyUnAlignPost(dstAddr, u0, 0);437+ AscendC::Reg::StoreUnAlignPost(dstAddr, u0, 0);
438 }438 }
439 AscendC::Reg::Adds(vd1, vd0, (U)(size0 * stride), p1);439 AscendC::Reg::Adds(vd1, vd0, (U)(size0 * stride), p1);
440- AscendC::Reg::DataCopyGather(vd4, srcAddr, vd1, p1);440+ AscendC::Reg::Gather(vd4, srcAddr, vd1, p1);
441 AscendC::Reg::Pack(vd5, vd4);441 AscendC::Reg::Pack(vd5, vd4);
442- AscendC::Reg::DataCopyUnAlign(dstAddr, vd5, u0, tail);442+ AscendC::Reg::StoreUnAlign(dstAddr, vd5, u0, tail);
443- AscendC::Reg::DataCopyUnAlignPost(dstAddr, u0, 0);443+ AscendC::Reg::StoreUnAlignPost(dstAddr, u0, 0);
444 }444 }
445 445 
446 inQueue_.FreeTensor(srcLocal);446 inQueue_.FreeTensor(srcLocal);
@@ -491,8 +491,8 @@ __aicore__ inline void OneAxisConcatNoAlignGather<T, U, TILINGDATA>::ComputeSpli
491 AscendC::Reg::RegTensor<T> vd2;491 AscendC::Reg::RegTensor<T> vd2;
492 AscendC::Reg::RegTensor<U> vd3;492 AscendC::Reg::RegTensor<U> vd3;
493 AscendC::Reg::RegTensor<T> vd4;493 AscendC::Reg::RegTensor<T> vd4;
494- AscendC::Reg::UnalignReg u0;494+ AscendC::Reg::UnalignRegForStore u0;
495- AscendC::Reg::UnalignReg u1;495+ AscendC::Reg::UnalignRegForStore u1;
496 496 
497 uint32_t num = static_cast<uint32_t>(regFactorDim1);497 uint32_t num = static_cast<uint32_t>(regFactorDim1);
498 uint32_t tailNum = static_cast<uint32_t>(tailRegFactorDim1);498 uint32_t tailNum = static_cast<uint32_t>(tailRegFactorDim1);
@@ -502,7 +502,7 @@ __aicore__ inline void OneAxisConcatNoAlignGather<T, U, TILINGDATA>::ComputeSpli
502 uint32_t main = num;502 uint32_t main = num;
503 uint32_t tail = tailNum;503 uint32_t tail = tailNum;
504 504 
505- AscendC::Reg::DataCopy(vd0, indexAddr);505+ AscendC::Reg::LoadAlign(vd0, indexAddr);
506 AscendC::Reg::MaskReg p0 = AscendC::Reg::UpdateMask<U>(pnum);506 AscendC::Reg::MaskReg p0 = AscendC::Reg::UpdateMask<U>(pnum);
507 AscendC::Reg::MaskReg p1 = AscendC::Reg::UpdateMask<U>(tailPnum);507 AscendC::Reg::MaskReg p1 = AscendC::Reg::UpdateMask<U>(tailPnum);
508 uint32_t stride = everyRegCalcTensorNum * tensorStride;508 uint32_t stride = everyRegCalcTensorNum * tensorStride;
@@ -513,15 +513,15 @@ __aicore__ inline void OneAxisConcatNoAlignGather<T, U, TILINGDATA>::ComputeSpli
513 rowStride = i * inputCols;513 rowStride = i * inputCols;
514 for (uint16_t j = 0; j < size1; j++) {514 for (uint16_t j = 0; j < size1; j++) {
515 AscendC::Reg::Adds(vd1, vd0, (U)(rowStride + j * stride), p0);515 AscendC::Reg::Adds(vd1, vd0, (U)(rowStride + j * stride), p0);
516- AscendC::Reg::DataCopyGather(vd2, srcAddr, vd1, p0);516+ AscendC::Reg::Gather(vd2, srcAddr, vd1, p0);
517- AscendC::Reg::DataCopyUnAlign(curDstAddr, vd2, u0, main);517+ AscendC::Reg::StoreUnAlign(curDstAddr, vd2, u0, main);
518- AscendC::Reg::DataCopyUnAlignPost(curDstAddr, u0, 0);518+ AscendC::Reg::StoreUnAlignPost(curDstAddr, u0, 0);
519 }519 }
520 tail = tailNum;520 tail = tailNum;
521 AscendC::Reg::Adds(vd3, vd0, (U)(rowStride + size1 * stride), p1);521 AscendC::Reg::Adds(vd3, vd0, (U)(rowStride + size1 * stride), p1);
522- AscendC::Reg::DataCopyGather(vd4, srcAddr, vd3, p1);522+ AscendC::Reg::Gather(vd4, srcAddr, vd3, p1);
523- AscendC::Reg::DataCopyUnAlign(curDstAddr, vd4, u1, tail);523+ AscendC::Reg::StoreUnAlign(curDstAddr, vd4, u1, tail);
524- AscendC::Reg::DataCopyUnAlignPost(curDstAddr, u1, 0);524+ AscendC::Reg::StoreUnAlignPost(curDstAddr, u1, 0);
525 }525 }
526 }526 }
527 527 
@@ -562,8 +562,8 @@ __aicore__ inline void OneAxisConcatNoAlignGather<T, U, TILINGDATA>::ComputeSpli
562 AscendC::Reg::RegTensor<uint16_t> vd4;562 AscendC::Reg::RegTensor<uint16_t> vd4;
563 AscendC::Reg::RegTensor<uint16_t> vd5;563 AscendC::Reg::RegTensor<uint16_t> vd5;
564 AscendC::Reg::RegTensor<uint8_t> vd6;564 AscendC::Reg::RegTensor<uint8_t> vd6;
565- AscendC::Reg::UnalignReg u0;565+ AscendC::Reg::UnalignRegForStore u0;
566- AscendC::Reg::UnalignReg u1;566+ AscendC::Reg::UnalignRegForStore u1;
567 567 
568 uint32_t num = static_cast<uint32_t>(regFactorDim1);568 uint32_t num = static_cast<uint32_t>(regFactorDim1);
569 uint32_t tailNum = static_cast<uint32_t>(tailRegFactorDim1);569 uint32_t tailNum = static_cast<uint32_t>(tailRegFactorDim1);
@@ -573,7 +573,7 @@ __aicore__ inline void OneAxisConcatNoAlignGather<T, U, TILINGDATA>::ComputeSpli
573 uint32_t main = num;573 uint32_t main = num;
574 uint32_t tail = tailNum;574 uint32_t tail = tailNum;
575 575 
576- AscendC::Reg::DataCopy(vd0, indexAddr);576+ AscendC::Reg::LoadAlign(vd0, indexAddr);
577 AscendC::Reg::MaskReg p1 = AscendC::Reg::UpdateMask<U>(tailPnum);577 AscendC::Reg::MaskReg p1 = AscendC::Reg::UpdateMask<U>(tailPnum);
578 AscendC::Reg::MaskReg p0 = AscendC::Reg::UpdateMask<U>(pnum);578 AscendC::Reg::MaskReg p0 = AscendC::Reg::UpdateMask<U>(pnum);
579 uint32_t stride = everyRegCalcTensorNum * tensorStride;579 uint32_t stride = everyRegCalcTensorNum * tensorStride;
@@ -584,17 +584,17 @@ __aicore__ inline void OneAxisConcatNoAlignGather<T, U, TILINGDATA>::ComputeSpli
584 rowStride = inputCols * i;584 rowStride = inputCols * i;
585 for (uint16_t j = 0; j < size1; j++) {585 for (uint16_t j = 0; j < size1; j++) {
586 AscendC::Reg::Adds(vd1, vd0, (U)(rowStride + j * stride), p0);586 AscendC::Reg::Adds(vd1, vd0, (U)(rowStride + j * stride), p0);
587- AscendC::Reg::DataCopyGather(vd2, srcAddr, vd1, p0);587+ AscendC::Reg::Gather(vd2, srcAddr, vd1, p0);
588 AscendC::Reg::Pack(vd3, vd2);588 AscendC::Reg::Pack(vd3, vd2);
589- AscendC::Reg::DataCopyUnAlign(curDstAddr, vd3, u0, main);589+ AscendC::Reg::StoreUnAlign(curDstAddr, vd3, u0, main);
590- AscendC::Reg::DataCopyUnAlignPost(curDstAddr, u0, 0);590+ AscendC::Reg::StoreUnAlignPost(curDstAddr, u0, 0);
591 }591 }
592 tail = tailNum;592 tail = tailNum;
593 AscendC::Reg::Adds(vd4, vd0, (U)(rowStride + size1 * stride), p1);593 AscendC::Reg::Adds(vd4, vd0, (U)(rowStride + size1 * stride), p1);
594- AscendC::Reg::DataCopyGather(vd5, srcAddr, vd4, p1);594+ AscendC::Reg::Gather(vd5, srcAddr, vd4, p1);
595 AscendC::Reg::Pack(vd6, vd5);595 AscendC::Reg::Pack(vd6, vd5);
596- AscendC::Reg::DataCopyUnAlign(curDstAddr, vd6, u1, tail);596+ AscendC::Reg::StoreUnAlign(curDstAddr, vd6, u1, tail);
597- AscendC::Reg::DataCopyUnAlignPost(curDstAddr, u1, 0);597+ AscendC::Reg::StoreUnAlignPost(curDstAddr, u1, 0);
598 }598 }
599 }599 }
600 600 
Mconversion/dynamic_partition/op_kernel/arch35/dynamic_partition_base.h+18-18
@@ -178,17 +178,17 @@ __aicore__ inline void DynPartBase<T>::RefreshOutputShapes(LocalTensor<uint64_t>
178 uint16_t lpCnt = static_cast<uint16_t>(shapeLpUnit_ - 1);178 uint16_t lpCnt = static_cast<uint16_t>(shapeLpUnit_ - 1);
179 if (lpCnt > 0) {179 if (lpCnt > 0) {
180 InsertSync(HardEvent::S_V);180 InsertSync(HardEvent::S_V);
181- __local_mem__ uint64_t* ptrFirstShape = (__local_mem__ uint64_t*)ubOutShape.GetPhyAddr();181+ __ubuf__ uint64_t* ptrFirstShape = (__ubuf__ uint64_t*)ubOutShape.GetPhyAddr();
182- __local_mem__ uint64_t* ptrSecondShape = ptrFirstShape + SHAPE_GAP;182+ __ubuf__ uint64_t* ptrSecondShape = ptrFirstShape + SHAPE_GAP;
183 __VEC_SCOPE__183 __VEC_SCOPE__
184 {184 {
185 RegTensor<uint64_t> shapeInfo;185 RegTensor<uint64_t> shapeInfo;
186- Reg::UnalignReg ureg;186+ Reg::UnalignRegForStore ureg;
187- Reg::DataCopy(shapeInfo, ptrFirstShape);187+ Reg::LoadAlign(shapeInfo, ptrFirstShape);
188 for (uint16_t j = 0; j < lpCnt; ++j) {188 for (uint16_t j = 0; j < lpCnt; ++j) {
189- Reg::DataCopyUnAlign(ptrSecondShape, shapeInfo, ureg, SHAPE_GAP);189+ Reg::StoreUnAlign(ptrSecondShape, shapeInfo, ureg, SHAPE_GAP);
190 }190 }
191- Reg::DataCopyUnAlignPost(ptrSecondShape, ureg);191+ Reg::StoreUnAlignPost(ptrSecondShape, ureg);
192 }192 }
193 InsertSync(HardEvent::V_S);193 InsertSync(HardEvent::V_S);
194 }194 }
@@ -263,14 +263,14 @@ __aicore__ inline void DynPartBase<T>::MultiplePartCopyOutX(LocalTensor<uint64_t
263 auto ubXIn = xInQue_.DeQue<T>();263 auto ubXIn = xInQue_.DeQue<T>();
264 auto ubPartIn = pR2InQue_.DeQue<int32_t>();264 auto ubPartIn = pR2InQue_.DeQue<int32_t>();
265 auto ubXOut = xOutQue_.AllocTensor<T>();265 auto ubXOut = xOutQue_.AllocTensor<T>();
266- __local_mem__ T* ptrXOut = (__local_mem__ T*)ubXOut.GetPhyAddr();266+ __ubuf__ T* ptrXOut = (__ubuf__ T*)ubXOut.GetPhyAddr();
267- __local_mem__ T* ptrXIn = (__local_mem__ T*)ubXIn.GetPhyAddr();267+ __ubuf__ T* ptrXIn = (__ubuf__ T*)ubXIn.GetPhyAddr();
268 uint32_t ubOffset = 0;268 uint32_t ubOffset = 0;
269 uint16_t partLpCnt = static_cast<uint16_t>(Ops::Base::CeilDiv(partLen, b32VLSize_));269 uint16_t partLpCnt = static_cast<uint16_t>(Ops::Base::CeilDiv(partLen, b32VLSize_));
270 int32_t int32VL = static_cast<int32_t>(b32VLSize_);270 int32_t int32VL = static_cast<int32_t>(b32VLSize_);
271 for (int32_t partID = begPart; partID < endPart; ++partID) {271 for (int32_t partID = begPart; partID < endPart; ++partID) {
272- __local_mem__ int32_t* ptrPartIn = (__local_mem__ int32_t*)ubPartIn.GetPhyAddr();272+ __ubuf__ int32_t* ptrPartIn = (__ubuf__ int32_t*)ubPartIn.GetPhyAddr();
273- __local_mem__ int32_t* ptrPartMid = (__local_mem__ int32_t*)ubPartMid.GetPhyAddr();273+ __ubuf__ int32_t* ptrPartMid = (__ubuf__ int32_t*)ubPartMid.GetPhyAddr();
274 uint32_t partSize = partLen;274 uint32_t partSize = partLen;
275 __VEC_SCOPE__275 __VEC_SCOPE__
276 {276 {
@@ -278,19 +278,19 @@ __aicore__ inline void DynPartBase<T>::MultiplePartCopyOutX(LocalTensor<uint64_t
278 RegTensor<int32_t> partMid;278 RegTensor<int32_t> partMid;
279 RegTensor<int32_t> alphaIdx;279 RegTensor<int32_t> alphaIdx;
280 Reg::Arange(alphaIdx, int32_t(0));280 Reg::Arange(alphaIdx, int32_t(0));
281- Reg::UnalignReg ureg;281+ Reg::UnalignRegForStore ureg;
282 MaskReg validMask;282 MaskReg validMask;
283 MaskReg cmpMask;283 MaskReg cmpMask;
284 Reg::ClearSpr<SpecialPurposeReg::AR>();284 Reg::ClearSpr<SpecialPurposeReg::AR>();
285 for (uint16_t partLpIdx = 0; partLpIdx < partLpCnt; ++partLpIdx) {285 for (uint16_t partLpIdx = 0; partLpIdx < partLpCnt; ++partLpIdx) {
286 validMask = UpdateMask<int32_t>(partSize);286 validMask = UpdateMask<int32_t>(partSize);
287- Reg::DataCopy<int32_t, Reg::PostLiteral::POST_MODE_UPDATE>(partIn, ptrPartIn, int32VL);287+ Reg::LoadAlign<int32_t, Reg::PostLiteral::POST_MODE_UPDATE>(partIn, ptrPartIn, int32VL);
288- Reg::CompareScalar(cmpMask, partIn, partID, validMask);288+ Reg::Compares(cmpMask, partIn, partID, validMask);
289- Reg::GatherMask<int32_t, Reg::GatherMaskMode::STORE_REG>(partMid, alphaIdx, cmpMask);289+ Reg::Squeeze<int32_t, Reg::GatherMaskMode::STORE_REG>(partMid, alphaIdx, cmpMask);
290- Reg::DataCopyUnAlign<int32_t, Reg::PostLiteral::POST_MODE_UPDATE>(ptrPartMid, partMid, ureg);290+ Reg::StoreUnAlign<int32_t, Reg::PostLiteral::POST_MODE_UPDATE>(ptrPartMid, partMid, ureg);
291 Reg::Adds(alphaIdx, alphaIdx, int32VL, validMask);291 Reg::Adds(alphaIdx, alphaIdx, int32VL, validMask);
292 }292 }
293- Reg::DataCopyUnAlignPost(ptrPartMid, ureg);293+ Reg::StoreUnAlignPost(ptrPartMid, ureg);
294 }294 }
295 295 
296 uint32_t vPartCnt = static_cast<uint32_t>(Reg::GetSpr<SpecialPurposeReg::AR>() / sizeof(int32_t));296 uint32_t vPartCnt = static_cast<uint32_t>(Reg::GetSpr<SpecialPurposeReg::AR>() / sizeof(int32_t));
@@ -308,8 +308,8 @@ __aicore__ inline void DynPartBase<T>::MultiplePartCopyOutX(LocalTensor<uint64_t
308 MaskReg mask;308 MaskReg mask;
309 for (uint16_t wLpIdx = 0; wLpIdx < wLpCnt; ++wLpIdx) {309 for (uint16_t wLpIdx = 0; wLpIdx < wLpCnt; ++wLpIdx) {
310 mask = UpdateMask<T>(wSize);310 mask = UpdateMask<T>(wSize);
311- Reg::DataCopy(xReg, ptrXIn + curWIdx * wAlign + wLpIdx * vlSize_);311+ Reg::LoadAlign(xReg, ptrXIn + curWIdx * wAlign + wLpIdx * vlSize_);
312- Reg::DataCopy(ptrXOut + vPIdx * wAlign + wLpIdx * vlSize_, xReg, mask);312+ Reg::StoreAlign(ptrXOut + vPIdx * wAlign + wLpIdx * vlSize_, xReg, mask);
313 }313 }
314 }314 }
315 }315 }
Mconversion/dynamic_partition/op_kernel/arch35/dynamic_partition_with_h_mc.h+8-8
@@ -110,9 +110,9 @@ __aicore__ inline void DynPartWithHMC<T, isSingleW>::InnerProcess4CalcPartCnt(ui
110 auto ubPart = pR1InQue_.template DeQue<int32_t>();110 auto ubPart = pR1InQue_.template DeQue<int32_t>();
111 auto ubPartCnt = this->pCntQue_.template DeQue<uint64_t>();111 auto ubPartCnt = this->pCntQue_.template DeQue<uint64_t>();
112 auto ubPartBase = this->pBaseQue_.template DeQue<uint64_t>();112 auto ubPartBase = this->pBaseQue_.template DeQue<uint64_t>();
113- __local_mem__ int32_t* ptrPart = (__local_mem__ int32_t*)ubPart.GetPhyAddr();113+ __ubuf__ int32_t* ptrPart = (__ubuf__ int32_t*)ubPart.GetPhyAddr();
114- __local_mem__ uint64_t* ptrPartBase = (__local_mem__ uint64_t*)ubPartBase.GetPhyAddr();114+ __ubuf__ uint64_t* ptrPartBase = (__ubuf__ uint64_t*)ubPartBase.GetPhyAddr();
115- __local_mem__ int32_t* ptrPartMid = reinterpret_cast<__local_mem__ int32_t*>(ptrPartBase);115+ __ubuf__ int32_t* ptrPartMid = reinterpret_cast<__ubuf__ int32_t*>(ptrPartBase);
116 constexpr auto dFactor = sizeof(uint64_t) / sizeof(int32_t);116 constexpr auto dFactor = sizeof(uint64_t) / sizeof(int32_t);
117 uint16_t pLpCntVF = static_cast<uint16_t>(Ops::Base::CeilDiv(ubPartLen, this->b32VLSize_));117 uint16_t pLpCntVF = static_cast<uint16_t>(Ops::Base::CeilDiv(ubPartLen, this->b32VLSize_));
118 118 
@@ -133,14 +133,14 @@ __aicore__ inline void DynPartWithHMC<T, isSingleW>::InnerProcess4CalcPartCnt(ui
133 133 
134 for (uint16_t i = 0; i < pLpCntVF; ++i) {134 for (uint16_t i = 0; i < pLpCntVF; ++i) {
135 mask = UpdateMask<int32_t>(partSize);135 mask = UpdateMask<int32_t>(partSize);
136- Reg::DataCopy(parts, ptrPart + i * this->b32VLSize_);136+ Reg::LoadAlign(parts, ptrPart + i * this->b32VLSize_);
137- Reg::CompareScalar(cmpMask, parts, partIdx, mask);137+ Reg::Compares(cmpMask, parts, partIdx, mask);
138- Reg::ReduceSum(midPartCnt, allTensor, cmpMask);138+ Reg::Reduce<Reg::ReduceType::SUM>(midPartCnt, allTensor, cmpMask);
139 Reg::Add(partCnt, partCnt, midPartCnt, vl1Mask);139 Reg::Add(partCnt, partCnt, midPartCnt, vl1Mask);
140 }140 }
141 // partCnt must be less than max int32_t141 // partCnt must be less than max int32_t
142- Reg::DataCopy<int32_t, Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(ptrPartMid + modPartIdx * dFactor, partCnt,142+ Reg::StoreAlign<int32_t, Reg::StoreDist::DIST_FIRST_ELEMENT_B32>(ptrPartMid + modPartIdx * dFactor, partCnt,
143- vl1Mask);143+ vl1Mask);
144 }144 }
145 }145 }
146 Add(ubPartCnt, ubPartCnt, ubPartBase, this->coreWS_);146 Add(ubPartCnt, ubPartCnt, ubPartBase, this->coreWS_);
Mconversion/dynamic_partition/op_kernel/arch35/dynamic_partition_with_h_mc_small_w.h+23-25
@@ -38,11 +38,9 @@ private:
38 __aicore__ inline void MultiplePartGatherOutX(LocalTensor<uint64_t>& ubPartBase, uint32_t partLen, int32_t begPart,38 __aicore__ inline void MultiplePartGatherOutX(LocalTensor<uint64_t>& ubPartBase, uint32_t partLen, int32_t begPart,
39 int32_t endPart);39 int32_t endPart);
40 __aicore__ inline void GatherB32B64(uint32_t processNum, uint16_t loopCnt, uint32_t loopTailNum,40 __aicore__ inline void GatherB32B64(uint32_t processNum, uint16_t loopCnt, uint32_t loopTailNum,
41- __local_mem__ int32_t* ptrPartMid, __local_mem__ T* ptrXIn,41+ __ubuf__ int32_t* ptrPartMid, __ubuf__ T* ptrXIn, __ubuf__ T* ptrXOut);
42- __local_mem__ T* ptrXOut);
43 __aicore__ inline void GatherB8B16(uint32_t processNum, uint16_t loopCnt, uint32_t loopTailNum,42 __aicore__ inline void GatherB8B16(uint32_t processNum, uint16_t loopCnt, uint32_t loopTailNum,
44- __local_mem__ int32_t* ptrPartMid, __local_mem__ T* ptrXIn,43+ __ubuf__ int32_t* ptrPartMid, __ubuf__ T* ptrXIn, __ubuf__ T* ptrXOut);
45- __local_mem__ T* ptrXOut);
46 44 
47private:45private:
48 const DynPartTilingData* tdPtr_ = nullptr;46 const DynPartTilingData* tdPtr_ = nullptr;
@@ -127,13 +125,13 @@ __aicore__ inline void DynPartWithHMCSMALLW<T>::CopyPInBrc(int64_t hLpIdx, uint3
127{125{
128 auto ubR2PIn = this->pR2InQue_.template AllocTensor<int32_t>();126 auto ubR2PIn = this->pR2InQue_.template AllocTensor<int32_t>();
129 constexpr uint8_t dim = NUM_TWO;127 constexpr uint8_t dim = NUM_TWO;
130- static constexpr MultiCopyConfig config = {false};128+ static constexpr NdDmaConfig config = {false};
131- MultiCopyLoopInfo<dim> loopInfo = {.loopSrcStride = {0, 1},129+ NdDmaLoopInfo<dim> loopInfo = {.loopSrcStride = {0, 1},
132- .loopDstStride = {1, wLen},130+ .loopDstStride = {1, wLen},
133- .loopSize = {wLen, hLen},131+ .loopSize = {wLen, hLen},
134- .loopLpSize = {0, 0},132+ .loopLpSize = {0, 0},
135- .loopRpSize = {0, 0}};133+ .loopRpSize = {0, 0}};
136- MultiCopyParams<int32_t, dim> copyInParams = {loopInfo, 0};134+ NdDmaParams<int32_t, dim> copyInParams = {loopInfo, 0};
137 DataCopy<int32_t, dim, config>(ubR2PIn, this->pInGM_[this->blockIdx_ * tdPtr_->hMSize + hLpIdx * tdPtr_->hLpUnit],135 DataCopy<int32_t, dim, config>(ubR2PIn, this->pInGM_[this->blockIdx_ * tdPtr_->hMSize + hLpIdx * tdPtr_->hLpUnit],
138 copyInParams);136 copyInParams);
139 this->pR2InQue_.EnQue(ubR2PIn);137 this->pR2InQue_.EnQue(ubR2PIn);
@@ -167,14 +165,14 @@ __aicore__ inline void DynPartWithHMCSMALLW<T>::MultiplePartGatherOutX(LocalTens
167 auto ubXIn = this->xInQue_.template DeQue<T>();165 auto ubXIn = this->xInQue_.template DeQue<T>();
168 auto ubPartIn = this->pR2InQue_.template DeQue<int32_t>();166 auto ubPartIn = this->pR2InQue_.template DeQue<int32_t>();
169 auto ubXOut = this->xOutQue_.template AllocTensor<T>();167 auto ubXOut = this->xOutQue_.template AllocTensor<T>();
170- __local_mem__ T* ptrXOut = (__local_mem__ T*)ubXOut.GetPhyAddr();168+ __ubuf__ T* ptrXOut = (__ubuf__ T*)ubXOut.GetPhyAddr();
171- __local_mem__ T* ptrXIn = (__local_mem__ T*)ubXIn.GetPhyAddr();169+ __ubuf__ T* ptrXIn = (__ubuf__ T*)ubXIn.GetPhyAddr();
172 uint32_t ubOffset = 0;170 uint32_t ubOffset = 0;
173 uint16_t partLpCnt = static_cast<uint16_t>(Ops::Base::CeilDiv(partLen * wLen, this->b32VLSize_));171 uint16_t partLpCnt = static_cast<uint16_t>(Ops::Base::CeilDiv(partLen * wLen, this->b32VLSize_));
174 int32_t int32VL = static_cast<int32_t>(this->b32VLSize_);172 int32_t int32VL = static_cast<int32_t>(this->b32VLSize_);
175 for (int32_t partID = begPart; partID < endPart; ++partID) {173 for (int32_t partID = begPart; partID < endPart; ++partID) {
176- __local_mem__ int32_t* ptrPartIn = (__local_mem__ int32_t*)ubPartIn.GetPhyAddr();174+ __ubuf__ int32_t* ptrPartIn = (__ubuf__ int32_t*)ubPartIn.GetPhyAddr();
177- __local_mem__ int32_t* ptrPartMid = (__local_mem__ int32_t*)ubPartMid.GetPhyAddr();175+ __ubuf__ int32_t* ptrPartMid = (__ubuf__ int32_t*)ubPartMid.GetPhyAddr();
178 uint32_t partSize = partLen * wLen;176 uint32_t partSize = partLen * wLen;
179 __VEC_SCOPE__177 __VEC_SCOPE__
180 {178 {
@@ -182,19 +180,19 @@ __aicore__ inline void DynPartWithHMCSMALLW<T>::MultiplePartGatherOutX(LocalTens
182 RegTensor<int32_t> partMid;180 RegTensor<int32_t> partMid;
183 RegTensor<int32_t> alphaIdx;181 RegTensor<int32_t> alphaIdx;
184 Reg::Arange(alphaIdx, int32_t(0));182 Reg::Arange(alphaIdx, int32_t(0));
185- Reg::UnalignReg ureg;183+ Reg::UnalignRegForStore ureg;
186 MaskReg validMask;184 MaskReg validMask;
187 MaskReg cmpMask;185 MaskReg cmpMask;
188 Reg::ClearSpr<SpecialPurposeReg::AR>();186 Reg::ClearSpr<SpecialPurposeReg::AR>();
189 for (uint16_t partLpIdx = 0; partLpIdx < partLpCnt; ++partLpIdx) {187 for (uint16_t partLpIdx = 0; partLpIdx < partLpCnt; ++partLpIdx) {
190 validMask = UpdateMask<int32_t>(partSize);188 validMask = UpdateMask<int32_t>(partSize);
191- Reg::DataCopy<int32_t, Reg::PostLiteral::POST_MODE_UPDATE>(partIn, ptrPartIn, int32VL);189+ Reg::LoadAlign<int32_t, Reg::PostLiteral::POST_MODE_UPDATE>(partIn, ptrPartIn, int32VL);
192- Reg::CompareScalar(cmpMask, partIn, partID, validMask);190+ Reg::Compares(cmpMask, partIn, partID, validMask);
193- Reg::GatherMask<int32_t, Reg::GatherMaskMode::STORE_REG>(partMid, alphaIdx, cmpMask);191+ Reg::Squeeze<int32_t, Reg::GatherMaskMode::STORE_REG>(partMid, alphaIdx, cmpMask);
194- Reg::DataCopyUnAlign<int32_t, Reg::PostLiteral::POST_MODE_UPDATE>(ptrPartMid, partMid, ureg);192+ Reg::StoreUnAlign<int32_t, Reg::PostLiteral::POST_MODE_UPDATE>(ptrPartMid, partMid, ureg);
195 Reg::Adds(alphaIdx, alphaIdx, int32VL, validMask);193 Reg::Adds(alphaIdx, alphaIdx, int32VL, validMask);
196 }194 }
197- Reg::DataCopyUnAlignPost(ptrPartMid, ureg);195+ Reg::StoreUnAlignPost(ptrPartMid, ureg);
198 }196 }
199 197 
200 uint32_t vPartCnt = static_cast<uint32_t>(Reg::GetSpr<SpecialPurposeReg::AR>() / sizeof(int32_t));198 uint32_t vPartCnt = static_cast<uint32_t>(Reg::GetSpr<SpecialPurposeReg::AR>() / sizeof(int32_t));
@@ -227,8 +225,8 @@ __aicore__ inline void DynPartWithHMCSMALLW<T>::MultiplePartGatherOutX(LocalTens
227 225 
228template <typename T>226template <typename T>
229__aicore__ inline void DynPartWithHMCSMALLW<T>::GatherB32B64(uint32_t processNum, uint16_t loopCnt,227__aicore__ inline void DynPartWithHMCSMALLW<T>::GatherB32B64(uint32_t processNum, uint16_t loopCnt,
230- uint32_t loopTailNum, __local_mem__ int32_t* ptrPartMid,228+ uint32_t loopTailNum, __ubuf__ int32_t* ptrPartMid,
231- __local_mem__ T* ptrXIn, __local_mem__ T* ptrXOut)229+ __ubuf__ T* ptrXIn, __ubuf__ T* ptrXOut)
232{230{
233 __VEC_SCOPE__231 __VEC_SCOPE__
234 {232 {
@@ -254,8 +252,8 @@ __aicore__ inline void DynPartWithHMCSMALLW<T>::GatherB32B64(uint32_t processNum
254 252 
255template <typename T>253template <typename T>
256__aicore__ inline void DynPartWithHMCSMALLW<T>::GatherB8B16(uint32_t processNum, uint16_t loopCnt, uint32_t loopTailNum,254__aicore__ inline void DynPartWithHMCSMALLW<T>::GatherB8B16(uint32_t processNum, uint16_t loopCnt, uint32_t loopTailNum,
257- __local_mem__ int32_t* ptrPartMid, __local_mem__ T* ptrXIn,255+ __ubuf__ int32_t* ptrPartMid, __ubuf__ T* ptrXIn,
258- __local_mem__ T* ptrXOut)256+ __ubuf__ T* ptrXOut)
259{257{
260 __VEC_SCOPE__258 __VEC_SCOPE__
261 {259 {
Mconversion/split_v/op_kernel/arch35/split_v_ub_split.h+31-32
@@ -148,7 +148,7 @@ __aicore__ inline void DataCopyGatherScope(int32_t vfLenU, int32_t ubSizeNum, in
148 AscendC::Reg::RegTensor<T1> tmp2;148 AscendC::Reg::RegTensor<T1> tmp2;
149 AscendC::Reg::RegTensor<T1> subReg;149 AscendC::Reg::RegTensor<T1> subReg;
150 AscendC::Reg::RegTensor<T1> niReg;150 AscendC::Reg::RegTensor<T1> niReg;
151- AscendC::Reg::UnalignReg u0;151+ AscendC::Reg::UnalignRegForStore u0;
152 p0 = AscendC::Reg::UpdateMask<T1>(mask1);152 p0 = AscendC::Reg::UpdateMask<T1>(mask1);
153 AscendC::Reg::Duplicate(niReg, (T1)niSize, p0);153 AscendC::Reg::Duplicate(niReg, (T1)niSize, p0);
154 AscendC::Reg::Arange(indexReg, 0);154 AscendC::Reg::Arange(indexReg, 0);
@@ -158,19 +158,19 @@ __aicore__ inline void DataCopyGatherScope(int32_t vfLenU, int32_t ubSizeNum, in
158 AscendC::Reg::Sub(tmp2, (AscendC::Reg::RegTensor<T1>&)indexReg, subReg, p0);158 AscendC::Reg::Sub(tmp2, (AscendC::Reg::RegTensor<T1>&)indexReg, subReg, p0);
159 AscendC::Reg::Add(addReg, tmp1, tmp2, p0);159 AscendC::Reg::Add(addReg, tmp1, tmp2, p0);
160 AscendC::Reg::Adds(addReg, addReg, (T1)ubStartOffset, p0);160 AscendC::Reg::Adds(addReg, addReg, (T1)ubStartOffset, p0);
161- AscendC::Reg::DataCopyGather(dstReg, srcPtr, addReg, p0);161+ AscendC::Reg::Gather(dstReg, srcPtr, addReg, p0);
162- AscendC::Reg::DataCopy(dstPtr, dstReg, p0);162+ AscendC::Reg::StoreAlign(dstPtr, dstReg, p0);
163 for (uint16_t ii = 0; ii < times; ii++) {163 for (uint16_t ii = 0; ii < times; ii++) {
164 AscendC::Reg::Adds(addReg1, addReg, (T1)(offset * (ii + 1)), p0);164 AscendC::Reg::Adds(addReg1, addReg, (T1)(offset * (ii + 1)), p0);
165- AscendC::Reg::DataCopyGather(dstReg, srcPtr, addReg1, p0);165+ AscendC::Reg::Gather(dstReg, srcPtr, addReg1, p0);
166- AscendC::Reg::DataCopyUnAlign(curDstPtr, dstReg, u0, num);166+ AscendC::Reg::StoreUnAlign(curDstPtr, dstReg, u0, num);
167- AscendC::Reg::DataCopyUnAlignPost(curDstPtr, u0, 0);167+ AscendC::Reg::StoreUnAlignPost(curDstPtr, u0, 0);
168 }168 }
169 p1 = AscendC::Reg::UpdateMask<T1>(mask2);169 p1 = AscendC::Reg::UpdateMask<T1>(mask2);
170 AscendC::Reg::Adds(addReg1, addReg, (T1)(offset * (times + 1)), p1);170 AscendC::Reg::Adds(addReg1, addReg, (T1)(offset * (times + 1)), p1);
171- AscendC::Reg::DataCopyGather(dstReg, srcPtr, addReg1, p1);171+ AscendC::Reg::Gather(dstReg, srcPtr, addReg1, p1);
172- AscendC::Reg::DataCopyUnAlign(curDstPtr, dstReg, u0, tail);172+ AscendC::Reg::StoreUnAlign(curDstPtr, dstReg, u0, tail);
173- AscendC::Reg::DataCopyUnAlignPost(curDstPtr, u0, 0);173+ AscendC::Reg::StoreUnAlignPost(curDstPtr, u0, 0);
174 }174 }
175}175}
176 176 
@@ -216,7 +216,7 @@ __aicore__ inline void SplitVUbSplit<T, U, Y>::DataCopyGatherVf(int64_t mFactor,
216 AscendC::Reg::RegTensor<U> subReg;216 AscendC::Reg::RegTensor<U> subReg;
217 AscendC::Reg::RegTensor<U> niReg;217 AscendC::Reg::RegTensor<U> niReg;
218 AscendC::Reg::RegTensor<T> dstRegO;218 AscendC::Reg::RegTensor<T> dstRegO;
219- AscendC::Reg::UnalignReg u0;219+ AscendC::Reg::UnalignRegForStore u0;
220 p0 = AscendC::Reg::UpdateMask<U>(mask1);220 p0 = AscendC::Reg::UpdateMask<U>(mask1);
221 AscendC::Reg::Duplicate(niReg, (U)niSize, p0);221 AscendC::Reg::Duplicate(niReg, (U)niSize, p0);
222 AscendC::Reg::Arange(indexReg, 0);222 AscendC::Reg::Arange(indexReg, 0);
@@ -226,34 +226,34 @@ __aicore__ inline void SplitVUbSplit<T, U, Y>::DataCopyGatherVf(int64_t mFactor,
226 AscendC::Reg::Sub(tmp2, (AscendC::Reg::RegTensor<U>&)indexReg, subReg, p0);226 AscendC::Reg::Sub(tmp2, (AscendC::Reg::RegTensor<U>&)indexReg, subReg, p0);
227 AscendC::Reg::Add(addReg, tmp1, tmp2, p0);227 AscendC::Reg::Add(addReg, tmp1, tmp2, p0);
228 AscendC::Reg::Adds(addReg, addReg, (U)ubStartOffset, p0);228 AscendC::Reg::Adds(addReg, addReg, (U)ubStartOffset, p0);
229- AscendC::Reg::DataCopyGather(dstReg, srcPtr, addReg, p0);229+ AscendC::Reg::Gather(dstReg, srcPtr, addReg, p0);
230 if constexpr (sizeof(T) == sizeof(int8_t)) {230 if constexpr (sizeof(T) == sizeof(int8_t)) {
231- AscendC::Reg::DataCopy<uint16_t, AscendC::Reg::StoreDist::DIST_PACK_B16>(dstPtr, dstReg, p0);231+ AscendC::Reg::StoreAlign<uint16_t, AscendC::Reg::StoreDist::DIST_PACK_B16>(dstPtr, dstReg, p0);
232 } else {232 } else {
233- AscendC::Reg::DataCopy(dstPtr, dstReg, p0);233+ AscendC::Reg::StoreAlign(dstPtr, dstReg, p0);
234 }234 }
235 for (uint16_t ii = 0; ii < times; ii++) {235 for (uint16_t ii = 0; ii < times; ii++) {
236 AscendC::Reg::Adds(addReg1, addReg, (U)(offset * (ii + 1)), p0);236 AscendC::Reg::Adds(addReg1, addReg, (U)(offset * (ii + 1)), p0);
237- AscendC::Reg::DataCopyGather(dstReg, srcPtr, addReg1, p0);237+ AscendC::Reg::Gather(dstReg, srcPtr, addReg1, p0);
238 if constexpr (sizeof(T) == sizeof(int8_t)) {238 if constexpr (sizeof(T) == sizeof(int8_t)) {
239 AscendC::Reg::Pack(dstRegO, dstReg);239 AscendC::Reg::Pack(dstRegO, dstReg);
240- AscendC::Reg::DataCopyUnAlign(curDstPtr, dstRegO, u0, num);240+ AscendC::Reg::StoreUnAlign(curDstPtr, dstRegO, u0, num);
241- AscendC::Reg::DataCopyUnAlignPost(curDstPtr, u0, 0);241+ AscendC::Reg::StoreUnAlignPost(curDstPtr, u0, 0);
242 } else {242 } else {
243- AscendC::Reg::DataCopyUnAlign(curDstPtr, dstReg, u0, num);243+ AscendC::Reg::StoreUnAlign(curDstPtr, dstReg, u0, num);
244- AscendC::Reg::DataCopyUnAlignPost(curDstPtr, u0, 0);244+ AscendC::Reg::StoreUnAlignPost(curDstPtr, u0, 0);
245 }245 }
246 }246 }
247 p1 = AscendC::Reg::UpdateMask<U>(mask2);247 p1 = AscendC::Reg::UpdateMask<U>(mask2);
248 AscendC::Reg::Adds(addReg1, addReg, (U)(offset * (times + 1)), p1);248 AscendC::Reg::Adds(addReg1, addReg, (U)(offset * (times + 1)), p1);
249- AscendC::Reg::DataCopyGather(dstReg, srcPtr, addReg1, p1);249+ AscendC::Reg::Gather(dstReg, srcPtr, addReg1, p1);
250 if constexpr (sizeof(T) == sizeof(int8_t)) {250 if constexpr (sizeof(T) == sizeof(int8_t)) {
251 AscendC::Reg::Pack(dstRegO, dstReg);251 AscendC::Reg::Pack(dstRegO, dstReg);
252- AscendC::Reg::DataCopyUnAlign(curDstPtr, dstRegO, u0, tail);252+ AscendC::Reg::StoreUnAlign(curDstPtr, dstRegO, u0, tail);
253- AscendC::Reg::DataCopyUnAlignPost(curDstPtr, u0, 0);253+ AscendC::Reg::StoreUnAlignPost(curDstPtr, u0, 0);
254 } else {254 } else {
255- AscendC::Reg::DataCopyUnAlign(curDstPtr, dstReg, u0, tail);255+ AscendC::Reg::StoreUnAlign(curDstPtr, dstReg, u0, tail);
256- AscendC::Reg::DataCopyUnAlignPost(curDstPtr, u0, 0);256+ AscendC::Reg::StoreUnAlignPost(curDstPtr, u0, 0);
257 }257 }
258 }258 }
259 }259 }
@@ -268,19 +268,18 @@ __aicore__ inline void DataCopyScope(uint16_t size0, int64_t offset, int64_t nis
268 __VEC_SCOPE__268 __VEC_SCOPE__
269 {269 {
270 AscendC::Reg::RegTensor<T1> vd0;270 AscendC::Reg::RegTensor<T1> vd0;
271- AscendC::Reg::UnalignReg u0;271+ AscendC::Reg::UnalignRegForLoad u0;
272- AscendC::Reg::UnalignReg u1;272+ AscendC::Reg::UnalignRegForStore u1;
273 for (uint16_t i = 0; i < size0; i++) {273 for (uint16_t i = 0; i < size0; i++) {
274 __ubuf__ T1* srcPtr1 = srcPtr + i * offset;274 __ubuf__ T1* srcPtr1 = srcPtr + i * offset;
275- AscendC::Reg::DataCopyUnAlignPre(u0, srcPtr1);275+ AscendC::Reg::LoadUnAlignPre(u0, srcPtr1);
276 for (uint16_t j = 0; j < repeatTimes; j++) {276 for (uint16_t j = 0; j < repeatTimes; j++) {
277- AscendC::Reg::DataCopyUnAlign<T1, AscendC::Reg::PostLiteral::POST_MODE_UPDATE>(vd0, u0, srcPtr1,277+ AscendC::Reg::LoadUnAlign<T1, AscendC::Reg::PostLiteral::POST_MODE_UPDATE>(vd0, u0, srcPtr1, vfLen1);
278- vfLen1);278+ AscendC::Reg::StoreUnAlign<T1, AscendC::Reg::PostLiteral::POST_MODE_UPDATE>(dstPtr, vd0, u1, vfLen1);
279- AscendC::Reg::DataCopyUnAlign<T1, AscendC::Reg::PostLiteral::POST_MODE_UPDATE>(dstPtr, vd0, u1, vfLen1);
280 }279 }
281- AscendC::Reg::DataCopyUnAlign<T1, AscendC::Reg::PostLiteral::POST_MODE_UPDATE>(vd0, u0, srcPtr1, vfLen1);280+ AscendC::Reg::LoadUnAlign<T1, AscendC::Reg::PostLiteral::POST_MODE_UPDATE>(vd0, u0, srcPtr1, vfLen1);
282- AscendC::Reg::DataCopyUnAlign<T1, AscendC::Reg::PostLiteral::POST_MODE_UPDATE>(dstPtr, vd0, u1, tail);281+ AscendC::Reg::StoreUnAlign<T1, AscendC::Reg::PostLiteral::POST_MODE_UPDATE>(dstPtr, vd0, u1, tail);
283- AscendC::Reg::DataCopyUnAlignPost<T1, AscendC::Reg::PostLiteral::POST_MODE_UPDATE>(dstPtr, u1, 0);282+ AscendC::Reg::StoreUnAlignPost<T1, AscendC::Reg::PostLiteral::POST_MODE_UPDATE>(dstPtr, u1, 0);
284 }283 }
285 }284 }
286}285}
Mconversion/split_v/op_kernel/arch35/split_v_ub_split_same_len.h+21-21
@@ -268,7 +268,7 @@ __aicore__ inline void SplitVUbSplitSameLen<T, U, Y>::ComputeIdx(int64_t process
268 AscendC::Reg::Sub(tmp2B64, (AscendC::Reg::RegTensor<U>&)indexRegB64, subRegB64, maskB64);268 AscendC::Reg::Sub(tmp2B64, (AscendC::Reg::RegTensor<U>&)indexRegB64, subRegB64, maskB64);
269 AscendC::Reg::Add(addRegB64, tmp1B64, tmp2B64, maskB64);269 AscendC::Reg::Add(addRegB64, tmp1B64, tmp2B64, maskB64);
270 270 
271- AscendC::Reg::DataCopy(idxPtr, addRegB64, maskB64);271+ AscendC::Reg::StoreAlign(idxPtr, addRegB64, maskB64);
272 }272 }
273 } else {273 } else {
274 __ubuf__ U* idxPtr = (__ubuf__ U*)idxUb.GetPhyAddr();274 __ubuf__ U* idxPtr = (__ubuf__ U*)idxUb.GetPhyAddr();
@@ -294,7 +294,7 @@ __aicore__ inline void SplitVUbSplitSameLen<T, U, Y>::ComputeIdx(int64_t process
294 AscendC::Reg::Sub(tmp2, (AscendC::Reg::RegTensor<U>&)indexReg, subReg, mask);294 AscendC::Reg::Sub(tmp2, (AscendC::Reg::RegTensor<U>&)indexReg, subReg, mask);
295 AscendC::Reg::Add(addReg, tmp1, tmp2, mask);295 AscendC::Reg::Add(addReg, tmp1, tmp2, mask);
296 296 
297- AscendC::Reg::DataCopy(idxPtr, addReg, mask);297+ AscendC::Reg::StoreAlign(idxPtr, addReg, mask);
298 }298 }
299 }299 }
300 idxQueue_.EnQue(idxUb);300 idxQueue_.EnQue(idxUb);
@@ -338,25 +338,25 @@ __aicore__ inline void SplitVUbSplitSameLen<T, U, Y>::Compute(uint32_t g, int64_
338 {338 {
339 AscendC::Reg::RegTensor<U> addReg;339 AscendC::Reg::RegTensor<U> addReg;
340 AscendC::Reg::RegTensor<U> dstReg;340 AscendC::Reg::RegTensor<U> dstReg;
341- AscendC::Reg::UnalignReg uDst;341+ AscendC::Reg::UnalignRegForStore uDst;
342 AscendC::Reg::MaskReg mask;342 AscendC::Reg::MaskReg mask;
343 AscendC::Reg::MaskReg maskTmp;343 AscendC::Reg::MaskReg maskTmp;
344 344 
345 mask = AscendC::Reg::UpdateMask<U>(processNum);345 mask = AscendC::Reg::UpdateMask<U>(processNum);
346 maskTmp = AscendC::Reg::UpdateMask<U>(nRegSize);346 maskTmp = AscendC::Reg::UpdateMask<U>(nRegSize);
347- AscendC::Reg::DataCopy(addReg, idxPtr);347+ AscendC::Reg::LoadAlign(addReg, idxPtr);
348 AscendC::Reg::Adds(addReg, addReg, (U)gnSize, mask);348 AscendC::Reg::Adds(addReg, addReg, (U)gnSize, mask);
349 349 
350 for (uint16_t i = 0; i < loopNum; i++) {350 for (uint16_t i = 0; i < loopNum; i++) {
351- AscendC::Reg::DataCopyGather(dstReg, srcPtr, addReg, maskTmp);351+ AscendC::Reg::Gather(dstReg, srcPtr, addReg, maskTmp);
352 // Copy out352 // Copy out
353- AscendC::Reg::DataCopyUnAlign(dstPtr, dstReg, uDst, mnSize);353+ AscendC::Reg::StoreUnAlign(dstPtr, dstReg, uDst, mnSize);
354 AscendC::Reg::Adds(addReg, addReg, (U)(tmpLoop * gnAlignSize), mask);354 AscendC::Reg::Adds(addReg, addReg, (U)(tmpLoop * gnAlignSize), mask);
355 }355 }
356 maskTmp = AscendC::Reg::UpdateMask<U>(processLastNum);356 maskTmp = AscendC::Reg::UpdateMask<U>(processLastNum);
357- AscendC::Reg::DataCopyGather(dstReg, srcPtr, addReg, maskTmp);357+ AscendC::Reg::Gather(dstReg, srcPtr, addReg, maskTmp);
358- AscendC::Reg::DataCopyUnAlign(dstPtr, dstReg, uDst, mnSizeLast);358+ AscendC::Reg::StoreUnAlign(dstPtr, dstReg, uDst, mnSizeLast);
359- AscendC::Reg::DataCopyUnAlignPost(dstPtr, uDst, 0);359+ AscendC::Reg::StoreUnAlignPost(dstPtr, uDst, 0);
360 }360 }
361 } else {361 } else {
362 __ubuf__ T* srcPtr = (__ubuf__ T*)srcUb.GetPhyAddr();362 __ubuf__ T* srcPtr = (__ubuf__ T*)srcUb.GetPhyAddr();
@@ -369,39 +369,39 @@ __aicore__ inline void SplitVUbSplitSameLen<T, U, Y>::Compute(uint32_t g, int64_
369 AscendC::Reg::RegTensor<U> addReg;369 AscendC::Reg::RegTensor<U> addReg;
370 AscendC::Reg::RegTensor<U> dstReg;370 AscendC::Reg::RegTensor<U> dstReg;
371 AscendC::Reg::RegTensor<T> dstRegT;371 AscendC::Reg::RegTensor<T> dstRegT;
372- AscendC::Reg::UnalignReg uDst;372+ AscendC::Reg::UnalignRegForStore uDst;
373 AscendC::Reg::MaskReg mask;373 AscendC::Reg::MaskReg mask;
374 AscendC::Reg::MaskReg maskTmp;374 AscendC::Reg::MaskReg maskTmp;
375 375 
376 mask = AscendC::Reg::UpdateMask<U>(processNum);376 mask = AscendC::Reg::UpdateMask<U>(processNum);
377 maskTmp = AscendC::Reg::UpdateMask<U>(nRegSize);377 maskTmp = AscendC::Reg::UpdateMask<U>(nRegSize);
378- AscendC::Reg::DataCopy(addReg, idxPtr);378+ AscendC::Reg::LoadAlign(addReg, idxPtr);
379 AscendC::Reg::Adds(addReg, addReg, (U)gnSize, mask);379 AscendC::Reg::Adds(addReg, addReg, (U)gnSize, mask);
380 380 
381 for (uint16_t i = 0; i < loopNum; i++) {381 for (uint16_t i = 0; i < loopNum; i++) {
382- AscendC::Reg::DataCopyGather(dstReg, srcPtr, addReg, maskTmp);382+ AscendC::Reg::Gather(dstReg, srcPtr, addReg, maskTmp);
383 // Copy out383 // Copy out
384 if constexpr (sizeof(T) == sizeof(int8_t)) {384 if constexpr (sizeof(T) == sizeof(int8_t)) {
385 // Convert B16 to B8385 // Convert B16 to B8
386 AscendC::Reg::Pack(dstRegT, dstReg);386 AscendC::Reg::Pack(dstRegT, dstReg);
387- AscendC::Reg::DataCopyUnAlign(dstPtrT, dstRegT, uDst, mnSize);387+ AscendC::Reg::StoreUnAlign(dstPtrT, dstRegT, uDst, mnSize);
388- AscendC::Reg::DataCopyUnAlignPost(dstPtrT, uDst, 0);388+ AscendC::Reg::StoreUnAlignPost(dstPtrT, uDst, 0);
389 } else {389 } else {
390- AscendC::Reg::DataCopyUnAlign(dstPtrU, dstReg, uDst, mnSize);390+ AscendC::Reg::StoreUnAlign(dstPtrU, dstReg, uDst, mnSize);
391- AscendC::Reg::DataCopyUnAlignPost(dstPtrU, uDst, 0);391+ AscendC::Reg::StoreUnAlignPost(dstPtrU, uDst, 0);
392 }392 }
393 AscendC::Reg::Adds(addReg, addReg, (U)(tmpLoop * gnAlignSize), mask);393 AscendC::Reg::Adds(addReg, addReg, (U)(tmpLoop * gnAlignSize), mask);
394 }394 }
395 maskTmp = AscendC::Reg::UpdateMask<U>(processLastNum);395 maskTmp = AscendC::Reg::UpdateMask<U>(processLastNum);
396- AscendC::Reg::DataCopyGather(dstReg, srcPtr, addReg, maskTmp);396+ AscendC::Reg::Gather(dstReg, srcPtr, addReg, maskTmp);
397 if constexpr (sizeof(T) == sizeof(int8_t)) {397 if constexpr (sizeof(T) == sizeof(int8_t)) {
398 // Convert B16 to B8398 // Convert B16 to B8
399 AscendC::Reg::Pack(dstRegT, dstReg);399 AscendC::Reg::Pack(dstRegT, dstReg);
400- AscendC::Reg::DataCopyUnAlign(dstPtrT, dstRegT, uDst, mnSizeLast);400+ AscendC::Reg::StoreUnAlign(dstPtrT, dstRegT, uDst, mnSizeLast);
401- AscendC::Reg::DataCopyUnAlignPost(dstPtrT, uDst, 0);401+ AscendC::Reg::StoreUnAlignPost(dstPtrT, uDst, 0);
402 } else {402 } else {
403- AscendC::Reg::DataCopyUnAlign(dstPtrU, dstReg, uDst, mnSizeLast);403+ AscendC::Reg::StoreUnAlign(dstPtrU, dstReg, uDst, mnSizeLast);
404- AscendC::Reg::DataCopyUnAlignPost(dstPtrU, uDst, 0);404+ AscendC::Reg::StoreUnAlignPost(dstPtrU, uDst, 0);
405 }405 }
406 }406 }
407 }407 }
Mconversion/split_v/op_kernel/arch35/split_v_ub_split_same_len_small_g.h+21-21
@@ -271,7 +271,7 @@ __aicore__ inline void SplitVUbSplitSameLenSmallG<T, U, Y>::ComputeIdx(int64_t p
271 AscendC::Reg::Sub(tmp2B64, (AscendC::Reg::RegTensor<U>&)indexRegB64, subRegB64, maskB64);271 AscendC::Reg::Sub(tmp2B64, (AscendC::Reg::RegTensor<U>&)indexRegB64, subRegB64, maskB64);
272 AscendC::Reg::Add(addRegB64, tmp1B64, tmp2B64, maskB64);272 AscendC::Reg::Add(addRegB64, tmp1B64, tmp2B64, maskB64);
273 273 
274- AscendC::Reg::DataCopy(idxPtr, addRegB64, maskB64);274+ AscendC::Reg::StoreAlign(idxPtr, addRegB64, maskB64);
275 }275 }
276 } else {276 } else {
277 __ubuf__ U* idxPtr = (__ubuf__ U*)idxUb.GetPhyAddr();277 __ubuf__ U* idxPtr = (__ubuf__ U*)idxUb.GetPhyAddr();
@@ -297,7 +297,7 @@ __aicore__ inline void SplitVUbSplitSameLenSmallG<T, U, Y>::ComputeIdx(int64_t p
297 AscendC::Reg::Sub(tmp2, (AscendC::Reg::RegTensor<U>&)indexReg, subReg, mask);297 AscendC::Reg::Sub(tmp2, (AscendC::Reg::RegTensor<U>&)indexReg, subReg, mask);
298 AscendC::Reg::Add(addReg, tmp1, tmp2, mask);298 AscendC::Reg::Add(addReg, tmp1, tmp2, mask);
299 299 
300- AscendC::Reg::DataCopy(idxPtr, addReg, mask);300+ AscendC::Reg::StoreAlign(idxPtr, addReg, mask);
301 }301 }
302 }302 }
303 idxQueue_.EnQue(idxUb);303 idxQueue_.EnQue(idxUb);
@@ -342,25 +342,25 @@ __aicore__ inline void SplitVUbSplitSameLenSmallG<T, U, Y>::Compute(uint32_t g,
342 {342 {
343 AscendC::Reg::RegTensor<U> addReg;343 AscendC::Reg::RegTensor<U> addReg;
344 AscendC::Reg::RegTensor<U> dstReg;344 AscendC::Reg::RegTensor<U> dstReg;
345- AscendC::Reg::UnalignReg uDst;345+ AscendC::Reg::UnalignRegForStore uDst;
346 AscendC::Reg::MaskReg mask;346 AscendC::Reg::MaskReg mask;
347 AscendC::Reg::MaskReg maskTmp;347 AscendC::Reg::MaskReg maskTmp;
348 348 
349 mask = AscendC::Reg::UpdateMask<U>(processNum);349 mask = AscendC::Reg::UpdateMask<U>(processNum);
350 maskTmp = AscendC::Reg::UpdateMask<U>(nRegSize);350 maskTmp = AscendC::Reg::UpdateMask<U>(nRegSize);
351- AscendC::Reg::DataCopy(addReg, idxPtr);351+ AscendC::Reg::LoadAlign(addReg, idxPtr);
352 AscendC::Reg::Adds(addReg, addReg, (U)gnSize, mask);352 AscendC::Reg::Adds(addReg, addReg, (U)gnSize, mask);
353 353 
354 for (uint16_t i = 0; i < loopNum; i++) {354 for (uint16_t i = 0; i < loopNum; i++) {
355- AscendC::Reg::DataCopyGather(dstReg, srcPtr, addReg, maskTmp);355+ AscendC::Reg::Gather(dstReg, srcPtr, addReg, maskTmp);
356 // Copy out356 // Copy out
357- AscendC::Reg::DataCopyUnAlign(dstPtr, dstReg, uDst, mnSize);357+ AscendC::Reg::StoreUnAlign(dstPtr, dstReg, uDst, mnSize);
358 AscendC::Reg::Adds(addReg, addReg, (U)(tmpLoop * gnAlignSize), mask);358 AscendC::Reg::Adds(addReg, addReg, (U)(tmpLoop * gnAlignSize), mask);
359 }359 }
360 maskTmp = AscendC::Reg::UpdateMask<U>(processLastNum);360 maskTmp = AscendC::Reg::UpdateMask<U>(processLastNum);
361- AscendC::Reg::DataCopyGather(dstReg, srcPtr, addReg, maskTmp);361+ AscendC::Reg::Gather(dstReg, srcPtr, addReg, maskTmp);
362- AscendC::Reg::DataCopyUnAlign(dstPtr, dstReg, uDst, mnSizeLast);362+ AscendC::Reg::StoreUnAlign(dstPtr, dstReg, uDst, mnSizeLast);
363- AscendC::Reg::DataCopyUnAlignPost(dstPtr, uDst, 0);363+ AscendC::Reg::StoreUnAlignPost(dstPtr, uDst, 0);
364 }364 }
365 } else {365 } else {
366 __ubuf__ T* srcPtr = (__ubuf__ T*)srcUb.GetPhyAddr();366 __ubuf__ T* srcPtr = (__ubuf__ T*)srcUb.GetPhyAddr();
@@ -373,39 +373,39 @@ __aicore__ inline void SplitVUbSplitSameLenSmallG<T, U, Y>::Compute(uint32_t g,
373 AscendC::Reg::RegTensor<U> addReg;373 AscendC::Reg::RegTensor<U> addReg;
374 AscendC::Reg::RegTensor<U> dstReg;374 AscendC::Reg::RegTensor<U> dstReg;
375 AscendC::Reg::RegTensor<T> dstRegT;375 AscendC::Reg::RegTensor<T> dstRegT;
376- AscendC::Reg::UnalignReg uDst;376+ AscendC::Reg::UnalignRegForStore uDst;
377 AscendC::Reg::MaskReg mask;377 AscendC::Reg::MaskReg mask;
378 AscendC::Reg::MaskReg maskTmp;378 AscendC::Reg::MaskReg maskTmp;
379 379 
380 mask = AscendC::Reg::UpdateMask<U>(processNum);380 mask = AscendC::Reg::UpdateMask<U>(processNum);
381 maskTmp = AscendC::Reg::UpdateMask<U>(nRegSize);381 maskTmp = AscendC::Reg::UpdateMask<U>(nRegSize);
382- AscendC::Reg::DataCopy(addReg, idxPtr);382+ AscendC::Reg::LoadAlign(addReg, idxPtr);
383 AscendC::Reg::Adds(addReg, addReg, (U)gnSize, mask);383 AscendC::Reg::Adds(addReg, addReg, (U)gnSize, mask);
384 384 
385 for (uint16_t i = 0; i < loopNum; i++) {385 for (uint16_t i = 0; i < loopNum; i++) {
386- AscendC::Reg::DataCopyGather(dstReg, srcPtr, addReg, maskTmp);386+ AscendC::Reg::Gather(dstReg, srcPtr, addReg, maskTmp);
387 // Copy out387 // Copy out
388 if constexpr (sizeof(T) == sizeof(int8_t)) {388 if constexpr (sizeof(T) == sizeof(int8_t)) {
389 // Convert B16 to B8389 // Convert B16 to B8
390 AscendC::Reg::Pack(dstRegT, dstReg);390 AscendC::Reg::Pack(dstRegT, dstReg);
391- AscendC::Reg::DataCopyUnAlign(dstPtrT, dstRegT, uDst, mnSize);391+ AscendC::Reg::StoreUnAlign(dstPtrT, dstRegT, uDst, mnSize);
392- AscendC::Reg::DataCopyUnAlignPost(dstPtrT, uDst, 0);392+ AscendC::Reg::StoreUnAlignPost(dstPtrT, uDst, 0);
393 } else {393 } else {
394- AscendC::Reg::DataCopyUnAlign(dstPtrU, dstReg, uDst, mnSize);394+ AscendC::Reg::StoreUnAlign(dstPtrU, dstReg, uDst, mnSize);
395- AscendC::Reg::DataCopyUnAlignPost(dstPtrU, uDst, 0);395+ AscendC::Reg::StoreUnAlignPost(dstPtrU, uDst, 0);
396 }396 }
397 AscendC::Reg::Adds(addReg, addReg, (U)(tmpLoop * gnAlignSize), mask);397 AscendC::Reg::Adds(addReg, addReg, (U)(tmpLoop * gnAlignSize), mask);
398 }398 }
399 maskTmp = AscendC::Reg::UpdateMask<U>(processLastNum);399 maskTmp = AscendC::Reg::UpdateMask<U>(processLastNum);
400- AscendC::Reg::DataCopyGather(dstReg, srcPtr, addReg, maskTmp);400+ AscendC::Reg::Gather(dstReg, srcPtr, addReg, maskTmp);
401 if constexpr (sizeof(T) == sizeof(int8_t)) {401 if constexpr (sizeof(T) == sizeof(int8_t)) {
402 // Convert B16 to B8402 // Convert B16 to B8
403 AscendC::Reg::Pack(dstRegT, dstReg);403 AscendC::Reg::Pack(dstRegT, dstReg);
404- AscendC::Reg::DataCopyUnAlign(dstPtrT, dstRegT, uDst, mnSizeLast);404+ AscendC::Reg::StoreUnAlign(dstPtrT, dstRegT, uDst, mnSizeLast);
405- AscendC::Reg::DataCopyUnAlignPost(dstPtrT, uDst, 0);405+ AscendC::Reg::StoreUnAlignPost(dstPtrT, uDst, 0);
406 } else {406 } else {
407- AscendC::Reg::DataCopyUnAlign(dstPtrU, dstReg, uDst, mnSizeLast);407+ AscendC::Reg::StoreUnAlign(dstPtrU, dstReg, uDst, mnSizeLast);
408- AscendC::Reg::DataCopyUnAlignPost(dstPtrU, uDst, 0);408+ AscendC::Reg::StoreUnAlignPost(dstPtrU, uDst, 0);
409 }409 }
410 }410 }
411 }411 }
Mconversion/strided_slice_grad/op_kernel/arch35/copy_out_simt.h+19-19
@@ -20,18 +20,19 @@
20#include "simt_api/asc_simt.h"20#include "simt_api/asc_simt.h"
21#include "simt_api/device_functions.h"21#include "simt_api/device_functions.h"
22 22 
23-namespace StridedSliceGrad23+namespace StridedSliceGrad {
24-{
25using namespace AscendC;24using namespace AscendC;
26constexpr int16_t SIMT_BEGIN_OFFSET = 8;25constexpr int16_t SIMT_BEGIN_OFFSET = 8;
27constexpr int16_t SIMT_STRIDES_OFFSET = 16;26constexpr int16_t SIMT_STRIDES_OFFSET = 16;
28constexpr int16_t SIMT_FUSEDOUTPUTINNERSHAPE_OFFSET = 24;27constexpr int16_t SIMT_FUSEDOUTPUTINNERSHAPE_OFFSET = 24;
29 28 
30template <typename T>29template <typename T>
31-__simt_vf__ __aicore__ LAUNCH_BOUND(THREAD_DIM) inline void CopyOut(__local_mem__ T* srcLocal, __gm__ T* dstGm,30+__simt_vf__ __aicore__ LAUNCH_BOUND(THREAD_DIM) inline void CopyOut(__ubuf__ T* srcLocal, __gm__ T* dstGm,
32- __local_mem__ uint32_t* simtB32Local, __local_mem__ int64_t* simtB64Local,31+ __ubuf__ uint32_t* simtB32Local,
33- uint32_t curLoopBlockNum, uint32_t blockIdx, uint32_t normalCoreProcessNum,32+ __ubuf__ int64_t* simtB64Local,
34- uint32_t loopNum, uint32_t normalLoopBlockNum, uint32_t inputDimNum)33+ uint32_t curLoopBlockNum, uint32_t blockIdx,
34+ uint32_t normalCoreProcessNum, uint32_t loopNum,
35+ uint32_t normalLoopBlockNum, uint32_t inputDimNum)
35{36{
36 for (int32_t i = threadIdx.x; i < curLoopBlockNum; i += blockDim.x) {37 for (int32_t i = threadIdx.x; i < curLoopBlockNum; i += blockDim.x) {
37 int32_t idx = blockIdx * normalCoreProcessNum + loopNum * normalLoopBlockNum + i;38 int32_t idx = blockIdx * normalCoreProcessNum + loopNum * normalLoopBlockNum + i;
@@ -53,11 +54,11 @@ __simt_vf__ __aicore__ LAUNCH_BOUND(THREAD_DIM) inline void CopyOut(__local_mem_
53}54}
54 55 
55template <typename T>56template <typename T>
56-class CopyOutSimt : public StridedSliceGradBase57+class CopyOutSimt : public StridedSliceGradBase {
57-{
58public:58public:
59 __aicore__ CopyOutSimt() {};59 __aicore__ CopyOutSimt() {};
60- __aicore__ inline void Init(GM_ADDR dy, GM_ADDR output, const StridedSliceGradTilingData& tilingData, TPipe& pipeIn);60+ __aicore__ inline void Init(GM_ADDR dy, GM_ADDR output, const StridedSliceGradTilingData& tilingData,
61+ TPipe& pipeIn);
61 __aicore__ inline void Process();62 __aicore__ inline void Process();
62 __aicore__ inline void InitCopyParams();63 __aicore__ inline void InitCopyParams();
63 __aicore__ inline void ConstructDivParams();64 __aicore__ inline void ConstructDivParams();
@@ -100,7 +101,7 @@ private:
100 101 
101template <typename T>102template <typename T>
102__aicore__ inline void CopyOutSimt<T>::Init(GM_ADDR dy, GM_ADDR output, const StridedSliceGradTilingData& tilingData,103__aicore__ inline void CopyOutSimt<T>::Init(GM_ADDR dy, GM_ADDR output, const StridedSliceGradTilingData& tilingData,
103- TPipe& pipeIn)104+ TPipe& pipeIn)
104{105{
105 // tiling_data106 // tiling_data
106 StridedSliceGradBase::ParseTilingData(tilingData);107 StridedSliceGradBase::ParseTilingData(tilingData);
@@ -165,23 +166,22 @@ __aicore__ inline void CopyOutSimt<T>::ProcessPerLoop()
165 LocalTensor<int64_t> simtB64Local = simtB64Buf_.Get<int64_t>();166 LocalTensor<int64_t> simtB64Local = simtB64Buf_.Get<int64_t>();
166 LocalTensor<uint32_t> simtB32Local = simtB32Buf_.Get<uint32_t>();167 LocalTensor<uint32_t> simtB32Local = simtB32Buf_.Get<uint32_t>();
167 168 
168- for(int i = 0; i < MAX_SUPPORT_DIM; i++) {169+ for (int i = 0; i < MAX_SUPPORT_DIM; i++) {
169 simtB32Local(i) = m_[i];170 simtB32Local(i) = m_[i];
170 simtB32Local(i + MAX_SUPPORT_DIM) = shift_[i];171 simtB32Local(i + MAX_SUPPORT_DIM) = shift_[i];
171 }172 }
172 173 
173- for(int i = 0; i < MAX_SUPPORT_DIM; i++) {174+ for (int i = 0; i < MAX_SUPPORT_DIM; i++) {
174 simtB64Local(i) = fusedSliceInnerShape_[i];175 simtB64Local(i) = fusedSliceInnerShape_[i];
175 simtB64Local(i + SIMT_BEGIN_OFFSET) = begin_[i];176 simtB64Local(i + SIMT_BEGIN_OFFSET) = begin_[i];
176 simtB64Local(i + SIMT_STRIDES_OFFSET) = strides_[i];177 simtB64Local(i + SIMT_STRIDES_OFFSET) = strides_[i];
177 simtB64Local(i + SIMT_FUSEDOUTPUTINNERSHAPE_OFFSET) = fusedOutputInnerShape_[i];178 simtB64Local(i + SIMT_FUSEDOUTPUTINNERSHAPE_OFFSET) = fusedOutputInnerShape_[i];
178 }179 }
179 PipeBarrier<PIPE_ALL>();180 PipeBarrier<PIPE_ALL>();
180- asc_vf_call<CopyOut<T>>(dim3(threadsPerBlock_), 181+ asc_vf_call<CopyOut<T>>(dim3(threadsPerBlock_), (__ubuf__ T*)(yLocal.GetPhyAddr()),
181- (__local_mem__ T*)(yLocal.GetPhyAddr()), (__gm__ T*)(dstGm_.GetPhyAddr()), 182+ (__gm__ T*)(dstGm_.GetPhyAddr()), (__ubuf__ uint32_t*)(simtB32Local.GetPhyAddr()),
182- (__local_mem__ uint32_t*)(simtB32Local.GetPhyAddr()), (__local_mem__ int64_t*)(simtB64Local.GetPhyAddr()),183+ (__ubuf__ int64_t*)(simtB64Local.GetPhyAddr()), curLoopBlockNum_, blockIdx_,
183- curLoopBlockNum_, blockIdx_, normalCoreProcessNum_,184+ normalCoreProcessNum_, loopNum_, normalLoopBlockNum_, inputDimNum_);
184- loopNum_, normalLoopBlockNum_, inputDimNum_);
185 dataQueue_.FreeTensor(yLocal);185 dataQueue_.FreeTensor(yLocal);
186}186}
187 187 
@@ -195,6 +195,6 @@ __aicore__ inline void CopyOutSimt<T>::InitCopyParams()
195 padParams_ = {false, static_cast<uint8_t>(0), static_cast<uint8_t>(0), static_cast<T>(0)};195 padParams_ = {false, static_cast<uint8_t>(0), static_cast<uint8_t>(0), static_cast<T>(0)};
196}196}
197 197 
198-} // namespace StridedSliceGrad198+} // namespace StridedSliceGrad
199 199 
200-#endif // COPY_OUT_SIMT_H200+#endif // COPY_OUT_SIMT_H
Mconversion/strided_slice_grad/op_kernel/arch35/split_all_axis_nddma.h+11-13
@@ -17,13 +17,11 @@
17 17 
18#include "strided_slice_grad_base.h"18#include "strided_slice_grad_base.h"
19 19 
20-namespace StridedSliceGrad20+namespace StridedSliceGrad {
21-{
22using namespace AscendC;21using namespace AscendC;
23 22 
24template <typename T>23template <typename T>
25-class SplitAllAxisNddma : public StridedSliceGradBase24+class SplitAllAxisNddma : public StridedSliceGradBase {
26-{
27public:25public:
28 __aicore__ inline SplitAllAxisNddma(){};26 __aicore__ inline SplitAllAxisNddma(){};
29 __aicore__ inline void Init(GM_ADDR dy, GM_ADDR output, const StridedSliceGradTilingData& tilingData,27 __aicore__ inline void Init(GM_ADDR dy, GM_ADDR output, const StridedSliceGradTilingData& tilingData,
@@ -41,8 +39,8 @@ private:
41 GlobalTensor<T> dstGm_;39 GlobalTensor<T> dstGm_;
42 GlobalTensor<T> srcGm_;40 GlobalTensor<T> srcGm_;
43 41 
44- MultiCopyParams<T, 1> dmaParam_;42+ NdDmaParams<T, 1> dmaParam_;
45- static constexpr MultiCopyConfig nddmaConfig_ = {false};43+ static constexpr NdDmaConfig nddmaConfig_ = {false};
46 44 
47 int64_t dealDataSize_;45 int64_t dealDataSize_;
48 46 
@@ -122,8 +120,8 @@ __aicore__ inline void SplitAllAxisNddma<T>::Process()
122 SetFlag<HardEvent::MTE3_V>(eventSecond);120 SetFlag<HardEvent::MTE3_V>(eventSecond);
123 for (loopNum_ = 0; loopNum_ < curCoreProcessNum_; loopNum_++) {121 for (loopNum_ = 0; loopNum_ < curCoreProcessNum_; loopNum_++) {
124 outerAxisIdx_ = (blockIdx_ * normalCoreProcessNum_ + loopNum_) / tailAxisOuter_;122 outerAxisIdx_ = (blockIdx_ * normalCoreProcessNum_ + loopNum_) / tailAxisOuter_;
125- outerAxisTail_ =123+ outerAxisTail_ = (blockIdx_ * normalCoreProcessNum_ + loopNum_) %
126- (blockIdx_ * normalCoreProcessNum_ + loopNum_) % tailAxisOuter_; // range [0--> tailAxisOuter_ - 1]124+ tailAxisOuter_; // range [0--> tailAxisOuter_ - 1]
127 125 
128 dealDataSize_ = (outerAxisTail_ == tailAxisOuter_ - 1) ? tailAxisTail_ : tailAxisInner_;126 dealDataSize_ = (outerAxisTail_ == tailAxisOuter_ - 1) ? tailAxisTail_ : tailAxisInner_;
129 127 
@@ -155,12 +153,12 @@ __aicore__ inline void SplitAllAxisNddma<T>::ProcessNddma(LocalTensor<T>& srcLoc
155 srcGmEndAxisOffset = 0;153 srcGmEndAxisOffset = 0;
156 }154 }
157 155 
158- int64_t elementSize = 0; // 本ubBuf内的元素数量，值只跟当前段尾和end的相对关系有关156+ int64_t elementSize = 0; // 本ubBuf内的元素数量，值只跟当前段尾和end的相对关系有关
159 if ((outerAxisTail_ + 1) * tailAxisInner_ < end_[DIM0]) {157 if ((outerAxisTail_ + 1) * tailAxisInner_ < end_[DIM0]) {
160- elementSize = (tailAxisInner_ - ubOffset) / strides_[DIM0] + 1; // 左闭不一定右开 （end-begin）//stride + 1158+ elementSize = (tailAxisInner_ - ubOffset) / strides_[DIM0] + 1; // 左闭不一定右开 （end-begin）//stride + 1
161 } else {159 } else {
162 elementSize = (end_[DIM0] - outerAxisTail_ * tailAxisInner_ - ubOffset - 1) / strides_[DIM0] +160 elementSize = (end_[DIM0] - outerAxisTail_ * tailAxisInner_ - ubOffset - 1) / strides_[DIM0] +
163- 1; // 左闭右开 （end-begin -1）//stride + 1161+ 1; // 左闭右开 （end-begin -1）//stride + 1
164 }162 }
165 163 
166 dmaParam_.constantValue = 0;164 dmaParam_.constantValue = 0;
@@ -203,6 +201,6 @@ __aicore__ inline void SplitAllAxisNddma<T>::ProcessPerLoop(LocalTensor<T>& srcL
203 SetFlag<HardEvent::MTE3_V>(eventId);201 SetFlag<HardEvent::MTE3_V>(eventId);
204}202}
205 203 
206-} // namespace StridedSliceGrad204+} // namespace StridedSliceGrad
207 205 
208-#endif // SPLIT_ALL_AXIS_NDDMA_H206+#endif // SPLIT_ALL_AXIS_NDDMA_H
Mconversion/strided_slice_grad/op_kernel/arch35/split_front_axis_nddma.h+9-11
@@ -17,13 +17,11 @@
17 17 
18#include "strided_slice_grad_base.h"18#include "strided_slice_grad_base.h"
19 19 
20-namespace StridedSliceGrad20+namespace StridedSliceGrad {
21-{
22using namespace AscendC;21using namespace AscendC;
23 22 
24template <typename T>23template <typename T>
25-class SplitFrontAxisNddma : public StridedSliceGradBase24+class SplitFrontAxisNddma : public StridedSliceGradBase {
26-{
27public:25public:
28 __aicore__ inline SplitFrontAxisNddma(){};26 __aicore__ inline SplitFrontAxisNddma(){};
29 __aicore__ inline void Init(GM_ADDR dy, GM_ADDR output, const StridedSliceGradTilingData& tilingData,27 __aicore__ inline void Init(GM_ADDR dy, GM_ADDR output, const StridedSliceGradTilingData& tilingData,
@@ -46,8 +44,8 @@ private:
46 44 
47 DataCopyExtParams copyOutCopyParams_;45 DataCopyExtParams copyOutCopyParams_;
48 46 
49- MultiCopyParams<T, NDDMA_MAX_DIMS> dmaParam_;47+ NdDmaParams<T, NDDMA_MAX_DIMS> dmaParam_;
50- static constexpr MultiCopyConfig nddmaConfig_ = {false};48+ static constexpr NdDmaConfig nddmaConfig_ = {false};
51 49 
52 uint32_t splitUbAxisNum_;50 uint32_t splitUbAxisNum_;
53 uint32_t splitUbAxisIndex_;51 uint32_t splitUbAxisIndex_;
@@ -70,7 +68,7 @@ __aicore__ inline void SplitFrontAxisNddma<T>::Init(GM_ADDR dy, GM_ADDR output,
70 return;68 return;
71 }69 }
72 70 
73- splitUbAxisNum_ = tilingData.splitUbAxisNum; // ub里面能放得下的维度数目 [1,5]71+ splitUbAxisNum_ = tilingData.splitUbAxisNum; // ub里面能放得下的维度数目 [1,5]
74 splitUbAxisIndex_ = MAX_SUPPORT_DIM - 1 - tilingData.splitUbAxisNum;72 splitUbAxisIndex_ = MAX_SUPPORT_DIM - 1 - tilingData.splitUbAxisNum;
75 ubDataSize_ = fusedOutputInnerShape_[splitUbAxisIndex_];73 ubDataSize_ = fusedOutputInnerShape_[splitUbAxisIndex_];
76 74 
@@ -182,8 +180,8 @@ __aicore__ inline void SplitFrontAxisNddma<T>::InitCopyParams()
182 // [低维 --> 高维]180 // [低维 --> 高维]
183 dmaParam_.loopInfo.loopSize[i] = inputShape_[MAX_SUPPORT_DIM - 1 - i];181 dmaParam_.loopInfo.loopSize[i] = inputShape_[MAX_SUPPORT_DIM - 1 - i];
184 dmaParam_.loopInfo.loopSrcStride[i] = fusedSliceInnerShape_[MAX_SUPPORT_DIM - 1 - i];182 dmaParam_.loopInfo.loopSrcStride[i] = fusedSliceInnerShape_[MAX_SUPPORT_DIM - 1 - i];
185- dmaParam_.loopInfo.loopDstStride[i] =183+ dmaParam_.loopInfo.loopDstStride[i] = fusedOutputInnerShape_[MAX_SUPPORT_DIM - 1 - i] *
186- fusedOutputInnerShape_[MAX_SUPPORT_DIM - 1 - i] * strides_[MAX_SUPPORT_DIM - 1 - i];184+ strides_[MAX_SUPPORT_DIM - 1 - i];
187 dmaParam_.loopInfo.loopLpSize[i] = 0;185 dmaParam_.loopInfo.loopLpSize[i] = 0;
188 dmaParam_.loopInfo.loopRpSize[i] = 0;186 dmaParam_.loopInfo.loopRpSize[i] = 0;
189 }187 }
@@ -198,6 +196,6 @@ __aicore__ inline void SplitFrontAxisNddma<T>::InitCopyParams()
198 }196 }
199}197}
200 198 
201-} // namespace StridedSliceGrad199+} // namespace StridedSliceGrad
202 200 
203-#endif // SPLIT_FRONT_AXIS_NDDMA_H201+#endif // SPLIT_FRONT_AXIS_NDDMA_H
Mconversion/triu/op_kernel/arch35/split_medium_shape.h+15-13
@@ -163,8 +163,8 @@ __aicore__ inline void SplitMediumShape<T, IS_LOWER>::DoVectorCopy(const MediumB
163 uint16_t headLoops = headRows_ * vfInnerLoops;163 uint16_t headLoops = headRows_ * vfInnerLoops;
164 uint16_t tailLoops = tailRows_ * vfInnerLoops;164 uint16_t tailLoops = tailRows_ * vfInnerLoops;
165 uint16_t midRows = midRows_;165 uint16_t midRows = midRows_;
166- __local_mem__ T* xAddr = (__local_mem__ T*)xLocal.GetPhyAddr();166+ __ubuf__ T* xAddr = (__ubuf__ T*)xLocal.GetPhyAddr();
167- __local_mem__ T* yAddr = (__local_mem__ T*)yLocal.GetPhyAddr();167+ __ubuf__ T* yAddr = (__ubuf__ T*)yLocal.GetPhyAddr();
168 168 
169 __VEC_SCOPE__169 __VEC_SCOPE__
170 {170 {
@@ -178,10 +178,11 @@ __aicore__ inline void SplitMediumShape<T, IS_LOWER>::DoVectorCopy(const MediumB
178 for (uint16_t m = 0; m < highActual; ++m) {178 for (uint16_t m = 0; m < highActual; ++m) {
179 for (uint16_t i = 0; i < headLoops; ++i) {179 for (uint16_t i = 0; i < headLoops; ++i) {
180 if constexpr (IS_LOWER) {180 if constexpr (IS_LOWER) {
181- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregZero, regCount_, rowMask);181+ AscendC::Reg::StoreAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregZero, regCount_,
182+ rowMask);
182 } else {183 } else {
183- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(vregSrc, xAddr, regCount_);184+ AscendC::Reg::LoadAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(vregSrc, xAddr, regCount_);
184- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregSrc, regCount_, rowMask);185+ AscendC::Reg::StoreAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregSrc, regCount_, rowMask);
185 }186 }
186 }187 }
187 if constexpr (IS_LOWER) {188 if constexpr (IS_LOWER) {
@@ -190,25 +191,26 @@ __aicore__ inline void SplitMediumShape<T, IS_LOWER>::DoVectorCopy(const MediumB
190 for (uint16_t i = 0; i < midRows; ++i) {191 for (uint16_t i = 0; i < midRows; ++i) {
191 uint32_t copyEleNum = copyCols_ + i;192 uint32_t copyEleNum = copyCols_ + i;
192 for (uint16_t j = 0; j < vfInnerLoops; ++j) {193 for (uint16_t j = 0; j < vfInnerLoops; ++j) {
193- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(vregSrc, xAddr, regCount_);194+ AscendC::Reg::LoadAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(vregSrc, xAddr, regCount_);
194 selectMask = AscendC::Reg::UpdateMask<T>(copyEleNum);195 selectMask = AscendC::Reg::UpdateMask<T>(copyEleNum);
195 if constexpr (IS_LOWER) {196 if constexpr (IS_LOWER) {
196 AscendC::Reg::Select(vregDst, vregSrc, vregZero, selectMask);197 AscendC::Reg::Select(vregDst, vregSrc, vregZero, selectMask);
197- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregDst, regCount_,198+ AscendC::Reg::StoreAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregDst, regCount_,
198- rowMask);199+ rowMask);
199 } else {200 } else {
200 AscendC::Reg::Select(vregDst, vregZero, vregSrc, selectMask);201 AscendC::Reg::Select(vregDst, vregZero, vregSrc, selectMask);
201- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregDst, regCount_,202+ AscendC::Reg::StoreAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregDst, regCount_,
202- rowMask);203+ rowMask);
203 }204 }
204 }205 }
205 }206 }
206 for (uint16_t i = 0; i < tailLoops; ++i) {207 for (uint16_t i = 0; i < tailLoops; ++i) {
207 if constexpr (IS_LOWER) {208 if constexpr (IS_LOWER) {
208- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(vregSrc, xAddr, regCount_);209+ AscendC::Reg::LoadAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(vregSrc, xAddr, regCount_);
209- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregSrc, regCount_, rowMask);210+ AscendC::Reg::StoreAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregSrc, regCount_, rowMask);
210 } else {211 } else {
211- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregZero, regCount_, rowMask);212+ AscendC::Reg::StoreAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregZero, regCount_,
213+ rowMask);
212 }214 }
213 }215 }
214 if constexpr (!IS_LOWER) {216 if constexpr (!IS_LOWER) {
Mconversion/triu/op_kernel/arch35/split_row_and_col.h+15-13
@@ -236,8 +236,8 @@ __aicore__ inline void SplitRowAndCol<T, IS_LOWER>::DoCopyVector(NormalBlockInfo
236 uint16_t tailLoops = tailRows * vfInnerLoops;236 uint16_t tailLoops = tailRows * vfInnerLoops;
237 uint16_t midRowsU16 = static_cast<uint16_t>(midRows);237 uint16_t midRowsU16 = static_cast<uint16_t>(midRows);
238 238 
239- __local_mem__ T* xAddr = (__local_mem__ T*)xLocal.GetPhyAddr();239+ __ubuf__ T* xAddr = (__ubuf__ T*)xLocal.GetPhyAddr();
240- __local_mem__ T* yAddr = (__local_mem__ T*)yLocal.GetPhyAddr();240+ __ubuf__ T* yAddr = (__ubuf__ T*)yLocal.GetPhyAddr();
241 241 
242 __VEC_SCOPE__242 __VEC_SCOPE__
243 {243 {
@@ -250,10 +250,11 @@ __aicore__ inline void SplitRowAndCol<T, IS_LOWER>::DoCopyVector(NormalBlockInfo
250 AscendC::Reg::MaskReg selectMask;250 AscendC::Reg::MaskReg selectMask;
251 for (uint16_t i = 0; i < headLoops; ++i) {251 for (uint16_t i = 0; i < headLoops; ++i) {
252 if constexpr (IS_LOWER) {252 if constexpr (IS_LOWER) {
253- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregZero, regCount_, rowMask);253+ AscendC::Reg::StoreAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregZero, regCount_,
254+ rowMask);
254 } else {255 } else {
255- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(vregSrc, xAddr, regCount_);256+ AscendC::Reg::LoadAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(vregSrc, xAddr, regCount_);
256- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregSrc, regCount_, rowMask);257+ AscendC::Reg::StoreAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregSrc, regCount_, rowMask);
257 }258 }
258 }259 }
259 if constexpr (IS_LOWER) {260 if constexpr (IS_LOWER) {
@@ -262,25 +263,26 @@ __aicore__ inline void SplitRowAndCol<T, IS_LOWER>::DoCopyVector(NormalBlockInfo
262 for (uint16_t i = 0; i < midRowsU16; ++i) {263 for (uint16_t i = 0; i < midRowsU16; ++i) {
263 uint32_t copyEleNum = copyCols + i;264 uint32_t copyEleNum = copyCols + i;
264 for (uint16_t j = 0; j < vfInnerLoops; ++j) {265 for (uint16_t j = 0; j < vfInnerLoops; ++j) {
265- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(vregSrc, xAddr, regCount_);266+ AscendC::Reg::LoadAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(vregSrc, xAddr, regCount_);
266 selectMask = AscendC::Reg::UpdateMask<T>(copyEleNum);267 selectMask = AscendC::Reg::UpdateMask<T>(copyEleNum);
267 if constexpr (IS_LOWER) {268 if constexpr (IS_LOWER) {
268 AscendC::Reg::Select(vregDst, vregSrc, vregZero, selectMask);269 AscendC::Reg::Select(vregDst, vregSrc, vregZero, selectMask);
269- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregDst, regCount_,270+ AscendC::Reg::StoreAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregDst, regCount_,
270- rowMask);271+ rowMask);
271 } else {272 } else {
272 AscendC::Reg::Select(vregDst, vregZero, vregSrc, selectMask);273 AscendC::Reg::Select(vregDst, vregZero, vregSrc, selectMask);
273- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregDst, regCount_,274+ AscendC::Reg::StoreAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregDst, regCount_,
274- rowMask);275+ rowMask);
275 }276 }
276 }277 }
277 }278 }
278 for (uint16_t i = 0; i < tailLoops; ++i) {279 for (uint16_t i = 0; i < tailLoops; ++i) {
279 if constexpr (IS_LOWER) {280 if constexpr (IS_LOWER) {
280- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(vregSrc, xAddr, regCount_);281+ AscendC::Reg::LoadAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(vregSrc, xAddr, regCount_);
281- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregSrc, regCount_, rowMask);282+ AscendC::Reg::StoreAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregSrc, regCount_, rowMask);
282 } else {283 } else {
283- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregZero, regCount_, rowMask);284+ AscendC::Reg::StoreAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(yAddr, vregZero, regCount_,
285+ rowMask);
284 }286 }
285 }287 }
286 if constexpr (!IS_LOWER) {288 if constexpr (!IS_LOWER) {
Mconversion/triu/op_kernel/arch35/split_tiny_shape.h+15-15
@@ -158,9 +158,9 @@ __aicore__ inline void SplitTinyShape<T>::DoVectorCopy(const TinyBlockInfo& info
158 LocalTensor<T> xLocal = inQueue_.DeQue<T>();158 LocalTensor<T> xLocal = inQueue_.DeQue<T>();
159 LocalTensor<T> yLocal = outQueue_.AllocTensor<T>();159 LocalTensor<T> yLocal = outQueue_.AllocTensor<T>();
160 160 
161- auto* srcAddr = (__local_mem__ T*)xLocal.GetPhyAddr();161+ auto* srcAddr = (__ubuf__ T*)xLocal.GetPhyAddr();
162- auto* dstAddr = (__local_mem__ T*)yLocal.GetPhyAddr();162+ auto* dstAddr = (__ubuf__ T*)yLocal.GetPhyAddr();
163- auto* maskUbAddr = (__local_mem__ uint8_t*)maskLocal_.GetPhyAddr();163+ auto* maskUbAddr = (__ubuf__ uint8_t*)maskLocal_.GetPhyAddr();
164 164 
165 constexpr int64_t dataCount = GetVRegSize() / sizeof(T);165 constexpr int64_t dataCount = GetVRegSize() / sizeof(T);
166 uint16_t repeatTimes = static_cast<uint16_t>(info.highActual);166 uint16_t repeatTimes = static_cast<uint16_t>(info.highActual);
@@ -180,24 +180,24 @@ __aicore__ inline void SplitTinyShape<T>::DoVectorCopy(const TinyBlockInfo& info
180 AscendC::Reg::Duplicate(sreg, 0);180 AscendC::Reg::Duplicate(sreg, 0);
181 181 
182 if constexpr (sizeof(T) / sizeof(int8_t) == DOUBLE) {182 if constexpr (sizeof(T) / sizeof(int8_t) == DOUBLE) {
183- AscendC::Reg::DataCopy<uint8_t, AscendC::Reg::MaskDist::DIST_US>(maskRegO, maskUbAddr);183+ AscendC::Reg::LoadAlign<uint8_t, AscendC::Reg::MaskDist::DIST_US>(maskRegO, maskUbAddr);
184 } else if constexpr (sizeof(T) / sizeof(int8_t) == QUADRUPLE) {184 } else if constexpr (sizeof(T) / sizeof(int8_t) == QUADRUPLE) {
185- AscendC::Reg::DataCopy(maskReg, maskUbAddr);185+ AscendC::Reg::LoadAlign(maskReg, maskUbAddr);
186- AscendC::Reg::MaskUnPack(maskRegD, maskReg);186+ AscendC::Reg::UnPack(maskRegD, maskReg);
187- AscendC::Reg::MaskUnPack(maskRegO, maskRegD);187+ AscendC::Reg::UnPack(maskRegO, maskRegD);
188 } else if constexpr (sizeof(T) / sizeof(int8_t) == OCTA) {188 } else if constexpr (sizeof(T) / sizeof(int8_t) == OCTA) {
189- AscendC::Reg::DataCopy(maskReg, maskUbAddr);189+ AscendC::Reg::LoadAlign(maskReg, maskUbAddr);
190- AscendC::Reg::MaskUnPack(maskRegD, maskReg);190+ AscendC::Reg::UnPack(maskRegD, maskReg);
191- AscendC::Reg::MaskUnPack(maskRegQ, maskRegD);191+ AscendC::Reg::UnPack(maskRegQ, maskRegD);
192- AscendC::Reg::MaskUnPack(maskRegO, maskRegQ);192+ AscendC::Reg::UnPack(maskRegO, maskRegQ);
193 } else {193 } else {
194- AscendC::Reg::DataCopy(maskRegO, maskUbAddr);194+ AscendC::Reg::LoadAlign(maskRegO, maskUbAddr);
195 }195 }
196 196 
197 for (uint16_t i = 0; i < repeatTimes; i++) {197 for (uint16_t i = 0; i < repeatTimes; i++) {
198- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(vreg, srcAddr, dataCount);198+ AscendC::Reg::LoadAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(vreg, srcAddr, dataCount);
199- AscendC::Reg::DataCopy(dstAddr, sreg, moveMask);199+ AscendC::Reg::StoreAlign(dstAddr, sreg, moveMask);
200- AscendC::Reg::DataCopy<T, Reg::PostLiteral::POST_MODE_UPDATE>(dstAddr, vreg, dataCount, maskRegO);200+ AscendC::Reg::StoreAlign<T, Reg::PostLiteral::POST_MODE_UPDATE>(dstAddr, vreg, dataCount, maskRegO);
201 }201 }
202 }202 }
203 inQueue_.FreeTensor(xLocal);203 inQueue_.FreeTensor(xLocal);
Mconversion/view_copy/op_kernel/arch35/view_copy_base.h+29-31
@@ -75,8 +75,8 @@ struct CustomCopyExtParams {
75};75};
76 76 
77template <typename T>77template <typename T>
78-__aicore__ inline void CopyGmToUbCompact(const LocalTensor<T> &dstLocal, const GlobalTensor<T> &srcGlobal,78+__aicore__ inline void CopyGmToUbCompact(const LocalTensor<T>& dstLocal, const GlobalTensor<T>& srcGlobal,
79- const DataCopyExtParams &params, bool isCompact=false)79+ const DataCopyExtParams& params, bool isCompact = false)
80{80{
81 DataCopyPadExtParams<T> padExtParams;81 DataCopyPadExtParams<T> padExtParams;
82 padExtParams.isPad = false;82 padExtParams.isPad = false;
@@ -94,27 +94,28 @@ template <typename T>
94class ViewCopyBase {94class ViewCopyBase {
95public:95public:
96 __aicore__ inline ViewCopyBase(){};96 __aicore__ inline ViewCopyBase(){};
97- __aicore__ inline int64_t GetGmOffset(int64_t loopIdx, const int64_t *blockStride, const int64_t *blockSrcStride,97+ __aicore__ inline int64_t GetGmOffset(int64_t loopIdx, const int64_t* blockStride, const int64_t* blockSrcStride,
98 int64_t blockFusedNumber);98 int64_t blockFusedNumber);
99+ 
99public:100public:
100 int64_t tailUbFactor_;101 int64_t tailUbFactor_;
101 int32_t tailNddmaSize_[NDDMA_ARRAY_LEN] = {1, 1, 1, 1, 1, 1, 1, 1};102 int32_t tailNddmaSize_[NDDMA_ARRAY_LEN] = {1, 1, 1, 1, 1, 1, 1, 1};
102 int32_t tailUbDstSize_[NDDMA_ARRAY_LEN] = {1, 1, 1, 1, 1, 1, 1, 1};103 int32_t tailUbDstSize_[NDDMA_ARRAY_LEN] = {1, 1, 1, 1, 1, 1, 1, 1};
103 104 
104protected:105protected:
105- __aicore__ inline void ParseTilingData(const ViewCopyTilingData *tilingData);106+ __aicore__ inline void ParseTilingData(const ViewCopyTilingData* tilingData);
106- __aicore__ inline void CopyArray(const int64_t *src, int64_t *dst, int16_t size);107+ __aicore__ inline void CopyArray(const int64_t* src, int64_t* dst, int16_t size);
107- __aicore__ inline void CopyArray(const int32_t *src, int32_t *dst, int16_t size);108+ __aicore__ inline void CopyArray(const int32_t* src, int32_t* dst, int16_t size);
108- __aicore__ inline void CopyInDim3(const GlobalTensor<T> &src, const LocalTensor<T> &dstLocal,109+ __aicore__ inline void CopyInDim3(const GlobalTensor<T>& src, const LocalTensor<T>& dstLocal,
109- const MultiCopyParams<T, DIM3> &dmaParams, bool enableMovAlign);110+ const NdDmaParams<T, DIM3>& dmaParams, bool enableMovAlign);
110- __aicore__ inline void CopyInDim4(const GlobalTensor<T> &src, const LocalTensor<T> &dstLocal,111+ __aicore__ inline void CopyInDim4(const GlobalTensor<T>& src, const LocalTensor<T>& dstLocal,
111- const MultiCopyParams<T, DIM4> &dmaParams, bool enableMovAlign);112+ const NdDmaParams<T, DIM4>& dmaParams, bool enableMovAlign);
112- __aicore__ inline void CopyInDim5(const GlobalTensor<T> &src, const LocalTensor<T> &dstLocal,113+ __aicore__ inline void CopyInDim5(const GlobalTensor<T>& src, const LocalTensor<T>& dstLocal,
113- const MultiCopyParams<T, DIM5> &dmaParams, bool enableMovAlign);114+ const NdDmaParams<T, DIM5>& dmaParams, bool enableMovAlign);
114};115};
115 116 
116template <typename T>117template <typename T>
117-__aicore__ inline void ViewCopyBase<T>::ParseTilingData(const ViewCopyTilingData *tilingData)118+__aicore__ inline void ViewCopyBase<T>::ParseTilingData(const ViewCopyTilingData* tilingData)
118{119{
119 CopyArray(tilingData->nddmaSize, tailNddmaSize_, tilingData->nddmaSizeLen);120 CopyArray(tilingData->nddmaSize, tailNddmaSize_, tilingData->nddmaSizeLen);
120 CopyArray(tilingData->ubDstSize, tailUbDstSize_, tilingData->ubDstSizeLen);121 CopyArray(tilingData->ubDstSize, tailUbDstSize_, tilingData->ubDstSizeLen);
@@ -124,7 +125,7 @@ __aicore__ inline void ViewCopyBase<T>::ParseTilingData(const ViewCopyTilingData
124}125}
125 126 
126template <typename T>127template <typename T>
127-__aicore__ inline void ViewCopyBase<T>::CopyArray(const int32_t *src, int32_t *dst, int16_t size)128+__aicore__ inline void ViewCopyBase<T>::CopyArray(const int32_t* src, int32_t* dst, int16_t size)
128{129{
129 for (int i = 0; i < size; i++) {130 for (int i = 0; i < size; i++) {
130 dst[i] = src[i];131 dst[i] = src[i];
@@ -132,12 +133,12 @@ __aicore__ inline void ViewCopyBase<T>::CopyArray(const int32_t *src, int32_t *d
132}133}
133 134 
134template <typename T>135template <typename T>
135-__aicore__ inline int64_t ViewCopyBase<T>::GetGmOffset(int64_t loopIdx, const int64_t *blockStride,136+__aicore__ inline int64_t ViewCopyBase<T>::GetGmOffset(int64_t loopIdx, const int64_t* blockStride,
136- const int64_t *blockSrcStride, int64_t blockFusedNumber)137+ const int64_t* blockSrcStride, int64_t blockFusedNumber)
137{138{
138 int64_t offset = 0;139 int64_t offset = 0;
139 int64_t curLoopIdx = loopIdx;140 int64_t curLoopIdx = loopIdx;
140- for (int64_t idx=0; idx < blockFusedNumber; idx++){141+ for (int64_t idx = 0; idx < blockFusedNumber; idx++) {
141 offset += (curLoopIdx / blockStride[idx] * blockSrcStride[idx]);142 offset += (curLoopIdx / blockStride[idx] * blockSrcStride[idx]);
142 curLoopIdx = curLoopIdx % blockStride[idx];143 curLoopIdx = curLoopIdx % blockStride[idx];
143 }144 }
@@ -145,8 +146,8 @@ __aicore__ inline int64_t ViewCopyBase<T>::GetGmOffset(int64_t loopIdx, const in
145}146}
146 147 
147template <typename T>148template <typename T>
148-__aicore__ inline void ViewCopyBase<T>::CopyInDim3(const GlobalTensor<T> &src, const LocalTensor<T> &dstLocal,149+__aicore__ inline void ViewCopyBase<T>::CopyInDim3(const GlobalTensor<T>& src, const LocalTensor<T>& dstLocal,
149- const MultiCopyParams<T, DIM3> &dmaParams, bool enableMovAlign)150+ const NdDmaParams<T, DIM3>& dmaParams, bool enableMovAlign)
150{151{
151 if (enableMovAlign) {152 if (enableMovAlign) {
152 int64_t srcOffset = 0;153 int64_t srcOffset = 0;
@@ -156,8 +157,7 @@ __aicore__ inline void ViewCopyBase<T>::CopyInDim3(const GlobalTensor<T> &src, c
156 copyParams.blockCount = loopInfo.loopSize[DIM1];157 copyParams.blockCount = loopInfo.loopSize[DIM1];
157 copyParams.blockLen = loopInfo.loopSize[DIM0] * sizeof(T);158 copyParams.blockLen = loopInfo.loopSize[DIM0] * sizeof(T);
158 copyParams.srcStride = (loopInfo.loopSrcStride[DIM1] - loopInfo.loopSize[DIM0]) * sizeof(T);159 copyParams.srcStride = (loopInfo.loopSrcStride[DIM1] - loopInfo.loopSize[DIM0]) * sizeof(T);
159- copyParams.dstStride = (loopInfo.loopDstStride[DIM1] - loopInfo.loopSize[DIM0]) * sizeof(T) /160+ copyParams.dstStride = (loopInfo.loopDstStride[DIM1] - loopInfo.loopSize[DIM0]) * sizeof(T) / GetUbBlockSize();
160- GetUbBlockSize();
161 bool isCompact = loopInfo.loopDstStride[DIM1] == loopInfo.loopSize[DIM0];161 bool isCompact = loopInfo.loopDstStride[DIM1] == loopInfo.loopSize[DIM0];
162 162 
163 for (uint32_t loopDim2 = 0; loopDim2 < loopInfo.loopSize[DIM2]; loopDim2++) {163 for (uint32_t loopDim2 = 0; loopDim2 < loopInfo.loopSize[DIM2]; loopDim2++) {
@@ -171,8 +171,8 @@ __aicore__ inline void ViewCopyBase<T>::CopyInDim3(const GlobalTensor<T> &src, c
171}171}
172 172 
173template <typename T>173template <typename T>
174-__aicore__ inline void ViewCopyBase<T>::CopyInDim4(const GlobalTensor<T> &src, const LocalTensor<T> &dstLocal,174+__aicore__ inline void ViewCopyBase<T>::CopyInDim4(const GlobalTensor<T>& src, const LocalTensor<T>& dstLocal,
175- const MultiCopyParams<T, DIM4> &dmaParams, bool enableMovAlign)175+ const NdDmaParams<T, DIM4>& dmaParams, bool enableMovAlign)
176{176{
177 if (enableMovAlign) {177 if (enableMovAlign) {
178 const auto& loopInfo = dmaParams.loopInfo;178 const auto& loopInfo = dmaParams.loopInfo;
@@ -182,8 +182,7 @@ __aicore__ inline void ViewCopyBase<T>::CopyInDim4(const GlobalTensor<T> &src, c
182 copyParams.blockCount = loopInfo.loopSize[DIM1];182 copyParams.blockCount = loopInfo.loopSize[DIM1];
183 copyParams.blockLen = loopInfo.loopSize[DIM0] * sizeof(T);183 copyParams.blockLen = loopInfo.loopSize[DIM0] * sizeof(T);
184 copyParams.srcStride = (loopInfo.loopSrcStride[DIM1] - loopInfo.loopSize[DIM0]) * sizeof(T);184 copyParams.srcStride = (loopInfo.loopSrcStride[DIM1] - loopInfo.loopSize[DIM0]) * sizeof(T);
185- copyParams.dstStride = (loopInfo.loopDstStride[DIM1] -185+ copyParams.dstStride = (loopInfo.loopDstStride[DIM1] - loopInfo.loopSize[DIM0]) * sizeof(T) / GetUbBlockSize();
186- loopInfo.loopSize[DIM0]) * sizeof(T) / GetUbBlockSize();
187 bool isCompact = loopInfo.loopDstStride[DIM1] == loopInfo.loopSize[DIM0];186 bool isCompact = loopInfo.loopDstStride[DIM1] == loopInfo.loopSize[DIM0];
188 for (uint32_t loopDim3 = 0; loopDim3 < loopInfo.loopSize[DIM3]; loopDim3++) {187 for (uint32_t loopDim3 = 0; loopDim3 < loopInfo.loopSize[DIM3]; loopDim3++) {
189 for (uint32_t loopDim2 = 0; loopDim2 < loopInfo.loopSize[DIM2]; loopDim2++) {188 for (uint32_t loopDim2 = 0; loopDim2 < loopInfo.loopSize[DIM2]; loopDim2++) {
@@ -198,8 +197,8 @@ __aicore__ inline void ViewCopyBase<T>::CopyInDim4(const GlobalTensor<T> &src, c
198}197}
199 198 
200template <typename T>199template <typename T>
201-__aicore__ inline void ViewCopyBase<T>::CopyInDim5(const GlobalTensor<T> &src, const LocalTensor<T> &dstLocal,200+__aicore__ inline void ViewCopyBase<T>::CopyInDim5(const GlobalTensor<T>& src, const LocalTensor<T>& dstLocal,
202- const MultiCopyParams<T, DIM5> &dmaParams, bool enableMovAlign)201+ const NdDmaParams<T, DIM5>& dmaParams, bool enableMovAlign)
203{202{
204 if (enableMovAlign) {203 if (enableMovAlign) {
205 const auto& loopInfo = dmaParams.loopInfo;204 const auto& loopInfo = dmaParams.loopInfo;
@@ -207,8 +206,7 @@ __aicore__ inline void ViewCopyBase<T>::CopyInDim5(const GlobalTensor<T> &src, c
207 copyParams.blockCount = loopInfo.loopSize[DIM1];206 copyParams.blockCount = loopInfo.loopSize[DIM1];
208 copyParams.blockLen = loopInfo.loopSize[DIM0] * sizeof(T);207 copyParams.blockLen = loopInfo.loopSize[DIM0] * sizeof(T);
209 copyParams.srcStride = (loopInfo.loopSrcStride[DIM1] - loopInfo.loopSize[DIM0]) * sizeof(T);208 copyParams.srcStride = (loopInfo.loopSrcStride[DIM1] - loopInfo.loopSize[DIM0]) * sizeof(T);
210- copyParams.dstStride = (loopInfo.loopDstStride[DIM1] -209+ copyParams.dstStride = (loopInfo.loopDstStride[DIM1] - loopInfo.loopSize[DIM0]) * sizeof(T) / GetUbBlockSize();
211- loopInfo.loopSize[DIM0]) * sizeof(T) / GetUbBlockSize();
212 bool isCompact = loopInfo.loopDstStride[DIM1] == loopInfo.loopSize[DIM0];210 bool isCompact = loopInfo.loopDstStride[DIM1] == loopInfo.loopSize[DIM0];
213 int64_t srcOffset = 0;211 int64_t srcOffset = 0;
214 int64_t dstOffset = 0;212 int64_t dstOffset = 0;
@@ -227,6 +225,6 @@ __aicore__ inline void ViewCopyBase<T>::CopyInDim5(const GlobalTensor<T> &src, c
227 DataCopy(dstLocal, src, dmaParams);225 DataCopy(dstLocal, src, dmaParams);
228 }226 }
229}227}
230-} // namespace ViewCopy228+} // namespace ViewCopy
231 229 
232-#endif // VIEW_COPY_BASE_H_230+#endif // VIEW_COPY_BASE_H_
Mconversion/view_copy/op_kernel/arch35/view_copy_dim1.h+36-47
@@ -24,41 +24,40 @@ using namespace AscendC;
24template <typename T>24template <typename T>
25class ViewCopyDim1 : public ViewCopyBase<T> {25class ViewCopyDim1 : public ViewCopyBase<T> {
26public:26public:
27- __aicore__ inline ViewCopyDim1(TPipe &pipe, const ViewCopyTilingData *tilingData) :27+ __aicore__ inline ViewCopyDim1(TPipe& pipe, const ViewCopyTilingData* tilingData)
28- pipe_(pipe), tilingData_(tilingData) {};28+ : pipe_(pipe), tilingData_(tilingData){};
29- __aicore__ inline void Init(29+ __aicore__ inline void Init(GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset, GM_ADDR src,
30- GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset,30+ GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset, GM_ADDR out);
31- GM_ADDR src, GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset,
32- GM_ADDR out);
33 __aicore__ inline void Process();31 __aicore__ inline void Process();
34 __aicore__ inline void InitCopyParams();32 __aicore__ inline void InitCopyParams();
35 33 
36private:34private:
37- __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM1> &dmaParam,35+ __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM1>& dmaParam,
38- const DataCopyExtParams &copyParams);36+ const DataCopyExtParams& copyParams);
37+ 
39private:38private:
40- TPipe &pipe_;39+ TPipe& pipe_;
41- const ViewCopyTilingData *tilingData_;40+ const ViewCopyTilingData* tilingData_;
42 TQueBind<QuePosition::VECIN, QuePosition::VECOUT, BUFFER_NUM> inQueue_;41 TQueBind<QuePosition::VECIN, QuePosition::VECOUT, BUFFER_NUM> inQueue_;
43 GlobalTensor<T> srcGm_;42 GlobalTensor<T> srcGm_;
44 GlobalTensor<T> dstGm_;43 GlobalTensor<T> dstGm_;
45 44 
46 int64_t blockOffset_ = 0;45 int64_t blockOffset_ = 0;
47- MultiCopyParams<T, DIM1> dmaParam_;46+ NdDmaParams<T, DIM1> dmaParam_;
48- MultiCopyParams<T, DIM1> tailDmaParam_;47+ NdDmaParams<T, DIM1> tailDmaParam_;
49 DataCopyExtParams copyParams_;48 DataCopyExtParams copyParams_;
50 DataCopyExtParams tailCopyParams_;49 DataCopyExtParams tailCopyParams_;
51};50};
52 51 
53template <typename T>52template <typename T>
54-__aicore__ inline void ViewCopyDim1<T>::Init(53+__aicore__ inline void ViewCopyDim1<T>::Init(GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset,
55- GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset,54+ GM_ADDR src, GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset,
56- GM_ADDR src, GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset, GM_ADDR out)55+ GM_ADDR out)
57{56{
58 this->ParseTilingData(tilingData_);57 this->ParseTilingData(tilingData_);
59 blockOffset_ = GetBlockIdx() * tilingData_->blockFactor;58 blockOffset_ = GetBlockIdx() * tilingData_->blockFactor;
60- dstGm_.SetGlobalBuffer((__gm__ T *)(dst + tilingData_->dstStorageOffset * sizeof(T)));59+ dstGm_.SetGlobalBuffer((__gm__ T*)(dst + tilingData_->dstStorageOffset * sizeof(T)));
61- srcGm_.SetGlobalBuffer((__gm__ T *)(src + tilingData_->srcStorageOffset * sizeof(T)));60+ srcGm_.SetGlobalBuffer((__gm__ T*)(src + tilingData_->srcStorageOffset * sizeof(T)));
62 61 
63 pipe_.InitBuffer(inQueue_, BUFFER_NUM, tilingData_->bufferSize);62 pipe_.InitBuffer(inQueue_, BUFFER_NUM, tilingData_->bufferSize);
64}63}
@@ -83,8 +82,8 @@ __aicore__ inline void ViewCopyDim1<T>::Process()
83 for (int64_t idx = 0; idx < loopSize; idx++) {82 for (int64_t idx = 0; idx < loopSize; idx++) {
84 globalLoopIdx = blockOffset_ + idx;83 globalLoopIdx = blockOffset_ + idx;
85 globalLoopIdxModUo = globalLoopIdx % tilingData_->uo;84 globalLoopIdxModUo = globalLoopIdx % tilingData_->uo;
86- if ((globalLoopIdxModUo) * tilingData_->ubFactor <= tilingData_->ubDimSize) {85+ if ((globalLoopIdxModUo)*tilingData_->ubFactor <= tilingData_->ubDimSize) {
87- if((globalLoopIdxModUo + 1) * tilingData_->ubFactor > tilingData_->ubDimSize) {86+ if ((globalLoopIdxModUo + 1) * tilingData_->ubFactor > tilingData_->ubDimSize) {
88 ProcessPerLoop(globalLoopIdx, tailDmaParam_, tailCopyParams_);87 ProcessPerLoop(globalLoopIdx, tailDmaParam_, tailCopyParams_);
89 } else {88 } else {
90 ProcessPerLoop(globalLoopIdx, dmaParam_, copyParams_);89 ProcessPerLoop(globalLoopIdx, dmaParam_, copyParams_);
@@ -94,8 +93,8 @@ __aicore__ inline void ViewCopyDim1<T>::Process()
94}93}
95 94 
96template <typename T>95template <typename T>
97-__aicore__ inline void ViewCopyDim1<T>::ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM1> &dmaParam,96+__aicore__ inline void ViewCopyDim1<T>::ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM1>& dmaParam,
98- const DataCopyExtParams &copyParams)97+ const DataCopyExtParams& copyParams)
99{98{
100 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,99 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,
101 tilingData_->blockFusedDimsNumber);100 tilingData_->blockFusedDimsNumber);
@@ -124,42 +123,32 @@ __aicore__ inline void ViewCopyDim1<T>::InitCopyParams()
124{123{
125 dmaParam_ = {124 dmaParam_ = {
126 {125 {
127- {static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX])}, // src stride126+ {static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX])}, // src stride
128 {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX])}, // dst stride127 {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX])}, // dst stride
129- {static_cast<uint32_t>(tilingData_->nddmaSize[DIM0_INDEX])}, // loop size128+ {static_cast<uint32_t>(tilingData_->nddmaSize[DIM0_INDEX])}, // loop size
130- {ZERO_U8}, // left pad129+ {ZERO_U8}, // left pad
131- {ZERO_U8} // right pad130+ {ZERO_U8} // right pad
132 },131 },
133- 0 //pad value132+ 0 // pad value
134 };133 };
135 134 
136 tailDmaParam_ = {135 tailDmaParam_ = {
137 {136 {
138- {static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX])}, // src stride137+ {static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX])}, // src stride
139 {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX])}, // dst stride138 {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX])}, // dst stride
140- {static_cast<uint32_t>(this->tailNddmaSize_[DIM0_INDEX])}, // loop size139+ {static_cast<uint32_t>(this->tailNddmaSize_[DIM0_INDEX])}, // loop size
141- {ZERO_U8}, // left pad140+ {ZERO_U8}, // left pad
142- {ZERO_U8} // right pad141+ {ZERO_U8} // right pad
143 },142 },
144- 0 //pad value143+ 0 // pad value
145 };144 };
146 145 
147- copyParams_ = {146+ copyParams_ = {static_cast<uint16_t>(1), static_cast<uint32_t>(tilingData_->ubDstSize[DIM0_INDEX] * sizeof(T)),
148- static_cast<uint16_t>(1),147+ static_cast<uint32_t>(0), 0, 0};
149- static_cast<uint32_t>(tilingData_->ubDstSize[DIM0_INDEX] * sizeof(T)),148+ tailCopyParams_ = {static_cast<uint16_t>(1), static_cast<uint32_t>(this->tailUbDstSize_[DIM0_INDEX] * sizeof(T)),
150- static_cast<uint32_t>(0),149+ static_cast<uint32_t>(0), 0, 0};
151- 0,
152- 0
153- };
154- tailCopyParams_ = {
155- static_cast<uint16_t>(1),
156- static_cast<uint32_t>(this->tailUbDstSize_[DIM0_INDEX] * sizeof(T)),
157- static_cast<uint32_t>(0),
158- 0,
159- 0
160- };
161}150}
162 151 
163-} // namespace ViewCopy152+} // namespace ViewCopy
164 153 
165-#endif // VIEW_COPY_DIM1_H_154+#endif // VIEW_COPY_DIM1_H_
Mconversion/view_copy/op_kernel/arch35/view_copy_dim2.h+4-4
@@ -34,7 +34,7 @@ public:
34 __aicore__ inline void InitCopyParams();34 __aicore__ inline void InitCopyParams();
35 35 
36private:36private:
37- __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM2>& dmaParam,37+ __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM2>& dmaParam,
38 const DataCopyExtParams& copyParams);38 const DataCopyExtParams& copyParams);
39 39 
40private:40private:
@@ -46,8 +46,8 @@ private:
46 GlobalTensor<T> srcGm_;46 GlobalTensor<T> srcGm_;
47 47 
48 int64_t blockOffset_ = 0;48 int64_t blockOffset_ = 0;
49- MultiCopyParams<T, DIM2> dmaParam_;49+ NdDmaParams<T, DIM2> dmaParam_;
50- MultiCopyParams<T, DIM2> tailDmaParam_;50+ NdDmaParams<T, DIM2> tailDmaParam_;
51 DataCopyExtParams copyParams_;51 DataCopyExtParams copyParams_;
52 DataCopyExtParams tailCopyParams_;52 DataCopyExtParams tailCopyParams_;
53};53};
@@ -96,7 +96,7 @@ __aicore__ inline void ViewCopyDim2<T>::Process()
96}96}
97 97 
98template <typename T>98template <typename T>
99-__aicore__ inline void ViewCopyDim2<T>::ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM2>& dmaParam,99+__aicore__ inline void ViewCopyDim2<T>::ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM2>& dmaParam,
100 const DataCopyExtParams& copyParams)100 const DataCopyExtParams& copyParams)
101{101{
102 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,102 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,
Mconversion/view_copy/op_kernel/arch35/view_copy_dim3.h+6-6
@@ -34,9 +34,9 @@ public:
34private:34private:
35 __aicore__ inline void CopyOut(const GlobalTensor<T>& dst, const DataCopyExtParams& copyParams,35 __aicore__ inline void CopyOut(const GlobalTensor<T>& dst, const DataCopyExtParams& copyParams,
36 const CustomCopyExtParams& extParams);36 const CustomCopyExtParams& extParams);
37- __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM3>& dmaParams,37+ __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM3>& dmaParams,
38 const DataCopyExtParams& copyParams, const int32_t* outLoopSize);38 const DataCopyExtParams& copyParams, const int32_t* outLoopSize);
39- __aicore__ inline void CopyIn(const GlobalTensor<T>& src, const MultiCopyParams<T, DIM3>& dmaParams);39+ __aicore__ inline void CopyIn(const GlobalTensor<T>& src, const NdDmaParams<T, DIM3>& dmaParams);
40 40 
41private:41private:
42 TPipe& pipe_;42 TPipe& pipe_;
@@ -47,8 +47,8 @@ private:
47 GlobalTensor<T> srcGm_;47 GlobalTensor<T> srcGm_;
48 int64_t blockOffset_ = 0;48 int64_t blockOffset_ = 0;
49 49 
50- MultiCopyParams<T, DIM3> dmaParam_;50+ NdDmaParams<T, DIM3> dmaParam_;
51- MultiCopyParams<T, DIM3> tailDmaParam_;51+ NdDmaParams<T, DIM3> tailDmaParam_;
52 DataCopyExtParams copyParams_;52 DataCopyExtParams copyParams_;
53 DataCopyExtParams tailCopyParams_;53 DataCopyExtParams tailCopyParams_;
54 CustomCopyExtParams outExtParams_;54 CustomCopyExtParams outExtParams_;
@@ -97,7 +97,7 @@ __aicore__ inline void ViewCopyDim3<T>::Process()
97}97}
98 98 
99template <typename T>99template <typename T>
100-__aicore__ inline void ViewCopyDim3<T>::CopyIn(const GlobalTensor<T>& src, const MultiCopyParams<T, DIM3>& dmaParams)100+__aicore__ inline void ViewCopyDim3<T>::CopyIn(const GlobalTensor<T>& src, const NdDmaParams<T, DIM3>& dmaParams)
101{101{
102 LocalTensor<T> dstLocal = inQueue_.AllocTensor<T>();102 LocalTensor<T> dstLocal = inQueue_.AllocTensor<T>();
103 this->CopyInDim3(src, dstLocal, dmaParams, tilingData_->enableMovAlign != 0);103 this->CopyInDim3(src, dstLocal, dmaParams, tilingData_->enableMovAlign != 0);
@@ -123,7 +123,7 @@ __aicore__ inline void ViewCopyDim3<T>::CopyOut(const GlobalTensor<T>& dst, cons
123}123}
124 124 
125template <typename T>125template <typename T>
126-__aicore__ inline void ViewCopyDim3<T>::ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM3>& dmaParams,126+__aicore__ inline void ViewCopyDim3<T>::ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM3>& dmaParams,
127 const DataCopyExtParams& copyParams, const int32_t* outLoopSize)127 const DataCopyExtParams& copyParams, const int32_t* outLoopSize)
128{128{
129 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,129 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,
Mconversion/view_copy/op_kernel/arch35/view_copy_dim4.h+6-6
@@ -34,8 +34,8 @@ public:
34private:34private:
35 __aicore__ inline void CopyOut(const GlobalTensor<T>& dst, const DataCopyExtParams& copyParams,35 __aicore__ inline void CopyOut(const GlobalTensor<T>& dst, const DataCopyExtParams& copyParams,
36 const CustomCopyExtParams& extParams);36 const CustomCopyExtParams& extParams);
37- __aicore__ inline void CopyIn(const GlobalTensor<T>& src, const MultiCopyParams<T, DIM4>& dmaParams);37+ __aicore__ inline void CopyIn(const GlobalTensor<T>& src, const NdDmaParams<T, DIM4>& dmaParams);
38- __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM4>& dmaParams,38+ __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM4>& dmaParams,
39 const DataCopyExtParams& copyParams, const int32_t* outLoopSize);39 const DataCopyExtParams& copyParams, const int32_t* outLoopSize);
40 40 
41private:41private:
@@ -47,8 +47,8 @@ private:
47 GlobalTensor<T> srcGm_;47 GlobalTensor<T> srcGm_;
48 48 
49 int64_t blockOffset_ = 0;49 int64_t blockOffset_ = 0;
50- MultiCopyParams<T, DIM4> dmaParam_;50+ NdDmaParams<T, DIM4> dmaParam_;
51- MultiCopyParams<T, DIM4> tailDmaParam_;51+ NdDmaParams<T, DIM4> tailDmaParam_;
52 DataCopyExtParams copyParams_;52 DataCopyExtParams copyParams_;
53 DataCopyExtParams tailCopyParams_;53 DataCopyExtParams tailCopyParams_;
54 CustomCopyExtParams outExtParams_;54 CustomCopyExtParams outExtParams_;
@@ -100,7 +100,7 @@ __aicore__ inline void ViewCopyDim4<T>::Process()
100}100}
101 101 
102template <typename T>102template <typename T>
103-__aicore__ inline void ViewCopyDim4<T>::CopyIn(const GlobalTensor<T>& src, const MultiCopyParams<T, DIM4>& dmaParams)103+__aicore__ inline void ViewCopyDim4<T>::CopyIn(const GlobalTensor<T>& src, const NdDmaParams<T, DIM4>& dmaParams)
104{104{
105 LocalTensor<T> dstLocal = inQueue_.AllocTensor<T>();105 LocalTensor<T> dstLocal = inQueue_.AllocTensor<T>();
106 this->CopyInDim4(src, dstLocal, dmaParams, tilingData_->enableMovAlign != 0);106 this->CopyInDim4(src, dstLocal, dmaParams, tilingData_->enableMovAlign != 0);
@@ -127,7 +127,7 @@ __aicore__ inline void ViewCopyDim4<T>::CopyOut(const GlobalTensor<T>& dst, cons
127}127}
128 128 
129template <typename T>129template <typename T>
130-__aicore__ inline void ViewCopyDim4<T>::ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM4>& dmaParams,130+__aicore__ inline void ViewCopyDim4<T>::ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM4>& dmaParams,
131 const DataCopyExtParams& copyParams, const int32_t* outLoopSize)131 const DataCopyExtParams& copyParams, const int32_t* outLoopSize)
132{132{
133 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,133 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,
Mconversion/view_copy/op_kernel/arch35/view_copy_dim5.h+6-6
@@ -32,10 +32,10 @@ public:
32 __aicore__ inline void InitCopyParams();32 __aicore__ inline void InitCopyParams();
33 33 
34private:34private:
35- __aicore__ inline void CopyIn(const GlobalTensor<T>& src, const MultiCopyParams<T, DIM5>& dmaParams);35+ __aicore__ inline void CopyIn(const GlobalTensor<T>& src, const NdDmaParams<T, DIM5>& dmaParams);
36 __aicore__ inline void CopyOut(const GlobalTensor<T>& dst, const DataCopyExtParams& copyParams,36 __aicore__ inline void CopyOut(const GlobalTensor<T>& dst, const DataCopyExtParams& copyParams,
37 const CustomCopyExtParams& extParams);37 const CustomCopyExtParams& extParams);
38- __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM5>& dmaParams,38+ __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM5>& dmaParams,
39 const DataCopyExtParams& copyParams, const int32_t* outLoopSize);39 const DataCopyExtParams& copyParams, const int32_t* outLoopSize);
40 40 
41private:41private:
@@ -47,8 +47,8 @@ private:
47 GlobalTensor<T> dstGm_;47 GlobalTensor<T> dstGm_;
48 GlobalTensor<T> srcGm_;48 GlobalTensor<T> srcGm_;
49 49 
50- MultiCopyParams<T, DIM5> dmaParam_;50+ NdDmaParams<T, DIM5> dmaParam_;
51- MultiCopyParams<T, DIM5> tailDmaParam_;51+ NdDmaParams<T, DIM5> tailDmaParam_;
52 DataCopyExtParams copyParams_;52 DataCopyExtParams copyParams_;
53 DataCopyExtParams tailCopyParams_;53 DataCopyExtParams tailCopyParams_;
54 CustomCopyExtParams outExtParams_;54 CustomCopyExtParams outExtParams_;
@@ -99,7 +99,7 @@ __aicore__ inline void ViewCopyDim5<T>::Process()
99}99}
100 100 
101template <typename T>101template <typename T>
102-__aicore__ inline void ViewCopyDim5<T>::CopyIn(const GlobalTensor<T>& src, const MultiCopyParams<T, DIM5>& dmaParams)102+__aicore__ inline void ViewCopyDim5<T>::CopyIn(const GlobalTensor<T>& src, const NdDmaParams<T, DIM5>& dmaParams)
103{103{
104 LocalTensor<T> dstLocal = inQueue_.AllocTensor<T>();104 LocalTensor<T> dstLocal = inQueue_.AllocTensor<T>();
105 this->CopyInDim5(src, dstLocal, dmaParams, tilingData_->enableMovAlign != 0);105 this->CopyInDim5(src, dstLocal, dmaParams, tilingData_->enableMovAlign != 0);
@@ -130,7 +130,7 @@ __aicore__ inline void ViewCopyDim5<T>::CopyOut(const GlobalTensor<T>& dst, cons
130}130}
131 131 
132template <typename T>132template <typename T>
133-__aicore__ inline void ViewCopyDim5<T>::ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM5>& dmaParams,133+__aicore__ inline void ViewCopyDim5<T>::ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM5>& dmaParams,
134 const DataCopyExtParams& copyParams, const int32_t* outLoopSize)134 const DataCopyExtParams& copyParams, const int32_t* outLoopSize)
135{135{
136 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,136 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,
Mconversion/view_copy/op_kernel/arch35/view_copy_dim8.h+9-9
@@ -33,10 +33,10 @@ public:
33 33 
34private:34private:
35 __aicore__ inline void CustomNddma(LocalTensor<T>& dst, const GlobalTensor<T>& src,35 __aicore__ inline void CustomNddma(LocalTensor<T>& dst, const GlobalTensor<T>& src,
36- const MultiCopyParams<T, 5>& dmaParam, const CustomCopyExtParams& extParams);36+ const NdDmaParams<T, 5>& dmaParam, const CustomCopyExtParams& extParams);
37 __aicore__ inline void CopyOut(const GlobalTensor<T>& dst, const DataCopyExtParams& copyParams,37 __aicore__ inline void CopyOut(const GlobalTensor<T>& dst, const DataCopyExtParams& copyParams,
38 const CustomCopyExtParams& extParams);38 const CustomCopyExtParams& extParams);
39- __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM5>& dmaParams,39+ __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM5>& dmaParams,
40 const DataCopyExtParams& copyParams, const int32_t* inLoopSize,40 const DataCopyExtParams& copyParams, const int32_t* inLoopSize,
41 const int32_t* outLoopSize);41 const int32_t* outLoopSize);
42 42 
@@ -51,8 +51,8 @@ private:
51 int64_t blockOffset_ = 0;51 int64_t blockOffset_ = 0;
52 CustomCopyExtParams outExtParams_;52 CustomCopyExtParams outExtParams_;
53 CustomCopyExtParams inExtParams_;53 CustomCopyExtParams inExtParams_;
54- MultiCopyParams<T, DIM5> dmaParam_;54+ NdDmaParams<T, DIM5> dmaParam_;
55- MultiCopyParams<T, DIM5> tailDmaParam_;55+ NdDmaParams<T, DIM5> tailDmaParam_;
56 DataCopyExtParams copyParams_;56 DataCopyExtParams copyParams_;
57 DataCopyExtParams tailCopyParams_;57 DataCopyExtParams tailCopyParams_;
58};58};
@@ -104,7 +104,7 @@ __aicore__ inline void ViewCopyDim8<T>::Process()
104}104}
105 105 
106template <typename T>106template <typename T>
107-__aicore__ inline void ViewCopyDim8<T>::ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM5>& dmaParams,107+__aicore__ inline void ViewCopyDim8<T>::ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM5>& dmaParams,
108 const DataCopyExtParams& copyParams, const int32_t* inLoopSize,108 const DataCopyExtParams& copyParams, const int32_t* inLoopSize,
109 const int32_t* outLoopSize)109 const int32_t* outLoopSize)
110{110{
@@ -122,7 +122,7 @@ __aicore__ inline void ViewCopyDim8<T>::ProcessPerLoop(int64_t globalLoopIdx, co
122 122 
123template <typename T>123template <typename T>
124__aicore__ inline void ViewCopyDim8<T>::CustomNddma(LocalTensor<T>& dst, const GlobalTensor<T>& src,124__aicore__ inline void ViewCopyDim8<T>::CustomNddma(LocalTensor<T>& dst, const GlobalTensor<T>& src,
125- const MultiCopyParams<T, DIM5>& dmaParam,125+ const NdDmaParams<T, DIM5>& dmaParam,
126 const CustomCopyExtParams& extParams)126 const CustomCopyExtParams& extParams)
127{127{
128 // srcSize (dim7, dim6, dim5, dim4, dim3, dim2, dim1, dim0)128 // srcSize (dim7, dim6, dim5, dim4, dim3, dim2, dim1, dim0)
@@ -209,7 +209,7 @@ __aicore__ inline void ViewCopyDim8<T>::InitCopyParams()
209 {ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8}, // left pad209 {ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8}, // left pad
210 {ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8} // right pad210 {ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8} // right pad
211 },211 },
212- 0 // pad value212+ 0 // pad value
213 };213 };
214 214 
215 tailDmaParam_ = {215 tailDmaParam_ = {
@@ -239,7 +239,7 @@ __aicore__ inline void ViewCopyDim8<T>::InitCopyParams()
239 {ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8}, // left pad239 {ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8}, // left pad
240 {ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8} // right pad240 {ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8, ZERO_U8} // right pad
241 },241 },
242- 0 // pad value242+ 0 // pad value
243 };243 };
244 int64_t copyDstStride = (tilingData_->ubDstStride[DIM1_INDEX] - tilingData_->ubDstSize[DIM0_INDEX]) * sizeof(T);244 int64_t copyDstStride = (tilingData_->ubDstStride[DIM1_INDEX] - tilingData_->ubDstSize[DIM0_INDEX]) * sizeof(T);
245 int64_t tailCopyDstStride = (tilingData_->ubDstStride[DIM1_INDEX] - this->tailUbDstSize_[DIM0_INDEX]) * sizeof(T);245 int64_t tailCopyDstStride = (tilingData_->ubDstStride[DIM1_INDEX] - this->tailUbDstSize_[DIM0_INDEX]) * sizeof(T);
@@ -252,4 +252,4 @@ __aicore__ inline void ViewCopyDim8<T>::InitCopyParams()
252}252}
253} // namespace ViewCopy253} // namespace ViewCopy
254 254 
255-#endif // VIEW_COPY_DIM8_H_255+#endif // VIEW_COPY_DIM8_H_
Mconversion/view_copy/op_kernel/arch35/view_copy_simt_dim1.h+38-49
@@ -35,22 +35,20 @@ __simt_vf__ LAUNCH_BOUND(THREAD_DIM) __aicore__
35template <typename T>35template <typename T>
36class ViewCopySimtDim1 : public ViewCopyBase<T> {36class ViewCopySimtDim1 : public ViewCopyBase<T> {
37public:37public:
38- __aicore__ inline ViewCopySimtDim1(TPipe &pipe, const ViewCopyTilingData *tilingData) :38+ __aicore__ inline ViewCopySimtDim1(TPipe& pipe, const ViewCopyTilingData* tilingData)
39- pipe_(pipe), tilingData_(tilingData) {};39+ : pipe_(pipe), tilingData_(tilingData){};
40- __aicore__ inline void Init(40+ __aicore__ inline void Init(GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset, GM_ADDR src,
41- GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset,41+ GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset, GM_ADDR out);
42- GM_ADDR src, GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset,
43- GM_ADDR out);
44 __aicore__ inline void Process();42 __aicore__ inline void Process();
45 __aicore__ inline void InitCopyParams();43 __aicore__ inline void InitCopyParams();
46 44 
47private:45private:
48- __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM1> &dmaParam,46+ __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM1>& dmaParam,
49- int32_t batchSize);47+ int32_t batchSize);
50 48 
51private:49private:
52- TPipe &pipe_;50+ TPipe& pipe_;
53- const ViewCopyTilingData *tilingData_;51+ const ViewCopyTilingData* tilingData_;
54 TQue<QuePosition::VECIN, BUFFER_NUM> inQueue_;52 TQue<QuePosition::VECIN, BUFFER_NUM> inQueue_;
55 53 
56 GlobalTensor<T> dstGm_;54 GlobalTensor<T> dstGm_;
@@ -59,19 +57,18 @@ private:
59 57 
60 DataCopyExtParams copyParams_;58 DataCopyExtParams copyParams_;
61 DataCopyExtParams tailCopyParams_;59 DataCopyExtParams tailCopyParams_;
62- MultiCopyParams<T, DIM1> dmaParam_;60+ NdDmaParams<T, DIM1> dmaParam_;
63- MultiCopyParams<T, DIM1> tailDmaParam_;61+ NdDmaParams<T, DIM1> tailDmaParam_;
64};62};
65 63 
66template <typename T>64template <typename T>
67-__aicore__ inline void ViewCopySimtDim1<T>::Init(65+__aicore__ inline void ViewCopySimtDim1<T>::Init(GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride,
68- GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset,66+ GM_ADDR dstStorageOffset, GM_ADDR src, GM_ADDR srcSize,
69- GM_ADDR src, GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset,67+ GM_ADDR srcStride, GM_ADDR srcStorageOffset, GM_ADDR out)
70- GM_ADDR out)
71{68{
72 this->ParseTilingData(tilingData_);69 this->ParseTilingData(tilingData_);
73- srcGm_.SetGlobalBuffer((__gm__ T *)(src + tilingData_->srcStorageOffset * sizeof(T)));70+ srcGm_.SetGlobalBuffer((__gm__ T*)(src + tilingData_->srcStorageOffset * sizeof(T)));
74- dstGm_.SetGlobalBuffer((__gm__ T *)(dst + tilingData_->dstStorageOffset * sizeof(T)));71+ dstGm_.SetGlobalBuffer((__gm__ T*)(dst + tilingData_->dstStorageOffset * sizeof(T)));
75 72 
76 pipe_.InitBuffer(inQueue_, BUFFER_NUM, tilingData_->bufferSize);73 pipe_.InitBuffer(inQueue_, BUFFER_NUM, tilingData_->bufferSize);
77 blockOffset_ = GetBlockIdx() * tilingData_->blockFactor;74 blockOffset_ = GetBlockIdx() * tilingData_->blockFactor;
@@ -97,7 +94,7 @@ __aicore__ inline void ViewCopySimtDim1<T>::Process()
97 for (int64_t idx = 0; idx < loopSize; idx++) {94 for (int64_t idx = 0; idx < loopSize; idx++) {
98 globalLoopIdx = blockOffset_ + idx;95 globalLoopIdx = blockOffset_ + idx;
99 globalLoopIdxModUo = globalLoopIdx % tilingData_->uo;96 globalLoopIdxModUo = globalLoopIdx % tilingData_->uo;
100- if ((globalLoopIdxModUo) * tilingData_->ubFactor <= tilingData_->ubDimSize) {97+ if ((globalLoopIdxModUo)*tilingData_->ubFactor <= tilingData_->ubDimSize) {
101 if ((globalLoopIdxModUo + 1) * tilingData_->ubFactor > tilingData_->ubDimSize) {98 if ((globalLoopIdxModUo + 1) * tilingData_->ubFactor > tilingData_->ubDimSize) {
102 ProcessPerLoop(globalLoopIdx, tailDmaParam_, this->tailUbDstSize_[DIM0_INDEX]);99 ProcessPerLoop(globalLoopIdx, tailDmaParam_, this->tailUbDstSize_[DIM0_INDEX]);
103 } else {100 } else {
@@ -108,11 +105,11 @@ __aicore__ inline void ViewCopySimtDim1<T>::Process()
108}105}
109 106 
110template <typename T>107template <typename T>
111-__aicore__ inline void ViewCopySimtDim1<T>::ProcessPerLoop(int64_t globalLoopIdx,108+__aicore__ inline void ViewCopySimtDim1<T>::ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM1>& dmaParam,
112- const MultiCopyParams<T, DIM1> &dmaParam, int32_t batchSize)109+ int32_t batchSize)
113{110{
114 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,111 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,
115- tilingData_->blockFusedDimsNumber);112+ tilingData_->blockFusedDimsNumber);
116 LocalTensor<T> srcLocal = inQueue_.AllocTensor<T>();113 LocalTensor<T> srcLocal = inQueue_.AllocTensor<T>();
117 if (tilingData_->enableMovAlign != 0) {114 if (tilingData_->enableMovAlign != 0) {
118 DataCopyExtParams copyParam;115 DataCopyExtParams copyParam;
@@ -126,19 +123,19 @@ __aicore__ inline void ViewCopySimtDim1<T>::ProcessPerLoop(int64_t globalLoopIdx
126 }123 }
127 inQueue_.EnQue(srcLocal);124 inQueue_.EnQue(srcLocal);
128 int64_t dstOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockDstStride,125 int64_t dstOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockDstStride,
129- tilingData_->blockFusedDimsNumber);126+ tilingData_->blockFusedDimsNumber);
130 LocalTensor<T> dstLocal = inQueue_.DeQue<T>();127 LocalTensor<T> dstLocal = inQueue_.DeQue<T>();
131 __ubuf__ T* srcAddr = (__ubuf__ T*)dstLocal.GetPhyAddr();128 __ubuf__ T* srcAddr = (__ubuf__ T*)dstLocal.GetPhyAddr();
132 __gm__ T* dstAddr = (__gm__ T*)(dstGm_.GetPhyAddr()) + dstOffset;129 __gm__ T* dstAddr = (__gm__ T*)(dstGm_.GetPhyAddr()) + dstOffset;
133 uint32_t srcStrideDim0 = tilingData_->contiguousUbDstStride[DIM0_INDEX];130 uint32_t srcStrideDim0 = tilingData_->contiguousUbDstStride[DIM0_INDEX];
134 if (tilingData_->enableDstInt64 != 0) {131 if (tilingData_->enableDstInt64 != 0) {
135 uint64_t dstStrideDim0 = static_cast<uint64_t>(tilingData_->ubDstStride[DIM0_INDEX]);132 uint64_t dstStrideDim0 = static_cast<uint64_t>(tilingData_->ubDstStride[DIM0_INDEX]);
136- asc_vf_call<CopyUbToGmDim1<T, uint64_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize,133+ asc_vf_call<CopyUbToGmDim1<T, uint64_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize, srcStrideDim0,
137- srcStrideDim0, dstStrideDim0);134+ dstStrideDim0);
138 } else {135 } else {
139 uint32_t dstStrideDim0 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM0_INDEX]);136 uint32_t dstStrideDim0 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM0_INDEX]);
140- asc_vf_call<CopyUbToGmDim1<T, uint32_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize,137+ asc_vf_call<CopyUbToGmDim1<T, uint32_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize, srcStrideDim0,
141- srcStrideDim0, dstStrideDim0);138+ dstStrideDim0);
142 }139 }
143 inQueue_.FreeTensor(dstLocal);140 inQueue_.FreeTensor(dstLocal);
144}141}
@@ -146,28 +143,20 @@ __aicore__ inline void ViewCopySimtDim1<T>::ProcessPerLoop(int64_t globalLoopIdx
146template <typename T>143template <typename T>
147__aicore__ inline void ViewCopySimtDim1<T>::InitCopyParams()144__aicore__ inline void ViewCopySimtDim1<T>::InitCopyParams()
148{145{
149- dmaParam_ = {146+ dmaParam_ = {{{static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX])},
150- {147+ {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX])},
151- {static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX])},148+ {static_cast<uint32_t>(tilingData_->nddmaSize[DIM0_INDEX])},
152- {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX])},149+ {0},
153- {static_cast<uint32_t>(tilingData_->nddmaSize[DIM0_INDEX])},150+ {0}},
154- {0},151+ 0};
155- {0}152+ tailDmaParam_ = {{{static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX])},
156- },153+ {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX])},
157- 0154+ {static_cast<uint32_t>(this->tailNddmaSize_[DIM0_INDEX])},
158- };155+ {0},
159- tailDmaParam_ = {156+ {0}},
160- {157+ 0};
161- {static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX])},
162- {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX])},
163- {static_cast<uint32_t>(this->tailNddmaSize_[DIM0_INDEX])},
164- {0},
165- {0}
166- },
167- 0
168- };
169}158}
170 159 
171-} // namespace ViewCopy160+} // namespace ViewCopy
172 161 
173-#endif // VIEW_COPY_SIMT_DIM1_H_162+#endif // VIEW_COPY_SIMT_DIM1_H_
Mconversion/view_copy/op_kernel/arch35/view_copy_simt_dim2.h+48-58
@@ -26,7 +26,8 @@ using namespace Ops::Base;
26template <typename T, typename U>26template <typename T, typename U>
27__simt_vf__ LAUNCH_BOUND(THREAD_DIM_1024) __aicore__27__simt_vf__ LAUNCH_BOUND(THREAD_DIM_1024) __aicore__
28 void CopyUbToGmDim2(__gm__ T* dst, __ubuf__ T* src, uint32_t batchSize, uint32_t srcStrideDim0,28 void CopyUbToGmDim2(__gm__ T* dst, __ubuf__ T* src, uint32_t batchSize, uint32_t srcStrideDim0,
29- uint32_t srcStrideDim1, uint32_t ubDim1Stride, uint32_t magic0, uint32_t shift0, U dstStrideDim0, U dstStrideDim1)29+ uint32_t srcStrideDim1, uint32_t ubDim1Stride, uint32_t magic0, uint32_t shift0,
30+ U dstStrideDim0, U dstStrideDim1)
30{31{
31 for (uint32_t i = threadIdx.x; i < batchSize; i += blockDim.x) {32 for (uint32_t i = threadIdx.x; i < batchSize; i += blockDim.x) {
32 uint32_t idx = i;33 uint32_t idx = i;
@@ -49,22 +50,20 @@ __simt_vf__ LAUNCH_BOUND(THREAD_DIM_1024) __aicore__
49template <typename T>50template <typename T>
50class ViewCopySimtDim2 : public ViewCopyBase<T> {51class ViewCopySimtDim2 : public ViewCopyBase<T> {
51public:52public:
52- __aicore__ inline ViewCopySimtDim2(TPipe &pipe, const ViewCopyTilingData *tilingData) :53+ __aicore__ inline ViewCopySimtDim2(TPipe& pipe, const ViewCopyTilingData* tilingData)
53- pipe_(pipe), tilingData_(tilingData) {};54+ : pipe_(pipe), tilingData_(tilingData){};
54- __aicore__ inline void Init(55+ __aicore__ inline void Init(GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset, GM_ADDR src,
55- GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset,56+ GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset, GM_ADDR out);
56- GM_ADDR src, GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset,
57- GM_ADDR out);
58 __aicore__ inline void Process();57 __aicore__ inline void Process();
59 __aicore__ inline void InitCopyParams();58 __aicore__ inline void InitCopyParams();
60 59 
61private:60private:
62- __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM2> &dmaParam,61+ __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM2>& dmaParam,
63- int32_t batchSize);62+ int32_t batchSize);
64 63 
65private:64private:
66- TPipe &pipe_;65+ TPipe& pipe_;
67- const ViewCopyTilingData *tilingData_;66+ const ViewCopyTilingData* tilingData_;
68 TQue<QuePosition::VECIN, BUFFER_NUM> inQueue_;67 TQue<QuePosition::VECIN, BUFFER_NUM> inQueue_;
69 68 
70 GlobalTensor<T> dstGm_;69 GlobalTensor<T> dstGm_;
@@ -72,8 +71,8 @@ private:
72 71 
73 DataCopyExtParams copyParams_;72 DataCopyExtParams copyParams_;
74 DataCopyExtParams tailCopyParams_;73 DataCopyExtParams tailCopyParams_;
75- MultiCopyParams<T, DIM2> dmaParam_;74+ NdDmaParams<T, DIM2> dmaParam_;
76- MultiCopyParams<T, DIM2> tailDmaParam_;75+ NdDmaParams<T, DIM2> tailDmaParam_;
77 76 
78 int64_t blockOffset_ = 0;77 int64_t blockOffset_ = 0;
79 int32_t batchSize_ = 1;78 int32_t batchSize_ = 1;
@@ -84,16 +83,15 @@ private:
84};83};
85 84 
86template <typename T>85template <typename T>
87-__aicore__ inline void ViewCopySimtDim2<T>::Init(86+__aicore__ inline void ViewCopySimtDim2<T>::Init(GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride,
88- GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset,87+ GM_ADDR dstStorageOffset, GM_ADDR src, GM_ADDR srcSize,
89- GM_ADDR src, GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset,88+ GM_ADDR srcStride, GM_ADDR srcStorageOffset, GM_ADDR out)
90- GM_ADDR out)
91{89{
92 this->ParseTilingData(tilingData_);90 this->ParseTilingData(tilingData_);
93 91 
94 blockOffset_ = GetBlockIdx() * tilingData_->blockFactor;92 blockOffset_ = GetBlockIdx() * tilingData_->blockFactor;
95- srcGm_.SetGlobalBuffer((__gm__ T *)(src + tilingData_->srcStorageOffset * sizeof(T)));93+ srcGm_.SetGlobalBuffer((__gm__ T*)(src + tilingData_->srcStorageOffset * sizeof(T)));
96- dstGm_.SetGlobalBuffer((__gm__ T *)(dst + tilingData_->dstStorageOffset * sizeof(T)));94+ dstGm_.SetGlobalBuffer((__gm__ T*)(dst + tilingData_->dstStorageOffset * sizeof(T)));
97 95 
98 pipe_.InitBuffer(inQueue_, BUFFER_NUM, tilingData_->bufferSize);96 pipe_.InitBuffer(inQueue_, BUFFER_NUM, tilingData_->bufferSize);
99 97 
@@ -124,7 +122,7 @@ __aicore__ inline void ViewCopySimtDim2<T>::Process()
124 for (int64_t idx = 0; idx < loopSize; idx++) {122 for (int64_t idx = 0; idx < loopSize; idx++) {
125 globalLoopIdx = blockOffset_ + idx;123 globalLoopIdx = blockOffset_ + idx;
126 globalLoopIdxModUo = globalLoopIdx % tilingData_->uo;124 globalLoopIdxModUo = globalLoopIdx % tilingData_->uo;
127- if ((globalLoopIdxModUo) * tilingData_->ubFactor <= tilingData_->ubDimSize) {125+ if ((globalLoopIdxModUo)*tilingData_->ubFactor <= tilingData_->ubDimSize) {
128 if ((globalLoopIdxModUo + 1) * tilingData_->ubFactor > tilingData_->ubDimSize) {126 if ((globalLoopIdxModUo + 1) * tilingData_->ubFactor > tilingData_->ubDimSize) {
129 ProcessPerLoop(globalLoopIdx, tailDmaParam_, tailBatchSize_);127 ProcessPerLoop(globalLoopIdx, tailDmaParam_, tailBatchSize_);
130 } else {128 } else {
@@ -135,8 +133,8 @@ __aicore__ inline void ViewCopySimtDim2<T>::Process()
135}133}
136 134 
137template <typename T>135template <typename T>
138-__aicore__ inline void ViewCopySimtDim2<T>::ProcessPerLoop(int64_t globalLoopIdx,136+__aicore__ inline void ViewCopySimtDim2<T>::ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM2>& dmaParam,
139- const MultiCopyParams<T, DIM2> &dmaParam, int32_t batchSize)137+ int32_t batchSize)
140{138{
141 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,139 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,
142 tilingData_->blockFusedDimsNumber);140 tilingData_->blockFusedDimsNumber);
@@ -146,8 +144,8 @@ __aicore__ inline void ViewCopySimtDim2<T>::ProcessPerLoop(int64_t globalLoopIdx
146 copyParam.blockCount = dmaParam.loopInfo.loopSize[DIM1];144 copyParam.blockCount = dmaParam.loopInfo.loopSize[DIM1];
147 copyParam.blockLen = dmaParam.loopInfo.loopSize[DIM0] * sizeof(T);145 copyParam.blockLen = dmaParam.loopInfo.loopSize[DIM0] * sizeof(T);
148 copyParam.srcStride = (dmaParam.loopInfo.loopSrcStride[DIM1] - dmaParam.loopInfo.loopSize[DIM0]) * sizeof(T);146 copyParam.srcStride = (dmaParam.loopInfo.loopSrcStride[DIM1] - dmaParam.loopInfo.loopSize[DIM0]) * sizeof(T);
149- copyParam.dstStride = (dmaParam.loopInfo.loopDstStride[DIM1] -147+ copyParam.dstStride = (dmaParam.loopInfo.loopDstStride[DIM1] - dmaParam.loopInfo.loopSize[DIM0]) * sizeof(T) /
150- dmaParam.loopInfo.loopSize[DIM0]) * sizeof(T) / GetUbBlockSize();148+ GetUbBlockSize();
151 bool isCompact = dmaParam.loopInfo.loopDstStride[DIM1] == dmaParam.loopInfo.loopSize[DIM0];149 bool isCompact = dmaParam.loopInfo.loopDstStride[DIM1] == dmaParam.loopInfo.loopSize[DIM0];
152 CopyGmToUbCompact(srcLocal, srcGm_[srcOffset], copyParam, isCompact);150 CopyGmToUbCompact(srcLocal, srcGm_[srcOffset], copyParam, isCompact);
153 } else {151 } else {
@@ -164,15 +162,15 @@ __aicore__ inline void ViewCopySimtDim2<T>::ProcessPerLoop(int64_t globalLoopIdx
164 if (tilingData_->enableDstInt64 != 0) {162 if (tilingData_->enableDstInt64 != 0) {
165 uint64_t dstStrideDim0 = static_cast<uint64_t>(tilingData_->ubDstStride[DIM0_INDEX]);163 uint64_t dstStrideDim0 = static_cast<uint64_t>(tilingData_->ubDstStride[DIM0_INDEX]);
166 uint64_t dstStrideDim1 = static_cast<uint64_t>(tilingData_->ubDstStride[DIM1_INDEX]);164 uint64_t dstStrideDim1 = static_cast<uint64_t>(tilingData_->ubDstStride[DIM1_INDEX]);
167- asc_vf_call<CopyUbToGmDim2<T, uint64_t>>(dim3(THREAD_DIM_1024), dstAddr, srcAddr, batchSize,165+ asc_vf_call<CopyUbToGmDim2<T, uint64_t>>(dim3(THREAD_DIM_1024), dstAddr, srcAddr, batchSize, srcStrideDim0,
168- srcStrideDim0, srcStrideDim1, dim1UbdstStride_, m_[MS_IDX0], shift_[MS_IDX0], dstStrideDim0,166+ srcStrideDim1, dim1UbdstStride_, m_[MS_IDX0], shift_[MS_IDX0],
169- dstStrideDim1);167+ dstStrideDim0, dstStrideDim1);
170 } else {168 } else {
171 uint32_t dstStrideDim0 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM0_INDEX]);169 uint32_t dstStrideDim0 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM0_INDEX]);
172 uint32_t dstStrideDim1 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM1_INDEX]);170 uint32_t dstStrideDim1 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM1_INDEX]);
173- asc_vf_call<CopyUbToGmDim2<T, uint32_t>>(dim3(THREAD_DIM_1024), dstAddr, srcAddr, batchSize,171+ asc_vf_call<CopyUbToGmDim2<T, uint32_t>>(dim3(THREAD_DIM_1024), dstAddr, srcAddr, batchSize, srcStrideDim0,
174- srcStrideDim0, srcStrideDim1, dim1UbdstStride_, m_[MS_IDX0], shift_[MS_IDX0], dstStrideDim0,172+ srcStrideDim1, dim1UbdstStride_, m_[MS_IDX0], shift_[MS_IDX0],
175- dstStrideDim1);173+ dstStrideDim0, dstStrideDim1);
176 }174 }
177 175 
178 inQueue_.FreeTensor(dstLocal);176 inQueue_.FreeTensor(dstLocal);
@@ -181,35 +179,27 @@ __aicore__ inline void ViewCopySimtDim2<T>::ProcessPerLoop(int64_t globalLoopIdx
181template <typename T>179template <typename T>
182__aicore__ inline void ViewCopySimtDim2<T>::InitCopyParams()180__aicore__ inline void ViewCopySimtDim2<T>::InitCopyParams()
183{181{
184- dmaParam_ = {182+ dmaParam_ = {{{static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),
185- {183+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX])},
186- {static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),184+ {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),
187- static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX])},185+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX])},
188- {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),186+ {static_cast<uint32_t>(tilingData_->nddmaSize[DIM0_INDEX]),
189- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX]) },187+ static_cast<uint32_t>(tilingData_->nddmaSize[DIM1_INDEX])},
190- { static_cast<uint32_t>(tilingData_->nddmaSize[DIM0_INDEX]),188+ {0, 0},
191- static_cast<uint32_t>(tilingData_->nddmaSize[DIM1_INDEX])},189+ {0, 0}},
192- {0, 0},190+ 0};
193- {0, 0}
194- },
195- 0
196- };
197 191 
198- tailDmaParam_ = {192+ tailDmaParam_ = {{{static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),
199- {193+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX])},
200- { static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),194+ {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),
201- static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX])},195+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX])},
202- {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),196+ {static_cast<uint32_t>(this->tailNddmaSize_[DIM0_INDEX]),
203- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX])},197+ static_cast<uint32_t>(this->tailNddmaSize_[DIM1_INDEX])},
204- {static_cast<uint32_t>(this->tailNddmaSize_[DIM0_INDEX]),198+ {0, 0},
205- static_cast<uint32_t>(this->tailNddmaSize_[DIM1_INDEX])},199+ {0, 0}},
206- {0, 0},200+ 0};
207- {0, 0}
208- },
209- 0
210- };
211}201}
212 202 
213-} // namespace ViewCopy203+} // namespace ViewCopy
214 204 
215-#endif // VIEW_COPY_SIMT_DIM2_H_205+#endif // VIEW_COPY_SIMT_DIM2_H_
Mconversion/view_copy/op_kernel/arch35/view_copy_simt_dim3.h+57-67
@@ -24,9 +24,9 @@ using namespace AscendC;
24template <typename T, typename U>24template <typename T, typename U>
25__simt_vf__ LAUNCH_BOUND(THREAD_DIM) __aicore__25__simt_vf__ LAUNCH_BOUND(THREAD_DIM) __aicore__
26 void CopyUbToGmDim3(__gm__ T* dst, __ubuf__ T* src, uint32_t batchSize, uint32_t srcStrideDim0,26 void CopyUbToGmDim3(__gm__ T* dst, __ubuf__ T* src, uint32_t batchSize, uint32_t srcStrideDim0,
27- uint32_t srcStrideDim1, uint32_t srcStrideDim2, uint32_t ubDim2Stride,27+ uint32_t srcStrideDim1, uint32_t srcStrideDim2, uint32_t ubDim2Stride, uint32_t ubDim1Stride,
28- uint32_t ubDim1Stride, uint32_t magic0, uint32_t magic1, uint32_t shift0, uint32_t shift1,28+ uint32_t magic0, uint32_t magic1, uint32_t shift0, uint32_t shift1, U dstStrideDim0,
29- U dstStrideDim0, U dstStrideDim1, U dstStrideDim2)29+ U dstStrideDim1, U dstStrideDim2)
30{30{
31 for (uint32_t i = threadIdx.x; i < batchSize; i += blockDim.x) {31 for (uint32_t i = threadIdx.x; i < batchSize; i += blockDim.x) {
32 uint32_t idx = i;32 uint32_t idx = i;
@@ -46,7 +46,7 @@ __simt_vf__ LAUNCH_BOUND(THREAD_DIM) __aicore__
46 uint32_t dim0Index = newIdx2 - dim1Index * ubDim1Stride;46 uint32_t dim0Index = newIdx2 - dim1Index * ubDim1Stride;
47 srcIndex += (dim1Index * srcStrideDim1);47 srcIndex += (dim1Index * srcStrideDim1);
48 dstIndex += (dim1Index * dstStrideDim1);48 dstIndex += (dim1Index * dstStrideDim1);
49- 49+ 
50 srcIndex += (dim0Index * srcStrideDim0);50 srcIndex += (dim0Index * srcStrideDim0);
51 dstIndex += (dim0Index * dstStrideDim0);51 dstIndex += (dim0Index * dstStrideDim0);
52 dst[dstIndex] = src[srcIndex];52 dst[dstIndex] = src[srcIndex];
@@ -56,23 +56,21 @@ __simt_vf__ LAUNCH_BOUND(THREAD_DIM) __aicore__
56template <typename T>56template <typename T>
57class ViewCopySimtDim3 : public ViewCopyBase<T> {57class ViewCopySimtDim3 : public ViewCopyBase<T> {
58public:58public:
59- __aicore__ inline ViewCopySimtDim3(TPipe &pipe, const ViewCopyTilingData *tilingData) :59+ __aicore__ inline ViewCopySimtDim3(TPipe& pipe, const ViewCopyTilingData* tilingData)
60- pipe_(pipe), tilingData_(tilingData) {};60+ : pipe_(pipe), tilingData_(tilingData){};
61- __aicore__ inline void Init(61+ __aicore__ inline void Init(GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset, GM_ADDR src,
62- GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset,62+ GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset, GM_ADDR out);
63- GM_ADDR src, GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset,
64- GM_ADDR out);
65 __aicore__ inline void Process();63 __aicore__ inline void Process();
66 __aicore__ inline void InitCopyParams();64 __aicore__ inline void InitCopyParams();
67 65 
68private:66private:
69- __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM3> &dmaParam,67+ __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM3>& dmaParam,
70- int32_t batchSize);68+ int32_t batchSize);
71- __aicore__ inline void CopyIn(const GlobalTensor<T> &src, const MultiCopyParams<T, DIM3> &dmaParams);69+ __aicore__ inline void CopyIn(const GlobalTensor<T>& src, const NdDmaParams<T, DIM3>& dmaParams);
72 70 
73private:71private:
74- TPipe &pipe_;72+ TPipe& pipe_;
75- const ViewCopyTilingData *tilingData_;73+ const ViewCopyTilingData* tilingData_;
76 TQue<QuePosition::VECIN, BUFFER_NUM> inQueue_;74 TQue<QuePosition::VECIN, BUFFER_NUM> inQueue_;
77 75 
78 GlobalTensor<T> dstGm_;76 GlobalTensor<T> dstGm_;
@@ -80,8 +78,8 @@ private:
80 78 
81 DataCopyExtParams copyParams_;79 DataCopyExtParams copyParams_;
82 DataCopyExtParams tailCopyParams_;80 DataCopyExtParams tailCopyParams_;
83- MultiCopyParams<T, DIM3> dmaParam_;81+ NdDmaParams<T, DIM3> dmaParam_;
84- MultiCopyParams<T, DIM3> tailDmaParam_;82+ NdDmaParams<T, DIM3> tailDmaParam_;
85 83 
86 int64_t blockOffset_ = 0;84 int64_t blockOffset_ = 0;
87 uint32_t batchSize_ = 1;85 uint32_t batchSize_ = 1;
@@ -93,15 +91,14 @@ private:
93};91};
94 92 
95template <typename T>93template <typename T>
96-__aicore__ inline void ViewCopySimtDim3<T>::Init(94+__aicore__ inline void ViewCopySimtDim3<T>::Init(GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride,
97- GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset,95+ GM_ADDR dstStorageOffset, GM_ADDR src, GM_ADDR srcSize,
98- GM_ADDR src, GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset,96+ GM_ADDR srcStride, GM_ADDR srcStorageOffset, GM_ADDR out)
99- GM_ADDR out)
100{97{
101 this->ParseTilingData(tilingData_);98 this->ParseTilingData(tilingData_);
102 blockOffset_ = GetBlockIdx() * tilingData_->blockFactor;99 blockOffset_ = GetBlockIdx() * tilingData_->blockFactor;
103- dstGm_.SetGlobalBuffer((__gm__ T *)(dst + tilingData_->dstStorageOffset * sizeof(T)));100+ dstGm_.SetGlobalBuffer((__gm__ T*)(dst + tilingData_->dstStorageOffset * sizeof(T)));
104- srcGm_.SetGlobalBuffer((__gm__ T *)(src + tilingData_->srcStorageOffset * sizeof(T)));101+ srcGm_.SetGlobalBuffer((__gm__ T*)(src + tilingData_->srcStorageOffset * sizeof(T)));
105 102 
106 pipe_.InitBuffer(inQueue_, BUFFER_NUM, tilingData_->bufferSize);103 pipe_.InitBuffer(inQueue_, BUFFER_NUM, tilingData_->bufferSize);
107 batchSize_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *104 batchSize_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *
@@ -132,19 +129,18 @@ __aicore__ inline void ViewCopySimtDim3<T>::Process()
132 for (int64_t idx = 0; idx < loopSize; idx++) {129 for (int64_t idx = 0; idx < loopSize; idx++) {
133 globalLoopIdx = blockOffset_ + idx;130 globalLoopIdx = blockOffset_ + idx;
134 globalLoopIdxModUo = globalLoopIdx % tilingData_->uo;131 globalLoopIdxModUo = globalLoopIdx % tilingData_->uo;
135- if ((globalLoopIdxModUo) * tilingData_->ubFactor <= tilingData_->ubDimSize) {132+ if ((globalLoopIdxModUo)*tilingData_->ubFactor <= tilingData_->ubDimSize) {
136 if ((globalLoopIdxModUo + 1) * tilingData_->ubFactor > tilingData_->ubDimSize) {133 if ((globalLoopIdxModUo + 1) * tilingData_->ubFactor > tilingData_->ubDimSize) {
137 ProcessPerLoop(globalLoopIdx, tailDmaParam_, tailBatchSize_);134 ProcessPerLoop(globalLoopIdx, tailDmaParam_, tailBatchSize_);
138 } else {135 } else {
139 ProcessPerLoop(globalLoopIdx, dmaParam_, batchSize_);136 ProcessPerLoop(globalLoopIdx, dmaParam_, batchSize_);
140 }137 }
141- }138+ }
142 }139 }
143}140}
144 141 
145template <typename T>142template <typename T>
146-__aicore__ inline void ViewCopySimtDim3<T>::CopyIn(const GlobalTensor<T> &src,143+__aicore__ inline void ViewCopySimtDim3<T>::CopyIn(const GlobalTensor<T>& src, const NdDmaParams<T, DIM3>& dmaParams)
147- const MultiCopyParams<T, DIM3> &dmaParams)
148{144{
149 LocalTensor<T> srcLocal = inQueue_.AllocTensor<T>();145 LocalTensor<T> srcLocal = inQueue_.AllocTensor<T>();
150 this->CopyInDim3(src, srcLocal, dmaParams, tilingData_->enableMovAlign != 0);146 this->CopyInDim3(src, srcLocal, dmaParams, tilingData_->enableMovAlign != 0);
@@ -152,8 +148,8 @@ __aicore__ inline void ViewCopySimtDim3<T>::CopyIn(const GlobalTensor<T> &src,
152}148}
153 149 
154template <typename T>150template <typename T>
155-__aicore__ inline void ViewCopySimtDim3<T>::ProcessPerLoop(int64_t globalLoopIdx,151+__aicore__ inline void ViewCopySimtDim3<T>::ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM3>& dmaParam,
156- const MultiCopyParams<T, DIM3> &dmaParam, int32_t batchSize)152+ int32_t batchSize)
157{153{
158 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,154 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,
159 tilingData_->blockFusedDimsNumber);155 tilingData_->blockFusedDimsNumber);
@@ -171,15 +167,17 @@ __aicore__ inline void ViewCopySimtDim3<T>::ProcessPerLoop(int64_t globalLoopIdx
171 uint64_t dstStrideDim1 = static_cast<uint64_t>(tilingData_->ubDstStride[DIM1_INDEX]);167 uint64_t dstStrideDim1 = static_cast<uint64_t>(tilingData_->ubDstStride[DIM1_INDEX]);
172 uint64_t dstStrideDim2 = static_cast<uint64_t>(tilingData_->ubDstStride[DIM2_INDEX]);168 uint64_t dstStrideDim2 = static_cast<uint64_t>(tilingData_->ubDstStride[DIM2_INDEX]);
173 asc_vf_call<CopyUbToGmDim3<T, uint64_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize, srcStrideDim0,169 asc_vf_call<CopyUbToGmDim3<T, uint64_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize, srcStrideDim0,
174- srcStrideDim1, srcStrideDim2, dim2UbdstStride_, dim1UbdstStride_, m_[MS_IDX0], m_[MS_IDX1],170+ srcStrideDim1, srcStrideDim2, dim2UbdstStride_, dim1UbdstStride_,
175- shift_[MS_IDX0], shift_[MS_IDX1], dstStrideDim0, dstStrideDim1, dstStrideDim2);171+ m_[MS_IDX0], m_[MS_IDX1], shift_[MS_IDX0], shift_[MS_IDX1],
172+ dstStrideDim0, dstStrideDim1, dstStrideDim2);
176 } else {173 } else {
177 uint32_t dstStrideDim0 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM0_INDEX]);174 uint32_t dstStrideDim0 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM0_INDEX]);
178 uint32_t dstStrideDim1 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM1_INDEX]);175 uint32_t dstStrideDim1 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM1_INDEX]);
179 uint32_t dstStrideDim2 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM2_INDEX]);176 uint32_t dstStrideDim2 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM2_INDEX]);
180 asc_vf_call<CopyUbToGmDim3<T, uint32_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize, srcStrideDim0,177 asc_vf_call<CopyUbToGmDim3<T, uint32_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize, srcStrideDim0,
181- srcStrideDim1, srcStrideDim2, dim2UbdstStride_, dim1UbdstStride_, m_[MS_IDX0], m_[MS_IDX1],178+ srcStrideDim1, srcStrideDim2, dim2UbdstStride_, dim1UbdstStride_,
182- shift_[MS_IDX0], shift_[MS_IDX1], dstStrideDim0, dstStrideDim1, dstStrideDim2);179+ m_[MS_IDX0], m_[MS_IDX1], shift_[MS_IDX0], shift_[MS_IDX1],
180+ dstStrideDim0, dstStrideDim1, dstStrideDim2);
183 }181 }
184 inQueue_.FreeTensor(dstLocal);182 inQueue_.FreeTensor(dstLocal);
185}183}
@@ -187,41 +185,33 @@ __aicore__ inline void ViewCopySimtDim3<T>::ProcessPerLoop(int64_t globalLoopIdx
187template <typename T>185template <typename T>
188__aicore__ inline void ViewCopySimtDim3<T>::InitCopyParams()186__aicore__ inline void ViewCopySimtDim3<T>::InitCopyParams()
189{187{
190- dmaParam_ = {188+ dmaParam_ = {{{static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),
191- {189+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX]),
192- { static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),190+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM2_INDEX])},
193- static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX]),191+ {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),
194- static_cast<uint64_t>(tilingData_->nddmaStride[DIM2_INDEX]) },192+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX]),
195- { static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),193+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM2_INDEX])},
196- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX]),194+ {static_cast<uint32_t>(tilingData_->nddmaSize[DIM0_INDEX]),
197- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM2_INDEX]) },195+ static_cast<uint32_t>(tilingData_->nddmaSize[DIM1_INDEX]),
198- { static_cast<uint32_t>(tilingData_->nddmaSize[DIM0_INDEX]),196+ static_cast<uint32_t>(tilingData_->nddmaSize[DIM2_INDEX])},
199- static_cast<uint32_t>(tilingData_->nddmaSize[DIM1_INDEX]),197+ {0, 0, 0},
200- static_cast<uint32_t>(tilingData_->nddmaSize[DIM2_INDEX]) },198+ {0, 0, 0}},
201- {0, 0, 0},199+ 0};
202- {0, 0, 0}
203- },
204- 0
205- };
206 200 
207- tailDmaParam_ = {201+ tailDmaParam_ = {{{static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),
208- {202+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX]),
209- { static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),203+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM2_INDEX])},
210- static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX]),204+ {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),
211- static_cast<uint64_t>(tilingData_->nddmaStride[DIM2_INDEX]) },205+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX]),
212- { static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),206+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM2_INDEX])},
213- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX]),207+ {static_cast<uint32_t>(this->tailNddmaSize_[DIM0_INDEX]),
214- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM2_INDEX]) },208+ static_cast<uint32_t>(this->tailNddmaSize_[DIM1_INDEX]),
215- { static_cast<uint32_t>(this->tailNddmaSize_[DIM0_INDEX]),209+ static_cast<uint32_t>(this->tailNddmaSize_[DIM2_INDEX])},
216- static_cast<uint32_t>(this->tailNddmaSize_[DIM1_INDEX]),210+ {0, 0, 0},
217- static_cast<uint32_t>(this->tailNddmaSize_[DIM2_INDEX]) },211+ {0, 0, 0}},
218- {0, 0, 0},212+ 0};
219- {0, 0, 0}
220- },
221- 0
222- };
223}213}
224 214 
225-} // namespace ViewCopy215+} // namespace ViewCopy
226 216 
227-#endif // VIEW_COPY_SIMT_DIM3_H_217+#endif // VIEW_COPY_SIMT_DIM3_H_
Mconversion/view_copy/op_kernel/arch35/view_copy_simt_dim4.h+71-80
@@ -23,11 +23,11 @@ using namespace AscendC;
23 23 
24template <typename T, typename U>24template <typename T, typename U>
25__simt_vf__ LAUNCH_BOUND(THREAD_DIM) __aicore__25__simt_vf__ LAUNCH_BOUND(THREAD_DIM) __aicore__
26- void CopyUbToGmDim4(__gm__ T* dst, __ubuf__ T* src, uint32_t batchSize,26+ void CopyUbToGmDim4(__gm__ T* dst, __ubuf__ T* src, uint32_t batchSize, uint32_t srcStrideDim0,
27- uint32_t srcStrideDim0, uint32_t srcStrideDim1, uint32_t srcStrideDim2, uint32_t srcStrideDim3,27+ uint32_t srcStrideDim1, uint32_t srcStrideDim2, uint32_t srcStrideDim3, uint32_t ubDim3Stride,
28- uint32_t ubDim3Stride, uint32_t ubDim2Stride, uint32_t ubDim1Stride, uint32_t magic0, uint32_t magic1,28+ uint32_t ubDim2Stride, uint32_t ubDim1Stride, uint32_t magic0, uint32_t magic1, uint32_t magic2,
29- uint32_t magic2, uint32_t shift0, uint32_t shift1, uint32_t shift2, U dstStrideDim0, U dstStrideDim1,29+ uint32_t shift0, uint32_t shift1, uint32_t shift2, U dstStrideDim0, U dstStrideDim1,
30- U dstStrideDim2, U dstStrideDim3)30+ U dstStrideDim2, U dstStrideDim3)
31{31{
32 for (uint32_t i = threadIdx.x; i < batchSize; i += blockDim.x) {32 for (uint32_t i = threadIdx.x; i < batchSize; i += blockDim.x) {
33 uint32_t idx = i;33 uint32_t idx = i;
@@ -54,7 +54,7 @@ __simt_vf__ LAUNCH_BOUND(THREAD_DIM) __aicore__
54 uint32_t dim0Index = newIdx2 - dim1Index * ubDim1Stride;54 uint32_t dim0Index = newIdx2 - dim1Index * ubDim1Stride;
55 srcIndex += (dim1Index * srcStrideDim1);55 srcIndex += (dim1Index * srcStrideDim1);
56 dstIndex += (dim1Index * dstStrideDim1);56 dstIndex += (dim1Index * dstStrideDim1);
57- 57+ 
58 srcIndex += (dim0Index * srcStrideDim0);58 srcIndex += (dim0Index * srcStrideDim0);
59 dstIndex += (dim0Index * dstStrideDim0);59 dstIndex += (dim0Index * dstStrideDim0);
60 dst[dstIndex] = src[srcIndex];60 dst[dstIndex] = src[srcIndex];
@@ -64,29 +64,28 @@ __simt_vf__ LAUNCH_BOUND(THREAD_DIM) __aicore__
64template <typename T>64template <typename T>
65class ViewCopySimtDim4 : public ViewCopyBase<T> {65class ViewCopySimtDim4 : public ViewCopyBase<T> {
66public:66public:
67- __aicore__ inline ViewCopySimtDim4(TPipe &pipe, const ViewCopyTilingData *tilingData) :67+ __aicore__ inline ViewCopySimtDim4(TPipe& pipe, const ViewCopyTilingData* tilingData)
68- pipe_(pipe), tilingData_(tilingData) {};68+ : pipe_(pipe), tilingData_(tilingData){};
69- __aicore__ inline void Init(69+ __aicore__ inline void Init(GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset, GM_ADDR src,
70- GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset,70+ GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset, GM_ADDR out);
71- GM_ADDR src, GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset,
72- GM_ADDR out);
73 __aicore__ inline void Process();71 __aicore__ inline void Process();
74 __aicore__ inline void InitCopyParams();72 __aicore__ inline void InitCopyParams();
75 73 
76private:74private:
77- __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM4> &dmaParam,75+ __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM4>& dmaParam,
78- int32_t batchSize);76+ int32_t batchSize);
79- __aicore__ inline void CopyIn(const GlobalTensor<T> &src, const MultiCopyParams<T, DIM4> &dmaParams);77+ __aicore__ inline void CopyIn(const GlobalTensor<T>& src, const NdDmaParams<T, DIM4>& dmaParams);
78+ 
80private:79private:
81- TPipe &pipe_;80+ TPipe& pipe_;
82- const ViewCopyTilingData *tilingData_;81+ const ViewCopyTilingData* tilingData_;
83 TQue<QuePosition::VECIN, BUFFER_NUM> inQueue_;82 TQue<QuePosition::VECIN, BUFFER_NUM> inQueue_;
84 83 
85 GlobalTensor<T> dstGm_;84 GlobalTensor<T> dstGm_;
86 GlobalTensor<T> srcGm_;85 GlobalTensor<T> srcGm_;
87 86 
88- MultiCopyParams<T, DIM4> tailDmaParam_;87+ NdDmaParams<T, DIM4> tailDmaParam_;
89- MultiCopyParams<T, DIM4> dmaParam_;88+ NdDmaParams<T, DIM4> dmaParam_;
90 DataCopyExtParams tailCopyParams_;89 DataCopyExtParams tailCopyParams_;
91 DataCopyExtParams copyParams_;90 DataCopyExtParams copyParams_;
92 91 
@@ -101,14 +100,13 @@ private:
101};100};
102 101 
103template <typename T>102template <typename T>
104-__aicore__ inline void ViewCopySimtDim4<T>::Init(103+__aicore__ inline void ViewCopySimtDim4<T>::Init(GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride,
105- GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset,104+ GM_ADDR dstStorageOffset, GM_ADDR src, GM_ADDR srcSize,
106- GM_ADDR src, GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset,105+ GM_ADDR srcStride, GM_ADDR srcStorageOffset, GM_ADDR out)
107- GM_ADDR out)
108{106{
109 this->ParseTilingData(tilingData_);107 this->ParseTilingData(tilingData_);
110- srcGm_.SetGlobalBuffer((__gm__ T *)(src + tilingData_->srcStorageOffset * sizeof(T)));108+ srcGm_.SetGlobalBuffer((__gm__ T*)(src + tilingData_->srcStorageOffset * sizeof(T)));
111- dstGm_.SetGlobalBuffer((__gm__ T *)(dst + tilingData_->dstStorageOffset * sizeof(T)));109+ dstGm_.SetGlobalBuffer((__gm__ T*)(dst + tilingData_->dstStorageOffset * sizeof(T)));
112 110 
113 pipe_.InitBuffer(inQueue_, BUFFER_NUM, tilingData_->bufferSize);111 pipe_.InitBuffer(inQueue_, BUFFER_NUM, tilingData_->bufferSize);
114 blockOffset_ = GetBlockIdx() * tilingData_->blockFactor;112 blockOffset_ = GetBlockIdx() * tilingData_->blockFactor;
@@ -118,13 +116,13 @@ __aicore__ inline void ViewCopySimtDim4<T>::Init(
118 }116 }
119 117 
120 dim3UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *118 dim3UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *
121- tilingData_->ubDstSize[DIM2_INDEX];119+ tilingData_->ubDstSize[DIM2_INDEX];
122 dim2UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX];120 dim2UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX];
123 dim1UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX];121 dim1UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX];
124- 122+ 
125 GetUintDivMagicAndShift<uint32_t>(m_[0], shift_[0], dim3UbdstStride_);123 GetUintDivMagicAndShift<uint32_t>(m_[0], shift_[0], dim3UbdstStride_);
126- GetUintDivMagicAndShift<uint32_t>(m_[1], shift_[1], dim2UbdstStride_); 124+ GetUintDivMagicAndShift<uint32_t>(m_[1], shift_[1], dim2UbdstStride_);
127- GetUintDivMagicAndShift<uint32_t>(m_[2], shift_[2], dim1UbdstStride_); 125+ GetUintDivMagicAndShift<uint32_t>(m_[2], shift_[2], dim1UbdstStride_);
128}126}
129 127 
130template <typename T>128template <typename T>
@@ -145,7 +143,7 @@ __aicore__ inline void ViewCopySimtDim4<T>::Process()
145 for (int64_t idx = 0; idx < loopSize; idx++) {143 for (int64_t idx = 0; idx < loopSize; idx++) {
146 globalLoopIdx = blockOffset_ + idx;144 globalLoopIdx = blockOffset_ + idx;
147 globalLoopIdxModUo = globalLoopIdx % tilingData_->uo;145 globalLoopIdxModUo = globalLoopIdx % tilingData_->uo;
148- if ((globalLoopIdxModUo) * tilingData_->ubFactor <= tilingData_->ubDimSize) {146+ if ((globalLoopIdxModUo)*tilingData_->ubFactor <= tilingData_->ubDimSize) {
149 if ((globalLoopIdxModUo + 1) * tilingData_->ubFactor > tilingData_->ubDimSize) {147 if ((globalLoopIdxModUo + 1) * tilingData_->ubFactor > tilingData_->ubDimSize) {
150 ProcessPerLoop(globalLoopIdx, tailDmaParam_, tailBatchSize_);148 ProcessPerLoop(globalLoopIdx, tailDmaParam_, tailBatchSize_);
151 } else {149 } else {
@@ -156,8 +154,7 @@ __aicore__ inline void ViewCopySimtDim4<T>::Process()
156}154}
157 155 
158template <typename T>156template <typename T>
159-__aicore__ inline void ViewCopySimtDim4<T>::CopyIn(const GlobalTensor<T> &src,157+__aicore__ inline void ViewCopySimtDim4<T>::CopyIn(const GlobalTensor<T>& src, const NdDmaParams<T, DIM4>& dmaParams)
160- const MultiCopyParams<T, DIM4> &dmaParams)
161{158{
162 LocalTensor<T> srcLocal = inQueue_.AllocTensor<T>();159 LocalTensor<T> srcLocal = inQueue_.AllocTensor<T>();
163 this->CopyInDim4(src, srcLocal, dmaParams, tilingData_->enableMovAlign != 0);160 this->CopyInDim4(src, srcLocal, dmaParams, tilingData_->enableMovAlign != 0);
@@ -165,8 +162,8 @@ __aicore__ inline void ViewCopySimtDim4<T>::CopyIn(const GlobalTensor<T> &src,
165}162}
166 163 
167template <typename T>164template <typename T>
168-__aicore__ inline void ViewCopySimtDim4<T>::ProcessPerLoop(int64_t globalLoopIdx,165+__aicore__ inline void ViewCopySimtDim4<T>::ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM4>& dmaParam,
169- const MultiCopyParams<T, DIM4> &dmaParam, int32_t batchSize)166+ int32_t batchSize)
170{167{
171 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,168 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,
172 tilingData_->blockFusedDimsNumber);169 tilingData_->blockFusedDimsNumber);
@@ -186,18 +183,20 @@ __aicore__ inline void ViewCopySimtDim4<T>::ProcessPerLoop(int64_t globalLoopIdx
186 uint64_t dstStrideDim2 = static_cast<uint64_t>(tilingData_->ubDstStride[DIM2_INDEX]);183 uint64_t dstStrideDim2 = static_cast<uint64_t>(tilingData_->ubDstStride[DIM2_INDEX]);
187 uint64_t dstStrideDim3 = static_cast<uint64_t>(tilingData_->ubDstStride[DIM3_INDEX]);184 uint64_t dstStrideDim3 = static_cast<uint64_t>(tilingData_->ubDstStride[DIM3_INDEX]);
188 asc_vf_call<CopyUbToGmDim4<T, uint64_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize, srcStrideDim0,185 asc_vf_call<CopyUbToGmDim4<T, uint64_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize, srcStrideDim0,
189- srcStrideDim1, srcStrideDim2, srcStrideDim3, dim3UbdstStride_, dim2UbdstStride_, dim1UbdstStride_,186+ srcStrideDim1, srcStrideDim2, srcStrideDim3, dim3UbdstStride_,
190- m_[MS_IDX0], m_[MS_IDX1], m_[MS_IDX2], shift_[MS_IDX0], shift_[MS_IDX1], shift_[MS_IDX2], dstStrideDim0,187+ dim2UbdstStride_, dim1UbdstStride_, m_[MS_IDX0], m_[MS_IDX1],
191- dstStrideDim1, dstStrideDim2, dstStrideDim3);188+ m_[MS_IDX2], shift_[MS_IDX0], shift_[MS_IDX1], shift_[MS_IDX2],
189+ dstStrideDim0, dstStrideDim1, dstStrideDim2, dstStrideDim3);
192 } else {190 } else {
193 uint32_t dstStrideDim0 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM0_INDEX]);191 uint32_t dstStrideDim0 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM0_INDEX]);
194 uint32_t dstStrideDim1 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM1_INDEX]);192 uint32_t dstStrideDim1 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM1_INDEX]);
195 uint32_t dstStrideDim2 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM2_INDEX]);193 uint32_t dstStrideDim2 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM2_INDEX]);
196 uint32_t dstStrideDim3 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM3_INDEX]);194 uint32_t dstStrideDim3 = static_cast<uint32_t>(tilingData_->ubDstStride[DIM3_INDEX]);
197 asc_vf_call<CopyUbToGmDim4<T, uint32_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize, srcStrideDim0,195 asc_vf_call<CopyUbToGmDim4<T, uint32_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize, srcStrideDim0,
198- srcStrideDim1, srcStrideDim2, srcStrideDim3, dim3UbdstStride_, dim2UbdstStride_, dim1UbdstStride_,196+ srcStrideDim1, srcStrideDim2, srcStrideDim3, dim3UbdstStride_,
199- m_[MS_IDX0], m_[MS_IDX1], m_[MS_IDX2], shift_[MS_IDX0], shift_[MS_IDX1], shift_[MS_IDX2],197+ dim2UbdstStride_, dim1UbdstStride_, m_[MS_IDX0], m_[MS_IDX1],
200- dstStrideDim0, dstStrideDim1, dstStrideDim2, dstStrideDim3);198+ m_[MS_IDX2], shift_[MS_IDX0], shift_[MS_IDX1], shift_[MS_IDX2],
199+ dstStrideDim0, dstStrideDim1, dstStrideDim2, dstStrideDim3);
201 }200 }
202 inQueue_.FreeTensor(dstLocal);201 inQueue_.FreeTensor(dstLocal);
203}202}
@@ -205,47 +204,39 @@ __aicore__ inline void ViewCopySimtDim4<T>::ProcessPerLoop(int64_t globalLoopIdx
205template <typename T>204template <typename T>
206__aicore__ inline void ViewCopySimtDim4<T>::InitCopyParams()205__aicore__ inline void ViewCopySimtDim4<T>::InitCopyParams()
207{206{
208- dmaParam_ = {207+ dmaParam_ = {{{static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),
209- {208+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX]),
210- { static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),209+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM2_INDEX]),
211- static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX]),210+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM3_INDEX])},
212- static_cast<uint64_t>(tilingData_->nddmaStride[DIM2_INDEX]),211+ {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),
213- static_cast<uint64_t>(tilingData_->nddmaStride[DIM3_INDEX]) },212+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX]),
214- { static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),213+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM2_INDEX]),
215- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX]),214+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM3_INDEX])},
216- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM2_INDEX]),215+ {static_cast<uint32_t>(tilingData_->nddmaSize[DIM0_INDEX]),
217- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM3_INDEX]) },216+ static_cast<uint32_t>(tilingData_->nddmaSize[DIM1_INDEX]),
218- {static_cast<uint32_t>(tilingData_->nddmaSize[DIM0_INDEX]),217+ static_cast<uint32_t>(tilingData_->nddmaSize[DIM2_INDEX]),
219- static_cast<uint32_t>(tilingData_->nddmaSize[DIM1_INDEX]),218+ static_cast<uint32_t>(tilingData_->nddmaSize[DIM3_INDEX])},
220- static_cast<uint32_t>(tilingData_->nddmaSize[DIM2_INDEX]),219+ {0, 0, 0, 0},
221- static_cast<uint32_t>(tilingData_->nddmaSize[DIM3_INDEX]) },220+ {0, 0, 0, 0}},
222- {0, 0, 0, 0},221+ 0};
223- {0, 0, 0, 0}
224- },
225- 0
226- };
227 222 
228- tailDmaParam_ = {223+ tailDmaParam_ = {{{static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),
229- {224+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX]),
230- {static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),225+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM2_INDEX]),
231- static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX]),226+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM3_INDEX])},
232- static_cast<uint64_t>(tilingData_->nddmaStride[DIM2_INDEX]),227+ {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),
233- static_cast<uint64_t>(tilingData_->nddmaStride[DIM3_INDEX])},228+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX]),
234- {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),229+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM2_INDEX]),
235- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX]),230+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM3_INDEX])},
236- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM2_INDEX]),231+ {static_cast<uint32_t>(this->tailNddmaSize_[DIM0_INDEX]),
237- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM3_INDEX])},232+ static_cast<uint32_t>(this->tailNddmaSize_[DIM1_INDEX]),
238- {static_cast<uint32_t>(this->tailNddmaSize_[DIM0_INDEX]),233+ static_cast<uint32_t>(this->tailNddmaSize_[DIM2_INDEX]),
239- static_cast<uint32_t>(this->tailNddmaSize_[DIM1_INDEX]),234+ static_cast<uint32_t>(this->tailNddmaSize_[DIM3_INDEX])},
240- static_cast<uint32_t>(this->tailNddmaSize_[DIM2_INDEX]),235+ {0, 0, 0, 0},
241- static_cast<uint32_t>(this->tailNddmaSize_[DIM3_INDEX])},236+ {0, 0, 0, 0}},
242- {0, 0, 0, 0},237+ 0};
243- {0, 0, 0, 0}
244- },
245- 0
246- };
247}238}
248 239 
249-} // namespace ViewCopy240+} // namespace ViewCopy
250 241 
251-#endif // VIEW_COPY_SIMT_DIM4_H_242+#endif // VIEW_COPY_SIMT_DIM4_H_
Mconversion/view_copy/op_kernel/arch35/view_copy_simt_dim5.h+81-91
@@ -23,10 +23,10 @@ using namespace AscendC;
23 23 
24template <typename T, typename U>24template <typename T, typename U>
25__simt_vf__ LAUNCH_BOUND(THREAD_DIM) __aicore__25__simt_vf__ LAUNCH_BOUND(THREAD_DIM) __aicore__
26- void CopyUbToGmDim5(__gm__ T* dst, __ubuf__ T* src, uint32_t batchSize,26+ void CopyUbToGmDim5(__gm__ T* dst, __ubuf__ T* src, uint32_t batchSize, uint32_t ubDim4Stride,
27- uint32_t ubDim4Stride, uint32_t ubDim3Stride, uint32_t ubDim2Stride, uint32_t ubDim1Stride,27+ uint32_t ubDim3Stride, uint32_t ubDim2Stride, uint32_t ubDim1Stride,
28- __ubuf__ uint32_t* ParamDim4MagicUb, __ubuf__ uint32_t* ParamDim4ShiftUb,28+ __ubuf__ uint32_t* ParamDim4MagicUb, __ubuf__ uint32_t* ParamDim4ShiftUb,
29- __ubuf__ uint32_t* ParamDim5SrcStrideUb, __ubuf__ U* ParamDim5DstStrideUb)29+ __ubuf__ uint32_t* ParamDim5SrcStrideUb, __ubuf__ U* ParamDim5DstStrideUb)
30{30{
31 for (uint32_t i = threadIdx.x; i < batchSize; i += blockDim.x) {31 for (uint32_t i = threadIdx.x; i < batchSize; i += blockDim.x) {
32 uint32_t idx = i;32 uint32_t idx = i;
@@ -60,7 +60,7 @@ __simt_vf__ LAUNCH_BOUND(THREAD_DIM) __aicore__
60 uint32_t dim0Index = newIdx2 - dim1Index * ubDim1Stride;60 uint32_t dim0Index = newIdx2 - dim1Index * ubDim1Stride;
61 srcIndex += (dim1Index * ParamDim5SrcStrideUb[DIM1]);61 srcIndex += (dim1Index * ParamDim5SrcStrideUb[DIM1]);
62 dstIndex += (dim1Index * ParamDim5DstStrideUb[DIM1]);62 dstIndex += (dim1Index * ParamDim5DstStrideUb[DIM1]);
63- 63+ 
64 srcIndex += (dim0Index * ParamDim5SrcStrideUb[DIM0]);64 srcIndex += (dim0Index * ParamDim5SrcStrideUb[DIM0]);
65 dstIndex += (dim0Index * ParamDim5DstStrideUb[DIM0]);65 dstIndex += (dim0Index * ParamDim5DstStrideUb[DIM0]);
66 dst[dstIndex] = src[srcIndex];66 dst[dstIndex] = src[srcIndex];
@@ -70,22 +70,21 @@ __simt_vf__ LAUNCH_BOUND(THREAD_DIM) __aicore__
70template <typename T>70template <typename T>
71class ViewCopySimtDim5 : public ViewCopyBase<T> {71class ViewCopySimtDim5 : public ViewCopyBase<T> {
72public:72public:
73- __aicore__ inline ViewCopySimtDim5(TPipe &pipe, const ViewCopyTilingData *tilingData) :73+ __aicore__ inline ViewCopySimtDim5(TPipe& pipe, const ViewCopyTilingData* tilingData)
74- pipe_(pipe), tilingData_(tilingData) {};74+ : pipe_(pipe), tilingData_(tilingData){};
75- __aicore__ inline void Init(75+ __aicore__ inline void Init(GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset, GM_ADDR src,
76- GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset,76+ GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset, GM_ADDR out);
77- GM_ADDR src, GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset,
78- GM_ADDR out);
79 __aicore__ inline void Process();77 __aicore__ inline void Process();
80 __aicore__ inline void InitCopyParams();78 __aicore__ inline void InitCopyParams();
81 79 
82private:80private:
83- __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM5> &dmaParam,81+ __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM5>& dmaParam,
84- int32_t batchSize);82+ int32_t batchSize);
85- __aicore__ inline void CopyIn(const GlobalTensor<T> &src, const MultiCopyParams<T, DIM5> &dmaParams);83+ __aicore__ inline void CopyIn(const GlobalTensor<T>& src, const NdDmaParams<T, DIM5>& dmaParams);
84+ 
86private:85private:
87- TPipe &pipe_;86+ TPipe& pipe_;
88- const ViewCopyTilingData *tilingData_;87+ const ViewCopyTilingData* tilingData_;
89 TQue<QuePosition::VECIN, BUFFER_NUM> inQueue_;88 TQue<QuePosition::VECIN, BUFFER_NUM> inQueue_;
90 89 
91 GlobalTensor<T> srcGm_;90 GlobalTensor<T> srcGm_;
@@ -95,8 +94,8 @@ private:
95 TBuf<TPosition::VECCALC> Dim5paramSrcStrideBuf_;94 TBuf<TPosition::VECCALC> Dim5paramSrcStrideBuf_;
96 TBuf<TPosition::VECCALC> Dim5paramDstStrideBuf_;95 TBuf<TPosition::VECCALC> Dim5paramDstStrideBuf_;
97 96 
98- MultiCopyParams<T, DIM5> tailDmaParam_;97+ NdDmaParams<T, DIM5> tailDmaParam_;
99- MultiCopyParams<T, DIM5> dmaParam_;98+ NdDmaParams<T, DIM5> dmaParam_;
100 DataCopyExtParams tailCopyParams_;99 DataCopyExtParams tailCopyParams_;
101 DataCopyExtParams copyParams_;100 DataCopyExtParams copyParams_;
102 101 
@@ -112,14 +111,13 @@ private:
112};111};
113 112 
114template <typename T>113template <typename T>
115-__aicore__ inline void ViewCopySimtDim5<T>::Init(114+__aicore__ inline void ViewCopySimtDim5<T>::Init(GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride,
116- GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset,115+ GM_ADDR dstStorageOffset, GM_ADDR src, GM_ADDR srcSize,
117- GM_ADDR src, GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset,116+ GM_ADDR srcStride, GM_ADDR srcStorageOffset, GM_ADDR out)
118- GM_ADDR out)
119{117{
120 this->ParseTilingData(tilingData_);118 this->ParseTilingData(tilingData_);
121- dstGm_.SetGlobalBuffer((__gm__ T *)(dst + tilingData_->dstStorageOffset * sizeof(T)));119+ dstGm_.SetGlobalBuffer((__gm__ T*)(dst + tilingData_->dstStorageOffset * sizeof(T)));
122- srcGm_.SetGlobalBuffer((__gm__ T *)(src + tilingData_->srcStorageOffset * sizeof(T)));120+ srcGm_.SetGlobalBuffer((__gm__ T*)(src + tilingData_->srcStorageOffset * sizeof(T)));
123 121 
124 blockOffset_ = GetBlockIdx() * tilingData_->blockFactor;122 blockOffset_ = GetBlockIdx() * tilingData_->blockFactor;
125 pipe_.InitBuffer(inQueue_, BUFFER_NUM, tilingData_->bufferSize);123 pipe_.InitBuffer(inQueue_, BUFFER_NUM, tilingData_->bufferSize);
@@ -135,15 +133,15 @@ __aicore__ inline void ViewCopySimtDim5<T>::Init(
135 int32_t tailBatchSize_ = 1;133 int32_t tailBatchSize_ = 1;
136 134 
137 dim4UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *135 dim4UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *
138- tilingData_->ubDstSize[DIM2_INDEX] * tilingData_->ubDstSize[DIM3_INDEX];136+ tilingData_->ubDstSize[DIM2_INDEX] * tilingData_->ubDstSize[DIM3_INDEX];
139 dim3UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *137 dim3UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *
140- tilingData_->ubDstSize[DIM2_INDEX];138+ tilingData_->ubDstSize[DIM2_INDEX];
141 dim2UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX];139 dim2UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX];
142 dim1UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX];140 dim1UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX];
143 GetUintDivMagicAndShift<uint32_t>(m_[0], shift_[0], dim4UbdstStride_);141 GetUintDivMagicAndShift<uint32_t>(m_[0], shift_[0], dim4UbdstStride_);
144- GetUintDivMagicAndShift<uint32_t>(m_[1], shift_[1], dim3UbdstStride_); 142+ GetUintDivMagicAndShift<uint32_t>(m_[1], shift_[1], dim3UbdstStride_);
145- GetUintDivMagicAndShift<uint32_t>(m_[2], shift_[2], dim2UbdstStride_); 143+ GetUintDivMagicAndShift<uint32_t>(m_[2], shift_[2], dim2UbdstStride_);
146- GetUintDivMagicAndShift<uint32_t>(m_[3], shift_[3], dim1UbdstStride_); 144+ GetUintDivMagicAndShift<uint32_t>(m_[3], shift_[3], dim1UbdstStride_);
147}145}
148 146 
149template <typename T>147template <typename T>
@@ -186,7 +184,7 @@ __aicore__ inline void ViewCopySimtDim5<T>::Process()
186 for (int64_t idx = 0; idx < loopSize; idx++) {184 for (int64_t idx = 0; idx < loopSize; idx++) {
187 globalLoopIdx = blockOffset_ + idx;185 globalLoopIdx = blockOffset_ + idx;
188 globalLoopIdxModUo = globalLoopIdx % tilingData_->uo;186 globalLoopIdxModUo = globalLoopIdx % tilingData_->uo;
189- if ((globalLoopIdxModUo) * tilingData_->ubFactor <= tilingData_->ubDimSize) {187+ if ((globalLoopIdxModUo)*tilingData_->ubFactor <= tilingData_->ubDimSize) {
190 if ((globalLoopIdxModUo + 1) * tilingData_->ubFactor > tilingData_->ubDimSize) {188 if ((globalLoopIdxModUo + 1) * tilingData_->ubFactor > tilingData_->ubDimSize) {
191 ProcessPerLoop(globalLoopIdx, tailDmaParam_, tailBatchSize_);189 ProcessPerLoop(globalLoopIdx, tailDmaParam_, tailBatchSize_);
192 } else {190 } else {
@@ -197,8 +195,7 @@ __aicore__ inline void ViewCopySimtDim5<T>::Process()
197}195}
198 196 
199template <typename T>197template <typename T>
200-__aicore__ inline void ViewCopySimtDim5<T>::CopyIn(const GlobalTensor<T> &src,198+__aicore__ inline void ViewCopySimtDim5<T>::CopyIn(const GlobalTensor<T>& src, const NdDmaParams<T, DIM5>& dmaParams)
201- const MultiCopyParams<T, DIM5> &dmaParams)
202{199{
203 LocalTensor<T> srcLocal = inQueue_.AllocTensor<T>();200 LocalTensor<T> srcLocal = inQueue_.AllocTensor<T>();
204 this->CopyInDim5(src, srcLocal, dmaParams, tilingData_->enableMovAlign != 0);201 this->CopyInDim5(src, srcLocal, dmaParams, tilingData_->enableMovAlign != 0);
@@ -206,8 +203,8 @@ __aicore__ inline void ViewCopySimtDim5<T>::CopyIn(const GlobalTensor<T> &src,
206}203}
207 204 
208template <typename T>205template <typename T>
209-__aicore__ inline void ViewCopySimtDim5<T>::ProcessPerLoop(int64_t globalLoopIdx,206+__aicore__ inline void ViewCopySimtDim5<T>::ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM5>& dmaParam,
210- const MultiCopyParams<T, DIM5> &dmaParam, int32_t batchSize)207+ int32_t batchSize)
211{208{
212 LocalTensor<uint32_t> ParamDim4MagicUb = Dim4paramMagicBuf_.Get<uint32_t>();209 LocalTensor<uint32_t> ParamDim4MagicUb = Dim4paramMagicBuf_.Get<uint32_t>();
213 LocalTensor<uint32_t> ParamDim4ShiftUb = Dim4paramShiftBuf_.Get<uint32_t>();210 LocalTensor<uint32_t> ParamDim4ShiftUb = Dim4paramShiftBuf_.Get<uint32_t>();
@@ -222,19 +219,20 @@ __aicore__ inline void ViewCopySimtDim5<T>::ProcessPerLoop(int64_t globalLoopIdx
222 __gm__ T* dstAddr = (__gm__ T*)(dstGm_.GetPhyAddr()) + dstOffset;219 __gm__ T* dstAddr = (__gm__ T*)(dstGm_.GetPhyAddr()) + dstOffset;
223 if (tilingData_->enableDstInt64 != 0) {220 if (tilingData_->enableDstInt64 != 0) {
224 LocalTensor<uint64_t> ParamDim5DstStrideUb = Dim5paramDstStrideBuf_.Get<uint64_t>();221 LocalTensor<uint64_t> ParamDim5DstStrideUb = Dim5paramDstStrideBuf_.Get<uint64_t>();
225- asc_vf_call<CopyUbToGmDim5<T, uint64_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize,222+ asc_vf_call<CopyUbToGmDim5<T, uint64_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize, dim4UbdstStride_,
226- dim4UbdstStride_, dim3UbdstStride_, dim2UbdstStride_, dim1UbdstStride_, 223+ dim3UbdstStride_, dim2UbdstStride_, dim1UbdstStride_,
227- (__ubuf__ uint32_t*)(ParamDim4MagicUb.GetPhyAddr()), (__ubuf__ uint32_t*)(ParamDim4ShiftUb.GetPhyAddr()),224+ (__ubuf__ uint32_t*)(ParamDim4MagicUb.GetPhyAddr()),
228- (__ubuf__ uint32_t*)(ParamDim5SrcStrideUb.GetPhyAddr()),225+ (__ubuf__ uint32_t*)(ParamDim4ShiftUb.GetPhyAddr()),
229- (__ubuf__ uint64_t*)(ParamDim5DstStrideUb.GetPhyAddr()));226+ (__ubuf__ uint32_t*)(ParamDim5SrcStrideUb.GetPhyAddr()),
227+ (__ubuf__ uint64_t*)(ParamDim5DstStrideUb.GetPhyAddr()));
230 } else {228 } else {
231 LocalTensor<uint32_t> ParamDim5DstStrideUb = Dim5paramDstStrideBuf_.Get<uint32_t>();229 LocalTensor<uint32_t> ParamDim5DstStrideUb = Dim5paramDstStrideBuf_.Get<uint32_t>();
232- asc_vf_call<CopyUbToGmDim5<T, uint32_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize,230+ asc_vf_call<CopyUbToGmDim5<T, uint32_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize, dim4UbdstStride_,
233- dim4UbdstStride_, dim3UbdstStride_, dim2UbdstStride_, dim1UbdstStride_,231+ dim3UbdstStride_, dim2UbdstStride_, dim1UbdstStride_,
234- (__ubuf__ uint32_t*)(ParamDim4MagicUb.GetPhyAddr()),232+ (__ubuf__ uint32_t*)(ParamDim4MagicUb.GetPhyAddr()),
235- (__ubuf__ uint32_t*)(ParamDim4ShiftUb.GetPhyAddr()),233+ (__ubuf__ uint32_t*)(ParamDim4ShiftUb.GetPhyAddr()),
236- (__ubuf__ uint32_t*)(ParamDim5SrcStrideUb.GetPhyAddr()),234+ (__ubuf__ uint32_t*)(ParamDim5SrcStrideUb.GetPhyAddr()),
237- (__ubuf__ uint32_t*)(ParamDim5DstStrideUb.GetPhyAddr()));235+ (__ubuf__ uint32_t*)(ParamDim5DstStrideUb.GetPhyAddr()));
238 }236 }
239 inQueue_.FreeTensor(dstLocal);237 inQueue_.FreeTensor(dstLocal);
240}238}
@@ -242,52 +240,44 @@ __aicore__ inline void ViewCopySimtDim5<T>::ProcessPerLoop(int64_t globalLoopIdx
242template <typename T>240template <typename T>
243__aicore__ inline void ViewCopySimtDim5<T>::InitCopyParams()241__aicore__ inline void ViewCopySimtDim5<T>::InitCopyParams()
244{242{
245- dmaParam_ = {243+ dmaParam_ = {{{static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),
246- {244+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX]),
247- {static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),245+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM2_INDEX]),
248- static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX]),246+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM3_INDEX]),
249- static_cast<uint64_t>(tilingData_->nddmaStride[DIM2_INDEX]),247+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM4_INDEX])},
250- static_cast<uint64_t>(tilingData_->nddmaStride[DIM3_INDEX]),248+ {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),
251- static_cast<uint64_t>(tilingData_->nddmaStride[DIM4_INDEX]) },249+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX]),
252- {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),250+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM2_INDEX]),
253- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX]),251+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM3_INDEX]),
254- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM2_INDEX]),252+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM4_INDEX])},
255- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM3_INDEX]),253+ {static_cast<uint32_t>(tilingData_->nddmaSize[DIM0_INDEX]),
256- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM4_INDEX])},254+ static_cast<uint32_t>(tilingData_->nddmaSize[DIM1_INDEX]),
257- {static_cast<uint32_t>(tilingData_->nddmaSize[DIM0_INDEX]),255+ static_cast<uint32_t>(tilingData_->nddmaSize[DIM2_INDEX]),
258- static_cast<uint32_t>(tilingData_->nddmaSize[DIM1_INDEX]),256+ static_cast<uint32_t>(tilingData_->nddmaSize[DIM3_INDEX]),
259- static_cast<uint32_t>(tilingData_->nddmaSize[DIM2_INDEX]),257+ static_cast<uint32_t>(tilingData_->nddmaSize[DIM4_INDEX])},
260- static_cast<uint32_t>(tilingData_->nddmaSize[DIM3_INDEX]),258+ {0, 0, 0, 0, 0},
261- static_cast<uint32_t>(tilingData_->nddmaSize[DIM4_INDEX])},259+ {0, 0, 0, 0, 0}},
262- {0, 0, 0, 0, 0},260+ 0};
263- {0, 0, 0, 0, 0}
264- },
265- 0
266- };
267 261 
268- tailDmaParam_ = {262+ tailDmaParam_ = {{{static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),
269- {263+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX]),
270- {static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),264+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM2_INDEX]),
271- static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX]),265+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM3_INDEX]),
272- static_cast<uint64_t>(tilingData_->nddmaStride[DIM2_INDEX]),266+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM4_INDEX])},
273- static_cast<uint64_t>(tilingData_->nddmaStride[DIM3_INDEX]),267+ {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),
274- static_cast<uint64_t>(tilingData_->nddmaStride[DIM4_INDEX])},268+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX]),
275- {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),269+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM2_INDEX]),
276- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX]),270+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM3_INDEX]),
277- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM2_INDEX]),271+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM4_INDEX])},
278- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM3_INDEX]),272+ {static_cast<uint32_t>(this->tailNddmaSize_[DIM0_INDEX]),
279- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM4_INDEX])},273+ static_cast<uint32_t>(this->tailNddmaSize_[DIM1_INDEX]),
280- {static_cast<uint32_t>(this->tailNddmaSize_[DIM0_INDEX]),274+ static_cast<uint32_t>(this->tailNddmaSize_[DIM2_INDEX]),
281- static_cast<uint32_t>(this->tailNddmaSize_[DIM1_INDEX]),275+ static_cast<uint32_t>(this->tailNddmaSize_[DIM3_INDEX]),
282- static_cast<uint32_t>(this->tailNddmaSize_[DIM2_INDEX]),276+ static_cast<uint32_t>(this->tailNddmaSize_[DIM4_INDEX])},
283- static_cast<uint32_t>(this->tailNddmaSize_[DIM3_INDEX]),277+ {0, 0, 0, 0, 0},
284- static_cast<uint32_t>(this->tailNddmaSize_[DIM4_INDEX])},278+ {0, 0, 0, 0, 0}},
285- {0, 0, 0, 0, 0},279+ 0};
286- {0, 0, 0, 0, 0}
287- },
288- 0
289- };
290}280}
291-} // namespace ViewCopy281+} // namespace ViewCopy
292 282 
293-#endif // VIEW_COPY_SIMT_DIM5_H_283+#endif // VIEW_COPY_SIMT_DIM5_H_
Mconversion/view_copy/op_kernel/arch35/view_copy_simt_dim8.h+93-91
@@ -23,10 +23,11 @@ using namespace AscendC;
23 23 
24template <typename T, typename U>24template <typename T, typename U>
25__simt_vf__ LAUNCH_BOUND(THREAD_DIM) __aicore__25__simt_vf__ LAUNCH_BOUND(THREAD_DIM) __aicore__
26- void CopyUbToGmDim8(__gm__ T* dst, __ubuf__ T* src, uint32_t batchSize,26+ void CopyUbToGmDim8(__gm__ T* dst, __ubuf__ T* src, uint32_t batchSize, uint32_t ubDim7Stride,
27- uint32_t ubDim7Stride, uint32_t ubDim6Stride, uint32_t ubDim5Stride, uint32_t ubDim4Stride, uint32_t ubDim3Stride,27+ uint32_t ubDim6Stride, uint32_t ubDim5Stride, uint32_t ubDim4Stride, uint32_t ubDim3Stride,
28- uint32_t ubDim2Stride, uint32_t ubDim1Stride, __ubuf__ uint32_t* ParamDim7MagicUb,28+ uint32_t ubDim2Stride, uint32_t ubDim1Stride, __ubuf__ uint32_t* ParamDim7MagicUb,
29- __ubuf__ uint32_t* ParamDim7ShiftUb, __ubuf__ uint32_t* ParamDim8SrcStrideUb, __ubuf__ U* ParamDim8DstStrideUb)29+ __ubuf__ uint32_t* ParamDim7ShiftUb, __ubuf__ uint32_t* ParamDim8SrcStrideUb,
30+ __ubuf__ U* ParamDim8DstStrideUb)
30{31{
31 for (uint32_t i = threadIdx.x; i < batchSize; i += blockDim.x) {32 for (uint32_t i = threadIdx.x; i < batchSize; i += blockDim.x) {
32 uint32_t idx = i;33 uint32_t idx = i;
@@ -88,23 +89,26 @@ __simt_vf__ LAUNCH_BOUND(THREAD_DIM) __aicore__
88 }89 }
89}90}
90 91 
91-template <typename T> class ViewCopySimtDim8 : public ViewCopyBase<T> {92+template <typename T>
93+class ViewCopySimtDim8 : public ViewCopyBase<T> {
92public:94public:
93- __aicore__ inline ViewCopySimtDim8(TPipe &pipe, const ViewCopyTilingData *tilingData) :95+ __aicore__ inline ViewCopySimtDim8(TPipe& pipe, const ViewCopyTilingData* tilingData)
94- pipe_(pipe), tilingData_(tilingData) {};96+ : pipe_(pipe), tilingData_(tilingData){};
95 __aicore__ inline void Init(GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset, GM_ADDR src,97 __aicore__ inline void Init(GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride, GM_ADDR dstStorageOffset, GM_ADDR src,
96- GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset, GM_ADDR out);98+ GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset, GM_ADDR out);
97 __aicore__ inline void Process();99 __aicore__ inline void Process();
98 __aicore__ inline void InitCopyParams();100 __aicore__ inline void InitCopyParams();
99 101 
100private:102private:
101- __aicore__ inline void CustomNddma(LocalTensor<T> &dst, const GlobalTensor<T> &src, const int32_t *loopSize,103+ __aicore__ inline void CustomNddma(LocalTensor<T>& dst, const GlobalTensor<T>& src, const int32_t* loopSize,
102- const int64_t *srcStride, const int64_t *dstStride, const MultiCopyParams<T, 5> &dmaParam);104+ const int64_t* srcStride, const int64_t* dstStride,
103- __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const MultiCopyParams<T, DIM5> &dmaParam,105+ const NdDmaParams<T, 5>& dmaParam);
104- const int32_t *copyInLoopSize, int32_t batchSize);106+ __aicore__ inline void ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM5>& dmaParam,
107+ const int32_t* copyInLoopSize, int32_t batchSize);
108+ 
105private:109private:
106- TPipe &pipe_;110+ TPipe& pipe_;
107- const ViewCopyTilingData *tilingData_;111+ const ViewCopyTilingData* tilingData_;
108 TQue<QuePosition::VECIN, BUFFER_NUM> inQueue_;112 TQue<QuePosition::VECIN, BUFFER_NUM> inQueue_;
109 113 
110 GlobalTensor<T> dstGm_;114 GlobalTensor<T> dstGm_;
@@ -114,8 +118,8 @@ private:
114 TBuf<TPosition::VECCALC> Dim8paramSrcStrideBuf_;118 TBuf<TPosition::VECCALC> Dim8paramSrcStrideBuf_;
115 TBuf<TPosition::VECCALC> Dim8paramDstStrideBuf_;119 TBuf<TPosition::VECCALC> Dim8paramDstStrideBuf_;
116 120 
117- MultiCopyParams<T, DIM5> tailDmaParam_;121+ NdDmaParams<T, DIM5> tailDmaParam_;
118- MultiCopyParams<T, DIM5> dmaParam_;122+ NdDmaParams<T, DIM5> dmaParam_;
119 DataCopyExtParams tailCopyParams_;123 DataCopyExtParams tailCopyParams_;
120 DataCopyExtParams copyParams_;124 DataCopyExtParams copyParams_;
121 125 
@@ -135,11 +139,12 @@ private:
135 139 
136template <typename T>140template <typename T>
137__aicore__ inline void ViewCopySimtDim8<T>::Init(GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride,141__aicore__ inline void ViewCopySimtDim8<T>::Init(GM_ADDR dst, GM_ADDR dstSize, GM_ADDR dstStride,
138- GM_ADDR dstStorageOffset, GM_ADDR src, GM_ADDR srcSize, GM_ADDR srcStride, GM_ADDR srcStorageOffset, GM_ADDR out)142+ GM_ADDR dstStorageOffset, GM_ADDR src, GM_ADDR srcSize,
143+ GM_ADDR srcStride, GM_ADDR srcStorageOffset, GM_ADDR out)
139{144{
140 this->ParseTilingData(tilingData_);145 this->ParseTilingData(tilingData_);
141- srcGm_.SetGlobalBuffer((__gm__ T *)(src + tilingData_->srcStorageOffset * sizeof(T)));146+ srcGm_.SetGlobalBuffer((__gm__ T*)(src + tilingData_->srcStorageOffset * sizeof(T)));
142- dstGm_.SetGlobalBuffer((__gm__ T *)(dst + tilingData_->dstStorageOffset * sizeof(T)));147+ dstGm_.SetGlobalBuffer((__gm__ T*)(dst + tilingData_->dstStorageOffset * sizeof(T)));
143 148 
144 pipe_.InitBuffer(inQueue_, BUFFER_NUM, tilingData_->bufferSize);149 pipe_.InitBuffer(inQueue_, BUFFER_NUM, tilingData_->bufferSize);
145 pipe_.InitBuffer(Dim7paramMagicBuf_, DIM7_PARAM_NUM * sizeof(uint32_t));150 pipe_.InitBuffer(Dim7paramMagicBuf_, DIM7_PARAM_NUM * sizeof(uint32_t));
@@ -153,17 +158,19 @@ __aicore__ inline void ViewCopySimtDim8<T>::Init(GM_ADDR dst, GM_ADDR dstSize, G
153 }158 }
154 159 
155 dim7UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *160 dim7UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *
156- tilingData_->ubDstSize[DIM2_INDEX] * tilingData_->ubDstSize[DIM3_INDEX] * tilingData_->ubDstSize[DIM4_INDEX] *161+ tilingData_->ubDstSize[DIM2_INDEX] * tilingData_->ubDstSize[DIM3_INDEX] *
157- tilingData_->ubDstSize[DIM5_INDEX] * tilingData_->ubDstSize[DIM6_INDEX];162+ tilingData_->ubDstSize[DIM4_INDEX] * tilingData_->ubDstSize[DIM5_INDEX] *
163+ tilingData_->ubDstSize[DIM6_INDEX];
158 dim6UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *164 dim6UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *
159- tilingData_->ubDstSize[DIM2_INDEX] * tilingData_->ubDstSize[DIM3_INDEX] * tilingData_->ubDstSize[DIM4_INDEX] *165+ tilingData_->ubDstSize[DIM2_INDEX] * tilingData_->ubDstSize[DIM3_INDEX] *
160- tilingData_->ubDstSize[DIM5_INDEX];166+ tilingData_->ubDstSize[DIM4_INDEX] * tilingData_->ubDstSize[DIM5_INDEX];
161 dim5UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *167 dim5UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *
162- tilingData_->ubDstSize[DIM2_INDEX] * tilingData_->ubDstSize[DIM3_INDEX] * tilingData_->ubDstSize[DIM4_INDEX];168+ tilingData_->ubDstSize[DIM2_INDEX] * tilingData_->ubDstSize[DIM3_INDEX] *
169+ tilingData_->ubDstSize[DIM4_INDEX];
163 dim4UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *170 dim4UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *
164- tilingData_->ubDstSize[DIM2_INDEX] * tilingData_->ubDstSize[DIM3_INDEX];171+ tilingData_->ubDstSize[DIM2_INDEX] * tilingData_->ubDstSize[DIM3_INDEX];
165 dim3UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *172 dim3UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX] *
166- tilingData_->ubDstSize[DIM2_INDEX];173+ tilingData_->ubDstSize[DIM2_INDEX];
167 dim2UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX];174 dim2UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX] * tilingData_->ubDstSize[DIM1_INDEX];
168 dim1UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX];175 dim1UbdstStride_ = tilingData_->ubDstSize[DIM0_INDEX];
169 GetUintDivMagicAndShift<uint32_t>(m_[0], shift_[0], dim7UbdstStride_);176 GetUintDivMagicAndShift<uint32_t>(m_[0], shift_[0], dim7UbdstStride_);
@@ -175,7 +182,8 @@ __aicore__ inline void ViewCopySimtDim8<T>::Init(GM_ADDR dst, GM_ADDR dstSize, G
175 GetUintDivMagicAndShift<uint32_t>(m_[6], shift_[6], dim1UbdstStride_);182 GetUintDivMagicAndShift<uint32_t>(m_[6], shift_[6], dim1UbdstStride_);
176}183}
177 184 
178-template <typename T> __aicore__ inline void ViewCopySimtDim8<T>::Process()185+template <typename T>
186+__aicore__ inline void ViewCopySimtDim8<T>::Process()
179{187{
180 if (GetBlockIdx() >= GetBlockNum()) {188 if (GetBlockIdx() >= GetBlockNum()) {
181 return;189 return;
@@ -214,7 +222,7 @@ template <typename T> __aicore__ inline void ViewCopySimtDim8<T>::Process()
214 for (int64_t idx = 0; idx < loopSize; idx++) {222 for (int64_t idx = 0; idx < loopSize; idx++) {
215 globalLoopIdx = blockOffset_ + idx;223 globalLoopIdx = blockOffset_ + idx;
216 globalLoopIdxModUo = globalLoopIdx % tilingData_->uo;224 globalLoopIdxModUo = globalLoopIdx % tilingData_->uo;
217- if ((globalLoopIdxModUo) * tilingData_->ubFactor <= tilingData_->ubDimSize) {225+ if ((globalLoopIdxModUo)*tilingData_->ubFactor <= tilingData_->ubDimSize) {
218 if ((globalLoopIdxModUo + 1) * tilingData_->ubFactor > tilingData_->ubDimSize) {226 if ((globalLoopIdxModUo + 1) * tilingData_->ubFactor > tilingData_->ubDimSize) {
219 ProcessPerLoop(globalLoopIdx, tailDmaParam_, this->tailNddmaSize_, tailBatchSize_);227 ProcessPerLoop(globalLoopIdx, tailDmaParam_, this->tailNddmaSize_, tailBatchSize_);
220 } else {228 } else {
@@ -225,38 +233,38 @@ template <typename T> __aicore__ inline void ViewCopySimtDim8<T>::Process()
225}233}
226 234 
227template <typename T>235template <typename T>
228-__aicore__ inline void ViewCopySimtDim8<T>::ProcessPerLoop(int64_t globalLoopIdx,236+__aicore__ inline void ViewCopySimtDim8<T>::ProcessPerLoop(int64_t globalLoopIdx, const NdDmaParams<T, DIM5>& dmaParam,
229- const MultiCopyParams<T, DIM5> &dmaParam, const int32_t *copyInLoopSize, int32_t batchSize)237+ const int32_t* copyInLoopSize, int32_t batchSize)
230{238{
231 LocalTensor<uint32_t> ParamDim7MagicUb = Dim7paramMagicBuf_.Get<uint32_t>();239 LocalTensor<uint32_t> ParamDim7MagicUb = Dim7paramMagicBuf_.Get<uint32_t>();
232 LocalTensor<uint32_t> ParamDim7ShiftUb = Dim7paramShiftBuf_.Get<uint32_t>();240 LocalTensor<uint32_t> ParamDim7ShiftUb = Dim7paramShiftBuf_.Get<uint32_t>();
233 LocalTensor<uint32_t> ParamDim8SrcStrideUb = Dim8paramSrcStrideBuf_.Get<uint32_t>();241 LocalTensor<uint32_t> ParamDim8SrcStrideUb = Dim8paramSrcStrideBuf_.Get<uint32_t>();
234 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,242 int64_t srcOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockSrcStride,
235- tilingData_->blockFusedDimsNumber);243+ tilingData_->blockFusedDimsNumber);
236 LocalTensor<T> srcLocal = inQueue_.AllocTensor<T>();244 LocalTensor<T> srcLocal = inQueue_.AllocTensor<T>();
237 CustomNddma(srcLocal, srcGm_[srcOffset], copyInLoopSize, tilingData_->nddmaStride,245 CustomNddma(srcLocal, srcGm_[srcOffset], copyInLoopSize, tilingData_->nddmaStride,
238- tilingData_->contiguousUbSrcStride, dmaParam);246+ tilingData_->contiguousUbSrcStride, dmaParam);
239 inQueue_.EnQue(srcLocal);247 inQueue_.EnQue(srcLocal);
240 int64_t dstOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockDstStride,248 int64_t dstOffset = this->GetGmOffset(globalLoopIdx, tilingData_->blockStride, tilingData_->blockDstStride,
241- tilingData_->blockFusedDimsNumber);249+ tilingData_->blockFusedDimsNumber);
242 LocalTensor<T> dstLocal = inQueue_.DeQue<T>();250 LocalTensor<T> dstLocal = inQueue_.DeQue<T>();
243 251 
244 __ubuf__ T* srcAddr = (__ubuf__ T*)srcLocal.GetPhyAddr();252 __ubuf__ T* srcAddr = (__ubuf__ T*)srcLocal.GetPhyAddr();
245 __gm__ T* dstAddr = (__gm__ T*)(dstGm_.GetPhyAddr()) + dstOffset;253 __gm__ T* dstAddr = (__gm__ T*)(dstGm_.GetPhyAddr()) + dstOffset;
246 if (tilingData_->enableDstInt64 != 0) {254 if (tilingData_->enableDstInt64 != 0) {
247 LocalTensor<uint64_t> ParamDim8DstStrideUb = Dim8paramDstStrideBuf_.Get<uint64_t>();255 LocalTensor<uint64_t> ParamDim8DstStrideUb = Dim8paramDstStrideBuf_.Get<uint64_t>();
248- asc_vf_call<CopyUbToGmDim8<T, uint64_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize,256+ asc_vf_call<CopyUbToGmDim8<T, uint64_t>>(
249- dim7UbdstStride_, dim6UbdstStride_, dim5UbdstStride_, dim4UbdstStride_, dim3UbdstStride_, dim2UbdstStride_,257+ dim3(THREAD_DIM), dstAddr, srcAddr, batchSize, dim7UbdstStride_, dim6UbdstStride_, dim5UbdstStride_,
250- dim1UbdstStride_, (__ubuf__ uint32_t*)(ParamDim7MagicUb.GetPhyAddr()),258+ dim4UbdstStride_, dim3UbdstStride_, dim2UbdstStride_, dim1UbdstStride_,
251- (__ubuf__ uint32_t*)(ParamDim7ShiftUb.GetPhyAddr()),259+ (__ubuf__ uint32_t*)(ParamDim7MagicUb.GetPhyAddr()), (__ubuf__ uint32_t*)(ParamDim7ShiftUb.GetPhyAddr()),
252 (__ubuf__ uint32_t*)(ParamDim8SrcStrideUb.GetPhyAddr()),260 (__ubuf__ uint32_t*)(ParamDim8SrcStrideUb.GetPhyAddr()),
253 (__ubuf__ uint64_t*)(ParamDim8DstStrideUb.GetPhyAddr()));261 (__ubuf__ uint64_t*)(ParamDim8DstStrideUb.GetPhyAddr()));
254 } else {262 } else {
255 LocalTensor<uint32_t> ParamDim8DstStrideUb = Dim8paramDstStrideBuf_.Get<uint32_t>();263 LocalTensor<uint32_t> ParamDim8DstStrideUb = Dim8paramDstStrideBuf_.Get<uint32_t>();
256- asc_vf_call<CopyUbToGmDim8<T, uint32_t>>(dim3(THREAD_DIM), dstAddr, srcAddr, batchSize,264+ asc_vf_call<CopyUbToGmDim8<T, uint32_t>>(
257- dim7UbdstStride_, dim6UbdstStride_, dim5UbdstStride_, dim4UbdstStride_, dim3UbdstStride_, dim2UbdstStride_,265+ dim3(THREAD_DIM), dstAddr, srcAddr, batchSize, dim7UbdstStride_, dim6UbdstStride_, dim5UbdstStride_,
258- dim1UbdstStride_, (__ubuf__ uint32_t*)(ParamDim7MagicUb.GetPhyAddr()),266+ dim4UbdstStride_, dim3UbdstStride_, dim2UbdstStride_, dim1UbdstStride_,
259- (__ubuf__ uint32_t*)(ParamDim7ShiftUb.GetPhyAddr()),267+ (__ubuf__ uint32_t*)(ParamDim7MagicUb.GetPhyAddr()), (__ubuf__ uint32_t*)(ParamDim7ShiftUb.GetPhyAddr()),
260 (__ubuf__ uint32_t*)(ParamDim8SrcStrideUb.GetPhyAddr()),268 (__ubuf__ uint32_t*)(ParamDim8SrcStrideUb.GetPhyAddr()),
261 (__ubuf__ uint32_t*)(ParamDim8DstStrideUb.GetPhyAddr()));269 (__ubuf__ uint32_t*)(ParamDim8DstStrideUb.GetPhyAddr()));
262 }270 }
@@ -264,8 +272,9 @@ __aicore__ inline void ViewCopySimtDim8<T>::ProcessPerLoop(int64_t globalLoopIdx
264}272}
265 273 
266template <typename T>274template <typename T>
267-__aicore__ inline void ViewCopySimtDim8<T>::CustomNddma(LocalTensor<T> &dst, const GlobalTensor<T> &src,275+__aicore__ inline void ViewCopySimtDim8<T>::CustomNddma(LocalTensor<T>& dst, const GlobalTensor<T>& src,
268- const int32_t *loopSize, const int64_t *srcStride, const int64_t *dstStride, const MultiCopyParams<T, 5> &dmaParam)276+ const int32_t* loopSize, const int64_t* srcStride,
277+ const int64_t* dstStride, const NdDmaParams<T, 5>& dmaParam)
269{278{
270 // srcSize (dim7, dim6, dim5, dim4, dim3, dim2, dim1, dim0)279 // srcSize (dim7, dim6, dim5, dim4, dim3, dim2, dim1, dim0)
271 int64_t srcOffset = 0;280 int64_t srcOffset = 0;
@@ -275,63 +284,56 @@ __aicore__ inline void ViewCopySimtDim8<T>::CustomNddma(LocalTensor<T> &dst, con
275 for (int64_t loopDim6 = 0; loopDim6 < loopSize[DIM6_INDEX]; loopDim6++) {284 for (int64_t loopDim6 = 0; loopDim6 < loopSize[DIM6_INDEX]; loopDim6++) {
276 for (int64_t loopDim5 = 0; loopDim5 < loopSize[DIM5_INDEX]; loopDim5++) {285 for (int64_t loopDim5 = 0; loopDim5 < loopSize[DIM5_INDEX]; loopDim5++) {
277 srcOffset = loopDim7 * srcStride[DIM7_INDEX] + loopDim6 * srcStride[DIM6_INDEX] +286 srcOffset = loopDim7 * srcStride[DIM7_INDEX] + loopDim6 * srcStride[DIM6_INDEX] +
278- loopDim5 * srcStride[DIM5_INDEX];287+ loopDim5 * srcStride[DIM5_INDEX];
279 dstOffset = loopDim7 * dstStride[DIM7_INDEX] + loopDim6 * dstStride[DIM6_INDEX] +288 dstOffset = loopDim7 * dstStride[DIM7_INDEX] + loopDim6 * dstStride[DIM6_INDEX] +
280- loopDim5 * dstStride[DIM5_INDEX];289+ loopDim5 * dstStride[DIM5_INDEX];
281 DataCopy(dst[dstOffset], src[srcOffset], dmaParam);290 DataCopy(dst[dstOffset], src[srcOffset], dmaParam);
282 }291 }
283 }292 }
284 }293 }
285}294}
286 295 
287-template <typename T> __aicore__ inline void ViewCopySimtDim8<T>::InitCopyParams()296+template <typename T>
297+__aicore__ inline void ViewCopySimtDim8<T>::InitCopyParams()
288{298{
289- dmaParam_ = { 299+ dmaParam_ = {{{static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),
290- { 300+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX]),
291- { static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),301+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM2_INDEX]),
292- static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX]),302+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM3_INDEX]),
293- static_cast<uint64_t>(tilingData_->nddmaStride[DIM2_INDEX]),303+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM4_INDEX])},
294- static_cast<uint64_t>(tilingData_->nddmaStride[DIM3_INDEX]),304+ {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),
295- static_cast<uint64_t>(tilingData_->nddmaStride[DIM4_INDEX]) },305+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX]),
296- { static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),306+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM2_INDEX]),
297- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX]),307+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM3_INDEX]),
298- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM2_INDEX]),308+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM4_INDEX])},
299- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM3_INDEX]),309+ {static_cast<uint32_t>(tilingData_->nddmaSize[DIM0_INDEX]),
300- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM4_INDEX]) },310+ static_cast<uint32_t>(tilingData_->nddmaSize[DIM1_INDEX]),
301- { static_cast<uint32_t>(tilingData_->nddmaSize[DIM0_INDEX]),311+ static_cast<uint32_t>(tilingData_->nddmaSize[DIM2_INDEX]),
302- static_cast<uint32_t>(tilingData_->nddmaSize[DIM1_INDEX]),312+ static_cast<uint32_t>(tilingData_->nddmaSize[DIM3_INDEX]),
303- static_cast<uint32_t>(tilingData_->nddmaSize[DIM2_INDEX]),313+ static_cast<uint32_t>(tilingData_->nddmaSize[DIM4_INDEX])},
304- static_cast<uint32_t>(tilingData_->nddmaSize[DIM3_INDEX]),314+ {0, 0, 0, 0, 0},
305- static_cast<uint32_t>(tilingData_->nddmaSize[DIM4_INDEX]) },315+ {0, 0, 0, 0, 0}},
306- { 0, 0, 0, 0, 0 },316+ 0};
307- { 0, 0, 0, 0, 0 }
308- },
309- 0
310- };
311 317 
312- tailDmaParam_ = {318+ tailDmaParam_ = {{{static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),
313- {319+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX]),
314- { static_cast<uint64_t>(tilingData_->nddmaStride[DIM0_INDEX]),320+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM2_INDEX]),
315- static_cast<uint64_t>(tilingData_->nddmaStride[DIM1_INDEX]),321+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM3_INDEX]),
316- static_cast<uint64_t>(tilingData_->nddmaStride[DIM2_INDEX]),322+ static_cast<uint64_t>(tilingData_->nddmaStride[DIM4_INDEX])},
317- static_cast<uint64_t>(tilingData_->nddmaStride[DIM3_INDEX]),323+ {static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),
318- static_cast<uint64_t>(tilingData_->nddmaStride[DIM4_INDEX]) },324+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX]),
319- { static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM0_INDEX]),325+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM2_INDEX]),
320- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM1_INDEX]),326+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM3_INDEX]),
321- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM2_INDEX]),327+ static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM4_INDEX])},
322- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM3_INDEX]),328+ {static_cast<uint32_t>(this->tailNddmaSize_[DIM0_INDEX]),
323- static_cast<uint32_t>(tilingData_->contiguousUbSrcStride[DIM4_INDEX]) },329+ static_cast<uint32_t>(this->tailNddmaSize_[DIM1_INDEX]),
324- { static_cast<uint32_t>(this->tailNddmaSize_[DIM0_INDEX]),330+ static_cast<uint32_t>(this->tailNddmaSize_[DIM2_INDEX]),
325- static_cast<uint32_t>(this->tailNddmaSize_[DIM1_INDEX]),331+ static_cast<uint32_t>(this->tailNddmaSize_[DIM3_INDEX]),
326- static_cast<uint32_t>(this->tailNddmaSize_[DIM2_INDEX]),332+ static_cast<uint32_t>(this->tailNddmaSize_[DIM4_INDEX])},
327- static_cast<uint32_t>(this->tailNddmaSize_[DIM3_INDEX]),333+ {0, 0, 0, 0, 0},
328- static_cast<uint32_t>(this->tailNddmaSize_[DIM4_INDEX]) },334+ {0, 0, 0, 0, 0}},
329- { 0, 0, 0, 0, 0 },335+ 0};
330- { 0, 0, 0, 0, 0 }
331- },
332- 0
333- };
334}336}
335} // namespace ViewCopy337} // namespace ViewCopy
336 338 
337-#endif // VIEW_COPY_SIMT_DIM8_H_339+#endif // VIEW_COPY_SIMT_DIM8_H_
Mmath/cumsum/op_kernel/arch35/cumsum_4_int.h+97-93
@@ -49,7 +49,7 @@ static __aicore__ inline void SetUBZero(const LocalTensor<T>& ubTensor, uint32_t
49{49{
50 static constexpr uint32_t regElem = GetVecLen() / sizeof(T);50 static constexpr uint32_t regElem = GetVecLen() / sizeof(T);
51 uint16_t lpCnt = Ops::Base::CeilDiv(len, regElem);51 uint16_t lpCnt = Ops::Base::CeilDiv(len, regElem);
52- __local_mem__ T* yPtr = (__local_mem__ T*)ubTensor.GetPhyAddr();52+ __ubuf__ T* yPtr = (__ubuf__ T*)ubTensor.GetPhyAddr();
53 __VEC_SCOPE__53 __VEC_SCOPE__
54 {54 {
55 Reg::MaskReg mask;55 Reg::MaskReg mask;
@@ -57,7 +57,7 @@ static __aicore__ inline void SetUBZero(const LocalTensor<T>& ubTensor, uint32_t
57 Reg::Duplicate(zeros, (T)0);57 Reg::Duplicate(zeros, (T)0);
58 for (uint16_t i = 0; i < lpCnt; i++) {58 for (uint16_t i = 0; i < lpCnt; i++) {
59 mask = Reg::UpdateMask<T>(len);59 mask = Reg::UpdateMask<T>(len);
60- Reg::DataCopy<T>(yPtr + i * regElem, zeros, mask);60+ Reg::StoreAlign<T>(yPtr + i * regElem, zeros, mask);
61 }61 }
62 }62 }
63}63}
@@ -247,8 +247,8 @@ __aicore__ inline void DataCopyTDRightACum<T>::CumCopyTDRightA(const LocalTensor
247 */247 */
248 uint16_t rLpCnt = (uint16_t)rLen;248 uint16_t rLpCnt = (uint16_t)rLen;
249 uint16_t aLpCnt = (uint16_t)Ops::Base::CeilDiv(rightALen, regElem);249 uint16_t aLpCnt = (uint16_t)Ops::Base::CeilDiv(rightALen, regElem);
250- __local_mem__ T* srcPtr = (__local_mem__ T*)inUB.GetPhyAddr();250+ __ubuf__ T* srcPtr = (__ubuf__ T*)inUB.GetPhyAddr();
251- __local_mem__ T* dstPtr = (__local_mem__ T*)outUB.GetPhyAddr();251+ __ubuf__ T* dstPtr = (__ubuf__ T*)outUB.GetPhyAddr();
252 252 
253 int32_t bakBegAddr = (rSize - 1) * raLenBA;253 int32_t bakBegAddr = (rSize - 1) * raLenBA;
254 int32_t srcBegAddr = 0;254 int32_t srcBegAddr = 0;
@@ -274,12 +274,12 @@ __aicore__ inline void DataCopyTDRightACum<T>::CumCopyTDRightA(const LocalTensor
274 mask = Reg::UpdateMask<T>(rightALen);274 mask = Reg::UpdateMask<T>(rightALen);
275 offset1 = aIdx * regElem;275 offset1 = aIdx * regElem;
276 // Get last loop backup result, the result is zero for first loop276 // Get last loop backup result, the result is zero for first loop
277- Reg::DataCopy<T>(yDst, dstPtr + bakBegAddr + offset1);277+ Reg::LoadAlign<T>(yDst, dstPtr + bakBegAddr + offset1);
278 for (uint16_t rIdx = 0; rIdx < rLpCnt; rIdx++) {278 for (uint16_t rIdx = 0; rIdx < rLpCnt; rIdx++) {
279 offset = offset1 + rIdx * rOffset;279 offset = offset1 + rIdx * rOffset;
280- Reg::DataCopy<T>(xSrc, srcPtr + srcBegAddr + offset);280+ Reg::LoadAlign<T>(xSrc, srcPtr + srcBegAddr + offset);
281 Reg::Add(yDst, yDst, xSrc, mask);281 Reg::Add(yDst, yDst, xSrc, mask);
282- Reg::DataCopy<T>(dstPtr + dstBegAddr + offset, yDst, mask);282+ Reg::StoreAlign<T>(dstPtr + dstBegAddr + offset, yDst, mask);
283 }283 }
284 }284 }
285 }285 }
@@ -436,20 +436,20 @@ private:
436 __aicore__ inline void TDLeftAInnerProcess(const GlobalTensor<T>& inGM, const GlobalTensor<T>& outGM,436 __aicore__ inline void TDLeftAInnerProcess(const GlobalTensor<T>& inGM, const GlobalTensor<T>& outGM,
437 const LocalTensor<T>& inLocal, const LocalTensor<T>& outLocal,437 const LocalTensor<T>& inLocal, const LocalTensor<T>& outLocal,
438 int64_t laLen, int64_t rLen, bool isExc = false);438 int64_t laLen, int64_t rLen, bool isExc = false);
439- __aicore__ inline void CleanInputDirtyData(__local_mem__ T*& srcPtr, int32_t zeroBase, uint32_t leftSize);439+ __aicore__ inline void CleanInputDirtyData(__ubuf__ T*& srcPtr, int32_t zeroBase, uint32_t leftSize);
440 __aicore__ inline void TDLeftAForwardProcess(const GlobalTensor<T>& inGM, const GlobalTensor<T>& outGM,440 __aicore__ inline void TDLeftAForwardProcess(const GlobalTensor<T>& inGM, const GlobalTensor<T>& outGM,
441 const LocalTensor<T>& inLocal, const LocalTensor<T>& outLocal,441 const LocalTensor<T>& inLocal, const LocalTensor<T>& outLocal,
442 int64_t laLpIdx, int64_t laSize);442 int64_t laLpIdx, int64_t laSize);
443 __aicore__ inline void TDLeftAReverseProcess(const GlobalTensor<T>& inGM, const GlobalTensor<T>& outGM,443 __aicore__ inline void TDLeftAReverseProcess(const GlobalTensor<T>& inGM, const GlobalTensor<T>& outGM,
444 const LocalTensor<T>& inLocal, const LocalTensor<T>& outLocal,444 const LocalTensor<T>& inLocal, const LocalTensor<T>& outLocal,
445 int64_t laLpIdx, int64_t laSize);445 int64_t laLpIdx, int64_t laSize);
446- __aicore__ inline void TDRReverseProcess(__local_mem__ T*& srcPtr, __local_mem__ T*& dstPtr, uint16_t rLpNum,446+ __aicore__ inline void TDRReverseProcess(__ubuf__ T*& srcPtr, __ubuf__ T*& dstPtr, uint16_t rLpNum,
447 int32_t comRightA, int32_t splitRASize, int32_t curBegAddrBase,447 int32_t comRightA, int32_t splitRASize, int32_t curBegAddrBase,
448 int32_t curBakBegAdd, uint16_t sndRLpCnt, int32_t phs1SrcBegIdx,448 int32_t curBakBegAdd, uint16_t sndRLpCnt, int32_t phs1SrcBegIdx,
449 int32_t phs2SrcBegIdx, int32_t phs1LpOffset, int32_t phs2LpOffset,449 int32_t phs2SrcBegIdx, int32_t phs1LpOffset, int32_t phs2LpOffset,
450 int32_t phs2TLpOffset, int32_t bakBegAddr, int32_t phs1DstBegIdx,450 int32_t phs2TLpOffset, int32_t bakBegAddr, int32_t phs1DstBegIdx,
451 int32_t phs2DstBegIdx, uint32_t comLpMaskVal, uint32_t sndLpLeftMaskVal);451 int32_t phs2DstBegIdx, uint32_t comLpMaskVal, uint32_t sndLpLeftMaskVal);
452- __aicore__ inline void TDRForwardProcess(__local_mem__ T*& srcPtr, __local_mem__ T*& dstPtr, uint16_t rLpNum,452+ __aicore__ inline void TDRForwardProcess(__ubuf__ T*& srcPtr, __ubuf__ T*& dstPtr, uint16_t rLpNum,
453 int32_t comRightA, int32_t splitRASize, int32_t curBegAddrBase,453 int32_t comRightA, int32_t splitRASize, int32_t curBegAddrBase,
454 int32_t curBakBegAdd, uint16_t sndRLpCnt, int32_t phs1SrcBegIdx,454 int32_t curBakBegAdd, uint16_t sndRLpCnt, int32_t phs1SrcBegIdx,
455 int32_t phs2SrcBegIdx, int32_t phs1LpOffset, int32_t phs2LpOffset,455 int32_t phs2SrcBegIdx, int32_t phs1LpOffset, int32_t phs2LpOffset,
@@ -514,8 +514,8 @@ __aicore__ inline void GatherCum<T>::CumGatherTDRightA(const LocalTensor<T>& inL
514 * 2. The position of result of last loop is at R.514 * 2. The position of result of last loop is at R.
515 */515 */
516 516 
517- __local_mem__ T* srcPtr = (__local_mem__ T*)inLocal.GetPhyAddr();517+ __ubuf__ T* srcPtr = (__ubuf__ T*)inLocal.GetPhyAddr();
518- __local_mem__ T* dstPtr = (__local_mem__ T*)outLocal.GetPhyAddr();518+ __ubuf__ T* dstPtr = (__ubuf__ T*)outLocal.GetPhyAddr();
519 uint16_t rLpCnt = (uint16_t)rLen;519 uint16_t rLpCnt = (uint16_t)rLen;
520 uint32_t aSize = tilingPtr->raLpUnit;520 uint32_t aSize = tilingPtr->raLpUnit;
521 // B8时Gather/Scatter中按照B16处理，mask控制RegTensor中的数据量，Index Tensor控制UB中的数据量，所以B8时mask需要翻倍521 // B8时Gather/Scatter中按照B16处理，mask控制RegTensor中的数据量，Index Tensor控制UB中的数据量，所以B8时mask需要翻倍
@@ -546,14 +546,14 @@ __aicore__ inline void GatherCum<T>::CumGatherTDRightA(const LocalTensor<T>& inL
546 546 
547 // get backup result of last loop547 // get backup result of last loop
548 Reg::Arange(gIdx, bakBegAddr);548 Reg::Arange(gIdx, bakBegAddr);
549- Reg::DataCopyGather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, mask);549+ Reg::Gather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, mask);
550 550 
551 Reg::Arange(gIdx, srcBegIdx);551 Reg::Arange(gIdx, srcBegIdx);
552 Reg::Arange(sIdx, dstBegIdx);552 Reg::Arange(sIdx, dstBegIdx);
553 for (uint16_t rIdx = 0; rIdx < rLpCnt; rIdx++) {553 for (uint16_t rIdx = 0; rIdx < rLpCnt; rIdx++) {
554- Reg::DataCopyGather((Reg::RegTensor<CastType>&)xSrc, srcPtr, (Reg::RegTensor<IdxType>&)gIdx, mask);554+ Reg::Gather((Reg::RegTensor<CastType>&)xSrc, srcPtr, (Reg::RegTensor<IdxType>&)gIdx, mask);
555 Reg::Add(yDst, yDst, xSrc, mask);555 Reg::Add(yDst, yDst, xSrc, mask);
556- Reg::DataCopyScatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, mask);556+ Reg::Scatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, mask);
557 Reg::Adds(gIdx, gIdx, lpOffset, idxMask);557 Reg::Adds(gIdx, gIdx, lpOffset, idxMask);
558 Reg::Adds(sIdx, sIdx, lpOffset, idxMask);558 Reg::Adds(sIdx, sIdx, lpOffset, idxMask);
559 }559 }
@@ -669,7 +669,7 @@ __aicore__ inline void GatherCum<T>::GatherNoSplitRProcess(
669}669}
670 670 
671template <typename T>671template <typename T>
672-__aicore__ inline void GatherCum<T>::CleanInputDirtyData(__local_mem__ T*& srcPtr, int32_t zeroBase, uint32_t leftSize)672+__aicore__ inline void GatherCum<T>::CleanInputDirtyData(__ubuf__ T*& srcPtr, int32_t zeroBase, uint32_t leftSize)
673{673{
674 uint16_t lpCnt = Ops::Base::CeilDiv(leftSize, regElem);674 uint16_t lpCnt = Ops::Base::CeilDiv(leftSize, regElem);
675 if constexpr (sizeof(T) == 1) {675 if constexpr (sizeof(T) == 1) {
@@ -685,18 +685,20 @@ __aicore__ inline void GatherCum<T>::CleanInputDirtyData(__local_mem__ T*& srcPt
685 Reg::MaskReg idxMask = Reg::CreateMask<RangeType, Reg::MaskPattern::ALL>();685 Reg::MaskReg idxMask = Reg::CreateMask<RangeType, Reg::MaskPattern::ALL>();
686 for (uint16_t i = 0; i < lpCnt; i++) {686 for (uint16_t i = 0; i < lpCnt; i++) {
687 mask = Reg::UpdateMask<T>(leftSize);687 mask = Reg::UpdateMask<T>(leftSize);
688- Reg::DataCopyScatter(srcPtr, zeros, (Reg::RegTensor<IdxType>&)sIdx, mask);688+ Reg::Scatter(srcPtr, zeros, (Reg::RegTensor<IdxType>&)sIdx, mask);
689 Reg::Adds(sIdx, sIdx, regElem, idxMask);689 Reg::Adds(sIdx, sIdx, regElem, idxMask);
690 }690 }
691 }691 }
692}692}
693 693 
694template <typename T>694template <typename T>
695-__aicore__ inline void GatherCum<T>::TDRReverseProcess(695+__aicore__ inline void GatherCum<T>::TDRReverseProcess(__ubuf__ T*& srcPtr, __ubuf__ T*& dstPtr, uint16_t rLpNum,
696- __local_mem__ T*& srcPtr, __local_mem__ T*& dstPtr, uint16_t rLpNum, int32_t comRightA, int32_t splitRASize,696+ int32_t comRightA, int32_t splitRASize, int32_t curBegAddrBase,
697- int32_t curBegAddrBase, int32_t curBakBegAdd, uint16_t sndRLpCnt, int32_t phs1SrcBegIdx, int32_t phs2SrcBegIdx,697+ int32_t curBakBegAdd, uint16_t sndRLpCnt, int32_t phs1SrcBegIdx,
698- int32_t phs1LpOffset, int32_t phs2LpOffset, int32_t phs2TLpOffset, int32_t bakBegAddr, int32_t phs1DstBegIdx,698+ int32_t phs2SrcBegIdx, int32_t phs1LpOffset,
699- int32_t phs2DstBegIdx, uint32_t comLpMaskVal, uint32_t sndLpLeftMaskVal)699+ int32_t phs2LpOffset, int32_t phs2TLpOffset, int32_t bakBegAddr,
700+ int32_t phs1DstBegIdx, int32_t phs2DstBegIdx,
701+ uint32_t comLpMaskVal, uint32_t sndLpLeftMaskVal)
700{702{
701 int32_t rightALen = tilingPtr->raLpUnit;703 int32_t rightALen = tilingPtr->raLpUnit;
702 __VEC_SCOPE__704 __VEC_SCOPE__
@@ -718,7 +720,7 @@ __aicore__ inline void GatherCum<T>::TDRReverseProcess(
718 Reg::Muls(tmp1, tmp1, (RangeType)rightALen, idxMask); // tmp1 is i//k*k720 Reg::Muls(tmp1, tmp1, (RangeType)rightALen, idxMask); // tmp1 is i//k*k
719 Reg::Sub(gIdx, gIdx, tmp1, idxMask);721 Reg::Sub(gIdx, gIdx, tmp1, idxMask);
720 // the index result format is: (0, 1, 2, 0, 1, 2, 0, 1, 2, ...), concat last loop backup result722 // the index result format is: (0, 1, 2, 0, 1, 2, 0, 1, 2, ...), concat last loop backup result
721- Reg::Copy(gComIdx, gIdx); // backup gComIdx723+ Reg::Move(gComIdx, gIdx); // backup gComIdx
722 // the index result format is: (0, 1, 2, 6, 7, 8, ...)724 // the index result format is: (0, 1, 2, 6, 7, 8, ...)
723 Reg::Add(gIdx, gIdx, tmp2, idxMask); // backup gIdx725 Reg::Add(gIdx, gIdx, tmp2, idxMask); // backup gIdx
724 726 
@@ -727,27 +729,27 @@ __aicore__ inline void GatherCum<T>::TDRReverseProcess(
727 Reg::RegTensor<RangeType> gBakIdx;729 Reg::RegTensor<RangeType> gBakIdx;
728 Reg::Adds(gBakIdx, gComIdx, bakBegAddr, idxMask);730 Reg::Adds(gBakIdx, gComIdx, bakBegAddr, idxMask);
729 Reg::MaskReg comMask = Reg::UpdateMask<T>(comLpMaskVal);731 Reg::MaskReg comMask = Reg::UpdateMask<T>(comLpMaskVal);
730- Reg::DataCopyGather((Reg::RegTensor<CastType>&)bakSrc, dstPtr, (Reg::RegTensor<IdxType>&)gBakIdx, comMask);732+ Reg::Gather((Reg::RegTensor<CastType>&)bakSrc, dstPtr, (Reg::RegTensor<IdxType>&)gBakIdx, comMask);
731 Reg::LocalMemBar<Reg::MemType::VEC_LOAD, Reg::MemType::VEC_STORE>();733 Reg::LocalMemBar<Reg::MemType::VEC_LOAD, Reg::MemType::VEC_STORE>();
732 734 
733 // first operation, split R735 // first operation, split R
734 Reg::Duplicate(yDst, (T)0, comMask);736 Reg::Duplicate(yDst, (T)0, comMask);
735- Reg::Copy(sIdx, gIdx);737+ Reg::Move(sIdx, gIdx);
736 Reg::Adds(gIdx, gIdx, phs1SrcBegIdx, idxMask);738 Reg::Adds(gIdx, gIdx, phs1SrcBegIdx, idxMask);
737 Reg::Adds(sIdx, sIdx, phs1DstBegIdx, idxMask);739 Reg::Adds(sIdx, sIdx, phs1DstBegIdx, idxMask);
738 for (uint16_t rIdx = 0; rIdx < rLpNum; rIdx++) {740 for (uint16_t rIdx = 0; rIdx < rLpNum; rIdx++) {
739- Reg::DataCopyGather((Reg::RegTensor<CastType>&)xSrc, srcPtr, (Reg::RegTensor<IdxType>&)gIdx, comMask);741+ Reg::Gather((Reg::RegTensor<CastType>&)xSrc, srcPtr, (Reg::RegTensor<IdxType>&)gIdx, comMask);
740 Reg::Add(yDst, yDst, xSrc, comMask);742 Reg::Add(yDst, yDst, xSrc, comMask);
741- Reg::DataCopyScatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, comMask);743+ Reg::Scatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, comMask);
742 Reg::Adds(gIdx, gIdx, phs1LpOffset, idxMask);744 Reg::Adds(gIdx, gIdx, phs1LpOffset, idxMask);
743 Reg::Adds(sIdx, sIdx, phs1LpOffset, idxMask);745 Reg::Adds(sIdx, sIdx, phs1LpOffset, idxMask);
744 }746 }
745 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();747 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
746 748 
747 // second operation, get result749 // second operation, get result
748- Reg::Copy(xSrc, bakSrc);750+ Reg::Move(xSrc, bakSrc);
749 Reg::Arange(sIdx, (RangeType)0);751 Reg::Arange(sIdx, (RangeType)0);
750- Reg::Copy(gIdx, sIdx);752+ Reg::Move(gIdx, sIdx);
751 Reg::Adds(gIdx, gIdx, phs2SrcBegIdx, idxMask);753 Reg::Adds(gIdx, gIdx, phs2SrcBegIdx, idxMask);
752 Reg::Adds(sIdx, sIdx, phs2DstBegIdx, idxMask);754 Reg::Adds(sIdx, sIdx, phs2DstBegIdx, idxMask);
753 // for exclusive is true755 // for exclusive is true
@@ -758,45 +760,47 @@ __aicore__ inline void GatherCum<T>::TDRReverseProcess(
758 for (uint16_t srIdx = 0; srIdx < sndRLpCnt; srIdx++) {760 for (uint16_t srIdx = 0; srIdx < sndRLpCnt; srIdx++) {
759 Reg::Adds(gIdx, gIdx, phs2LpOffset, idxMask);761 Reg::Adds(gIdx, gIdx, phs2LpOffset, idxMask);
760 Reg::Adds(sIdx, sIdx, phs2LpOffset, idxMask);762 Reg::Adds(sIdx, sIdx, phs2LpOffset, idxMask);
761- Reg::DataCopyGather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, comMask);763+ Reg::Gather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, comMask);
762 Reg::Add(yDst, yDst, xSrc, comMask);764 Reg::Add(yDst, yDst, xSrc, comMask);
763- Reg::DataCopyScatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, comMask);765+ Reg::Scatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, comMask);
764 }766 }
765 Reg::Adds(gIdx, gIdx, phs2TLpOffset, idxMask);767 Reg::Adds(gIdx, gIdx, phs2TLpOffset, idxMask);
766 Reg::Adds(sIdx, sIdx, phs2TLpOffset, idxMask);768 Reg::Adds(sIdx, sIdx, phs2TLpOffset, idxMask);
767- Reg::DataCopyGather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, sndLpLeftMask);769+ Reg::Gather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, sndLpLeftMask);
768 Reg::Add(yDst, yDst, xSrc, sndLpLeftMask);770 Reg::Add(yDst, yDst, xSrc, sndLpLeftMask);
769- Reg::DataCopyScatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, sndLpLeftMask);771+ Reg::Scatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, sndLpLeftMask);
770 Reg::Adds(gComIdx, gComIdx, curBegAddrBase, idxMask);772 Reg::Adds(gComIdx, gComIdx, curBegAddrBase, idxMask);
771 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();773 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
772- Reg::DataCopyGather((Reg::RegTensor<CastType>&)xSrc, dstPtr, (Reg::RegTensor<IdxType>&)gComIdx, comMask);774+ Reg::Gather((Reg::RegTensor<CastType>&)xSrc, dstPtr, (Reg::RegTensor<IdxType>&)gComIdx, comMask);
773 775 
774 for (uint16_t comIdx = 1; comIdx < (uint16_t)comRightA; comIdx++) {776 for (uint16_t comIdx = 1; comIdx < (uint16_t)comRightA; comIdx++) {
775 for (uint16_t srIdx = 0; srIdx < sndRLpCnt; srIdx++) {777 for (uint16_t srIdx = 0; srIdx < sndRLpCnt; srIdx++) {
776 Reg::Adds(gIdx, gIdx, phs2LpOffset, idxMask);778 Reg::Adds(gIdx, gIdx, phs2LpOffset, idxMask);
777 Reg::Adds(sIdx, sIdx, phs2LpOffset, idxMask);779 Reg::Adds(sIdx, sIdx, phs2LpOffset, idxMask);
778- Reg::DataCopyGather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, comMask);780+ Reg::Gather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, comMask);
779 Reg::Add(yDst, yDst, xSrc, comMask);781 Reg::Add(yDst, yDst, xSrc, comMask);
780- Reg::DataCopyScatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, comMask);782+ Reg::Scatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, comMask);
781 }783 }
782 Reg::Adds(gIdx, gIdx, phs2TLpOffset, idxMask);784 Reg::Adds(gIdx, gIdx, phs2TLpOffset, idxMask);
783 Reg::Adds(sIdx, sIdx, phs2TLpOffset, idxMask);785 Reg::Adds(sIdx, sIdx, phs2TLpOffset, idxMask);
784- Reg::DataCopyGather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, sndLpLeftMask);786+ Reg::Gather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, sndLpLeftMask);
785 Reg::Add(yDst, yDst, xSrc, sndLpLeftMask);787 Reg::Add(yDst, yDst, xSrc, sndLpLeftMask);
786- Reg::DataCopyScatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, sndLpLeftMask);788+ Reg::Scatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, sndLpLeftMask);
787 Reg::Adds(gComIdx, gComIdx, curBakBegAdd, idxMask);789 Reg::Adds(gComIdx, gComIdx, curBakBegAdd, idxMask);
788 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();790 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
789- Reg::DataCopyGather((Reg::RegTensor<CastType>&)xSrc, dstPtr, (Reg::RegTensor<IdxType>&)gComIdx, comMask);791+ Reg::Gather((Reg::RegTensor<CastType>&)xSrc, dstPtr, (Reg::RegTensor<IdxType>&)gComIdx, comMask);
790 }792 }
791 }793 }
792}794}
793 795 
794template <typename T>796template <typename T>
795-__aicore__ inline void GatherCum<T>::TDRForwardProcess(797+__aicore__ inline void GatherCum<T>::TDRForwardProcess(__ubuf__ T*& srcPtr, __ubuf__ T*& dstPtr, uint16_t rLpNum,
796- __local_mem__ T*& srcPtr, __local_mem__ T*& dstPtr, uint16_t rLpNum, int32_t comRightA, int32_t splitRASize,798+ int32_t comRightA, int32_t splitRASize, int32_t curBegAddrBase,
797- int32_t curBegAddrBase, int32_t curBakBegAdd, uint16_t sndRLpCnt, int32_t phs1SrcBegIdx, int32_t phs2SrcBegIdx,799+ int32_t curBakBegAdd, uint16_t sndRLpCnt, int32_t phs1SrcBegIdx,
798- int32_t phs1LpOffset, int32_t phs2LpOffset, int32_t phs2TLpOffset, int32_t bakBegAddr, int32_t phs1DstBegIdx,800+ int32_t phs2SrcBegIdx, int32_t phs1LpOffset,
799- int32_t phs2DstBegIdx, uint32_t comLpMaskVal, uint32_t sndLpLeftMaskVal)801+ int32_t phs2LpOffset, int32_t phs2TLpOffset, int32_t bakBegAddr,
802+ int32_t phs1DstBegIdx, int32_t phs2DstBegIdx,
803+ uint32_t comLpMaskVal, uint32_t sndLpLeftMaskVal)
800{804{
801 int32_t rightALen = tilingPtr->raLpUnit;805 int32_t rightALen = tilingPtr->raLpUnit;
802 __VEC_SCOPE__806 __VEC_SCOPE__
@@ -818,7 +822,7 @@ __aicore__ inline void GatherCum<T>::TDRForwardProcess(
818 Reg::Muls(tmp1, tmp1, (RangeType)rightALen, idxMask); // tmp1 is i//k*k822 Reg::Muls(tmp1, tmp1, (RangeType)rightALen, idxMask); // tmp1 is i//k*k
819 Reg::Sub(gIdx, gIdx, tmp1, idxMask);823 Reg::Sub(gIdx, gIdx, tmp1, idxMask);
820 // the index result format is: (0, 1, 2, 0, 1, 2, 0, 1, 2, ...), concat last loop backup result824 // the index result format is: (0, 1, 2, 0, 1, 2, 0, 1, 2, ...), concat last loop backup result
821- Reg::Copy(gComIdx, gIdx); // backup gComIdx825+ Reg::Move(gComIdx, gIdx); // backup gComIdx
822 // the index result format is: (0, 1, 2, 6, 7, 8, ...)826 // the index result format is: (0, 1, 2, 6, 7, 8, ...)
823 Reg::Add(gIdx, gIdx, tmp2, idxMask); // backup gIdx827 Reg::Add(gIdx, gIdx, tmp2, idxMask); // backup gIdx
824 828 
@@ -827,27 +831,27 @@ __aicore__ inline void GatherCum<T>::TDRForwardProcess(
827 Reg::RegTensor<RangeType> gBakIdx;831 Reg::RegTensor<RangeType> gBakIdx;
828 Reg::Adds(gBakIdx, gComIdx, bakBegAddr, idxMask);832 Reg::Adds(gBakIdx, gComIdx, bakBegAddr, idxMask);
829 Reg::MaskReg comMask = Reg::UpdateMask<T>(comLpMaskVal);833 Reg::MaskReg comMask = Reg::UpdateMask<T>(comLpMaskVal);
830- Reg::DataCopyGather((Reg::RegTensor<CastType>&)bakSrc, dstPtr, (Reg::RegTensor<IdxType>&)gBakIdx, comMask);834+ Reg::Gather((Reg::RegTensor<CastType>&)bakSrc, dstPtr, (Reg::RegTensor<IdxType>&)gBakIdx, comMask);
831 Reg::LocalMemBar<Reg::MemType::VEC_LOAD, Reg::MemType::VEC_STORE>();835 Reg::LocalMemBar<Reg::MemType::VEC_LOAD, Reg::MemType::VEC_STORE>();
832 836 
833 // first operation, split R837 // first operation, split R
834 Reg::Duplicate(yDst, (T)0, comMask);838 Reg::Duplicate(yDst, (T)0, comMask);
835- Reg::Copy(sIdx, gIdx);839+ Reg::Move(sIdx, gIdx);
836 Reg::Adds(gIdx, gIdx, phs1SrcBegIdx, idxMask);840 Reg::Adds(gIdx, gIdx, phs1SrcBegIdx, idxMask);
837 Reg::Adds(sIdx, sIdx, phs1DstBegIdx, idxMask);841 Reg::Adds(sIdx, sIdx, phs1DstBegIdx, idxMask);
838 for (uint16_t rIdx = 0; rIdx < rLpNum; rIdx++) {842 for (uint16_t rIdx = 0; rIdx < rLpNum; rIdx++) {
839- Reg::DataCopyGather((Reg::RegTensor<CastType>&)xSrc, srcPtr, (Reg::RegTensor<IdxType>&)gIdx, comMask);843+ Reg::Gather((Reg::RegTensor<CastType>&)xSrc, srcPtr, (Reg::RegTensor<IdxType>&)gIdx, comMask);
840 Reg::Add(yDst, yDst, xSrc, comMask);844 Reg::Add(yDst, yDst, xSrc, comMask);
841- Reg::DataCopyScatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, comMask);845+ Reg::Scatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, comMask);
842 Reg::Adds(gIdx, gIdx, phs1LpOffset, idxMask);846 Reg::Adds(gIdx, gIdx, phs1LpOffset, idxMask);
843 Reg::Adds(sIdx, sIdx, phs1LpOffset, idxMask);847 Reg::Adds(sIdx, sIdx, phs1LpOffset, idxMask);
844 }848 }
845 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();849 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
846 850 
847 // second operation, get result851 // second operation, get result
848- Reg::Copy(xSrc, bakSrc);852+ Reg::Move(xSrc, bakSrc);
849 Reg::Arange(sIdx, (RangeType)0);853 Reg::Arange(sIdx, (RangeType)0);
850- Reg::Copy(gIdx, sIdx);854+ Reg::Move(gIdx, sIdx);
851 Reg::Adds(gIdx, gIdx, phs2SrcBegIdx, idxMask);855 Reg::Adds(gIdx, gIdx, phs2SrcBegIdx, idxMask);
852 Reg::Adds(sIdx, sIdx, phs2DstBegIdx, idxMask);856 Reg::Adds(sIdx, sIdx, phs2DstBegIdx, idxMask);
853 // for exclusive is true857 // for exclusive is true
@@ -856,37 +860,37 @@ __aicore__ inline void GatherCum<T>::TDRForwardProcess(
856 860 
857 // first split RA of second phase861 // first split RA of second phase
858 for (uint16_t srIdx = 0; srIdx < sndRLpCnt; srIdx++) {862 for (uint16_t srIdx = 0; srIdx < sndRLpCnt; srIdx++) {
859- Reg::DataCopyGather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, comMask);863+ Reg::Gather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, comMask);
860 Reg::Add(yDst, yDst, xSrc, comMask);864 Reg::Add(yDst, yDst, xSrc, comMask);
861- Reg::DataCopyScatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, comMask);865+ Reg::Scatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, comMask);
862 Reg::Adds(gIdx, gIdx, phs2LpOffset, idxMask);866 Reg::Adds(gIdx, gIdx, phs2LpOffset, idxMask);
863 Reg::Adds(sIdx, sIdx, phs2LpOffset, idxMask);867 Reg::Adds(sIdx, sIdx, phs2LpOffset, idxMask);
864 }868 }
865- Reg::DataCopyGather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, sndLpLeftMask);869+ Reg::Gather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, sndLpLeftMask);
866 Reg::Add(yDst, yDst, xSrc, sndLpLeftMask);870 Reg::Add(yDst, yDst, xSrc, sndLpLeftMask);
867- Reg::DataCopyScatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, sndLpLeftMask);871+ Reg::Scatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, sndLpLeftMask);
868 Reg::Adds(gIdx, gIdx, phs2TLpOffset, idxMask);872 Reg::Adds(gIdx, gIdx, phs2TLpOffset, idxMask);
869 Reg::Adds(sIdx, sIdx, phs2TLpOffset, idxMask);873 Reg::Adds(sIdx, sIdx, phs2TLpOffset, idxMask);
870 Reg::Adds(gComIdx, gComIdx, curBegAddrBase, idxMask);874 Reg::Adds(gComIdx, gComIdx, curBegAddrBase, idxMask);
871 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();875 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
872- Reg::DataCopyGather((Reg::RegTensor<CastType>&)xSrc, dstPtr, (Reg::RegTensor<IdxType>&)gComIdx, comMask);876+ Reg::Gather((Reg::RegTensor<CastType>&)xSrc, dstPtr, (Reg::RegTensor<IdxType>&)gComIdx, comMask);
873 877 
874 for (uint16_t comIdx = 1; comIdx < (uint16_t)comRightA; comIdx++) {878 for (uint16_t comIdx = 1; comIdx < (uint16_t)comRightA; comIdx++) {
875 for (uint16_t srIdx = 0; srIdx < sndRLpCnt; srIdx++) {879 for (uint16_t srIdx = 0; srIdx < sndRLpCnt; srIdx++) {
876- Reg::DataCopyGather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, comMask);880+ Reg::Gather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, comMask);
877 Reg::Add(yDst, yDst, xSrc, comMask);881 Reg::Add(yDst, yDst, xSrc, comMask);
878- Reg::DataCopyScatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, comMask);882+ Reg::Scatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, comMask);
879 Reg::Adds(gIdx, gIdx, phs2LpOffset, idxMask);883 Reg::Adds(gIdx, gIdx, phs2LpOffset, idxMask);
880 Reg::Adds(sIdx, sIdx, phs2LpOffset, idxMask);884 Reg::Adds(sIdx, sIdx, phs2LpOffset, idxMask);
881 }885 }
882- Reg::DataCopyGather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, sndLpLeftMask);886+ Reg::Gather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gIdx, sndLpLeftMask);
883 Reg::Add(yDst, yDst, xSrc, sndLpLeftMask);887 Reg::Add(yDst, yDst, xSrc, sndLpLeftMask);
884- Reg::DataCopyScatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, sndLpLeftMask);888+ Reg::Scatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, sndLpLeftMask);
885 Reg::Adds(gIdx, gIdx, phs2TLpOffset, idxMask);889 Reg::Adds(gIdx, gIdx, phs2TLpOffset, idxMask);
886 Reg::Adds(sIdx, sIdx, phs2TLpOffset, idxMask);890 Reg::Adds(sIdx, sIdx, phs2TLpOffset, idxMask);
887 Reg::Adds(gComIdx, gComIdx, curBakBegAdd, idxMask);891 Reg::Adds(gComIdx, gComIdx, curBakBegAdd, idxMask);
888 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();892 Reg::LocalMemBar<Reg::MemType::VEC_STORE, Reg::MemType::VEC_LOAD>();
889- Reg::DataCopyGather((Reg::RegTensor<CastType>&)xSrc, dstPtr, (Reg::RegTensor<IdxType>&)gComIdx, comMask);893+ Reg::Gather((Reg::RegTensor<CastType>&)xSrc, dstPtr, (Reg::RegTensor<IdxType>&)gComIdx, comMask);
890 }894 }
891 }895 }
892}896}
@@ -904,8 +908,8 @@ __aicore__ inline void GatherCum<T>::CumGatherRightATDR(const LocalTensor<T>& in
904 * 3. The rLen will be split into multiple parts.908 * 3. The rLen will be split into multiple parts.
905 */909 */
906 910 
907- __local_mem__ T* srcPtr = (__local_mem__ T*)inLocal.GetPhyAddr();911+ __ubuf__ T* srcPtr = (__ubuf__ T*)inLocal.GetPhyAddr();
908- __local_mem__ T* dstPtr = (__local_mem__ T*)outLocal.GetPhyAddr();912+ __ubuf__ T* dstPtr = (__ubuf__ T*)outLocal.GetPhyAddr();
909 913 
910 int32_t rightALen = tilingPtr->raLpUnit;914 int32_t rightALen = tilingPtr->raLpUnit;
911 uint32_t tmp = (regElem / rightALen > rLen) ? rLen : regElem / rightALen;915 uint32_t tmp = (regElem / rightALen > rLen) ? rLen : regElem / rightALen;
@@ -1001,8 +1005,8 @@ __aicore__ inline void GatherCum<T>::CumGatherRightATDLeftA(const LocalTensor<T>
1001 * 3. The laOffset always be main loop size of RA.1005 * 3. The laOffset always be main loop size of RA.
1002 */1006 */
1003 1007 
1004- __local_mem__ T* srcPtr = (__local_mem__ T*)inLocal.GetPhyAddr();1008+ __ubuf__ T* srcPtr = (__ubuf__ T*)inLocal.GetPhyAddr();
1005- __local_mem__ T* dstPtr = (__local_mem__ T*)outLocal.GetPhyAddr();1009+ __ubuf__ T* dstPtr = (__ubuf__ T*)outLocal.GetPhyAddr();
1006 uint16_t rLpCnt = (uint16_t)rLen;1010 uint16_t rLpCnt = (uint16_t)rLen;
1007 uint32_t rightALen = tilingPtr->raLpUnit;1011 uint32_t rightALen = tilingPtr->raLpUnit;
1008 uint32_t comRightACnt = (regElem / rightALen > leftALen) ? leftALen : regElem / rightALen;1012 uint32_t comRightACnt = (regElem / rightALen > leftALen) ? leftALen : regElem / rightALen;
@@ -1057,9 +1061,9 @@ __aicore__ inline void GatherCum<T>::CumGatherRightATDLeftA(const LocalTensor<T>
1057 Reg::Muls(tmp1, tmp1, (RangeType)rightALen, idxMask);1061 Reg::Muls(tmp1, tmp1, (RangeType)rightALen, idxMask);
1058 Reg::Sub(gIdx, gIdx, tmp1, idxMask);1062 Reg::Sub(gIdx, gIdx, tmp1, idxMask);
1059 Reg::Add(gIdx, gIdx, tmp2, idxMask); // backup gIdx1063 Reg::Add(gIdx, gIdx, tmp2, idxMask); // backup gIdx
1060- Reg::Copy(gBakIdx, gIdx);1064+ Reg::Move(gBakIdx, gIdx);
1061- Reg::Copy(sIdx, gIdx);1065+ Reg::Move(sIdx, gIdx);
1062- Reg::Copy(sBakIdx, gIdx);1066+ Reg::Move(sBakIdx, gIdx);
1063 Reg::Adds(gBakIdx, gBakIdx, bakBegAddr, idxMask);1067 Reg::Adds(gBakIdx, gBakIdx, bakBegAddr, idxMask);
1064 1068 
1065 Reg::Adds(gIdx, gIdx, srcBegAddr, idxMask);1069 Reg::Adds(gIdx, gIdx, srcBegAddr, idxMask);
@@ -1068,26 +1072,26 @@ __aicore__ inline void GatherCum<T>::CumGatherRightATDLeftA(const LocalTensor<T>
1068 Reg::MaskReg tMask = Reg::UpdateMask<T>(tailComSize);1072 Reg::MaskReg tMask = Reg::UpdateMask<T>(tailComSize);
1069 for (uint16_t aIdx = 0; aIdx < aMainLpCnt; aIdx++) {1073 for (uint16_t aIdx = 0; aIdx < aMainLpCnt; aIdx++) {
1070 // get backup result of last loop1074 // get backup result of last loop
1071- Reg::DataCopyGather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gBakIdx, mMask);1075+ Reg::Gather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gBakIdx, mMask);
1072 for (uint16_t rIdx = 0; rIdx < rLpCnt; rIdx++) {1076 for (uint16_t rIdx = 0; rIdx < rLpCnt; rIdx++) {
1073- Reg::DataCopyGather((Reg::RegTensor<CastType>&)xSrc, srcPtr, (Reg::RegTensor<IdxType>&)gIdx, mMask);1077+ Reg::Gather((Reg::RegTensor<CastType>&)xSrc, srcPtr, (Reg::RegTensor<IdxType>&)gIdx, mMask);
1074 Reg::Add(yDst, yDst, xSrc, mMask);1078 Reg::Add(yDst, yDst, xSrc, mMask);
1075- Reg::DataCopyScatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, mMask);1079+ Reg::Scatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, mMask);
1076 Reg::Adds(gIdx, gIdx, lpOffset, idxMask);1080 Reg::Adds(gIdx, gIdx, lpOffset, idxMask);
1077 Reg::Adds(sIdx, sIdx, lpOffset, idxMask);1081 Reg::Adds(sIdx, sIdx, lpOffset, idxMask);
1078 }1082 }
1079 Reg::Adds(gBakIdx, gBakIdx, aLpIdxOffset, idxMask);1083 Reg::Adds(gBakIdx, gBakIdx, aLpIdxOffset, idxMask);
1080 Reg::Adds(sBakIdx, sBakIdx, aLpIdxOffset, idxMask);1084 Reg::Adds(sBakIdx, sBakIdx, aLpIdxOffset, idxMask);
1081- Reg::Copy(gIdx, sBakIdx);1085+ Reg::Move(gIdx, sBakIdx);
1082 Reg::Adds(sIdx, gIdx, dstBegIdx, idxMask);1086 Reg::Adds(sIdx, gIdx, dstBegIdx, idxMask);
1083 Reg::Adds(gIdx, gIdx, srcBegAddr, idxMask);1087 Reg::Adds(gIdx, gIdx, srcBegAddr, idxMask);
1084 }1088 }
1085 1089 
1086- Reg::DataCopyGather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gBakIdx, tMask);1090+ Reg::Gather((Reg::RegTensor<CastType>&)yDst, dstPtr, (Reg::RegTensor<IdxType>&)gBakIdx, tMask);
1087 for (uint16_t rIdx = 0; rIdx < rLpCnt; rIdx++) {1091 for (uint16_t rIdx = 0; rIdx < rLpCnt; rIdx++) {
1088- Reg::DataCopyGather((Reg::RegTensor<CastType>&)xSrc, srcPtr, (Reg::RegTensor<IdxType>&)gIdx, tMask);1092+ Reg::Gather((Reg::RegTensor<CastType>&)xSrc, srcPtr, (Reg::RegTensor<IdxType>&)gIdx, tMask);
1089 Reg::Add(yDst, yDst, xSrc, tMask);1093 Reg::Add(yDst, yDst, xSrc, tMask);
1090- Reg::DataCopyScatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, tMask);1094+ Reg::Scatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)sIdx, tMask);
1091 Reg::Adds(gIdx, gIdx, lpOffset, idxMask);1095 Reg::Adds(gIdx, gIdx, lpOffset, idxMask);
1092 Reg::Adds(sIdx, sIdx, lpOffset, idxMask);1096 Reg::Adds(sIdx, sIdx, lpOffset, idxMask);
1093 }1097 }
@@ -1433,8 +1437,8 @@ __aicore__ inline void CumWithGroup<T>::GetPrevCoresResult(LocalTensor<T>& midUB
1433 InsertSync(HardEvent::MTE2_V);1437 InsertSync(HardEvent::MTE2_V);
1434 1438 
1435 // rightA in UB is block align1439 // rightA in UB is block align
1436- __local_mem__ T* midPtr = (__local_mem__ T*)midUB.GetPhyAddr();1440+ __ubuf__ T* midPtr = (__ubuf__ T*)midUB.GetPhyAddr();
1437- __local_mem__ T* inPtr = (__local_mem__ T*)inUB.GetPhyAddr();1441+ __ubuf__ T* inPtr = (__ubuf__ T*)inUB.GetPhyAddr();
1438 uint32_t rightALen = raSize;1442 uint32_t rightALen = raSize;
1439 uint16_t raLpCnt_ = Ops::Base::CeilDiv(raSize, realRegElem);1443 uint16_t raLpCnt_ = Ops::Base::CeilDiv(raSize, realRegElem);
1440 __VEC_SCOPE__1444 __VEC_SCOPE__
@@ -1446,10 +1450,10 @@ __aicore__ inline void CumWithGroup<T>::GetPrevCoresResult(LocalTensor<T>& midUB
1446 for (uint16_t raIdx = 0; raIdx < raLpCnt_; raIdx++) {1450 for (uint16_t raIdx = 0; raIdx < raLpCnt_; raIdx++) {
1447 mask = Reg::UpdateMask<T>(rightALen);1451 mask = Reg::UpdateMask<T>(rightALen);
1448 for (uint16_t idx = 0; idx < prevCoreCnt; idx++) {1452 for (uint16_t idx = 0; idx < prevCoreCnt; idx++) {
1449- Reg::DataCopy(src, inPtr + raIdx * realRegElem + idx * raBA);1453+ Reg::LoadAlign(src, inPtr + raIdx * realRegElem + idx * raBA);
1450 Reg::Add(dst, dst, src, mask);1454 Reg::Add(dst, dst, src, mask);
1451 }1455 }
1452- Reg::DataCopy<T>(midPtr + raIdx * realRegElem, dst, mask);1456+ Reg::StoreAlign<T>(midPtr + raIdx * realRegElem, dst, mask);
1453 }1457 }
1454 }1458 }
1455}1459}
@@ -1458,9 +1462,9 @@ template <typename T>
1458__aicore__ inline void CumWithGroup<T>::CrossCoreCum(const LocalTensor<T>& inUB, const LocalTensor<T>& midUB,1462__aicore__ inline void CumWithGroup<T>::CrossCoreCum(const LocalTensor<T>& inUB, const LocalTensor<T>& midUB,
1459 LocalTensor<T>& outUB, uint32_t rLen, uint32_t raSize)1463 LocalTensor<T>& outUB, uint32_t rLen, uint32_t raSize)
1460{1464{
1461- __local_mem__ T* srcPtr = (__local_mem__ T*)inUB.GetPhyAddr();1465+ __ubuf__ T* srcPtr = (__ubuf__ T*)inUB.GetPhyAddr();
1462- __local_mem__ T* midPtr = (__local_mem__ T*)midUB.GetPhyAddr();1466+ __ubuf__ T* midPtr = (__ubuf__ T*)midUB.GetPhyAddr();
1463- __local_mem__ T* dstPtr = (__local_mem__ T*)outUB.GetPhyAddr();1467+ __ubuf__ T* dstPtr = (__ubuf__ T*)outUB.GetPhyAddr();
1464 // A must be smaller than VL1468 // A must be smaller than VL
1465 uint32_t rightALen = raSize;1469 uint32_t rightALen = raSize;
1466 uint32_t comA = regElem / rightALen;1470 uint32_t comA = regElem / rightALen;
@@ -1498,17 +1502,17 @@ __aicore__ inline void CumWithGroup<T>::CrossCoreCum(const LocalTensor<T>& inUB,
1498 1502 
1499 Reg::MaskReg mMask = Reg::UpdateMask<T>(mMaskVal);1503 Reg::MaskReg mMask = Reg::UpdateMask<T>(mMaskVal);
1500 // get last R result of previous cores1504 // get last R result of previous cores
1501- Reg::DataCopyGather((Reg::RegTensor<CastType>&)mSrc, midPtr, (Reg::RegTensor<IdxType>&)gBakIdx, mMask);1505+ Reg::Gather((Reg::RegTensor<CastType>&)mSrc, midPtr, (Reg::RegTensor<IdxType>&)gBakIdx, mMask);
1502 for (uint16_t rIdx = 0; rIdx < rLpCnt; rIdx++) {1506 for (uint16_t rIdx = 0; rIdx < rLpCnt; rIdx++) {
1503- Reg::DataCopyGather((Reg::RegTensor<CastType>&)xSrc, srcPtr, (Reg::RegTensor<IdxType>&)gIdx, mMask);1507+ Reg::Gather((Reg::RegTensor<CastType>&)xSrc, srcPtr, (Reg::RegTensor<IdxType>&)gIdx, mMask);
1504 Reg::Add(yDst, xSrc, mSrc, mMask);1508 Reg::Add(yDst, xSrc, mSrc, mMask);
1505- Reg::DataCopyScatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)gIdx, mMask);1509+ Reg::Scatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)gIdx, mMask);
1506 Reg::Adds(gIdx, gIdx, mComA, idxMask);1510 Reg::Adds(gIdx, gIdx, mComA, idxMask);
1507 }1511 }
1508 Reg::MaskReg tMask = Reg::UpdateMask<T>(tMaskVal);1512 Reg::MaskReg tMask = Reg::UpdateMask<T>(tMaskVal);
1509- Reg::DataCopyGather((Reg::RegTensor<CastType>&)xSrc, srcPtr, (Reg::RegTensor<IdxType>&)gIdx, tMask);1513+ Reg::Gather((Reg::RegTensor<CastType>&)xSrc, srcPtr, (Reg::RegTensor<IdxType>&)gIdx, tMask);
1510 Reg::Add(yDst, xSrc, mSrc, tMask);1514 Reg::Add(yDst, xSrc, mSrc, tMask);
1511- Reg::DataCopyScatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)gIdx, tMask);1515+ Reg::Scatter(dstPtr, yDst, (Reg::RegTensor<IdxType>&)gIdx, tMask);
1512 }1516 }
1513}1517}
1514 1518 
@@ -1516,9 +1520,9 @@ template <typename T>
1516__aicore__ inline void CumWithGroup<T>::CrossCoreCum4BigRA(const LocalTensor<T>& inUB, const LocalTensor<T>& midUB,1520__aicore__ inline void CumWithGroup<T>::CrossCoreCum4BigRA(const LocalTensor<T>& inUB, const LocalTensor<T>& midUB,
1517 LocalTensor<T>& outUB, uint32_t rLen, uint32_t raSize)1521 LocalTensor<T>& outUB, uint32_t rLen, uint32_t raSize)
1518{1522{
1519- __local_mem__ T* srcPtr = (__local_mem__ T*)inUB.GetPhyAddr();1523+ __ubuf__ T* srcPtr = (__ubuf__ T*)inUB.GetPhyAddr();
1520- __local_mem__ T* midPtr = (__local_mem__ T*)midUB.GetPhyAddr();1524+ __ubuf__ T* midPtr = (__ubuf__ T*)midUB.GetPhyAddr();
1521- __local_mem__ T* dstPtr = (__local_mem__ T*)outUB.GetPhyAddr();1525+ __ubuf__ T* dstPtr = (__ubuf__ T*)outUB.GetPhyAddr();
1522 1526 
1523 uint32_t rightALen = raSize;1527 uint32_t rightALen = raSize;
1524 uint16_t raLpCnt = (uint16_t)Ops::Base::CeilDiv(raSize, realRegElem);1528 uint16_t raLpCnt = (uint16_t)Ops::Base::CeilDiv(raSize, realRegElem);
@@ -1532,15 +1536,15 @@ __aicore__ inline void CumWithGroup<T>::CrossCoreCum4BigRA(const LocalTensor<T>&
1532 Reg::MaskReg mask;1536 Reg::MaskReg mask;
1533 1537 
1534 // get last R result of previous cores1538 // get last R result of previous cores
1535- Reg::DataCopy(mSrc, midPtr);1539+ Reg::LoadAlign(mSrc, midPtr);
1536 uint32_t offset = 0;1540 uint32_t offset = 0;
1537 for (uint16_t raIdx = 0; raIdx < raLpCnt; raIdx++) {1541 for (uint16_t raIdx = 0; raIdx < raLpCnt; raIdx++) {
1538 mask = Reg::UpdateMask<T>(rightALen);1542 mask = Reg::UpdateMask<T>(rightALen);
1539 for (uint16_t rIdx = 0; rIdx < rLpCnt; rIdx++) {1543 for (uint16_t rIdx = 0; rIdx < rLpCnt; rIdx++) {
1540 offset = raIdx * realRegElem + rIdx * raBA;1544 offset = raIdx * realRegElem + rIdx * raBA;
1541- Reg::DataCopy(xSrc, srcPtr + offset);1545+ Reg::LoadAlign(xSrc, srcPtr + offset);
1542 Reg::Add(yDst, xSrc, mSrc, mask);1546 Reg::Add(yDst, xSrc, mSrc, mask);
1543- Reg::DataCopy(dstPtr + offset, yDst, mask);1547+ Reg::StoreAlign(dstPtr + offset, yDst, mask);
1544 }1548 }
1545 }1549 }
1546 }1550 }
Mmath/cumsum/op_kernel/arch35/cumsum_base/cumsum_core_sklansky.h+13-12
@@ -117,7 +117,7 @@ private:
117 TBuf<> addDataCasted_; // 与前一R分组中的最大值的一行，进行行相加的每一行数据，精度提升后117 TBuf<> addDataCasted_; // 与前一R分组中的最大值的一行，进行行相加的每一行数据，精度提升后
118 TBuf<> addFactorOffset_; // 每次copy addFactor数据的偏移量118 TBuf<> addFactorOffset_; // 每次copy addFactor数据的偏移量
119 119 
120- static constexpr MultiCopyConfig nddmaConfig_ = {false};120+ static constexpr NdDmaConfig nddmaConfig_ = {false};
121}; // class CumsumCoreSklansky121}; // class CumsumCoreSklansky
122 122 
123template <typename T, typename PromtT>123template <typename T, typename PromtT>
@@ -243,7 +243,7 @@ __aicore__ inline void CumsumCoreSklansky<T, PromtT>::CopyInAddFactor(int32_t ss
243 243 
244 // 搬入UB，使用nddma broadcast244 // 搬入UB，使用nddma broadcast
245 NdDmaDci();245 NdDmaDci();
246- MultiCopyParams<T, ADD_FACTOR_NDDMA_DIM> copyParam;246+ NdDmaParams<T, ADD_FACTOR_NDDMA_DIM> copyParam;
247 copyParam.constantValue = 0;247 copyParam.constantValue = 0;
248 copyParam.loopInfo = {248 copyParam.loopInfo = {
249 {1, 0},249 {1, 0},
@@ -482,18 +482,19 @@ __aicore__ inline void CumsumCoreSklansky<T, PromtT>::DoAddTailN(DataCopyExtPara
482 for (uint16_t j = 0; j < addFactorCircleCount_; j++) {482 for (uint16_t j = 0; j < addFactorCircleCount_; j++) {
483 // 搬入addFactor483 // 搬入addFactor
484 AscendC::Reg::AddrReg addrReg = AscendC::Reg::CreateAddrReg<uint32_t>(j, vlElementCount);484 AscendC::Reg::AddrReg addrReg = AscendC::Reg::CreateAddrReg<uint32_t>(j, vlElementCount);
485- AscendC::Reg::DataCopy(addFactorIndexRegTensor, addFactorOffset, addrReg);485+ AscendC::Reg::LoadAlign(addFactorIndexRegTensor, addFactorOffset, addrReg);
486- AscendC::Reg::DataCopyGather(addFactorRegTensor, addFactor, addFactorIndexRegTensor, p0);486+ AscendC::Reg::Gather(addFactorRegTensor, addFactor, addFactorIndexRegTensor, p0);
487 487 
488 // 对齐连续搬入addData488 // 对齐连续搬入addData
489- DataCopy(addDataRegTensor, addData + (i * addFactorCircleCount_ + j) * vlElementCount);489+ MicroAPI::LoadAlign(addDataRegTensor, addData + (i * addFactorCircleCount_ + j) * vlElementCount);
490 490 
491 // Add491 // Add
492 mask = AscendC::Reg::UpdateMask<uint32_t>(addTotal);492 mask = AscendC::Reg::UpdateMask<uint32_t>(addTotal);
493 Add(addDataRegTensor, addFactorRegTensor, addDataRegTensor, mask);493 Add(addDataRegTensor, addFactorRegTensor, addDataRegTensor, mask);
494 494 
495 // 对齐连续搬出addData495 // 对齐连续搬出addData
496- DataCopy(addData + (i * addFactorCircleCount_ + j) * vlElementCount, addDataRegTensor, mask);496+ MicroAPI::StoreAlign(addData + (i * addFactorCircleCount_ + j) * vlElementCount, addDataRegTensor,
497+ mask);
497 }498 }
498 }499 }
499 500 
@@ -501,20 +502,20 @@ __aicore__ inline void CumsumCoreSklansky<T, PromtT>::DoAddTailN(DataCopyExtPara
501 for (uint16_t j = 0; j < addFactorCircleCountTail; j++) {502 for (uint16_t j = 0; j < addFactorCircleCountTail; j++) {
502 // 搬入addFactor503 // 搬入addFactor
503 AscendC::Reg::AddrReg addrReg = AscendC::Reg::CreateAddrReg<uint32_t>(j, vlElementCount);504 AscendC::Reg::AddrReg addrReg = AscendC::Reg::CreateAddrReg<uint32_t>(j, vlElementCount);
504- AscendC::Reg::DataCopy(addFactorIndexRegTensor, addFactorOffset, addrReg);505+ AscendC::Reg::LoadAlign(addFactorIndexRegTensor, addFactorOffset, addrReg);
505- AscendC::Reg::DataCopyGather(addFactorRegTensor, addFactor, addFactorIndexRegTensor, p0);506+ AscendC::Reg::Gather(addFactorRegTensor, addFactor, addFactorIndexRegTensor, p0);
506 507 
507 // 对齐连续搬入addData508 // 对齐连续搬入addData
508- DataCopy(addDataRegTensor,509+ MicroAPI::LoadAlign(addDataRegTensor,
509- addData + ((addFactorCircleLoop - 1) * addFactorCircleCount_ + j) * vlElementCount);510+ addData + ((addFactorCircleLoop - 1) * addFactorCircleCount_ + j) * vlElementCount);
510 511 
511 // Add512 // Add
512 mask = AscendC::Reg::UpdateMask<uint32_t>(addTotalTail);513 mask = AscendC::Reg::UpdateMask<uint32_t>(addTotalTail);
513 Add(addDataRegTensor, addFactorRegTensor, addDataRegTensor, mask);514 Add(addDataRegTensor, addFactorRegTensor, addDataRegTensor, mask);
514 515 
515 // 对齐连续搬出addData516 // 对齐连续搬出addData
516- DataCopy(addData + ((addFactorCircleLoop - 1) * addFactorCircleCount_ + j) * vlElementCount,517+ MicroAPI::StoreAlign(addData + ((addFactorCircleLoop - 1) * addFactorCircleCount_ + j) * vlElementCount,
517- addDataRegTensor, mask);518+ addDataRegTensor, mask);
518 }519 }
519 } // __VEC_SCOPE__520 } // __VEC_SCOPE__
520}521}
Mmath/cumsum/op_kernel/arch35/cumsum_base/cumsum_oneway_sklansky.h+19-14
@@ -59,7 +59,7 @@ private:
59 __aicore__ inline void CopyInExclusive();59 __aicore__ inline void CopyInExclusive();
60 __aicore__ inline void CopyInDataCast();60 __aicore__ inline void CopyInDataCast();
61 __aicore__ inline void Compute();61 __aicore__ inline void Compute();
62- __aicore__ inline void ComputeWithFlag(int32_t addOffset, uint32_t realDupSize, __local_mem__ PromoteDataType* xBuf,62+ __aicore__ inline void ComputeWithFlag(int32_t addOffset, uint32_t realDupSize, __ubuf__ PromoteDataType* xBuf,
63 int32_t flag);63 int32_t flag);
64 __aicore__ inline void ComputeInner(int32_t addLoop, uint32_t realDupSize);64 __aicore__ inline void ComputeInner(int32_t addLoop, uint32_t realDupSize);
65 65 
@@ -219,8 +219,10 @@ __aicore__ inline void CumsumOnewaySklansky<DataType, PromoteDataType>::CopyInDa
219}219}
220 220 
221template <typename DataType, typename PromoteDataType>221template <typename DataType, typename PromoteDataType>
222-__aicore__ inline void CumsumOnewaySklansky<DataType, PromoteDataType>::ComputeWithFlag(222+__aicore__ inline void CumsumOnewaySklansky<DataType, PromoteDataType>::ComputeWithFlag(int32_t addOffset,
223- int32_t addOffset, uint32_t realDupSize, __local_mem__ PromoteDataType* xBuf, int32_t flag)223+ uint32_t realDupSize,
224+ __ubuf__ PromoteDataType* xBuf,
225+ int32_t flag)
224{226{
225 uint16_t groupMainCount = groupMainCount_;227 uint16_t groupMainCount = groupMainCount_;
226 uint16_t groupTailCount = groupTailCount_;228 uint16_t groupTailCount = groupTailCount_;
@@ -242,23 +244,26 @@ __aicore__ inline void CumsumOnewaySklansky<DataType, PromoteDataType>::ComputeW
242 for (uint16_t j = 0; j < nLoop; j++) {244 for (uint16_t j = 0; j < nLoop; j++) {
243 mask = Reg::UpdateMask<uint32_t>(addTotal);245 mask = Reg::UpdateMask<uint32_t>(addTotal);
244 for (uint16_t m = 0; m < groupMainCount; m++) {246 for (uint16_t m = 0; m < groupMainCount; m++) {
245- DataCopy(x1RegTensor, xBuf + j * ADD_ELEM + flag * m * addOffset + i * mFactor);247+ MicroAPI::LoadAlign(x1RegTensor, xBuf + j * ADD_ELEM + flag * m * addOffset + i * mFactor);
246 for (uint16_t n = 1; n <= addCount; n++) {248 for (uint16_t n = 1; n <= addCount; n++) {
247- DataCopy(x2RegTensor,249+ MicroAPI::LoadAlign(
248- xBuf + j * ADD_ELEM + flag * (m * addOffset + n * computeLen) + i * mFactor);250+ x2RegTensor, xBuf + j * ADD_ELEM + flag * (m * addOffset + n * computeLen) + i * mFactor);
249 Add(x2RegTensor, x1RegTensor, x2RegTensor, mask);251 Add(x2RegTensor, x1RegTensor, x2RegTensor, mask);
250- DataCopy(xBuf + j * ADD_ELEM + flag * (m * addOffset + n * computeLen) + i * mFactor,252+ MicroAPI::StoreAlign(
251- x2RegTensor, mask);253+ xBuf + j * ADD_ELEM + flag * (m * addOffset + n * computeLen) + i * mFactor, x2RegTensor,
254+ mask);
252 }255 }
253 }256 }
254 257 
255 for (uint16_t m = 0; m < groupTailCount; m++) {258 for (uint16_t m = 0; m < groupTailCount; m++) {
256- DataCopy(x1RegTensor, xBuf + j * ADD_ELEM + flag * groupMainCount * addOffset + i * mFactor);259+ MicroAPI::LoadAlign(x1RegTensor,
260+ xBuf + j * ADD_ELEM + flag * groupMainCount * addOffset + i * mFactor);
257 for (uint16_t n = 1; n <= addTailCount; n++) {261 for (uint16_t n = 1; n <= addTailCount; n++) {
258- DataCopy(x2RegTensor, xBuf + j * ADD_ELEM +262+ MicroAPI::LoadAlign(
259- flag * (groupMainCount * addOffset + n * computeLen) + i * mFactor);263+ x2RegTensor,
264+ xBuf + j * ADD_ELEM + flag * (groupMainCount * addOffset + n * computeLen) + i * mFactor);
260 Add(x2RegTensor, x1RegTensor, x2RegTensor, mask);265 Add(x2RegTensor, x1RegTensor, x2RegTensor, mask);
261- DataCopy(266+ MicroAPI::StoreAlign(
262 xBuf + j * ADD_ELEM + flag * (groupMainCount * addOffset + n * computeLen) + i * mFactor,267 xBuf + j * ADD_ELEM + flag * (groupMainCount * addOffset + n * computeLen) + i * mFactor,
263 x2RegTensor, mask);268 x2RegTensor, mask);
264 }269 }
@@ -282,8 +287,8 @@ __aicore__ inline void CumsumOnewaySklansky<DataType, PromoteDataType>::ComputeI
282 groupMainCount_ = Ops::Base::FloorDiv(rFactor_, nextAddLoopPow);287 groupMainCount_ = Ops::Base::FloorDiv(rFactor_, nextAddLoopPow);
283 groupTailCount_ = (rFactor_ % nextAddLoopPow > addCount_) ? 1 : 0;288 groupTailCount_ = (rFactor_ % nextAddLoopPow > addCount_) ? 1 : 0;
284 addTailCount_ = rFactor_ - groupMainCount_ * nextAddLoopPow - addLoopPow;289 addTailCount_ = rFactor_ - groupMainCount_ * nextAddLoopPow - addLoopPow;
285- __local_mem__ PromoteDataType* xBuf = (__local_mem__ PromoteDataType*)computerBuffer_[startOffset + startAddOffset]290+ __ubuf__ PromoteDataType* xBuf = (__ubuf__ PromoteDataType*)computerBuffer_[startOffset + startAddOffset]
286- .GetPhyAddr();291+ .GetPhyAddr();
287 292 
288 int32_t flag = reverse_ ? -1 : 1;293 int32_t flag = reverse_ ? -1 : 1;
289 ComputeWithFlag(addOffset, realDupSize, xBuf, flag);294 ComputeWithFlag(addOffset, realDupSize, xBuf, flag);
Mmath/cumsum/op_kernel/arch35/cumsum_base/cumsum_twoway_sklansky.h+24-24
@@ -281,12 +281,12 @@ __aicore__ inline void CumsumTwowaySklansky<DataType, PromoteDataType>::VfNoAlig
281 __VEC_SCOPE__281 __VEC_SCOPE__
282 {282 {
283 AscendC::Reg::MaskReg p0 = AscendC::Reg::CreateMask<DataType, AscendC::Reg::MaskPattern::ALL>();283 AscendC::Reg::MaskReg p0 = AscendC::Reg::CreateMask<DataType, AscendC::Reg::MaskPattern::ALL>();
284- AscendC::Reg::UnalignReg u0;284+ AscendC::Reg::UnalignRegForLoad u0;
285 AscendC::Reg::RegTensor<DataType> outNoAlignReg;285 AscendC::Reg::RegTensor<DataType> outNoAlignReg;
286 auto srcUbT = outputPtr + dirtyDataNum; // 非对齐的有效数据286 auto srcUbT = outputPtr + dirtyDataNum; // 非对齐的有效数据
287- AscendC::Reg::DataCopyUnAlignPre(u0, srcUbT);287+ AscendC::Reg::LoadUnAlignPre(u0, srcUbT);
288- AscendC::Reg::DataCopyUnAlign(outNoAlignReg, u0, srcUbT);288+ AscendC::Reg::LoadUnAlign(outNoAlignReg, u0, srcUbT);
289- AscendC::Reg::DataCopy(outNoAlignPtr, outNoAlignReg, p0);289+ AscendC::Reg::StoreAlign(outNoAlignPtr, outNoAlignReg, p0);
290 }290 }
291 DataCopyExtParams dataCopyNoAlignParams;291 DataCopyExtParams dataCopyNoAlignParams;
292 dataCopyNoAlignParams.blockCount = 1;292 dataCopyNoAlignParams.blockCount = 1;
@@ -382,10 +382,10 @@ __aicore__ inline void CumsumTwowaySklansky<DataType, PromoteDataType>::VfFirstA
382 addReg1Offset = (startOffset + reverseFlag * m * addOffset) * VL_ELEM;382 addReg1Offset = (startOffset + reverseFlag * m * addOffset) * VL_ELEM;
383 for (uint16_t n = 0; n < addCount; n++) {383 for (uint16_t n = 0; n < addCount; n++) {
384 addReg2Offset = (startOffset + reverseFlag * (m * addOffset + n + 1)) * VL_ELEM;384 addReg2Offset = (startOffset + reverseFlag * (m * addOffset + n + 1)) * VL_ELEM;
385- AscendC::Reg::DataCopy(addReg1, foldPtr + addReg1Offset);385+ AscendC::Reg::LoadAlign(addReg1, foldPtr + addReg1Offset);
386- AscendC::Reg::DataCopy(addReg2, foldPtr + addReg2Offset);386+ AscendC::Reg::LoadAlign(addReg2, foldPtr + addReg2Offset);
387 AscendC::Reg::Add(sumReg, addReg1, addReg2, p0);387 AscendC::Reg::Add(sumReg, addReg1, addReg2, p0);
388- AscendC::Reg::DataCopy(foldPtr + addReg2Offset, sumReg, p0);388+ AscendC::Reg::StoreAlign(foldPtr + addReg2Offset, sumReg, p0);
389 }389 }
390 }390 }
391 }391 }
@@ -436,9 +436,9 @@ __aicore__ inline void CumsumTwowaySklansky<DataType, PromoteDataType>::VfGather
436 p1 = AscendC::Reg::UpdateMask<int32_t>(num1);436 p1 = AscendC::Reg::UpdateMask<int32_t>(num1);
437 for (uint16_t j = 0; j < foldCount; j++) {437 for (uint16_t j = 0; j < foldCount; j++) {
438 AscendC::Reg::Adds(loopAddReg, addReg, (int32_t)nAlignBlockElem_ * j, p0);438 AscendC::Reg::Adds(loopAddReg, addReg, (int32_t)nAlignBlockElem_ * j, p0);
439- AscendC::Reg::DataCopyGather(dstReg, srcPtr, (AscendC::Reg::RegTensor<uint32_t>&)loopAddReg, p1);439+ AscendC::Reg::Gather(dstReg, srcPtr, (AscendC::Reg::RegTensor<uint32_t>&)loopAddReg, p1);
440 dstOffset = dstBaseOffset + i * VL_ELEM + reverseFlag * j * numAlignBlock;440 dstOffset = dstBaseOffset + i * VL_ELEM + reverseFlag * j * numAlignBlock;
441- AscendC::Reg::DataCopy(dstPtr + dstOffset, dstReg, p1);441+ AscendC::Reg::StoreAlign(dstPtr + dstOffset, dstReg, p1);
442 }442 }
443 }443 }
444 }444 }
@@ -507,14 +507,14 @@ __aicore__ inline void CumsumTwowaySklansky<DataType, PromoteDataType>::VfSecond
507 p1 = AscendC::Reg::UpdateMask<int32_t>(baseNum1);507 p1 = AscendC::Reg::UpdateMask<int32_t>(baseNum1);
508 // 连续搬入AddFactor的index和data508 // 连续搬入AddFactor的index和data
509 AscendC::Reg::AddrReg addrOffset = AscendC::Reg::CreateAddrReg<PromoteDataType>(j, VL_ELEM);509 AscendC::Reg::AddrReg addrOffset = AscendC::Reg::CreateAddrReg<PromoteDataType>(j, VL_ELEM);
510- AscendC::Reg::DataCopy(addFactorIndexReg, addFactorIndexPtr, addrOffset);510+ AscendC::Reg::LoadAlign(addFactorIndexReg, addFactorIndexPtr, addrOffset);
511 AscendC::Reg::Adds(addFactorIndexReg, addFactorIndexReg, m * VL_ELEM * 2, p0);511 AscendC::Reg::Adds(addFactorIndexReg, addFactorIndexReg, m * VL_ELEM * 2, p0);
512- AscendC::Reg::DataCopyGather(addFactorDataReg, addFactorDataPtr,512+ AscendC::Reg::Gather(addFactorDataReg, addFactorDataPtr,
513- (AscendC::Reg::RegTensor<uint32_t>&)addFactorIndexReg, p0);513+ (AscendC::Reg::RegTensor<uint32_t>&)addFactorIndexReg, p0);
514 int32_t addDataOffset = addDataMainReserveOffset + (i * indexLoopCount_ + j) * VL_ELEM;514 int32_t addDataOffset = addDataMainReserveOffset + (i * indexLoopCount_ + j) * VL_ELEM;
515- AscendC::Reg::DataCopy(addDataReg, secondfoldPtr + addDataOffset); // 连续对齐搬入addData515+ AscendC::Reg::LoadAlign(addDataReg, secondfoldPtr + addDataOffset); // 连续对齐搬入addData
516- AscendC::Reg::Add(addDataReg, addDataReg, addFactorDataReg, p1); // Add516+ AscendC::Reg::Add(addDataReg, addDataReg, addFactorDataReg, p1); // Add
517- AscendC::Reg::DataCopy(secondfoldPtr + addDataOffset, addDataReg, p1); // 搬出517+ AscendC::Reg::StoreAlign(secondfoldPtr + addDataOffset, addDataReg, p1); // 搬出
518 }518 }
519 }519 }
520 // 尾块，只需执行一次即可520 // 尾块，只需执行一次即可
@@ -522,14 +522,14 @@ __aicore__ inline void CumsumTwowaySklansky<DataType, PromoteDataType>::VfSecond
522 for (uint16_t j = 0; j < indexLoopCountTail; j++) {522 for (uint16_t j = 0; j < indexLoopCountTail; j++) {
523 p2 = AscendC::Reg::UpdateMask<int32_t>(baseNum2);523 p2 = AscendC::Reg::UpdateMask<int32_t>(baseNum2);
524 AscendC::Reg::AddrReg addrOffset = AscendC::Reg::CreateAddrReg<PromoteDataType>(j, VL_ELEM);524 AscendC::Reg::AddrReg addrOffset = AscendC::Reg::CreateAddrReg<PromoteDataType>(j, VL_ELEM);
525- AscendC::Reg::DataCopy(addFactorIndexReg, addFactorIndexPtr, addrOffset);525+ AscendC::Reg::LoadAlign(addFactorIndexReg, addFactorIndexPtr, addrOffset);
526 AscendC::Reg::Adds(addFactorIndexReg, addFactorIndexReg, m * VL_ELEM * 2, p0);526 AscendC::Reg::Adds(addFactorIndexReg, addFactorIndexReg, m * VL_ELEM * 2, p0);
527- AscendC::Reg::DataCopyGather(addFactorDataReg, addFactorDataPtr,527+ AscendC::Reg::Gather(addFactorDataReg, addFactorDataPtr,
528- (AscendC::Reg::RegTensor<uint32_t>&)addFactorIndexReg, p0);528+ (AscendC::Reg::RegTensor<uint32_t>&)addFactorIndexReg, p0);
529 int32_t addDataOffset = addDataTailReserveOffset + (i * indexLoopCount_ + j) * VL_ELEM;529 int32_t addDataOffset = addDataTailReserveOffset + (i * indexLoopCount_ + j) * VL_ELEM;
530- AscendC::Reg::DataCopy(addDataReg, secondfoldPtr + addDataOffset);530+ AscendC::Reg::LoadAlign(addDataReg, secondfoldPtr + addDataOffset);
531 AscendC::Reg::Add(addDataReg, addDataReg, addFactorDataReg, p2);531 AscendC::Reg::Add(addDataReg, addDataReg, addFactorDataReg, p2);
532- AscendC::Reg::DataCopy(secondfoldPtr + addDataOffset, addDataReg, p2);532+ AscendC::Reg::StoreAlign(secondfoldPtr + addDataOffset, addDataReg, p2);
533 }533 }
534 }534 }
535 }535 }
@@ -572,10 +572,10 @@ __aicore__ inline void CumsumTwowaySklansky<DataType, PromoteDataType>::VfSecond
572 AscendC::Reg::Mul(tmp3, tmp, nReg, p0); // [000...99999.....63]572 AscendC::Reg::Mul(tmp3, tmp, nReg, p0); // [000...99999.....63]
573 AscendC::Reg::Sub(tmp2, indexReg, tmp3, p0); // [0123...0123...0123..]573 AscendC::Reg::Sub(tmp2, indexReg, tmp3, p0); // [0123...0123...0123..]
574 AscendC::Reg::Adds(secondFoldReg, tmp2, addFactorDataStart + ReserveStartOffset, p0);574 AscendC::Reg::Adds(secondFoldReg, tmp2, addFactorDataStart + ReserveStartOffset, p0);
575- AscendC::Reg::DataCopyGather(addFactorDataReg, secondfoldPtr,575+ AscendC::Reg::Gather(addFactorDataReg, secondfoldPtr, (AscendC::Reg::RegTensor<uint32_t>&)secondFoldReg,
576- (AscendC::Reg::RegTensor<uint32_t>&)secondFoldReg, p0);576+ p0);
577- AscendC::Reg::DataCopy(addFactorDataPtr + (int32_t)VL_ELEM * 2 * m + (int32_t)VL_ELEM * i,577+ AscendC::Reg::StoreAlign(addFactorDataPtr + (int32_t)VL_ELEM * 2 * m + (int32_t)VL_ELEM * i,
578- addFactorDataReg, p0);578+ addFactorDataReg, p0);
579 startIdx = startIdx + singleRemainder;579 startIdx = startIdx + singleRemainder;
580 }580 }
581 }581 }
Mmath/cumsum/op_kernel/arch35/cumsum_base/cumsum_ub_sklansky.h+39-48
@@ -90,24 +90,19 @@ private:
90 __aicore__ inline void SetOnewaySklanskyParam(int32_t ubIndex,90 __aicore__ inline void SetOnewaySklanskyParam(int32_t ubIndex,
91 OnewaySklanskyProcessData& onewaySklanskyProcessData);91 OnewaySklanskyProcessData& onewaySklanskyProcessData);
92 __aicore__ inline bool CheckOffsetAndLen(int32_t ubIndex);92 __aicore__ inline bool CheckOffsetAndLen(int32_t ubIndex);
93- __aicore__ inline void VfBetweenUbAddTwoway(__local_mem__ PromoteDataType* dstPtr,93+ __aicore__ inline void VfBetweenUbAddTwoway(__ubuf__ PromoteDataType* dstPtr, __ubuf__ PromoteDataType* srcPtr,
94- __local_mem__ PromoteDataType* srcPtr, uint32_t cacheStartInRes,94+ uint32_t cacheStartInRes, uint16_t memCacheId);
95- uint16_t memCacheId);95+ __aicore__ inline void VfBetweenUbAddTwowayBak(__ubuf__ PromoteDataType* dstPtr, __ubuf__ PromoteDataType* srcPtr,
96- __aicore__ inline void VfBetweenUbAddTwowayBak(__local_mem__ PromoteDataType* dstPtr,96+ uint32_t cacheStartInRes, uint16_t memCacheId);
97- __local_mem__ PromoteDataType* srcPtr, uint32_t cacheStartInRes,97+ __aicore__ inline void VfBetweenUbWriteMemTwoway(__ubuf__ PromoteDataType* dstPtr, __ubuf__ PromoteDataType* srcPtr,
98- uint16_t memCacheId);98+ uint32_t cacheStartInRes, uint16_t memCacheId);
99- __aicore__ inline void VfBetweenUbWriteMemTwoway(__local_mem__ PromoteDataType* dstPtr,99+ __aicore__ inline void VfBetweenUbAddOneway(__ubuf__ PromoteDataType* dstPtr, __ubuf__ PromoteDataType* srcPtr,
100- __local_mem__ PromoteDataType* srcPtr, uint32_t cacheStartInRes,100+ uint32_t cacheStartInRes, uint16_t memCacheId);
101- uint16_t memCacheId);101+ __aicore__ inline void VfBetweenUbWriteMemOneway(__ubuf__ PromoteDataType* dstPtr, __ubuf__ PromoteDataType* srcPtr,
102- __aicore__ inline void VfBetweenUbAddOneway(__local_mem__ PromoteDataType* dstPtr,102+ uint32_t cacheStartInRes, uint16_t memCacheId);
103- __local_mem__ PromoteDataType* srcPtr, uint32_t cacheStartInRes,
104- uint16_t memCacheId);
105- __aicore__ inline void VfBetweenUbWriteMemOneway(__local_mem__ PromoteDataType* dstPtr,
106- __local_mem__ PromoteDataType* srcPtr, uint32_t cacheStartInRes,
107- uint16_t memCacheId);
108 __aicore__ inline uint16_t CalAddFactorIndex();103 __aicore__ inline uint16_t CalAddFactorIndex();
109- __aicore__ inline void CalAddFactorValue(__local_mem__ PromoteDataType* dstPtr,104+ __aicore__ inline void CalAddFactorValue(__ubuf__ PromoteDataType* dstPtr,
110- __local_mem__ PromoteDataType* addFactorDataPtr, int32_t singleRemainder,105+ __ubuf__ PromoteDataType* addFactorDataPtr, int32_t singleRemainder,
111 uint16_t memCacheId);106 uint16_t memCacheId);
112 107 
113private:108private:
@@ -294,7 +289,7 @@ __aicore__ inline uint16_t CumsumUbSklansky<DataType, PromoteDataType, UbInner>:
294 289 
295template <typename DataType, typename PromoteDataType, typename UbInner>290template <typename DataType, typename PromoteDataType, typename UbInner>
296__aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::CalAddFactorValue(291__aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::CalAddFactorValue(
297- __local_mem__ PromoteDataType* dstPtr, __local_mem__ PromoteDataType* addFactorDataPtr, int32_t singleRemainder,292+ __ubuf__ PromoteDataType* dstPtr, __ubuf__ PromoteDataType* addFactorDataPtr, int32_t singleRemainder,
298 uint16_t memCacheId)293 uint16_t memCacheId)
299{294{
300 uint16_t dataFactorSize = DATA_FACTOR_SIZE;295 uint16_t dataFactorSize = DATA_FACTOR_SIZE;
@@ -317,9 +312,9 @@ __aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::Cal
317 AscendC::Reg::Muls(tmp1, tmp0, (int32_t)VL_ELEM, p0); // [000..64...448]312 AscendC::Reg::Muls(tmp1, tmp0, (int32_t)VL_ELEM, p0); // [000..64...448]
318 AscendC::Reg::Mul(tmp3, tmp0, nReg, p0); // [000...99999.....63]313 AscendC::Reg::Mul(tmp3, tmp0, nReg, p0); // [000...99999.....63]
319 AscendC::Reg::Sub(tmp2, indexReg, tmp3, p0); // [0123...0123...0123..]314 AscendC::Reg::Sub(tmp2, indexReg, tmp3, p0); // [0123...0123...0123..]
320- AscendC::Reg::DataCopyGather(addFactorDataReg, dstPtr + memCacheId * initData_.perMemoSize,315+ AscendC::Reg::Gather(addFactorDataReg, dstPtr + memCacheId * initData_.perMemoSize,
321- (AscendC::Reg::RegTensor<uint32_t>&)tmp2, p0);316+ (AscendC::Reg::RegTensor<uint32_t>&)tmp2, p0);
322- AscendC::Reg::DataCopy(addFactorDataPtr + (int32_t)VL_ELEM * ii, addFactorDataReg, p0);317+ AscendC::Reg::StoreAlign(addFactorDataPtr + (int32_t)VL_ELEM * ii, addFactorDataReg, p0);
323 startIdx = startIdx + singleRemainder;318 startIdx = startIdx + singleRemainder;
324 }319 }
325 }320 }
@@ -327,13 +322,12 @@ __aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::Cal
327 322 
328template <typename DataType, typename PromoteDataType, typename UbInner>323template <typename DataType, typename PromoteDataType, typename UbInner>
329__aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::VfBetweenUbAddTwoway(324__aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::VfBetweenUbAddTwoway(
330- __local_mem__ PromoteDataType* dstPtr, __local_mem__ PromoteDataType* srcPtr, uint32_t cacheStartInRes,325+ __ubuf__ PromoteDataType* dstPtr, __ubuf__ PromoteDataType* srcPtr, uint32_t cacheStartInRes, uint16_t memCacheId)
331- uint16_t memCacheId)
332{326{
333 uint16_t indexLoopCount = CalAddFactorIndex();327 uint16_t indexLoopCount = CalAddFactorIndex();
334 int32_t singleRemainder = VL_ELEM % initData_.ubFactorN;328 int32_t singleRemainder = VL_ELEM % initData_.ubFactorN;
335- __local_mem__ PromoteDataType* addFactorDataPtr = (__local_mem__ PromoteDataType*)addFactorDataBuffer_.GetPhyAddr();329+ __ubuf__ PromoteDataType* addFactorDataPtr = (__ubuf__ PromoteDataType*)addFactorDataBuffer_.GetPhyAddr();
336- __local_mem__ uint32_t* addFactorIndexPtr = (__local_mem__ uint32_t*)addFactorIndexBuffer_.GetPhyAddr();330+ __ubuf__ uint32_t* addFactorIndexPtr = (__ubuf__ uint32_t*)addFactorIndexBuffer_.GetPhyAddr();
337 331 
338 CalAddFactorValue(dstPtr, addFactorDataPtr, singleRemainder, memCacheId);332 CalAddFactorValue(dstPtr, addFactorDataPtr, singleRemainder, memCacheId);
339 333 
@@ -358,13 +352,13 @@ __aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::VfB
358 // 连续搬入AddFactor的index和data352 // 连续搬入AddFactor的index和data
359 p3 = AscendC::Reg::UpdateMask<int32_t>(mainNum);353 p3 = AscendC::Reg::UpdateMask<int32_t>(mainNum);
360 AscendC::Reg::AddrReg addrOffset = AscendC::Reg::CreateAddrReg<PromoteDataType>(j, VL_ELEM);354 AscendC::Reg::AddrReg addrOffset = AscendC::Reg::CreateAddrReg<PromoteDataType>(j, VL_ELEM);
361- AscendC::Reg::DataCopy(addFactorIndexReg, addFactorIndexPtr, addrOffset);355+ AscendC::Reg::LoadAlign(addFactorIndexReg, addFactorIndexPtr, addrOffset);
362- AscendC::Reg::DataCopyGather(addFactorDataReg, addFactorDataPtr, addFactorIndexReg, p0);356+ AscendC::Reg::Gather(addFactorDataReg, addFactorDataPtr, addFactorIndexReg, p0);
363 // 搬入第二块数据357 // 搬入第二块数据
364 int32_t addDataOffset = (i * indexLoopCount + j) * VL_ELEM;358 int32_t addDataOffset = (i * indexLoopCount + j) * VL_ELEM;
365- AscendC::Reg::DataCopy(curReg, srcPtr + addDataOffset);359+ AscendC::Reg::LoadAlign(curReg, srcPtr + addDataOffset);
366 AscendC::Reg::Add(curReg, curReg, addFactorDataReg, p3);360 AscendC::Reg::Add(curReg, curReg, addFactorDataReg, p3);
367- AscendC::Reg::DataCopy(srcPtr + addDataOffset, curReg, p3);361+ AscendC::Reg::StoreAlign(srcPtr + addDataOffset, curReg, p3);
368 }362 }
369 }363 }
370 364 
@@ -372,21 +366,20 @@ __aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::VfB
372 // 连续搬入AddFactor的index和data366 // 连续搬入AddFactor的index和data
373 p3 = AscendC::Reg::UpdateMask<int32_t>(baseNumOver);367 p3 = AscendC::Reg::UpdateMask<int32_t>(baseNumOver);
374 AscendC::Reg::AddrReg addrOffset = AscendC::Reg::CreateAddrReg<PromoteDataType>(j, VL_ELEM);368 AscendC::Reg::AddrReg addrOffset = AscendC::Reg::CreateAddrReg<PromoteDataType>(j, VL_ELEM);
375- AscendC::Reg::DataCopy(addFactorIndexReg, addFactorIndexPtr, addrOffset);369+ AscendC::Reg::LoadAlign(addFactorIndexReg, addFactorIndexPtr, addrOffset);
376- AscendC::Reg::DataCopyGather(addFactorDataReg, addFactorDataPtr, addFactorIndexReg, p0);370+ AscendC::Reg::Gather(addFactorDataReg, addFactorDataPtr, addFactorIndexReg, p0);
377 // 搬入第二块数据371 // 搬入第二块数据
378 int32_t addDataOffset = (mainCount + j) * VL_ELEM;372 int32_t addDataOffset = (mainCount + j) * VL_ELEM;
379- AscendC::Reg::DataCopy(curReg, srcPtr + addDataOffset);373+ AscendC::Reg::LoadAlign(curReg, srcPtr + addDataOffset);
380 AscendC::Reg::Add(curReg, curReg, addFactorDataReg, p3);374 AscendC::Reg::Add(curReg, curReg, addFactorDataReg, p3);
381- AscendC::Reg::DataCopy(srcPtr + addDataOffset, curReg, p3);375+ AscendC::Reg::StoreAlign(srcPtr + addDataOffset, curReg, p3);
382 }376 }
383 }377 }
384}378}
385 379 
386template <typename DataType, typename PromoteDataType, typename UbInner>380template <typename DataType, typename PromoteDataType, typename UbInner>
387__aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::VfBetweenUbAddOneway(381__aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::VfBetweenUbAddOneway(
388- __local_mem__ PromoteDataType* dstPtr, __local_mem__ PromoteDataType* srcPtr, uint32_t cacheStartInRes,382+ __ubuf__ PromoteDataType* dstPtr, __ubuf__ PromoteDataType* srcPtr, uint32_t cacheStartInRes, uint16_t memCacheId)
389- uint16_t memCacheId)
390{383{
391 uint16_t rCount = ubBetweenAddR_;384 uint16_t rCount = ubBetweenAddR_;
392 uint32_t addLen = addLenN_ * initData_.mFold;385 uint32_t addLen = addLenN_ * initData_.mFold;
@@ -399,14 +392,14 @@ __aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::VfB
399 AscendC::Reg::RegTensor<PromoteDataType> curReg;392 AscendC::Reg::RegTensor<PromoteDataType> curReg;
400 AscendC::Reg::MaskReg p0;393 AscendC::Reg::MaskReg p0;
401 p0 = AscendC::Reg::UpdateMask<int32_t>(addLen);394 p0 = AscendC::Reg::UpdateMask<int32_t>(addLen);
402- AscendC::Reg::DataCopy(memoReg, dstPtr + memCacheId * initData_.perMemoSize);395+ AscendC::Reg::LoadAlign(memoReg, dstPtr + memCacheId * initData_.perMemoSize);
403 for (uint16_t i = 0; i < mLoop; i++) {396 for (uint16_t i = 0; i < mLoop; i++) {
404 for (uint16_t ii = 0; ii < rCount; ii++) {397 for (uint16_t ii = 0; ii < rCount; ii++) {
405 uint16_t j = i * rCount + ii;398 uint16_t j = i * rCount + ii;
406 AscendC::Reg::AddrReg addrOffset = AscendC::Reg::CreateAddrReg<PromoteDataType>(j, addLenAlign);399 AscendC::Reg::AddrReg addrOffset = AscendC::Reg::CreateAddrReg<PromoteDataType>(j, addLenAlign);
407- AscendC::Reg::DataCopy(curReg, srcPtr, addrOffset);400+ AscendC::Reg::LoadAlign(curReg, srcPtr, addrOffset);
408 AscendC::Reg::Add(curReg, curReg, memoReg, p0);401 AscendC::Reg::Add(curReg, curReg, memoReg, p0);
409- AscendC::Reg::DataCopy(srcPtr, curReg, addrOffset, p0);402+ AscendC::Reg::StoreAlign(srcPtr, curReg, addrOffset, p0);
410 }403 }
411 }404 }
412 }405 }
@@ -414,8 +407,7 @@ __aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::VfB
414 407 
415template <typename DataType, typename PromoteDataType, typename UbInner>408template <typename DataType, typename PromoteDataType, typename UbInner>
416__aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::VfBetweenUbWriteMemTwoway(409__aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::VfBetweenUbWriteMemTwoway(
417- __local_mem__ PromoteDataType* dstPtr, __local_mem__ PromoteDataType* srcPtr, uint32_t cacheStartInRes,410+ __ubuf__ PromoteDataType* dstPtr, __ubuf__ PromoteDataType* srcPtr, uint32_t cacheStartInRes, uint16_t memCacheId)
418- uint16_t memCacheId)
419{411{
420 __VEC_SCOPE__412 __VEC_SCOPE__
421 {413 {
@@ -426,15 +418,14 @@ __aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::VfB
426 AscendC::Reg::Arange(indexReg2, cacheStartInRes);418 AscendC::Reg::Arange(indexReg2, cacheStartInRes);
427 uint32_t nLen = initData_.ubFactorN;419 uint32_t nLen = initData_.ubFactorN;
428 p0 = AscendC::Reg::UpdateMask<int32_t>(nLen);420 p0 = AscendC::Reg::UpdateMask<int32_t>(nLen);
429- AscendC::Reg::DataCopyGather(dstCaheReg, srcPtr, (AscendC::Reg::RegTensor<uint32_t>&)indexReg2, p0);421+ AscendC::Reg::Gather(dstCaheReg, srcPtr, (AscendC::Reg::RegTensor<uint32_t>&)indexReg2, p0);
430- AscendC::Reg::DataCopy(dstPtr + memCacheId * initData_.perMemoSize, dstCaheReg, p0);422+ AscendC::Reg::StoreAlign(dstPtr + memCacheId * initData_.perMemoSize, dstCaheReg, p0);
431 }423 }
432}424}
433 425 
434template <typename DataType, typename PromoteDataType, typename UbInner>426template <typename DataType, typename PromoteDataType, typename UbInner>
435__aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::VfBetweenUbWriteMemOneway(427__aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::VfBetweenUbWriteMemOneway(
436- __local_mem__ PromoteDataType* dstPtr, __local_mem__ PromoteDataType* srcPtr, uint32_t cacheStartInRes,428+ __ubuf__ PromoteDataType* dstPtr, __ubuf__ PromoteDataType* srcPtr, uint32_t cacheStartInRes, uint16_t memCacheId)
437- uint16_t memCacheId)
438{429{
439 uint32_t addLen = addLenN_ * initData_.mFold;430 uint32_t addLen = addLenN_ * initData_.mFold;
440 uint32_t addLenAlign = addLenN_ * initData_.mFold; // block对齐431 uint32_t addLenAlign = addLenN_ * initData_.mFold; // block对齐
@@ -443,8 +434,8 @@ __aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::VfB
443 AscendC::Reg::MaskReg p0;434 AscendC::Reg::MaskReg p0;
444 AscendC::Reg::RegTensor<PromoteDataType> dstCaheReg;435 AscendC::Reg::RegTensor<PromoteDataType> dstCaheReg;
445 p0 = AscendC::Reg::UpdateMask<int32_t>(addLen);436 p0 = AscendC::Reg::UpdateMask<int32_t>(addLen);
446- AscendC::Reg::DataCopy(dstCaheReg, srcPtr + cacheStartInRes * addLenAlign);437+ AscendC::Reg::LoadAlign(dstCaheReg, srcPtr + cacheStartInRes * addLenAlign);
447- AscendC::Reg::DataCopy(dstPtr + memCacheId * initData_.perMemoSize, dstCaheReg, p0);438+ AscendC::Reg::StoreAlign(dstPtr + memCacheId * initData_.perMemoSize, dstCaheReg, p0);
448 }439 }
449}440}
450 441 
@@ -452,9 +443,9 @@ template <typename DataType, typename PromoteDataType, typename UbInner>
452__aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::SkalanskyBetweenUb(int32_t ubIndex)443__aicore__ inline void CumsumUbSklansky<DataType, PromoteDataType, UbInner>::SkalanskyBetweenUb(int32_t ubIndex)
453{444{
454 int32_t cacheID = __CumsumUtil::GetCacheID(ubIndex);445 int32_t cacheID = __CumsumUtil::GetCacheID(ubIndex);
455- __local_mem__ PromoteDataType* dstPtr = (__local_mem__ PromoteDataType*)memoBuffer_.GetPhyAddr();446+ __ubuf__ PromoteDataType* dstPtr = (__ubuf__ PromoteDataType*)memoBuffer_.GetPhyAddr();
456 LocalTensor<PromoteDataType> computeBuffer = ubInner_.GetBaseResTensor();447 LocalTensor<PromoteDataType> computeBuffer = ubInner_.GetBaseResTensor();
457- __local_mem__ PromoteDataType* srcPtr = (__local_mem__ PromoteDataType*)computeBuffer.GetPhyAddr();448+ __ubuf__ PromoteDataType* srcPtr = (__ubuf__ PromoteDataType*)computeBuffer.GetPhyAddr();
458 449 
459 uint32_t cacheStartInRes = 0;450 uint32_t cacheStartInRes = 0;
460 if constexpr (IsSameType<UbInner, CumsumTwowaySklansky<DataType, PromoteDataType>>::value) {451 if constexpr (IsSameType<UbInner, CumsumTwowaySklansky<DataType, PromoteDataType>>::value) {
Mmath/eye/op_kernel/arch35/eye_simd.h+7-7
@@ -106,7 +106,7 @@ __aicore__ inline void EyeKernelSimd<T, U>::GenScatterIndex()
106 Reg::Mul(vd5, vd2, v2, p0);106 Reg::Mul(vd5, vd2, v2, p0);
107 Reg::Mul(vd1, vd4, v0, p0);107 Reg::Mul(vd1, vd4, v0, p0);
108 Reg::Add(vd6, vd5, vd1, p0);108 Reg::Add(vd6, vd5, vd1, p0);
109- Reg::DataCopy(dstAddr, vd6, p0);109+ Reg::StoreAlign(dstAddr, vd6, p0);
110 }110 }
111}111}
112 112 
@@ -142,13 +142,13 @@ __aicore__ inline void EyeKernelSimd<T, U>::GenEyeBuf()
142 Reg::RegTensor<U> vd1;142 Reg::RegTensor<U> vd1;
143 Reg::RegTensor<T> one;143 Reg::RegTensor<T> one;
144 Reg::Duplicate(one, (T)(1), p0);144 Reg::Duplicate(one, (T)(1), p0);
145- Reg::DataCopy(vd0, indexAddr);145+ Reg::LoadAlign(vd0, indexAddr);
146 for (uint16_t i = 0; i < size0; i++) {146 for (uint16_t i = 0; i < size0; i++) {
147 Reg::Adds(vd1, vd0, (U)(i * stride), p0);147 Reg::Adds(vd1, vd0, (U)(i * stride), p0);
148- Reg::DataCopyScatter(zeroAddr, one, vd1, p0);148+ Reg::Scatter(zeroAddr, one, vd1, p0);
149 }149 }
150 Reg::Adds(vd1, vd0, (U)(size0 * stride), p1);150 Reg::Adds(vd1, vd0, (U)(size0 * stride), p1);
151- Reg::DataCopyScatter(zeroAddr, one, vd1, p1);151+ Reg::Scatter(zeroAddr, one, vd1, p1);
152 }152 }
153 } else {153 } else {
154 uint16_t regFactor1 = vfLen;154 uint16_t regFactor1 = vfLen;
@@ -169,15 +169,15 @@ __aicore__ inline void EyeKernelSimd<T, U>::GenEyeBuf()
169 Reg::RegTensor<U> index;169 Reg::RegTensor<U> index;
170 Reg::RegTensor<T> one;170 Reg::RegTensor<T> one;
171 Reg::Duplicate(one, (T)(1), p0);171 Reg::Duplicate(one, (T)(1), p0);
172- Reg::DataCopy(index, indexAddr);172+ Reg::LoadAlign(index, indexAddr);
173 for (uint16_t i = 0; i < size0; i++) {173 for (uint16_t i = 0; i < size0; i++) {
174 Reg::Adds(vd0, index, (U)(i * batchEleNum), p0);174 Reg::Adds(vd0, index, (U)(i * batchEleNum), p0);
175 for (uint16_t j = 0; j < size1; j++) {175 for (uint16_t j = 0; j < size1; j++) {
176 Reg::Adds(vd1, vd0, (U)(j * stride), p0);176 Reg::Adds(vd1, vd0, (U)(j * stride), p0);
177- Reg::DataCopyScatter(zeroAddr, one, vd1, p0);177+ Reg::Scatter(zeroAddr, one, vd1, p0);
178 }178 }
179 Reg::Adds(vd2, vd0, (U)(size1 * stride), p1);179 Reg::Adds(vd2, vd0, (U)(size1 * stride), p1);
180- Reg::DataCopyScatter(zeroAddr, one, vd2, p1);180+ Reg::Scatter(zeroAddr, one, vd2, p1);
181 }181 }
182 }182 }
183 }183 }
Mmath/power/op_kernel/arch35/power_dag.h+2-2
@@ -164,7 +164,7 @@ struct PowerGenericCompute : public Vec::ElemwiseTernaryOP<float, float, float,
164 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {164 for (uint16_t loopIdx = 0; loopIdx < loopNum; loopIdx++) {
165 // 更新尾元素 mask 并把当前 batch 的 base 搬入寄存器。165 // 更新尾元素 mask 并把当前 batch 的 base 搬入寄存器。
166 mask = AscendC::Reg::UpdateMask<float, AscendC::Reg::RegTraitNumOne>(count);166 mask = AscendC::Reg::UpdateMask<float, AscendC::Reg::RegTraitNumOne>(count);
167- AscendC::Reg::DataCopy(regBase, (__ubuf__ float*)(baseAddr + loopIdx * vl));167+ AscendC::Reg::LoadAlign(regBase, (__ubuf__ float*)(baseAddr + loopIdx * vl));
168 168 
169 // ----- 主路径：rawExp = exp(power * log(|base|))，全程 in-place 复用 regMain -----169 // ----- 主路径：rawExp = exp(power * log(|base|))，全程 in-place 复用 regMain -----
170 AscendC::Reg::Abs(regMain, regBase, mask); // regMain = |base|170 AscendC::Reg::Abs(regMain, regBase, mask); // regMain = |base|
@@ -194,7 +194,7 @@ struct PowerGenericCompute : public Vec::ElemwiseTernaryOP<float, float, float,
194 AscendC::Reg::Select(regBranch, regMain, regBranch, maskZero);194 AscendC::Reg::Select(regBranch, regMain, regBranch, maskZero);
195 195 
196 // 把当前 batch 的最终结果写回 UB。196 // 把当前 batch 的最终结果写回 UB。
197- AscendC::Reg::DataCopy((__ubuf__ float*)(dstAddr + loopIdx * vl), regBranch, mask);197+ AscendC::Reg::StoreAlign((__ubuf__ float*)(dstAddr + loopIdx * vl), regBranch, mask);
198 }198 }
199 }199 }
200#endif200#endif