Star861
1736
代码
代码
Issues165
Pull Requests293
Actions
项目讨论
Wiki
分析
Star861
1736
已合并
opt maxpool3dgradwithargmax perf. #6893
liuchuangdev创建于 7月2日
opt maxpool3dgradwithargmax perf. #6893
已合并
liuchuangdev创建于 7月2日
从liuchuangdev/ops-nnperf/maxpool3dgradwithargmax-optimize-910合入到cann/ops-nn9.1.0
共 5 个文件变更+1247-1884
Mpooling/max_pool3d_grad_with_argmax/op_host/max_pool3d_grad_with_argmax_simd_tiling.cpp+7-5
@@ -25,8 +25,7 @@ static constexpr int64_t INT64_SIZE = 8;
25static constexpr int64_t UB_RESVERVED_SIZE = 2048;25static constexpr int64_t UB_RESVERVED_SIZE = 2048;
26static constexpr int64_t T3_INT64 = 10;26static constexpr int64_t T3_INT64 = 10;
27static constexpr int64_t DOUBLE_BUFFER = 2;27static constexpr int64_t DOUBLE_BUFFER = 2;
28-static constexpr int64_t THRESHOLD = 2;28+static constexpr int64_t KSIZE_STRIDE_RATIO_THRESHOLD = 16;
29-static constexpr int64_t HELP_BUFFER_SIZE = 2048;
30 29 
31void MaxPool3DGradWithArgmaxNCDHWTiling::InitializationVars()30void MaxPool3DGradWithArgmaxNCDHWTiling::InitializationVars()
32{31{
@@ -126,7 +125,7 @@ void MaxPool3DGradWithArgmaxNCDHWTiling::DoBufferCalculate()
126 splitData.outputBufferSize = splitData.highAxisInner * outputPlaneSizeDHW * FLOAT32_SIZE;125 splitData.outputBufferSize = splitData.highAxisInner * outputPlaneSizeDHW * FLOAT32_SIZE;
127 126 
128 int64_t tmpTotalBufferSize = splitData.outputBufferSize + splitData.gradBufferSize + splitData.argmaxBufferSize;127 int64_t tmpTotalBufferSize = splitData.outputBufferSize + splitData.gradBufferSize + splitData.argmaxBufferSize;
129- splitData.totalBufferSize = tmpTotalBufferSize * DOUBLE_BUFFER + HELP_BUFFER_SIZE;128+ splitData.totalBufferSize = tmpTotalBufferSize * DOUBLE_BUFFER;
130}129}
131 130 
132void MaxPool3DGradWithArgmaxNCDHWTiling::DoBufferCalculateNC()131void MaxPool3DGradWithArgmaxNCDHWTiling::DoBufferCalculateNC()
@@ -153,7 +152,7 @@ void MaxPool3DGradWithArgmaxNCDHWTiling::DoBufferCalculateNC()
153 splitData.outputBufferSize = splitData.highAxisInner * outputPlaneSizeDHW * FLOAT32_SIZE;152 splitData.outputBufferSize = splitData.highAxisInner * outputPlaneSizeDHW * FLOAT32_SIZE;
154 153 
155 int64_t tmpTotalBufferSize = splitData.outputBufferSize + splitData.gradBufferSize + splitData.argmaxBufferSize;154 int64_t tmpTotalBufferSize = splitData.outputBufferSize + splitData.gradBufferSize + splitData.argmaxBufferSize;
156- splitData.totalBufferSize = tmpTotalBufferSize * DOUBLE_BUFFER + HELP_BUFFER_SIZE;155+ splitData.totalBufferSize = tmpTotalBufferSize * DOUBLE_BUFFER;
157}156}
158 157 
159bool MaxPool3DGradWithArgmaxNCDHWTiling::IsMeetTargetCoreNum() const158bool MaxPool3DGradWithArgmaxNCDHWTiling::IsMeetTargetCoreNum() const
@@ -358,7 +357,9 @@ void MaxPool3DGradWithArgmaxNCDHWTiling::SearchBestTiling()
358 splitData.isFull = 1;357 splitData.isFull = 1;
359 return;358 return;
360 }359 }
361- splitData.isCheckRange = 1;360+ if (inputData.dDilation != 1 || inputData.hDilation != 1 || inputData.wDilation != 1) {
361+ splitData.isCheckRange = 1;
362+ }
362 if (baseData.isPad == 0 && baseData.isOverlap == 0) {363 if (baseData.isPad == 0 && baseData.isOverlap == 0) {
363 if (TrySplitAlignD()) {364 if (TrySplitAlignD()) {
364 return;365 return;
@@ -372,6 +373,7 @@ void MaxPool3DGradWithArgmaxNCDHWTiling::SearchBestTiling()
372 return;373 return;
373 }374 }
374 }375 }
376+ splitData.isCheckRange = 1;
375 SplitUnalignDHW();377 SplitUnalignDHW();
376 return;378 return;
377}379}
Mpooling/max_pool3d_grad_with_argmax/op_kernel/arch35/max_pool3d_grad_with_argmax_simd.h+383-134
@@ -26,7 +26,6 @@ using namespace AscendC;
26using Pool3D::FastDivImpl;26using Pool3D::FastDivImpl;
27constexpr uint32_t BUFFER_NUM = 2;27constexpr uint32_t BUFFER_NUM = 2;
28constexpr int64_t DOUBLE = 2;28constexpr int64_t DOUBLE = 2;
29-constexpr uint32_t HELP_BUFFER = 2048;
30 29 
31constexpr uint32_t INDEX_TWO = 2;30constexpr uint32_t INDEX_TWO = 2;
32constexpr uint32_t INDEX_THREE = 3;31constexpr uint32_t INDEX_THREE = 3;
@@ -130,39 +129,31 @@ namespace MaxPool3DGradWithArgmaxNCDHWNameSpace {
130 129 
131template <const uint32_t IS_MUL_NC = 0>130template <const uint32_t IS_MUL_NC = 0>
132__aicore__ inline void IndexConvNcdhwFastDiv(131__aicore__ inline void IndexConvNcdhwFastDiv(
133- MicroAPI::RegTensor<int32_t>& argmaxReg, MicroAPI::RegTensor<int32_t>& dIndexReg,132+ MicroAPI::RegTensor<int32_t>& argmaxReg, MicroAPI::RegTensor<uint32_t>& dTmpReg,
134- MicroAPI::RegTensor<int32_t>& hIndexReg, MicroAPI::RegTensor<int32_t>& wIndexReg,133+ MicroAPI::RegTensor<uint32_t>& hTmpReg, MicroAPI::RegTensor<uint32_t>& wTmpReg,
135 MicroAPI::RegTensor<uint32_t>& magicHWReg, int16_t shiftHW, MicroAPI::RegTensor<uint32_t>& magicWReg,134 MicroAPI::RegTensor<uint32_t>& magicHWReg, int16_t shiftHW, MicroAPI::RegTensor<uint32_t>& magicWReg,
136- int16_t shiftW, int64_t curDIndex, int64_t curHIndex, int64_t curWIndex, int32_t hOutputActual,135+ int16_t shiftW, int32_t hwOutputAligned,
137- int32_t wOutputAligned, int32_t wOutput, int32_t hwOutput, int32_t highOutputOffset, int32_t highOutputPlaneActual,136+ int32_t wOutputAligned, int32_t wOutput, int32_t hwOutput, int32_t baseOffset,
138- int32_t highArgmaxPlaneActual, MicroAPI::RegTensor<uint32_t>& magicHighReg, int16_t shiftHigh)137+ int32_t highOutputPlaneActual, int32_t highArgmaxPlaneActual,
138+ MicroAPI::RegTensor<uint32_t>& magicHighReg, int16_t shiftHigh)
139{139{
140 MicroAPI::MaskReg allMask = MicroAPI::CreateMask<uint32_t, MicroAPI::MaskPattern::ALL>();140 MicroAPI::MaskReg allMask = MicroAPI::CreateMask<uint32_t, MicroAPI::MaskPattern::ALL>();
141- MicroAPI::RegTensor<uint32_t> dTmpU32;
142- MicroAPI::RegTensor<uint32_t> hTmpU32;
143- MicroAPI::RegTensor<uint32_t> wTmpU32;
144 MicroAPI::RegTensor<uint32_t> remU32;141 MicroAPI::RegTensor<uint32_t> remU32;
145 142 
146- FastDivImpl(dTmpU32, (MicroAPI::RegTensor<uint32_t>&)argmaxReg, magicHWReg, shiftHW, allMask);143+ FastDivImpl(dTmpReg, (MicroAPI::RegTensor<uint32_t>&)argmaxReg, magicHWReg, shiftHW, allMask);
147- MicroAPI::Muls(remU32, dTmpU32, uint32_t(hwOutput), allMask);144+ MicroAPI::Muls(remU32, dTmpReg, uint32_t(hwOutput), allMask);
148 MicroAPI::Sub(remU32, (MicroAPI::RegTensor<uint32_t>&)argmaxReg, remU32, allMask);145 MicroAPI::Sub(remU32, (MicroAPI::RegTensor<uint32_t>&)argmaxReg, remU32, allMask);
149 146 
150- FastDivImpl(hTmpU32, remU32, magicWReg, shiftW, allMask);147+ FastDivImpl(hTmpReg, remU32, magicWReg, shiftW, allMask);
151- MicroAPI::Muls(wTmpU32, hTmpU32, uint32_t(wOutput), allMask);148+ MicroAPI::Muls(wTmpReg, hTmpReg, uint32_t(wOutput), allMask);
152- MicroAPI::Sub(wTmpU32, remU32, wTmpU32, allMask);149+ MicroAPI::Sub(wTmpReg, remU32, wTmpReg, allMask);
153- 
154- MicroAPI::Adds(dIndexReg, (MicroAPI::RegTensor<int32_t>&)dTmpU32, int32_t(-curDIndex), allMask);
155- MicroAPI::Adds(hIndexReg, (MicroAPI::RegTensor<int32_t>&)hTmpU32, int32_t(-curHIndex), allMask);
156- MicroAPI::Adds(wIndexReg, (MicroAPI::RegTensor<int32_t>&)wTmpU32, int32_t(-curWIndex), allMask);
157- 
158- int32_t hwOutputAligned = hOutputActual * wOutputAligned;
159- MicroAPI::Muls(argmaxReg, hIndexReg, int32_t(wOutputAligned), allMask);
160- MicroAPI::Add(argmaxReg, argmaxReg, wIndexReg, allMask);
161- MicroAPI::Adds(argmaxReg, argmaxReg, highOutputOffset, allMask);
162 150 
151+ MicroAPI::Muls(argmaxReg, (MicroAPI::RegTensor<int32_t>&)hTmpReg, int32_t(wOutputAligned), allMask);
152+ MicroAPI::Add(argmaxReg, argmaxReg, (MicroAPI::RegTensor<int32_t>&)wTmpReg, allMask);
163 MicroAPI::RegTensor<int32_t> dhwTmpIndexReg;153 MicroAPI::RegTensor<int32_t> dhwTmpIndexReg;
164- MicroAPI::Muls(dhwTmpIndexReg, dIndexReg, int32_t(hwOutputAligned), allMask);154+ MicroAPI::Muls(dhwTmpIndexReg, (MicroAPI::RegTensor<int32_t>&)dTmpReg, int32_t(hwOutputAligned), allMask);
165 MicroAPI::Add(argmaxReg, argmaxReg, dhwTmpIndexReg, allMask);155 MicroAPI::Add(argmaxReg, argmaxReg, dhwTmpIndexReg, allMask);
156+ MicroAPI::Adds(argmaxReg, argmaxReg, baseOffset, allMask);
166 157 
167 if constexpr (IS_MUL_NC == 1) {158 if constexpr (IS_MUL_NC == 1) {
168 MicroAPI::RegTensor<int32_t> highIncRegI32;159 MicroAPI::RegTensor<int32_t> highIncRegI32;
@@ -192,21 +183,25 @@ __aicore__ inline int64_t PEnd(int64_t index, int64_t pad, int64_t stride, int64
192};183};
193 184 
194__aicore__ inline void FilterMask3D(185__aicore__ inline void FilterMask3D(
195- MicroAPI::MaskReg& preg, MicroAPI::RegTensor<int32_t>& dIndexReg, MicroAPI::RegTensor<int32_t>& hIndexReg,186+ MicroAPI::MaskReg& preg, MicroAPI::RegTensor<uint32_t>& dTmpReg, MicroAPI::RegTensor<uint32_t>& hTmpReg,
196- MicroAPI::RegTensor<int32_t>& wIndexReg, MicroAPI::RegTensor<int32_t>& zeroConstReg,187+ MicroAPI::RegTensor<uint32_t>& wTmpReg, MicroAPI::RegTensor<int32_t>& dLowerReg,
197- MicroAPI::RegTensor<int32_t>& dMaxReg, MicroAPI::RegTensor<int32_t>& hMaxReg, MicroAPI::RegTensor<int32_t>& wMaxReg)188+ MicroAPI::RegTensor<int32_t>& hLowerReg, MicroAPI::RegTensor<int32_t>& wLowerReg,
189+ MicroAPI::RegTensor<int32_t>& dUpperReg, MicroAPI::RegTensor<int32_t>& hUpperReg,
190+ MicroAPI::RegTensor<int32_t>& wUpperReg)
198{191{
199- AscendC::MicroAPI::MaskReg gtMask = AscendC::MicroAPI::CreateMask<int32_t, AscendC::MicroAPI::MaskPattern::ALL>();
200 AscendC::MicroAPI::MaskReg allMask = AscendC::MicroAPI::CreateMask<int32_t, AscendC::MicroAPI::MaskPattern::ALL>();192 AscendC::MicroAPI::MaskReg allMask = AscendC::MicroAPI::CreateMask<int32_t, AscendC::MicroAPI::MaskPattern::ALL>();
201- AscendC::MicroAPI::Compare<int32_t, CMPMODE::GE>(gtMask, hIndexReg, zeroConstReg, gtMask);193+ AscendC::MicroAPI::MaskReg hMask;
202- AscendC::MicroAPI::Compare<int32_t, CMPMODE::GT>(gtMask, hMaxReg, hIndexReg, gtMask);194+ AscendC::MicroAPI::MaskReg wMask;
203- 195+ AscendC::MicroAPI::MaskReg dMask;
204- AscendC::MicroAPI::Compare<int32_t, CMPMODE::GE>(gtMask, wIndexReg, zeroConstReg, gtMask);196+ AscendC::MicroAPI::Compare<int32_t, CMPMODE::GE>(hMask, (AscendC::MicroAPI::RegTensor<int32_t>&)hTmpReg, hLowerReg, allMask);
205- AscendC::MicroAPI::Compare<int32_t, CMPMODE::GT>(gtMask, wMaxReg, wIndexReg, gtMask);197+ AscendC::MicroAPI::Compare<int32_t, CMPMODE::GE>(wMask, (AscendC::MicroAPI::RegTensor<int32_t>&)wTmpReg, wLowerReg, allMask);
206- 198+ AscendC::MicroAPI::Compare<int32_t, CMPMODE::GE>(dMask, (AscendC::MicroAPI::RegTensor<int32_t>&)dTmpReg, dLowerReg, allMask);
207- AscendC::MicroAPI::Compare<int32_t, CMPMODE::GE>(gtMask, dIndexReg, zeroConstReg, gtMask);199+ AscendC::MicroAPI::Compare<int32_t, CMPMODE::GT>(hMask, hUpperReg, (AscendC::MicroAPI::RegTensor<int32_t>&)hTmpReg, hMask);
208- AscendC::MicroAPI::Compare<int32_t, CMPMODE::GT>(gtMask, dMaxReg, dIndexReg, gtMask);200+ AscendC::MicroAPI::Compare<int32_t, CMPMODE::GT>(wMask, wUpperReg, (AscendC::MicroAPI::RegTensor<int32_t>&)wTmpReg, wMask);
209- AscendC::MicroAPI::MaskAnd(preg, preg, gtMask, allMask);201+ AscendC::MicroAPI::Compare<int32_t, CMPMODE::GT>(dMask, dUpperReg, (AscendC::MicroAPI::RegTensor<int32_t>&)dTmpReg, dMask);
202+ AscendC::MicroAPI::MaskAnd(hMask, hMask, wMask, allMask);
203+ AscendC::MicroAPI::MaskAnd(dMask, dMask, hMask, allMask);
204+ AscendC::MicroAPI::MaskAnd(preg, preg, dMask, allMask);
210}205}
211 206 
212template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>207template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>
@@ -214,17 +209,17 @@ __aicore__ inline void DoSingleNCNchwFastDiv(
214 __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr,209 __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr,
215 MicroAPI::RegTensor<uint32_t>& parallelRegIndex, uint32_t argmaxMaskCount,210 MicroAPI::RegTensor<uint32_t>& parallelRegIndex, uint32_t argmaxMaskCount,
216 MicroAPI::RegTensor<uint32_t>& magicHWReg, int16_t shiftHW, MicroAPI::RegTensor<uint32_t>& magicWReg,211 MicroAPI::RegTensor<uint32_t>& magicHWReg, int16_t shiftHW, MicroAPI::RegTensor<uint32_t>& magicWReg,
217- int16_t shiftW, int64_t curDIndex, int64_t curHIndex, int64_t curWIndex, int32_t hOutputActual,212+ int16_t shiftW, int32_t hwOutputAligned, int32_t wOutputAligned, int32_t wOutput, int32_t hwOutput,
218- int32_t wOutputAligned, int32_t wOutput, int32_t hwOutput, int32_t highOutputOffset,213+ int32_t baseOffset,
219- MicroAPI::RegTensor<int32_t>& zeroConstReg, MicroAPI::RegTensor<int32_t>& wMaxReg,214+ MicroAPI::RegTensor<int32_t>& dLowerReg, MicroAPI::RegTensor<int32_t>& hLowerReg,
220- MicroAPI::RegTensor<int32_t>& hMaxReg, MicroAPI::RegTensor<int32_t>& dMaxReg)215+ MicroAPI::RegTensor<int32_t>& wLowerReg, MicroAPI::RegTensor<int32_t>& dUpperReg,
216+ MicroAPI::RegTensor<int32_t>& hUpperReg, MicroAPI::RegTensor<int32_t>& wUpperReg)
221{217{
222 AscendC::MicroAPI::RegTensor<computeType> gradReg;218 AscendC::MicroAPI::RegTensor<computeType> gradReg;
223 AscendC::MicroAPI::RegTensor<int32_t> argmaxReg;219 AscendC::MicroAPI::RegTensor<int32_t> argmaxReg;
224- 220+ AscendC::MicroAPI::RegTensor<uint32_t> dTmpReg;
225- AscendC::MicroAPI::RegTensor<int32_t> dIndexReg;221+ AscendC::MicroAPI::RegTensor<uint32_t> hTmpReg;
226- AscendC::MicroAPI::RegTensor<int32_t> hIndexReg;222+ AscendC::MicroAPI::RegTensor<uint32_t> wTmpReg;
227- AscendC::MicroAPI::RegTensor<int32_t> wIndexReg;
228 223 
229 MicroAPI::RegTensor<uint32_t> dummyMagicHighReg;224 MicroAPI::RegTensor<uint32_t> dummyMagicHighReg;
230 int16_t dummyShiftHigh = 0;225 int16_t dummyShiftHigh = 0;
@@ -236,15 +231,24 @@ __aicore__ inline void DoSingleNCNchwFastDiv(
236 231 
237 GetConCurrentInput<T1, T2>(argmaxReg, gradReg, gradAddr, argmaxAddr, parallelRegIndex, pregT1, pregT2);232 GetConCurrentInput<T1, T2>(argmaxReg, gradReg, gradAddr, argmaxAddr, parallelRegIndex, pregT1, pregT2);
238 IndexConvNcdhwFastDiv<0>(233 IndexConvNcdhwFastDiv<0>(
239- argmaxReg, dIndexReg, hIndexReg, wIndexReg, magicHWReg, shiftHW, magicWReg, shiftW, curDIndex, curHIndex,234+ argmaxReg, dTmpReg, hTmpReg, wTmpReg, magicHWReg, shiftHW, magicWReg, shiftW,
240- curWIndex, hOutputActual, wOutputAligned, wOutput, hwOutput, highOutputOffset, 0, 0, dummyMagicHighReg,235+ hwOutputAligned, wOutputAligned, wOutput, hwOutput, baseOffset, 0, 0,
241- dummyShiftHigh);236+ dummyMagicHighReg, dummyShiftHigh);
242- uint32_t argmaxMask = argmaxMaskCount;237+ if constexpr (std::is_same<T2, int32_t>::value) {
243- AscendC::MicroAPI::MaskReg pregArgmax = AscendC::MicroAPI::UpdateMask<int32_t>(argmaxMask);238+ if constexpr (IS_CHECK_RANGE == 1) {
244- if constexpr (IS_CHECK_RANGE == 1) {239+ FilterMask3D(pregT2, dTmpReg, hTmpReg, wTmpReg, dLowerReg, hLowerReg, wLowerReg,
245- FilterMask3D(pregArgmax, dIndexReg, hIndexReg, wIndexReg, zeroConstReg, dMaxReg, hMaxReg, wMaxReg);240+ dUpperReg, hUpperReg, wUpperReg);
241+ }
242+ GradientAcc<int32_t>(yAddr, gradReg, argmaxReg, pregT2);
243+ } else {
244+ uint32_t argmaxMask = argmaxMaskCount;
245+ AscendC::MicroAPI::MaskReg pregArgmax = AscendC::MicroAPI::UpdateMask<int32_t>(argmaxMask);
246+ if constexpr (IS_CHECK_RANGE == 1) {
247+ FilterMask3D(pregArgmax, dTmpReg, hTmpReg, wTmpReg, dLowerReg, hLowerReg, wLowerReg,
248+ dUpperReg, hUpperReg, wUpperReg);
249+ }
250+ GradientAcc<int32_t>(yAddr, gradReg, argmaxReg, pregArgmax);
246 }251 }
247- GradientAcc<int32_t>(yAddr, gradReg, argmaxReg, pregArgmax);
248}252}
249 253 
250template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>254template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>
@@ -252,36 +256,16 @@ __aicore__ inline void DoSingleNCNcdhwFastDiv(
252 __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr,256 __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr,
253 MicroAPI::RegTensor<uint32_t>& parallelRegIndex, uint32_t argmaxMaskCount,257 MicroAPI::RegTensor<uint32_t>& parallelRegIndex, uint32_t argmaxMaskCount,
254 MicroAPI::RegTensor<uint32_t>& magicHWReg, int16_t shiftHW, MicroAPI::RegTensor<uint32_t>& magicWReg,258 MicroAPI::RegTensor<uint32_t>& magicHWReg, int16_t shiftHW, MicroAPI::RegTensor<uint32_t>& magicWReg,
255- int16_t shiftW, int64_t curDIndex, int64_t curHIndex, int64_t curWIndex, int32_t wOutputAligned,259+ int16_t shiftW, int32_t hwOutputAligned, int32_t wOutputAligned, int32_t wOutput, int32_t hwOutput,
256- int32_t highOutputOffset, int32_t hOutputActual, int32_t wOutput, int32_t hwOutput,260+ int32_t baseOffset,
257- MicroAPI::RegTensor<int32_t>& zeroConstReg, MicroAPI::RegTensor<int32_t>& dMaxReg,261+ MicroAPI::RegTensor<int32_t>& dLowerReg, MicroAPI::RegTensor<int32_t>& hLowerReg,
258- MicroAPI::RegTensor<int32_t>& hMaxReg, MicroAPI::RegTensor<int32_t>& wMaxReg)262+ MicroAPI::RegTensor<int32_t>& wLowerReg, MicroAPI::RegTensor<int32_t>& dUpperReg,
263+ MicroAPI::RegTensor<int32_t>& hUpperReg, MicroAPI::RegTensor<int32_t>& wUpperReg)
259{264{
260- AscendC::MicroAPI::RegTensor<computeType> gradReg;265+ DoSingleNCNchwFastDiv<T1, T2, IS_CHECK_RANGE>(
261- AscendC::MicroAPI::RegTensor<int32_t> argmaxReg;266+ yAddr, gradAddr, argmaxAddr, parallelRegIndex, argmaxMaskCount, magicHWReg, shiftHW, magicWReg, shiftW,
262- AscendC::MicroAPI::RegTensor<int32_t> dIndexReg;267+ hwOutputAligned, wOutputAligned, wOutput, hwOutput, baseOffset,
263- AscendC::MicroAPI::RegTensor<int32_t> hIndexReg;268+ dLowerReg, hLowerReg, wLowerReg, dUpperReg, hUpperReg, wUpperReg);
264- AscendC::MicroAPI::RegTensor<int32_t> wIndexReg;
265- 
266- MicroAPI::RegTensor<uint32_t> dummyMagicHighReg;
267- int16_t dummyShiftHigh = 0;
268- 
269- uint32_t maskT1 = argmaxMaskCount;
270- uint32_t maskT2 = argmaxMaskCount;
271- AscendC::MicroAPI::MaskReg pregT1 = AscendC::MicroAPI::UpdateMask<T1>(maskT1);
272- AscendC::MicroAPI::MaskReg pregT2 = GenT2Mask<T2>(maskT2);
273- GetConCurrentInput<T1, T2>(argmaxReg, gradReg, gradAddr, argmaxAddr, parallelRegIndex, pregT1, pregT2);
274- IndexConvNcdhwFastDiv<0>(
275- argmaxReg, dIndexReg, hIndexReg, wIndexReg, magicHWReg, shiftHW, magicWReg, shiftW, curDIndex, curHIndex,
276- curWIndex, hOutputActual, wOutputAligned, wOutput, hwOutput, highOutputOffset, 0, 0, dummyMagicHighReg,
277- dummyShiftHigh);
278- uint32_t argmaxMask = argmaxMaskCount;
279- AscendC::MicroAPI::MaskReg pregArgmax = AscendC::MicroAPI::UpdateMask<int32_t>(argmaxMask);
280- if constexpr (IS_CHECK_RANGE == 1) {
281- FilterMask3D(pregArgmax, dIndexReg, hIndexReg, wIndexReg, zeroConstReg, dMaxReg, hMaxReg, wMaxReg);
282- }
283- 
284- GradientAcc<int32_t>(yAddr, gradReg, argmaxReg, pregArgmax);
285}269}
286 270 
287template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>271template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>
@@ -289,18 +273,19 @@ __aicore__ inline void DoMulNCNcdhwFastDiv(
289 __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr,273 __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr,
290 MicroAPI::RegTensor<uint32_t>& parallelRegIndex, uint32_t argmaxMaskCount,274 MicroAPI::RegTensor<uint32_t>& parallelRegIndex, uint32_t argmaxMaskCount,
291 MicroAPI::RegTensor<uint32_t>& magicHWReg, int16_t shiftHW, MicroAPI::RegTensor<uint32_t>& magicWReg,275 MicroAPI::RegTensor<uint32_t>& magicHWReg, int16_t shiftHW, MicroAPI::RegTensor<uint32_t>& magicWReg,
292- int16_t shiftW, int64_t curDIndex, int64_t curHIndex, int64_t curWIndex, int32_t wOutputAligned,276+ int16_t shiftW, int32_t hwOutputAligned, int32_t wOutputAligned, int32_t wOutput, int32_t hwOutput,
293- int32_t highOutputOffset, int32_t hOutputActual, int32_t wOutput, int32_t hwOutput,277+ int32_t baseOffset,
294- MicroAPI::RegTensor<int32_t>& zeroConstReg, MicroAPI::RegTensor<int32_t>& dMaxReg,278+ MicroAPI::RegTensor<int32_t>& dLowerReg, MicroAPI::RegTensor<int32_t>& hLowerReg,
295- MicroAPI::RegTensor<int32_t>& hMaxReg, MicroAPI::RegTensor<int32_t>& wMaxReg, int32_t highOutputPlaneActual,279+ MicroAPI::RegTensor<int32_t>& wLowerReg, MicroAPI::RegTensor<int32_t>& dUpperReg,
296- int32_t highArgmaxPlaneActual, MicroAPI::RegTensor<uint32_t>& magicHighReg, int16_t shiftHigh,280+ MicroAPI::RegTensor<int32_t>& hUpperReg, MicroAPI::RegTensor<int32_t>& wUpperReg,
297- __local_mem__ uint32_t* helpAddr)281+ int32_t highOutputPlaneActual, int32_t highArgmaxPlaneActual,
282+ MicroAPI::RegTensor<uint32_t>& magicHighReg, int16_t shiftHigh)
298{283{
299 AscendC::MicroAPI::RegTensor<computeType> gradReg;284 AscendC::MicroAPI::RegTensor<computeType> gradReg;
300 AscendC::MicroAPI::RegTensor<int32_t> argmaxReg;285 AscendC::MicroAPI::RegTensor<int32_t> argmaxReg;
301- AscendC::MicroAPI::RegTensor<int32_t> dIndexReg;286+ AscendC::MicroAPI::RegTensor<uint32_t> dTmpReg;
302- AscendC::MicroAPI::RegTensor<int32_t> hIndexReg;287+ AscendC::MicroAPI::RegTensor<uint32_t> hTmpReg;
303- AscendC::MicroAPI::RegTensor<int32_t> wIndexReg;288+ AscendC::MicroAPI::RegTensor<uint32_t> wTmpReg;
304 uint32_t maskT1 = argmaxMaskCount;289 uint32_t maskT1 = argmaxMaskCount;
305 uint32_t maskT2 = argmaxMaskCount;290 uint32_t maskT2 = argmaxMaskCount;
306 AscendC::MicroAPI::MaskReg pregT1 = AscendC::MicroAPI::UpdateMask<T1>(maskT1);291 AscendC::MicroAPI::MaskReg pregT1 = AscendC::MicroAPI::UpdateMask<T1>(maskT1);
@@ -308,17 +293,25 @@ __aicore__ inline void DoMulNCNcdhwFastDiv(
308 GetConCurrentInput<T1, T2>(argmaxReg, gradReg, gradAddr, argmaxAddr, parallelRegIndex, pregT1, pregT2);293 GetConCurrentInput<T1, T2>(argmaxReg, gradReg, gradAddr, argmaxAddr, parallelRegIndex, pregT1, pregT2);
309 294 
310 IndexConvNcdhwFastDiv<1>(295 IndexConvNcdhwFastDiv<1>(
311- argmaxReg, dIndexReg, hIndexReg, wIndexReg, magicHWReg, shiftHW, magicWReg, shiftW, curDIndex, curHIndex,296+ argmaxReg, dTmpReg, hTmpReg, wTmpReg, magicHWReg, shiftHW, magicWReg, shiftW,
312- curWIndex, hOutputActual, wOutputAligned, wOutput, hwOutput, highOutputOffset, highOutputPlaneActual,297+ hwOutputAligned, wOutputAligned, wOutput, hwOutput, baseOffset,
313- highArgmaxPlaneActual, magicHighReg, shiftHigh);298+ highOutputPlaneActual, highArgmaxPlaneActual, magicHighReg, shiftHigh);
314 299 
315- uint32_t argmaxMask = argmaxMaskCount;300+ if constexpr (std::is_same<T2, int32_t>::value) {
316- AscendC::MicroAPI::MaskReg pregArgmax = AscendC::MicroAPI::UpdateMask<int32_t>(argmaxMask);301+ if constexpr (IS_CHECK_RANGE == 1) {
317- if constexpr (IS_CHECK_RANGE == 1) {302+ FilterMask3D(pregT2, dTmpReg, hTmpReg, wTmpReg, dLowerReg, hLowerReg, wLowerReg,
318- FilterMask3D(pregArgmax, dIndexReg, hIndexReg, wIndexReg, zeroConstReg, dMaxReg, hMaxReg, wMaxReg);303+ dUpperReg, hUpperReg, wUpperReg);
304+ }
305+ GradientAcc<int32_t>(yAddr, gradReg, argmaxReg, pregT2);
306+ } else {
307+ uint32_t argmaxMask = argmaxMaskCount;
308+ AscendC::MicroAPI::MaskReg pregArgmax = AscendC::MicroAPI::UpdateMask<int32_t>(argmaxMask);
309+ if constexpr (IS_CHECK_RANGE == 1) {
310+ FilterMask3D(pregArgmax, dTmpReg, hTmpReg, wTmpReg, dLowerReg, hLowerReg, wLowerReg,
311+ dUpperReg, hUpperReg, wUpperReg);
312+ }
313+ GradientAcc<int32_t>(yAddr, gradReg, argmaxReg, pregArgmax);
319 }314 }
320- 
321- GradientAcc<int32_t>(yAddr, gradReg, argmaxReg, pregArgmax);
322}315}
323 316 
324template <typename T>317template <typename T>
@@ -329,6 +322,32 @@ __aicore__ inline void GenInitial1DIndices(MicroAPI::RegTensor<T>& indexReg, int
329 AscendC::MicroAPI::Muls(indexReg, indexReg, T(colGenRate), preg);322 AscendC::MicroAPI::Muls(indexReg, indexReg, T(colGenRate), preg);
330}323}
331 324 
325+template <typename T>
326+__aicore__ inline void GenInitial2DHighIndices(
327+ MicroAPI::RegTensor<T>& indexReg, int64_t highStride, int64_t colGenRate, int64_t fullBatchColNum)
328+{
329+ AscendC::MicroAPI::Arange(indexReg, 0);
330+ AscendC::MicroAPI::RegTensor<T> segmentScalarReg;
331+ AscendC::MicroAPI::RegTensor<T> segmentIncReg;
332+ AscendC::MicroAPI::RegTensor<T> constReg;
333+ AscendC::MicroAPI::Duplicate(constReg, T(fullBatchColNum));
334+ AscendC::MicroAPI::MaskReg preg = AscendC::MicroAPI::CreateMask<T, AscendC::MicroAPI::MaskPattern::ALL>();
335+ AscendC::MicroAPI::Div(segmentScalarReg, indexReg, constReg, preg);
336+ AscendC::MicroAPI::Muls(segmentIncReg, segmentScalarReg, T(fullBatchColNum), preg);
337+ AscendC::MicroAPI::Sub(segmentIncReg, indexReg, segmentIncReg, preg);
338+ AscendC::MicroAPI::Muls(segmentIncReg, segmentIncReg, T(colGenRate), preg);
339+ AscendC::MicroAPI::Muls(segmentScalarReg, segmentScalarReg, T(highStride), preg);
340+ AscendC::MicroAPI::Add(indexReg, segmentScalarReg, segmentIncReg, preg);
341+}
342+ 
343+template <typename T>
344+__aicore__ inline void Gen2DHighIndexOne(MicroAPI::RegTensor<T>& indexReg, int64_t highStride)
345+{
346+ AscendC::MicroAPI::Arange(indexReg, 0);
347+ AscendC::MicroAPI::MaskReg preg = AscendC::MicroAPI::CreateMask<T, AscendC::MicroAPI::MaskPattern::ALL>();
348+ AscendC::MicroAPI::Muls(indexReg, indexReg, T(highStride), preg);
349+}
350+ 
332template <typename T>351template <typename T>
333__aicore__ inline void GenInitial2DIndices(352__aicore__ inline void GenInitial2DIndices(
334 MicroAPI::RegTensor<T>& indexReg, int64_t colGenRate, int64_t rowGenRate, int64_t colNumAligned,353 MicroAPI::RegTensor<T>& indexReg, int64_t colGenRate, int64_t rowGenRate, int64_t colNumAligned,
@@ -352,30 +371,7 @@ __aicore__ inline void DhwGenInitial2DIndices(
352 MicroAPI::RegTensor<T>& indexReg, int64_t colGenRate, int64_t rowGenRate, int64_t colNumAligned,371 MicroAPI::RegTensor<T>& indexReg, int64_t colGenRate, int64_t rowGenRate, int64_t colNumAligned,
353 int64_t fullBatchColNum)372 int64_t fullBatchColNum)
354{373{
355- AscendC::MicroAPI::Arange(indexReg, 0);374+ GenInitial2DIndices<T>(indexReg, colGenRate, rowGenRate, colNumAligned, fullBatchColNum);
356- AscendC::MicroAPI::RegTensor<T> segmentScalarReg;
357- AscendC::MicroAPI::RegTensor<T> segmentIncReg;
358- AscendC::MicroAPI::RegTensor<T> constReg;
359- AscendC::MicroAPI::Duplicate(constReg, T(fullBatchColNum));
360- AscendC::MicroAPI::MaskReg preg = AscendC::MicroAPI::CreateMask<T, AscendC::MicroAPI::MaskPattern::ALL>();
361- 
362- AscendC::MicroAPI::Div(segmentScalarReg, indexReg, constReg, preg);
363- 
364- AscendC::MicroAPI::Muls(segmentIncReg, segmentScalarReg, T(fullBatchColNum), preg);
365- AscendC::MicroAPI::Sub(segmentIncReg, indexReg, segmentIncReg, preg);
366- 
367- AscendC::MicroAPI::Muls(segmentIncReg, segmentIncReg, T(colGenRate), preg);
368- AscendC::MicroAPI::Muls(segmentScalarReg, segmentScalarReg, T(rowGenRate * colNumAligned), preg);
369- 
370- AscendC::MicroAPI::Add(indexReg, segmentScalarReg, segmentIncReg, preg);
371-}
372- 
373-template <typename T>
374-__aicore__ inline void DhwGen2DIndexOne(MicroAPI::RegTensor<T>& indexReg, int64_t rowGenRate, int64_t colNumAligned)
375-{
376- AscendC::MicroAPI::Arange(indexReg, 0);
377- AscendC::MicroAPI::MaskReg preg = AscendC::MicroAPI::CreateMask<T, AscendC::MicroAPI::MaskPattern::ALL>();
378- AscendC::MicroAPI::Muls(indexReg, indexReg, T(rowGenRate * colNumAligned), preg);
379}375}
380 376 
381template <typename T>377template <typename T>
@@ -386,6 +382,12 @@ __aicore__ inline void Gen2DIndexOne(MicroAPI::RegTensor<T>& indexReg, int64_t r
386 AscendC::MicroAPI::Muls(indexReg, indexReg, T(rowGenRate * colNumAligned), preg);382 AscendC::MicroAPI::Muls(indexReg, indexReg, T(rowGenRate * colNumAligned), preg);
387}383}
388 384 
385+template <typename T>
386+__aicore__ inline void DhwGen2DIndexOne(MicroAPI::RegTensor<T>& indexReg, int64_t rowGenRate, int64_t colNumAligned)
387+{
388+ Gen2DIndexOne<T>(indexReg, rowGenRate, colNumAligned);
389+}
390+ 
389template <typename T>391template <typename T>
390__aicore__ inline void GenInitial3DIndices(392__aicore__ inline void GenInitial3DIndices(
391 MicroAPI::RegTensor<T>& indexReg, int64_t dGenRate, int64_t rowGenRate, int64_t colGenRate, int64_t fullBatchRowNum,393 MicroAPI::RegTensor<T>& indexReg, int64_t dGenRate, int64_t rowGenRate, int64_t colGenRate, int64_t fullBatchRowNum,
@@ -587,6 +589,259 @@ __aicore__ inline void Gen4DIndexOne(
587 AscendC::MicroAPI::Add(indexReg, indexReg, hPartReg, preg);589 AscendC::MicroAPI::Add(indexReg, indexReg, hPartReg, preg);
588}590}
589 591 
592+struct DivMagic {
593+ uint32_t magic;
594+ int16_t shift;
595+};
596+ 
597+__aicore__ inline DivMagic PrecomputeDiv(uint32_t divisor)
598+{
599+ DivMagic dm;
600+ uint32_t m = 0, s = 0;
601+ GetUintDivMagicAndShift<uint32_t>(m, s, divisor);
602+ dm.magic = m;
603+ dm.shift = static_cast<int16_t>(s);
604+ return dm;
605+}
606+ 
607+__aicore__ inline void FastDivInt32(
608+ MicroAPI::RegTensor<int32_t>& res, MicroAPI::RegTensor<int32_t>& src, const DivMagic& dm)
609+{
610+ MicroAPI::RegTensor<uint32_t> tmp;
611+ MicroAPI::RegTensor<uint32_t> magicReg;
612+ MicroAPI::Duplicate(magicReg, dm.magic);
613+ MicroAPI::MaskReg allMask = MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
614+ FastDivImpl(tmp, (MicroAPI::RegTensor<uint32_t>&)src, magicReg, dm.shift, allMask);
615+ res = (MicroAPI::RegTensor<int32_t>&)tmp;
616+}
617+ 
618+template <typename T>
619+__aicore__ inline void GenInitial2DHighIndicesFast(
620+ MicroAPI::RegTensor<T>& indexReg, int64_t highStride, int64_t colGenRate, int64_t fullBatchColNum,
621+ const DivMagic& divW)
622+{
623+ AscendC::MicroAPI::Arange(indexReg, 0);
624+ AscendC::MicroAPI::RegTensor<T> segmentScalarReg;
625+ AscendC::MicroAPI::RegTensor<T> segmentIncReg;
626+ AscendC::MicroAPI::MaskReg preg = AscendC::MicroAPI::CreateMask<T, AscendC::MicroAPI::MaskPattern::ALL>();
627+ FastDivInt32(segmentScalarReg, indexReg, divW);
628+ AscendC::MicroAPI::Muls(segmentIncReg, segmentScalarReg, T(fullBatchColNum), preg);
629+ AscendC::MicroAPI::Sub(segmentIncReg, indexReg, segmentIncReg, preg);
630+ AscendC::MicroAPI::Muls(segmentIncReg, segmentIncReg, T(colGenRate), preg);
631+ AscendC::MicroAPI::Muls(segmentScalarReg, segmentScalarReg, T(highStride), preg);
632+ AscendC::MicroAPI::Add(indexReg, segmentScalarReg, segmentIncReg, preg);
633+}
634+ 
635+template <typename T>
636+__aicore__ inline void GenInitial2DIndicesFast(
637+ MicroAPI::RegTensor<T>& indexReg, int64_t colGenRate, int64_t rowGenRate, int64_t colNumAligned,
638+ int64_t fullBatchColNum, const DivMagic& divW)
639+{
640+ AscendC::MicroAPI::Arange(indexReg, 0);
641+ AscendC::MicroAPI::RegTensor<T> segmentScalarReg;
642+ AscendC::MicroAPI::RegTensor<T> segmentIncReg;
643+ AscendC::MicroAPI::MaskReg preg = AscendC::MicroAPI::CreateMask<T, AscendC::MicroAPI::MaskPattern::ALL>();
644+ FastDivInt32(segmentScalarReg, indexReg, divW);
645+ AscendC::MicroAPI::Muls(segmentIncReg, segmentScalarReg, T(fullBatchColNum), preg);
646+ AscendC::MicroAPI::Sub(segmentIncReg, indexReg, segmentIncReg, preg);
647+ AscendC::MicroAPI::Muls(segmentIncReg, segmentIncReg, T(colGenRate), preg);
648+ AscendC::MicroAPI::Muls(segmentScalarReg, segmentScalarReg, T(rowGenRate * colNumAligned), preg);
649+ AscendC::MicroAPI::Add(indexReg, segmentScalarReg, segmentIncReg, preg);
650+}
651+ 
652+template <typename T>
653+__aicore__ inline void DhwGenInitial2DIndicesFast(
654+ MicroAPI::RegTensor<T>& indexReg, int64_t colGenRate, int64_t rowGenRate, int64_t colNumAligned,
655+ int64_t fullBatchColNum, const DivMagic& divW)
656+{
657+ GenInitial2DIndicesFast<T>(indexReg, colGenRate, rowGenRate, colNumAligned, fullBatchColNum, divW);
658+}
659+ 
660+template <typename T>
661+__aicore__ inline void GenInitial3DIndicesFast(
662+ MicroAPI::RegTensor<T>& indexReg, int64_t dGenRate, int64_t rowGenRate, int64_t colGenRate, int64_t fullBatchRowNum,
663+ int64_t rowNumCount, int64_t fullBatchColNum, int64_t colNumAligned,
664+ const DivMagic& divWH, const DivMagic& divW)
665+{
666+ AscendC::MicroAPI::Arange(indexReg, 0);
667+ AscendC::MicroAPI::RegTensor<T> segmentScalarReg;
668+ AscendC::MicroAPI::RegTensor<T> segmentIncReg;
669+ AscendC::MicroAPI::RegTensor<T> segmentScalarReg2;
670+ AscendC::MicroAPI::RegTensor<T> segmentIncReg2;
671+ AscendC::MicroAPI::MaskReg preg = AscendC::MicroAPI::CreateMask<T, AscendC::MicroAPI::MaskPattern::ALL>();
672+ 
673+ FastDivInt32(segmentScalarReg, indexReg, divWH);
674+ AscendC::MicroAPI::Muls(segmentIncReg, segmentScalarReg, T(fullBatchColNum * fullBatchRowNum), preg);
675+ AscendC::MicroAPI::Sub(segmentIncReg, indexReg, segmentIncReg, preg);
676+ 
677+ AscendC::MicroAPI::Muls(segmentScalarReg, segmentScalarReg, T(dGenRate * rowNumCount * colNumAligned), preg);
678+ 
679+ FastDivInt32(segmentScalarReg2, segmentIncReg, divW);
680+ AscendC::MicroAPI::Muls(segmentIncReg2, segmentScalarReg2, T(fullBatchColNum), preg);
681+ AscendC::MicroAPI::Sub(segmentIncReg2, segmentIncReg, segmentIncReg2, preg);
682+ AscendC::MicroAPI::Muls(segmentIncReg2, segmentIncReg2, colGenRate, preg);
683+ 
684+ AscendC::MicroAPI::Muls(segmentScalarReg2, segmentScalarReg2, T(rowGenRate * colNumAligned), preg);
685+ 
686+ AscendC::MicroAPI::Add(indexReg, segmentIncReg2, segmentScalarReg2, preg);
687+ AscendC::MicroAPI::Add(indexReg, indexReg, segmentScalarReg, preg);
688+}
689+ 
690+template <typename T>
691+__aicore__ inline void Gen3DIndexOneFast(
692+ MicroAPI::RegTensor<T>& indexReg, int64_t dGenRate, int64_t rowGenRate, int64_t colNumAligned,
693+ int64_t fullBatchRowNum, int64_t rowNumCount, const DivMagic& divH)
694+{
695+ AscendC::MicroAPI::Arange(indexReg, 0);
696+ AscendC::MicroAPI::RegTensor<T> segmentScalarReg;
697+ AscendC::MicroAPI::RegTensor<T> segmentIncReg;
698+ AscendC::MicroAPI::MaskReg preg = AscendC::MicroAPI::CreateMask<T, AscendC::MicroAPI::MaskPattern::ALL>();
699+ 
700+ FastDivInt32(segmentScalarReg, indexReg, divH);
701+ AscendC::MicroAPI::Muls(segmentIncReg, segmentScalarReg, T(1 * fullBatchRowNum), preg);
702+ AscendC::MicroAPI::Sub(segmentIncReg, indexReg, segmentIncReg, preg);
703+ 
704+ AscendC::MicroAPI::Muls(segmentScalarReg, segmentScalarReg, T(dGenRate * rowNumCount * colNumAligned), preg);
705+ AscendC::MicroAPI::Muls(segmentIncReg, segmentIncReg, T(rowGenRate * colNumAligned), preg);
706+ 
707+ AscendC::MicroAPI::Add(indexReg, segmentIncReg, segmentScalarReg, preg);
708+}
709+ 
710+template <typename T>
711+__aicore__ inline void GenInitial3DHighIndicesFast(
712+ MicroAPI::RegTensor<T>& indexReg, int64_t highStride, int64_t colGenRate, int64_t rowGenRate, int64_t colNumAligned,
713+ int64_t fullBatchColNum, int64_t fullBatchRowNum,
714+ const DivMagic& divWH, const DivMagic& divW)
715+{
716+ AscendC::MicroAPI::Arange(indexReg, 0);
717+ AscendC::MicroAPI::RegTensor<T> highReg;
718+ AscendC::MicroAPI::RegTensor<T> hwReg;
719+ AscendC::MicroAPI::RegTensor<T> hReg;
720+ AscendC::MicroAPI::RegTensor<T> wReg;
721+ AscendC::MicroAPI::RegTensor<T> tmpReg;
722+ AscendC::MicroAPI::MaskReg preg = AscendC::MicroAPI::CreateMask<T, AscendC::MicroAPI::MaskPattern::ALL>();
723+ const uint64_t hStride = rowGenRate * colNumAligned;
724+ const uint64_t wStride = colGenRate;
725+ 
726+ FastDivInt32(highReg, indexReg, divWH);
727+ AscendC::MicroAPI::Muls(tmpReg, highReg, T(fullBatchColNum * fullBatchRowNum), preg);
728+ AscendC::MicroAPI::Sub(hwReg, indexReg, tmpReg, preg);
729+ 
730+ FastDivInt32(hReg, hwReg, divW);
731+ AscendC::MicroAPI::Muls(tmpReg, hReg, T(fullBatchColNum), preg);
732+ AscendC::MicroAPI::Sub(wReg, hwReg, tmpReg, preg);
733+ 
734+ AscendC::MicroAPI::RegTensor<T> highPartReg;
735+ AscendC::MicroAPI::RegTensor<T> hPartReg;
736+ AscendC::MicroAPI::RegTensor<T> wPartReg;
737+ AscendC::MicroAPI::Muls(highPartReg, highReg, T(highStride), preg);
738+ AscendC::MicroAPI::Muls(hPartReg, hReg, T(hStride), preg);
739+ AscendC::MicroAPI::Muls(wPartReg, wReg, T(wStride), preg);
740+ 
741+ AscendC::MicroAPI::Add(indexReg, highPartReg, hPartReg, preg);
742+ AscendC::MicroAPI::Add(indexReg, indexReg, wPartReg, preg);
743+}
744+ 
745+template <typename T>
746+__aicore__ inline void Gen3DHighIndexOneFast(
747+ MicroAPI::RegTensor<T>& indexReg, int64_t highStride, int64_t rowGenRate, int64_t colNumAligned,
748+ int64_t fullBatchRowNum, const DivMagic& divH)
749+{
750+ AscendC::MicroAPI::Arange(indexReg, 0);
751+ AscendC::MicroAPI::RegTensor<T> highReg;
752+ AscendC::MicroAPI::RegTensor<T> hReg;
753+ AscendC::MicroAPI::RegTensor<T> tmpReg;
754+ AscendC::MicroAPI::MaskReg preg = AscendC::MicroAPI::CreateMask<T, AscendC::MicroAPI::MaskPattern::ALL>();
755+ const uint64_t hStride = rowGenRate * colNumAligned;
756+ 
757+ FastDivInt32(highReg, indexReg, divH);
758+ AscendC::MicroAPI::Muls(tmpReg, highReg, T(1 * fullBatchRowNum), preg);
759+ AscendC::MicroAPI::Sub(hReg, indexReg, tmpReg, preg);
760+ 
761+ AscendC::MicroAPI::RegTensor<T> highPartReg;
762+ AscendC::MicroAPI::RegTensor<T> hPartReg;
763+ AscendC::MicroAPI::Muls(highPartReg, highReg, T(highStride), preg);
764+ AscendC::MicroAPI::Muls(hPartReg, hReg, T(hStride), preg);
765+ AscendC::MicroAPI::Add(indexReg, highPartReg, hPartReg, preg);
766+}
767+ 
768+template <typename T>
769+__aicore__ inline void GenInitial4DIndicesFast(
770+ MicroAPI::RegTensor<T>& indexReg, int64_t colGenRate, int64_t rowGenRate, int64_t colNumAligned,
771+ int64_t fullBatchColNum, int64_t fullBatchRowNum, int64_t fullBatchDepthNum, int64_t depthStride,
772+ int64_t highStride,
773+ const DivMagic& divDHW, const DivMagic& divHW, const DivMagic& divW)
774+{
775+ AscendC::MicroAPI::Arange(indexReg, 0);
776+ AscendC::MicroAPI::MaskReg preg = AscendC::MicroAPI::CreateMask<T, AscendC::MicroAPI::MaskPattern::ALL>();
777+ 
778+ AscendC::MicroAPI::RegTensor<T> highReg;
779+ AscendC::MicroAPI::RegTensor<T> dReg;
780+ AscendC::MicroAPI::RegTensor<T> dhwReg;
781+ AscendC::MicroAPI::RegTensor<T> hwReg;
782+ AscendC::MicroAPI::RegTensor<T> hReg;
783+ AscendC::MicroAPI::RegTensor<T> wReg;
784+ AscendC::MicroAPI::RegTensor<T> tmpReg;
785+ 
786+ FastDivInt32(highReg, indexReg, divDHW);
787+ AscendC::MicroAPI::Muls(dReg, highReg, T(fullBatchColNum * fullBatchRowNum * fullBatchDepthNum), preg);
788+ AscendC::MicroAPI::Sub(dhwReg, indexReg, dReg, preg);
789+ 
790+ FastDivInt32(dReg, dhwReg, divHW);
791+ AscendC::MicroAPI::Muls(hwReg, dReg, T(fullBatchRowNum * fullBatchColNum), preg);
792+ AscendC::MicroAPI::Sub(hwReg, dhwReg, hwReg, preg);
793+ 
794+ FastDivInt32(hReg, hwReg, divW);
795+ AscendC::MicroAPI::Muls(wReg, hReg, T(fullBatchColNum), preg);
796+ AscendC::MicroAPI::Sub(wReg, hwReg, wReg, preg);
797+ 
798+ AscendC::MicroAPI::RegTensor<T> highPartReg;
799+ AscendC::MicroAPI::RegTensor<T> dPartReg;
800+ AscendC::MicroAPI::RegTensor<T> hPartReg;
801+ AscendC::MicroAPI::RegTensor<T> wPartReg;
802+ 
803+ AscendC::MicroAPI::Muls(highPartReg, highReg, T(highStride), preg);
804+ AscendC::MicroAPI::Muls(dPartReg, dReg, T(depthStride), preg);
805+ AscendC::MicroAPI::Muls(hPartReg, hReg, T(rowGenRate * colNumAligned), preg);
806+ AscendC::MicroAPI::Muls(wPartReg, wReg, T(colGenRate), preg);
807+ AscendC::MicroAPI::Add(indexReg, highPartReg, dPartReg, preg);
808+ AscendC::MicroAPI::Add(indexReg, indexReg, hPartReg, preg);
809+ AscendC::MicroAPI::Add(indexReg, indexReg, wPartReg, preg);
810+}
811+ 
812+template <typename T>
813+__aicore__ inline void Gen4DIndexOneFast(
814+ MicroAPI::RegTensor<T>& indexReg, int64_t rowGenRate, int64_t colNumAligned, int64_t fullBatchRowNum,
815+ int64_t fullBatchDepthNum, int64_t depthStride, int64_t highStride,
816+ const DivMagic& divHD, const DivMagic& divH)
817+{
818+ AscendC::MicroAPI::Arange(indexReg, 0);
819+ AscendC::MicroAPI::MaskReg preg = AscendC::MicroAPI::CreateMask<T, AscendC::MicroAPI::MaskPattern::ALL>();
820+ AscendC::MicroAPI::RegTensor<T> highReg;
821+ AscendC::MicroAPI::RegTensor<T> dReg;
822+ AscendC::MicroAPI::RegTensor<T> dhwReg;
823+ AscendC::MicroAPI::RegTensor<T> tmpReg;
824+ AscendC::MicroAPI::RegTensor<T> hReg;
825+ 
826+ FastDivInt32(highReg, indexReg, divHD);
827+ AscendC::MicroAPI::Muls(dReg, highReg, T(1 * fullBatchRowNum * fullBatchDepthNum), preg);
828+ AscendC::MicroAPI::Sub(dhwReg, indexReg, dReg, preg);
829+ 
830+ FastDivInt32(dReg, dhwReg, divH);
831+ AscendC::MicroAPI::Muls(tmpReg, dReg, T(1 * fullBatchRowNum), preg);
832+ AscendC::MicroAPI::Sub(hReg, dhwReg, tmpReg, preg);
833+ 
834+ AscendC::MicroAPI::RegTensor<T> highPartReg;
835+ AscendC::MicroAPI::RegTensor<T> dPartReg;
836+ AscendC::MicroAPI::RegTensor<T> hPartReg;
837+ 
838+ AscendC::MicroAPI::Muls(highPartReg, highReg, T(highStride), preg);
839+ AscendC::MicroAPI::Muls(dPartReg, dReg, T(depthStride), preg);
840+ AscendC::MicroAPI::Muls(hPartReg, hReg, T(rowGenRate * colNumAligned), preg);
841+ AscendC::MicroAPI::Add(indexReg, highPartReg, dPartReg, preg);
842+ AscendC::MicroAPI::Add(indexReg, indexReg, hPartReg, preg);
843+}
844+ 
590template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>845template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>
591class MaxPool3DGradWithArgmaxNCDHWKernel {846class MaxPool3DGradWithArgmaxNCDHWKernel {
592public:847public:
@@ -602,17 +857,13 @@ public:
602 __aicore__ inline void CopyIn();857 __aicore__ inline void CopyIn();
603 __aicore__ inline void Compute();858 __aicore__ inline void Compute();
604 __aicore__ inline void singleLineProcessVF(859 __aicore__ inline void singleLineProcessVF(
605- __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr,860+ __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr);
606- __local_mem__ uint32_t* helpAddr);
607 __aicore__ inline void multipleLineProcessVF2(861 __aicore__ inline void multipleLineProcessVF2(
608- __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr,862+ __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr);
609- __local_mem__ uint32_t* helpAddr);
610 __aicore__ inline void multipleLineHwProcessVF(863 __aicore__ inline void multipleLineHwProcessVF(
611- __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr,864+ __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr);
612- __local_mem__ uint32_t* helpAddr);
613 __aicore__ inline void multipleLineDhwProcessVF(865 __aicore__ inline void multipleLineDhwProcessVF(
614- __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr,866+ __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr);
615- __local_mem__ uint32_t* helpAddr);
616 __aicore__ inline void ProcessNoArgmaxBlock();867 __aicore__ inline void ProcessNoArgmaxBlock();
617 __aicore__ inline void CopyOut();868 __aicore__ inline void CopyOut();
618 869 
@@ -620,7 +871,6 @@ public:
620 TQue<QuePosition::VECIN, BUFFER_NUM> gradQue_;871 TQue<QuePosition::VECIN, BUFFER_NUM> gradQue_;
621 TQue<QuePosition::VECIN, BUFFER_NUM> argmaxQue_;872 TQue<QuePosition::VECIN, BUFFER_NUM> argmaxQue_;
622 TQue<QuePosition::VECOUT, BUFFER_NUM> outputQue_;873 TQue<QuePosition::VECOUT, BUFFER_NUM> outputQue_;
623- TBuf<QuePosition::VECCALC> helpBuf_;
624 874 
625 GlobalTensor<T1> gradGm_;875 GlobalTensor<T1> gradGm_;
626 GlobalTensor<T1> yGm_;876 GlobalTensor<T1> yGm_;
@@ -792,7 +1042,6 @@ __aicore__ inline void MaxPool3DGradWithArgmaxNCDHWKernel<T1, T2, IS_CHECK_RANGE
792 pipe_.InitBuffer(outputQue_, BUFFER_NUM, outputBufferSize_);1042 pipe_.InitBuffer(outputQue_, BUFFER_NUM, outputBufferSize_);
793 pipe_.InitBuffer(gradQue_, BUFFER_NUM, gradBufferSize_);1043 pipe_.InitBuffer(gradQue_, BUFFER_NUM, gradBufferSize_);
794 pipe_.InitBuffer(argmaxQue_, BUFFER_NUM, argmaxBufferSize_);1044 pipe_.InitBuffer(argmaxQue_, BUFFER_NUM, argmaxBufferSize_);
795- pipe_.InitBuffer(helpBuf_, HELP_BUFFER);
796}1045}
797 1046 
798template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>1047template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>
Mpooling/max_pool3d_grad_with_argmax/op_kernel/arch35/max_pool3d_grad_with_argmax_simd_full_load.h+368-647
@@ -36,13 +36,13 @@ public:
36 __aicore__ inline void Compute();36 __aicore__ inline void Compute();
37 37 
38 __aicore__ inline void fullLoadSingleLineProcessVF(__local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr,38 __aicore__ inline void fullLoadSingleLineProcessVF(__local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr,
39- __local_mem__ T2* argmaxAddr, __local_mem__ uint32_t* helpAddr);39+ __local_mem__ T2* argmaxAddr);
40 __aicore__ inline void fullLoadMultipleLineHwProcessVF(__local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr,40 __aicore__ inline void fullLoadMultipleLineHwProcessVF(__local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr,
41- __local_mem__ T2* argmaxAddr, __local_mem__ uint32_t* helpAddr);41+ __local_mem__ T2* argmaxAddr);
42 __aicore__ inline void fullLoadMultipleLineDhwProcessVF(__local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr,42 __aicore__ inline void fullLoadMultipleLineDhwProcessVF(__local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr,
43- __local_mem__ T2* argmaxAddr, __local_mem__ uint32_t* helpAddr);43+ __local_mem__ T2* argmaxAddr);
44 __aicore__ inline void fullLoadMultipleLineProcessVF2(__local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr,44 __aicore__ inline void fullLoadMultipleLineProcessVF2(__local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr,
45- __local_mem__ T2* argmaxAddr, __local_mem__ uint32_t* helpAddr);45+ __local_mem__ T2* argmaxAddr);
46 __aicore__ inline void ProcessNoArgmaxBlock();46 __aicore__ inline void ProcessNoArgmaxBlock();
47 __aicore__ inline void CopyOut();47 __aicore__ inline void CopyOut();
48 48 
@@ -50,7 +50,6 @@ public:
50 TQue<QuePosition::VECIN, BUFFER_NUM> gradQue_;50 TQue<QuePosition::VECIN, BUFFER_NUM> gradQue_;
51 TQue<QuePosition::VECIN, BUFFER_NUM> argmaxQue_;51 TQue<QuePosition::VECIN, BUFFER_NUM> argmaxQue_;
52 TQue<QuePosition::VECOUT, BUFFER_NUM> outputQue_;52 TQue<QuePosition::VECOUT, BUFFER_NUM> outputQue_;
53- TBuf<QuePosition::VECCALC> helpBuf_;
54 53 
55 GlobalTensor<T1> gradGm_;54 GlobalTensor<T1> gradGm_;
56 GlobalTensor<T1> yGm_;55 GlobalTensor<T1> yGm_;
@@ -121,6 +120,7 @@ public:
121 int64_t dArgmaxActual_ = 0;120 int64_t dArgmaxActual_ = 0;
122 int64_t wArgmaxActual_ = 0;121 int64_t wArgmaxActual_ = 0;
123 int64_t wArgmaxAligned_ = 0;122 int64_t wArgmaxAligned_ = 0;
123+ int64_t dhwPlaneAligned_ = 0;
124 124 
125 int64_t highAxisArgmaxOffset_ = 0;125 int64_t highAxisArgmaxOffset_ = 0;
126 int64_t hAxisArgmaxOffset_ = 0;126 int64_t hAxisArgmaxOffset_ = 0;
@@ -169,7 +169,6 @@ __aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHE
169 pipe_.InitBuffer(outputQue_, BUFFER_NUM, outputBufferSize_);169 pipe_.InitBuffer(outputQue_, BUFFER_NUM, outputBufferSize_);
170 pipe_.InitBuffer(gradQue_, BUFFER_NUM, gradBufferSize_);170 pipe_.InitBuffer(gradQue_, BUFFER_NUM, gradBufferSize_);
171 pipe_.InitBuffer(argmaxQue_, BUFFER_NUM, argmaxBufferSize_);171 pipe_.InitBuffer(argmaxQue_, BUFFER_NUM, argmaxBufferSize_);
172- pipe_.InitBuffer(helpBuf_, HELP_BUFFER);
173}172}
174 173 
175template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>174template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>
@@ -265,6 +264,10 @@ __aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHE
265 hArgmaxActual_ = hArgmaxActualEnd - hArgmaxActualStart;264 hArgmaxActual_ = hArgmaxActualEnd - hArgmaxActualStart;
266 dArgmaxActual_ = dArgmaxActualEnd - dArgmaxActualStart;265 dArgmaxActual_ = dArgmaxActualEnd - dArgmaxActualStart;
267 266 
267+ int64_t dhwPlaneElems = dArgmaxActual_ * hArgmaxActual_ * wArgmaxActual_;
268+ dhwPlaneAligned_ = (dhwPlaneElems + MAX_DATA_NUM_IN_ONE_BLOCK - 1)
269+ / MAX_DATA_NUM_IN_ONE_BLOCK * MAX_DATA_NUM_IN_ONE_BLOCK;
270+ 
268 curDProBatchSize_ = dProBatchSize_ > dArgmaxActual_ ? dArgmaxActual_ : dProBatchSize_;271 curDProBatchSize_ = dProBatchSize_ > dArgmaxActual_ ? dArgmaxActual_ : dProBatchSize_;
269 curHProBatchSize_ = hProBatchSize_ > hArgmaxActual_ ? hArgmaxActual_ : hProBatchSize_;272 curHProBatchSize_ = hProBatchSize_ > hArgmaxActual_ ? hArgmaxActual_ : hProBatchSize_;
270 curWProBatchSize_ = wProBatchSize_ > wArgmaxActual_ ? wArgmaxActual_ : wProBatchSize_;273 curWProBatchSize_ = wProBatchSize_ > wArgmaxActual_ ? wArgmaxActual_ : wProBatchSize_;
@@ -280,45 +283,34 @@ __aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHE
280{283{
281 LocalTensor<T1> gradLocal = gradQue_.AllocTensor<T1>();284 LocalTensor<T1> gradLocal = gradQue_.AllocTensor<T1>();
282 LocalTensor<T2> argmaxLocal = argmaxQue_.AllocTensor<T2>();285 LocalTensor<T2> argmaxLocal = argmaxQue_.AllocTensor<T2>();
283- int64_t planeHW = hArgmax_ * wArgmax_;
284 int64_t argmaxGmOffset = highAxisArgmaxOffset_ + dAxisArgmaxOffset_ + hAxisArgmaxOffset_ + wAxisArgmaxOffset_;286 int64_t argmaxGmOffset = highAxisArgmaxOffset_ + dAxisArgmaxOffset_ + hAxisArgmaxOffset_ + wAxisArgmaxOffset_;
285- DataCopyPadExtParams<T1> paramsT1 = {false, 0, 0, 0};
286- LoopModeParams loopModeParamsT1;
287- loopModeParamsT1.loop1Size = dArgmaxActual_;
288- loopModeParamsT1.loop2Size = highAxisActual_;
289- loopModeParamsT1.loop1SrcStride = planeHW * sizeof(T1);
290- loopModeParamsT1.loop2SrcStride = argmaxPlaneSize_ * sizeof(T1);
291- loopModeParamsT1.loop1DstStride = hArgmaxActual_ * wArgmaxAligned_ * sizeof(T1);
292- loopModeParamsT1.loop2DstStride = dArgmaxActual_ * hArgmaxActual_ * wArgmaxAligned_ * sizeof(T1);
293 287 
294- SetLoopModePara(loopModeParamsT1, DataCopyMVType::OUT_TO_UB);288+ int64_t dhwPlaneElems = dArgmaxActual_ * hArgmaxActual_ * wArgmaxActual_;
289+ 
290+ int64_t dhwPlaneBytesT1 = dhwPlaneElems * sizeof(T1);
291+ int64_t dhwPlaneBytesT2 = dhwPlaneElems * sizeof(T2);
292+ int64_t gmSrcStrideT1 = (argmaxPlaneSize_ - dhwPlaneElems) * sizeof(T1);
293+ int64_t gmSrcStrideT2 = (argmaxPlaneSize_ - dhwPlaneElems) * sizeof(T2);
294+ int64_t ubDstStrideT1 = (dhwPlaneAligned_ - dhwPlaneElems) * sizeof(T1) / BLOCK_SIZE;
295+ int64_t ubDstStrideT2 = (dhwPlaneAligned_ - dhwPlaneElems) * sizeof(T2) / BLOCK_SIZE;
296+ 
297+ DataCopyPadExtParams<T1> paramsT1 = {false, 0, 0, 0};
295 DataCopyExtParams copyOutParamT1 = {298 DataCopyExtParams copyOutParamT1 = {
296- static_cast<uint16_t>(hArgmaxActual_),299+ static_cast<uint16_t>(highAxisActual_),
297- static_cast<uint32_t>(wArgmaxActual_ * sizeof(T1)),300+ static_cast<uint32_t>(dhwPlaneBytesT1),
298- static_cast<uint32_t>((wArgmax_ - wArgmaxActual_) * sizeof(T1)),301+ static_cast<uint32_t>(gmSrcStrideT1),
299- static_cast<uint32_t>(0), static_cast<uint32_t>(0)};302+ static_cast<uint32_t>(ubDstStrideT1), static_cast<uint32_t>(0)};
300 303 
301 DataCopyPad(gradLocal, gradGm_[argmaxGmOffset], copyOutParamT1, paramsT1);304 DataCopyPad(gradLocal, gradGm_[argmaxGmOffset], copyOutParamT1, paramsT1);
302 305 
303 DataCopyPadExtParams<T2> paramsT2 = {false, 0, 0, 0};306 DataCopyPadExtParams<T2> paramsT2 = {false, 0, 0, 0};
304- LoopModeParams loopModeParamsT2;
305- loopModeParamsT2.loop1Size = dArgmaxActual_;
306- loopModeParamsT2.loop2Size = highAxisActual_;
307- loopModeParamsT2.loop1SrcStride = planeHW * sizeof(T2);
308- loopModeParamsT2.loop2SrcStride = argmaxPlaneSize_ * sizeof(T2);
309- loopModeParamsT2.loop1DstStride = hArgmaxActual_ * wArgmaxAligned_ * sizeof(T2);
310- loopModeParamsT2.loop2DstStride = dArgmaxActual_ * hArgmaxActual_ * wArgmaxAligned_ * sizeof(T2);
311- 
312- uint32_t dstStrideT2 = (wArgmaxAligned_ - wArgmaxActual_) * sizeof(T2) / BLOCK_SIZE;
313- SetLoopModePara(loopModeParamsT2, DataCopyMVType::OUT_TO_UB);
314 DataCopyExtParams copyOutParamT2 = {307 DataCopyExtParams copyOutParamT2 = {
315- static_cast<uint16_t>(hArgmaxActual_),308+ static_cast<uint16_t>(highAxisActual_),
316- static_cast<uint32_t>(wArgmaxActual_ * sizeof(T2)),309+ static_cast<uint32_t>(dhwPlaneBytesT2),
317- static_cast<uint32_t>((wArgmax_ - wArgmaxActual_) * sizeof(T2)),310+ static_cast<uint32_t>(gmSrcStrideT2),
318- static_cast<uint32_t>(dstStrideT2), static_cast<uint32_t>(0)};311+ static_cast<uint32_t>(ubDstStrideT2), static_cast<uint32_t>(0)};
319 312 
320 DataCopyPad(argmaxLocal, argmaxGm_[argmaxGmOffset], copyOutParamT2, paramsT2);313 DataCopyPad(argmaxLocal, argmaxGm_[argmaxGmOffset], copyOutParamT2, paramsT2);
321- ResetLoopModePara(DataCopyMVType::OUT_TO_UB);
322 314 
323 gradQue_.EnQue(gradLocal);315 gradQue_.EnQue(gradLocal);
324 argmaxQue_.EnQue(argmaxLocal);316 argmaxQue_.EnQue(argmaxLocal);
@@ -373,16 +365,14 @@ __aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHE
373 uint32_t hConcurrentCount = hArgmaxActual_ / curHProBatchSize_;365 uint32_t hConcurrentCount = hArgmaxActual_ / curHProBatchSize_;
374 uint32_t dConcurrentCount = dArgmaxActual_ / curDProBatchSize_;366 uint32_t dConcurrentCount = dArgmaxActual_ / curDProBatchSize_;
375 367 
376- LocalTensor<uint32_t> helpTensor = helpBuf_.Get<uint32_t>();
377- __local_mem__ uint32_t* helpAddr = (__local_mem__ uint32_t*)helpTensor.GetPhyAddr();
378 if (wConcurrentCount * DOUBLE * sizeof(T2) > V_REG_SIZE) {368 if (wConcurrentCount * DOUBLE * sizeof(T2) > V_REG_SIZE) {
379- fullLoadSingleLineProcessVF(yAddr, gradAddr, argmaxAddr, helpAddr);369+ fullLoadSingleLineProcessVF(yAddr, gradAddr, argmaxAddr);
380 } else if (wConcurrentCount * hConcurrentCount * DOUBLE * sizeof(T2) > V_REG_SIZE) {370 } else if (wConcurrentCount * hConcurrentCount * DOUBLE * sizeof(T2) > V_REG_SIZE) {
381- fullLoadMultipleLineHwProcessVF(yAddr, gradAddr, argmaxAddr, helpAddr);371+ fullLoadMultipleLineHwProcessVF(yAddr, gradAddr, argmaxAddr);
382 } else if (wConcurrentCount * hConcurrentCount * dConcurrentCount * DOUBLE * sizeof(T2) > V_REG_SIZE) {372 } else if (wConcurrentCount * hConcurrentCount * dConcurrentCount * DOUBLE * sizeof(T2) > V_REG_SIZE) {
383- fullLoadMultipleLineDhwProcessVF(yAddr, gradAddr, argmaxAddr, helpAddr);373+ fullLoadMultipleLineDhwProcessVF(yAddr, gradAddr, argmaxAddr);
384 } else {374 } else {
385- fullLoadMultipleLineProcessVF2(yAddr, gradAddr, argmaxAddr, helpAddr);375+ fullLoadMultipleLineProcessVF2(yAddr, gradAddr, argmaxAddr);
386 }376 }
387 377 
388 if constexpr (std::negation<std::is_same<T1, float>>::value) {378 if constexpr (std::negation<std::is_same<T1, float>>::value) {
@@ -419,23 +409,22 @@ __aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHE
419 409 
420template <typename T>410template <typename T>
421__aicore__ inline void IndexConvNcNcdhwFullLoad(MicroAPI::RegTensor<T>& argmaxReg,411__aicore__ inline void IndexConvNcNcdhwFullLoad(MicroAPI::RegTensor<T>& argmaxReg,
422- int64_t fullBatchCount,412+ int64_t highOutputOffset, int64_t highOutputSize,
423- int64_t highOutputOffset, int64_t highOutputSize, MicroAPI::MaskReg& pregT3)413+ MicroAPI::RegTensor<uint32_t>& magicHighReg, int16_t shiftHigh,
414+ MicroAPI::MaskReg& pregT3,
415+ MicroAPI::RegTensor<int32_t>& indexIncReg)
424{416{
425- AscendC::MicroAPI::RegTensor<T> constReg;417+ AscendC::MicroAPI::RegTensor<int32_t> localIndexIncReg;
426- AscendC::MicroAPI::Duplicate(constReg, fullBatchCount);418+ AscendC::MicroAPI::Arange(localIndexIncReg, 0);
419+ AscendC::MicroAPI::RegTensor<uint32_t> ncIndexIncRegU32;
420+ FastDivImpl(ncIndexIncRegU32, (MicroAPI::RegTensor<uint32_t>&)localIndexIncReg, magicHighReg, shiftHigh, pregT3);
427 421 
428- AscendC::MicroAPI::RegTensor<T> indexIncReg;422+ AscendC::MicroAPI::Muls((MicroAPI::RegTensor<int32_t>&)ncIndexIncRegU32,
429- AscendC::MicroAPI::Arange(indexIncReg, 0);423+ (MicroAPI::RegTensor<int32_t>&)ncIndexIncRegU32,
430- 424+ highOutputSize, pregT3);
431- AscendC::MicroAPI::RegTensor<T> ncIndexIncReg;
432- AscendC::MicroAPI::Div(ncIndexIncReg, indexIncReg, constReg, pregT3);
433- 
434- AscendC::MicroAPI::Muls(ncIndexIncReg, ncIndexIncReg, highOutputSize,
435- pregT3);
436 425 
437 AscendC::MicroAPI::Adds(argmaxReg, argmaxReg, highOutputOffset, pregT3);426 AscendC::MicroAPI::Adds(argmaxReg, argmaxReg, highOutputOffset, pregT3);
438- AscendC::MicroAPI::Add(argmaxReg, argmaxReg, ncIndexIncReg, pregT3);427+ AscendC::MicroAPI::Add(argmaxReg, argmaxReg, (MicroAPI::RegTensor<int32_t>&)ncIndexIncRegU32, pregT3);
439}428}
440 429 
441 430 
@@ -450,7 +439,9 @@ template <typename T1, typename T2>
450__aicore__ inline void DoMulNCNcdhwFullLoad(__local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr,439__aicore__ inline void DoMulNCNcdhwFullLoad(__local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr,
451 __local_mem__ T2* argmaxAddr, MicroAPI::RegTensor<uint32_t>& parallelRegIndex,440 __local_mem__ T2* argmaxAddr, MicroAPI::RegTensor<uint32_t>& parallelRegIndex,
452 uint32_t argmaxMaskCount, int32_t highOutputOffset,441 uint32_t argmaxMaskCount, int32_t highOutputOffset,
453- int32_t highOutputPlaneActual, int32_t highArgmaxPlaneActual, __local_mem__ uint32_t* helpAddr)442+ int32_t highOutputPlaneActual, int32_t highArgmaxPlaneActual,
443+ MicroAPI::RegTensor<uint32_t>& magicHighReg, int16_t shiftHigh,
444+ MicroAPI::RegTensor<int32_t>& indexIncReg)
454{445{
455 AscendC::MicroAPI::RegTensor<computeType> gradReg;446 AscendC::MicroAPI::RegTensor<computeType> gradReg;
456 AscendC::MicroAPI::RegTensor<int32_t> argmaxReg;447 AscendC::MicroAPI::RegTensor<int32_t> argmaxReg;
@@ -463,7 +454,7 @@ __aicore__ inline void DoMulNCNcdhwFullLoad(__local_mem__ computeType* yAddr, __
463 AscendC::MicroAPI::MaskReg pregT3 = AscendC::MicroAPI::UpdateMask<int32_t>(maskT3);454 AscendC::MicroAPI::MaskReg pregT3 = AscendC::MicroAPI::UpdateMask<int32_t>(maskT3);
464 GetConCurrentInput<T1, T2>(argmaxReg, gradReg, gradAddr, argmaxAddr, parallelRegIndex, pregT1, pregT2);455 GetConCurrentInput<T1, T2>(argmaxReg, gradReg, gradAddr, argmaxAddr, parallelRegIndex, pregT1, pregT2);
465 456 
466- IndexConvNcNcdhwFullLoad<int32_t>(argmaxReg, highArgmaxPlaneActual, highOutputOffset, highOutputPlaneActual, pregT3);457+ IndexConvNcNcdhwFullLoad<int32_t>(argmaxReg, highOutputOffset, highOutputPlaneActual, magicHighReg, shiftHigh, pregT3, indexIncReg);
467 uint32_t argmaxMask = argmaxMaskCount;458 uint32_t argmaxMask = argmaxMaskCount;
468 AscendC::MicroAPI::MaskReg pregArgmax = AscendC::MicroAPI::UpdateMask<int32_t>(argmaxMask);459 AscendC::MicroAPI::MaskReg pregArgmax = AscendC::MicroAPI::UpdateMask<int32_t>(argmaxMask);
469 GradientAcc<int32_t>(yAddr, gradReg, argmaxReg, pregArgmax);460 GradientAcc<int32_t>(yAddr, gradReg, argmaxReg, pregArgmax);
@@ -498,20 +489,15 @@ __aicore__ inline void DoSingleNchwFullLoad(__local_mem__ computeType* yAddr, __
498 489 
499template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>490template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>
500__aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHECK_RANGE>::fullLoadSingleLineProcessVF(491__aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHECK_RANGE>::fullLoadSingleLineProcessVF(
501- __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr, __local_mem__ uint32_t* helpAddr)492+ __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr)
502{493{
503- int64_t wOutput = wOutput_;
504 int64_t wOutputActual = wOutputActual_;494 int64_t wOutputActual = wOutputActual_;
505- int64_t wOutputAligned = wOutputAligned_;
506 int64_t hOutputActual = hOutputActual_;495 int64_t hOutputActual = hOutputActual_;
507 int64_t dOutputActual = dOutputActual_;496 int64_t dOutputActual = dOutputActual_;
508 uint16_t highAxisActual = static_cast<uint16_t>(highAxisActual_);497 uint16_t highAxisActual = static_cast<uint16_t>(highAxisActual_);
509- int64_t curDIndex = dAxisIndex_ * dOutputInner_;
510- int64_t curHIndex = hAxisIndex_ * hOutputInner_;
511- int64_t curWIndex = wAxisIndex_ * wOutputInner_;
512 uint16_t dArgmaxActual = dArgmaxActual_;498 uint16_t dArgmaxActual = dArgmaxActual_;
513 int64_t wArgmaxActual = wArgmaxActual_;499 int64_t wArgmaxActual = wArgmaxActual_;
514- int64_t wArgmaxAligned = wArgmaxAligned_;500+ int64_t wArgmaxAligned = wArgmaxActual_;
515 uint16_t hArgmaxActual = hArgmaxActual_;501 uint16_t hArgmaxActual = hArgmaxActual_;
516 uint16_t wProBatchSize = curWProBatchSize_;502 uint16_t wProBatchSize = curWProBatchSize_;
517 uint32_t wFullBatchCount = wArgmaxActual / wProBatchSize;503 uint32_t wFullBatchCount = wArgmaxActual / wProBatchSize;
@@ -526,57 +512,38 @@ __aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHE
526 {512 {
527 AscendC::MicroAPI::RegTensor<uint32_t> initialRegIndex;513 AscendC::MicroAPI::RegTensor<uint32_t> initialRegIndex;
528 GenInitial1DIndices((AscendC::MicroAPI::RegTensor<int32_t>&)initialRegIndex, wProBatchSize);514 GenInitial1DIndices((AscendC::MicroAPI::RegTensor<int32_t>&)initialRegIndex, wProBatchSize);
529- 515+ AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
530- AscendC::MicroAPI::MaskReg allMask =516+ AscendC::MicroAPI::MaskReg allMaskU32 =
531 AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();517 AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
532- AscendC::MicroAPI::DataCopy(helpAddr, initialRegIndex, allMask);
533- }
534- for (uint16_t highIdx = 0; highIdx < highAxisActual; ++highIdx) {
535- uint32_t highArgmaxOffset = highIdx * dArgmaxActual * hArgmaxActual * wArgmaxAligned;
536- uint32_t highOutputOffset = highIdx * dOutputActual * hOutputActual * wOutputActual;
537- for (uint16_t dIdx = 0; dIdx < dArgmaxActual; dIdx++) {
538- uint32_t dArgmaxOffset = dIdx * hArgmaxActual * wArgmaxAligned;
539- for (uint16_t hIdx = 0; hIdx < hArgmaxActual; hIdx++) {
540- {
541- __VEC_SCOPE__
542- {
543- AscendC::MicroAPI::RegTensor<uint32_t> initialRegIndex;
544- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
545- AscendC::MicroAPI::MaskReg allMaskU32 =
546- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
547- AscendC::MicroAPI::DataCopy(initialRegIndex, helpAddr);
548- for (uint16_t wRepeatIdx = 0; wRepeatIdx < repeatimes; wRepeatIdx++) {
549- for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
550- uint32_t offset = (wBatchIdx + wRepeatIdx * computeSizeT2 * wProBatchSize +
551- hIdx * wArgmaxAligned + dArgmaxOffset + highArgmaxOffset);
552- AscendC::MicroAPI::Adds(parallelRegIndex, initialRegIndex, offset, allMaskU32);
553- DoSingleNchwFullLoad<T1, T2>(
554- yAddr, gradAddr, argmaxAddr, parallelRegIndex, all, highOutputOffset);
555- }
556- }
557- }
558 518 
559- __VEC_SCOPE__519+ for (uint16_t highIdx = 0; highIdx < highAxisActual; ++highIdx) {
560- {520+ uint32_t highArgmaxOffset = highIdx * dhwPlaneAligned_;
561- AscendC::MicroAPI::RegTensor<uint32_t> initialRegIndex;521+ uint32_t highOutputOffset = highIdx * dOutputActual * hOutputActual * wOutputActual;
562- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;522+ for (uint16_t dIdx = 0; dIdx < dArgmaxActual; dIdx++) {
563- AscendC::MicroAPI::MaskReg allMaskU32 =523+ uint32_t dArgmaxOffset = dIdx * hArgmaxActual * wArgmaxAligned;
564- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();524+ for (uint16_t hIdx = 0; hIdx < hArgmaxActual; hIdx++) {
565- AscendC::MicroAPI::DataCopy(initialRegIndex, helpAddr);525+ for (uint16_t wRepeatIdx = 0; wRepeatIdx < repeatimes; wRepeatIdx++) {
566 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {526 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
567- uint32_t offset = (wBatchIdx + repeatimes * computeSizeT2 * wProBatchSize + hIdx * wArgmaxAligned + dArgmaxOffset +527+ uint32_t offset = (wBatchIdx + wRepeatIdx * computeSizeT2 * wProBatchSize +
568- highArgmaxOffset);528+ hIdx * wArgmaxAligned + dArgmaxOffset + highArgmaxOffset);
569 AscendC::MicroAPI::Adds(parallelRegIndex, initialRegIndex, offset, allMaskU32);529 AscendC::MicroAPI::Adds(parallelRegIndex, initialRegIndex, offset, allMaskU32);
570 DoSingleNchwFullLoad<T1, T2>(530 DoSingleNchwFullLoad<T1, T2>(
571- yAddr, gradAddr, argmaxAddr, parallelRegIndex, wRemainBatchCount, highOutputOffset);531+ yAddr, gradAddr, argmaxAddr, parallelRegIndex, all, highOutputOffset);
572 }532 }
533+ }
534+ for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
535+ uint32_t offset = (wBatchIdx + repeatimes * computeSizeT2 * wProBatchSize + hIdx * wArgmaxAligned + dArgmaxOffset +
536+ highArgmaxOffset);
537+ AscendC::MicroAPI::Adds(parallelRegIndex, initialRegIndex, offset, allMaskU32);
538+ DoSingleNchwFullLoad<T1, T2>(
539+ yAddr, gradAddr, argmaxAddr, parallelRegIndex, wRemainBatchCount, highOutputOffset);
540+ }
573 541 
574- for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {542+ for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {
575- uint32_t offset = (wBatchIdx + wRemainBatchCount * wProBatchSize +543+ uint32_t offset = (wBatchIdx + wRemainBatchCount * wProBatchSize +
576- repeatimes * computeSizeT2 * wProBatchSize + hIdx * wArgmaxAligned + dArgmaxOffset + highArgmaxOffset);544+ repeatimes * computeSizeT2 * wProBatchSize + hIdx * wArgmaxAligned + dArgmaxOffset + highArgmaxOffset);
577- AscendC::MicroAPI::Adds(parallelRegIndex, initialRegIndex, offset, allMaskU32);545+ AscendC::MicroAPI::Adds(parallelRegIndex, initialRegIndex, offset, allMaskU32);
578- DoSingleNchwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, one, highOutputOffset);546+ DoSingleNchwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, one, highOutputOffset);
579- }
580 }547 }
581 }548 }
582 }549 }
@@ -586,23 +553,15 @@ __aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHE
586 553 
587template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>554template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>
588__aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHECK_RANGE>::fullLoadMultipleLineHwProcessVF(555__aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHECK_RANGE>::fullLoadMultipleLineHwProcessVF(
589- __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr, __local_mem__ uint32_t* helpAddr)556+ __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr)
590{557{
591- int64_t wOutput = wOutput_;
592- int64_t hOutput = hOutput_;
593- 
594 int64_t wOutputActual = wOutputActual_;558 int64_t wOutputActual = wOutputActual_;
595- int64_t wOutputAligned = wOutputAligned_;
596 int64_t hOutputActual = hOutputActual_;559 int64_t hOutputActual = hOutputActual_;
597 int64_t dOutputActual = dOutputActual_;560 int64_t dOutputActual = dOutputActual_;
598 561 
599 uint16_t highAxisActual = static_cast<uint16_t>(highAxisActual_);562 uint16_t highAxisActual = static_cast<uint16_t>(highAxisActual_);
600 563 
601- int64_t curDIndex = dAxisIndex_ * dOutputInner_;564+ int64_t wArgmaxAligned = wArgmaxActual_;
602- int64_t curHIndex = hAxisIndex_ * hOutputInner_;
603- int64_t curWIndex = wAxisIndex_ * wOutputInner_;
604- 
605- int64_t wArgmaxAligned = wArgmaxAligned_;
606 int64_t wArgmaxActual = wArgmaxActual_;565 int64_t wArgmaxActual = wArgmaxActual_;
607 uint16_t hArgmaxActual = hArgmaxActual_;566 uint16_t hArgmaxActual = hArgmaxActual_;
608 uint16_t dArgmaxActual = dArgmaxActual_;567 uint16_t dArgmaxActual = dArgmaxActual_;
@@ -627,34 +586,25 @@ __aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHE
627 uint32_t maskBlock = wFullBatchCount * hConcurrentCount;586 uint32_t maskBlock = wFullBatchCount * hConcurrentCount;
628 uint32_t maskRemainBatch = wFullBatchCount * hRemainBatchCount;587 uint32_t maskRemainBatch = wFullBatchCount * hRemainBatchCount;
629 uint32_t maskRemainTail = wFullBatchCount;588 uint32_t maskRemainTail = wFullBatchCount;
589+ 
590+ DivMagic divW = PrecomputeDiv(static_cast<uint32_t>(wFullBatchCount));
591+ 
630 __VEC_SCOPE__592 __VEC_SCOPE__
631 {593 {
632 AscendC::MicroAPI::RegTensor<uint32_t> initialRegIndex;594 AscendC::MicroAPI::RegTensor<uint32_t> initialRegIndex;
633 AscendC::MicroAPI::RegTensor<uint32_t> initialRegIndexOne;595 AscendC::MicroAPI::RegTensor<uint32_t> initialRegIndexOne;
634- DhwGenInitial2DIndices((AscendC::MicroAPI::RegTensor<int32_t>&)initialRegIndex, wProBatchSize, hProBatchSize,596+ DhwGenInitial2DIndicesFast((AscendC::MicroAPI::RegTensor<int32_t>&)initialRegIndex, wProBatchSize, hProBatchSize,
635- wArgmaxAligned, wFullBatchCount);597+ wArgmaxAligned, wFullBatchCount, divW);
636 DhwGen2DIndexOne((AscendC::MicroAPI::RegTensor<int32_t>&)initialRegIndexOne, hProBatchSize, wArgmaxAligned);598 DhwGen2DIndexOne((AscendC::MicroAPI::RegTensor<int32_t>&)initialRegIndexOne, hProBatchSize, wArgmaxAligned);
637- AscendC::MicroAPI::MaskReg allMask =599+ AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
600+ AscendC::MicroAPI::MaskReg allMaskU32 =
638 AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();601 AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
639- AscendC::MicroAPI::DataCopy(helpAddr, initialRegIndex, allMask);
640- AscendC::MicroAPI::DataCopy(helpAddr + V_REG_SIZE / sizeof(uint32_t), initialRegIndexOne, allMask);
641- }
642- for (uint16_t highIdx = 0; highIdx < highAxisActual; ++highIdx) {
643- uint32_t highArgmaxOffset = highIdx * dArgmaxActual * hArgmaxActual * wArgmaxAligned;
644- uint32_t highOutputOffset = highIdx * dOutputActual * hOutputActual * wOutputActual;
645- for (uint16_t dIdx = 0; dIdx < dArgmaxActual; dIdx++) {
646- uint32_t dArgmaxOffset = dIdx * hArgmaxActual * wArgmaxAligned;
647- __VEC_SCOPE__
648- {
649- AscendC::MicroAPI::RegTensor<uint32_t> initialRegIndex;
650- AscendC::MicroAPI::RegTensor<uint32_t> initialRegIndexOne;
651- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
652- 
653- AscendC::MicroAPI::MaskReg allMaskU32 =
654- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
655- AscendC::MicroAPI::DataCopy(initialRegIndex, helpAddr);
656- AscendC::MicroAPI::DataCopy(initialRegIndexOne, helpAddr + V_REG_SIZE / sizeof(uint32_t));
657 602 
603+ for (uint16_t highIdx = 0; highIdx < highAxisActual; ++highIdx) {
604+ uint32_t highArgmaxOffset = highIdx * dhwPlaneAligned_;
605+ uint32_t highOutputOffset = highIdx * dOutputActual * hOutputActual * wOutputActual;
606+ for (uint16_t dIdx = 0; dIdx < dArgmaxActual; dIdx++) {
607+ uint32_t dArgmaxOffset = dIdx * hArgmaxActual * wArgmaxAligned;
658 for (uint16_t hIdx = 0; hIdx < blockConcurrentCount; hIdx++) {608 for (uint16_t hIdx = 0; hIdx < blockConcurrentCount; hIdx++) {
659 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {609 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {
660 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {610 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
@@ -673,18 +623,7 @@ __aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHE
673 }623 }
674 }624 }
675 }625 }
676- }
677 626 
678- __VEC_SCOPE__
679- {
680- AscendC::MicroAPI::RegTensor<uint32_t> initialRegIndex;
681- AscendC::MicroAPI::RegTensor<uint32_t> initialRegIndexOne;
682- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
683- 
684- AscendC::MicroAPI::MaskReg allMaskU32 =
685- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
686- AscendC::MicroAPI::DataCopy(initialRegIndex, helpAddr);
687- AscendC::MicroAPI::DataCopy(initialRegIndexOne, helpAddr + V_REG_SIZE / sizeof(uint32_t));
688 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {627 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {
689 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {628 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
690 T2 offset =629 T2 offset =
@@ -703,17 +642,6 @@ __aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHE
703 yAddr, gradAddr, argmaxAddr, parallelRegIndex, remainBatchOne, highOutputOffset);642 yAddr, gradAddr, argmaxAddr, parallelRegIndex, remainBatchOne, highOutputOffset);
704 }643 }
705 }644 }
706- }
707- __VEC_SCOPE__
708- {
709- AscendC::MicroAPI::RegTensor<uint32_t> initialRegIndex;
710- AscendC::MicroAPI::RegTensor<uint32_t> initialRegIndexOne;
711- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
712- 
713- AscendC::MicroAPI::MaskReg allMaskU32 =
714- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
715- AscendC::MicroAPI::DataCopy(initialRegIndex, helpAddr);
716- AscendC::MicroAPI::DataCopy(initialRegIndexOne, helpAddr + V_REG_SIZE / sizeof(uint32_t));
717 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hRemainTail; hProBatchIdx++) {645 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hRemainTail; hProBatchIdx++) {
718 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {646 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
719 T2 offset =647 T2 offset =
@@ -740,28 +668,17 @@ __aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHE
740}668}
741 669 
742template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>670template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>
743-__aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHECK_RANGE>::fullLoadMultipleLineDhwProcessVF(671+ __aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHECK_RANGE>::fullLoadMultipleLineDhwProcessVF(
744- __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr, __local_mem__ uint32_t* helpAddr)672+ __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr)
745{673{
746- int64_t wOutput = wOutput_;
747- int64_t hOutput = hOutput_;
748- int64_t dOutput = dOutput_;
749- 
750 int64_t wOutputActual = wOutputActual_;674 int64_t wOutputActual = wOutputActual_;
751- int64_t wOutputAligned = wOutputAligned_;
752 int64_t hOutputActual = hOutputActual_;675 int64_t hOutputActual = hOutputActual_;
753 int64_t dOutputActual = dOutputActual_;676 int64_t dOutputActual = dOutputActual_;
754- 
755 uint16_t highAxisActual = static_cast<uint16_t>(highAxisActual_);677 uint16_t highAxisActual = static_cast<uint16_t>(highAxisActual_);
756- 
757- int64_t curDIndex = dAxisIndex_ * dOutputInner_;
758- int64_t curHIndex = hAxisIndex_ * hOutputInner_;
759- int64_t curWIndex = wAxisIndex_ * wOutputInner_;
760- 
761- int64_t wArgmaxAligned = wArgmaxAligned_;
762- int64_t wArgmaxActual = wArgmaxActual_;
763- uint16_t hArgmaxActual = hArgmaxActual_;
764 uint16_t dArgmaxActual = dArgmaxActual_;678 uint16_t dArgmaxActual = dArgmaxActual_;
679+ int64_t wArgmaxActual = wArgmaxActual_;
680+ int64_t wArgmaxAligned = wArgmaxActual_;
681+ uint16_t hArgmaxActual = hArgmaxActual_;
765 682 
766 uint16_t dProBatchSize = curDProBatchSize_;683 uint16_t dProBatchSize = curDProBatchSize_;
767 uint16_t hProBatchSize = curHProBatchSize_;684 uint16_t hProBatchSize = curHProBatchSize_;
@@ -784,20 +701,28 @@ __aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHE
784 701 
785 uint16_t hRemainTail = hArgmaxActual - hFullBatchCount * hProBatchSize;702 uint16_t hRemainTail = hArgmaxActual - hFullBatchCount * hProBatchSize;
786 703 
704+ uint32_t highArgmaxStride = dhwPlaneAligned_;
705+ uint32_t highOutputStride = dOutputActual * hOutputActual * wOutputActual;
706+ 
787 uint32_t mask0 = hwConcurrentCount * hwFullBatchCount;707 uint32_t mask0 = hwConcurrentCount * hwFullBatchCount;
788- uint32_t mask1 = hwConcurrentCount * hFullBatchCount * 1;708+ uint32_t mask1 = hwConcurrentCount * hFullBatchCount;
789- uint32_t mask2 = hwConcurrentCount * 1 * wFullBatchCount;709+ uint32_t mask2 = hwConcurrentCount * wFullBatchCount;
790- uint32_t mask3 = hwConcurrentCount * 1 * 1;710+ uint32_t mask3 = hwConcurrentCount;
791 711 
792 uint32_t mask4 = dRemainBatchCount * hwFullBatchCount;712 uint32_t mask4 = dRemainBatchCount * hwFullBatchCount;
793- uint32_t mask5 = dRemainBatchCount * hFullBatchCount * 1;713+ uint32_t mask5 = dRemainBatchCount * hFullBatchCount;
794- uint32_t mask6 = dRemainBatchCount * 1 * wFullBatchCount;714+ uint32_t mask6 = dRemainBatchCount * wFullBatchCount;
795- uint32_t mask7 = dRemainBatchCount * 1 * 1;715+ uint32_t mask7 = dRemainBatchCount;
796 716 
797- uint32_t mask8 = 1 * hwFullBatchCount;717+ uint32_t mask8 = hwFullBatchCount;
798- uint32_t mask9 = 1 * hFullBatchCount * 1;718+ uint32_t mask9 = hFullBatchCount;
799- uint32_t mask10 = 1 * 1 * wFullBatchCount;719+ uint32_t mask10 = wFullBatchCount;
800- uint32_t mask11 = 1 * 1 * 1;720+ uint32_t mask11 = 1;
721+ 
722+ DivMagic divW = PrecomputeDiv(static_cast<uint32_t>(wFullBatchCount));
723+ DivMagic divH = PrecomputeDiv(static_cast<uint32_t>(hFullBatchCount));
724+ DivMagic divWH = PrecomputeDiv(static_cast<uint32_t>(wFullBatchCount * hFullBatchCount));
725+ DivMagic div1 = PrecomputeDiv(1);
801 726 
802 __VEC_SCOPE__727 __VEC_SCOPE__
803 {728 {
@@ -805,290 +730,175 @@ __aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHE
805 AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndexOne;730 AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndexOne;
806 AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndexDw;731 AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndexDw;
807 AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndexOneDw;732 AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndexOneDw;
733+ GenInitial3DIndicesFast((AscendC::MicroAPI::RegTensor<int32_t>&)initial3DRegIndex, dProBatchSize, hProBatchSize, wProBatchSize,
734+ hFullBatchCount, hArgmaxActual, wFullBatchCount, wArgmaxAligned, divWH, divW);
735+ Gen3DIndexOneFast((AscendC::MicroAPI::RegTensor<int32_t>&)initial3DRegIndexOne, dProBatchSize, hProBatchSize, wArgmaxAligned,
736+ hFullBatchCount, hArgmaxActual, divH);
737+ GenInitial3DIndicesFast((AscendC::MicroAPI::RegTensor<int32_t>&)initial3DRegIndexDw, dProBatchSize, hProBatchSize, wProBatchSize,
738+ 1, hArgmaxActual, wFullBatchCount, wArgmaxAligned, divW, divW);
739+ Gen3DIndexOneFast((AscendC::MicroAPI::RegTensor<int32_t>&)initial3DRegIndexOneDw, dProBatchSize, hProBatchSize, wArgmaxAligned,
740+ 1, hArgmaxActual, div1);
808 AscendC::MicroAPI::RegTensor<uint32_t> initial2DRegIndex;741 AscendC::MicroAPI::RegTensor<uint32_t> initial2DRegIndex;
809 AscendC::MicroAPI::RegTensor<uint32_t> initial2DRegIndexOne;742 AscendC::MicroAPI::RegTensor<uint32_t> initial2DRegIndexOne;
810- 743+ DhwGenInitial2DIndicesFast((AscendC::MicroAPI::RegTensor<int32_t>&)initial2DRegIndex, wProBatchSize, hProBatchSize,
811- GenInitial3DIndices((AscendC::MicroAPI::RegTensor<int32_t>&)initial3DRegIndex, dProBatchSize, hProBatchSize, wProBatchSize,744+ wArgmaxAligned, wFullBatchCount, divW);
812- hFullBatchCount, hArgmaxActual, wFullBatchCount, wArgmaxAligned);
813- Gen3DIndexOne((AscendC::MicroAPI::RegTensor<int32_t>&)initial3DRegIndexOne, dProBatchSize, hProBatchSize, wArgmaxAligned,
814- hFullBatchCount, hArgmaxActual);
815- GenInitial3DIndices((AscendC::MicroAPI::RegTensor<int32_t>&)initial3DRegIndexDw, dProBatchSize, hProBatchSize, wProBatchSize,
816- 1, hArgmaxActual, wFullBatchCount, wArgmaxAligned);
817- Gen3DIndexOne((AscendC::MicroAPI::RegTensor<int32_t>&)initial3DRegIndexOneDw, dProBatchSize, hProBatchSize, wArgmaxAligned,
818- 1, hArgmaxActual);
819- DhwGenInitial2DIndices((AscendC::MicroAPI::RegTensor<int32_t>&)initial2DRegIndex, wProBatchSize, hProBatchSize,
820- wArgmaxAligned, wFullBatchCount);
821 DhwGen2DIndexOne((AscendC::MicroAPI::RegTensor<int32_t>&)initial2DRegIndexOne, hProBatchSize, wArgmaxAligned);745 DhwGen2DIndexOne((AscendC::MicroAPI::RegTensor<int32_t>&)initial2DRegIndexOne, hProBatchSize, wArgmaxAligned);
822- 746+ AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
823- AscendC::MicroAPI::MaskReg allMask =747+ AscendC::MicroAPI::MaskReg allMaskU32 =
824 AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();748 AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
825- AscendC::MicroAPI::DataCopy(helpAddr, initial3DRegIndex, allMask);
826- AscendC::MicroAPI::DataCopy(helpAddr + V_REG_SIZE / sizeof(uint32_t), initial3DRegIndexOne, allMask);
827 749 
828- AscendC::MicroAPI::DataCopy(helpAddr + INDEX_TWO * V_REG_SIZE / sizeof(uint32_t), initial3DRegIndexDw, allMask);750+ uint32_t highArgmaxOffset = 0;
829- AscendC::MicroAPI::DataCopy(751+ uint32_t highOutputOffset = 0;
830- helpAddr + INDEX_THREE * V_REG_SIZE / sizeof(uint32_t), initial3DRegIndexOneDw, allMask);752+ for (uint16_t highIdx = 0; highIdx < highAxisActual; ++highIdx) {
831- 753+ for (uint16_t dIdx = 0; dIdx < dBlockConcurrentCount; dIdx++) {
832- AscendC::MicroAPI::DataCopy(helpAddr + INDEX_FOUR * V_REG_SIZE / sizeof(uint32_t), initial2DRegIndex, allMask);754+ for (uint16_t dProBatchIdx = 0; dProBatchIdx < dProBatchSize; dProBatchIdx++) {
833- AscendC::MicroAPI::DataCopy(755+ T2 dhOffset = dProBatchIdx * hArgmaxActual * wArgmaxAligned
834- helpAddr + INDEX_FIVE * V_REG_SIZE / sizeof(uint32_t), initial2DRegIndexOne, allMask);756+ + dIdx * dProBatchSize * hArgmaxActual * wArgmaxAligned * hwConcurrentCount
835- }757+ + highArgmaxOffset;
836- 
837- for (uint16_t highIdx = 0; highIdx < highAxisActual; ++highIdx) {
838- uint32_t highArgmaxOffset = highIdx * dArgmaxActual * hArgmaxActual * wArgmaxAligned;
839- uint32_t highOutputOffset = highIdx * dOutputActual * hOutputActual * wOutputActual;
840- for (uint16_t dIdx = 0; dIdx < dBlockConcurrentCount; dIdx++) {
841- for (uint16_t dProBatchIdx = 0; dProBatchIdx < dProBatchSize; dProBatchIdx++) {
842- __VEC_SCOPE__
843- {
844- AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndex;
845- AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndexOne;
846- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
847- AscendC::MicroAPI::MaskReg allMaskU32 =
848- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
849- AscendC::MicroAPI::DataCopy(initial3DRegIndex, helpAddr);
850- AscendC::MicroAPI::DataCopy(initial3DRegIndexOne, helpAddr + V_REG_SIZE / sizeof(uint32_t));
851 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {758 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {
759+ T2 hOffset = hProBatchIdx * wArgmaxAligned + dhOffset;
852 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {760 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
853- T2 offset = (wBatchIdx761+ T2 offset = wBatchIdx + hOffset;
854- + hProBatchIdx * wArgmaxAligned762+ AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndex, offset, allMaskU32);
855- + dProBatchIdx * hArgmaxActual * wArgmaxAligned763+ DoSingleNchwFullLoad<T1, T2>(
856- + dIdx * dProBatchSize * hArgmaxActual * wArgmaxAligned * hwConcurrentCount764+ yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask0, highOutputOffset);
857- + highArgmaxOffset);
858- 
859- AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndex, offset, allMaskU32);
860- DoSingleNchwFullLoad<T1, T2>(
861- yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask0, highOutputOffset);
862 }765 }
863- 
864 for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {766 for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {
865- T2 offset = (wBatchIdx + wProBatchSize * wFullBatchCount767+ T2 offset = wBatchIdx + wProBatchSize * wFullBatchCount + hOffset;
866- + hProBatchIdx * wArgmaxAligned
867- + dProBatchIdx * hArgmaxActual * wArgmaxAligned
868- + dIdx * dProBatchSize * hArgmaxActual * wArgmaxAligned * hwConcurrentCount
869- + highArgmaxOffset);
870- 
871 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndexOne, offset, allMaskU32);768 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndexOne, offset, allMaskU32);
872 DoSingleNchwFullLoad<T1, T2>(769 DoSingleNchwFullLoad<T1, T2>(
873- yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask1, highOutputOffset);770+ yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask1, highOutputOffset);
874 }771 }
875 }772 }
876- }
877- 
878- __VEC_SCOPE__
879- {
880- AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndexDw;
881- AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndexOneDw;
882- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
883- AscendC::MicroAPI::MaskReg allMaskU32 =
884- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
885- AscendC::MicroAPI::DataCopy(initial3DRegIndexDw, helpAddr + INDEX_TWO * V_REG_SIZE / sizeof(uint32_t));
886- AscendC::MicroAPI::DataCopy(initial3DRegIndexOneDw, helpAddr + INDEX_THREE * V_REG_SIZE / sizeof(uint32_t));
887 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hRemainTail; hProBatchIdx++) {773 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hRemainTail; hProBatchIdx++) {
774+ T2 hOffset = (hProBatchIdx + hFullBatchCount * hProBatchSize) * wArgmaxAligned + dhOffset;
888 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {775 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
889- T2 offset = (wBatchIdx776+ T2 offset = wBatchIdx + hOffset;
890- + (hProBatchIdx + hFullBatchCount * hProBatchSize) * wArgmaxAligned
891- + dProBatchIdx * hArgmaxActual * wArgmaxAligned
892- + dIdx * dProBatchSize * hArgmaxActual * wArgmaxAligned * hwConcurrentCount
893- + highArgmaxOffset);
894- 
895 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndexDw, offset, allMaskU32);777 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndexDw, offset, allMaskU32);
896 DoSingleNchwFullLoad<T1, T2>(778 DoSingleNchwFullLoad<T1, T2>(
897- yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask2, highOutputOffset);779+ yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask2, highOutputOffset);
898 }780 }
899 for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {781 for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {
900- T2 offset = (wBatchIdx + wProBatchSize * wFullBatchCount782+ T2 offset = wBatchIdx + wProBatchSize * wFullBatchCount + hOffset;
901- + (hProBatchIdx + hFullBatchCount * hProBatchSize) * wArgmaxAligned
902- + dProBatchIdx * hArgmaxActual * wArgmaxAligned
903- + dIdx * dProBatchSize * hArgmaxActual * wArgmaxAligned * hwConcurrentCount
904- + highArgmaxOffset);
905 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndexOneDw, offset, allMaskU32);783 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndexOneDw, offset, allMaskU32);
906 DoSingleNchwFullLoad<T1, T2>(784 DoSingleNchwFullLoad<T1, T2>(
907- yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask3, highOutputOffset);785+ yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask3, highOutputOffset);
908 }786 }
909 }787 }
910 }788 }
911 }789 }
912- }
913 790 
914- for (uint16_t dProBatchIdx = 0; dProBatchIdx < dProBatchSize; dProBatchIdx++) {791+ for (uint16_t dProBatchIdx = 0; dProBatchIdx < dProBatchSize; dProBatchIdx++) {
915- __VEC_SCOPE__792+ T2 dhOffset = dProBatchIdx * hArgmaxActual * wArgmaxAligned
916- {793+ + (dBlockConcurrentCount * hwConcurrentCount) * dProBatchSize * hArgmaxActual * wArgmaxAligned
917- AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndex;794+ + highArgmaxOffset;
918- AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndexOne;
919- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
920- AscendC::MicroAPI::MaskReg allMaskU32 =
921- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
922- AscendC::MicroAPI::DataCopy(initial3DRegIndex, helpAddr);
923- AscendC::MicroAPI::DataCopy(initial3DRegIndexOne, helpAddr + V_REG_SIZE / sizeof(uint32_t));
924 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {795 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {
796+ T2 hOffset = hProBatchIdx * wArgmaxAligned + dhOffset;
925 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {797 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
926- T2 offset = (wBatchIdx798+ T2 offset = wBatchIdx + hOffset;
927- + hProBatchIdx * wArgmaxAligned
928- + dProBatchIdx * hArgmaxActual * wArgmaxAligned
929- + (dBlockConcurrentCount * hwConcurrentCount) * dProBatchSize * hArgmaxActual * wArgmaxAligned
930- + highArgmaxOffset);
931- 
932 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndex, offset, allMaskU32);799 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndex, offset, allMaskU32);
933 DoSingleNchwFullLoad<T1, T2>(800 DoSingleNchwFullLoad<T1, T2>(
934- yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask4, highOutputOffset);801+ yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask4, highOutputOffset);
935 }802 }
936 for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {803 for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {
937- T2 offset = (wBatchIdx + wProBatchSize * wFullBatchCount804+ T2 offset = wBatchIdx + wProBatchSize * wFullBatchCount + hOffset;
938- + hProBatchIdx * wArgmaxAligned
939- + dProBatchIdx * hArgmaxActual * wArgmaxAligned
940- + (dBlockConcurrentCount * hwConcurrentCount) * dProBatchSize * hArgmaxActual * wArgmaxAligned
941- + highArgmaxOffset);
942 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndexOne, offset, allMaskU32);805 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndexOne, offset, allMaskU32);
943 DoSingleNchwFullLoad<T1, T2>(806 DoSingleNchwFullLoad<T1, T2>(
944- yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask5, highOutputOffset);807+ yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask5, highOutputOffset);
945 }808 }
946 }809 }
947- }
948- __VEC_SCOPE__
949- {
950- AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndexDw;
951- AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndexOneDw;
952- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
953- AscendC::MicroAPI::MaskReg allMaskU32 =
954- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
955- AscendC::MicroAPI::DataCopy(initial3DRegIndexDw, helpAddr + INDEX_TWO * V_REG_SIZE / sizeof(uint32_t));
956- AscendC::MicroAPI::DataCopy(initial3DRegIndexOneDw, helpAddr + INDEX_THREE * V_REG_SIZE / sizeof(uint32_t));
957 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hRemainTail; hProBatchIdx++) {810 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hRemainTail; hProBatchIdx++) {
811+ T2 hOffset = (hProBatchIdx + hFullBatchCount * hProBatchSize) * wArgmaxAligned + dhOffset;
958 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {812 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
959- T2 offset = (wBatchIdx813+ T2 offset = wBatchIdx + hOffset;
960- + (hProBatchIdx + hFullBatchCount * hProBatchSize) * wArgmaxAligned
961- + dProBatchIdx * hArgmaxActual * wArgmaxAligned
962- + (dBlockConcurrentCount * hwConcurrentCount) * dProBatchSize * hArgmaxActual * wArgmaxAligned
963- + highArgmaxOffset);
964- 
965 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndexDw, offset, allMaskU32);814 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndexDw, offset, allMaskU32);
966 DoSingleNchwFullLoad<T1, T2>(815 DoSingleNchwFullLoad<T1, T2>(
967- yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask6, highOutputOffset);816+ yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask6, highOutputOffset);
968 }817 }
969 for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {818 for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {
970- T2 offset = (wBatchIdx + wProBatchSize * wFullBatchCount819+ T2 offset = wBatchIdx + wProBatchSize * wFullBatchCount + hOffset;
971- + (hProBatchIdx + hFullBatchCount * hProBatchSize) * wArgmaxAligned
972- + dProBatchIdx * hArgmaxActual * wArgmaxAligned
973- + (dBlockConcurrentCount * hwConcurrentCount) * dProBatchSize * hArgmaxActual * wArgmaxAligned
974- + highArgmaxOffset);
975- 
976 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndexOneDw, offset, allMaskU32);820 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndexOneDw, offset, allMaskU32);
977 DoSingleNchwFullLoad<T1, T2>(821 DoSingleNchwFullLoad<T1, T2>(
978- yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask7, highOutputOffset);822+ yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask7, highOutputOffset);
979 }823 }
980 }824 }
981 }825 }
982- }
983 826 
984- for (uint16_t dProBatchIdx = 0; dProBatchIdx < dRemainTail; dProBatchIdx++) {827+ for (uint16_t dProBatchIdx = 0; dProBatchIdx < dRemainTail; dProBatchIdx++) {
985- __VEC_SCOPE__828+ T2 dhOffset = dProBatchIdx * hArgmaxActual * wArgmaxAligned
986- {829+ + (dRemainBatchCount + dBlockConcurrentCount * hwConcurrentCount) * dProBatchSize * hArgmaxActual * wArgmaxAligned
987- AscendC::MicroAPI::RegTensor<uint32_t> initial2DRegIndex;830+ + highArgmaxOffset;
988- AscendC::MicroAPI::RegTensor<uint32_t> initial2DRegIndexOne;
989- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
990- AscendC::MicroAPI::MaskReg allMaskU32 =
991- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
992- AscendC::MicroAPI::DataCopy(initial2DRegIndex, helpAddr + INDEX_FOUR * V_REG_SIZE / sizeof(uint32_t));
993- AscendC::MicroAPI::DataCopy(initial2DRegIndexOne, helpAddr + INDEX_FIVE * V_REG_SIZE / sizeof(uint32_t));
994 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {831 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {
832+ T2 hOffset = hProBatchIdx * wArgmaxAligned + dhOffset;
995 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {833 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
996- T2 offset = (wBatchIdx834+ T2 offset = wBatchIdx + hOffset;
997- + hProBatchIdx * wArgmaxAligned
998- + dProBatchIdx * hArgmaxActual * wArgmaxAligned
999- + (dRemainBatchCount + dBlockConcurrentCount * hwConcurrentCount) * dProBatchSize * hArgmaxActual * wArgmaxAligned
1000- + highArgmaxOffset);
1001 AscendC::MicroAPI::Adds(parallelRegIndex, initial2DRegIndex, offset, allMaskU32);835 AscendC::MicroAPI::Adds(parallelRegIndex, initial2DRegIndex, offset, allMaskU32);
1002 DoSingleNchwFullLoad<T1, T2>(836 DoSingleNchwFullLoad<T1, T2>(
1003- yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask8, highOutputOffset);837+ yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask8, highOutputOffset);
1004 }838 }
1005 for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {839 for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {
1006- T2 offset = (wBatchIdx + wProBatchSize * wFullBatchCount840+ T2 offset = wBatchIdx + wProBatchSize * wFullBatchCount + hOffset;
1007- + hProBatchIdx * wArgmaxAligned
1008- + dProBatchIdx * hArgmaxActual * wArgmaxAligned
1009- + (dRemainBatchCount + dBlockConcurrentCount * hwConcurrentCount) * dProBatchSize * hArgmaxActual * wArgmaxAligned
1010- + highArgmaxOffset);
1011- 
1012 AscendC::MicroAPI::Adds(parallelRegIndex, initial2DRegIndexOne, offset, allMaskU32);841 AscendC::MicroAPI::Adds(parallelRegIndex, initial2DRegIndexOne, offset, allMaskU32);
1013 DoSingleNchwFullLoad<T1, T2>(842 DoSingleNchwFullLoad<T1, T2>(
1014- yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask9, highOutputOffset);843+ yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask9, highOutputOffset);
1015 }844 }
1016 }845 }
1017- }
1018- __VEC_SCOPE__
1019- {
1020- AscendC::MicroAPI::RegTensor<uint32_t> initial2DRegIndex;
1021- AscendC::MicroAPI::RegTensor<uint32_t> initial2DRegIndexOne;
1022- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
1023- AscendC::MicroAPI::MaskReg allMaskU32 =
1024- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1025- AscendC::MicroAPI::DataCopy(initial2DRegIndex, helpAddr + INDEX_FOUR * V_REG_SIZE / sizeof(uint32_t));
1026- AscendC::MicroAPI::DataCopy(initial2DRegIndexOne, helpAddr + INDEX_FIVE * V_REG_SIZE / sizeof(uint32_t));
1027 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hRemainTail; hProBatchIdx++) {846 for (uint16_t hProBatchIdx = 0; hProBatchIdx < hRemainTail; hProBatchIdx++) {
847+ T2 hOffset = (hProBatchIdx + hFullBatchCount * hProBatchSize) * wArgmaxAligned + dhOffset;
1028 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {848 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
1029- T2 offset = (wBatchIdx849+ T2 offset = wBatchIdx + hOffset;
1030- + (hProBatchIdx + hFullBatchCount * hProBatchSize) * wArgmaxAligned
1031- + dProBatchIdx * hArgmaxActual * wArgmaxAligned
1032- + (dRemainBatchCount + dBlockConcurrentCount * hwConcurrentCount) * dProBatchSize * hArgmaxActual * wArgmaxAligned
1033- + highArgmaxOffset);
1034 AscendC::MicroAPI::Adds(parallelRegIndex, initial2DRegIndex, offset, allMaskU32);850 AscendC::MicroAPI::Adds(parallelRegIndex, initial2DRegIndex, offset, allMaskU32);
1035 DoSingleNchwFullLoad<T1, T2>(851 DoSingleNchwFullLoad<T1, T2>(
1036- yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask10, highOutputOffset);852+ yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask10, highOutputOffset);
1037- }853+ }
1038 for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {854 for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {
1039- T2 offset = (wBatchIdx + wProBatchSize * wFullBatchCount855+ T2 offset = wBatchIdx + wProBatchSize * wFullBatchCount + hOffset;
1040- + (hProBatchIdx + hFullBatchCount * hProBatchSize) * wArgmaxAligned
1041- + dProBatchIdx * hArgmaxActual * wArgmaxAligned
1042- + (dRemainBatchCount + dBlockConcurrentCount * hwConcurrentCount) * dProBatchSize * hArgmaxActual * wArgmaxAligned
1043- + highArgmaxOffset);
1044- 
1045 AscendC::MicroAPI::Adds(parallelRegIndex, initial2DRegIndexOne, offset, allMaskU32);856 AscendC::MicroAPI::Adds(parallelRegIndex, initial2DRegIndexOne, offset, allMaskU32);
1046 DoSingleNchwFullLoad<T1, T2>(857 DoSingleNchwFullLoad<T1, T2>(
1047- yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask11, highOutputOffset);858+ yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask11, highOutputOffset);
1048 }859 }
1049 }860 }
1050 }861 }
1051- }862+ highArgmaxOffset += highArgmaxStride;
863+ highOutputOffset += highOutputStride;
864+ }
1052 }865 }
1053}866}
1054 867 
1055 868 
1056template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>869template <typename T1, typename T2, const uint32_t IS_CHECK_RANGE>
1057-__aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHECK_RANGE>::fullLoadMultipleLineProcessVF2(870+ __aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHECK_RANGE>::fullLoadMultipleLineProcessVF2(
1058- __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr, __local_mem__ uint32_t* helpAddr)871+ __local_mem__ computeType* yAddr, __local_mem__ T1* gradAddr, __local_mem__ T2* argmaxAddr)
1059{872{
1060- int64_t curOutPutGmoffset = outPutGmOffset;
1061- int64_t curArgmaxGmoffset = argmaxGmOffset;
1062- 
1063 int64_t wOutputActual = wOutputActual_;873 int64_t wOutputActual = wOutputActual_;
1064- int64_t wOutputAligned = wOutputAligned_;
1065 int64_t hOutputActual = hOutputActual_;874 int64_t hOutputActual = hOutputActual_;
1066 int64_t dOutputActual = dOutputActual_;875 int64_t dOutputActual = dOutputActual_;
1067 int32_t highOutputPlaneActual = wOutputActual * hOutputActual * dOutputActual;876 int32_t highOutputPlaneActual = wOutputActual * hOutputActual * dOutputActual;
1068 int64_t highAxisActual = highAxisActual_;877 int64_t highAxisActual = highAxisActual_;
1069 878 
1070- int64_t wArgmaxAligned = wArgmaxAligned_;879+ int64_t wArgmaxAligned = wArgmaxActual_;
1071- int64_t wArgmaxActual = wArgmaxActual_;
1072 uint16_t hArgmaxActual = hArgmaxActual_;880 uint16_t hArgmaxActual = hArgmaxActual_;
1073 uint16_t dArgmaxActual = dArgmaxActual_;881 uint16_t dArgmaxActual = dArgmaxActual_;
1074 uint16_t hProBatchSize = curHProBatchSize_;882 uint16_t hProBatchSize = curHProBatchSize_;
1075 uint16_t wProBatchSize = curWProBatchSize_;883 uint16_t wProBatchSize = curWProBatchSize_;
1076 uint16_t dProBatchSize = curDProBatchSize_;884 uint16_t dProBatchSize = curDProBatchSize_;
1077 885 
1078- uint32_t wFullBatchCount = wArgmaxActual / wProBatchSize;886+ uint32_t wFullBatchCount = wArgmaxActual_ / wProBatchSize;
1079- uint16_t wRemainTail = wArgmaxActual - (wProBatchSize * wFullBatchCount);887+ uint16_t wRemainTail = wArgmaxActual_ - (wProBatchSize * wFullBatchCount);
1080 uint32_t dFullBatchCount = dArgmaxActual / dProBatchSize;888 uint32_t dFullBatchCount = dArgmaxActual / dProBatchSize;
1081 uint16_t dRemainTail = dArgmaxActual - (dProBatchSize * dFullBatchCount);889 uint16_t dRemainTail = dArgmaxActual - (dProBatchSize * dFullBatchCount);
1082 uint32_t hFullBatchCount = hArgmaxActual / hProBatchSize;890 uint32_t hFullBatchCount = hArgmaxActual / hProBatchSize;
1083 uint16_t hRemainTail = hArgmaxActual - (hProBatchSize * hFullBatchCount);891 uint16_t hRemainTail = hArgmaxActual - (hProBatchSize * hFullBatchCount);
1084 892 
1085- uint32_t dhwFullBatchCount = wFullBatchCount * hFullBatchCount * dFullBatchCount;893+ uint16_t highConcurrentCount = V_REG_SIZE / (wFullBatchCount * hFullBatchCount * dFullBatchCount * sizeof(T2));
1086- uint16_t highConcurrentCount = V_REG_SIZE / (dhwFullBatchCount * sizeof(T2));
1087 uint16_t highBlockConcurrentCount = highAxisActual / highConcurrentCount;894 uint16_t highBlockConcurrentCount = highAxisActual / highConcurrentCount;
1088 uint16_t highBlockRemainTail = highAxisActual - highBlockConcurrentCount * highConcurrentCount;895 uint16_t highBlockRemainTail = highAxisActual - highBlockConcurrentCount * highConcurrentCount;
1089 896 
1090 int64_t depthStride = hArgmaxActual * wArgmaxAligned * dProBatchSize;897 int64_t depthStride = hArgmaxActual * wArgmaxAligned * dProBatchSize;
1091- int64_t highStride = dArgmaxActual * hArgmaxActual * wArgmaxAligned;898+ int64_t highStride = dhwPlaneAligned_;
899+ 
900+ uint32_t highArgmaxStride = highConcurrentCount * dhwPlaneAligned_;
901+ uint32_t highOutputStride = highConcurrentCount * dOutputActual * hOutputActual * wOutputActual;
1092 902 
1093 uint32_t mask0 = highConcurrentCount * dFullBatchCount * hFullBatchCount * wFullBatchCount;903 uint32_t mask0 = highConcurrentCount * dFullBatchCount * hFullBatchCount * wFullBatchCount;
1094 uint32_t mask1 = highConcurrentCount * dFullBatchCount * hFullBatchCount;904 uint32_t mask1 = highConcurrentCount * dFullBatchCount * hFullBatchCount;
@@ -1107,343 +917,254 @@ __aicore__ inline void MaxPool3DGradWithArgmaxNCDHWFullLoadKernel<T1, T2, IS_CHE
1107 uint32_t mask14 = highBlockRemainTail * wFullBatchCount;917 uint32_t mask14 = highBlockRemainTail * wFullBatchCount;
1108 uint32_t mask15 = highBlockRemainTail;918 uint32_t mask15 = highBlockRemainTail;
1109 919 
920+ uint32_t divisor_dhw = dFullBatchCount * hFullBatchCount * wFullBatchCount;
921+ uint32_t divisor_dh = dFullBatchCount * hFullBatchCount;
922+ uint32_t divisor_dw = dFullBatchCount * wFullBatchCount;
923+ uint32_t divisor_d = dFullBatchCount;
924+ uint32_t divisor_hw = hFullBatchCount * wFullBatchCount;
925+ uint32_t divisor_h = hFullBatchCount;
926+ uint32_t divisor_w = wFullBatchCount;
927+ uint32_t divisor_1 = 1;
928+ 
929+ uint32_t magicHigh_dhw = 0, shiftHigh_dhw = 0;
930+ uint32_t magicHigh_dh = 0, shiftHigh_dh = 0;
931+ uint32_t magicHigh_dw = 0, shiftHigh_dw = 0;
932+ uint32_t magicHigh_d = 0, shiftHigh_d = 0;
933+ uint32_t magicHigh_hw = 0, shiftHigh_hw = 0;
934+ uint32_t magicHigh_h = 0, shiftHigh_h = 0;
935+ uint32_t magicHigh_w = 0, shiftHigh_w = 0;
936+ uint32_t magicHigh_1 = 0, shiftHigh_1 = 0;
937+ 
938+ GetUintDivMagicAndShift<uint32_t>(magicHigh_dhw, shiftHigh_dhw, divisor_dhw);
939+ GetUintDivMagicAndShift<uint32_t>(magicHigh_dh, shiftHigh_dh, divisor_dh);
940+ GetUintDivMagicAndShift<uint32_t>(magicHigh_dw, shiftHigh_dw, divisor_dw);
941+ GetUintDivMagicAndShift<uint32_t>(magicHigh_d, shiftHigh_d, divisor_d);
942+ GetUintDivMagicAndShift<uint32_t>(magicHigh_hw, shiftHigh_hw, divisor_hw);
943+ GetUintDivMagicAndShift<uint32_t>(magicHigh_h, shiftHigh_h, divisor_h);
944+ GetUintDivMagicAndShift<uint32_t>(magicHigh_w, shiftHigh_w, divisor_w);
945+ GetUintDivMagicAndShift<uint32_t>(magicHigh_1, shiftHigh_1, divisor_1);
946+ 
947+ DivMagic divW = PrecomputeDiv(static_cast<uint32_t>(wFullBatchCount));
948+ DivMagic divH = PrecomputeDiv(static_cast<uint32_t>(hFullBatchCount));
949+ DivMagic divWH = PrecomputeDiv(static_cast<uint32_t>(wFullBatchCount * hFullBatchCount));
950+ DivMagic divHD = PrecomputeDiv(static_cast<uint32_t>(hFullBatchCount * dFullBatchCount));
951+ DivMagic divDHW = PrecomputeDiv(static_cast<uint32_t>(wFullBatchCount * hFullBatchCount * dFullBatchCount));
952+ DivMagic divWD = PrecomputeDiv(static_cast<uint32_t>(wFullBatchCount * dFullBatchCount));
953+ DivMagic divD = PrecomputeDiv(static_cast<uint32_t>(dFullBatchCount));
954+ DivMagic div1 = PrecomputeDiv(1);
955+ 
1110 __VEC_SCOPE__956 __VEC_SCOPE__
1111 {957 {
1112 AscendC::MicroAPI::RegTensor<uint32_t> initial4DRegIndex;958 AscendC::MicroAPI::RegTensor<uint32_t> initial4DRegIndex;
1113 AscendC::MicroAPI::RegTensor<uint32_t> initial4DRegIndexOne;959 AscendC::MicroAPI::RegTensor<uint32_t> initial4DRegIndexOne;
960+ GenInitial4DIndicesFast((AscendC::MicroAPI::RegTensor<int32_t>&)initial4DRegIndex, wProBatchSize, hProBatchSize,
961+ wArgmaxAligned, wFullBatchCount, hFullBatchCount, dFullBatchCount, depthStride, highStride, divDHW, divWH, divW);
962+ Gen4DIndexOneFast((AscendC::MicroAPI::RegTensor<int32_t>&)initial4DRegIndexOne, hProBatchSize, wArgmaxAligned,
963+ hFullBatchCount, dFullBatchCount, depthStride, highStride, divHD, divH);
1114 AscendC::MicroAPI::RegTensor<uint32_t> initial4DRegIndexDW;964 AscendC::MicroAPI::RegTensor<uint32_t> initial4DRegIndexDW;
1115 AscendC::MicroAPI::RegTensor<uint32_t> initial4DRegIndexOneHD;965 AscendC::MicroAPI::RegTensor<uint32_t> initial4DRegIndexOneHD;
966+ GenInitial4DIndicesFast((AscendC::MicroAPI::RegTensor<int32_t>&)initial4DRegIndexDW, wProBatchSize, hProBatchSize,
967+ wArgmaxAligned, wFullBatchCount, 1, dFullBatchCount, depthStride, highStride, divWD, divW, divW);
968+ Gen4DIndexOneFast((AscendC::MicroAPI::RegTensor<int32_t>&)initial4DRegIndexOneHD, hProBatchSize, wArgmaxAligned,
969+ 1, dFullBatchCount, depthStride, highStride, divD, div1);
1116 AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndex;970 AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndex;
1117 AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndexOne;971 AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndexOne;
972+ GenInitial3DHighIndicesFast((AscendC::MicroAPI::RegTensor<int32_t>&)initial3DRegIndex, highStride, wProBatchSize, hProBatchSize,
973+ wArgmaxAligned, wFullBatchCount, hFullBatchCount, divWH, divW);
974+ Gen3DHighIndexOneFast((AscendC::MicroAPI::RegTensor<int32_t>&)initial3DRegIndexOne, highStride, hProBatchSize, wArgmaxAligned,
975+ hFullBatchCount, divH);
1118 AscendC::MicroAPI::RegTensor<uint32_t> initial2DRegIndex;976 AscendC::MicroAPI::RegTensor<uint32_t> initial2DRegIndex;
1119 AscendC::MicroAPI::RegTensor<uint32_t> initial2DRegIndexOne;977 AscendC::MicroAPI::RegTensor<uint32_t> initial2DRegIndexOne;
1120- GenInitial4DIndices((AscendC::MicroAPI::RegTensor<int32_t>&)initial4DRegIndex, wProBatchSize, hProBatchSize,978+ GenInitial2DHighIndicesFast((AscendC::MicroAPI::RegTensor<int32_t>&)initial2DRegIndex, highStride, wProBatchSize, wFullBatchCount, divW);
1121- wArgmaxAligned, wFullBatchCount, hFullBatchCount, dFullBatchCount, depthStride, highStride);979+ Gen2DHighIndexOne((AscendC::MicroAPI::RegTensor<int32_t>&)initial2DRegIndexOne, highStride);
1122- Gen4DIndexOne((AscendC::MicroAPI::RegTensor<int32_t>&)initial4DRegIndexOne, hProBatchSize, wArgmaxAligned,980+ AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
1123- hFullBatchCount, dFullBatchCount, depthStride, highStride);981+ AscendC::MicroAPI::MaskReg allMaskU32 =
1124- GenInitial4DIndices((AscendC::MicroAPI::RegTensor<int32_t>&)initial4DRegIndexDW, wProBatchSize, hProBatchSize,982+ AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1125- wArgmaxAligned, wFullBatchCount, 1, dFullBatchCount, depthStride, highStride);983+ AscendC::MicroAPI::RegTensor<uint32_t> magicHighReg;
1126- Gen4DIndexOne((AscendC::MicroAPI::RegTensor<int32_t>&)initial4DRegIndexOneHD, hProBatchSize, wArgmaxAligned,984+ AscendC::MicroAPI::RegTensor<int32_t> indexIncReg;
1127- 1, dFullBatchCount, depthStride, highStride);985+ AscendC::MicroAPI::Arange(indexIncReg, 0);
1128- GenInitial3DHighIndices((AscendC::MicroAPI::RegTensor<int32_t>&)initial3DRegIndex, highStride, wProBatchSize, hProBatchSize,
1129- wArgmaxAligned, wFullBatchCount, hFullBatchCount);
1130- Gen3DHighIndexOne((AscendC::MicroAPI::RegTensor<int32_t>&)initial3DRegIndexOne, highStride, hProBatchSize, wArgmaxAligned,
1131- hFullBatchCount);
1132- GenInitial2DIndices((AscendC::MicroAPI::RegTensor<int32_t>&)initial2DRegIndex, wProBatchSize, dArgmaxActual * hArgmaxActual,
1133- wArgmaxAligned, wFullBatchCount);
1134- Gen2DIndexOne((AscendC::MicroAPI::RegTensor<int32_t>&)initial2DRegIndexOne, dArgmaxActual * hArgmaxActual, wArgmaxAligned);
1135- AscendC::MicroAPI::MaskReg allMask =
1136- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1137- AscendC::MicroAPI::DataCopy(helpAddr, initial4DRegIndex, allMask);
1138- AscendC::MicroAPI::DataCopy(helpAddr + V_REG_SIZE / sizeof(uint32_t), initial4DRegIndexOne, allMask);
1139- AscendC::MicroAPI::DataCopy(helpAddr + INDEX_TWO * V_REG_SIZE / sizeof(uint32_t), initial3DRegIndex, allMask);
1140- AscendC::MicroAPI::DataCopy(helpAddr + INDEX_THREE * V_REG_SIZE / sizeof(uint32_t), initial3DRegIndexOne, allMask);
1141- AscendC::MicroAPI::DataCopy(helpAddr + INDEX_FOUR * V_REG_SIZE / sizeof(uint32_t), initial2DRegIndex, allMask);
1142- AscendC::MicroAPI::DataCopy(helpAddr + INDEX_FIVE * V_REG_SIZE / sizeof(uint32_t), initial2DRegIndexOne, allMask);
1143- AscendC::MicroAPI::DataCopy(helpAddr + INDEX_SIX * V_REG_SIZE / sizeof(uint32_t), initial4DRegIndexDW, allMask);
1144- AscendC::MicroAPI::DataCopy(helpAddr + INDEX_SEVEN * V_REG_SIZE / sizeof(uint32_t), initial4DRegIndexOneHD, allMask);
1145- }
1146- 
1147- for (uint16_t highBlockIdx = 0; highBlockIdx < highBlockConcurrentCount; ++highBlockIdx) {
1148- uint32_t highArgmaxOffset = highBlockIdx * highConcurrentCount * dArgmaxActual * hArgmaxActual * wArgmaxAligned;
1149- uint32_t highOutputOffset = highBlockIdx * highConcurrentCount * dOutputActual * hOutputActual * wOutputActual;
1150- for (uint16_t dProBatchIdx = 0; dProBatchIdx < dProBatchSize; dProBatchIdx++) {
1151- for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {
1152- for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
1153- T2 offset = (wBatchIdx + hProBatchIdx * wArgmaxAligned + dProBatchIdx * hArgmaxActual * wArgmaxAligned + highArgmaxOffset);
1154- __VEC_SCOPE__
1155- {
1156- 
1157- AscendC::MicroAPI::RegTensor<uint32_t> initial4DRegIndex;
1158- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
1159- AscendC::MicroAPI::MaskReg allMaskU32 =
1160- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1161- AscendC::MicroAPI::DataCopy(initial4DRegIndex, helpAddr);
1162 986 
987+ for (uint16_t highBlockIdx = 0; highBlockIdx < highBlockConcurrentCount; ++highBlockIdx) {
988+ uint32_t curHighArgmaxOffset = highBlockIdx * highArgmaxStride;
989+ uint32_t curHighOutputOffset = highBlockIdx * highOutputStride;
990+ for (uint16_t dProBatchIdx = 0; dProBatchIdx < dProBatchSize; dProBatchIdx++) {
991+ T2 dhOffset = dProBatchIdx * hArgmaxActual * wArgmaxAligned + curHighArgmaxOffset;
992+ for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {
993+ T2 hOffset = hProBatchIdx * wArgmaxAligned + dhOffset;
994+ for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
995+ T2 offset = wBatchIdx + hOffset;
1163 AscendC::MicroAPI::Adds(parallelRegIndex, initial4DRegIndex, offset, allMaskU32);996 AscendC::MicroAPI::Adds(parallelRegIndex, initial4DRegIndex, offset, allMaskU32);
1164- DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask0, highOutputOffset,997+ AscendC::MicroAPI::Duplicate(magicHighReg, magicHigh_dhw);
1165- highOutputPlaneActual, dFullBatchCount * hFullBatchCount * wFullBatchCount, helpAddr);998+ DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask0, curHighOutputOffset,
999+ highOutputPlaneActual, dFullBatchCount * hFullBatchCount * wFullBatchCount,
1000+ magicHighReg, static_cast<int16_t>(shiftHigh_dhw), indexIncReg);
1166 }1001 }
1167- }1002+ for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {
1168- for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {1003+ T2 offset = wBatchIdx + wProBatchSize * wFullBatchCount + hOffset;
1169- T2 offset = (wBatchIdx + wProBatchSize * wFullBatchCount + hProBatchIdx * wArgmaxAligned + dProBatchIdx * hArgmaxActual * wArgmaxAligned + highArgmaxOffset);
1170- __VEC_SCOPE__
1171- {
1172- AscendC::MicroAPI::RegTensor<uint32_t> initial4DRegIndexOne;
1173- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
1174- AscendC::MicroAPI::MaskReg allMaskU32 =
1175- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1176- AscendC::MicroAPI::DataCopy(initial4DRegIndexOne, helpAddr + V_REG_SIZE / sizeof(uint32_t));
1177- 
1178 AscendC::MicroAPI::Adds(parallelRegIndex, initial4DRegIndexOne, offset, allMaskU32);1004 AscendC::MicroAPI::Adds(parallelRegIndex, initial4DRegIndexOne, offset, allMaskU32);
1179- DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask1, highOutputOffset,1005+ AscendC::MicroAPI::Duplicate(magicHighReg, magicHigh_dh);
1180- highOutputPlaneActual, dFullBatchCount * hFullBatchCount, helpAddr);1006+ DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask1, curHighOutputOffset,
1007+ highOutputPlaneActual, dFullBatchCount * hFullBatchCount,
1008+ magicHighReg, static_cast<int16_t>(shiftHigh_dh), indexIncReg);
1181 }1009 }
1182 }1010 }
1183- }1011+ for (uint16_t hTailIdx = 0; hTailIdx < hRemainTail; hTailIdx++) {
1184- }1012+ T2 hOffset = (hProBatchSize * hFullBatchCount + hTailIdx) * wArgmaxAligned + dhOffset;
1185- }1013+ for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
1186- 1014+ T2 offset = wBatchIdx + hOffset;
1187- for (uint16_t highBlockIdx = 0; highBlockIdx < highBlockConcurrentCount; ++highBlockIdx) {
1188- uint32_t highArgmaxOffset = highBlockIdx * highConcurrentCount * dArgmaxActual * hArgmaxActual * wArgmaxAligned;
1189- uint32_t highOutputOffset = highBlockIdx * highConcurrentCount * dOutputActual * hOutputActual * wOutputActual;
1190- for (uint16_t dProBatchIdx = 0; dProBatchIdx < dProBatchSize; dProBatchIdx++) {
1191- for (uint16_t hTailIdx = 0; hTailIdx < hRemainTail; hTailIdx++) {
1192- for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
1193- T2 offset = (wBatchIdx + (hProBatchSize * hFullBatchCount + hTailIdx) * wArgmaxAligned + dProBatchIdx * hArgmaxActual * wArgmaxAligned + highArgmaxOffset);
1194- __VEC_SCOPE__
1195- {
1196- AscendC::MicroAPI::RegTensor<uint32_t> initial4DRegIndexDW;
1197- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
1198- AscendC::MicroAPI::MaskReg allMaskU32 =
1199- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1200- AscendC::MicroAPI::DataCopy(initial4DRegIndexDW, helpAddr + INDEX_SIX * V_REG_SIZE / sizeof(uint32_t));
1201- 
1202 AscendC::MicroAPI::Adds(parallelRegIndex, initial4DRegIndexDW, offset, allMaskU32);1015 AscendC::MicroAPI::Adds(parallelRegIndex, initial4DRegIndexDW, offset, allMaskU32);
1203- DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask2, highOutputOffset,1016+ AscendC::MicroAPI::Duplicate(magicHighReg, magicHigh_dw);
1204- highOutputPlaneActual, dFullBatchCount * wFullBatchCount, helpAddr);1017+ DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask2, curHighOutputOffset,
1018+ highOutputPlaneActual, dFullBatchCount * wFullBatchCount,
1019+ magicHighReg, static_cast<int16_t>(shiftHigh_dw), indexIncReg);
1205 }1020 }
1206- }1021+ for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {
1207- for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {1022+ T2 offset = wBatchIdx + wProBatchSize * wFullBatchCount + hOffset;
1208- T2 offset = wBatchIdx + wProBatchSize * wFullBatchCount + (hProBatchSize * hFullBatchCount + hTailIdx) * wArgmaxAligned + dProBatchIdx * hArgmaxActual * wArgmaxAligned + highArgmaxOffset;
1209- __VEC_SCOPE__
1210- {
1211- AscendC::MicroAPI::RegTensor<uint32_t> initial4DRegIndexOneHD;
1212- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
1213- AscendC::MicroAPI::MaskReg allMaskU32 =
1214- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1215- AscendC::MicroAPI::DataCopy(initial4DRegIndexOneHD, helpAddr + INDEX_SEVEN * V_REG_SIZE / sizeof(uint32_t));
1216- 
1217 AscendC::MicroAPI::Adds(parallelRegIndex, initial4DRegIndexOneHD, offset, allMaskU32);1023 AscendC::MicroAPI::Adds(parallelRegIndex, initial4DRegIndexOneHD, offset, allMaskU32);
1218- DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask3, highOutputOffset,1024+ AscendC::MicroAPI::Duplicate(magicHighReg, magicHigh_d);
1219- highOutputPlaneActual, dFullBatchCount, helpAddr);1025+ DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask3, curHighOutputOffset,
1026+ highOutputPlaneActual, dFullBatchCount,
1027+ magicHighReg, static_cast<int16_t>(shiftHigh_d), indexIncReg);
1220 }1028 }
1221 }1029 }
1222 }1030 }
1223 }1031 }
1224- }
1225- 
1226- for (uint16_t highBlockIdx = 0; highBlockIdx < highBlockConcurrentCount; ++highBlockIdx) {
1227- uint32_t highArgmaxOffset = highBlockIdx * highConcurrentCount * dArgmaxActual * hArgmaxActual * wArgmaxAligned;
1228- uint32_t highOutputOffset = highBlockIdx * highConcurrentCount * dOutputActual * hOutputActual * wOutputActual;
1229- for (uint16_t dTailIdx = 0; dTailIdx < dRemainTail; dTailIdx++) {
1230- for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {
1231- for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
1232- T2 offset = (wBatchIdx + hProBatchIdx * wArgmaxAligned + (dFullBatchCount * dProBatchSize + dTailIdx) * hArgmaxActual * wArgmaxAligned + highArgmaxOffset);
1233- __VEC_SCOPE__
1234- {
1235- AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndex;
1236- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
1237- AscendC::MicroAPI::MaskReg allMaskU32 =
1238- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1239- AscendC::MicroAPI::DataCopy(initial3DRegIndex, helpAddr + INDEX_TWO * V_REG_SIZE / sizeof(uint32_t));
1240 1032 
1033+ for (uint16_t highBlockIdx = 0; highBlockIdx < highBlockConcurrentCount; ++highBlockIdx) {
1034+ uint32_t curHighArgmaxOffset = highBlockIdx * highArgmaxStride;
1035+ uint32_t curHighOutputOffset = highBlockIdx * highOutputStride;
1036+ for (uint16_t dTailIdx = 0; dTailIdx < dRemainTail; dTailIdx++) {
1037+ T2 dOffset = (dFullBatchCount * dProBatchSize + dTailIdx) * hArgmaxActual * wArgmaxAligned + curHighArgmaxOffset;
1038+ for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {
1039+ T2 hOffset = hProBatchIdx * wArgmaxAligned + dOffset;
1040+ for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
1041+ T2 offset = wBatchIdx + hOffset;
1241 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndex, offset, allMaskU32);1042 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndex, offset, allMaskU32);
1242- DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask4, highOutputOffset,1043+ AscendC::MicroAPI::Duplicate(magicHighReg, magicHigh_hw);
1243- highOutputPlaneActual, hFullBatchCount * wFullBatchCount, helpAddr);1044+ DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask4, curHighOutputOffset,
1045+ highOutputPlaneActual, hFullBatchCount * wFullBatchCount,
1046+ magicHighReg, static_cast<int16_t>(shiftHigh_hw), indexIncReg);
1244 }1047 }
1245- }1048+ for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {
1246- for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {1049+ T2 offset = wBatchIdx + wProBatchSize * wFullBatchCount + hOffset;
1247- T2 offset = (wBatchIdx + wProBatchSize * wFullBatchCount + hProBatchIdx * wArgmaxAligned + (dFullBatchCount * dProBatchSize + dTailIdx) * hArgmaxActual * wArgmaxAligned + highArgmaxOffset);
1248- __VEC_SCOPE__
1249- {
1250- AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndexOne;
1251- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
1252- AscendC::MicroAPI::MaskReg allMaskU32 =
1253- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1254- AscendC::MicroAPI::DataCopy(initial3DRegIndexOne, helpAddr + INDEX_THREE * V_REG_SIZE / sizeof(uint32_t));
1255- 
1256 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndexOne, offset, allMaskU32);1050 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndexOne, offset, allMaskU32);
1257- DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask5, highOutputOffset,1051+ AscendC::MicroAPI::Duplicate(magicHighReg, magicHigh_h);
1258- highOutputPlaneActual, hFullBatchCount, helpAddr);1052+ DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask5, curHighOutputOffset,
1053+ highOutputPlaneActual, hFullBatchCount,
1054+ magicHighReg, static_cast<int16_t>(shiftHigh_h), indexIncReg);
1055+ }
1056+ }
1057+ for (uint16_t hTailIdx = 0; hTailIdx < hRemainTail; hTailIdx++) {
1058+ T2 hOffset = (hProBatchSize * hFullBatchCount + hTailIdx) * wArgmaxAligned + dOffset;
1059+ for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
1060+ T2 offset = wBatchIdx + hOffset;
1061+ AscendC::MicroAPI::Adds(parallelRegIndex, initial2DRegIndex, offset, allMaskU32);
1062+ AscendC::MicroAPI::Duplicate(magicHighReg, magicHigh_w);
1063+ DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask6, curHighOutputOffset,
1064+ highOutputPlaneActual, wFullBatchCount,
1065+ magicHighReg, static_cast<int16_t>(shiftHigh_w), indexIncReg);
1066+ }
1067+ for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {
1068+ T2 offset = wBatchIdx + wProBatchSize * wFullBatchCount + hOffset;
1069+ AscendC::MicroAPI::Adds(parallelRegIndex, initial2DRegIndexOne, offset, allMaskU32);
1070+ AscendC::MicroAPI::Duplicate(magicHighReg, magicHigh_1);
1071+ DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask7, curHighOutputOffset,
1072+ highOutputPlaneActual, 1,
1073+ magicHighReg, static_cast<int16_t>(shiftHigh_1), indexIncReg);
1259 }1074 }
1260 }1075 }
1261 }1076 }
1262 }1077 }
1263- }
1264 1078 
1265- for (uint16_t highBlockIdx = 0; highBlockIdx < highBlockConcurrentCount; ++highBlockIdx) {1079+ uint32_t highArgmaxOffset = highBlockConcurrentCount * highArgmaxStride;
1266- uint32_t highArgmaxOffset = highBlockIdx * highConcurrentCount * dArgmaxActual * hArgmaxActual * wArgmaxAligned;1080+ uint32_t highOutputOffset = highBlockConcurrentCount * highOutputStride;
1267- uint32_t highOutputOffset = highBlockIdx * highConcurrentCount * dOutputActual * hOutputActual * wOutputActual;1081+ for (uint16_t dProBatchIdx = 0; dProBatchIdx < dProBatchSize; dProBatchIdx++) {
1268- for (uint16_t dTailIdx = 0; dTailIdx < dRemainTail; dTailIdx++) {1082+ T2 dhOffset = dProBatchIdx * hArgmaxActual * wArgmaxAligned + highArgmaxOffset;
1269- for (uint16_t hTailIdx = 0; hTailIdx < hRemainTail; hTailIdx++) {1083+ for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {
1084+ T2 hOffset = hProBatchIdx * wArgmaxAligned + dhOffset;
1270 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {1085 for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
1271- T2 offset = (wBatchIdx + (hProBatchSize * hFullBatchCount + hTailIdx) * wArgmaxAligned + (dFullBatchCount * dProBatchSize + dTailIdx) * hArgmaxActual * wArgmaxAligned + highArgmaxOffset);1086+ T2 offset = wBatchIdx + hOffset;
1272- __VEC_SCOPE__1087+ AscendC::MicroAPI::Adds(parallelRegIndex, initial4DRegIndex, offset, allMaskU32);
1273- {1088+ AscendC::MicroAPI::Duplicate(magicHighReg, magicHigh_dhw);
1274- AscendC::MicroAPI::RegTensor<uint32_t> initial2DRegIndex;1089+ DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask8, highOutputOffset,
1275- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;1090+ highOutputPlaneActual, dFullBatchCount * hFullBatchCount * wFullBatchCount,
1276- AscendC::MicroAPI::MaskReg allMaskU32 =1091+ magicHighReg, static_cast<int16_t>(shiftHigh_dhw), indexIncReg);
1277- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1278- AscendC::MicroAPI::DataCopy(initial2DRegIndex, helpAddr + INDEX_FOUR * V_REG_SIZE / sizeof(uint32_t));
1279- AscendC::MicroAPI::Adds(parallelRegIndex, initial2DRegIndex, offset, allMaskU32);
1280- DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask6, highOutputOffset,
1281- highOutputPlaneActual, wFullBatchCount, helpAddr);
1282- }
1283 }1092 }
1284 for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {1093 for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {
1285- T2 offset = (wBatchIdx + wProBatchSize * wFullBatchCount + (hProBatchSize * hFullBatchCount + hTailIdx) * wArgmaxAligned + (dProBatchSize * dFullBatchCount + dTailIdx) * hArgmaxActual * wArgmaxAligned + highArgmaxOffset);1094+ T2 offset = wBatchIdx + wProBatchSize * wFullBatchCount + hOffset;
1286- __VEC_SCOPE__
1287- {
1288- AscendC::MicroAPI::RegTensor<uint32_t> initial2DRegIndexOne;
1289- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
1290- AscendC::MicroAPI::MaskReg allMaskU32 =
1291- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1292- AscendC::MicroAPI::DataCopy(initial2DRegIndexOne, helpAddr + INDEX_FIVE * V_REG_SIZE / sizeof(uint32_t));
1293- 
1294- AscendC::MicroAPI::Adds(parallelRegIndex, initial2DRegIndexOne, offset, allMaskU32);
1295- DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask7, highOutputOffset,
1296- highOutputPlaneActual, 1, helpAddr);
1297- }
1298- }
1299- }
1300- }
1301- }
1302- 
1303- 
1304- 
1305- uint32_t highArgmaxOffset = highBlockConcurrentCount * highConcurrentCount * dArgmaxActual * hArgmaxActual * wArgmaxAligned;
1306- uint32_t highOutputOffset= highBlockConcurrentCount * highConcurrentCount * dOutputActual * hOutputActual * wOutputActual;
1307- for (uint16_t dProBatchIdx = 0; dProBatchIdx < dProBatchSize; dProBatchIdx++) {
1308- for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {
1309- for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
1310- T2 offset = (wBatchIdx + hProBatchIdx * wArgmaxAligned + dProBatchIdx * hArgmaxActual * wArgmaxAligned + highArgmaxOffset);
1311- __VEC_SCOPE__
1312- {
1313- AscendC::MicroAPI::RegTensor<uint32_t> initial4DRegIndex;
1314- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
1315- AscendC::MicroAPI::MaskReg allMaskU32 =
1316- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1317- AscendC::MicroAPI::DataCopy(initial4DRegIndex, helpAddr);
1318- AscendC::MicroAPI::Adds(parallelRegIndex, initial4DRegIndex, offset, allMaskU32);
1319- DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask8, highOutputOffset,
1320- highOutputPlaneActual, dFullBatchCount * hFullBatchCount * wFullBatchCount, helpAddr);
1321- }
1322- }
1323- for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {
1324- T2 offset = (wBatchIdx + wProBatchSize * wFullBatchCount + hProBatchIdx * wArgmaxAligned + dProBatchIdx * hArgmaxActual * wArgmaxAligned + highArgmaxOffset);
1325- __VEC_SCOPE__
1326- {
1327- AscendC::MicroAPI::RegTensor<uint32_t> initial4DRegIndexOne;
1328- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
1329- AscendC::MicroAPI::MaskReg allMaskU32 =
1330- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1331- AscendC::MicroAPI::DataCopy(initial4DRegIndexOne, helpAddr + V_REG_SIZE / sizeof(uint32_t));
1332- 
1333 AscendC::MicroAPI::Adds(parallelRegIndex, initial4DRegIndexOne, offset, allMaskU32);1095 AscendC::MicroAPI::Adds(parallelRegIndex, initial4DRegIndexOne, offset, allMaskU32);
1096+ AscendC::MicroAPI::Duplicate(magicHighReg, magicHigh_dh);
1334 DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask9, highOutputOffset,1097 DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask9, highOutputOffset,
1335- highOutputPlaneActual, dFullBatchCount * hFullBatchCount, helpAddr);1098+ highOutputPlaneActual, dFullBatchCount * hFullBatchCount,
1099+ magicHighReg, static_cast<int16_t>(shiftHigh_dh), indexIncReg);
1336 }1100 }
1337 }1101 }
1338- }1102+ for (uint16_t hTailIdx = 0; hTailIdx < hRemainTail; hTailIdx++) {
1339- }1103+ T2 hOffset = (hProBatchSize * hFullBatchCount + hTailIdx) * wArgmaxAligned + dhOffset;
1340- 1104+ for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
1341- for (uint16_t dProBatchIdx = 0; dProBatchIdx < dProBatchSize; dProBatchIdx++) {1105+ T2 offset = wBatchIdx + hOffset;
1342- for (uint16_t hProBatchIdx = 0; hProBatchIdx < hRemainTail; hProBatchIdx++) {
1343- for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
1344- T2 offset = (wBatchIdx + (hProBatchSize * hFullBatchCount + hProBatchIdx) * wArgmaxAligned + dProBatchIdx * hArgmaxActual * wArgmaxAligned + highArgmaxOffset);
1345- __VEC_SCOPE__
1346- {
1347- AscendC::MicroAPI::RegTensor<uint32_t> initial4DRegIndexDW;
1348- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
1349- AscendC::MicroAPI::MaskReg allMaskU32 =
1350- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1351- AscendC::MicroAPI::DataCopy(initial4DRegIndexDW, helpAddr + INDEX_SIX * V_REG_SIZE / sizeof(uint32_t));
1352- 
1353 AscendC::MicroAPI::Adds(parallelRegIndex, initial4DRegIndexDW, offset, allMaskU32);1106 AscendC::MicroAPI::Adds(parallelRegIndex, initial4DRegIndexDW, offset, allMaskU32);
1107+ AscendC::MicroAPI::Duplicate(magicHighReg, magicHigh_dw);
1354 DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask10, highOutputOffset,1108 DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask10, highOutputOffset,
1355- highOutputPlaneActual, dFullBatchCount * wFullBatchCount, helpAddr);1109+ highOutputPlaneActual, dFullBatchCount * wFullBatchCount,
1110+ magicHighReg, static_cast<int16_t>(shiftHigh_dw), indexIncReg);
1356 }1111 }
1357- }1112+ for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {
1358- for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {1113+ T2 offset = wBatchIdx + wProBatchSize * wFullBatchCount + hOffset;
1359- T2 offset = (wBatchIdx + wProBatchSize * wFullBatchCount + (hProBatchSize * hFullBatchCount + hProBatchIdx) * wArgmaxAligned + dProBatchIdx * hArgmaxActual * wArgmaxAligned + highArgmaxOffset);
1360- __VEC_SCOPE__
1361- {
1362- AscendC::MicroAPI::RegTensor<uint32_t> initial4DRegIndexOneHD;
1363- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
1364- AscendC::MicroAPI::MaskReg allMaskU32 =
1365- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1366- AscendC::MicroAPI::DataCopy(initial4DRegIndexOneHD, helpAddr + INDEX_SEVEN * V_REG_SIZE / sizeof(uint32_t));
1367- 
1368 AscendC::MicroAPI::Adds(parallelRegIndex, initial4DRegIndexOneHD, offset, allMaskU32);1114 AscendC::MicroAPI::Adds(parallelRegIndex, initial4DRegIndexOneHD, offset, allMaskU32);
1115+ AscendC::MicroAPI::Duplicate(magicHighReg, magicHigh_d);
1369 DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask11, highOutputOffset,1116 DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask11, highOutputOffset,
1370- highOutputPlaneActual, dFullBatchCount, helpAddr);1117+ highOutputPlaneActual, dFullBatchCount,
1118+ magicHighReg, static_cast<int16_t>(shiftHigh_d), indexIncReg);
1371 }1119 }
1372 }1120 }
1373 }1121 }
1374- }
1375 1122 
1376- for (uint16_t dTailIdx = 0; dTailIdx < dRemainTail; dTailIdx++) {
1377- for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {
1378- for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
1379- T2 offset = (wBatchIdx + hProBatchIdx * wArgmaxAligned + (dFullBatchCount * dProBatchSize + dTailIdx) * hArgmaxActual * wArgmaxAligned + highArgmaxOffset);
1380- __VEC_SCOPE__
1381- {
1382- 
1383- AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndex;
1384- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
1385- AscendC::MicroAPI::MaskReg allMaskU32 =
1386- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1387- AscendC::MicroAPI::DataCopy(initial3DRegIndex, helpAddr + INDEX_TWO * V_REG_SIZE / sizeof(uint32_t));
1388 1123 
1124+ for (uint16_t dTailIdx = 0; dTailIdx < dRemainTail; dTailIdx++) {
1125+ T2 dOffset = (dFullBatchCount * dProBatchSize + dTailIdx) * hArgmaxActual * wArgmaxAligned + highArgmaxOffset;
1126+ for (uint16_t hProBatchIdx = 0; hProBatchIdx < hProBatchSize; hProBatchIdx++) {
1127+ T2 hOffset = hProBatchIdx * wArgmaxAligned + dOffset;
1128+ for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
1129+ T2 offset = wBatchIdx + hOffset;
1389 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndex, offset, allMaskU32);1130 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndex, offset, allMaskU32);
1131+ AscendC::MicroAPI::Duplicate(magicHighReg, magicHigh_hw);
1390 DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask12, highOutputOffset,1132 DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask12, highOutputOffset,
1391- highOutputPlaneActual, hFullBatchCount * wFullBatchCount, helpAddr);1133+ highOutputPlaneActual, hFullBatchCount * wFullBatchCount,
1134+ magicHighReg, static_cast<int16_t>(shiftHigh_hw), indexIncReg);
1392 }1135 }
1393- }1136+ for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {
1394- for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {1137+ T2 offset = wBatchIdx + wProBatchSize * wFullBatchCount + hOffset;
1395- T2 offset = (wBatchIdx + wProBatchSize * wFullBatchCount + hProBatchIdx * wArgmaxAligned + (dFullBatchCount * dProBatchSize + dTailIdx) * hArgmaxActual * wArgmaxAligned + highArgmaxOffset);
1396- __VEC_SCOPE__
1397- {
1398- AscendC::MicroAPI::RegTensor<uint32_t> initial3DRegIndexOne;
1399- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
1400- AscendC::MicroAPI::MaskReg allMaskU32 =
1401- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1402- AscendC::MicroAPI::DataCopy(initial3DRegIndexOne, helpAddr + INDEX_THREE * V_REG_SIZE / sizeof(uint32_t));
1403- 
1404 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndexOne, offset, allMaskU32);1138 AscendC::MicroAPI::Adds(parallelRegIndex, initial3DRegIndexOne, offset, allMaskU32);
1139+ AscendC::MicroAPI::Duplicate(magicHighReg, magicHigh_h);
1405 DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask13, highOutputOffset,1140 DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask13, highOutputOffset,
1406- highOutputPlaneActual, hFullBatchCount, helpAddr);1141+ highOutputPlaneActual, hFullBatchCount,
1142+ magicHighReg, static_cast<int16_t>(shiftHigh_h), indexIncReg);
1407 }1143 }
1408 }1144 }
1409- }1145+ for (uint16_t hTailIdx = 0; hTailIdx < hRemainTail; hTailIdx++) {
1410- }1146+ T2 hOffset = (hProBatchSize * hFullBatchCount + hTailIdx) * wArgmaxAligned + dOffset;
1411- 1147+ for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
1412- for (uint16_t dTailIdx = 0; dTailIdx < dRemainTail; dTailIdx++) {1148+ T2 offset = wBatchIdx + hOffset;
1413- for (uint16_t hTailIdx = 0; hTailIdx < hRemainTail; hTailIdx++) {
1414- for (uint16_t wBatchIdx = 0; wBatchIdx < wProBatchSize; wBatchIdx++) {
1415- T2 offset = (wBatchIdx + (hProBatchSize * hFullBatchCount + hTailIdx) * wArgmaxAligned + (dFullBatchCount * dProBatchSize + dTailIdx) * hArgmaxActual * wArgmaxAligned + highArgmaxOffset);
1416- __VEC_SCOPE__
1417- {
1418- AscendC::MicroAPI::RegTensor<uint32_t> initial2DRegIndex;
1419- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
1420- AscendC::MicroAPI::MaskReg allMaskU32 =
1421- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1422- AscendC::MicroAPI::DataCopy(initial2DRegIndex, helpAddr + INDEX_FOUR * V_REG_SIZE / sizeof(uint32_t));
1423- 
1424 AscendC::MicroAPI::Adds(parallelRegIndex, initial2DRegIndex, offset, allMaskU32);1149 AscendC::MicroAPI::Adds(parallelRegIndex, initial2DRegIndex, offset, allMaskU32);
1150+ AscendC::MicroAPI::Duplicate(magicHighReg, magicHigh_w);
1425 DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask14, highOutputOffset,1151 DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask14, highOutputOffset,
1426- highOutputPlaneActual, wFullBatchCount, helpAddr);1152+ highOutputPlaneActual, wFullBatchCount,
1153+ magicHighReg, static_cast<int16_t>(shiftHigh_w), indexIncReg);
1427 }1154 }
1428- }1155+ for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {
1429- for (uint16_t wBatchIdx = 0; wBatchIdx < wRemainTail; wBatchIdx++) {1156+ T2 offset = wBatchIdx + wProBatchSize * wFullBatchCount + hOffset;
1430- T2 offset = (wBatchIdx + wProBatchSize * wFullBatchCount + (hProBatchSize * hFullBatchCount + hTailIdx) * wArgmaxAligned + (dProBatchSize * dFullBatchCount + dTailIdx) * hArgmaxActual * wArgmaxAligned + highArgmaxOffset);
1431- __VEC_SCOPE__
1432- {
1433- AscendC::MicroAPI::RegTensor<uint32_t> initial2DRegIndexOne;
1434- AscendC::MicroAPI::RegTensor<uint32_t> parallelRegIndex;
1435- AscendC::MicroAPI::MaskReg allMaskU32 =
1436- AscendC::MicroAPI::CreateMask<uint32_t, AscendC::MicroAPI::MaskPattern::ALL>();
1437- AscendC::MicroAPI::DataCopy(initial2DRegIndexOne, helpAddr + INDEX_FIVE * V_REG_SIZE / sizeof(uint32_t));
1438- 
1439 AscendC::MicroAPI::Adds(parallelRegIndex, initial2DRegIndexOne, offset, allMaskU32);1157 AscendC::MicroAPI::Adds(parallelRegIndex, initial2DRegIndexOne, offset, allMaskU32);
1158+ AscendC::MicroAPI::Duplicate(magicHighReg, magicHigh_1);
1440 DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask15, highOutputOffset,1159 DoMulNCNcdhwFullLoad<T1, T2>(yAddr, gradAddr, argmaxAddr, parallelRegIndex, mask15, highOutputOffset,
1441- highOutputPlaneActual, 1, helpAddr);1160+ highOutputPlaneActual, 1,
1161+ magicHighReg, static_cast<int16_t>(shiftHigh_1), indexIncReg);
1442 }1162 }
1443 }1163 }
1444 }1164 }
1445 }1165 }
1446}1166}
1447 1167 
1168+ 
1448} // namespace MaxPool3DGradWithArgmaxNCDHWNameSpace1169} // namespace MaxPool3DGradWithArgmaxNCDHWNameSpace
1449-#endif // MAX_POOL_GRAD_WITH_ARGMAX_SIMD_IMPL_H_1170+#endif // MAX_POOL_GRAD_WITH_ARGMAX_SIMD_IMPL_H_
Mpooling/max_pool3d_grad_with_argmax/op_kernel/arch35/max_pool3d_grad_with_argmax_simd_impl.h+484-1093
Mpooling/max_pool3d_grad_with_argmax/tests/ut/op_host/test_max_pool3d_grad_with_argmax_tiling.cpp+5-5
@@ -553,7 +553,7 @@ static void ExecuteTestCase(
553 ge::DataType dtype, int64_t index_dtype, bool ceil_mode, std::string data_format, uint64_t except_tilingkey,553 ge::DataType dtype, int64_t index_dtype, bool ceil_mode, std::string data_format, uint64_t except_tilingkey,
554 std::string expect)554 std::string expect)
555{555{
556- dlog_setlevel(0, 0, 0);556+ // dlog_setlevel(0, 0, 0);
557 557 
558 string compile_info_string = R"({558 string compile_info_string = R"({
559 "hardware_info": {"BT_SIZE": 0, "load3d_constraints": "1",559 "hardware_info": {"BT_SIZE": 0, "load3d_constraints": "1",
@@ -644,7 +644,7 @@ static void ExecuteTestCase(
644 ASSERT_EQ(tiling_key, except_tilingkey);644 ASSERT_EQ(tiling_key, except_tilingkey);
645 auto tilingData = tiling_context->GetRawTilingData();645 auto tilingData = tiling_context->GetRawTilingData();
646 ASSERT_NE(tilingData, nullptr);646 ASSERT_NE(tilingData, nullptr);
647- dlog_setlevel(0, 3, 0);647+ // dlog_setlevel(0, 3, 0);
648}648}
649 649 
650static void ExecuteFailTestCase(650static void ExecuteFailTestCase(
@@ -653,7 +653,7 @@ static void ExecuteFailTestCase(
653 ge::DataType dtype, ge::DataType argmaxDtype, ge::DataType yDtype, bool ceil_mode, std::string data_format,653 ge::DataType dtype, ge::DataType argmaxDtype, ge::DataType yDtype, bool ceil_mode, std::string data_format,
654 int64_t index_dtype, ge::graphStatus expectResult)654 int64_t index_dtype, ge::graphStatus expectResult)
655{655{
656- dlog_setlevel(0, 0, 0);656+ // dlog_setlevel(0, 0, 0);
657 657 
658 string compile_info_string = R"({658 string compile_info_string = R"({
659 "hardware_info": {"BT_SIZE": 0, "load3d_constraints": "1",659 "hardware_info": {"BT_SIZE": 0, "load3d_constraints": "1",
@@ -735,7 +735,7 @@ static void ExecuteFailTestCase(
735 735 
736 auto ret = tiling_func(tiling_context);736 auto ret = tiling_func(tiling_context);
737 ASSERT_EQ(ret, expectResult);737 ASSERT_EQ(ret, expectResult);
738- dlog_setlevel(0, 3, 0);738+ // dlog_setlevel(0, 3, 0);
739}739}
740 740 
741TEST_F(MaxPool3dGradWithArgmaxTiling, tiling_invalid_format)741TEST_F(MaxPool3dGradWithArgmaxTiling, tiling_invalid_format)
@@ -982,7 +982,7 @@ TEST_F(MaxPool3dGradWithArgmaxTiling, MaxPool3DGradWithArgmax_tiling_simd_test_0
982 int64_t index_dtype = 3;982 int64_t index_dtype = 3;
983 bool ceil_mode = false;983 bool ceil_mode = false;
984 std::string data_format = "NCDHW";984 std::string data_format = "NCDHW";
985- uint64_t except_tilingkey = 1025;985+ uint64_t except_tilingkey = 1;
986 std::string expect = " ";986 std::string expect = " ";
987 ExecuteTestCase(987 ExecuteTestCase(
988 xShape, gradShape, argmaxShape, yShape, ksize, strides, pads, dilation, dtype, index_dtype, ceil_mode,988 xShape, gradShape, argmaxShape, yShape, ksize, strides, pads, dilation, dtype, index_dtype, ceil_mode,