avgpool3d、adaptivemaxpool3d池化算子cleancode清理-ops-nn-AtomGit

已合并

avgpool3d、adaptivemaxpool3d池化算子cleancode清理 #2534

chenyanbin8创建于 3月11日从chenyanbin8/ops-nn_2： fix/pool_cleancode_9.0.0 合入到 cann/ops-nn： 9.0.0

avgpool3d、adaptivemaxpool3d池化算子cleancode清理 #2534

已合并

chenyanbin8创建于 3月11日

从chenyanbin8/ops-nn_2 fix/pool_cleancode_9.0.0合入到cann/ops-nn 9.0.0

共 9 个文件变更+879-1146

Mpooling/adaptive_max_pool3d/CMakeLists.txt+1-1

	@@ -8,4 +8,4 @@
8	# See LICENSE in the root of the software repository for the full text of the License.	8	# See LICENSE in the root of the software repository for the full text of the License.
9	# ----------------------------------------------------------------------------	9	# ----------------------------------------------------------------------------
10		10
11	-add_modules_sources(HOSTNAME ${OPHOST_NAME} MODE PRIVATE DIR ${CMAKE_CURRENT_SOURCE_DIR} OPTYPE adaptive_max_pool3d ACLNNTYPE aclnn_exclude DEPENDENCIES adaptive_pool3d_common batch_mat_mul_v3 max_pool3d_with_argmax_v2 max_pool_with_argmax_v3)	11	+add_modules_sources(HOSTNAME ${OPHOST_NAME} MODE PRIVATE DIR ${CMAKE_CURRENT_SOURCE_DIR} OPTYPE adaptive_max_pool3d ACLNNTYPE aclnn_exclude DEPENDENCIES adaptive_pool3d_common batch_mat_mul_v3 max_pool3d_with_argmax_v2 max_pool_with_argmax_v3 pool_3d_common)

Mpooling/adaptive_max_pool3d/op_kernel/adaptive_max_pool3d_small_pool.h+104-270文件内容审核中，请稍后刷新重试

Mpooling/avg_pool3_d/op_kernel/avg_pool3d_common.h+174-0

	@@ -225,6 +225,42 @@ struct IndexBuffer {
225	}	225	}
226	};	226	};
227		227
		228	+template <typename T, int32_t QUEUE_DEPTH>
		229	+struct PoolMem {
		230	+ TPipe* pipe;
		231	+ TQue<QuePosition::VECIN, QUEUE_DEPTH> inputQueue;
		232	+ TQue<QuePosition::VECOUT, QUEUE_DEPTH> outputQueue;
		233	+
		234	+ TBuf<QuePosition::VECCALC> tmpPattern;
		235	+ TBuf<TPosition::VECCALC> sumBuf;
		236	+ LocalTensor<float> sumBufLocal;
		237	+
		238	+ GlobalTensor<T> inputGlobal;
		239	+ GlobalTensor<T> outputGlobal;
		240	+
		241	+ TQue<QuePosition::VECIN, QUEUE_DEPTH> syncWorkQueue;
		242	+ GlobalTensor<int32_t> syncTensorsGM;
		243	+ TBuf<TPosition::VECCALC> clearTensorBuff;
		244	+
		245	+ int64_t inC;
		246	+ int64_t alignC;
		247	+ int64_t outputPointNum;
		248	+ int64_t outputPointOffset;
		249	+ int64_t lastPointOffset;
		250	+ int64_t atomicAddNum;
		251	+ int64_t nextCoreAddrOffset;
		252	+ uint32_t inputBufLen;
		253	+
		254	+ PoolShape inputShape;
		255	+ PoolShape outputShape;
		256	+ int64_t indexBufLen;
		257	+ IndexBuffer indexBuf;
		258	+ PoolParameter poolParam;
		259	+ uint32_t numPerBlock;
		260	+ int32_t validTailLen;
		261	+ uint32_t usedCoreNum;
		262	+};
		263	+
228	__aicore__ inline void SToMTE2Sync() {	264	__aicore__ inline void SToMTE2Sync() {
229	event_t eventIDSToMTE2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE2));	265	event_t eventIDSToMTE2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE2));
230	SetFlag<HardEvent::S_MTE2>(eventIDSToMTE2);	266	SetFlag<HardEvent::S_MTE2>(eventIDSToMTE2);
	@@ -274,6 +310,144 @@ __aicore__ inline void MTE3ToMTE2Sync()
274	WaitFlag<HardEvent::MTE3_MTE2>(eventIDMTE3ToMTE2);	310	WaitFlag<HardEvent::MTE3_MTE2>(eventIDMTE3ToMTE2);
275	}	311	}
276		312
		313	+template <typename T, int32_t QUEUE_DEPTH>
		314	+__aicore__ inline void HandleTailMask( LocalTensor<T>& outputLocal, int64_t gatherOffset, PoolMem<T, QUEUE_DEPTH>& poolMem, uint32_t mask) {
		315	+ MTE3ToVSync();
		316	+ int32_t lastLeftShift = poolMem.validTailLen;
		317	+ uint64_t rsvdCnt = 0;
		318	+ if constexpr (std::is_same_v<T, float>) {
		319	+ LocalTensor<uint32_t> bufPattern = poolMem.tmpPattern.template Get<uint32_t>();
		320	+ int32_t preLeftShift = poolMem.numPerBlock + lastLeftShift;
		321	+
		322	+ bufPattern.SetValue(0, (1u << preLeftShift) - (1u << lastLeftShift));
		323	+ SToVSync();
		324	+ GatherMask(outputLocal[gatherOffset], outputLocal[gatherOffset], bufPattern, true, mask, {1, 1, 8, 8}, rsvdCnt);
		325	+ } else {
		326	+ LocalTensor<uint16_t> bufPattern = poolMem.tmpPattern.template Get<uint16_t>();
		327	+ int32_t preLeftShift = poolMem.numPerBlock - lastLeftShift;
		328	+
		329	+ bufPattern.SetValue(0, ((1u << preLeftShift) - 1u) << lastLeftShift);
		330	+ bufPattern.SetValue(1, (1u << lastLeftShift) - 1u);
		331	+ SToVSync();
		332	+ GatherMask(outputLocal[gatherOffset], outputLocal[gatherOffset], bufPattern, true, mask, {1, 1, 8, 8}, rsvdCnt);
		333	+ }
		334	+ VToMTE3Sync();
		335	+}
		336	+
		337	+template <typename T, int32_t QUEUE_DEPTH>
		338	+__aicore__ inline void CastAndEnqueueOutput(
		339	+ PoolMem<T, QUEUE_DEPTH>& poolMem, int64_t count, float factor) {
		340	+ Muls(poolMem.sumBufLocal, poolMem.sumBufLocal, factor, count);
		341	+
		342	+ LocalTensor<T> outputLocal = poolMem.outputQueue.template AllocTensor<T>();
		343	+ if constexpr (std::is_same_v<T, float>) {
		344	+#if __CCE_AICORE__ < 220
		345	+ Adds(outputLocal, poolMem.sumBufLocal, 0.0f, AlignUp(count, poolMem.numPerBlock));
		346	+#else
		347	+ DataCopy(outputLocal, poolMem.sumBufLocal, AlignUp(count, poolMem.numPerBlock));
		348	+#endif
		349	+ } else if constexpr (std::is_same_v<T, half>) {
		350	+ Cast(outputLocal, poolMem.sumBufLocal, RoundMode::CAST_NONE, count);
		351	+ } else {
		352	+ Cast(outputLocal, poolMem.sumBufLocal, RoundMode::CAST_RINT, count);
		353	+ }
		354	+ poolMem.outputQueue.EnQue(outputLocal);
		355	+}
		356	+
		357	+template <typename T, int32_t QUEUE_DEPTH>
		358	+__aicore__ inline void InitCommonBuffers(
		359	+ PoolMem<T, QUEUE_DEPTH>& poolMem, GM_ADDR workspace) {
		360	+#if __CCE_AICORE__ < 220
		361	+ if (poolMem.atomicAddNum) {
		362	+ poolMem.pipe->InitBuffer(poolMem.tmpPattern, poolMem.numPerBlock * sizeof(T));
		363	+
		364	+ poolMem.pipe->InitBuffer(poolMem.syncWorkQueue, QUEUE_DEPTH, 8 * 32 * sizeof(int32_t));
		365	+ poolMem.syncTensorsGM.SetGlobalBuffer((__gm__ int32_t )workspace, poolMem.usedCoreNum 8 * 32);
		366	+ poolMem.pipe->InitBuffer(poolMem.clearTensorBuff, DEFAULT_CLEAR_UB_SIZE * sizeof(T));
		367	+ } else if (poolMem.validTailLen != 0) {
		368	+ poolMem.pipe->InitBuffer(poolMem.tmpPattern, poolMem.numPerBlock * sizeof(T));
		369	+ }
		370	+#endif
		371	+}
		372	+
		373	+template <typename T, int32_t QUEUE_DEPTH>
		374	+__aicore__ inline void CopyInTemplate(PoolMem<T, QUEUE_DEPTH>& poolMem, int64_t offset, uint16_t blockCount, uint32_t blockLen, uint8_t rightPadding) {
		375	+ LocalTensor<T> inputLocal = poolMem.inputQueue.template AllocTensor<T>();
		376	+#if __CCE_AICORE__ < 220
		377	+ if constexpr (std::is_same_v<T, float>) {
		378	+ if (blockLen == poolMem.alignC) {
		379	+ DataCopyParams copyParams{blockCount, static_cast<uint16_t>(blockLen / poolMem.numPerBlock), 0, 0};
		380	+ DataCopy(inputLocal, poolMem.inputGlobal[offset], copyParams);
		381	+ } else {
		382	+ for (int i = 0; i < blockCount; i++) {
		383	+ DataCopy(inputLocal[i * poolMem.alignC], poolMem.inputGlobal[offset + i * blockLen], poolMem.alignC);
		384	+ }
		385	+ }
		386	+ } else {
		387	+ if (blockLen == poolMem.alignC) {
		388	+ DataCopyParams copyParams{blockCount, static_cast<uint16_t>(blockLen / poolMem.numPerBlock), 0, 0};
		389	+ DataCopy(inputLocal[poolMem.inputBufLen], poolMem.inputGlobal[offset], copyParams);
		390	+ } else {
		391	+ for (int i = 0; i < blockCount; i++) {
		392	+ DataCopy(inputLocal[poolMem.inputBufLen + i * poolMem.alignC], poolMem.inputGlobal[offset + i * blockLen], poolMem.alignC);
		393	+ }
		394	+ }
		395	+ }
		396	+#else
		397	+ DataCopyExtParams copyParams{blockCount, static_cast<uint32_t>(blockLen * sizeof(T)), 0, 0, 0};
		398	+ DataCopyPadExtParams<T> padParams{true, 0, rightPadding, 0};
		399	+ if constexpr (std::is_same_v<T, float>) {
		400	+ DataCopyPad(inputLocal, poolMem.inputGlobal[offset], copyParams, padParams);
		401	+ } else {
		402	+ DataCopyPad(inputLocal[poolMem.inputBufLen], poolMem.inputGlobal[offset], copyParams, padParams);
		403	+ }
		404	+#endif
		405	+ poolMem.inputQueue.EnQue(inputLocal);
		406	+}
		407	+
		408	+template <typename T, int32_t QUEUE_DEPTH>
		409	+__aicore__ inline void HandleAtomicAdd(
		410	+ PoolMem<T, QUEUE_DEPTH>& poolMem) {
		411	+#if __CCE_AICORE__ < 220
		412	+ if (poolMem.atomicAddNum) {
		413	+ LocalTensor<T> clearUb = poolMem.clearTensorBuff.template Get<T>();
		414	+ Duplicate(clearUb, (T)0, DEFAULT_CLEAR_UB_SIZE);
		415	+
		416	+ VToMTE3Sync();
		417	+ int64_t curOutputPointIdx = poolMem.lastPointOffset;
		418	+ for (int i = 0; i < poolMem.atomicAddNum; i++, curOutputPointIdx--) {
		419	+ DataCopy<T>(poolMem.outputGlobal[curOutputPointIdx * poolMem.inC], clearUb, poolMem.numPerBlock);
		420	+ }
		421	+
		422	+ DataCopy(poolMem.syncTensorsGM[0], clearUb.template ReinterpretCast<int32_t>(), poolMem.usedCoreNum * 8 * 32);
		423	+ LocalTensor<int32_t> syncLocalTensor = poolMem.syncWorkQueue.template AllocTensor<int32_t>();
		424	+ AscendC::SyncAll(poolMem.syncTensorsGM, syncLocalTensor, int32_t(poolMem.usedCoreNum));
		425	+ poolMem.syncWorkQueue.FreeTensor(syncLocalTensor);
		426	+ }
		427	+#endif
		428	+}
		429	+
		430	+template <typename T, int32_t QUEUE_DEPTH>
		431	+__aicore__ inline void HandleAtomicAddWithTail(
		432	+ PoolMem<T, QUEUE_DEPTH>& poolMem, int64_t curOutputPointIdx, int64_t tailLength) {
		433	+#if __CCE_AICORE__ < 220
		434	+ if (poolMem.atomicAddNum) {
		435	+ LocalTensor<T> clearUb = poolMem.clearTensorBuff.template Get<T>();
		436	+ Duplicate(clearUb, (T)0, DEFAULT_CLEAR_UB_SIZE);
		437	+
		438	+ VToMTE3Sync();
		439	+ for (int i = 0; i < poolMem.atomicAddNum; i++, curOutputPointIdx--) {
		440	+ DataCopy<T>(poolMem.outputGlobal[curOutputPointIdx * tailLength], clearUb, poolMem.numPerBlock);
		441	+ }
		442	+
		443	+ DataCopy(poolMem.syncTensorsGM[0], clearUb.template ReinterpretCast<int32_t>(), poolMem.usedCoreNum * 8 * 32);
		444	+ LocalTensor<int32_t> syncLocalTensor = poolMem.syncWorkQueue.template AllocTensor<int32_t>();
		445	+ AscendC::SyncAll(poolMem.syncTensorsGM, syncLocalTensor, int32_t(poolMem.usedCoreNum));
		446	+ poolMem.syncWorkQueue.FreeTensor(syncLocalTensor);
		447	+ }
		448	+#endif
		449	+}
		450	+
277	} // namespace AvgPool3d	451	} // namespace AvgPool3d
278		452
279	#endif // AVG_POOL3D_COMMON_H_	453	#endif // AVG_POOL3D_COMMON_H_

Mpooling/avg_pool3_d/op_kernel/avg_pool3d_ncdhw_reduce_d.h+60-139

	@@ -37,51 +37,25 @@ private:
37	__aicore__ inline void ReduceSumDWindow(	37	__aicore__ inline void ReduceSumDWindow(
38	const Index& index, LocalTensor<float>& sumBufLocal, int64_t startOffset, int64_t len);	38	const Index& index, LocalTensor<float>& sumBufLocal, int64_t startOffset, int64_t len);
39		39
40	- TPipe* pipe;
41	- TQue<QuePosition::VECIN, QUEUE_DEPTH> inputQueue;
42	- TQue<QuePosition::VECOUT, QUEUE_DEPTH> outputQueue;
43	-
44	- TBuf<QuePosition::VECCALC> tmpPattern;
45	- TBuf<TPosition::VECCALC> sumBuf;
46	- LocalTensor<float> sumBufLocal;
47	-
48	- GlobalTensor<T> inputGlobal;
49	- GlobalTensor<T> outputGlobal;
50	-
51	int64_t hwLength;	40	int64_t hwLength;
52	int64_t tileHW;	41	int64_t tileHW;
53	int64_t ncdBlockLength;	42	int64_t ncdBlockLength;
54	int64_t ncdOffset;	43	int64_t ncdOffset;
55	- int64_t nextCoreAddrOffset;
56	- int64_t atomicAddNum;
57	int64_t hwTailLength;	44	int64_t hwTailLength;
58	int64_t hwTailAlign;	45	int64_t hwTailAlign;
59		46
60	- PoolShape inputShape;	47	+ PoolMem<T, QUEUE_DEPTH> poolMem;
61	- PoolShape outputShape;
62	-
63	- int64_t indexBufLen;
64	- IndexBuffer indexBuf;
65	- PoolParameter poolParam;
66	-
67	- uint32_t numPerBlock;
68	- int32_t validTailLen;
69	-
70	- TQue<QuePosition::VECIN, QUEUE_DEPTH> syncWorkQueue;
71	- GlobalTensor<int32_t> syncTensorsGM;
72	- TBuf<TPosition::VECCALC> clearTensorBuff;
73	- uint32_t usedCoreNum;
74	};	48	};
75		49
76	template <typename T, int32_t QUEUE_DEPTH>	50	template <typename T, int32_t QUEUE_DEPTH>
77	__aicore__ inline void KernelAvgPool3dReduceD<T, QUEUE_DEPTH>::InitTiling(const AvgPool3DTilingData* __restrict__ tiling) {	51	__aicore__ inline void KernelAvgPool3dReduceD<T, QUEUE_DEPTH>::InitTiling(const AvgPool3DTilingData* __restrict__ tiling) {
78	- inputShape = PoolShape(tiling->inN, tiling->inC, tiling->inD, tiling->inH, tiling->inW);	52	+ poolMem.inputShape = PoolShape(tiling->inN, tiling->inC, tiling->inD, tiling->inH, tiling->inW);
79	- outputShape = PoolShape(tiling->inN, tiling->inC, tiling->outD, tiling->outH, tiling->outW);	53	+ poolMem.outputShape = PoolShape(tiling->inN, tiling->inC, tiling->outD, tiling->outH, tiling->outW);
80		54
81	- poolParam = PoolParameter(tiling->kD, tiling->kH, tiling->kW, tiling->dD, tiling->dH, tiling->dW,	55	+ poolMem.poolParam = PoolParameter(tiling->kD, tiling->kH, tiling->kW, tiling->dD, tiling->dH, tiling->dW,
82	tiling->pD, tiling->pH, tiling->pW, tiling->divisorOverride, tiling->countIncludePad);	56	tiling->pD, tiling->pH, tiling->pW, tiling->divisorOverride, tiling->countIncludePad);
83		57
84	- numPerBlock = GetDataBlockSizeInBytes() / sizeof(T);	58	+ poolMem.numPerBlock = GetDataBlockSizeInBytes() / sizeof(T);
85		59
86	hwLength = tiling->inH * tiling->inW;	60	hwLength = tiling->inH * tiling->inW;
87	tileHW = tiling->tileHW;	61	tileHW = tiling->tileHW;
	@@ -90,102 +64,85 @@ __aicore__ inline void KernelAvgPool3dReduceD<T, QUEUE_DEPTH>::InitTiling(const
90	ncdOffset = GetBlockIdx() < tiling->formerNum	64	ncdOffset = GetBlockIdx() < tiling->formerNum
91	? tiling->formerLength * GetBlockIdx()	65	? tiling->formerLength * GetBlockIdx()
92	: tiling->formerNum * tiling->formerLength + tiling->tailLength * (GetBlockIdx() - tiling->formerNum);	66	: tiling->formerNum * tiling->formerLength + tiling->tailLength * (GetBlockIdx() - tiling->formerNum);
93	- nextCoreAddrOffset = (ncdOffset + ncdBlockLength) * hwLength;	67	+ poolMem.nextCoreAddrOffset = (ncdOffset + ncdBlockLength) * hwLength;
94	- atomicAddNum = tiling->atomicAddNum;	68	+ poolMem.atomicAddNum = tiling->atomicAddNum;
95	hwTailLength = hwLength % tileHW;	69	hwTailLength = hwLength % tileHW;
96	- hwTailAlign = AlignUp(hwTailLength, numPerBlock);	70	+ hwTailAlign = AlignUp(hwTailLength, poolMem.numPerBlock);
97	- validTailLen = hwTailLength % numPerBlock;	71	+ poolMem.validTailLen = hwTailLength % poolMem.numPerBlock;
98	- usedCoreNum = tiling->usedCoreNum;	72	+ poolMem.usedCoreNum = tiling->usedCoreNum;
99	}	73	}
100		74
101	template <typename T, int32_t QUEUE_DEPTH>	75	template <typename T, int32_t QUEUE_DEPTH>
102	__aicore__ inline void KernelAvgPool3dReduceD<T, QUEUE_DEPTH>::CopyIn(int64_t offset, int64_t len) {	76	__aicore__ inline void KernelAvgPool3dReduceD<T, QUEUE_DEPTH>::CopyIn(int64_t offset, int64_t len) {
103	- LocalTensor<T> inputLocal = inputQueue.template AllocTensor<T>();	77	+ LocalTensor<T> inputLocal = poolMem.inputQueue.template AllocTensor<T>();
104	#if __CCE_AICORE__ < 220	78	#if __CCE_AICORE__ < 220
105	if constexpr (std::is_same_v<T, float>) {	79	if constexpr (std::is_same_v<T, float>) {
106	if (len == tileHW) {	80	if (len == tileHW) {
107	- DataCopyParams copyParams{1, static_cast<uint16_t>(len / numPerBlock), 0, 0};	81	+ DataCopyParams copyParams{1, static_cast<uint16_t>(len / poolMem.numPerBlock), 0, 0};
108	- DataCopy(inputLocal, inputGlobal[offset], copyParams);	82	+ DataCopy(inputLocal, poolMem.inputGlobal[offset], copyParams);
109	} else {	83	} else {
110	- DataCopy(inputLocal, inputGlobal[offset], hwTailAlign);	84	+ DataCopy(inputLocal, poolMem.inputGlobal[offset], hwTailAlign);
111	}	85	}
112	} else {	86	} else {
113	if (len == tileHW) {	87	if (len == tileHW) {
114	- DataCopyParams copyParams{1, static_cast<uint16_t>(len / numPerBlock), 0, 0};	88	+ DataCopyParams copyParams{1, static_cast<uint16_t>(len / poolMem.numPerBlock), 0, 0};
115	- DataCopy(inputLocal[tileHW], inputGlobal[offset], copyParams);	89	+ DataCopy(inputLocal[tileHW], poolMem.inputGlobal[offset], copyParams);
116	} else {	90	} else {
117	- DataCopy(inputLocal[tileHW], inputGlobal[offset], hwTailAlign);	91	+ DataCopy(inputLocal[tileHW], poolMem.inputGlobal[offset], hwTailAlign);
118	}	92	}
119	}	93	}
120	#else	94	#else
121	DataCopyExtParams copyParams{1, static_cast<uint32_t>(len * sizeof(T)), 0, 0, 0};	95	DataCopyExtParams copyParams{1, static_cast<uint32_t>(len * sizeof(T)), 0, 0, 0};
122	DataCopyPadExtParams<T> padParams{false, 0, 0, 0};	96	DataCopyPadExtParams<T> padParams{false, 0, 0, 0};
123	if constexpr (std::is_same_v<T, float>) {	97	if constexpr (std::is_same_v<T, float>) {
124	- DataCopyPad(inputLocal, inputGlobal[offset], copyParams, padParams);	98	+ DataCopyPad(inputLocal, poolMem.inputGlobal[offset], copyParams, padParams);
125	} else {	99	} else {
126	- DataCopyPad(inputLocal[tileHW], inputGlobal[offset], copyParams, padParams);	100	+ DataCopyPad(inputLocal[tileHW], poolMem.inputGlobal[offset], copyParams, padParams);
127	}	101	}
128	#endif	102	#endif
129	- inputQueue.EnQue(inputLocal);	103	+ poolMem.inputQueue.EnQue(inputLocal);
130	}	104	}
131		105
132	template <typename T, int32_t QUEUE_DEPTH>	106	template <typename T, int32_t QUEUE_DEPTH>
133	__aicore__ inline void KernelAvgPool3dReduceD<T, QUEUE_DEPTH>::DataCopyOutNonPad(	107	__aicore__ inline void KernelAvgPool3dReduceD<T, QUEUE_DEPTH>::DataCopyOutNonPad(
134	LocalTensor<T>& outputLocal, int64_t offset, int64_t validDataLen) {	108	LocalTensor<T>& outputLocal, int64_t offset, int64_t validDataLen) {
135	- if ((validDataLen < numPerBlock) && (offset + validDataLen * atomicAddNum >= nextCoreAddrOffset)) {	109	+ if ((validDataLen < poolMem.numPerBlock) && (offset + validDataLen * poolMem.atomicAddNum >= poolMem.nextCoreAddrOffset)) {
136	- uint64_t mask0 = (1ul << numPerBlock) - (1ul << validDataLen);	110	+ uint64_t mask0 = (1ul << poolMem.numPerBlock) - (1ul << validDataLen);
137	uint64_t mask[2] = {mask0, 0};	111	uint64_t mask[2] = {mask0, 0};
138	Duplicate<T>(outputLocal, 0, mask, 1, 1, 1);	112	Duplicate<T>(outputLocal, 0, mask, 1, 1, 1);
139	VToMTE3Sync();	113	VToMTE3Sync();
140	SetAtomicAdd<T>();	114	SetAtomicAdd<T>();
141	- DataCopy(outputGlobal[offset], outputLocal, hwTailAlign);	115	+ DataCopy(poolMem.outputGlobal[offset], outputLocal, hwTailAlign);
142	SetAtomicNone();	116	SetAtomicNone();
143	AscendC::PipeBarrier<PIPE_MTE3>();	117	AscendC::PipeBarrier<PIPE_MTE3>();
144	- } else if ((validTailLen != 0) && (offset + validDataLen == nextCoreAddrOffset)) {	118	+ } else if ((poolMem.validTailLen != 0) && (offset + validDataLen == poolMem.nextCoreAddrOffset)) {
145	- DataCopy(outputGlobal[offset], outputLocal, hwTailAlign - numPerBlock);	119	+ DataCopy(poolMem.outputGlobal[offset], outputLocal, hwTailAlign - poolMem.numPerBlock);
146	- int32_t lastLeftShift = validTailLen;	120	+ uint32_t mask = poolMem.numPerBlock * 2;
147	- uint32_t mask = numPerBlock * 2;
148	- uint64_t rsvdCnt = 0;
149	uint64_t gatherOffset = hwTailAlign - mask;	121	uint64_t gatherOffset = hwTailAlign - mask;
150	- MTE3ToVSync();	122	+ HandleTailMask(outputLocal, gatherOffset, poolMem, mask);
151	- if constexpr (std::is_same_v<T, float>) {	123	+ DataCopy(poolMem.outputGlobal[poolMem.nextCoreAddrOffset - poolMem.numPerBlock], outputLocal[gatherOffset], poolMem.numPerBlock);
152	- LocalTensor<uint32_t> bufPattern = tmpPattern.Get<uint32_t>();
153	- int32_t preLeftShift = numPerBlock + lastLeftShift;
154	-
155	- bufPattern.SetValue(0, (1u << preLeftShift) - (1u << lastLeftShift));
156	- SToVSync();
157	- GatherMask(outputLocal[gatherOffset], outputLocal[gatherOffset], bufPattern, true, mask, {1, 1, 8, 8}, rsvdCnt);
158	- } else {
159	- LocalTensor<uint16_t> bufPattern = tmpPattern.Get<uint16_t>();
160	- int32_t preLeftShift = numPerBlock - lastLeftShift;
161	-
162	- bufPattern.SetValue(0, ((1u << preLeftShift) - 1u) << lastLeftShift);
163	- bufPattern.SetValue(1, (1u << lastLeftShift) - 1u);
164	- SToVSync();
165	- GatherMask(outputLocal[gatherOffset], outputLocal[gatherOffset], bufPattern, true, mask, {1, 1, 8, 8}, rsvdCnt);
166	- }
167	- VToMTE3Sync();
168	- DataCopy(outputGlobal[nextCoreAddrOffset - numPerBlock], outputLocal[gatherOffset], numPerBlock);
169	} else {	124	} else {
170	- DataCopy(outputGlobal[offset], outputLocal, hwTailAlign);	125	+ DataCopy(poolMem.outputGlobal[offset], outputLocal, hwTailAlign);
171	}	126	}
172	}	127	}
173		128
174	template <typename T, int32_t QUEUE_DEPTH>	129	template <typename T, int32_t QUEUE_DEPTH>
175	__aicore__ inline void KernelAvgPool3dReduceD<T, QUEUE_DEPTH>::CopyOut(int64_t offset, int64_t len) {	130	__aicore__ inline void KernelAvgPool3dReduceD<T, QUEUE_DEPTH>::CopyOut(int64_t offset, int64_t len) {
176	- LocalTensor<T> outputLocal = outputQueue.template DeQue<T>();	131	+ LocalTensor<T> outputLocal = poolMem.outputQueue.template DeQue<T>();
177	#if __CCE_AICORE__ < 220	132	#if __CCE_AICORE__ < 220
178	if (len == tileHW) {	133	if (len == tileHW) {
179	- DataCopyParams copyParams{1, static_cast<uint16_t>(len / numPerBlock), 0, 0};	134	+ DataCopyParams copyParams{1, static_cast<uint16_t>(len / poolMem.numPerBlock), 0, 0};
180	- DataCopy(outputGlobal[offset], outputLocal, copyParams);	135	+ DataCopy(poolMem.outputGlobal[offset], outputLocal, copyParams);
181	} else {	136	} else {
182	DataCopyOutNonPad(outputLocal, offset, len);	137	DataCopyOutNonPad(outputLocal, offset, len);
183	}	138	}
184	#else	139	#else
185	- DataCopyExtParams copyParams{1, static_cast<uint32_t>(len * sizeof(T)), 0, 0, 0};	140	+ DataCopyExtParams copyParams{
186	- DataCopyPad(outputGlobal[offset], outputLocal, copyParams);	141	+ static_cast<uint16_t>(1), static_cast<uint32_t>(len * sizeof(T)), static_cast<uint32_t>(0),
		142	+ static_cast<uint32_t>(0), static_cast<uint32_t>(0)};
		143	+ DataCopyPad(poolMem.outputGlobal[offset], outputLocal, copyParams);
187	#endif	144	#endif
188	- outputQueue.FreeTensor(outputLocal);	145	+ poolMem.outputQueue.FreeTensor(outputLocal);
189	}	146	}
190		147
191	template <typename T, int32_t QUEUE_DEPTH>	148	template <typename T, int32_t QUEUE_DEPTH>
	@@ -195,18 +152,18 @@ __aicore__ inline void KernelAvgPool3dReduceD<T, QUEUE_DEPTH>::ReduceSumDWindow(
195	int64_t dend = index.D.end;	152	int64_t dend = index.D.end;
196		153
197	for (int64_t id = dstart; id < dend; ++id) {	154	for (int64_t id = dstart; id < dend; ++id) {
198	- int64_t dOffset = id * inputShape.strideD;	155	+ int64_t dOffset = id * poolMem.inputShape.strideD;
199		156
200	CopyIn(startOffset + dOffset, len);	157	CopyIn(startOffset + dOffset, len);
201		158
202	- LocalTensor<T> inputLocal = inputQueue.template DeQue<T>();	159	+ LocalTensor<T> inputLocal = poolMem.inputQueue.template DeQue<T>();
203	if constexpr (std::is_same_v<T, float>) {	160	if constexpr (std::is_same_v<T, float>) {
204	Add(sumBufLocal, sumBufLocal, inputLocal, len);	161	Add(sumBufLocal, sumBufLocal, inputLocal, len);
205	} else {	162	} else {
206	Cast(inputLocal.template ReinterpretCast<float>(), inputLocal[tileHW], RoundMode::CAST_NONE, len);	163	Cast(inputLocal.template ReinterpretCast<float>(), inputLocal[tileHW], RoundMode::CAST_NONE, len);
207	Add(sumBufLocal, sumBufLocal, inputLocal.template ReinterpretCast<float>(), len);	164	Add(sumBufLocal, sumBufLocal, inputLocal.template ReinterpretCast<float>(), len);
208	}	165	}
209	- inputQueue.FreeTensor(inputLocal);	166	+ poolMem.inputQueue.FreeTensor(inputLocal);
210	}	167	}
211	}	168	}
212		169
	@@ -214,12 +171,12 @@ template <typename T, int32_t QUEUE_DEPTH>
214	__aicore__ inline void KernelAvgPool3dReduceD<T, QUEUE_DEPTH>::ReduceMeanDWindow(int64_t dIdx) {	171	__aicore__ inline void KernelAvgPool3dReduceD<T, QUEUE_DEPTH>::ReduceMeanDWindow(int64_t dIdx) {
215	Index index;	172	Index index;
216		173
217	- uint64_t ncIdx = dIdx / outputShape.D;	174	+ uint64_t ncIdx = dIdx / poolMem.outputShape.D;
218	- uint64_t outputDIdx = dIdx % outputShape.D;	175	+ uint64_t outputDIdx = dIdx % poolMem.outputShape.D;
219	- index.D.Compute(outputDIdx, inputShape.D, poolParam.kernelD, poolParam.strideD, poolParam.padD,	176	+ index.D.Compute(outputDIdx, poolMem.inputShape.D, poolMem.poolParam.kernelD, poolMem.poolParam.strideD, poolMem.poolParam.padD,
220	- poolParam.countIncludePad);	177	+ poolMem.poolParam.countIncludePad);
221		178
222	- int64_t poolSize = poolParam.divisorOverride ? poolParam.divisorOverride : index.D.poolSize;	179	+ int64_t poolSize = poolMem.poolParam.divisorOverride ? poolMem.poolParam.divisorOverride : index.D.poolSize;
223	float factor = 1.0f / static_cast<float>(poolSize);	180	float factor = 1.0f / static_cast<float>(poolSize);
224		181
225	SToVSync();	182	SToVSync();
	@@ -229,28 +186,14 @@ __aicore__ inline void KernelAvgPool3dReduceD<T, QUEUE_DEPTH>::ReduceMeanDWindow
229	for (int64_t i = 0; i < hwLoop; ++i) {	186	for (int64_t i = 0; i < hwLoop; ++i) {
230	int64_t count = i < hwLoop - 1 ? tileHW : hwLength - (hwLoop - 1) * tileHW;	187	int64_t count = i < hwLoop - 1 ? tileHW : hwLength - (hwLoop - 1) * tileHW;
231		188
232	- Duplicate(sumBufLocal, 0.0f, count);	189	+ Duplicate(poolMem.sumBufLocal, 0.0f, count);
233		190
234	- int64_t startOffset = ncIdx * inputShape.strideC + hwOffset;	191	+ int64_t startOffset = ncIdx * poolMem.inputShape.strideC + hwOffset;
235		192
236	- ReduceSumDWindow(index, sumBufLocal, startOffset, count);	193	+ ReduceSumDWindow(index, poolMem.sumBufLocal, startOffset, count);
237	- Muls(sumBufLocal, sumBufLocal, factor, count);	194	+ CastAndEnqueueOutput<T, QUEUE_DEPTH>(poolMem, count, factor);
238		195
239	- LocalTensor<T> outputLocal = outputQueue.template AllocTensor<T>();	196	+ CopyOut(ncIdx * poolMem.outputShape.strideC + outputDIdx * hwLength + hwOffset, count);
240	- if constexpr (std::is_same_v<T, float>) {
241	-#if __CCE_AICORE__ < 220
242	- Adds(outputLocal, sumBufLocal, 0.0f, AlignUp(count, numPerBlock));
243	-#else
244	- DataCopy(outputLocal, sumBufLocal, AlignUp(count, numPerBlock));
245	-#endif
246	- } else if constexpr (std::is_same_v<T, half>) {
247	- Cast(outputLocal, sumBufLocal, RoundMode::CAST_NONE, count);
248	- } else {
249	- Cast(outputLocal, sumBufLocal, RoundMode::CAST_RINT, count);
250	- }
251	- outputQueue.EnQue(outputLocal);
252	-
253	- CopyOut(ncIdx * outputShape.strideC + outputDIdx * hwLength + hwOffset, count);
254		197
255	hwOffset += count;	198	hwOffset += count;
256	}	199	}
	@@ -261,46 +204,24 @@ __aicore__ inline void KernelAvgPool3dReduceD<T, QUEUE_DEPTH>::Init(
261	GM_ADDR x, GM_ADDR y, GM_ADDR workspace, const AvgPool3DTilingData* __restrict__ tiling, TPipe* pipe) {	204	GM_ADDR x, GM_ADDR y, GM_ADDR workspace, const AvgPool3DTilingData* __restrict__ tiling, TPipe* pipe) {
262	InitTiling(tiling);	205	InitTiling(tiling);
263		206
264	- inputGlobal.SetGlobalBuffer((__gm__ T*)x);	207	+ poolMem.pipe = pipe;
265	- outputGlobal.SetGlobalBuffer((__gm__ T*)y);	208	+ poolMem.inputGlobal.SetGlobalBuffer((__gm__ T*)x);
		209	+ poolMem.outputGlobal.SetGlobalBuffer((__gm__ T*)y);
266		210
267	- pipe->InitBuffer(inputQueue, QUEUE_DEPTH, tileHW * sizeof(float));	211	+ pipe->InitBuffer(poolMem.inputQueue, QUEUE_DEPTH, tileHW * sizeof(float));
268	- pipe->InitBuffer(outputQueue, QUEUE_DEPTH, tileHW * sizeof(T));	212	+ pipe->InitBuffer(poolMem.outputQueue, QUEUE_DEPTH, tileHW * sizeof(T));
269		213
270	-#if __CCE_AICORE__ < 220	214	+ InitCommonBuffers(poolMem, workspace);
271	- if (atomicAddNum) {
272	- pipe->InitBuffer(tmpPattern, numPerBlock * sizeof(T));
273		215
274	- pipe->InitBuffer(syncWorkQueue, QUEUE_DEPTH, 8 * 32 * sizeof(int32_t));	216	+ pipe->InitBuffer(poolMem.sumBuf, tileHW * sizeof(float));
275	- syncTensorsGM.SetGlobalBuffer((__gm__ int32_t )workspace, usedCoreNum 8 * 32);	217	+ poolMem.sumBufLocal = poolMem.sumBuf.template Get<float>();
276	- pipe->InitBuffer(clearTensorBuff, DEFAULT_CLEAR_UB_SIZE * sizeof(T));
277	- } else if (validTailLen != 0) {
278	- pipe->InitBuffer(tmpPattern, numPerBlock * sizeof(T));
279	- }
280	-#endif
281	-
282	- pipe->InitBuffer(sumBuf, tileHW * sizeof(float));
283	- sumBufLocal = sumBuf.Get<float>();
284	}	218	}
285		219
286	template <typename T, int32_t QUEUE_DEPTH>	220	template <typename T, int32_t QUEUE_DEPTH>
287	__aicore__ inline void KernelAvgPool3dReduceD<T, QUEUE_DEPTH>::Process() {	221	__aicore__ inline void KernelAvgPool3dReduceD<T, QUEUE_DEPTH>::Process() {
288	#if __CCE_AICORE__ < 220	222	#if __CCE_AICORE__ < 220
289	- if (atomicAddNum) {	223	+ int64_t curOutputPointIdx = ncdBlockLength + ncdOffset - 1;
290	- LocalTensor<T> clearUb = clearTensorBuff.Get<T>();	224	+ AvgPool3d::HandleAtomicAddWithTail(poolMem, curOutputPointIdx, hwTailLength);
291	- Duplicate(clearUb, (T)0, DEFAULT_CLEAR_UB_SIZE);
292	-
293	- VToMTE3Sync();
294	- int64_t curOutputPointIdx = ncdBlockLength + ncdOffset - 1;
295	- for (int i = 0; i < atomicAddNum; i++, curOutputPointIdx--) {
296	- DataCopy<T>(outputGlobal[curOutputPointIdx * hwTailLength], clearUb, numPerBlock);
297	- }
298	-
299	- DataCopy(syncTensorsGM[0], clearUb.template ReinterpretCast<int32_t>(), usedCoreNum * 8 * 32);
300	- LocalTensor<int32_t> syncLocalTensor = syncWorkQueue.template AllocTensor<int32_t>();
301	- AscendC::SyncAll(syncTensorsGM, syncLocalTensor, int32_t(usedCoreNum));
302	- syncWorkQueue.FreeTensor(syncLocalTensor);
303	- }
304	#endif	225	#endif
305		226
306	for (int64_t dIdx = ncdOffset; dIdx < ncdOffset + ncdBlockLength; ++dIdx) {	227	for (int64_t dIdx = ncdOffset; dIdx < ncdOffset + ncdBlockLength; ++dIdx) {

Mpooling/avg_pool3_d/op_kernel/avg_pool3d_ndhwc_multi_w.h+90-193

	@@ -43,113 +43,52 @@ private:
43	__aicore__ inline void ReduceSumRowRepeat(	43	__aicore__ inline void ReduceSumRowRepeat(
44	const Index& startIndex, LocalTensor<float>& sumBufLocal, LocalTensor<T>& inputLocal, int64_t windowNum);	44	const Index& startIndex, LocalTensor<float>& sumBufLocal, LocalTensor<T>& inputLocal, int64_t windowNum);
45		45
46	- TPipe* pipe;
47	- TQue<QuePosition::VECIN, QUEUE_DEPTH> inputQueue;
48	- TQue<QuePosition::VECOUT, QUEUE_DEPTH> outputQueue;
49		46
50	- TBuf<QuePosition::VECCALC> tmpPattern;
51	- TBuf<TPosition::VECCALC> sumBuf;
52	- LocalTensor<float> sumBufLocal;
53	-
54	- GlobalTensor<T> inputGlobal;
55	- GlobalTensor<T> outputGlobal;
56	-
57	- int64_t inC;
58	- int64_t alignC;
59	- int64_t outputPointNum;
60	- int64_t outputPointOffset;
61	- int64_t lastPointOffset;
62	int64_t windowWNum;	47	int64_t windowWNum;
63	- int64_t atomicAddNum;	48	+ PoolMem<T, QUEUE_DEPTH> poolMem;
64	-
65	- PoolShape inputShape;
66	- PoolShape outputShape;
67	-
68	- int64_t indexBufLen;
69	- IndexBuffer indexBuf;
70	- PoolParameter poolParam;
71	-
72	- uint32_t numPerBlock;
73	- uint32_t inputBufLen;
74	- int32_t validTailLen;
75		49
76	bool isSumWithRepeat;	50	bool isSumWithRepeat;
77	bool isSamePoolSize;	51	bool isSamePoolSize;
78	-
79	- TQue<QuePosition::VECIN, QUEUE_DEPTH> syncWorkQueue;
80	- GlobalTensor<int32_t> syncTensorsGM;
81	- TBuf<TPosition::VECCALC> clearTensorBuff;
82	- uint32_t usedCoreNum;
83	};	52	};
84		53
85	template <typename T, int32_t QUEUE_DEPTH>	54	template <typename T, int32_t QUEUE_DEPTH>
86	__aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::InitTiling(const AvgPool3DTilingData* __restrict__ tiling) {	55	__aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::InitTiling(const AvgPool3DTilingData* __restrict__ tiling) {
87	- inputShape = PoolShape(tiling->inN, tiling->inC, tiling->inD, tiling->inH, tiling->inW);	56	+ poolMem.inputShape = PoolShape(tiling->inN, tiling->inC, tiling->inD, tiling->inH, tiling->inW);
88	- outputShape = PoolShape(tiling->inN, tiling->inC, tiling->outD, tiling->outH, tiling->outW);	57	+ poolMem.outputShape = PoolShape(tiling->inN, tiling->inC, tiling->outD, tiling->outH, tiling->outW);
89		58
90	- poolParam = PoolParameter(tiling->kD, tiling->kH, tiling->kW, tiling->dD, tiling->dH, tiling->dW,	59	+ poolMem.poolParam = PoolParameter(tiling->kD, tiling->kH, tiling->kW, tiling->dD, tiling->dH, tiling->dW,
91	tiling->pD, tiling->pH, tiling->pW, tiling->divisorOverride, tiling->countIncludePad);	60	tiling->pD, tiling->pH, tiling->pW, tiling->divisorOverride, tiling->countIncludePad);
92		61
93	- indexBuf.SetComputeParameter(outputShape, inputShape, poolParam);	62	+ poolMem.indexBuf.SetComputeParameter(poolMem.outputShape, poolMem.inputShape, poolMem.poolParam);
94		63
95	- numPerBlock = GetDataBlockSizeInBytes() / sizeof(T);	64	+ poolMem.numPerBlock = GetDataBlockSizeInBytes() / sizeof(T);
96	- inC = tiling->inC;	65	+ poolMem.inC = tiling->inC;
97	- alignC = AlignUp(inC, numPerBlock);	66	+ poolMem.alignC = AlignUp(poolMem.inC, poolMem.numPerBlock);
98	windowWNum = tiling->windowWNum;	67	windowWNum = tiling->windowWNum;
99		68
100	- outputPointNum = GetBlockIdx() < tiling->formerNum ? tiling->formerLength : tiling->tailLength;	69	+ poolMem.outputPointNum = GetBlockIdx() < tiling->formerNum ? tiling->formerLength : tiling->tailLength;
101	- outputPointOffset = GetBlockIdx() < tiling->formerNum	70	+ poolMem.outputPointOffset = GetBlockIdx() < tiling->formerNum
102	? tiling->formerLength * GetBlockIdx()	71	? tiling->formerLength * GetBlockIdx()
103	: tiling->formerNum * tiling->formerLength + tiling->tailLength * (GetBlockIdx() - tiling->formerNum);	72	: tiling->formerNum * tiling->formerLength + tiling->tailLength * (GetBlockIdx() - tiling->formerNum);
104	- lastPointOffset = outputPointNum + outputPointOffset - 1;	73	+ poolMem.lastPointOffset = poolMem.outputPointNum + poolMem.outputPointOffset - 1;
105	- atomicAddNum = outputPointNum < tiling->atomicAddNum ? outputPointNum : tiling->atomicAddNum;	74	+ poolMem.atomicAddNum = poolMem.outputPointNum < tiling->atomicAddNum ? poolMem.outputPointNum : tiling->atomicAddNum;
106	- validTailLen = inC % numPerBlock;	75	+ poolMem.validTailLen = poolMem.inC % poolMem.numPerBlock;
107	- usedCoreNum = tiling->usedCoreNum;	76	+ poolMem.usedCoreNum = tiling->usedCoreNum;
108		77
109	- indexBufLen = tiling->indexBufLen;	78	+ poolMem.indexBufLen = tiling->indexBufLen;
110		79
111	uint32_t floatNumPerBlock = GetDataBlockSizeInBytes() / sizeof(float);	80	uint32_t floatNumPerBlock = GetDataBlockSizeInBytes() / sizeof(float);
112	- uint32_t src1RepStride = alignC / floatNumPerBlock * poolParam.strideW;	81	+ uint32_t src1RepStride = poolMem.alignC / floatNumPerBlock * poolMem.poolParam.strideW;
113		82
114	- isSumWithRepeat = (poolParam.padW == 0 && !tiling->ceilMode) && src1RepStride <= UINT8_MAX;	83	+ isSumWithRepeat = (poolMem.poolParam.padW == 0 && !tiling->ceilMode) && src1RepStride <= UINT8_MAX;
115	isSamePoolSize =	84	isSamePoolSize =
116	- poolParam.divisorOverride \|\| ((poolParam.countIncludePad \|\| poolParam.padW == 0) && !tiling->ceilMode);	85	+ poolMem.poolParam.divisorOverride \|\| ((poolMem.poolParam.countIncludePad \|\| poolMem.poolParam.padW == 0) && !tiling->ceilMode);
117	}	86	}
118		87
119	template <typename T, int32_t QUEUE_DEPTH>	88	template <typename T, int32_t QUEUE_DEPTH>
120	__aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::CopyIn(	89	__aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::CopyIn(
121	int64_t offset, uint16_t blockCount, uint32_t blockLen, uint8_t rightPadding) {	90	int64_t offset, uint16_t blockCount, uint32_t blockLen, uint8_t rightPadding) {
122	- LocalTensor<T> inputLocal = inputQueue.template AllocTensor<T>();	91	+ AvgPool3d::CopyInTemplate(poolMem, offset, blockCount, blockLen, rightPadding);
123	-#if __CCE_AICORE__ < 220
124	- if constexpr (std::is_same_v<T, float>) {
125	- if (blockLen == alignC) {
126	- DataCopyParams copyParams{blockCount, static_cast<uint16_t>(blockLen / numPerBlock), 0, 0};
127	- DataCopy(inputLocal, inputGlobal[offset], copyParams);
128	- } else {
129	- for (int i = 0; i < blockCount; i++) {
130	- DataCopy(inputLocal[i * alignC], inputGlobal[offset + i * blockLen], alignC);
131	- }
132	- }
133	- } else {
134	- if (blockLen == alignC) {
135	- DataCopyParams copyParams{blockCount, static_cast<uint16_t>(blockLen / numPerBlock), 0, 0};
136	- DataCopy(inputLocal[inputBufLen], inputGlobal[offset], copyParams);
137	- } else {
138	- for (int i = 0; i < blockCount; i++) {
139	- DataCopy(inputLocal[inputBufLen + i * alignC], inputGlobal[offset + i * blockLen], alignC);
140	- }
141	- }
142	- }
143	-#else
144	- DataCopyExtParams copyParams{blockCount, static_cast<uint32_t>(blockLen * sizeof(T)), 0, 0, 0};
145	- DataCopyPadExtParams<T> padParams{true, 0, rightPadding, 0};
146	- if constexpr (std::is_same_v<T, float>) {
147	- DataCopyPad(inputLocal, inputGlobal[offset], copyParams, padParams);
148	- } else {
149	- DataCopyPad(inputLocal[inputBufLen], inputGlobal[offset], copyParams, padParams);
150	- }
151	-#endif
152	- inputQueue.EnQue(inputLocal);
153	}	92	}
154		93
155	template <typename T, int32_t QUEUE_DEPTH>	94	template <typename T, int32_t QUEUE_DEPTH>
	@@ -158,46 +97,27 @@ __aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::DataCopyOutNonPad(
158	int64_t curPointIdx = outputPointIdx;	97	int64_t curPointIdx = outputPointIdx;
159	for (int i = 0; i < blockCount; i++, curPointIdx++) {	98	for (int i = 0; i < blockCount; i++, curPointIdx++) {
160	PipeBarrier<PIPE_MTE3>();	99	PipeBarrier<PIPE_MTE3>();
161	- if ((validDataLen < numPerBlock) && (curPointIdx >= lastPointOffset - atomicAddNum)) {	100	+ if ((validDataLen < poolMem.numPerBlock) && (curPointIdx >= poolMem.lastPointOffset - poolMem.atomicAddNum)) {
162	- uint64_t mask0 = (1ul << numPerBlock) - (1ul << validDataLen);	101	+ uint64_t mask0 = (1ul << poolMem.numPerBlock) - (1ul << validDataLen);
163	uint64_t mask[2] = {mask0, 0};	102	uint64_t mask[2] = {mask0, 0};
164	- Duplicate<T>(outputLocal[i * alignC], 0, mask, 1, 1, 1);	103	+ Duplicate<T>(outputLocal[i * poolMem.alignC], 0, mask, 1, 1, 1);
165	VToMTE3Sync();	104	VToMTE3Sync();
166	- if (curPointIdx > lastPointOffset - atomicAddNum) {	105	+ if (curPointIdx > poolMem.lastPointOffset - poolMem.atomicAddNum) {
167	SetAtomicAdd<T>();	106	SetAtomicAdd<T>();
168	- DataCopy(outputGlobal[curPointIdx * validDataLen], outputLocal[i * alignC], alignC);	107	+ DataCopy(poolMem.outputGlobal[curPointIdx * validDataLen], outputLocal[i * poolMem.alignC], poolMem.alignC);
169	SetAtomicNone();	108	SetAtomicNone();
170	AscendC::PipeBarrier<PIPE_MTE3>();	109	AscendC::PipeBarrier<PIPE_MTE3>();
171	} else {	110	} else {
172	- DataCopy(outputGlobal[curPointIdx * validDataLen], outputLocal[i * alignC], alignC);	111	+ DataCopy(poolMem.outputGlobal[curPointIdx * validDataLen], outputLocal[i * poolMem.alignC], poolMem.alignC);
173	}	112	}
174	- } else if (curPointIdx == lastPointOffset) {	113	+ } else if (curPointIdx == poolMem.lastPointOffset) {
175	- DataCopy(outputGlobal[curPointIdx * validDataLen], outputLocal[i * alignC], alignC - numPerBlock);	114	+ DataCopy(poolMem.outputGlobal[curPointIdx * validDataLen], outputLocal[i * poolMem.alignC], poolMem.alignC - poolMem.numPerBlock);
176	- int32_t lastLeftShift = validTailLen;	115	+ uint32_t mask = poolMem.numPerBlock * 2;
177	- uint32_t mask = numPerBlock * 2;	116	+ uint64_t gatherOffset = blockCount * poolMem.alignC - mask;
178	- uint64_t rsvdCnt = 0;	117	+ HandleTailMask(outputLocal, gatherOffset, poolMem, mask);
179	- uint64_t gatherOffset = blockCount * alignC - mask;	118	+ DataCopy(poolMem.outputGlobal[(curPointIdx + 1) * validDataLen - poolMem.numPerBlock], outputLocal[gatherOffset], poolMem.numPerBlock);
180	- MTE3ToVSync();
181	- if constexpr (std::is_same_v<T, float>) {
182	- LocalTensor<uint32_t> bufPattern = tmpPattern.Get<uint32_t>();
183	- int32_t preLeftShift = numPerBlock + lastLeftShift;
184	-
185	- bufPattern.SetValue(0, (1u << preLeftShift) - (1u << lastLeftShift));
186	- SToVSync();
187	- GatherMask(outputLocal[gatherOffset], outputLocal[gatherOffset], bufPattern, true, mask, {1, 1, 8, 8}, rsvdCnt);
188	- } else {
189	- LocalTensor<uint16_t> bufPattern = tmpPattern.Get<uint16_t>();
190	- int32_t preLeftShift = numPerBlock - lastLeftShift;
191	-
192	- bufPattern.SetValue(0, ((1u << preLeftShift) - 1u) << lastLeftShift);
193	- bufPattern.SetValue(1, (1u << lastLeftShift) - 1u);
194	- SToVSync();
195	- GatherMask(outputLocal[gatherOffset], outputLocal[gatherOffset], bufPattern, true, mask, {1, 1, 8, 8}, rsvdCnt);
196	- }
197	- VToMTE3Sync();
198	- DataCopy(outputGlobal[(curPointIdx + 1) * validDataLen - numPerBlock], outputLocal[gatherOffset], numPerBlock);
199	} else {	119	} else {
200	- DataCopy(outputGlobal[curPointIdx * validDataLen], outputLocal[i * alignC], alignC);	120	+ DataCopy(poolMem.outputGlobal[curPointIdx * validDataLen], outputLocal[i * poolMem.alignC], poolMem.alignC);
201	}	121	}
202	}	122	}
203	}	123	}
	@@ -205,37 +125,37 @@ __aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::DataCopyOutNonPad(
205	template <typename T, int32_t QUEUE_DEPTH>	125	template <typename T, int32_t QUEUE_DEPTH>
206	__aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::CopyOut(	126	__aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::CopyOut(
207	int64_t offset, uint16_t blockCount, uint32_t blockLen) {	127	int64_t offset, uint16_t blockCount, uint32_t blockLen) {
208	- LocalTensor<T> outputLocal = outputQueue.template DeQue<T>();	128	+ LocalTensor<T> outputLocal = poolMem.outputQueue.template DeQue<T>();
209	#if __CCE_AICORE__ < 220	129	#if __CCE_AICORE__ < 220
210	- if (blockLen == alignC) {	130	+ if (blockLen == poolMem.alignC) {
211	- DataCopyParams copyParams{blockCount, static_cast<uint16_t>(blockLen / numPerBlock), 0, 0};	131	+ DataCopyParams copyParams{blockCount, static_cast<uint16_t>(blockLen / poolMem.numPerBlock), 0, 0};
212	- DataCopy(outputGlobal[offset * blockLen], outputLocal, copyParams);	132	+ DataCopy(poolMem.outputGlobal[offset * blockLen], outputLocal, copyParams);
213	} else {	133	} else {
214	DataCopyOutNonPad(outputLocal, offset, blockCount, blockLen);	134	DataCopyOutNonPad(outputLocal, offset, blockCount, blockLen);
215	}	135	}
216	#else	136	#else
217	DataCopyExtParams copyParams{blockCount, static_cast<uint32_t>(blockLen * sizeof(T)), 0, 0, 0};	137	DataCopyExtParams copyParams{blockCount, static_cast<uint32_t>(blockLen * sizeof(T)), 0, 0, 0};
218	- DataCopyPad(outputGlobal[offset * blockLen], outputLocal, copyParams);	138	+ DataCopyPad(poolMem.outputGlobal[offset * blockLen], outputLocal, copyParams);
219	#endif	139	#endif
220	- outputQueue.FreeTensor(outputLocal);	140	+ poolMem.outputQueue.FreeTensor(outputLocal);
221	}	141	}
222		142
223	template <typename T, int32_t QUEUE_DEPTH>	143	template <typename T, int32_t QUEUE_DEPTH>
224	__aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::ReduceSumRow(	144	__aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::ReduceSumRow(
225	const Index& startIndex, LocalTensor<float>& sumBufLocal, LocalTensor<T>& inputLocal,	145	const Index& startIndex, LocalTensor<float>& sumBufLocal, LocalTensor<T>& inputLocal,
226	int64_t outputPointIdx, int64_t windowNum) {	146	int64_t outputPointIdx, int64_t windowNum) {
227	- for (int64_t in = outputPointIdx, offset = 0; in < outputPointIdx + windowNum; ++in, offset += alignC) {	147	+ for (int64_t in = outputPointIdx, offset = 0; in < outputPointIdx + windowNum; ++in, offset += poolMem.alignC) {
228	Index index;	148	Index index;
229	- indexBuf.GetWIndex(in, index);	149	+ poolMem.indexBuf.GetWIndex(in, index);
230		150
231	SToVSync();	151	SToVSync();
232		152
233	for (int64_t iw = index.W.start - startIndex.W.start; iw < index.W.end - startIndex.W.start; ++iw) {	153	for (int64_t iw = index.W.start - startIndex.W.start; iw < index.W.end - startIndex.W.start; ++iw) {
234	if constexpr (std::is_same_v<T, float>) {	154	if constexpr (std::is_same_v<T, float>) {
235	- Add(sumBufLocal[offset], sumBufLocal[offset], inputLocal[iw * alignC], alignC);	155	+ Add(sumBufLocal[offset], sumBufLocal[offset], inputLocal[iw * poolMem.alignC], poolMem.alignC);
236	} else {	156	} else {
237	Add(sumBufLocal[offset], sumBufLocal[offset],	157	Add(sumBufLocal[offset], sumBufLocal[offset],
238	- inputLocal.template ReinterpretCast<float>()[iw * alignC], alignC);	158	+ inputLocal.template ReinterpretCast<float>()[iw * poolMem.alignC], poolMem.alignC);
239	}	159	}
240	}	160	}
241	}	161	}
	@@ -247,14 +167,14 @@ __aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::ReduceSumRowRepeat
247	int64_t poolSize = startIndex.W.end - startIndex.W.start;	167	int64_t poolSize = startIndex.W.end - startIndex.W.start;
248		168
249	uint32_t floatNumPerBlock = GetDataBlockSizeInBytes() / sizeof(float);	169	uint32_t floatNumPerBlock = GetDataBlockSizeInBytes() / sizeof(float);
250	- int64_t loop = (alignC + floatNumPerBlock * 8 - 1) / (floatNumPerBlock * 8);	170	+ int64_t loop = (poolMem.alignC + floatNumPerBlock * 8 - 1) / (floatNumPerBlock * 8);
251		171
252	- uint8_t repStride = alignC / floatNumPerBlock;	172	+ uint8_t repStride = poolMem.alignC / floatNumPerBlock;
253	- uint8_t src1RepStride = alignC / floatNumPerBlock * poolParam.strideW;	173	+ uint8_t src1RepStride = poolMem.alignC / floatNumPerBlock * poolMem.poolParam.strideW;
254		174
255	for (int64_t i = 0; i < poolSize; ++i) {	175	for (int64_t i = 0; i < poolSize; ++i) {
256	for (int64_t j = 0; j < loop; ++j) {	176	for (int64_t j = 0; j < loop; ++j) {
257	- int64_t mask = j < loop - 1 ? floatNumPerBlock * 8 : alignC - (loop - 1) * floatNumPerBlock * 8;	177	+ int64_t mask = j < loop - 1 ? floatNumPerBlock * 8 : poolMem.alignC - (loop - 1) * floatNumPerBlock * 8;
258		178
259	BinaryRepeatParams repeatParams;	179	BinaryRepeatParams repeatParams;
260	repeatParams.dstBlkStride = 1;	180	repeatParams.dstBlkStride = 1;
	@@ -265,7 +185,7 @@ __aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::ReduceSumRowRepeat
265	repeatParams.src1RepStride = src1RepStride;	185	repeatParams.src1RepStride = src1RepStride;
266		186
267	int64_t offset = j * floatNumPerBlock * 8;	187	int64_t offset = j * floatNumPerBlock * 8;
268	- int64_t src1Offset = i * alignC + offset;	188	+ int64_t src1Offset = i * poolMem.alignC + offset;
269		189
270	if constexpr (std::is_same_v<T, float>) {	190	if constexpr (std::is_same_v<T, float>) {
271	Add(sumBufLocal[offset], sumBufLocal[offset], inputLocal[src1Offset], mask, windowNum, repeatParams);	191	Add(sumBufLocal[offset], sumBufLocal[offset], inputLocal[src1Offset], mask, windowNum, repeatParams);
	@@ -285,22 +205,22 @@ __aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::ReduceSumMultiWind
285	int64_t dend = startIndex.D.end;	205	int64_t dend = startIndex.D.end;
286	int64_t hstart = startIndex.H.start;	206	int64_t hstart = startIndex.H.start;
287	int64_t hend = startIndex.H.end;	207	int64_t hend = startIndex.H.end;
288	- int64_t wStartOffset = startIndex.W.start * inC;	208	+ int64_t wStartOffset = startIndex.W.start * poolMem.inC;
289		209
290	uint16_t blockCount = static_cast<uint16_t>(endIndex.W.end - startIndex.W.start);	210	uint16_t blockCount = static_cast<uint16_t>(endIndex.W.end - startIndex.W.start);
291	- uint8_t rightPadding = static_cast<uint8_t>(alignC - inC);	211	+ uint8_t rightPadding = static_cast<uint8_t>(poolMem.alignC - poolMem.inC);
292		212
293	for (int64_t id = dstart; id < dend; ++id) {	213	for (int64_t id = dstart; id < dend; ++id) {
294	- int64_t dOffset = id * inputShape.strideD * inC;	214	+ int64_t dOffset = id * poolMem.inputShape.strideD * poolMem.inC;
295	for (int64_t ih = hstart; ih < hend; ++ih) {	215	for (int64_t ih = hstart; ih < hend; ++ih) {
296	- int64_t hOffset = ih * inputShape.strideH * inC;	216	+ int64_t hOffset = ih * poolMem.inputShape.strideH * poolMem.inC;
297		217
298	- CopyIn(nOffset * inputShape.strideN + dOffset + hOffset + wStartOffset, blockCount, inC, rightPadding);	218	+ CopyIn(nOffset * poolMem.inputShape.strideN + dOffset + hOffset + wStartOffset, blockCount, poolMem.inC, rightPadding);
299	- LocalTensor<T> inputLocal = inputQueue.template DeQue<T>();	219	+ LocalTensor<T> inputLocal = poolMem.inputQueue.template DeQue<T>();
300		220
301	if constexpr (!std::is_same_v<T, float>) {	221	if constexpr (!std::is_same_v<T, float>) {
302	- Cast(inputLocal.template ReinterpretCast<float>(), inputLocal[inputBufLen], RoundMode::CAST_NONE,	222	+ Cast(inputLocal.template ReinterpretCast<float>(), inputLocal[poolMem.inputBufLen], RoundMode::CAST_NONE,
303	- inputBufLen);	223	+ poolMem.inputBufLen);
304	}	224	}
305		225
306	if (isSumWithRepeat) [[likely]] {	226	if (isSumWithRepeat) [[likely]] {
	@@ -309,7 +229,7 @@ __aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::ReduceSumMultiWind
309	ReduceSumRow(startIndex, sumBufLocal, inputLocal, outputPointIdx, windowNum);	229	ReduceSumRow(startIndex, sumBufLocal, inputLocal, outputPointIdx, windowNum);
310	}	230	}
311		231
312	- inputQueue.FreeTensor(inputLocal);	232	+ poolMem.inputQueue.FreeTensor(inputLocal);
313	}	233	}
314	}	234	}
315	}	235	}
	@@ -318,54 +238,54 @@ template <typename T, int32_t QUEUE_DEPTH>
318	__aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::ReduceMeanMultiWindow(	238	__aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::ReduceMeanMultiWindow(
319	int64_t outputPointIdx, int64_t windowNum) {	239	int64_t outputPointIdx, int64_t windowNum) {
320	Index startIndex;	240	Index startIndex;
321	- indexBuf.GetIndex(outputPointIdx, startIndex);	241	+ poolMem.indexBuf.GetIndex(outputPointIdx, startIndex);
322	Index endIndex;	242	Index endIndex;
323	- indexBuf.GetIndex(outputPointIdx + windowNum - 1, endIndex);	243	+ poolMem.indexBuf.GetIndex(outputPointIdx + windowNum - 1, endIndex);
324		244
325	- int64_t len = windowNum * alignC;	245	+ int64_t len = windowNum * poolMem.alignC;
326		246
327	SToVSync();	247	SToVSync();
328		248
329	- Duplicate(sumBufLocal, 0.0f, len);	249	+ Duplicate(poolMem.sumBufLocal, 0.0f, len);
330		250
331	- ReduceSumMultiWindow(startIndex, endIndex, sumBufLocal, outputPointIdx,	251	+ ReduceSumMultiWindow(startIndex, endIndex, poolMem.sumBufLocal, outputPointIdx,
332	- outputPointIdx / outputShape.strideC, windowNum);	252	+ outputPointIdx / poolMem.outputShape.strideC, windowNum);
333		253
334	if (isSamePoolSize) [[likely]] {	254	if (isSamePoolSize) [[likely]] {
335	- int64_t poolSize = poolParam.divisorOverride	255	+ int64_t poolSize = poolMem.poolParam.divisorOverride
336	- ? poolParam.divisorOverride	256	+ ? poolMem.poolParam.divisorOverride
337	: startIndex.D.poolSize * startIndex.H.poolSize * startIndex.W.poolSize;	257	: startIndex.D.poolSize * startIndex.H.poolSize * startIndex.W.poolSize;
338	float factor = 1.0f / static_cast<float>(poolSize);	258	float factor = 1.0f / static_cast<float>(poolSize);
339		259
340	- Muls(sumBufLocal, sumBufLocal, factor, windowWNum * alignC);	260	+ Muls(poolMem.sumBufLocal, poolMem.sumBufLocal, factor, windowWNum * poolMem.alignC);
341	} else {	261	} else {
342	- for (int64_t i = outputPointIdx, offset = 0; i < outputPointIdx + windowNum; ++i, offset += alignC) {	262	+ for (int64_t i = outputPointIdx, offset = 0; i < outputPointIdx + windowNum; ++i, offset += poolMem.alignC) {
343	Index index;	263	Index index;
344	- indexBuf.GetWIndex(i, index);	264	+ poolMem.indexBuf.GetWIndex(i, index);
345	int64_t poolSize = startIndex.D.poolSize * startIndex.H.poolSize * index.W.poolSize;	265	int64_t poolSize = startIndex.D.poolSize * startIndex.H.poolSize * index.W.poolSize;
346	float factor = 1.0f / static_cast<float>(poolSize);	266	float factor = 1.0f / static_cast<float>(poolSize);
347		267
348	SToVSync();	268	SToVSync();
349		269
350	- Muls(sumBufLocal[offset], sumBufLocal[offset], factor, alignC);	270	+ Muls(poolMem.sumBufLocal[offset], poolMem.sumBufLocal[offset], factor, poolMem.alignC);
351	}	271	}
352	}	272	}
353		273
354	- LocalTensor<T> outputLocal = outputQueue.template AllocTensor<T>();	274	+ LocalTensor<T> outputLocal = poolMem.outputQueue.template AllocTensor<T>();
355	if constexpr (std::is_same_v<T, float>) {	275	if constexpr (std::is_same_v<T, float>) {
356	#if __CCE_AICORE__ < 220	276	#if __CCE_AICORE__ < 220
357	- Adds(outputLocal, sumBufLocal, 0.0f, len);	277	+ Adds(outputLocal, poolMem.sumBufLocal, 0.0f, len);
358	#else	278	#else
359	- DataCopy(outputLocal, sumBufLocal, len);	279	+ DataCopy(outputLocal, poolMem.sumBufLocal, len);
360	#endif	280	#endif
361	} else if constexpr (std::is_same_v<T, half>) {	281	} else if constexpr (std::is_same_v<T, half>) {
362	- Cast(outputLocal, sumBufLocal, RoundMode::CAST_NONE, len);	282	+ Cast(outputLocal, poolMem.sumBufLocal, RoundMode::CAST_NONE, len);
363	} else {	283	} else {
364	- Cast(outputLocal, sumBufLocal, RoundMode::CAST_RINT, len);	284	+ Cast(outputLocal, poolMem.sumBufLocal, RoundMode::CAST_RINT, len);
365	}	285	}
366	- outputQueue.EnQue(outputLocal);	286	+ poolMem.outputQueue.EnQue(outputLocal);
367		287
368	- CopyOut(outputPointIdx, static_cast<uint16_t>(windowNum), static_cast<uint32_t>(inC));	288	+ CopyOut(outputPointIdx, static_cast<uint16_t>(windowNum), static_cast<uint32_t>(poolMem.inC));
369	}	289	}
370		290
371	template <typename T, int32_t QUEUE_DEPTH>	291	template <typename T, int32_t QUEUE_DEPTH>
	@@ -373,56 +293,33 @@ __aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::Init(
373	GM_ADDR x, GM_ADDR y, GM_ADDR workspace, const AvgPool3DTilingData* __restrict__ tiling, TPipe* pipe) {	293	GM_ADDR x, GM_ADDR y, GM_ADDR workspace, const AvgPool3DTilingData* __restrict__ tiling, TPipe* pipe) {
374	InitTiling(tiling);	294	InitTiling(tiling);
375		295
376	- inputGlobal.SetGlobalBuffer((__gm__ T*)x);	296	+ poolMem.pipe = pipe;
377	- outputGlobal.SetGlobalBuffer((__gm__ T*)y);	297	+ poolMem.inputGlobal.SetGlobalBuffer((__gm__ T*)x);
		298	+ poolMem.outputGlobal.SetGlobalBuffer((__gm__ T*)y);
378		299
379	- inputBufLen = (windowWNum * poolParam.strideW + poolParam.kernelW) * alignC;	300	+ poolMem.inputBufLen = (windowWNum * poolMem.poolParam.strideW + poolMem.poolParam.kernelW) * poolMem.alignC;
380		301
381	- pipe->InitBuffer(inputQueue, QUEUE_DEPTH, inputBufLen * sizeof(float));	302	+ pipe->InitBuffer(poolMem.inputQueue, QUEUE_DEPTH, poolMem.inputBufLen * sizeof(float));
382	- pipe->InitBuffer(outputQueue, QUEUE_DEPTH, windowWNum * alignC * sizeof(T));	303	+ pipe->InitBuffer(poolMem.outputQueue, QUEUE_DEPTH, windowWNum * poolMem.alignC * sizeof(T));
383		304
384	-#if __CCE_AICORE__ < 220	305	+ InitCommonBuffers(poolMem, workspace);
385	- if (atomicAddNum) {	306	+ pipe->InitBuffer(poolMem.sumBuf, windowWNum * poolMem.alignC * sizeof(float));
386	- pipe->InitBuffer(tmpPattern, numPerBlock * sizeof(T));	307	+ poolMem.sumBufLocal = poolMem.sumBuf.template Get<float>();
387		308
388	- pipe->InitBuffer(syncWorkQueue, QUEUE_DEPTH, 8 * 32 * sizeof(int32_t));	309	+ poolMem.indexBuf.Init(pipe, poolMem.indexBufLen);
389	- syncTensorsGM.SetGlobalBuffer((__gm__ int32_t )workspace, usedCoreNum 8 * 32);
390	- pipe->InitBuffer(clearTensorBuff, DEFAULT_CLEAR_UB_SIZE * sizeof(T));
391	- } else if (validTailLen != 0) {
392	- pipe->InitBuffer(tmpPattern, numPerBlock * sizeof(T));
393	- }
394	-#endif
395	- pipe->InitBuffer(sumBuf, windowWNum * alignC * sizeof(float));
396	- sumBufLocal = sumBuf.Get<float>();
397	-
398	- indexBuf.Init(pipe, indexBufLen);
399	}	310	}
400		311
401	template <typename T, int32_t QUEUE_DEPTH>	312	template <typename T, int32_t QUEUE_DEPTH>
402	__aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::Process() {	313	__aicore__ inline void KernelAvgPool3dMultiW<T, QUEUE_DEPTH>::Process() {
403	#if __CCE_AICORE__ < 220	314	#if __CCE_AICORE__ < 220
404	- if (atomicAddNum) {	315	+ AvgPool3d::HandleAtomicAdd(poolMem);
405	- LocalTensor<T> clearUb = clearTensorBuff.Get<T>();
406	- Duplicate(clearUb, (T)0, DEFAULT_CLEAR_UB_SIZE);
407	-
408	- VToMTE3Sync();
409	- int64_t curOutputPointIdx = lastPointOffset;
410	- for (int i = 0; i < atomicAddNum; i++, curOutputPointIdx--) {
411	- DataCopy<T>(outputGlobal[curOutputPointIdx * inC], clearUb, numPerBlock);
412	- }
413	-
414	- DataCopy(syncTensorsGM[0], clearUb.template ReinterpretCast<int32_t>(), usedCoreNum * 8 * 32);
415	- LocalTensor<int32_t> syncLocalTensor = syncWorkQueue.template AllocTensor<int32_t>();
416	- AscendC::SyncAll(syncTensorsGM, syncLocalTensor, int32_t(usedCoreNum));
417	- syncWorkQueue.FreeTensor(syncLocalTensor);
418	- }
419	#endif	316	#endif
420	int64_t curWindowWNum = windowWNum;	317	int64_t curWindowWNum = windowWNum;
421	- for (int64_t outputPointIdx = outputPointOffset, count = 0;	318	+ for (int64_t outputPointIdx = poolMem.outputPointOffset, count = 0;
422	- outputPointIdx < outputPointOffset + outputPointNum; outputPointIdx += curWindowWNum, count += curWindowWNum) {	319	+ outputPointIdx < poolMem.outputPointOffset + poolMem.outputPointNum; outputPointIdx += curWindowWNum, count += curWindowWNum) {
423	- curWindowWNum = (count + windowWNum) < outputPointNum ? windowWNum : outputPointNum - count;	320	+ curWindowWNum = (count + windowWNum) < poolMem.outputPointNum ? windowWNum : poolMem.outputPointNum - count;
424		321
425	- int64_t newRowWindowWNum = (outputPointIdx + curWindowWNum) % outputShape.W;	322	+ int64_t newRowWindowWNum = (outputPointIdx + curWindowWNum) % poolMem.outputShape.W;
426	curWindowWNum = newRowWindowWNum != 0 && newRowWindowWNum < curWindowWNum	323	curWindowWNum = newRowWindowWNum != 0 && newRowWindowWNum < curWindowWNum
427	? curWindowWNum - newRowWindowWNum : curWindowWNum;	324	? curWindowWNum - newRowWindowWNum : curWindowWNum;
428		325

Mpooling/avg_pool3_d/op_kernel/avg_pool3d_ndhwc_split_c.h+72-137

	@@ -36,130 +36,101 @@ private:
36	__aicore__ inline void ReduceSumWindow(	36	__aicore__ inline void ReduceSumWindow(
37	const Index& index, LocalTensor<float>& sumBufLocal, int64_t nOffset, int64_t cOffset, int64_t len);	37	const Index& index, LocalTensor<float>& sumBufLocal, int64_t nOffset, int64_t cOffset, int64_t len);
38		38
39	- TPipe* pipe;
40	- TQue<QuePosition::VECIN, QUEUE_DEPTH> inputQueue;
41	- TQue<QuePosition::VECOUT, QUEUE_DEPTH> outputQueue;
42	-
43	- TBuf<QuePosition::VECCALC> tmpPattern;
44	- TBuf<TPosition::VECCALC> sumBuf;
45	- LocalTensor<float> sumBufLocal;
46	-
47	- GlobalTensor<T> inputGlobal;
48	- GlobalTensor<T> outputGlobal;
49	-
50	- int64_t inC;
51	int64_t tileC;	39	int64_t tileC;
52	- int64_t outputPointNum;
53	- int64_t outputPointOffset;
54	- int64_t nextCoreAddrOffset;
55	- int64_t atomicAddNum;
56	int64_t cTailLength;	40	int64_t cTailLength;
57	int64_t cTailAlign;	41	int64_t cTailAlign;
58		42
59	- PoolShape inputShape;	43	+ PoolMem<T, QUEUE_DEPTH> poolMem;
60	- PoolShape outputShape;
61	-
62	- int64_t indexBufLen;
63	- IndexBuffer indexBuf;
64	- PoolParameter poolParam;
65	-
66	- uint32_t numPerBlock;
67	- int32_t validTailLen;
68	-
69	- TQue<QuePosition::VECIN, QUEUE_DEPTH> syncWorkQueue;
70	- GlobalTensor<int32_t> syncTensorsGM;
71	- TBuf<TPosition::VECCALC> clearTensorBuff;
72	- uint32_t usedCoreNum;
73	};	44	};
74		45
75	template <typename T, int32_t QUEUE_DEPTH>	46	template <typename T, int32_t QUEUE_DEPTH>
76	__aicore__ inline void KernelAvgPool3dSplitC<T, QUEUE_DEPTH>::InitTiling(const AvgPool3DTilingData* __restrict__ tiling) {	47	__aicore__ inline void KernelAvgPool3dSplitC<T, QUEUE_DEPTH>::InitTiling(const AvgPool3DTilingData* __restrict__ tiling) {
77	- inputShape = PoolShape(tiling->inN, tiling->inC, tiling->inD, tiling->inH, tiling->inW);	48	+ poolMem.inputShape = PoolShape(tiling->inN, tiling->inC, tiling->inD, tiling->inH, tiling->inW);
78	- outputShape = PoolShape(tiling->inN, tiling->inC, tiling->outD, tiling->outH, tiling->outW);	49	+ poolMem.outputShape = PoolShape(tiling->inN, tiling->inC, tiling->outD, tiling->outH, tiling->outW);
79		50
80	- poolParam = PoolParameter(tiling->kD, tiling->kH, tiling->kW, tiling->dD, tiling->dH, tiling->dW,	51	+ poolMem.poolParam = PoolParameter(tiling->kD, tiling->kH, tiling->kW, tiling->dD, tiling->dH, tiling->dW,
81	tiling->pD, tiling->pH, tiling->pW, tiling->divisorOverride, tiling->countIncludePad);	52	tiling->pD, tiling->pH, tiling->pW, tiling->divisorOverride, tiling->countIncludePad);
82		53
83	- indexBuf.SetComputeParameter(outputShape, inputShape, poolParam);	54	+ poolMem.indexBuf.SetComputeParameter(poolMem.outputShape, poolMem.inputShape, poolMem.poolParam);
84		55
85	- numPerBlock = GetDataBlockSizeInBytes() / sizeof(T);	56	+ poolMem.numPerBlock = GetDataBlockSizeInBytes() / sizeof(T);
86	- inC = tiling->inC;	57	+ poolMem.inC = tiling->inC;
87	tileC = tiling->tileC;	58	tileC = tiling->tileC;
88	- indexBufLen = tiling->indexBufLen;	59	+ poolMem.indexBufLen = tiling->indexBufLen;
89		60
90	- outputPointNum = GetBlockIdx() < tiling->formerNum ? tiling->formerLength : tiling->tailLength;	61	+ poolMem.outputPointNum = GetBlockIdx() < tiling->formerNum ? tiling->formerLength : tiling->tailLength;
91	- outputPointOffset = GetBlockIdx() < tiling->formerNum	62	+ poolMem.outputPointOffset = GetBlockIdx() < tiling->formerNum
92	? tiling->formerLength * GetBlockIdx()	63	? tiling->formerLength * GetBlockIdx()
93	: tiling->formerNum * tiling->formerLength + tiling->tailLength * (GetBlockIdx() - tiling->formerNum);	64	: tiling->formerNum * tiling->formerLength + tiling->tailLength * (GetBlockIdx() - tiling->formerNum);
94	- nextCoreAddrOffset = (outputPointOffset + outputPointNum) * inC;	65	+ poolMem.nextCoreAddrOffset = (poolMem.outputPointOffset + poolMem.outputPointNum) * poolMem.inC;
95	- atomicAddNum = tiling->atomicAddNum;	66	+ poolMem.atomicAddNum = tiling->atomicAddNum;
96	- cTailLength = inC % tileC;	67	+ cTailLength = poolMem.inC % tileC;
97	- cTailAlign = AlignUp(cTailLength, numPerBlock);	68	+ cTailAlign = AlignUp(cTailLength, poolMem.numPerBlock);
98	- validTailLen = cTailLength % numPerBlock;	69	+ poolMem.validTailLen = cTailLength % poolMem.numPerBlock;
99	- usedCoreNum = tiling->usedCoreNum;	70	+ poolMem.usedCoreNum = tiling->usedCoreNum;
100	}	71	}
101		72
102	template <typename T, int32_t QUEUE_DEPTH>	73	template <typename T, int32_t QUEUE_DEPTH>
103	__aicore__ inline void KernelAvgPool3dSplitC<T, QUEUE_DEPTH>::CopyIn(int64_t offset, int64_t len) {	74	__aicore__ inline void KernelAvgPool3dSplitC<T, QUEUE_DEPTH>::CopyIn(int64_t offset, int64_t len) {
104	- LocalTensor<T> inputLocal = inputQueue.template AllocTensor<T>();	75	+ LocalTensor<T> inputLocal = poolMem.inputQueue.template AllocTensor<T>();
105	#if __CCE_AICORE__ < 220	76	#if __CCE_AICORE__ < 220
106	if constexpr (std::is_same_v<T, float>) {	77	if constexpr (std::is_same_v<T, float>) {
107	if (len == tileC) {	78	if (len == tileC) {
108	- DataCopyParams copyParams{1, static_cast<uint16_t>(len / numPerBlock), 0, 0};	79	+ DataCopyParams copyParams{1, static_cast<uint16_t>(len / poolMem.numPerBlock), 0, 0};
109	- DataCopy(inputLocal, inputGlobal[offset], copyParams);	80	+ DataCopy(inputLocal, poolMem.inputGlobal[offset], copyParams);
110	} else {	81	} else {
111	- DataCopy(inputLocal, inputGlobal[offset], cTailAlign);	82	+ DataCopy(inputLocal, poolMem.inputGlobal[offset], cTailAlign);
112	}	83	}
113	} else {	84	} else {
114	if (len == tileC) {	85	if (len == tileC) {
115	- DataCopyParams copyParams{1, static_cast<uint16_t>(len / numPerBlock), 0, 0};	86	+ DataCopyParams copyParams{1, static_cast<uint16_t>(len / poolMem.numPerBlock), 0, 0};
116	- DataCopy(inputLocal[tileC], inputGlobal[offset], copyParams);	87	+ DataCopy(inputLocal[tileC], poolMem.inputGlobal[offset], copyParams);
117	} else {	88	} else {
118	- DataCopy(inputLocal[tileC], inputGlobal[offset], cTailAlign);	89	+ DataCopy(inputLocal[tileC], poolMem.inputGlobal[offset], cTailAlign);
119	}	90	}
120	}	91	}
121	#else	92	#else
122	DataCopyExtParams copyParams{1, static_cast<uint32_t>(len * sizeof(T)), 0, 0, 0};	93	DataCopyExtParams copyParams{1, static_cast<uint32_t>(len * sizeof(T)), 0, 0, 0};
123	DataCopyPadExtParams<T> padParams{false, 0, 0, 0};	94	DataCopyPadExtParams<T> padParams{false, 0, 0, 0};
124	if constexpr (std::is_same_v<T, float>) {	95	if constexpr (std::is_same_v<T, float>) {
125	- DataCopyPad(inputLocal, inputGlobal[offset], copyParams, padParams);	96	+ DataCopyPad(inputLocal, poolMem.inputGlobal[offset], copyParams, padParams);
126	} else {	97	} else {
127	- DataCopyPad(inputLocal[tileC], inputGlobal[offset], copyParams, padParams);	98	+ DataCopyPad(inputLocal[tileC], poolMem.inputGlobal[offset], copyParams, padParams);
128	}	99	}
129	#endif	100	#endif
130	- inputQueue.EnQue(inputLocal);	101	+ poolMem.inputQueue.EnQue(inputLocal);
131	}	102	}
132		103
133	template <typename T, int32_t QUEUE_DEPTH>	104	template <typename T, int32_t QUEUE_DEPTH>
134	__aicore__ inline void KernelAvgPool3dSplitC<T, QUEUE_DEPTH>::DataCopyOutNonPad(	105	__aicore__ inline void KernelAvgPool3dSplitC<T, QUEUE_DEPTH>::DataCopyOutNonPad(
135	LocalTensor<T>& outputLocal, int64_t offset, int64_t validDataLen) {	106	LocalTensor<T>& outputLocal, int64_t offset, int64_t validDataLen) {
136	- if ((validDataLen < numPerBlock) && (offset + validDataLen * atomicAddNum >= nextCoreAddrOffset)) {	107	+ if ((validDataLen < poolMem.numPerBlock) && (offset + validDataLen * poolMem.atomicAddNum >= poolMem.nextCoreAddrOffset)) {
137	- uint64_t mask0 = (1ul << numPerBlock) - (1ul << validDataLen);	108	+ uint64_t mask0 = (1ul << poolMem.numPerBlock) - (1ul << validDataLen);
138	uint64_t mask[2] = {mask0, 0};	109	uint64_t mask[2] = {mask0, 0};
139	Duplicate<T>(outputLocal, 0, mask, 1, 1, 1);	110	Duplicate<T>(outputLocal, 0, mask, 1, 1, 1);
140	VToMTE3Sync();	111	VToMTE3Sync();
141	SetAtomicAdd<T>();	112	SetAtomicAdd<T>();
142	- DataCopy(outputGlobal[offset], outputLocal, cTailAlign);	113	+ DataCopy(poolMem.outputGlobal[offset], outputLocal, cTailAlign);
143	SetAtomicNone();	114	SetAtomicNone();
144	AscendC::PipeBarrier<PIPE_MTE3>();	115	AscendC::PipeBarrier<PIPE_MTE3>();
145	- } else if ((validTailLen != 0) && (offset + validDataLen == nextCoreAddrOffset)) {	116	+ } else if ((poolMem.validTailLen != 0) && (offset + validDataLen == poolMem.nextCoreAddrOffset)) {
146	- DataCopy(outputGlobal[offset], outputLocal, cTailAlign - numPerBlock);	117	+ DataCopy(poolMem.outputGlobal[offset], outputLocal, cTailAlign - poolMem.numPerBlock);
147	- int32_t lastLeftShift = validTailLen;	118	+ int32_t lastLeftShift = poolMem.validTailLen;
148	- uint32_t mask = numPerBlock * 2;	119	+ uint32_t mask = poolMem.numPerBlock * 2;
149	uint64_t rsvdCnt = 0;	120	uint64_t rsvdCnt = 0;
150	uint64_t gatherOffset = cTailAlign - mask;	121	uint64_t gatherOffset = cTailAlign - mask;
151	MTE3ToVSync();	122	MTE3ToVSync();
152	if constexpr (std::is_same_v<T, float>) {	123	if constexpr (std::is_same_v<T, float>) {
153	- LocalTensor<uint32_t> bufPattern = tmpPattern.Get<uint32_t>();	124	+ LocalTensor<uint32_t> bufPattern = poolMem.tmpPattern.template Get<uint32_t>();
154	- int32_t preLeftShift = numPerBlock + lastLeftShift;	125	+ int32_t preLeftShift = poolMem.numPerBlock + lastLeftShift;
155		126
156	bufPattern.SetValue(0, (1u << preLeftShift) - (1u << lastLeftShift));	127	bufPattern.SetValue(0, (1u << preLeftShift) - (1u << lastLeftShift));
157	SToVSync();	128	SToVSync();
158	GatherMask(outputLocal[gatherOffset], outputLocal[gatherOffset], bufPattern, true, mask, {1, 1, 8, 8},	129	GatherMask(outputLocal[gatherOffset], outputLocal[gatherOffset], bufPattern, true, mask, {1, 1, 8, 8},
159	rsvdCnt);	130	rsvdCnt);
160	} else {	131	} else {
161	- LocalTensor<uint16_t> bufPattern = tmpPattern.Get<uint16_t>();	132	+ LocalTensor<uint16_t> bufPattern = poolMem.tmpPattern.template Get<uint16_t>();
162	- int32_t preLeftShift = numPerBlock - lastLeftShift;	133	+ int32_t preLeftShift = poolMem.numPerBlock - lastLeftShift;
163		134
164	bufPattern.SetValue(0, ((1u << preLeftShift) - 1u) << lastLeftShift);	135	bufPattern.SetValue(0, ((1u << preLeftShift) - 1u) << lastLeftShift);
165	bufPattern.SetValue(1, (1u << lastLeftShift) - 1u);	136	bufPattern.SetValue(1, (1u << lastLeftShift) - 1u);
	@@ -168,27 +139,27 @@ __aicore__ inline void KernelAvgPool3dSplitC<T, QUEUE_DEPTH>::DataCopyOutNonPad(
168	rsvdCnt);	139	rsvdCnt);
169	}	140	}
170	VToMTE3Sync();	141	VToMTE3Sync();
171	- DataCopy(outputGlobal[nextCoreAddrOffset - numPerBlock], outputLocal[gatherOffset], numPerBlock);	142	+ DataCopy(poolMem.outputGlobal[poolMem.nextCoreAddrOffset - poolMem.numPerBlock], outputLocal[gatherOffset], poolMem.numPerBlock);
172	} else {	143	} else {
173	- DataCopy(outputGlobal[offset], outputLocal, cTailAlign);	144	+ DataCopy(poolMem.outputGlobal[offset], outputLocal, cTailAlign);
174	}	145	}
175	}	146	}
176		147
177	template <typename T, int32_t QUEUE_DEPTH>	148	template <typename T, int32_t QUEUE_DEPTH>
178	__aicore__ inline void KernelAvgPool3dSplitC<T, QUEUE_DEPTH>::CopyOut(int64_t offset, int64_t len) {	149	__aicore__ inline void KernelAvgPool3dSplitC<T, QUEUE_DEPTH>::CopyOut(int64_t offset, int64_t len) {
179	- LocalTensor<T> outputLocal = outputQueue.template DeQue<T>();	150	+ LocalTensor<T> outputLocal = poolMem.outputQueue.template DeQue<T>();
180	#if __CCE_AICORE__ < 220	151	#if __CCE_AICORE__ < 220
181	if (len == tileC) {	152	if (len == tileC) {
182	- DataCopyParams copyParams{1, static_cast<uint16_t>(len / numPerBlock), 0, 0};	153	+ DataCopyParams copyParams{1, static_cast<uint16_t>(len / poolMem.numPerBlock), 0, 0};
183	- DataCopy(outputGlobal[offset], outputLocal, copyParams);	154	+ DataCopy(poolMem.outputGlobal[offset], outputLocal, copyParams);
184	} else {	155	} else {
185	DataCopyOutNonPad(outputLocal, offset, len);	156	DataCopyOutNonPad(outputLocal, offset, len);
186	}	157	}
187	#else	158	#else
188	DataCopyExtParams copyParams{1, static_cast<uint32_t>(len * sizeof(T)), 0, 0, 0};	159	DataCopyExtParams copyParams{1, static_cast<uint32_t>(len * sizeof(T)), 0, 0, 0};
189	- DataCopyPad(outputGlobal[offset], outputLocal, copyParams);	160	+ DataCopyPad(poolMem.outputGlobal[offset], outputLocal, copyParams);
190	#endif	161	#endif
191	- outputQueue.FreeTensor(outputLocal);	162	+ poolMem.outputQueue.FreeTensor(outputLocal);
192	}	163	}
193		164
194	template <typename T, int32_t QUEUE_DEPTH>	165	template <typename T, int32_t QUEUE_DEPTH>
	@@ -201,22 +172,22 @@ __aicore__ inline void KernelAvgPool3dSplitC<T, QUEUE_DEPTH>::ReduceSumWindow(
201	int64_t wstart = index.W.start;	172	int64_t wstart = index.W.start;
202	int64_t wend = index.W.end;	173	int64_t wend = index.W.end;
203		174
204	- int64_t startOffset = nOffset * inputShape.strideN + cOffset;	175	+ int64_t startOffset = nOffset * poolMem.inputShape.strideN + cOffset;
205	for (int64_t id = dstart; id < dend; ++id) {	176	for (int64_t id = dstart; id < dend; ++id) {
206	- int64_t dOffset = id * inputShape.strideD;	177	+ int64_t dOffset = id * poolMem.inputShape.strideD;
207	for (int64_t ih = hstart; ih < hend; ++ih) {	178	for (int64_t ih = hstart; ih < hend; ++ih) {
208	- int64_t hOffset = ih * inputShape.strideH;	179	+ int64_t hOffset = ih * poolMem.inputShape.strideH;
209	for (int64_t iw = wstart; iw < wend; ++iw) {	180	for (int64_t iw = wstart; iw < wend; ++iw) {
210	- CopyIn(startOffset + (dOffset + hOffset + iw * inputShape.strideW) * inC, len);	181	+ CopyIn(startOffset + (dOffset + hOffset + iw * poolMem.inputShape.strideW) * poolMem.inC, len);
211		182
212	- LocalTensor<T> inputLocal = inputQueue.template DeQue<T>();	183	+ LocalTensor<T> inputLocal = poolMem.inputQueue.template DeQue<T>();
213	if constexpr (std::is_same_v<T, float>) {	184	if constexpr (std::is_same_v<T, float>) {
214	Add(sumBufLocal, sumBufLocal, inputLocal, len);	185	Add(sumBufLocal, sumBufLocal, inputLocal, len);
215	} else {	186	} else {
216	Cast(inputLocal.template ReinterpretCast<float>(), inputLocal[tileC], RoundMode::CAST_NONE, len);	187	Cast(inputLocal.template ReinterpretCast<float>(), inputLocal[tileC], RoundMode::CAST_NONE, len);
217	Add(sumBufLocal, sumBufLocal, inputLocal.template ReinterpretCast<float>(), len);	188	Add(sumBufLocal, sumBufLocal, inputLocal.template ReinterpretCast<float>(), len);
218	}	189	}
219	- inputQueue.FreeTensor(inputLocal);	190	+ poolMem.inputQueue.FreeTensor(inputLocal);
220	}	191	}
221	}	192	}
222	}	193	}
	@@ -225,39 +196,25 @@ __aicore__ inline void KernelAvgPool3dSplitC<T, QUEUE_DEPTH>::ReduceSumWindow(
225	template <typename T, int32_t QUEUE_DEPTH>	196	template <typename T, int32_t QUEUE_DEPTH>
226	__aicore__ inline void KernelAvgPool3dSplitC<T, QUEUE_DEPTH>::ReduceMeanWindow(int64_t outputPointIdx) {	197	__aicore__ inline void KernelAvgPool3dSplitC<T, QUEUE_DEPTH>::ReduceMeanWindow(int64_t outputPointIdx) {
227	Index index;	198	Index index;
228	- indexBuf.GetIndex(outputPointIdx, index);	199	+ poolMem.indexBuf.GetIndex(outputPointIdx, index);
229		200
230	- int64_t poolSize = poolParam.divisorOverride ?	201	+ int64_t poolSize = poolMem.poolParam.divisorOverride ?
231	- poolParam.divisorOverride : index.D.poolSize * index.H.poolSize * index.W.poolSize;	202	+ poolMem.poolParam.divisorOverride : index.D.poolSize * index.H.poolSize * index.W.poolSize;
232	float factor = 1.0f / static_cast<float>(poolSize);	203	float factor = 1.0f / static_cast<float>(poolSize);
233		204
234	SToVSync();	205	SToVSync();
235		206
236	- int64_t cLoop = (inC + tileC - 1) / tileC;	207	+ int64_t cLoop = (poolMem.inC + tileC - 1) / tileC;
237	int64_t cOffset = 0;	208	int64_t cOffset = 0;
238	for (int64_t i = 0; i < cLoop; ++i) {	209	for (int64_t i = 0; i < cLoop; ++i) {
239	- int64_t count = i < cLoop - 1 ? tileC : inC - (cLoop - 1) * tileC;	210	+ int64_t count = i < cLoop - 1 ? tileC : poolMem.inC - (cLoop - 1) * tileC;
240		211
241	- Duplicate(sumBufLocal, 0.0f, count);	212	+ Duplicate(poolMem.sumBufLocal, 0.0f, count);
242		213
243	- ReduceSumWindow(index, sumBufLocal, outputPointIdx / outputShape.strideC, cOffset, count);	214	+ ReduceSumWindow(index, poolMem.sumBufLocal, outputPointIdx / poolMem.outputShape.strideC, cOffset, count);
244	- Muls(sumBufLocal, sumBufLocal, factor, count);	215	+ CastAndEnqueueOutput<T, QUEUE_DEPTH>(poolMem, count, factor);
245		216
246	- LocalTensor<T> outputLocal = outputQueue.template AllocTensor<T>();	217	+ CopyOut(outputPointIdx * poolMem.inC + cOffset, count);
247	- if constexpr (std::is_same_v<T, float>) {
248	-#if __CCE_AICORE__ < 220
249	- Adds(outputLocal, sumBufLocal, 0.0f, AlignUp(count, numPerBlock));
250	-#else
251	- DataCopy(outputLocal, sumBufLocal, AlignUp(count, numPerBlock));
252	-#endif
253	- } else if constexpr (std::is_same_v<T, half>) {
254	- Cast(outputLocal, sumBufLocal, RoundMode::CAST_NONE, count);
255	- } else {
256	- Cast(outputLocal, sumBufLocal, RoundMode::CAST_RINT, count);
257	- }
258	- outputQueue.EnQue(outputLocal);
259	-
260	- CopyOut(outputPointIdx * inC + cOffset, count);
261		218
262	cOffset += count;	219	cOffset += count;
263	}	220	}
	@@ -268,51 +225,29 @@ __aicore__ inline void KernelAvgPool3dSplitC<T, QUEUE_DEPTH>::Init(
268	GM_ADDR x, GM_ADDR y, GM_ADDR workspace, const AvgPool3DTilingData* __restrict__ tiling, TPipe* pipe) {	225	GM_ADDR x, GM_ADDR y, GM_ADDR workspace, const AvgPool3DTilingData* __restrict__ tiling, TPipe* pipe) {
269	InitTiling(tiling);	226	InitTiling(tiling);
270		227
271	- inputGlobal.SetGlobalBuffer((__gm__ T*)x);	228	+ poolMem.pipe = pipe;
272	- outputGlobal.SetGlobalBuffer((__gm__ T*)y);	229	+ poolMem.inputGlobal.SetGlobalBuffer((__gm__ T*)x);
		230	+ poolMem.outputGlobal.SetGlobalBuffer((__gm__ T*)y);
273		231
274	- pipe->InitBuffer(inputQueue, QUEUE_DEPTH, tileC * sizeof(float));	232	+ pipe->InitBuffer(poolMem.inputQueue, QUEUE_DEPTH, tileC * sizeof(float));
275	- pipe->InitBuffer(outputQueue, QUEUE_DEPTH, tileC * sizeof(T));	233	+ pipe->InitBuffer(poolMem.outputQueue, QUEUE_DEPTH, tileC * sizeof(T));
276		234
277	-#if __CCE_AICORE__ < 220	235	+ InitCommonBuffers(poolMem, workspace);
278	- if (atomicAddNum) {
279	- pipe->InitBuffer(tmpPattern, numPerBlock * sizeof(T));
280		236
281	- pipe->InitBuffer(syncWorkQueue, QUEUE_DEPTH, 8 * 32 * sizeof(int32_t));	237	+ pipe->InitBuffer(poolMem.sumBuf, tileC * sizeof(float));
282	- syncTensorsGM.SetGlobalBuffer((__gm__ int32_t )workspace, usedCoreNum 8 * 32);	238	+ poolMem.sumBufLocal = poolMem.sumBuf.template Get<float>();
283	- pipe->InitBuffer(clearTensorBuff, DEFAULT_CLEAR_UB_SIZE * sizeof(T));
284	- } else if (validTailLen != 0) {
285	- pipe->InitBuffer(tmpPattern, numPerBlock * sizeof(T));
286	- }
287	-#endif
288		239
289	- pipe->InitBuffer(sumBuf, tileC * sizeof(float));	240	+ poolMem.indexBuf.Init(pipe, poolMem.indexBufLen);
290	- sumBufLocal = sumBuf.Get<float>();
291	-
292	- indexBuf.Init(pipe, indexBufLen);
293	}	241	}
294		242
295	template <typename T, int32_t QUEUE_DEPTH>	243	template <typename T, int32_t QUEUE_DEPTH>
296	__aicore__ inline void KernelAvgPool3dSplitC<T, QUEUE_DEPTH>::Process() {	244	__aicore__ inline void KernelAvgPool3dSplitC<T, QUEUE_DEPTH>::Process() {
297	#if __CCE_AICORE__ < 220	245	#if __CCE_AICORE__ < 220
298	- if (atomicAddNum) {	246	+ int64_t curOutputPointIdx = poolMem.outputPointNum + poolMem.outputPointOffset - 1;
299	- LocalTensor<T> clearUb = clearTensorBuff.Get<T>();	247	+ AvgPool3d::HandleAtomicAddWithTail(poolMem, curOutputPointIdx, cTailLength);
300	- Duplicate(clearUb, (T)0, DEFAULT_CLEAR_UB_SIZE);
301	-
302	- VToMTE3Sync();
303	- int64_t curOutputPointIdx = outputPointNum + outputPointOffset - 1;
304	- for (int i = 0; i < atomicAddNum; i++, curOutputPointIdx--) {
305	- DataCopy<T>(outputGlobal[curOutputPointIdx * cTailLength], clearUb, numPerBlock);
306	- }
307	-
308	- DataCopy(syncTensorsGM[0], clearUb.template ReinterpretCast<int32_t>(), usedCoreNum * 8 * 32);
309	- LocalTensor<int32_t> syncLocalTensor = syncWorkQueue.template AllocTensor<int32_t>();
310	- AscendC::SyncAll(syncTensorsGM, syncLocalTensor, int32_t(usedCoreNum));
311	- syncWorkQueue.FreeTensor(syncLocalTensor);
312	- }
313	#endif	248	#endif
314	- for (int64_t outputPointIdx = outputPointOffset;	249	+ for (int64_t outputPointIdx = poolMem.outputPointOffset;
315	- outputPointIdx < outputPointOffset + outputPointNum; ++outputPointIdx) {	250	+ outputPointIdx < poolMem.outputPointOffset + poolMem.outputPointNum; ++outputPointIdx) {
316	ReduceMeanWindow(outputPointIdx);	251	ReduceMeanWindow(outputPointIdx);
317	}	252	}
318	}	253	}

Mpooling/avg_pool3_d/op_kernel/avg_pool3d_ndhwc_split_w.h+71-175

	@@ -37,165 +37,84 @@ private:
37	__aicore__ inline void ReduceMeanWindow(int64_t outputPointIdx);	37	__aicore__ inline void ReduceMeanWindow(int64_t outputPointIdx);
38	__aicore__ inline void ReduceSumWindow(const Index& index, LocalTensor<float>& sumBufLocal, int64_t nOffset);	38	__aicore__ inline void ReduceSumWindow(const Index& index, LocalTensor<float>& sumBufLocal, int64_t nOffset);
39		39
40	- TPipe* pipe;
41	- TQue<QuePosition::VECIN, QUEUE_DEPTH> inputQueue;
42	- TQue<QuePosition::VECOUT, QUEUE_DEPTH> outputQueue;
43	-
44	- TBuf<QuePosition::VECCALC> tmpPattern;
45	- TBuf<TPosition::VECCALC> sumBuf;
46	- LocalTensor<float> sumBufLocal;
47	-
48	- GlobalTensor<T> inputGlobal;
49	- GlobalTensor<T> outputGlobal;
50	-
51	- int64_t inC;
52	- int64_t alignC;
53	- int64_t outputPointNum;
54	- int64_t outputPointOffset;
55	- int64_t lastPointOffset;
56	int64_t tileInput;	40	int64_t tileInput;
57	- int64_t atomicAddNum;	41	+ PoolMem<T, QUEUE_DEPTH> poolMem;
58	-
59	- PoolShape inputShape;
60	- PoolShape outputShape;
61	-
62	- int64_t indexBufLen;
63	- IndexBuffer indexBuf;
64	- PoolParameter poolParam;
65	-
66	- uint32_t numPerBlock;
67	- uint32_t inputBufLen;
68	- int32_t validTailLen;
69	-
70	- TQue<QuePosition::VECIN, QUEUE_DEPTH> syncWorkQueue;
71	- GlobalTensor<int32_t> syncTensorsGM;
72	- TBuf<TPosition::VECCALC> clearTensorBuff;
73	- uint32_t usedCoreNum;
74	};	42	};
75		43
76	template <typename T, int32_t QUEUE_DEPTH>	44	template <typename T, int32_t QUEUE_DEPTH>
77	__aicore__ inline void KernelAvgPool3dSplitW<T, QUEUE_DEPTH>::InitTiling(const AvgPool3DTilingData* __restrict__ tiling) {	45	__aicore__ inline void KernelAvgPool3dSplitW<T, QUEUE_DEPTH>::InitTiling(const AvgPool3DTilingData* __restrict__ tiling) {
78	- inputShape = PoolShape(tiling->inN, tiling->inC, tiling->inD, tiling->inH, tiling->inW);	46	+ poolMem.inputShape = PoolShape(tiling->inN, tiling->inC, tiling->inD, tiling->inH, tiling->inW);
79	- outputShape = PoolShape(tiling->inN, tiling->inC, tiling->outD, tiling->outH, tiling->outW);	47	+ poolMem.outputShape = PoolShape(tiling->inN, tiling->inC, tiling->outD, tiling->outH, tiling->outW);
80		48
81	- poolParam = PoolParameter(tiling->kD, tiling->kH, tiling->kW, tiling->dD, tiling->dH, tiling->dW,	49	+ poolMem.poolParam = PoolParameter(tiling->kD, tiling->kH, tiling->kW, tiling->dD, tiling->dH, tiling->dW,
82	tiling->pD, tiling->pH, tiling->pW, tiling->divisorOverride, tiling->countIncludePad);	50	tiling->pD, tiling->pH, tiling->pW, tiling->divisorOverride, tiling->countIncludePad);
83		51
84	- indexBuf.SetComputeParameter(outputShape, inputShape, poolParam);	52	+ poolMem.indexBuf.SetComputeParameter(poolMem.outputShape, poolMem.inputShape, poolMem.poolParam);
85		53
86	- numPerBlock = GetDataBlockSizeInBytes() / sizeof(T);	54	+ poolMem.numPerBlock = GetDataBlockSizeInBytes() / sizeof(T);
87	- inC = tiling->inC;	55	+ poolMem.inC = tiling->inC;
88	- alignC = AlignUp(inC, numPerBlock);	56	+ poolMem.alignC = AlignUp(poolMem.inC, poolMem.numPerBlock);
89	tileInput = tiling->tileInput;	57	tileInput = tiling->tileInput;
90		58
91	- outputPointNum = GetBlockIdx() < tiling->formerNum ? tiling->formerLength : tiling->tailLength;	59	+ poolMem.outputPointNum = GetBlockIdx() < tiling->formerNum ? tiling->formerLength : tiling->tailLength;
92	- outputPointOffset = GetBlockIdx() < tiling->formerNum	60	+ poolMem.outputPointOffset = GetBlockIdx() < tiling->formerNum
93	? tiling->formerLength * GetBlockIdx()	61	? tiling->formerLength * GetBlockIdx()
94	: tiling->formerNum * tiling->formerLength + tiling->tailLength * (GetBlockIdx() - tiling->formerNum);	62	: tiling->formerNum * tiling->formerLength + tiling->tailLength * (GetBlockIdx() - tiling->formerNum);
95	- lastPointOffset = outputPointNum + outputPointOffset - 1;	63	+ poolMem.lastPointOffset = poolMem.outputPointNum + poolMem.outputPointOffset - 1;
96	- atomicAddNum = outputPointNum < tiling->atomicAddNum ? outputPointNum : tiling->atomicAddNum;	64	+ poolMem.atomicAddNum = poolMem.outputPointNum < tiling->atomicAddNum ? poolMem.outputPointNum : tiling->atomicAddNum;
97	- indexBufLen = tiling->indexBufLen;	65	+ poolMem.indexBufLen = tiling->indexBufLen;
98	- validTailLen = inC % numPerBlock;	66	+ poolMem.validTailLen = poolMem.inC % poolMem.numPerBlock;
99	- usedCoreNum = tiling->usedCoreNum;	67	+ poolMem.usedCoreNum = tiling->usedCoreNum;
100	}	68	}
101		69
102	template <typename T, int32_t QUEUE_DEPTH>	70	template <typename T, int32_t QUEUE_DEPTH>
103	__aicore__ inline void KernelAvgPool3dSplitW<T, QUEUE_DEPTH>::CopyIn(	71	__aicore__ inline void KernelAvgPool3dSplitW<T, QUEUE_DEPTH>::CopyIn(
104	int64_t offset, uint16_t blockCount, uint32_t blockLen, uint8_t rightPadding) {	72	int64_t offset, uint16_t blockCount, uint32_t blockLen, uint8_t rightPadding) {
105	- LocalTensor<T> inputLocal = inputQueue.template AllocTensor<T>();	73	+ AvgPool3d::CopyInTemplate(poolMem, offset, blockCount, blockLen, rightPadding);
106	-#if __CCE_AICORE__ < 220
107	- if constexpr (std::is_same_v<T, float>) {
108	- if (blockLen == alignC) {
109	- DataCopyParams copyParams{blockCount, static_cast<uint16_t>(blockLen / numPerBlock), 0, 0};
110	- DataCopy(inputLocal, inputGlobal[offset], copyParams);
111	- } else {
112	- for (int i = 0; i < blockCount; i++) {
113	- DataCopy(inputLocal[i * alignC], inputGlobal[offset + i * blockLen], alignC);
114	- }
115	- }
116	- } else {
117	- if (blockLen == alignC) {
118	- DataCopyParams copyParams{blockCount, static_cast<uint16_t>(blockLen / numPerBlock), 0, 0};
119	- DataCopy(inputLocal[inputBufLen], inputGlobal[offset], copyParams);
120	- } else {
121	- for (int i = 0; i < blockCount; i++) {
122	- DataCopy(inputLocal[inputBufLen + i * alignC], inputGlobal[offset + i * blockLen], alignC);
123	- }
124	- }
125	- }
126	-#else
127	- DataCopyExtParams copyParams{blockCount, static_cast<uint32_t>(blockLen * sizeof(T)), 0, 0, 0};
128	- DataCopyPadExtParams<T> padParams{true, 0, rightPadding, 0};
129	- if constexpr (std::is_same_v<T, float>) {
130	- DataCopyPad(inputLocal, inputGlobal[offset], copyParams, padParams);
131	- } else {
132	- DataCopyPad(inputLocal[inputBufLen], inputGlobal[offset], copyParams, padParams);
133	- }
134	-#endif
135	- inputQueue.EnQue(inputLocal);
136	}	74	}
137		75
138	template <typename T, int32_t QUEUE_DEPTH>	76	template <typename T, int32_t QUEUE_DEPTH>
139	__aicore__ inline void KernelAvgPool3dSplitW<T, QUEUE_DEPTH>::DataCopyOutNonPad(	77	__aicore__ inline void KernelAvgPool3dSplitW<T, QUEUE_DEPTH>::DataCopyOutNonPad(
140	LocalTensor<T>& outputLocal, int64_t outputPointIdx, uint32_t validDataLen) {	78	LocalTensor<T>& outputLocal, int64_t outputPointIdx, uint32_t validDataLen) {
141	- if ((validDataLen < numPerBlock) && (outputPointIdx >= lastPointOffset - atomicAddNum)) {	79	+ if ((validDataLen < poolMem.numPerBlock) && (outputPointIdx >= poolMem.lastPointOffset - poolMem.atomicAddNum)) {
142	- uint64_t mask0 = (1ul << numPerBlock) - (1ul << validDataLen);	80	+ uint64_t mask0 = (1ul << poolMem.numPerBlock) - (1ul << validDataLen);
143	uint64_t mask[2] = {mask0, 0};	81	uint64_t mask[2] = {mask0, 0};
144	Duplicate<T>(outputLocal, 0, mask, 1, 1, 1);	82	Duplicate<T>(outputLocal, 0, mask, 1, 1, 1);
145	VToMTE3Sync();	83	VToMTE3Sync();
146	- if (outputPointIdx > lastPointOffset - atomicAddNum) {	84	+ if (outputPointIdx > poolMem.lastPointOffset - poolMem.atomicAddNum) {
147	SetAtomicAdd<T>();	85	SetAtomicAdd<T>();
148	- DataCopy(outputGlobal[outputPointIdx * validDataLen], outputLocal, alignC);	86	+ DataCopy(poolMem.outputGlobal[outputPointIdx * validDataLen], outputLocal, poolMem.alignC);
149	SetAtomicNone();	87	SetAtomicNone();
150	AscendC::PipeBarrier<PIPE_MTE3>();	88	AscendC::PipeBarrier<PIPE_MTE3>();
151	} else {	89	} else {
152	- DataCopy(outputGlobal[outputPointIdx * validDataLen], outputLocal, alignC);	90	+ DataCopy(poolMem.outputGlobal[outputPointIdx * validDataLen], outputLocal, poolMem.alignC);
153	}	91	}
154	- } else if (outputPointIdx == lastPointOffset) {	92	+ } else if (outputPointIdx == poolMem.lastPointOffset) {
155	- DataCopy(outputGlobal[outputPointIdx * validDataLen], outputLocal, alignC - numPerBlock);	93	+ DataCopy(poolMem.outputGlobal[outputPointIdx * validDataLen], outputLocal, poolMem.alignC - poolMem.numPerBlock);
156	- int32_t lastLeftShift = validTailLen;	94	+ uint32_t mask = poolMem.numPerBlock * 2;
157	- uint32_t mask = numPerBlock * 2;	95	+ uint64_t gatherOffset = poolMem.alignC - mask;
158	- uint64_t rsvdCnt = 0;	96	+ HandleTailMask(outputLocal, gatherOffset, poolMem, mask);
159	- uint64_t gatherOffset = alignC - mask;	97	+ DataCopy(poolMem.outputGlobal[(outputPointIdx + 1) * validDataLen - poolMem.numPerBlock], outputLocal[gatherOffset], poolMem.numPerBlock);
160	- MTE3ToVSync();
161	- if constexpr (std::is_same_v<T, float>) {
162	- LocalTensor<uint32_t> bufPattern = tmpPattern.Get<uint32_t>();
163	- int32_t preLeftShift = numPerBlock + lastLeftShift;
164	-
165	- bufPattern.SetValue(0, (1u << preLeftShift) - (1u << lastLeftShift));
166	- SToVSync();
167	- GatherMask(outputLocal[gatherOffset], outputLocal[gatherOffset], bufPattern, true, mask, {1, 1, 8, 8}, rsvdCnt);
168	- } else {
169	- LocalTensor<uint16_t> bufPattern = tmpPattern.Get<uint16_t>();
170	- int32_t preLeftShift = numPerBlock - lastLeftShift;
171	-
172	- bufPattern.SetValue(0, ((1u << preLeftShift) - 1u) << lastLeftShift);
173	- bufPattern.SetValue(1, (1u << lastLeftShift) - 1u);
174	- SToVSync();
175	- GatherMask(outputLocal[gatherOffset], outputLocal[gatherOffset], bufPattern, true, mask, {1, 1, 8, 8}, rsvdCnt);
176	- }
177	- VToMTE3Sync();
178	- DataCopy(outputGlobal[(outputPointIdx + 1) * validDataLen - numPerBlock], outputLocal[gatherOffset], numPerBlock);
179	} else {	98	} else {
180	- DataCopy(outputGlobal[outputPointIdx * validDataLen], outputLocal, alignC);	99	+ DataCopy(poolMem.outputGlobal[outputPointIdx * validDataLen], outputLocal, poolMem.alignC);
181	}	100	}
182	}	101	}
183		102
184	template <typename T, int32_t QUEUE_DEPTH>	103	template <typename T, int32_t QUEUE_DEPTH>
185	__aicore__ inline void KernelAvgPool3dSplitW<T, QUEUE_DEPTH>::CopyOut(int64_t offset, int64_t len) {	104	__aicore__ inline void KernelAvgPool3dSplitW<T, QUEUE_DEPTH>::CopyOut(int64_t offset, int64_t len) {
186	- LocalTensor<T> outputLocal = outputQueue.template DeQue<T>();	105	+ LocalTensor<T> outputLocal = poolMem.outputQueue.template DeQue<T>();
187	#if __CCE_AICORE__ < 220	106	#if __CCE_AICORE__ < 220
188	- if (len == alignC) {	107	+ if (len == poolMem.alignC) {
189	- DataCopyParams copyParams{1, static_cast<uint16_t>(len / numPerBlock), 0, 0};	108	+ DataCopyParams copyParams{1, static_cast<uint16_t>(len / poolMem.numPerBlock), 0, 0};
190	- DataCopy(outputGlobal[offset * len], outputLocal, copyParams);	109	+ DataCopy(poolMem.outputGlobal[offset * len], outputLocal, copyParams);
191	} else {	110	} else {
192	DataCopyOutNonPad(outputLocal, offset, len);	111	DataCopyOutNonPad(outputLocal, offset, len);
193	}	112	}
194	#else	113	#else
195	DataCopyExtParams copyParams{1, static_cast<uint32_t>(len * sizeof(T)), 0, 0, 0};	114	DataCopyExtParams copyParams{1, static_cast<uint32_t>(len * sizeof(T)), 0, 0, 0};
196	- DataCopyPad(outputGlobal[offset * len], outputLocal, copyParams);	115	+ DataCopyPad(poolMem.outputGlobal[offset * len], outputLocal, copyParams);
197	#endif	116	#endif
198	- outputQueue.FreeTensor(outputLocal);	117	+ poolMem.outputQueue.FreeTensor(outputLocal);
199	}	118	}
200		119
201	template <typename T, int32_t QUEUE_DEPTH>	120	template <typename T, int32_t QUEUE_DEPTH>
	@@ -209,35 +128,35 @@ __aicore__ inline void KernelAvgPool3dSplitW<T, QUEUE_DEPTH>::ReduceSumWindow(
209	int64_t wend = index.W.end;	128	int64_t wend = index.W.end;
210		129
211	int64_t kW = (wend - wstart + tileInput - 1) / tileInput;	130	int64_t kW = (wend - wstart + tileInput - 1) / tileInput;
212	- uint8_t rightPadding = static_cast<uint8_t>(alignC - inC);	131	+ uint8_t rightPadding = static_cast<uint8_t>(poolMem.alignC - poolMem.inC);
213		132
214	for (int64_t id = dstart; id < dend; ++id) {	133	for (int64_t id = dstart; id < dend; ++id) {
215	- int64_t dOffset = id * inputShape.strideD * inC;	134	+ int64_t dOffset = id * poolMem.inputShape.strideD * poolMem.inC;
216	for (int64_t ih = hstart; ih < hend; ++ih) {	135	for (int64_t ih = hstart; ih < hend; ++ih) {
217	- int64_t hOffset = ih * inputShape.strideH * inC;	136	+ int64_t hOffset = ih * poolMem.inputShape.strideH * poolMem.inC;
218	for (int64_t j = 0, iw = wstart; j < kW; ++j) {	137	for (int64_t j = 0, iw = wstart; j < kW; ++j) {
219	int64_t tileNum = j < kW - 1 ? tileInput : wend - iw;	138	int64_t tileNum = j < kW - 1 ? tileInput : wend - iw;
220		139
221	- CopyIn(nOffset * inputShape.strideN + dOffset + hOffset + iw * inC,	140	+ CopyIn(nOffset * poolMem.inputShape.strideN + dOffset + hOffset + iw * poolMem.inC,
222	- static_cast<uint16_t>(tileNum), static_cast<uint32_t>(inC), rightPadding);	141	+ static_cast<uint16_t>(tileNum), static_cast<uint32_t>(poolMem.inC), rightPadding);
223	- LocalTensor<T> inputLocal = inputQueue.template DeQue<T>();	142	+ LocalTensor<T> inputLocal = poolMem.inputQueue.template DeQue<T>();
224		143
225	if constexpr (!std::is_same_v<T, float>) {	144	if constexpr (!std::is_same_v<T, float>) {
226	- Cast(inputLocal.template ReinterpretCast<float>(), inputLocal[inputBufLen],	145	+ Cast(inputLocal.template ReinterpretCast<float>(), inputLocal[poolMem.inputBufLen],
227	- RoundMode::CAST_NONE, inputBufLen);	146	+ RoundMode::CAST_NONE, poolMem.inputBufLen);
228	}	147	}
229		148
230	for (int64_t i = 0; i < tileNum; ++i) {	149	for (int64_t i = 0; i < tileNum; ++i) {
231	if constexpr (std::is_same_v<T, float>) {	150	if constexpr (std::is_same_v<T, float>) {
232	- Add(sumBufLocal, sumBufLocal, inputLocal[i * alignC], alignC);	151	+ Add(sumBufLocal, sumBufLocal, inputLocal[i * poolMem.alignC], poolMem.alignC);
233	} else {	152	} else {
234	- Add(sumBufLocal, sumBufLocal, inputLocal.template ReinterpretCast<float>()[i * alignC], alignC);	153	+ Add(sumBufLocal, sumBufLocal, inputLocal.template ReinterpretCast<float>()[i * poolMem.alignC], poolMem.alignC);
235	}	154	}
236	}	155	}
237		156
238	iw += tileNum;	157	iw += tileNum;
239		158
240	- inputQueue.FreeTensor(inputLocal);	159	+ poolMem.inputQueue.FreeTensor(inputLocal);
241	}	160	}
242	}	161	}
243	}	162	}
	@@ -246,34 +165,34 @@ __aicore__ inline void KernelAvgPool3dSplitW<T, QUEUE_DEPTH>::ReduceSumWindow(
246	template <typename T, int32_t QUEUE_DEPTH>	165	template <typename T, int32_t QUEUE_DEPTH>
247	__aicore__ inline void KernelAvgPool3dSplitW<T, QUEUE_DEPTH>::ReduceMeanWindow(int64_t outputPointIdx) {	166	__aicore__ inline void KernelAvgPool3dSplitW<T, QUEUE_DEPTH>::ReduceMeanWindow(int64_t outputPointIdx) {
248	Index index;	167	Index index;
249	- indexBuf.GetIndex(outputPointIdx, index);	168	+ poolMem.indexBuf.GetIndex(outputPointIdx, index);
250		169
251	- int64_t poolSize = poolParam.divisorOverride ?	170	+ int64_t poolSize = poolMem.poolParam.divisorOverride ?
252	- poolParam.divisorOverride : index.D.poolSize * index.H.poolSize * index.W.poolSize;	171	+ poolMem.poolParam.divisorOverride : index.D.poolSize * index.H.poolSize * index.W.poolSize;
253	float factor = 1.0f / static_cast<float>(poolSize);	172	float factor = 1.0f / static_cast<float>(poolSize);
254		173
255	SToVSync();	174	SToVSync();
256		175
257	- Duplicate(sumBufLocal, 0.0f, alignC);	176	+ Duplicate(poolMem.sumBufLocal, 0.0f, poolMem.alignC);
258		177
259	- ReduceSumWindow(index, sumBufLocal, outputPointIdx / outputShape.strideC);	178	+ ReduceSumWindow(index, poolMem.sumBufLocal, outputPointIdx / poolMem.outputShape.strideC);
260	- Muls(sumBufLocal, sumBufLocal, factor, alignC);	179	+ Muls(poolMem.sumBufLocal, poolMem.sumBufLocal, factor, poolMem.alignC);
261		180
262	- LocalTensor<T> outputLocal = outputQueue.template AllocTensor<T>();	181	+ LocalTensor<T> outputLocal = poolMem.outputQueue.template AllocTensor<T>();
263	if constexpr (std::is_same_v<T, float>) {	182	if constexpr (std::is_same_v<T, float>) {
264	#if __CCE_AICORE__ < 220	183	#if __CCE_AICORE__ < 220
265	- Adds(outputLocal, sumBufLocal, 0.0f, alignC);	184	+ Adds(outputLocal, poolMem.sumBufLocal, 0.0f, poolMem.alignC);
266	#else	185	#else
267	- DataCopy(outputLocal, sumBufLocal, alignC);	186	+ DataCopy(outputLocal, poolMem.sumBufLocal, poolMem.alignC);
268	#endif	187	#endif
269	} else if constexpr (std::is_same_v<T, half>) {	188	} else if constexpr (std::is_same_v<T, half>) {
270	- Cast(outputLocal, sumBufLocal, RoundMode::CAST_NONE, alignC);	189	+ Cast(outputLocal, poolMem.sumBufLocal, RoundMode::CAST_NONE, poolMem.alignC);
271	} else {	190	} else {
272	- Cast(outputLocal, sumBufLocal, RoundMode::CAST_RINT, alignC);	191	+ Cast(outputLocal, poolMem.sumBufLocal, RoundMode::CAST_RINT, poolMem.alignC);
273	}	192	}
274	- outputQueue.EnQue(outputLocal);	193	+ poolMem.outputQueue.EnQue(outputLocal);
275		194
276	- CopyOut(outputPointIdx, inC);	195	+ CopyOut(outputPointIdx, poolMem.inC);
277	}	196	}
278		197
279	template <typename T, int32_t QUEUE_DEPTH>	198	template <typename T, int32_t QUEUE_DEPTH>
	@@ -281,51 +200,28 @@ __aicore__ inline void KernelAvgPool3dSplitW<T, QUEUE_DEPTH>::Init(
281	GM_ADDR x, GM_ADDR y, GM_ADDR workspace, const AvgPool3DTilingData* __restrict__ tiling, TPipe* pipe) {	200	GM_ADDR x, GM_ADDR y, GM_ADDR workspace, const AvgPool3DTilingData* __restrict__ tiling, TPipe* pipe) {
282	InitTiling(tiling);	201	InitTiling(tiling);
283		202
284	- inputGlobal.SetGlobalBuffer((__gm__ T*)x);	203	+ poolMem.pipe = pipe;
285	- outputGlobal.SetGlobalBuffer((__gm__ T*)y);	204	+ poolMem.inputGlobal.SetGlobalBuffer((__gm__ T*)x);
		205	+ poolMem.outputGlobal.SetGlobalBuffer((__gm__ T*)y);
286		206
287	- inputBufLen = tileInput * alignC;	207	+ poolMem.inputBufLen = tileInput * poolMem.alignC;
288	- pipe->InitBuffer(inputQueue, QUEUE_DEPTH, inputBufLen * sizeof(float));	208	+ pipe->InitBuffer(poolMem.inputQueue, QUEUE_DEPTH, poolMem.inputBufLen * sizeof(float));
289	- pipe->InitBuffer(outputQueue, QUEUE_DEPTH, alignC * sizeof(T));	209	+ pipe->InitBuffer(poolMem.outputQueue, QUEUE_DEPTH, poolMem.alignC * sizeof(T));
290		210
291	- pipe->InitBuffer(sumBuf, alignC * sizeof(float));	211	+ pipe->InitBuffer(poolMem.sumBuf, poolMem.alignC * sizeof(float));
292	- sumBufLocal = sumBuf.Get<float>();	212	+ poolMem.sumBufLocal = poolMem.sumBuf.template Get<float>();
293		213
294	- indexBuf.Init(pipe, indexBufLen);	214	+ poolMem.indexBuf.Init(pipe, poolMem.indexBufLen);
295	-#if __CCE_AICORE__ < 220	215	+ InitCommonBuffers(poolMem, workspace);
296	- if (atomicAddNum) {
297	- pipe->InitBuffer(tmpPattern, numPerBlock * sizeof(T));
298	-
299	- pipe->InitBuffer(syncWorkQueue, QUEUE_DEPTH, 8 * 32 * sizeof(int32_t));
300	- syncTensorsGM.SetGlobalBuffer((__gm__ int32_t )workspace, usedCoreNum 8 * 32);
301	- pipe->InitBuffer(clearTensorBuff, DEFAULT_CLEAR_UB_SIZE * sizeof(T));
302	- } else if (validTailLen != 0) {
303	- pipe->InitBuffer(tmpPattern, numPerBlock * sizeof(T));
304	- }
305	-#endif
306	}	216	}
307		217
308	template <typename T, int32_t QUEUE_DEPTH>	218	template <typename T, int32_t QUEUE_DEPTH>
309	__aicore__ inline void KernelAvgPool3dSplitW<T, QUEUE_DEPTH>::Process() {	219	__aicore__ inline void KernelAvgPool3dSplitW<T, QUEUE_DEPTH>::Process() {
310	#if __CCE_AICORE__ < 220	220	#if __CCE_AICORE__ < 220
311	- if (atomicAddNum) {	221	+ AvgPool3d::HandleAtomicAdd(poolMem);
312	- LocalTensor<T> clearUb = clearTensorBuff.Get<T>();
313	- Duplicate(clearUb, (T)0, DEFAULT_CLEAR_UB_SIZE);
314	-
315	- VToMTE3Sync();
316	- int64_t curOutputPointIdx = lastPointOffset;
317	- for (int i = 0; i < atomicAddNum; i++, curOutputPointIdx--) {
318	- DataCopy<T>(outputGlobal[curOutputPointIdx * inC], clearUb, numPerBlock);
319	- }
320	-
321	- DataCopy(syncTensorsGM[0], clearUb.template ReinterpretCast<int32_t>(), usedCoreNum * 8 * 32);
322	- LocalTensor<int32_t> syncLocalTensor = syncWorkQueue.template AllocTensor<int32_t>();
323	- AscendC::SyncAll(syncTensorsGM, syncLocalTensor, int32_t(usedCoreNum));
324	- syncWorkQueue.FreeTensor(syncLocalTensor);
325	- }
326	#endif	222	#endif
327	- for (int64_t outputPointIdx = outputPointOffset;	223	+ for (int64_t outputPointIdx = poolMem.outputPointOffset;
328	- outputPointIdx < outputPointOffset + outputPointNum; ++outputPointIdx) {	224	+ outputPointIdx < poolMem.outputPointOffset + poolMem.outputPointNum; ++outputPointIdx) {
329	ReduceMeanWindow(outputPointIdx);	225	ReduceMeanWindow(outputPointIdx);
330	}	226	}
331	}	227	}

Mpooling/avg_pool3_d/op_kernel/avg_pool3d_normal.h+77-231

	@@ -19,6 +19,7 @@
19	#include "kernel_operator.h"	19	#include "kernel_operator.h"
20	#include "kernel_tiling/kernel_tiling.h"	20	#include "kernel_tiling/kernel_tiling.h"
21	#include "avg_pool3d_common.h"	21	#include "avg_pool3d_common.h"
		22	+ #include "../pool_3d_common/arch32/pool_3d_memory_optimized_utils.h"
22		23
23	namespace AvgPool3d {	24	namespace AvgPool3d {
24	template <typename T>	25	template <typename T>
	@@ -44,8 +45,6 @@ private:
44	__aicore__ inline void TransOut(int64_t curDoFactor, int64_t curHoFactor, int64_t curWoFactor);	45	__aicore__ inline void TransOut(int64_t curDoFactor, int64_t curHoFactor, int64_t curWoFactor);
45	__aicore__ inline void CopyOut(	46	__aicore__ inline void CopyOut(
46	int64_t curNcFactor, int64_t curDoFactor, int64_t curHoFactor, int64_t curWoFactor, int64_t yGmOffset);	47	int64_t curNcFactor, int64_t curDoFactor, int64_t curHoFactor, int64_t curWoFactor, int64_t yGmOffset);
47	- __aicore__ inline void OutTranspose(
48	- LocalTensor<float> xLocalTrans, LocalTensor<float> xLocal, int32_t rowNum, int32_t colNum);
49		48
50	TQue<QuePosition::VECIN, 1> inputQue;	49	TQue<QuePosition::VECIN, 1> inputQue;
51	TQue<QuePosition::VECOUT, 1> yQue;	50	TQue<QuePosition::VECOUT, 1> yQue;
	@@ -53,42 +52,7 @@ private:
53	TBuf<> mulWBuffer;	52	TBuf<> mulWBuffer;
54	GlobalTensor<T> xGm;	53	GlobalTensor<T> xGm;
55	GlobalTensor<T> yGm;	54	GlobalTensor<T> yGm;
56	- uint32_t cBlockIdx = 0;	55	+ Pool3dMemCommon::PoolVars poolVars;
57	-
58	- int64_t N = 1;
59	- int64_t C = 1;
60	- int64_t Di = 1;
61	- int64_t Hi = 1;
62	- int64_t Wi = 1;
63	- int64_t Do = 1;
64	- int64_t Ho = 1;
65	- int64_t Wo = 1;
66	- int64_t DiHiWi = 1;
67	- int64_t HiWi = 1;
68	- const int32_t VL_NUM = 64; // Vector calculate length / float size
69	-
70	- // 多核切分的整尾块
71	- int64_t ncFactor = 0;
72	- int64_t doFactor = 0;
73	- int64_t hoFactor = 0;
74	- int64_t woFactor = 0;
75	- int64_t ncTail = 0;
76	- int64_t doTail = 0;
77	- int64_t hoTail = 0;
78	- int64_t woTail = 0;
79	-
80	- // 多核切分的数量
81	- int64_t ncOuter = 0;
82	- int64_t doOuter = 0;
83	- int64_t hoOuter = 0;
84	- int64_t woOuter = 0;
85	-
86	- int64_t totalIdx = 0; // 总UB计算块
87	- int64_t blockFactor = 0; // 每个核最多计算的UB块
88	- int64_t useCoreNum = 0; // 使用核数
89	- int64_t blockTail = 0; // 多核尾块
90	- int64_t beginIdx = 0; // 当前核计算块起始id
91	- int64_t endIdx = 0; // 当前核计算块终止id
92	int64_t kernelsize = 0;	56	int64_t kernelsize = 0;
93	int64_t kW = 0;	57	int64_t kW = 0;
94	int64_t kH = 0;	58	int64_t kH = 0;
	@@ -116,30 +80,16 @@ private:
116		80
117	template <typename T>	81	template <typename T>
118	__aicore__ inline void AvgPool3dNormal<T>::InitTiling(const AvgPool3DTilingData* __restrict__ tiling) {	82	__aicore__ inline void AvgPool3dNormal<T>::InitTiling(const AvgPool3DTilingData* __restrict__ tiling) {
119	- useCoreNum = tiling->useCoreNum;	83	+ poolVars.useCoreNum = tiling->useCoreNum;
120	- N = tiling->inN;	84	+ poolVars.N = tiling->inN;
121	- C = tiling->inC;	85	+ poolVars.C = tiling->inC;
122	- Di = tiling->inD;	86	+ poolVars.Di = tiling->inD;
123	- Hi = tiling->inH;	87	+ poolVars.Hi = tiling->inH;
124	- Wi = tiling->inW;	88	+ poolVars.Wi = tiling->inW;
125	- Do = tiling->outD;	89	+ poolVars.Do = tiling->outD;
126	- Ho = tiling->outH;	90	+ poolVars.Ho = tiling->outH;
127	- Wo = tiling->outW;	91	+ poolVars.Wo = tiling->outW;
128	- totalIdx = tiling->totalIdx;	92	+ Pool3dMemCommon::InitPoolVars(poolVars, tiling);
129	- blockFactor = tiling->blockFactor;
130	- blockTail = tiling->blockTail;
131	- ncFactor = tiling->ncFactor;
132	- woFactor = tiling->woFactor;
133	- hoFactor = tiling->hoFactor;
134	- doFactor = tiling->doFactor;
135	- doOuter = tiling->doOuter;
136	- doTail = tiling->doTail;
137	- hoOuter = tiling->hoOuter;
138	- hoTail = tiling->hoTail;
139	- woOuter = tiling->woOuter;
140	- woTail = tiling->woTail;
141	- ncOuter = tiling->ncOuter;
142	- ncTail = tiling->ncTail;
143	kernelsize = tiling->kD * tiling->kH * tiling->kW;	93	kernelsize = tiling->kD * tiling->kH * tiling->kW;
144	kW = tiling->kW;	94	kW = tiling->kW;
145	kD = tiling->kD;	95	kD = tiling->kD;
	@@ -162,110 +112,33 @@ __aicore__ inline void AvgPool3dNormal<T>::Init(
162	GM_ADDR x, GM_ADDR y, GM_ADDR workspace, const AvgPool3DTilingData* __restrict__ tiling, TPipe* pipe) {	112	GM_ADDR x, GM_ADDR y, GM_ADDR workspace, const AvgPool3DTilingData* __restrict__ tiling, TPipe* pipe) {
163	InitTiling(tiling);	113	InitTiling(tiling);
164		114
165	- cBlockIdx = GetBlockIdx();	115	+ poolVars.cBlockIdx = GetBlockIdx();
166	- if (cBlockIdx >= useCoreNum) {	116	+ if (poolVars.cBlockIdx >= poolVars.useCoreNum) {
167	return;	117	return;
168	}	118	}
169		119
170	- DiHiWi = Di * Hi * Wi;	120	+ poolVars.DiHiWi = poolVars.Di * poolVars.Hi * poolVars.Wi;
171	- HiWi = Hi * Wi;	121	+ poolVars.HiWi = poolVars.Hi * poolVars.Wi;
172	- int64_t calBlockNum = blockFactor;	122	+ int64_t calBlockNum = (poolVars.cBlockIdx == poolVars.useCoreNum - 1) ? poolVars.blockTail : poolVars.blockFactor;
173	- if (cBlockIdx == useCoreNum - 1) {	123	+ poolVars.beginIdx = poolVars.cBlockIdx * poolVars.blockFactor;
174	- calBlockNum = blockTail;	124	+ poolVars.endIdx = poolVars.cBlockIdx * poolVars.blockFactor + calBlockNum;
175	- }
176	- beginIdx = cBlockIdx * blockFactor;
177	- endIdx = cBlockIdx * blockFactor + calBlockNum;
178	xGm.SetGlobalBuffer((__gm__ T*)x);	125	xGm.SetGlobalBuffer((__gm__ T*)x);
179	yGm.SetGlobalBuffer((__gm__ T*)y);	126	yGm.SetGlobalBuffer((__gm__ T*)y);
180		127
181	// 初始化que	128	// 初始化que
182	- pipe->InitBuffer(inputQue, 1, 32 * 1024); // VL_NUMdiFactorhiFactorwiFactorAlignsizeof(T) 有问题？	129	+ pipe->InitBuffer(inputQue, 1, 32 * 1024); // VL_NUMdiFactorhiFactorwiFactorAlignsizeof(T)
183	pipe->InitBuffer(yQue, 1, 8 * 1024); // VL_NUMdoFactorhoFactorwoFactorAlignsizeof(T)	130	pipe->InitBuffer(yQue, 1, 8 * 1024); // VL_NUMdoFactorhoFactorwoFactorAlignsizeof(T)
184	pipe->InitBuffer(inputTransBuffer, 64 * 1024); // VL_NUMdiFactorhiFactorwiFactorAlignsizeof(float)	131	pipe->InitBuffer(inputTransBuffer, 64 * 1024); // VL_NUMdiFactorhiFactorwiFactorAlignsizeof(float)
185	pipe->InitBuffer(mulWBuffer, 64 * 1024); // VL_NUMdiFactorhiFactorwiFactor16Alignsizeof(float)	132	pipe->InitBuffer(mulWBuffer, 64 * 1024); // VL_NUMdiFactorhiFactorwiFactor16Alignsizeof(float)
186	}	133	}
187		134
188	-template <typename T>
189	-__aicore__ inline void AvgPool3dNormal<T>::OutTranspose(
190	- LocalTensor<float> xLocalTrans, LocalTensor<float> xLocal, int32_t rowNum, int32_t colNum) {
191	- LocalTensor<float> dstList[16];
192	- LocalTensor<float> srcList[16];
193	-
194	- event_t eventVS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S));
195	- event_t eventSV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V));
196	-
197	- TransDataTo5HDParams transDataParams;
198	- transDataParams.dstHighHalf = false;
199	- transDataParams.srcHighHalf = false;
200	- if (colNum == 8) {
201	- transDataParams.repeatTimes = rowNum / 16;
202	- transDataParams.dstRepStride = 2;
203	- transDataParams.srcRepStride = 16;
204	-
205	- for (int32_t i = 0; i < 16; i++) {
206	- srcList[i] = xLocal[i * 8];
207	- }
208	-
209	- for (int32_t i = 0; i < 8; i++) {
210	- dstList[i * 2] = xLocalTrans[i * rowNum];
211	- dstList[i * 2 + 1] = xLocalTrans[i * rowNum + 8];
212	- }
213	-
214	- SetFlag<HardEvent::S_V>(eventSV);
215	- WaitFlag<HardEvent::S_V>(eventSV);
216	- TransDataTo5HD<float>(dstList, srcList, transDataParams);
217	- SetFlag<HardEvent::V_S>(eventVS);
218	- WaitFlag<HardEvent::V_S>(eventVS);
219	- } else {
220	- transDataParams.repeatTimes = colNum / 8;
221	- transDataParams.dstRepStride = rowNum;
222	- transDataParams.srcRepStride = 1;
223	- for (int32_t j = 0; j < rowNum / 16; j++) {
224	- for (int32_t i = 0; i < 16; i++) {
225	- srcList[i] = xLocal[i * colNum + j * 16 * colNum];
226	- }
227	-
228	- for (int32_t i = 0; i < 8; i++) {
229	- dstList[i * 2] = xLocalTrans[i * rowNum + j * 16];
230	- dstList[i * 2 + 1] = xLocalTrans[i * rowNum + 8 + j * 16];
231	- }
232	-
233	- SetFlag<HardEvent::S_V>(eventSV);
234	- WaitFlag<HardEvent::S_V>(eventSV);
235	- TransDataTo5HD<float>(dstList, srcList, transDataParams);
236	- SetFlag<HardEvent::V_S>(eventVS);
237	- WaitFlag<HardEvent::V_S>(eventVS);
238	- }
239	- }
240	-}
241	/*	135	/*
242	* 功能：input类型转换 <T> -> <fp32>类型, 并转置，把[VL, D, H, W] 转为[D, H, W, VL]	136	* 功能：input类型转换 <T> -> <fp32>类型, 并转置，把[VL, D, H, W] 转为[D, H, W, VL]
243	*/	137	*/
244	template <typename T>	138	template <typename T>
245	__aicore__ inline void AvgPool3dNormal<T>::TransInput(	139	__aicore__ inline void AvgPool3dNormal<T>::TransInput(
246	int64_t curNcFactor, const uint8_t diFactor, const uint8_t hiFactor, const uint8_t wiFactor) {	140	int64_t curNcFactor, const uint8_t diFactor, const uint8_t hiFactor, const uint8_t wiFactor) {
247	- const uint8_t wiFactor16Align = Ceil(wiFactor, 32 / sizeof(T)) * 32 / sizeof(T);	141	+ Pool3dMemCommon::TransposeInput<T, 1>(inputQue, inputTransBuffer, mulWBuffer, curNcFactor, diFactor, hiFactor, wiFactor, poolVars);
248	- const uint8_t wiFactorAlign = Ceil(wiFactor, 8) * 8;
249	- LocalTensor<T> xLocal = inputQue.DeQue<T>();
250	- LocalTensor<float> xLocalTransVL = inputTransBuffer.Get<float>();
251	- if constexpr (IsSameType<T, float>::value) {
252	- OutTranspose(xLocalTransVL, xLocal, VL_NUM, diFactor * hiFactor * wiFactorAlign);
253	- } else {
254	- LocalTensor<float> xLocalCast = mulWBuffer.Get<float>();
255	- UnaryRepeatParams repeatCastParams{(uint16_t)(wiFactorAlign / 8), (uint16_t)(wiFactor16Align / 8),
256	- (uint8_t)(wiFactorAlign / 8 * Ceil(diFactor * hiFactor, 2)),
257	- (uint8_t)(wiFactor16Align / 8 * Ceil(diFactor * hiFactor, 2))};
258	-
259	- Cast(xLocalCast, xLocal, RoundMode::CAST_NONE, wiFactor16Align * curNcFactor * diFactor * hiFactor);
260	- AscendC::PipeBarrier<PIPE_V>();
261	- Adds(xLocalCast, xLocalCast, float(0.0), uint8_t(wiFactorAlign * Ceil(diFactor * hiFactor, 2)), curNcFactor * 2,
262	- repeatCastParams);
263	-
264	- AscendC::PipeBarrier<PIPE_V>();
265	- OutTranspose(xLocalTransVL, xLocalCast, VL_NUM, diFactor * hiFactor * wiFactorAlign);
266	- }
267	- AscendC::PipeBarrier<PIPE_V>();
268	- inputQue.FreeTensor(xLocal);
269	}	142	}
270		143
271	/*	144	/*
	@@ -282,23 +155,23 @@ __aicore__ inline void AvgPool3dNormal<T>::TransOut(int64_t curDoFactor, int64_t
282	LocalTensor<float> mulDUb = mulWBuffer.Get<float>();	155	LocalTensor<float> mulDUb = mulWBuffer.Get<float>();
283	LocalTensor<float> mulHUb = inputTransBuffer.Get<float>();	156	LocalTensor<float> mulHUb = inputTransBuffer.Get<float>();
284	if constexpr (IsSameType<T, float>::value) {	157	if constexpr (IsSameType<T, float>::value) {
285	- OutTranspose(yLocal, mulDUb, Ceil(curDoFactor * curHoFactor * curWoFactorAlign, 16) * 16, VL_NUM);	158	+ Pool3dMemCommon::OutTranspose<T>(yLocal, mulDUb, Ceil(curDoFactor * curHoFactor * curWoFactorAlign, 16) * 16, poolVars.VL_NUM);
286	} else {	159	} else {
287	if (curWoFactorAlign == curWoFactorAlign16) {	160	if (curWoFactorAlign == curWoFactorAlign16) {
288	- OutTranspose(mulHUb, mulDUb, Ceil(curDoFactor * curHoFactor * curWoFactorAlign, 16) * 16, VL_NUM);	161	+ Pool3dMemCommon::OutTranspose<T>(mulHUb, mulDUb, Ceil(curDoFactor * curHoFactor * curWoFactorAlign, 16) * 16, poolVars.VL_NUM);
289	} else {	162	} else {
290	UnaryRepeatParams repeatCastParams2{(uint16_t)(curWoFactorAlign16 / 8), (uint16_t)(curWoFactorAlign / 8),	163	UnaryRepeatParams repeatCastParams2{(uint16_t)(curWoFactorAlign16 / 8), (uint16_t)(curWoFactorAlign / 8),
291	(uint8_t)(Ceil(curDoFactor * curHoFactor * curWoFactorAlign16, 16) * 2),	164	(uint8_t)(Ceil(curDoFactor * curHoFactor * curWoFactorAlign16, 16) * 2),
292	(uint8_t)(Ceil(curDoFactor * curHoFactor * curWoFactorAlign, 16) * 2)};	165	(uint8_t)(Ceil(curDoFactor * curHoFactor * curWoFactorAlign, 16) * 2)};
293	- OutTranspose(mulHUb[4096], mulDUb, Ceil(curDoFactor * curHoFactor * curWoFactorAlign, 16) * 16, VL_NUM);	166	+ Pool3dMemCommon::OutTranspose<T>(mulHUb[4096], mulDUb, Ceil(curDoFactor * curHoFactor * curWoFactorAlign, 16) * 16, poolVars.VL_NUM);
294	AscendC::PipeBarrier<PIPE_V>();	167	AscendC::PipeBarrier<PIPE_V>();
295		168
296	- Adds(mulHUb, mulHUb[4096], (float)0.0, (uint8_t)(curWoFactorAlign * curDoFactor * curHoFactor), VL_NUM,	169	+ Adds(mulHUb, mulHUb[4096], (float)0.0, (uint8_t)(curWoFactorAlign * curDoFactor * curHoFactor), poolVars.VL_NUM,
297	repeatCastParams2);	170	repeatCastParams2);
298	}	171	}
299	AscendC::PipeBarrier<PIPE_V>();	172	AscendC::PipeBarrier<PIPE_V>();
300		173
301	- Cast(yLocal, mulHUb, RoundMode::CAST_ROUND, VL_NUM * curWoFactorAlign16 * curDoFactor * curHoFactor);	174	+ Cast(yLocal, mulHUb, RoundMode::CAST_ROUND, poolVars.VL_NUM * curWoFactorAlign16 * curDoFactor * curHoFactor);
302	}	175	}
303	yQue.EnQue(yLocal);	176	yQue.EnQue(yLocal);
304	}	177	}
	@@ -307,17 +180,14 @@ template <typename T>
307	__aicore__ inline void AvgPool3dNormal<T>::CopyOut(	180	__aicore__ inline void AvgPool3dNormal<T>::CopyOut(
308	int64_t curNcFactor, int64_t curDoFactor, int64_t curHoFactor, int64_t curWoFactor, int64_t yGmOffset) {	181	int64_t curNcFactor, int64_t curDoFactor, int64_t curHoFactor, int64_t curWoFactor, int64_t yGmOffset) {
309	auto curWoFactorAlign16 = Ceil(curWoFactor, 32 / sizeof(T)) * 32 / sizeof(T);	182	auto curWoFactorAlign16 = Ceil(curWoFactor, 32 / sizeof(T)) * 32 / sizeof(T);
310	-
311	LocalTensor<T> yLocal = yQue.DeQue<T>();	183	LocalTensor<T> yLocal = yQue.DeQue<T>();
312		184
313	- DataCopyExtParams paramsOut2;	185	+ DataCopyExtParams paramsOut2 = {
314	- paramsOut2.blockCount = curHoFactor;	186	+ static_cast<uint16_t>(curHoFactor), static_cast<uint32_t>(curWoFactor * sizeof(T)), static_cast<uint32_t>(0),
315	- paramsOut2.blockLen = curWoFactor * sizeof(T);	187	+ static_cast<uint32_t>((poolVars.Wo - curWoFactor) * sizeof(T)), static_cast<uint32_t>(0)};
316	- paramsOut2.srcStride = 0;
317	- paramsOut2.dstStride = (Wo - curWoFactor) * sizeof(T);
318	for (int64_t ncCopyi = 0; ncCopyi < curNcFactor; ncCopyi++) {	188	for (int64_t ncCopyi = 0; ncCopyi < curNcFactor; ncCopyi++) {
319	for (int64_t dCopyi = 0; dCopyi < curDoFactor; dCopyi++) {	189	for (int64_t dCopyi = 0; dCopyi < curDoFactor; dCopyi++) {
320	- auto dstAddr = yGmOffset + ncCopyi * Do * Ho * Wo + dCopyi * Ho * Wo;	190	+ auto dstAddr = yGmOffset + ncCopyi * poolVars.Do * poolVars.Ho * poolVars.Wo + dCopyi * poolVars.Ho * poolVars.Wo;
321	auto srcAddr = ncCopyi * Ceil(curDoFactor * curHoFactor * curWoFactorAlign16, 16) * 16 +	191	auto srcAddr = ncCopyi * Ceil(curDoFactor * curHoFactor * curWoFactorAlign16, 16) * 16 +
322	dCopyi * curHoFactor * curWoFactorAlign16;	192	dCopyi * curHoFactor * curWoFactorAlign16;
323	DataCopyPad(yGm[dstAddr], yLocal[srcAddr], paramsOut2);	193	DataCopyPad(yGm[dstAddr], yLocal[srcAddr], paramsOut2);
	@@ -335,12 +205,12 @@ __aicore__ inline void AvgPool3dNormal<T>::AvgPoolW(
335	const uint8_t wiFactorAlign = Ceil(wiFactor, 8) * 8;	205	const uint8_t wiFactorAlign = Ceil(wiFactor, 8) * 8;
336	uint64_t mask = 256 / sizeof(float);	206	uint64_t mask = 256 / sizeof(float);
337	auto repeat = hiFactor * diFactor;	207	auto repeat = hiFactor * diFactor;
338	- UnaryRepeatParams repeatCopyParams{1, 1, 8, (uint8_t)(VL_NUM / 8 * wiFactorAlign)};	208	+ UnaryRepeatParams repeatCopyParams{1, 1, 8, (uint8_t)(poolVars.VL_NUM / 8 * wiFactorAlign)};
339	BinaryRepeatParams repeatParams{1, 1, 1, 8, 8, (uint8_t)(8 * wiFactorAlign)};	209	BinaryRepeatParams repeatParams{1, 1, 1, 8, 8, (uint8_t)(8 * wiFactorAlign)};
340	LocalTensor<float> mulWUb = mulWBuffer.Get<float>();	210	LocalTensor<float> mulWUb = mulWBuffer.Get<float>();
341		211
342	for (int kernelIdx = 0; kernelIdx < curWoFactor; kernelIdx++) {	212	for (int kernelIdx = 0; kernelIdx < curWoFactor; kernelIdx++) {
343	- int32_t kerWEndIdx = Min(Wi, wEnd + kernelIdx * dW);	213	+ int32_t kerWEndIdx = Min(poolVars.Wi, wEnd + kernelIdx * dW);
344	int32_t kerWStartIdx = Max(wStart + kernelIdx * dW, kerWEndIdx - kW);	214	int32_t kerWStartIdx = Max(wStart + kernelIdx * dW, kerWEndIdx - kW);
345	if (wStart == 0) {	215	if (wStart == 0) {
346	kerWStartIdx = Max(wStart, wEnd + kernelIdx * dW - kW);	216	kerWStartIdx = Max(wStart, wEnd + kernelIdx * dW - kW);
	@@ -349,19 +219,19 @@ __aicore__ inline void AvgPool3dNormal<T>::AvgPoolW(
349	if(wStart == 0) {	219	if(wStart == 0) {
350	kerWStartIdx = Max(wEnd + kernelIdx * dW - kW, 0);	220	kerWStartIdx = Max(wEnd + kernelIdx * dW - kW, 0);
351	} else {	221	} else {
352	- kerWStartIdx = Min(wStart + kernelIdx * dW, Wi);	222	+ kerWStartIdx = Min(wStart + kernelIdx * dW, poolVars.Wi);
353	}	223	}
354	if(curWoFactor == 1) {	224	if(curWoFactor == 1) {
355	kerWEndIdx = wEnd;	225	kerWEndIdx = wEnd;
356	kerWStartIdx = wStart;	226	kerWStartIdx = wStart;
357	}	227	}
358	}	228	}
359	- auto mulWOffset = kernelIdx * diFactor * hiFactor * VL_NUM;	229	+ auto mulWOffset = kernelIdx * diFactor * hiFactor * poolVars.VL_NUM;
360	- auto inputOffset = VL_NUM * (kerWStartIdx - wStart);	230	+ auto inputOffset = poolVars.VL_NUM * (kerWStartIdx - wStart);
361	- Adds(mulWUb[mulWOffset], xLocalTransVL[inputOffset], (float)0.0, VL_NUM, repeat, repeatCopyParams);	231	+ Adds(mulWUb[mulWOffset], xLocalTransVL[inputOffset], (float)0.0, poolVars.VL_NUM, repeat, repeatCopyParams);
362	AscendC::PipeBarrier<PIPE_V>();	232	AscendC::PipeBarrier<PIPE_V>();
363	for (int i = kerWStartIdx + 1; i < kerWEndIdx; i++) {	233	for (int i = kerWStartIdx + 1; i < kerWEndIdx; i++) {
364	- auto nexAddOffset = VL_NUM * (i - wStart);	234	+ auto nexAddOffset = poolVars.VL_NUM * (i - wStart);
365	Add(mulWUb[mulWOffset], mulWUb[mulWOffset], xLocalTransVL[nexAddOffset], mask, repeat, repeatParams);	235	Add(mulWUb[mulWOffset], mulWUb[mulWOffset], xLocalTransVL[nexAddOffset], mask, repeat, repeatParams);
366	AscendC::PipeBarrier<PIPE_V>();	236	AscendC::PipeBarrier<PIPE_V>();
367	}	237	}
	@@ -375,13 +245,13 @@ __aicore__ inline void AvgPool3dNormal<T>::AvgPoolH(
375		245
376	uint64_t mask = 256 / sizeof(float);	246	uint64_t mask = 256 / sizeof(float);
377	auto repeat = woFactorAlign * diFactor;	247	auto repeat = woFactorAlign * diFactor;
378	- UnaryRepeatParams repeatCopyParams{1, 1, 8, (uint8_t)(VL_NUM / 8 * hiFactor)};	248	+ UnaryRepeatParams repeatCopyParams{1, 1, 8, (uint8_t)(poolVars.VL_NUM / 8 * hiFactor)};
379	BinaryRepeatParams repeatParams{1, 1, 1, 8, 8, (uint8_t)(8 * hiFactor)};	249	BinaryRepeatParams repeatParams{1, 1, 1, 8, 8, (uint8_t)(8 * hiFactor)};
380	LocalTensor<float> mulWUb = mulWBuffer.Get<float>();	250	LocalTensor<float> mulWUb = mulWBuffer.Get<float>();
381	LocalTensor<float> mulHUb = inputTransBuffer.Get<float>();	251	LocalTensor<float> mulHUb = inputTransBuffer.Get<float>();
382		252
383	for (int kernelIdx = 0; kernelIdx < curHoFactor; kernelIdx++) {	253	for (int kernelIdx = 0; kernelIdx < curHoFactor; kernelIdx++) {
384	- int32_t kerHEndIdx = Min(Hi, hEnd + kernelIdx * dH);	254	+ int32_t kerHEndIdx = Min(poolVars.Hi, hEnd + kernelIdx * dH);
385	int32_t kerHStartIdx = Max(hStart + kernelIdx * dH, kerHEndIdx - kH);	255	int32_t kerHStartIdx = Max(hStart + kernelIdx * dH, kerHEndIdx - kH);
386	if (hStart == 0) {	256	if (hStart == 0) {
387	kerHStartIdx = Max(hStart, hEnd + kernelIdx * dH -kH);	257	kerHStartIdx = Max(hStart, hEnd + kernelIdx * dH -kH);
	@@ -390,19 +260,19 @@ __aicore__ inline void AvgPool3dNormal<T>::AvgPoolH(
390	if(hStart == 0) {	260	if(hStart == 0) {
391	kerHStartIdx = Max(hEnd + kernelIdx * dH - kH, 0);	261	kerHStartIdx = Max(hEnd + kernelIdx * dH - kH, 0);
392	} else {	262	} else {
393	- kerHStartIdx = Min(hStart + kernelIdx * dH, Hi);	263	+ kerHStartIdx = Min(hStart + kernelIdx * dH, poolVars.Hi);
394	}	264	}
395	if(curHoFactor == 1) {	265	if(curHoFactor == 1) {
396	kerHEndIdx = hEnd;	266	kerHEndIdx = hEnd;
397	kerHStartIdx = hStart;	267	kerHStartIdx = hStart;
398	}	268	}
399	}	269	}
400	- auto mulHOffset = kernelIdx * repeat * VL_NUM;	270	+ auto mulHOffset = kernelIdx * repeat * poolVars.VL_NUM;
401	- auto mulWOffset = VL_NUM * (kerHStartIdx - hStart);	271	+ auto mulWOffset = poolVars.VL_NUM * (kerHStartIdx - hStart);
402	- Adds(mulHUb[mulHOffset], mulWUb[mulWOffset], (float)0.0, VL_NUM, repeat, repeatCopyParams);	272	+ Adds(mulHUb[mulHOffset], mulWUb[mulWOffset], (float)0.0, poolVars.VL_NUM, repeat, repeatCopyParams);
403	AscendC::PipeBarrier<PIPE_V>();	273	AscendC::PipeBarrier<PIPE_V>();
404	for (int i = kerHStartIdx + 1; i < kerHEndIdx; i++) {	274	for (int i = kerHStartIdx + 1; i < kerHEndIdx; i++) {
405	- auto nexAddOffset = VL_NUM * (i - hStart);	275	+ auto nexAddOffset = poolVars.VL_NUM * (i - hStart);
406	Add(mulHUb[mulHOffset], mulHUb[mulHOffset], mulWUb[nexAddOffset], mask, repeat, repeatParams);	276	Add(mulHUb[mulHOffset], mulHUb[mulHOffset], mulWUb[nexAddOffset], mask, repeat, repeatParams);
407	AscendC::PipeBarrier<PIPE_V>();	277	AscendC::PipeBarrier<PIPE_V>();
408	}	278	}
	@@ -416,7 +286,7 @@ __aicore__ inline void AvgPool3dNormal<T>::AvgPoolD(
416		286
417	uint64_t mask = 256 / sizeof(float);	287	uint64_t mask = 256 / sizeof(float);
418	auto repeat = curHoFactor * woFactorAlign;	288	auto repeat = curHoFactor * woFactorAlign;
419	- UnaryRepeatParams repeatCopyParams{1, 1, 8, (uint8_t)(VL_NUM / 8 * diFactor)};	289	+ UnaryRepeatParams repeatCopyParams{1, 1, 8, (uint8_t)(poolVars.VL_NUM / 8 * diFactor)};
420	BinaryRepeatParams repeatParams{1, 1, 1, 8, 8, (uint8_t)(8 * diFactor)};	290	BinaryRepeatParams repeatParams{1, 1, 1, 8, 8, (uint8_t)(8 * diFactor)};
421	LocalTensor<float> mulHUb = inputTransBuffer.Get<float>();	291	LocalTensor<float> mulHUb = inputTransBuffer.Get<float>();
422	LocalTensor<float> mulDUb = mulWBuffer.Get<float>();	292	LocalTensor<float> mulDUb = mulWBuffer.Get<float>();
	@@ -424,10 +294,10 @@ __aicore__ inline void AvgPool3dNormal<T>::AvgPoolD(
424	int32_t kerDEndIdx = dEnd;	294	int32_t kerDEndIdx = dEnd;
425	auto mulDOffset = 0;	295	auto mulDOffset = 0;
426	auto mulHOffset = 0;	296	auto mulHOffset = 0;
427	- Adds(mulDUb[mulDOffset], mulHUb[mulHOffset], (float)0.0, VL_NUM, repeat, repeatCopyParams);	297	+ Adds(mulDUb[mulDOffset], mulHUb[mulHOffset], (float)0.0, poolVars.VL_NUM, repeat, repeatCopyParams);
428	AscendC::PipeBarrier<PIPE_V>();	298	AscendC::PipeBarrier<PIPE_V>();
429	for (int i = 1; i < kerDEndIdx - kerDStartIdx; i++) {	299	for (int i = 1; i < kerDEndIdx - kerDStartIdx; i++) {
430	- auto nexAddOffset = VL_NUM * i;	300	+ auto nexAddOffset = poolVars.VL_NUM * i;
431	Add(mulDUb[mulDOffset], mulDUb[mulDOffset], mulHUb[nexAddOffset], mask, repeat, repeatParams);	301	Add(mulDUb[mulDOffset], mulDUb[mulDOffset], mulHUb[nexAddOffset], mask, repeat, repeatParams);
432	AscendC::PipeBarrier<PIPE_V>();	302	AscendC::PipeBarrier<PIPE_V>();
433	}	303	}
	@@ -438,26 +308,26 @@ __aicore__ inline void AvgPool3dNormal<T>::AvgPoolD(
438		308
439	template <typename T>	309	template <typename T>
440	__aicore__ inline void AvgPool3dNormal<T>::CalcIndex(int64_t index,int64_t baseW, int64_t baseH) {	310	__aicore__ inline void AvgPool3dNormal<T>::CalcIndex(int64_t index,int64_t baseW, int64_t baseH) {
441	- auto indexD = (index / (Ho * Wo)) % Do;	311	+ auto indexD = (index / (poolVars.Ho * poolVars.Wo)) % poolVars.Do;
442	- auto indexH = (index / Wo) % Ho;	312	+ auto indexH = (index / poolVars.Wo) % poolVars.Ho;
443	- auto indexW = index % Wo;	313	+ auto indexW = index % poolVars.Wo;
444	dStart = indexD * dD -padD;	314	dStart = indexD * dD -padD;
445	hStart = indexH * dH -padH;	315	hStart = indexH * dH -padH;
446	wStart = indexW * dW -padW;	316	wStart = indexW * dW -padW;
447	- dEnd = Min(dStart + kD, Di + padD);	317	+ dEnd = Min(dStart + kD, poolVars.Di + padD);
448	- hEnd = Min(hStart + kH, Hi + padH);	318	+ hEnd = Min(hStart + kH, poolVars.Hi + padH);
449	- wEnd = Min(wStart + kW, Wi + padW);	319	+ wEnd = Min(wStart + kW, poolVars.Wi + padW);
450	auto poolSize = (dEnd - dStart) * (hEnd - hStart) * (wEnd - wStart);	320	auto poolSize = (dEnd - dStart) * (hEnd - hStart) * (wEnd - wStart);
451	dStart = Max(dStart , 0);	321	dStart = Max(dStart , 0);
452	hStart = Max(hStart , 0);	322	hStart = Max(hStart , 0);
453	wStart = Max(wStart , 0);	323	wStart = Max(wStart , 0);
454	- dEnd = Min(dEnd, Di);	324	+ dEnd = Min(dEnd, poolVars.Di);
455	- hEnd = Min(hEnd, Hi);	325	+ hEnd = Min(hEnd, poolVars.Hi);
456	- wEnd = Min(wEnd, Wi);	326	+ wEnd = Min(wEnd, poolVars.Wi);
457	kernelsize = (dEnd - dStart) * (hEnd - hStart) * (wEnd - wStart);	327	kernelsize = (dEnd - dStart) * (hEnd - hStart) * (wEnd - wStart);
458	diFactor = dEnd - dStart;	328	diFactor = dEnd - dStart;
459	- hiFactor = Min(Hi, ((hEnd - hStart) + (baseH - 1) * dH));	329	+ hiFactor = Min(poolVars.Hi, ((hEnd - hStart) + (baseH - 1) * dH));
460	- wiFactor = Min(Wi, ((wEnd - wStart) + (baseW - 1) * dW));	330	+ wiFactor = Min(poolVars.Wi, ((wEnd - wStart) + (baseW - 1) * dW));
461	if (divisorOverride) {	331	if (divisorOverride) {
462	mulsFactor = (float)1.0 / static_cast<float>(divisorOverride);	332	mulsFactor = (float)1.0 / static_cast<float>(divisorOverride);
463	} else if (countIncludePad) {	333	} else if (countIncludePad) {
	@@ -469,73 +339,49 @@ __aicore__ inline void AvgPool3dNormal<T>::CalcIndex(int64_t index,int64_t baseW
469		339
470	template <typename T>	340	template <typename T>
471	__aicore__ inline void AvgPool3dNormal<T>::CopyInX(int64_t curNcFactor,int64_t xGmOffset) {	341	__aicore__ inline void AvgPool3dNormal<T>::CopyInX(int64_t curNcFactor,int64_t xGmOffset) {
472	- LocalTensor<T> xLocal = inputQue.AllocTensor<T>();	342	+ Pool3dMemCommon::CopyInputData<T, 1>(inputQue, xGm, curNcFactor, diFactor, hiFactor, wiFactor, xGmOffset, poolVars);
473	- const uint8_t wiFactor16Align = Ceil(wiFactor, 32 / sizeof(T)) * 32 / sizeof(T);
474	- DataCopyPadExtParams<T> padParams{false, 0, 0, 0};
475	- DataCopyExtParams paramsIn;
476	- paramsIn.blockCount = hiFactor;
477	- paramsIn.blockLen = wiFactor * sizeof(T);
478	- paramsIn.srcStride = (Wi - wiFactor) * sizeof(T);
479	- paramsIn.dstStride = 0;
480	- for (int64_t ncCopyi = 0; ncCopyi < curNcFactor; ncCopyi++) {
481	- for (int64_t dCopyi = 0; dCopyi < diFactor; dCopyi++) {
482	- auto srcAddr = xGmOffset + ncCopyi * DiHiWi + dCopyi * HiWi;
483	- auto dstAddr = (ncCopyi * diFactor + dCopyi) * hiFactor * wiFactor16Align;
484	- DataCopyPad(xLocal[dstAddr], xGm[srcAddr], paramsIn, padParams);
485	- }
486	- }
487	- inputQue.EnQue(xLocal);
488	}	343	}
489		344
490	template <typename T>	345	template <typename T>
491	__aicore__ inline void AvgPool3dNormal<T>::Process() {	346	__aicore__ inline void AvgPool3dNormal<T>::Process() {
492	- if (cBlockIdx >= useCoreNum) {	347	+ if (poolVars.cBlockIdx >= poolVars.useCoreNum) {
493	return;	348	return;
494	}	349	}
495	- for (auto curIdx = beginIdx; curIdx < endIdx; curIdx++) {	350	+ for (auto curIdx = poolVars.beginIdx; curIdx < poolVars.endIdx; curIdx++) {
496	- auto curNcIdx = curIdx / (doOuter * hoOuter * woOuter);	351	+ auto blockVar = Pool3dMemCommon::CalcBlockVar(curIdx, poolVars);
497	- auto curNcFactor = curNcIdx == (ncOuter - 1) ? ncTail : ncFactor;
498	- auto tmpIdx = curIdx % (doOuter * hoOuter * woOuter);
499	- auto curDoIdx = tmpIdx / (hoOuter * woOuter);
500	- auto curDoFactor = curDoIdx == (doOuter - 1) ? doTail : doFactor;
501	- tmpIdx = tmpIdx % (hoOuter * woOuter);
502	- auto curHoIdx = tmpIdx / woOuter;
503	- auto curHoFactor = curHoIdx == (hoOuter - 1) ? hoTail : hoFactor;
504	- auto curWoIdx = tmpIdx % woOuter;
505	- auto curWoFactor = curWoIdx == (woOuter - 1) ? woTail : woFactor;
506	auto kernelWMaxAlign = (kD * kH <= 8) ? 16 : 8;	352	auto kernelWMaxAlign = (kD * kH <= 8) ? 16 : 8;
507	auto baseW = ((kernelWMaxAlign - kW) / dW) + 1;	353	auto baseW = ((kernelWMaxAlign - kW) / dW) + 1;
508	auto baseH = ((128 / kD / kernelWMaxAlign) - kH) / dH + 1;	354	auto baseH = ((128 / kD / kernelWMaxAlign) - kH) / dH + 1;
509	auto baseD = 1;	355	auto baseD = 1;
510	- auto ncCoreIdx = curNcIdx * ncFactor;	356	+ auto ncCoreIdx = blockVar.curNcIdx * poolVars.ncFactor;
511	- auto doCoreIdx = curDoIdx * doFactor;	357	+ auto doCoreIdx = blockVar.curDoIdx * poolVars.doFactor;
512	- auto hoCoreIdx = curHoIdx * hoFactor;	358	+ auto hoCoreIdx = blockVar.curHoIdx * poolVars.hoFactor;
513	- auto woCoreIdx = curWoIdx * woFactor;	359	+ auto woCoreIdx = blockVar.curWoIdx * poolVars.woFactor;
514	- auto incoreDCnt = CeilDiv(curDoFactor, baseD);	360	+ auto incoreDCnt = CeilDiv(blockVar.curDoFactor, baseD);
515	- auto incoreHCnt = CeilDiv(curHoFactor, baseH);	361	+ auto incoreHCnt = CeilDiv(blockVar.curHoFactor, baseH);
516	- auto incoreWCnt = CeilDiv(curWoFactor, baseW);	362	+ auto incoreWCnt = CeilDiv(blockVar.curWoFactor, baseW);
517	event_t eventIDMTE3ToMTE2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE3_MTE2));	363	event_t eventIDMTE3ToMTE2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE3_MTE2));
518	for (auto doLoop = 0; doLoop < incoreDCnt; doLoop++) {	364	for (auto doLoop = 0; doLoop < incoreDCnt; doLoop++) {
519	auto doBlockIdx = doCoreIdx + doLoop;	365	auto doBlockIdx = doCoreIdx + doLoop;
520	for (auto hoLoop = 0; hoLoop < incoreHCnt; hoLoop++) {	366	for (auto hoLoop = 0; hoLoop < incoreHCnt; hoLoop++) {
521	auto nowH = baseH;	367	auto nowH = baseH;
522	auto hoBlockIdx = hoCoreIdx + hoLoop * nowH;	368	auto hoBlockIdx = hoCoreIdx + hoLoop * nowH;
523	- nowH = (hoLoop == incoreHCnt - 1) ? (curHoFactor - hoLoop * baseH) : baseH;	369	+ nowH = (hoLoop == incoreHCnt - 1) ? (blockVar.curHoFactor - hoLoop * baseH) : baseH;
524	auto nowW = baseW;	370	auto nowW = baseW;
525	for(auto woLoop = 0; woLoop < incoreWCnt; woLoop++) {	371	for(auto woLoop = 0; woLoop < incoreWCnt; woLoop++) {
526	auto woBlockIdx = woCoreIdx + woLoop * nowW;	372	auto woBlockIdx = woCoreIdx + woLoop * nowW;
527	- auto BlockIdx = doBlockIdx * Ho * Wo + hoBlockIdx * Wo +woBlockIdx;	373	+ auto BlockIdx = doBlockIdx * poolVars.Ho * poolVars.Wo + hoBlockIdx * poolVars.Wo +woBlockIdx;
528	- auto yGmOffset = ncCoreIdx * Do * Ho * Wo + BlockIdx;	374	+ auto yGmOffset = ncCoreIdx * poolVars.Do * poolVars.Ho * poolVars.Wo + BlockIdx;
529	- nowW = (woLoop == incoreWCnt - 1) ? (curWoFactor - woLoop * baseW) : baseW;	375	+ nowW = (woLoop == incoreWCnt - 1) ? (blockVar.curWoFactor - woLoop * baseW) : baseW;
530	CalcIndex(yGmOffset, nowW, nowH);	376	CalcIndex(yGmOffset, nowW, nowH);
531	- auto xGmOffset = ncCoreIdx * DiHiWi + dStart * HiWi + hStart * Wi + wStart;	377	+ auto xGmOffset = ncCoreIdx * poolVars.DiHiWi + dStart * poolVars.HiWi + hStart * poolVars.Wi + wStart;
532	- CopyInX(curNcFactor, xGmOffset);	378	+ CopyInX(blockVar.curNcFactor, xGmOffset);
533	- TransInput(curNcFactor, diFactor, hiFactor, wiFactor);	379	+ TransInput(blockVar.curNcFactor, diFactor, hiFactor, wiFactor);
534	AvgPoolW(diFactor, hiFactor, wiFactor, nowW);	380	AvgPoolW(diFactor, hiFactor, wiFactor, nowW);
535	AvgPoolH(diFactor, hiFactor, nowW, nowH);	381	AvgPoolH(diFactor, hiFactor, nowW, nowH);
536	AvgPoolD(diFactor, hiFactor, nowW, nowH);	382	AvgPoolD(diFactor, hiFactor, nowW, nowH);
537	TransOut(baseD, nowH, nowW);	383	TransOut(baseD, nowH, nowW);
538	- CopyOut(curNcFactor, baseD, nowH, nowW, yGmOffset);	384	+ CopyOut(blockVar.curNcFactor, baseD, nowH, nowW, yGmOffset);
539	SetFlag<HardEvent::MTE3_MTE2>(eventIDMTE3ToMTE2);	385	SetFlag<HardEvent::MTE3_MTE2>(eventIDMTE3ToMTE2);
540	WaitFlag<HardEvent::MTE3_MTE2>(eventIDMTE3ToMTE2);	386	WaitFlag<HardEvent::MTE3_MTE2>(eventIDMTE3ToMTE2);
541	}	387	}

Apooling/pool_3d_common/op_kernel/arch32/pool_3d_memory_optimized_utils.h+230-0

@@ -0,0 +1,230 @@
	1	+/**
	2	+ * Copyright (c) 2025 Huawei Technologies Co., Ltd.
	3	+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
	4	+ * CANN Open Software License Agreement Version 2.0 (the "License").
	5	+ * Please refer to the License for details. You may not use this file except in compliance with the License.
	6	+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
	7	+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
	8	+ * See LICENSE in the root of the software repository for the full text of the License.
	9	+ */
	10	+
	11	+/*!
	12	+ * \file pool_3d_memory_optimized_utils.h
	13	+ * \brief
	14	+ */
	15	+
	16	+#ifndef POOL_3D_MEMORY_OPTIMIZED_UTILS_H_
	17	+#define POOL_3D_MEMORY_OPTIMIZED_UTILS_H_
	18	+
	19	+#include "kernel_tiling/kernel_tiling.h"
	20	+
	21	+namespace Pool3dMemCommon {
	22	+using namespace AscendC;
	23	+struct PoolVars {
	24	+ uint32_t cBlockIdx = 0;
	25	+
	26	+ int64_t N = 1;
	27	+ int64_t C = 1;
	28	+ int64_t Di = 1;
	29	+ int64_t Hi = 1;
	30	+ int64_t Wi = 1;
	31	+ int64_t Do = 1;
	32	+ int64_t Ho = 1;
	33	+ int64_t Wo = 1;
	34	+ int64_t DiHiWi = 1;
	35	+ int64_t HiWi = 1;
	36	+ const int32_t VL_NUM = 64; // Vector calculate length / float size
	37	+
	38	+ // 多核切分的整尾块
	39	+ int64_t ncFactor = 0;
	40	+ int64_t doFactor = 0;
	41	+ int64_t hoFactor = 0;
	42	+ int64_t woFactor = 0;
	43	+ int64_t ncTail = 0;
	44	+ int64_t doTail = 0;
	45	+ int64_t hoTail = 0;
	46	+ int64_t woTail = 0;
	47	+
	48	+ // 多核切分的数量
	49	+ int64_t ncOuter = 0;
	50	+ int64_t doOuter = 0;
	51	+ int64_t hoOuter = 0;
	52	+ int64_t woOuter = 0;
	53	+
	54	+ int64_t totalIdx = 0; // 总UB计算块
	55	+ int64_t blockFactor = 0; // 每个核最多计算的UB块
	56	+ int64_t useCoreNum = 0; // 使用核数
	57	+ int64_t blockTail = 0; // 多核尾块
	58	+
	59	+ int64_t beginIdx = 0; // 当前核计算块起始id
	60	+ int64_t endIdx = 0; // 当前核计算块终止id
	61	+};
	62	+
	63	+struct BlockVar {
	64	+ int64_t curNcIdx = 0; // 当前nc块索引
	65	+ int64_t curNcFactor = 0; // 当前nc块因子
	66	+ int64_t curDoIdx = 0; // 当前do块索引
	67	+ int64_t curDoFactor = 0; // 当前do块因子
	68	+ int64_t curHoIdx = 0; // 当前ho块索引
	69	+ int64_t curHoFactor = 0; // 当前ho块因子
	70	+ int64_t curWoIdx = 0; // 当前wo块索引
	71	+ int64_t curWoFactor = 0; // 当前wo块因子
	72	+};
	73	+
	74	+template <typename TilingDataT>
	75	+__aicore__ inline void InitPoolVars(PoolVars& poolVars, const TilingDataT* __restrict__ tiling) {
	76	+ poolVars.totalIdx = tiling->totalIdx;
	77	+ poolVars.blockFactor = tiling->blockFactor;
	78	+ poolVars.blockTail = tiling->blockTail;
	79	+ poolVars.ncFactor = tiling->ncFactor;
	80	+ poolVars.woFactor = tiling->woFactor;
	81	+ poolVars.hoFactor = tiling->hoFactor;
	82	+ poolVars.doFactor = tiling->doFactor;
	83	+ poolVars.doOuter = tiling->doOuter;
	84	+ poolVars.doTail = tiling->doTail;
	85	+ poolVars.hoOuter = tiling->hoOuter;
	86	+ poolVars.hoTail = tiling->hoTail;
	87	+ poolVars.woOuter = tiling->woOuter;
	88	+ poolVars.woTail = tiling->woTail;
	89	+ poolVars.ncOuter = tiling->ncOuter;
	90	+ poolVars.ncTail = tiling->ncTail;
	91	+}
	92	+
	93	+__aicore__ inline BlockVar
	94	+CalcBlockVar(int64_t curIdx, const PoolVars& poolVars) {
	95	+ BlockVar indices;
	96	+ indices.curNcIdx = curIdx / (poolVars.doOuter * poolVars.hoOuter * poolVars.woOuter);
	97	+ indices.curNcFactor = indices.curNcIdx == (poolVars.ncOuter - 1) ? poolVars.ncTail : poolVars.ncFactor;
	98	+ auto tmpIdx = curIdx % (poolVars.doOuter * poolVars.hoOuter * poolVars.woOuter);
	99	+ indices.curDoIdx = tmpIdx / (poolVars.hoOuter * poolVars.woOuter);
	100	+ indices.curDoFactor = indices.curDoIdx == (poolVars.doOuter - 1) ? poolVars.doTail : poolVars.doFactor;
	101	+ tmpIdx = tmpIdx % (poolVars.hoOuter * poolVars.woOuter);
	102	+ indices.curHoIdx = tmpIdx / poolVars.woOuter;
	103	+ indices.curHoFactor = indices.curHoIdx == (poolVars.hoOuter - 1) ? poolVars.hoTail : poolVars.hoFactor;
	104	+ indices.curWoIdx = tmpIdx % poolVars.woOuter;
	105	+ indices.curWoFactor = indices.curWoIdx == (poolVars.woOuter - 1) ? poolVars.woTail : poolVars.woFactor;
	106	+ return indices;
	107	+}
	108	+
	109	+template <typename T, int32_t QUEUE_DEPTH>
	110	+__aicore__ inline void CopyInputData(
	111	+ TQue<QuePosition::VECIN, QUEUE_DEPTH>& inputQue,
	112	+ GlobalTensor<T>& xGm,
	113	+ int64_t curNcFactor,
	114	+ uint8_t diFactor,
	115	+ uint8_t hiFactor,
	116	+ uint8_t wiFactor,
	117	+ int64_t xGmOffset,
	118	+ const PoolVars& poolVars) {
	119	+ LocalTensor<T> xLocal = inputQue.template AllocTensor<T>();
	120	+ const uint8_t wiFactor16Align = Ceil(wiFactor, 32 / sizeof(T)) * 32 / sizeof(T);
	121	+ DataCopyPadExtParams<T> padParams{false, 0, 0, 0};
	122	+ DataCopyExtParams paramsIn;
	123	+ paramsIn.blockCount = hiFactor;
	124	+ paramsIn.blockLen = wiFactor * sizeof(T);
	125	+ paramsIn.srcStride = (poolVars.Wi - wiFactor) * sizeof(T);
	126	+ paramsIn.dstStride = 0;
	127	+ for (int64_t ncCopyi = 0; ncCopyi < curNcFactor; ncCopyi++) {
	128	+ for (int64_t dCopyi = 0; dCopyi < diFactor; dCopyi++) {
	129	+ auto srcAddr = xGmOffset + ncCopyi * poolVars.DiHiWi + dCopyi * poolVars.HiWi;
	130	+ auto dstAddr = (ncCopyi * diFactor + dCopyi) * hiFactor * wiFactor16Align;
	131	+ DataCopyPad(xLocal[dstAddr], xGm[srcAddr], paramsIn, padParams);
	132	+ }
	133	+ }
	134	+ inputQue.EnQue(xLocal);
	135	+}
	136	+
	137	+
	138	+template <typename T>
	139	+__aicore__ inline void OutTranspose(
	140	+ LocalTensor<float> xLocalTrans, LocalTensor<float> xLocal, int32_t rowNum, int32_t colNum) {
	141	+ LocalTensor<float> dstList[16];
	142	+ LocalTensor<float> srcList[16];
	143	+
	144	+ event_t eventVS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S));
	145	+ event_t eventSV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V));
	146	+
	147	+ TransDataTo5HDParams transDataParams;
	148	+ transDataParams.dstHighHalf = false;
	149	+ transDataParams.srcHighHalf = false;
	150	+ if (colNum == 8) {
	151	+ transDataParams.repeatTimes = rowNum / 16;
	152	+ transDataParams.dstRepStride = 2;
	153	+ transDataParams.srcRepStride = 16;
	154	+
	155	+ for (int32_t i = 0; i < 16; i++) {
	156	+ srcList[i] = xLocal[i * 8];
	157	+ }
	158	+
	159	+ for (int32_t i = 0; i < 8; i++) {
	160	+ dstList[i * 2] = xLocalTrans[i * rowNum];
	161	+ dstList[i * 2 + 1] = xLocalTrans[i * rowNum + 8];
	162	+ }
	163	+
	164	+ SetFlag<HardEvent::S_V>(eventSV);
	165	+ WaitFlag<HardEvent::S_V>(eventSV);
	166	+ TransDataTo5HD<float>(dstList, srcList, transDataParams);
	167	+ SetFlag<HardEvent::V_S>(eventVS);
	168	+ WaitFlag<HardEvent::V_S>(eventVS);
	169	+ } else {
	170	+ transDataParams.repeatTimes = colNum / 8;
	171	+ transDataParams.dstRepStride = rowNum;
	172	+ transDataParams.srcRepStride = 1;
	173	+ for (int32_t j = 0; j < rowNum / 16; j++) {
	174	+ for (int32_t i = 0; i < 16; i++) {
	175	+ srcList[i] = xLocal[i * colNum + j * 16 * colNum];
	176	+ }
	177	+
	178	+ for (int32_t i = 0; i < 8; i++) {
	179	+ dstList[i * 2] = xLocalTrans[i * rowNum + j * 16];
	180	+ dstList[i * 2 + 1] = xLocalTrans[i * rowNum + 8 + j * 16];
	181	+ }
	182	+
	183	+ SetFlag<HardEvent::S_V>(eventSV);
	184	+ WaitFlag<HardEvent::S_V>(eventSV);
	185	+ TransDataTo5HD<float>(dstList, srcList, transDataParams);
	186	+ SetFlag<HardEvent::V_S>(eventVS);
	187	+ WaitFlag<HardEvent::V_S>(eventVS);
	188	+ }
	189	+ }
	190	+}
	191	+
	192	+template <typename T, int32_t QUEUE_DEPTH>
	193	+__aicore__ inline void TransposeInput(
	194	+ TQue<QuePosition::VECIN, QUEUE_DEPTH>& inputQue,
	195	+ TBuf<TPosition::VECCALC>& inputTransBuffer,
	196	+ TBuf<TPosition::VECCALC>& mulWBuffer,
	197	+ int64_t curNcFactor,
	198	+ uint8_t diFactor,
	199	+ uint8_t hiFactor,
	200	+ uint8_t wiFactor,
	201	+ const PoolVars& poolVars) {
	202	+ const uint8_t wiFactor16Align = Ceil(wiFactor, 32 / sizeof(T)) * 32 / sizeof(T);
	203	+ const uint8_t wiFactorAlign = Ceil(wiFactor, 8) * 8;
	204	+ LocalTensor<T> xLocal = inputQue.template DeQue<T>();
	205	+ LocalTensor<float> xLocalTransVL = inputTransBuffer.Get<float>();
	206	+ if constexpr (IsSameType<T, float>::value) {
	207	+ OutTranspose<T>(xLocalTransVL, xLocal, poolVars.VL_NUM, diFactor * hiFactor * wiFactorAlign);
	208	+ } else {
	209	+ LocalTensor<float> xLocalCast = mulWBuffer.Get<float>();
	210	+ UnaryRepeatParams repeatCastParams{
	211	+ (uint16_t)(wiFactorAlign / 8), (uint16_t)(wiFactor16Align / 8),
	212	+ (uint8_t)(wiFactorAlign / 8 * Ceil(diFactor * hiFactor, 2)),
	213	+ (uint8_t)(wiFactor16Align / 8 * Ceil(diFactor * hiFactor, 2))};
	214	+
	215	+ Cast(xLocalCast, xLocal, RoundMode::CAST_NONE, wiFactor16Align * curNcFactor * diFactor * hiFactor);
	216	+ PipeBarrier<PIPE_V>();
	217	+ Adds(
	218	+ xLocalCast, xLocalCast, float(0.0), uint8_t(wiFactorAlign * Ceil(diFactor * hiFactor, 2)), curNcFactor * 2,
	219	+ repeatCastParams);
	220	+
	221	+ PipeBarrier<PIPE_V>();
	222	+ OutTranspose<T>(xLocalTransVL, xLocalCast, poolVars.VL_NUM, diFactor * hiFactor * wiFactorAlign);
	223	+ }
	224	+ PipeBarrier<PIPE_V>();
	225	+ inputQue.FreeTensor(xLocal);
	226	+}
	227	+
	228	+} // namespace Pool3dMemCommon
	229	+
	230	+#endif // POOL_3D_MEMORY_OPTIMIZED_UTILS_H_