仓内重名（ffn / torch_extension / mhc / posembedding）整改-ops-transformer-AtomGit

Mffn/ffn/op_kernel/ffn_antiquant.h+7-6

	@@ -16,8 +16,7 @@
16	#ifndef ASCENDC_FFN_ANTI_QUANT_H	16	#ifndef ASCENDC_FFN_ANTI_QUANT_H
17	#define ASCENDC_FFN_ANTI_QUANT_H	17	#define ASCENDC_FFN_ANTI_QUANT_H
18		18
19	-#include "ffn.h"	19	+#include "ffn_kernel.h"
20	-
21		20
22	namespace FFN {	21	namespace FFN {
23	/*@brief store variables for castWeight configuration	22	/*@brief store variables for castWeight configuration
	@@ -42,7 +41,9 @@ template <typename T, typename wT, typename mm1Type, typename mm2Type = mm1Type,
42	typename biasT = T, bool isPerGroup = false>	41	typename biasT = T, bool isPerGroup = false>
43	class FFNAntiQuant {	42	class FFNAntiQuant {
44	public:	43	public:
45	- __aicore__ inline FFNAntiQuant(mm1Type &mm1_, mm2Type &mm2_) : mm1(mm1_), mm2(mm2_)	44	+ __aicore__ inline FFNAntiQuant(mm1Type &mm1_, mm2Type &mm2_)
		45	+ : mm1(mm1_),
		46	+ mm2(mm2_)
46	{	47	{
47	}	48	}
48	__aicore__ inline void Init(__gm__ uint8_t x, __gm__ uint8_t weight1, __gm__ uint8_t *weight2,	49	__aicore__ inline void Init(__gm__ uint8_t x, __gm__ uint8_t weight1, __gm__ uint8_t *weight2,
	@@ -317,8 +318,8 @@ protected:
317	if (isMatMul1) {	318	if (isMatMul1) {
318	n1Loops = nLoops;	319	n1Loops = nLoops;
319	m1Loops = mLoops;	320	m1Loops = mLoops;
320	- singleM1 = singleM; // compute C matrix block length along m direction for each cube	321	+ singleM1 = singleM; // compute C matrix block length along m direction for each cube
321	- singleN1 = singleN; // compute C matrix block length along n direction for each cube	322	+ singleN1 = singleN; // compute C matrix block length along n direction for each cube
322	singleM1Tail = tokens - (m1Loops - 1) * singleM1; // recompute last block length along m direction	323	singleM1Tail = tokens - (m1Loops - 1) * singleM1; // recompute last block length along m direction
323	singleN1Tail = n1 - (n1Loops - 1) * singleN1; // recompute last block length along n direction	324	singleN1Tail = n1 - (n1Loops - 1) * singleN1; // recompute last block length along n direction
324	castWeightSingleN1 = n1;	325	castWeightSingleN1 = n1;
	@@ -769,4 +770,4 @@ protected:
769	};	770	};
770	} // namespace FFN	771	} // namespace FFN
771		772
772	-#endif // ASCENDC_FFN_ANTI_QUANT_H	773	+#endif // ASCENDC_FFN_ANTI_QUANT_H

Mffn/ffn/op_kernel/ffn_antiquant_msd.h+9-6

	@@ -17,7 +17,7 @@
17	#define ASCENDC_FFN_ANTIQUANT_MSD_H	17	#define ASCENDC_FFN_ANTIQUANT_MSD_H
18		18
19	#include "ffn_base.h"	19	#include "ffn_base.h"
20	-#include "ffn.h"	20	+#include "ffn_kernel.h"
21		21
22	namespace FFN {	22	namespace FFN {
23	struct TilingConfig {	23	struct TilingConfig {
	@@ -95,7 +95,9 @@ __aicore__ inline void DataCopyPad2D(const GlobalTensor<T> dst, const LocalTenso
95	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>	95	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>
96	class FFNAntiQuantMSD {	96	class FFNAntiQuantMSD {
97	public:	97	public:
98	- __aicore__ inline FFNAntiQuantMSD(mm1Type &mm1_, mm2Type &mm2_) : mm1(mm1_), mm2(mm2_)	98	+ __aicore__ inline FFNAntiQuantMSD(mm1Type &mm1_, mm2Type &mm2_)
		99	+ : mm1(mm1_),
		100	+ mm2(mm2_)
99	{	101	{
100	}	102	}
101	__aicore__ inline void Init(__gm__ uint8_t x, __gm__ uint8_t weight1, __gm__ uint8_t *weight2,	103	__aicore__ inline void Init(__gm__ uint8_t x, __gm__ uint8_t weight1, __gm__ uint8_t *weight2,
	@@ -427,7 +429,7 @@ FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::MM1VectorTiling(Tilin
427	vecBlockDimK_ = Ceil(tilingParams.k, vecBaseK_); // recompute coreNum in K-axis	429	vecBlockDimK_ = Ceil(tilingParams.k, vecBaseK_); // recompute coreNum in K-axis
428	uint32_t vecBlockDimM_ =	430	uint32_t vecBlockDimM_ =
429	tilingParams.aivNumPerExpert / vecBlockDimK_; // recompute coreNum in M-axis	431	tilingParams.aivNumPerExpert / vecBlockDimK_; // recompute coreNum in M-axis
430	- // recompute singleM and M-axis coreNum	432	+ // recompute singleM and M-axis coreNum
431	uint32_t vecSingleM_ = Ceil(tilingParams.mVec, vecBlockDimM_);	433	uint32_t vecSingleM_ = Ceil(tilingParams.mVec, vecBlockDimM_);
432	vecBlockDimM_ = Ceil(tilingParams.mVec, vecSingleM_);	434	vecBlockDimM_ = Ceil(tilingParams.mVec, vecSingleM_);
433	uint32_t vecSingleMTail_ = tilingParams.mVec - (vecBlockDimM_ - 1) * vecSingleM_;	435	uint32_t vecSingleMTail_ = tilingParams.mVec - (vecBlockDimM_ - 1) * vecSingleM_;
	@@ -558,7 +560,7 @@ FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::PreProcessMM1(TilingC
558		560
559	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>	561	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>
560	__aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::PreProcessMM2(	562	__aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::PreProcessMM2(
561	- uint32_t offsetM, uint32_t curBaseK, TilingConfig tilingParams, uint32_t &syncCount,	563	+ uint32_t offsetM, uint32_t curBaseK, TilingConfig tilingParams, uint32_t &syncCount,
562	ExpertParallInfo mmExpertParallInfo)	564	ExpertParallInfo mmExpertParallInfo)
563	{	565	{
564	uint32_t curBaseM = tilingParams.vecBaseM;	566	uint32_t curBaseM = tilingParams.vecBaseM;
	@@ -582,7 +584,8 @@ __aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>
582	tilingParams.vecBlockDimM * tilingParams.vecBlockDimK &&	584	tilingParams.vecBlockDimM * tilingParams.vecBlockDimK &&
583	subBlockIdx_ == 0) {	585	subBlockIdx_ == 0) {
584	aOffsetGm = (mmExpertParallInfo.LocalOffset[expertIdxInParaGroupMM1] * ANTIQUANT_MSD_STEP +	586	aOffsetGm = (mmExpertParallInfo.LocalOffset[expertIdxInParaGroupMM1] * ANTIQUANT_MSD_STEP +
585	- vec1BlockMIdx_ * tilingParams.vecSingleM + offsetM) * tilingParams.k +	587	+ vec1BlockMIdx_ * tilingParams.vecSingleM + offsetM) *
		588	+ tilingParams.k +
586	vec1BlockKIdx_ * tilingParams.vecBaseK;	589	vec1BlockKIdx_ * tilingParams.vecBaseK;
587	CalcAMax(tilingParams, gmReduceOffset, curBaseM, offsetM, reduceMax2WorkspaceGm_);	590	CalcAMax(tilingParams, gmReduceOffset, curBaseM, offsetM, reduceMax2WorkspaceGm_);
588	CalcA1A2(tilingParams, offsetM, curBaseM, curBaseK, aOffsetGm, workspaceMM2AMatrixGm_);	591	CalcA1A2(tilingParams, offsetM, curBaseM, curBaseK, aOffsetGm, workspaceMM2AMatrixGm_);
	@@ -1347,4 +1350,4 @@ __aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>
1347		1350
1348	} // namespace FFN	1351	} // namespace FFN
1349		1352
1350	-#endif // ASCENDC_FFN_ANTIQUANT_MSD_H	1353	+#endif // ASCENDC_FFN_ANTIQUANT_MSD_H

Mffn/ffn/op_kernel/ffn_base.h+4-2

	@@ -16,7 +16,7 @@
16	#ifndef ASCENDC_FFN_BASE_H	16	#ifndef ASCENDC_FFN_BASE_H
17	#define ASCENDC_FFN_BASE_H	17	#define ASCENDC_FFN_BASE_H
18		18
19	-#include "ffn.h"	19	+#include "ffn_kernel.h"
20		20
21	namespace FFN {	21	namespace FFN {
22		22
	@@ -67,7 +67,9 @@ protected:
67		67
68	public:	68	public:
69	/** @brief constructor */	69	/** @brief constructor */
70	- __aicore__ inline FFNBase(typename mm1Type::MT &mm1_, typename mm2Type::MT &mm2_) : mm1(mm1_), mm2(mm2_)	70	+ __aicore__ inline FFNBase(typename mm1Type::MT &mm1_, typename mm2Type::MT &mm2_)
		71	+ : mm1(mm1_),
		72	+ mm2(mm2_)
71	{	73	{
72	}	74	}
73		75

Mffn/ffn/op_kernel/ffn_glu.h+2-3

	@@ -16,8 +16,7 @@
16	#ifndef ASCENDC_FFN_GLU_H	16	#ifndef ASCENDC_FFN_GLU_H
17	#define ASCENDC_FFN_GLU_H	17	#define ASCENDC_FFN_GLU_H
18		18
19	-#include "ffn.h"	19	+#include "ffn_kernel.h"
20	-
21		20
22	namespace FFN {	21	namespace FFN {
23	using namespace matmul;	22	using namespace matmul;
	@@ -121,4 +120,4 @@ protected:
121	};	120	};
122	} // namespace FFN	121	} // namespace FFN
123		122
124	-#endif // ASCENDC_FFN_GLU_H	123	+#endif // ASCENDC_FFN_GLU_H

Rffn/ffn/op_kernel/ffn.h→ffn/ffn/op_kernel/ffn_kernel.h+17-17

	@@ -9,18 +9,17 @@
9	*/	9	*/
10		10
11	/*!	11	/*!
12	- * \file ffn.h	12	+ * \file ffn_kernel.h
13	* \brief	13	* \brief
14	*/	14	*/
15		15
16	-#ifndef ASCENDC_FFN_H	16	+#ifndef ASCENDC_FFN_KERNEL_H
17	-#define ASCENDC_FFN_H	17	+#define ASCENDC_FFN_KERNEL_H
18		18
19	#include "kernel_tiling/kernel_tiling.h"	19	#include "kernel_tiling/kernel_tiling.h"
20	#include "kernel_operator.h"	20	#include "kernel_operator.h"
21	#include "lib/matmul_intf.h"	21	#include "lib/matmul_intf.h"
22		22
23	-
24	namespace FFN {	23	namespace FFN {
25	using namespace AscendC;	24	using namespace AscendC;
26		25
	@@ -72,25 +71,25 @@ public:
72	GluActiveFuncPtr<T> gluFuncPointer;	71	GluActiveFuncPtr<T> gluFuncPointer;
73	};	72	};
74		73
75	-constexpr float BETA_ = 1.0; // beta param of swiglu	74	+constexpr float BETA_ = 1.0; // beta param of swiglu
76	-constexpr uint32_t MAX_EXPERT_PARALLELISM = 10; // allow `MAX_EXPERT_PARALLELISM` experts to compute together	75	+constexpr uint32_t MAX_EXPERT_PARALLELISM = 10; // allow `MAX_EXPERT_PARALLELISM` experts to compute together
77	-constexpr uint32_t UB_BLOCK_UNIT_SIZE = 32; // 32: a block has 32 bytes data	76	+constexpr uint32_t UB_BLOCK_UNIT_SIZE = 32; // 32: a block has 32 bytes data
78	-constexpr uint32_t CUBE_BASE_ALIGN_FACTOR = 16; // 16: baseM align requirement for ai cube	77	+constexpr uint32_t CUBE_BASE_ALIGN_FACTOR = 16; // 16: baseM align requirement for ai cube
79	-constexpr uint32_t CUBE_QUANT_BASE_ALIGN_FACTOR = 32; // 32: quant matmul baseM align requirement for ai cube	78	+constexpr uint32_t CUBE_QUANT_BASE_ALIGN_FACTOR = 32; // 32: quant matmul baseM align requirement for ai cube
80	-constexpr uint32_t INT8_BITS = 8; // 8: a int8 data has 8 bits	79	+constexpr uint32_t INT8_BITS = 8; // 8: a int8 data has 8 bits
81	constexpr uint32_t FP16_INT8_BEST_DATACOPY_BASE_SIZE = 512; // 512: can copy 512 elements of fp16 int8 type every time	80	constexpr uint32_t FP16_INT8_BEST_DATACOPY_BASE_SIZE = 512; // 512: can copy 512 elements of fp16 int8 type every time
82	constexpr uint32_t BF16_INT8_BEST_DATACOPY_BASE_SIZE = 256; // 256: can copy 256 elements of bf16 int8 type every time	81	constexpr uint32_t BF16_INT8_BEST_DATACOPY_BASE_SIZE = 256; // 256: can copy 256 elements of bf16 int8 type every time
83	-constexpr uint32_t INT8_SYNC_N1_SIZE = 256; // 256: when n1 is small than 256, should enable SyncbeforeMM1	82	+constexpr uint32_t INT8_SYNC_N1_SIZE = 256; // 256: when n1 is small than 256, should enable SyncbeforeMM1
84	// a unit block can contain `EXPERT_NUM_ALIGN` int64_t elements	83	// a unit block can contain `EXPERT_NUM_ALIGN` int64_t elements
85	constexpr uint32_t EXPERT_NUM_ALIGN = UB_BLOCK_UNIT_SIZE / sizeof(int64_t);	84	constexpr uint32_t EXPERT_NUM_ALIGN = UB_BLOCK_UNIT_SIZE / sizeof(int64_t);
86	constexpr uint32_t ANTIQUANT_MSD_STEP = 2;	85	constexpr uint32_t ANTIQUANT_MSD_STEP = 2;
87	constexpr uint32_t NUM_ALIGN_TO_THIRTYTWO = 31; // used to align to 32	86	constexpr uint32_t NUM_ALIGN_TO_THIRTYTWO = 31; // used to align to 32
88	constexpr uint32_t NUM_ALIGN_TO_SIXTEEN = 15; // used to align to 16	87	constexpr uint32_t NUM_ALIGN_TO_SIXTEEN = 15; // used to align to 16
89	constexpr uint32_t NUM_ALIGN_TO_ONE_HUNDRED_TWEENTY_EIGHT = 127; // used to align to 128	88	constexpr uint32_t NUM_ALIGN_TO_ONE_HUNDRED_TWEENTY_EIGHT = 127; // used to align to 128
90	-constexpr uint32_t FACTOR_FOR_FLOAT_ALIGN_TO_32BYTE = 8; // a float type num need to duplicate 8 times to align 32	89	+constexpr uint32_t FACTOR_FOR_FLOAT_ALIGN_TO_32BYTE = 8; // a float type num need to duplicate 8 times to align 32
91	-constexpr uint32_t DATASIZE_EACH_REPEAT_TIME = 256; // each repeat time can calc 256Byte data	90	+constexpr uint32_t DATASIZE_EACH_REPEAT_TIME = 256; // each repeat time can calc 256Byte data
92	-constexpr uint32_t MSD_EACH_UB_BLOCK_SIZR = 6 * 1024; // each repeat time can calc 256Byte data	91	+constexpr uint32_t MSD_EACH_UB_BLOCK_SIZR = 6 * 1024; // each repeat time can calc 256Byte data
93	-constexpr uint32_t DATABLOCK_NUM_IN_GATHER = 8; // In Gather API, each repeat collects 8 data blocks	92	+constexpr uint32_t DATABLOCK_NUM_IN_GATHER = 8; // In Gather API, each repeat collects 8 data blocks
94		93
95	template <class T>	94	template <class T>
96	__aicore__ inline constexpr uint32_t GetNumInUbBlock()	95	__aicore__ inline constexpr uint32_t GetNumInUbBlock()
	@@ -122,7 +121,8 @@ __aicore__ inline T AlignUp(T a, T base)
122	return (a + base - 1) / base * base;	121	return (a + base - 1) / base * base;
123	}	122	}
124		123
125	-template <typename T> __aicore__ inline T AlignDown(T a, T base)	124	+template <typename T>
		125	+__aicore__ inline T AlignDown(T a, T base)
126	{	126	{
127	if (unlikely(base == 0)) {	127	if (unlikely(base == 0)) {
128	return a;	128	return a;
	@@ -408,4 +408,4 @@ __aicore__ inline void ApplyActivation(ActiveType activationType, LocalTensor<T>
408	}	408	}
409	} // namespace FFN	409	} // namespace FFN
410		410
411	-#endif // ASCENDC_FFN_H	411	+#endif // ASCENDC_FFN_H

Mffn/ffn/op_kernel/ffn_nonquant_nz.h+6-3

	@@ -16,7 +16,7 @@
16	#ifndef ASCENDC_FFN_NONQUANT_NZ_H	16	#ifndef ASCENDC_FFN_NONQUANT_NZ_H
17	#define ASCENDC_FFN_NONQUANT_NZ_H	17	#define ASCENDC_FFN_NONQUANT_NZ_H
18		18
19	-#include "ffn.h"	19	+#include "ffn_kernel.h"
20		20
21	namespace FFN {	21	namespace FFN {
22		22
	@@ -39,7 +39,8 @@ private:
39		39
40	public:	40	public:
41	/** @brief constructor */	41	/** @brief constructor */
42	- __aicore__ inline FFNProcess(ComputeType &computeOp_) : computeOp(computeOp_)	42	+ __aicore__ inline FFNProcess(ComputeType &computeOp_)
		43	+ : computeOp(computeOp_)
43	{	44	{
44	}	45	}
45		46
	@@ -189,7 +190,9 @@ private:
189		190
190	public:	191	public:
191	/** @brief constructor */	192	/** @brief constructor */
192	- __aicore__ inline FFNCompute(typename mm1Type::MT &mm1_, typename mm2Type::MT &mm2_) : mm1(mm1_), mm2(mm2_)	193	+ __aicore__ inline FFNCompute(typename mm1Type::MT &mm1_, typename mm2Type::MT &mm2_)
		194	+ : mm1(mm1_),
		195	+ mm2(mm2_)
193	{	196	{
194	}	197	}
195		198

Mffn/ffn/op_kernel/ffn_quant.h+5-4

	@@ -16,15 +16,16 @@
16	#ifndef ASCENDC_FFN_QUANT_H	16	#ifndef ASCENDC_FFN_QUANT_H
17	#define ASCENDC_FFN_QUANT_H	17	#define ASCENDC_FFN_QUANT_H
18		18
19	-#include "ffn.h"	19	+#include "ffn_kernel.h"
20	-
21		20
22	namespace FFN {	21	namespace FFN {
23	template <typename T, typename mm1Type, typename mm2Type, typename c1T, typename c2T, typename biasT, typename actT,	22	template <typename T, typename mm1Type, typename mm2Type, typename c1T, typename c2T, typename biasT, typename actT,
24	typename dequantT, bool isSmooth = false>	23	typename dequantT, bool isSmooth = false>
25	class FFNQuant {	24	class FFNQuant {
26	public:	25	public:
27	- __aicore__ inline FFNQuant(mm1Type &mm1_, mm2Type &mm2_) : mm1(mm1_), mm2(mm2_){};	26	+ __aicore__ inline FFNQuant(mm1Type &mm1_, mm2Type &mm2_)
		27	+ : mm1(mm1_),
		28	+ mm2(mm2_){};
28	__aicore__ inline void Init(__gm__ uint8_t x, __gm__ uint8_t weight1, __gm__ uint8_t *weight2,	29	__aicore__ inline void Init(__gm__ uint8_t x, __gm__ uint8_t weight1, __gm__ uint8_t *weight2,
29	__gm__ uint8_t expertTokens, __gm__ uint8_t bias1, __gm__ uint8_t *bias2,	30	__gm__ uint8_t expertTokens, __gm__ uint8_t bias1, __gm__ uint8_t *bias2,
30	__gm__ uint8_t scale, __gm__ uint8_t offset, __gm__ uint8_t *deqScale1,	31	__gm__ uint8_t scale, __gm__ uint8_t offset, __gm__ uint8_t *deqScale1,
	@@ -995,4 +996,4 @@ protected:
995	};	996	};
996	} // namespace FFN	997	} // namespace FFN
997		998
998	-#endif // ASCENDC_FFN_QUANT_H	999	+#endif // ASCENDC_FFN_QUANT_H

Rmhc/mhc_post/op_kernel/arch22/mhc_post.h→mhc/mhc_post/op_kernel/arch22/mhc_post_arch22.h+6-6

	@@ -9,15 +9,15 @@
9	*/	9	*/
10		10
11	/*!	11	/*!
12	- * \file mhc_post.h	12	+ * \file mhc_post_arch22.h
13	* \brief MhcPost kernel implementation	13	* \brief MhcPost kernel implementation
14	* Formula: x_{l+1} = (H_{l}^{res})^{T} * x_l + h_{l}^{out} * H_{t}^{post}	14	* Formula: x_{l+1} = (H_{l}^{res})^{T} * x_l + h_{l}^{out} * H_{t}^{post}
15	* where: (H_{l}^{res})^{T} * x_l represents matrix multiplication with transposed h_res	15	* where: (H_{l}^{res})^{T} * x_l represents matrix multiplication with transposed h_res
16	* h_{l}^{out} * H_{t}^{post} represents element-wise multiplication and broadcasting	16	* h_{l}^{out} * H_{t}^{post} represents element-wise multiplication and broadcasting
17	*/	17	*/
18		18
19	-#ifndef ASCENDC_MHC_POST_H	19	+#ifndef ASCENDC_MHC_POST_ARCH22_H
20	-#define ASCENDC_MHC_POST_H	20	+#define ASCENDC_MHC_POST_ARCH22_H
21		21
22	#include "kernel_operator.h"	22	#include "kernel_operator.h"
23	#include "kernel_utils.h"	23	#include "kernel_utils.h"
	@@ -29,7 +29,7 @@ namespace MhcPost {
29	using namespace AscendC;	29	using namespace AscendC;
30		30
31	// Double Buffer configuration - Double Buffer提升Memory Bound算子性能	31	// Double Buffer configuration - Double Buffer提升Memory Bound算子性能
32	-constexpr uint32_t DOUBLE_BUFFER_DEPTH = 2; // Double Buffer depth for data tiles	32	+constexpr uint32_t DOUBLE_BUFFER_DEPTH = 2; // Double Buffer depth for data tiles
33		33
34	#define TEMPLATE_DECLARE template <typename T, uint16_t USE_PERMANENT_X>	34	#define TEMPLATE_DECLARE template <typename T, uint16_t USE_PERMANENT_X>
35	#define TEMPLATE_ARGS T, USE_PERMANENT_X	35	#define TEMPLATE_ARGS T, USE_PERMANENT_X
	@@ -268,6 +268,6 @@ __aicore__ inline void MhcPostKernel<TEMPLATE_ARGS>::CopyOutTile(int64_t bsIdx,
268	outputTileQueue_.FreeTensor(outputTile);	268	outputTileQueue_.FreeTensor(outputTile);
269	}	269	}
270		270
271	-} // namespace MhcPost	271	+} // namespace MhcPost
272		272
273	-#endif // ASCENDC_MHC_POST_H	273	+#endif // ASCENDC_MHC_POST_H

Rmhc/mhc_post/op_kernel/arch35/mhc_post.h→mhc/mhc_post/op_kernel/arch35/mhc_post_arch35.h+5-5

	@@ -9,15 +9,15 @@
9	*/	9	*/
10		10
11	/*!	11	/*!
12	- * \file mhc_post.h	12	+ * \file mhc_post_arch35.h
13	* \brief MhcPost kernel implementation	13	* \brief MhcPost kernel implementation
14	* Formula: x_{l+1} = (H_{l}^{res})^{T} * x_l + h_{l}^{out} * H_{t}^{post}	14	* Formula: x_{l+1} = (H_{l}^{res})^{T} * x_l + h_{l}^{out} * H_{t}^{post}
15	* where: (H_{l}^{res})^{T} * x_l represents matrix multiplication with transposed h_res	15	* where: (H_{l}^{res})^{T} * x_l represents matrix multiplication with transposed h_res
16	* h_{l}^{out} * H_{t}^{post} represents element-wise multiplication and broadcasting	16	* h_{l}^{out} * H_{t}^{post} represents element-wise multiplication and broadcasting
17	*/	17	*/
18		18
19	-#ifndef ASCENDC_MHC_POST_H	19	+#ifndef ASCENDC_MHC_POST_ARCH35_H
20	-#define ASCENDC_MHC_POST_H	20	+#define ASCENDC_MHC_POST_ARCH35_H
21		21
22	#include "kernel_operator.h"	22	#include "kernel_operator.h"
23	#include "kernel_tiling/kernel_tiling.h"	23	#include "kernel_tiling/kernel_tiling.h"
	@@ -257,6 +257,6 @@ __aicore__ inline void MhcPostKernel<TEMPLATE_ARGS>::CopyOutTile(int64_t bsIdx,
257	outputTileQueue_.FreeTensor(outputTile);	257	outputTileQueue_.FreeTensor(outputTile);
258	}	258	}
259		259
260	-} // namespace MhcPost	260	+} // namespace MhcPost
261		261
262	-#endif // ASCENDC_MHC_POST_H	262	+#endif // ASCENDC_MHC_POST_H

Mmhc/mhc_post/op_kernel/mhc_post.cpp+2-2

	@@ -16,7 +16,7 @@
16		16
17	#include "kernel_operator.h"	17	#include "kernel_operator.h"
18	#include "kernel_tiling/kernel_tiling.h"	18	#include "kernel_tiling/kernel_tiling.h"
19	-#include "arch22/mhc_post.h"	19	+#include "arch22/mhc_post_arch22.h"
20		20
21	using namespace AscendC;	21	using namespace AscendC;
22	using namespace MhcPost;	22	using namespace MhcPost;
	@@ -33,4 +33,4 @@ __global__ __aicore__ void mhc_post(GM_ADDR x, GM_ADDR hRes, GM_ADDR hOut, GM_AD
33	MhcPostKernel<DTYPE_X, usePermanentX> op(&tPipe, &tilingData);	33	MhcPostKernel<DTYPE_X, usePermanentX> op(&tPipe, &tilingData);
34	op.Init(x, hRes, hOut, hPost, output, workspace);	34	op.Init(x, hRes, hOut, hPost, output, workspace);
35	op.Process();	35	op.Process();
36	-}	36	+}

Mmhc/mhc_post/op_kernel/mhc_post_apt.cpp+1-1

	@@ -17,7 +17,7 @@
17		17
18	#include "kernel_operator.h"	18	#include "kernel_operator.h"
19	#include "kernel_tiling/kernel_tiling.h"	19	#include "kernel_tiling/kernel_tiling.h"
20	-#include "arch35/mhc_post.h"	20	+#include "arch35/mhc_post_arch35.h"
21	#include "arch35/mhc_post_regbase.h"	21	#include "arch35/mhc_post_regbase.h"
22	#include "arch35/mhc_post_nohres.h"	22	#include "arch35/mhc_post_nohres.h"
23		23

Rmhc/mhc_post_backward/op_kernel/arch22/mhc_post_backward.h→mhc/mhc_post_backward/op_kernel/arch22/mhc_post_backward_arch22.h+59-63

	@@ -9,12 +9,12 @@
9	*/	9	*/
10		10
11	/*!	11	/*!
12	- * \file mhc_post_backward.h	12	+ * \file mhc_post_backward_arch22.h
13	* \brief	13	* \brief
14	*/	14	*/
15		15
16	-#ifndef MHC_POST_BACKWARD_H	16	+#ifndef MHC_POST_BACKWARD_ARCH22_H
17	-#define MHC_POST_BACKWARD_H	17	+#define MHC_POST_BACKWARD_ARCH22_H
18		18
19	#include "kernel_operator.h"	19	#include "kernel_operator.h"
20	#include "mhc_post_backward_tiling_data_arch22.h"	20	#include "mhc_post_backward_tiling_data_arch22.h"
	@@ -23,27 +23,26 @@ using namespace AscendC;
23	constexpr float ZERO = 0;	23	constexpr float ZERO = 0;
24		24
25	template <typename T>	25	template <typename T>
26	-class KernelMhcPostBackward	26	+class KernelMhcPostBackward {
27	-{
28	public:	27	public:
29	__aicore__ inline KernelMhcPostBackward() {}	28	__aicore__ inline KernelMhcPostBackward() {}
30		29
31	__aicore__ inline void Init(	30	__aicore__ inline void Init(
32	GM_ADDR grad_y, GM_ADDR x, GM_ADDR h_res, GM_ADDR h_out, GM_ADDR h_post,	31	GM_ADDR grad_y, GM_ADDR x, GM_ADDR h_res, GM_ADDR h_out, GM_ADDR h_post,
33	GM_ADDR grad_x, GM_ADDR grad_h_res, GM_ADDR grad_h_out, GM_ADDR grad_h_post,	32	GM_ADDR grad_x, GM_ADDR grad_h_res, GM_ADDR grad_h_out, GM_ADDR grad_h_post,
34	- const MhcPostBackwardTilingDataArch22& tilingData, TPipe* pipe);	33	+ const MhcPostBackwardTilingDataArch22 &tilingData, TPipe *pipe);
35	__aicore__ inline void Process();	34	__aicore__ inline void Process();
36		35
37	protected:	36	protected:
38	static constexpr uint64_t BUFFER_NUM = 1;	37	static constexpr uint64_t BUFFER_NUM = 1;
39		38
40	__aicore__ inline void VecMatmulMknk(	39	__aicore__ inline void VecMatmulMknk(
41	- LocalTensor<float>& A, LocalTensor<float>& B, LocalTensor<float>& C,	40	+ LocalTensor<float> &A, LocalTensor<float> &B, LocalTensor<float> &C,
42	- LocalTensor<float>& broadcastBuffer, LocalTensor<float>& reduceBuffer, uint32_t m, uint32_t k,	41	+ LocalTensor<float> &broadcastBuffer, LocalTensor<float> &reduceBuffer, uint32_t m, uint32_t k,
43	uint32_t n, uint32_t alignN);	42	uint32_t n, uint32_t alignN);
44	__aicore__ inline void VecMatmulMkkn(	43	__aicore__ inline void VecMatmulMkkn(
45	- LocalTensor<float>& A, LocalTensor<float>& B, LocalTensor<float>& C,	44	+ LocalTensor<float> &A, LocalTensor<float> &B, LocalTensor<float> &C,
46	- LocalTensor<float>& broadcastBuffer, uint32_t m, uint32_t k, uint32_t n);	45	+ LocalTensor<float> &broadcastBuffer, uint32_t m, uint32_t k, uint32_t n);
47		46
48	TBuf<TPosition::VECCALC> dFPostResCastBuf, FOutCastBuf, HLPostBuf, xLCastBuf, HLResBuf;	47	TBuf<TPosition::VECCALC> dFPostResCastBuf, FOutCastBuf, HLPostBuf, xLCastBuf, HLResBuf;
49	TBuf<TPosition::VECCALC> dHLPostBuf, dFOutCastBuf, dHLResBuf, dxLCastBuf;	48	TBuf<TPosition::VECCALC> dHLPostBuf, dFOutCastBuf, dHLResBuf, dxLCastBuf;
	@@ -56,7 +55,7 @@ protected:
56		55
57	GlobalTensor<T> dxLGm, dFOutGm;	56	GlobalTensor<T> dxLGm, dFOutGm;
58	GlobalTensor<float> dHResGm, dHPostGm;	57	GlobalTensor<float> dHResGm, dHPostGm;
59	-	58	+
60	LocalTensor<float> dFPostResUb, FOutUb, HLPostUb, xLUb, HLResUb;	59	LocalTensor<float> dFPostResUb, FOutUb, HLPostUb, xLUb, HLResUb;
61	LocalTensor<T> dFPostResCastUb, FOutCastUb, xLCastUb, dFOutCastUb, dxLCastUb;	60	LocalTensor<T> dFPostResCastUb, FOutCastUb, xLCastUb, dFOutCastUb, dxLCastUb;
62	LocalTensor<float> dHLPostUb, dFOutUb, dHLResUb, dxLUb;	61	LocalTensor<float> dHLPostUb, dFOutUb, dHLResUb, dxLUb;
	@@ -92,8 +91,7 @@ template <typename T>
92	__aicore__ inline void KernelMhcPostBackward<T>::Init(	91	__aicore__ inline void KernelMhcPostBackward<T>::Init(
93	GM_ADDR grad_y, GM_ADDR x, GM_ADDR h_res, GM_ADDR h_out, GM_ADDR h_post,	92	GM_ADDR grad_y, GM_ADDR x, GM_ADDR h_res, GM_ADDR h_out, GM_ADDR h_post,
94	GM_ADDR grad_x, GM_ADDR grad_h_res, GM_ADDR grad_h_out, GM_ADDR grad_h_post,	93	GM_ADDR grad_x, GM_ADDR grad_h_res, GM_ADDR grad_h_out, GM_ADDR grad_h_post,
95	- const MhcPostBackwardTilingDataArch22& tilingData, TPipe* pipe	94	+ const MhcPostBackwardTilingDataArch22 &tilingData, TPipe *pipe)
96	-)
97	{	95	{
98	this->coreUsed = tilingData.coreUsed;	96	this->coreUsed = tilingData.coreUsed;
99	this->singleCoreBS = tilingData.singleCoreBS;	97	this->singleCoreBS = tilingData.singleCoreBS;
	@@ -115,16 +113,16 @@ __aicore__ inline void KernelMhcPostBackward<T>::Init(
115	this->hOutSize = tilingData.hOutSize;	113	this->hOutSize = tilingData.hOutSize;
116	this->hPostSize = tilingData.hPostSize;	114	this->hPostSize = tilingData.hPostSize;
117		115
118	- this->dFPostResGm.SetGlobalBuffer((__gm__ T*)grad_y, this->dFPostResSize);	116	+ this->dFPostResGm.SetGlobalBuffer((__gm__ T *)grad_y, this->dFPostResSize);
119	- this->FOutGm.SetGlobalBuffer((__gm__ T*)h_out, this->hOutSize);	117	+ this->FOutGm.SetGlobalBuffer((__gm__ T *)h_out, this->hOutSize);
120	- this->xLGm.SetGlobalBuffer((__gm__ T*)x, this->xSize);	118	+ this->xLGm.SetGlobalBuffer((__gm__ T *)x, this->xSize);
121	- this->HLResGm.SetGlobalBuffer((__gm__ float*)h_res, this->hResSize);	119	+ this->HLResGm.SetGlobalBuffer((__gm__ float *)h_res, this->hResSize);
122	- this->HLPostGm.SetGlobalBuffer((__gm__ float*)h_post, this->hPostSize);	120	+ this->HLPostGm.SetGlobalBuffer((__gm__ float *)h_post, this->hPostSize);
123		121
124	- this->dxLGm.SetGlobalBuffer((__gm__ T*)grad_x, this->xSize);	122	+ this->dxLGm.SetGlobalBuffer((__gm__ T *)grad_x, this->xSize);
125	- this->dFOutGm.SetGlobalBuffer((__gm__ T*)grad_h_out, this->hOutSize);	123	+ this->dFOutGm.SetGlobalBuffer((__gm__ T *)grad_h_out, this->hOutSize);
126	- this->dHResGm.SetGlobalBuffer((__gm__ float*)grad_h_res, this->hResSize);	124	+ this->dHResGm.SetGlobalBuffer((__gm__ float *)grad_h_res, this->hResSize);
127	- this->dHPostGm.SetGlobalBuffer((__gm__ float*)grad_h_post, this->hPostSize);	125	+ this->dHPostGm.SetGlobalBuffer((__gm__ float *)grad_h_post, this->hPostSize);
128		126
129	pipe->InitBuffer(this->dFPostResCastBuf, this->n * this->blockChannel * sizeof(T));	127	pipe->InitBuffer(this->dFPostResCastBuf, this->n * this->blockChannel * sizeof(T));
130	pipe->InitBuffer(this->FOutCastBuf, this->blockChannel * sizeof(T));	128	pipe->InitBuffer(this->FOutCastBuf, this->blockChannel * sizeof(T));
	@@ -138,15 +136,15 @@ __aicore__ inline void KernelMhcPostBackward<T>::Init(
138	pipe->InitBuffer(this->dxLCastBuf, this->n * this->blockChannel * sizeof(T));	136	pipe->InitBuffer(this->dxLCastBuf, this->n * this->blockChannel * sizeof(T));
139		137
140	pipe->InitBuffer(this->dFPostResBuf, this->n * this->blockChannel * sizeof(float)); // [n, tileC]	138	pipe->InitBuffer(this->dFPostResBuf, this->n * this->blockChannel * sizeof(float)); // [n, tileC]
141	- pipe->InitBuffer(this->FOutBuf, this->blockChannel * sizeof(float)); // [1, tileC]	139	+ pipe->InitBuffer(this->FOutBuf, this->blockChannel * sizeof(float)); // [1, tileC]
142	- pipe->InitBuffer(this->xLBuf, this->n * this->blockChannel * sizeof(float)); // [n, tileC]	140	+ pipe->InitBuffer(this->xLBuf, this->n * this->blockChannel * sizeof(float)); // [n, tileC]
143	- pipe->InitBuffer(this->dFOutBuf, this->blockChannel * sizeof(float)); // [1, tileC]	141	+ pipe->InitBuffer(this->dFOutBuf, this->blockChannel * sizeof(float)); // [1, tileC]
144	- pipe->InitBuffer(this->dxLBuf, this->n * this->blockChannel * sizeof(float)); // [n, tileC]	142	+ pipe->InitBuffer(this->dxLBuf, this->n * this->blockChannel * sizeof(float)); // [n, tileC]
145		143
146	pipe->InitBuffer(this->dHLResTmpBuf1, this->blockChannel * this->n * sizeof(float)); // [tilC, n] [k, n]	144	pipe->InitBuffer(this->dHLResTmpBuf1, this->blockChannel * this->n * sizeof(float)); // [tilC, n] [k, n]
147	pipe->InitBuffer(this->dHLResTmpBuf2, this->blockChannel * this->n * this->n * sizeof(float));	145	pipe->InitBuffer(this->dHLResTmpBuf2, this->blockChannel * this->n * this->n * sizeof(float));
148	pipe->InitBuffer(this->dHLResTmpBuf3, this->n * this->alignN * sizeof(float)); // [n, n] [m, n]	146	pipe->InitBuffer(this->dHLResTmpBuf3, this->n * this->alignN * sizeof(float)); // [n, n] [m, n]
149	- pipe->InitBuffer(this->dHLResTmpBuf4, this->n * this->n * sizeof(uint8_t)); // [n, n] [m, n]	147	+ pipe->InitBuffer(this->dHLResTmpBuf4, this->n * this->n * sizeof(uint8_t)); // [n, n] [m, n]
150		148
151	pipe->InitBuffer(this->dxLTmpBuf2, this->n * this->n * sizeof(float));	149	pipe->InitBuffer(this->dxLTmpBuf2, this->n * this->n * sizeof(float));
152	}	150	}
	@@ -161,7 +159,8 @@ __aicore__ inline void KernelMhcPostBackward<T>::Process()
161	}	159	}
162		160
163	uint64_t startIdx = coreId > this->frontCore ? (coreId - this->frontCore) * this->tailBS +	161	uint64_t startIdx = coreId > this->frontCore ? (coreId - this->frontCore) * this->tailBS +
164	- this->frontCore * this->singleCoreBS : coreId * this->singleCoreBS;	162	+ this->frontCore * this->singleCoreBS :
		163	+ coreId * this->singleCoreBS;
165		164
166	uint64_t endIdx = startIdx + ((coreId < this->frontCore) ? this->singleCoreBS : this->tailBS);	165	uint64_t endIdx = startIdx + ((coreId < this->frontCore) ? this->singleCoreBS : this->tailBS);
167		166
	@@ -197,7 +196,7 @@ __aicore__ inline void KernelMhcPostBackward<T>::Process()
197	DataCopyPad(this->HLPostUb, this->HLPostGm[i * this->n], copyParamsHLPostUb, this->padParamsFloat);	196	DataCopyPad(this->HLPostUb, this->HLPostGm[i * this->n], copyParamsHLPostUb, this->padParamsFloat);
198	DataCopyExtParams copyParamsHLResUb{1, static_cast<uint32_t>(this->n * this->n * sizeof(float)), 0, 0, 0};	197	DataCopyExtParams copyParamsHLResUb{1, static_cast<uint32_t>(this->n * this->n * sizeof(float)), 0, 0, 0};
199	DataCopyPad(this->HLResUb, this->HLResGm[i * this->n * this->n], copyParamsHLResUb, this->padParamsFloat);	198	DataCopyPad(this->HLResUb, this->HLResGm[i * this->n * this->n], copyParamsHLResUb, this->padParamsFloat);
200	-	199	+
201	// 清0	200	// 清0
202	SetFlag<HardEvent::MTE3_V>(0);	201	SetFlag<HardEvent::MTE3_V>(0);
203	WaitFlag<HardEvent::MTE3_V>(0);	202	WaitFlag<HardEvent::MTE3_V>(0);
	@@ -205,7 +204,7 @@ __aicore__ inline void KernelMhcPostBackward<T>::Process()
205	Duplicate(this->dHLPostUb, float(0.0), this->alignN);	204	Duplicate(this->dHLPostUb, float(0.0), this->alignN);
206	Duplicate(this->dHLResUb, float(0.0), this->n * this->alignN);	205	Duplicate(this->dHLResUb, float(0.0), this->n * this->alignN);
207		206
208	- for (int j = 0; j < this->loopC; j ++) {	207	+ for (int j = 0; j < this->loopC; j++) {
209	uint32_t channelStride = this->channel - this->blockChannel;	208	uint32_t channelStride = this->channel - this->blockChannel;
210	SetFlag<HardEvent::V_MTE2>(0);	209	SetFlag<HardEvent::V_MTE2>(0);
211	WaitFlag<HardEvent::V_MTE2>(0);	210	WaitFlag<HardEvent::V_MTE2>(0);
	@@ -223,7 +222,7 @@ __aicore__ inline void KernelMhcPostBackward<T>::Process()
223	DataCopyPad(	222	DataCopyPad(
224	this->FOutCastUb, this->FOutGm[i * this->channel + j * this->blockChannel],	223	this->FOutCastUb, this->FOutGm[i * this->channel + j * this->blockChannel],
225	copyParamsFOutUb, this->padParams);	224	copyParamsFOutUb, this->padParams);
226	-	225	+
227	DataCopyExtParams copyParamsXLUb{	226	DataCopyExtParams copyParamsXLUb{
228	static_cast<uint16_t>(this->n), static_cast<uint32_t>(this->blockChannel * sizeof(T)),	227	static_cast<uint16_t>(this->n), static_cast<uint32_t>(this->blockChannel * sizeof(T)),
229	static_cast<uint32_t>(channelStride * sizeof(T)), 0, 0};	228	static_cast<uint32_t>(channelStride * sizeof(T)), 0, 0};
	@@ -242,19 +241,19 @@ __aicore__ inline void KernelMhcPostBackward<T>::Process()
242	// 输出：[1, n] 驻留在ub做累加	241	// 输出：[1, n] 驻留在ub做累加
243	Cast(	242	Cast(
244	this->dFPostResUb, this->dFPostResCastUb, RoundMode::CAST_NONE,	243	this->dFPostResUb, this->dFPostResCastUb, RoundMode::CAST_NONE,
245	- this->n * this->blockChannel); // bf16--> fp32	244	+ this->n * this->blockChannel); // bf16--> fp32
246	- Cast(this->FOutUb, this->FOutCastUb, RoundMode::CAST_NONE, this->blockChannel); // bf16--> fp32	245	+ Cast(this->FOutUb, this->FOutCastUb, RoundMode::CAST_NONE, this->blockChannel); // bf16--> fp32
247	Cast(this->xLUb, this->xLCastUb, RoundMode::CAST_NONE, this->n * this->blockChannel); // bf16--> fp32	246	Cast(this->xLUb, this->xLCastUb, RoundMode::CAST_NONE, this->n * this->blockChannel); // bf16--> fp32
248		247
249	VecMatmulMknk(	248	VecMatmulMknk(
250	this->FOutUb, this->dFPostResUb, this->dHLPostUb, this->dHLResTmp1,	249	this->FOutUb, this->dFPostResUb, this->dHLPostUb, this->dHLResTmp1,
251	- this->dHLResTmp3, 1, this->blockChannel, this->n, this->alignN);	250	+ this->dHLResTmp3, 1, this->blockChannel, this->n, this->alignN);
252		251
253	// dHres: dF@x^T //x@dF^T	252	// dHres: dF@x^T //x@dF^T
254	// [n, C]@[C, n] = [n, n] 驻留在ub做累加	253	// [n, C]@[C, n] = [n, n] 驻留在ub做累加
255	VecMatmulMknk(	254	VecMatmulMknk(
256	this->xLUb, this->dFPostResUb, this->dHLResUb, this->dHLResTmp1,	255	this->xLUb, this->dFPostResUb, this->dHLResUb, this->dHLResTmp1,
257	- this->dHLResTmp3, this->n, this->blockChannel, this->n, this->alignN);	256	+ this->dHLResTmp3, this->n, this->blockChannel, this->n, this->alignN);
258		257
259	// dFout: H_post@dF	258	// dFout: H_post@dF
260	// [1, n]@[n, C] = [1, C] 直接搬出	259	// [1, n]@[n, C] = [1, C] 直接搬出
	@@ -270,14 +269,13 @@ __aicore__ inline void KernelMhcPostBackward<T>::Process()
270	this->HLResUb, this->dFPostResUb, this->dxLUb, this->dHLResTmp2,	269	this->HLResUb, this->dFPostResUb, this->dxLUb, this->dHLResTmp2,
271	this->n, this->n, this->blockChannel);	270	this->n, this->n, this->blockChannel);
272		271
273	-
274	DataCopyExtParams copyParamsdxLGm{	272	DataCopyExtParams copyParamsdxLGm{
275	static_cast<uint16_t>(this->n), static_cast<uint32_t>(this->blockChannel * sizeof(T)), 0,	273	static_cast<uint16_t>(this->n), static_cast<uint32_t>(this->blockChannel * sizeof(T)), 0,
276	static_cast<uint32_t>(channelStride * sizeof(T)), 0};	274	static_cast<uint32_t>(channelStride * sizeof(T)), 0};
277	DataCopyExtParams copyParamsdFOutGm{1, static_cast<uint32_t>(this->blockChannel * sizeof(T)), 0, 0, 0};	275	DataCopyExtParams copyParamsdFOutGm{1, static_cast<uint32_t>(this->blockChannel * sizeof(T)), 0, 0, 0};
278	- Cast(this->dFOutCastUb, this->dFOutUb, RoundMode::CAST_ROUND, this->blockChannel); // float--> bf16	276	+ Cast(this->dFOutCastUb, this->dFOutUb, RoundMode::CAST_ROUND, this->blockChannel); // float--> bf16
279	Cast(this->dxLCastUb, this->dxLUb, RoundMode::CAST_ROUND, this->n * this->blockChannel); // float--> bf16	277	Cast(this->dxLCastUb, this->dxLUb, RoundMode::CAST_ROUND, this->n * this->blockChannel); // float--> bf16
280	-	278	+
281	SetFlag<HardEvent::V_MTE3>(0);	279	SetFlag<HardEvent::V_MTE3>(0);
282	WaitFlag<HardEvent::V_MTE3>(0);	280	WaitFlag<HardEvent::V_MTE3>(0);
283	DataCopyPad(	281	DataCopyPad(
	@@ -291,7 +289,7 @@ __aicore__ inline void KernelMhcPostBackward<T>::Process()
291	if (this->tailC != 0) {	289	if (this->tailC != 0) {
292	// 尾部C的处理	290	// 尾部C的处理
293	uint32_t channelStride = this->channel - this->tailC;	291	uint32_t channelStride = this->channel - this->tailC;
294	-	292	+
295	SetFlag<HardEvent::V_MTE2>(0);	293	SetFlag<HardEvent::V_MTE2>(0);
296	WaitFlag<HardEvent::V_MTE2>(0);	294	WaitFlag<HardEvent::V_MTE2>(0);
297		295
	@@ -305,7 +303,7 @@ __aicore__ inline void KernelMhcPostBackward<T>::Process()
305	this->dFPostResCastUb,	303	this->dFPostResCastUb,
306	this->dFPostResGm[i * this->n * this->channel + this->loopC * this->blockChannel],	304	this->dFPostResGm[i * this->n * this->channel + this->loopC * this->blockChannel],
307	copyParamsdFPostResUb, this->padParams);	305	copyParamsdFPostResUb, this->padParams);
308	-	306	+
309	DataCopyExtParams copyParamsFOutUb{1, static_cast<uint32_t>(this->tailC * sizeof(T)), 0, 0, 0};	307	DataCopyExtParams copyParamsFOutUb{1, static_cast<uint32_t>(this->tailC * sizeof(T)), 0, 0, 0};
310	DataCopyPad(	308	DataCopyPad(
311	this->FOutCastUb,	309	this->FOutCastUb,
	@@ -328,18 +326,18 @@ __aicore__ inline void KernelMhcPostBackward<T>::Process()
328	// 搬运量：[1, n, tailC] + [1, 1, tailC]	326	// 搬运量：[1, n, tailC] + [1, 1, tailC]
329	// 输出：[1, n] 驻留在ub做累加	327	// 输出：[1, n] 驻留在ub做累加
330	Cast(this->dFPostResUb, this->dFPostResCastUb, RoundMode::CAST_NONE, this->n * this->tailC); // bf16--> fp32	328	Cast(this->dFPostResUb, this->dFPostResCastUb, RoundMode::CAST_NONE, this->n * this->tailC); // bf16--> fp32
331	- Cast(this->FOutUb, this->FOutCastUb, RoundMode::CAST_NONE, this->tailC); // bf16--> fp32	329	+ Cast(this->FOutUb, this->FOutCastUb, RoundMode::CAST_NONE, this->tailC); // bf16--> fp32
332	- Cast(this->xLUb, this->xLCastUb, RoundMode::CAST_NONE, this->n * this->tailC); // bf16--> fp32	330	+ Cast(this->xLUb, this->xLCastUb, RoundMode::CAST_NONE, this->n * this->tailC); // bf16--> fp32
333		331
334	VecMatmulMknk(	332	VecMatmulMknk(
335	this->FOutUb, this->dFPostResUb, this->dHLPostUb, this->dHLResTmp1,	333	this->FOutUb, this->dFPostResUb, this->dHLPostUb, this->dHLResTmp1,
336	- this->dHLResTmp3, 1, this->tailC, this->n, this->alignN);	334	+ this->dHLResTmp3, 1, this->tailC, this->n, this->alignN);
337		335
338	// dHres: dF@x^T	336	// dHres: dF@x^T
339	// [n, tailC]@[tailC, n] = [n, n] 驻留在ub做累加	337	// [n, tailC]@[tailC, n] = [n, n] 驻留在ub做累加
340	VecMatmulMknk(	338	VecMatmulMknk(
341	this->xLUb, this->dFPostResUb, this->dHLResUb, this->dHLResTmp1,	339	this->xLUb, this->dFPostResUb, this->dHLResUb, this->dHLResTmp1,
342	- this->dHLResTmp3, this->n, this->tailC, this->n, this->alignN);	340	+ this->dHLResTmp3, this->n, this->tailC, this->n, this->alignN);
343		341
344	// dFout: H_post@dF	342	// dFout: H_post@dF
345	// [1, n]@[n, tailC] = [1, tailC] 直接搬出	343	// [1, n]@[n, tailC] = [1, tailC] 直接搬出
	@@ -358,9 +356,9 @@ __aicore__ inline void KernelMhcPostBackward<T>::Process()
358	static_cast<uint16_t>(this->n), static_cast<uint32_t>(this->tailC * sizeof(T)), 0,	356	static_cast<uint16_t>(this->n), static_cast<uint32_t>(this->tailC * sizeof(T)), 0,
359	static_cast<uint32_t>(channelStride * sizeof(T)), 0};	357	static_cast<uint32_t>(channelStride * sizeof(T)), 0};
360	DataCopyExtParams copyParamsdFOutGm{1, static_cast<uint32_t>(this->tailC * sizeof(T)), 0, 0, 0};	358	DataCopyExtParams copyParamsdFOutGm{1, static_cast<uint32_t>(this->tailC * sizeof(T)), 0, 0, 0};
361	- Cast(this->dFOutCastUb, this->dFOutUb, RoundMode::CAST_ROUND, this->tailC); // float--> bf16	359	+ Cast(this->dFOutCastUb, this->dFOutUb, RoundMode::CAST_ROUND, this->tailC); // float--> bf16
362	Cast(this->dxLCastUb, this->dxLUb, RoundMode::CAST_ROUND, this->n * this->tailC); // float--> bf16	360	Cast(this->dxLCastUb, this->dxLUb, RoundMode::CAST_ROUND, this->n * this->tailC); // float--> bf16
363	-	361	+
364	SetFlag<HardEvent::V_MTE3>(0);	362	SetFlag<HardEvent::V_MTE3>(0);
365	WaitFlag<HardEvent::V_MTE3>(0);	363	WaitFlag<HardEvent::V_MTE3>(0);
366	DataCopyPad(	364	DataCopyPad(
	@@ -384,11 +382,11 @@ __aicore__ inline void KernelMhcPostBackward<T>::Process()
384		382
385	template <typename T>	383	template <typename T>
386	__aicore__ inline void KernelMhcPostBackward<T>::VecMatmulMknk(	384	__aicore__ inline void KernelMhcPostBackward<T>::VecMatmulMknk(
387	- LocalTensor<float>& A,	385	+ LocalTensor<float> &A,
388	- LocalTensor<float>& B,	386	+ LocalTensor<float> &B,
389	- LocalTensor<float>& C,	387	+ LocalTensor<float> &C,
390	- LocalTensor<float>& broadcastBuffer,	388	+ LocalTensor<float> &broadcastBuffer,
391	- LocalTensor<float>& reduceBuffer,	389	+ LocalTensor<float> &reduceBuffer,
392	uint32_t m, uint32_t k, uint32_t n, uint32_t alignN)	390	uint32_t m, uint32_t k, uint32_t n, uint32_t alignN)
393	{	391	{
394	// Compute matrix multiplication using vector instructions.	392	// Compute matrix multiplication using vector instructions.
	@@ -398,7 +396,6 @@ __aicore__ inline void KernelMhcPostBackward<T>::VecMatmulMknk(
398		396
399	auto tempBuffer = this->dHLResTmpBuf4.template Get<uint8_t>();	397	auto tempBuffer = this->dHLResTmpBuf4.template Get<uint8_t>();
400	for (int32_t i = 0; i < m; i++) {	398	for (int32_t i = 0; i < m; i++) {
401	-
402	// 逐元素相乘	399	// 逐元素相乘
403	PipeBarrier<PIPE_V>();	400	PipeBarrier<PIPE_V>();
404	Mul(broadcastBuffer[0], A[i * k], B[0], k);	401	Mul(broadcastBuffer[0], A[i * k], B[0], k);
	@@ -411,7 +408,7 @@ __aicore__ inline void KernelMhcPostBackward<T>::VecMatmulMknk(
411	reduceBuffer[i * alignN], broadcastBuffer,	408	reduceBuffer[i * alignN], broadcastBuffer,
412	tempBuffer, reduceShape, true);	409	tempBuffer, reduceShape, true);
413	}	410	}
414	-	411	+
415	// 累加到输出 C	412	// 累加到输出 C
416	PipeBarrier<PIPE_V>();	413	PipeBarrier<PIPE_V>();
417	Add(C, C, reduceBuffer, m * alignN);	414	Add(C, C, reduceBuffer, m * alignN);
	@@ -419,23 +416,22 @@ __aicore__ inline void KernelMhcPostBackward<T>::VecMatmulMknk(
419		416
420	template <typename T>	417	template <typename T>
421	__aicore__ inline void KernelMhcPostBackward<T>::VecMatmulMkkn(	418	__aicore__ inline void KernelMhcPostBackward<T>::VecMatmulMkkn(
422	- LocalTensor<float>&A,	419	+ LocalTensor<float> &A,
423	- LocalTensor<float>&B,	420	+ LocalTensor<float> &B,
424	- LocalTensor<float>&C,	421	+ LocalTensor<float> &C,
425	- LocalTensor<float>&broadcastBuffer,	422	+ LocalTensor<float> &broadcastBuffer,
426	uint32_t m,	423	uint32_t m,
427	uint32_t k,	424	uint32_t k,
428	- uint32_t n	425	+ uint32_t n)
429	-)
430	{	426	{
431	- uint32_t broadcastSrcShape[] = {m*k, 1};	427	+ uint32_t broadcastSrcShape[] = {m * k, 1};
432	- uint32_t broadcastDstShape[] = {m*k, n};	428	+ uint32_t broadcastDstShape[] = {m * k, n};
433	Broadcast<float, 2, 1>(broadcastBuffer, A, broadcastDstShape, broadcastSrcShape);	429	Broadcast<float, 2, 1>(broadcastBuffer, A, broadcastDstShape, broadcastSrcShape);
434	PipeBarrier<PIPE_V>();	430	PipeBarrier<PIPE_V>();
435	for (int32_t i = 0; i < m; i++) {	431	for (int32_t i = 0; i < m; i++) {
436	for (int32_t j = 0; j < k; j++) {	432	for (int32_t j = 0; j < k; j++) {
437	- MulAddDst(C[i * n], broadcastBuffer[(im+j)n], B[j * n], n);	433	+ MulAddDst(C[i * n], broadcastBuffer[(i * m + j) * n], B[j * n], n);
438	}	434	}
439	}	435	}
440	}	436	}
441	-#endif	437	+#endif

Mmhc/mhc_post_backward/op_kernel/mhc_post_backward.cpp+2-2

	@@ -13,7 +13,7 @@
13	* \brief	13	* \brief
14	*/	14	*/
15		15
16	-#include "./arch22/mhc_post_backward.h"	16	+#include "./arch22/mhc_post_backward_arch22.h"
17	#include "./arch22/mhc_post_backward_tiling_data_arch22.h"	17	#include "./arch22/mhc_post_backward_tiling_data_arch22.h"
18	#include "./arch22/mhc_post_backward_tiling_key_arch22.h"	18	#include "./arch22/mhc_post_backward_tiling_key_arch22.h"
19	#include "kernel_operator.h"	19	#include "kernel_operator.h"
	@@ -33,4 +33,4 @@ __global__ __aicore__ void mhc_post_backward(
33	op.Init(grad_y, x, h_res, h_out, h_post, grad_x, grad_h_res, grad_h_out,	33	op.Init(grad_y, x, h_res, h_out, h_post, grad_x, grad_h_res, grad_h_out,
34	grad_h_post, tiling_data, &pipe);	34	grad_h_post, tiling_data, &pipe);
35	op.Process();	35	op.Process();
36	-}	36	+}

Rmhc/mhc_sinkhorn/op_kernel/arch35/mhc_sinkhorn.h→mhc/mhc_sinkhorn/op_kernel/arch35/mhc_sinkhorn_arch35.h+5-5

	@@ -8,12 +8,12 @@
8	* See LICENSE in the root of the software repository for the full text of the License.	8	* See LICENSE in the root of the software repository for the full text of the License.
9	*/	9	*/
10	/*!	10	/*!
11	- * \file mhc_sinkhorn.h	11	+ * \file mhc_sinkhorn_arch35.h
12	* \brief mhc_sinkhorn	12	* \brief mhc_sinkhorn
13	*/	13	*/
14		14
15	-#ifndef ASCENDC_MHC_SINKHORN_H	15	+#ifndef ASCENDC_MHC_SINKHORN_ARCH35_H
16	-#define ASCENDC_MHC_SINKHORN_H	16	+#define ASCENDC_MHC_SINKHORN_ARCH35_H
17		17
18	#include "kernel_operator.h"	18	#include "kernel_operator.h"
19	#include "op_kernel/math_util.h"	19	#include "op_kernel/math_util.h"
	@@ -42,7 +42,8 @@ template <typename T, bool OUT_FLAG>
42	class MhcSinkhornSimd {	42	class MhcSinkhornSimd {
43	public:	43	public:
44	__aicore__ inline MhcSinkhornSimd(TPipe &pipe, const MhcSinkhornTilingData &tilingData)	44	__aicore__ inline MhcSinkhornSimd(TPipe &pipe, const MhcSinkhornTilingData &tilingData)
45	- : pipe_(pipe), tilingData_(tilingData){};	45	+ : pipe_(pipe),
		46	+ tilingData_(tilingData){};
46	__aicore__ inline void Init(GM_ADDR h_res, GM_ADDR y, GM_ADDR norm_out, GM_ADDR sum_out, GM_ADDR tiling);	47	__aicore__ inline void Init(GM_ADDR h_res, GM_ADDR y, GM_ADDR norm_out, GM_ADDR sum_out, GM_ADDR tiling);
47	__aicore__ inline void Process();	48	__aicore__ inline void Process();
48		49
	@@ -383,7 +384,6 @@ __aicore__ inline void MhcSinkhornSimd<T, OUT_FLAG>::CalcRowNorm(int32_t handleN
383	sumRowQue_.EnQue(sumRowLocal);	384	sumRowQue_.EnQue(sumRowLocal);
384	}	385	}
385		386
386	-
387	template <typename T, bool OUT_FLAG>	387	template <typename T, bool OUT_FLAG>
388	__aicore__ inline void MhcSinkhornSimd<T, OUT_FLAG>::ScatterOutFromNorm(int32_t handleNum)	388	__aicore__ inline void MhcSinkhornSimd<T, OUT_FLAG>::ScatterOutFromNorm(int32_t handleNum)
389	{	389	{

Mmhc/mhc_sinkhorn/op_kernel/mhc_sinkhorn_apt.cpp+5-5

	@@ -10,16 +10,16 @@
10	/*!	10	/*!
11	* \file mhc_sinkhorn_apt.cpp	11	* \file mhc_sinkhorn_apt.cpp
12	* \brief mhc_sinkhorn	12	* \brief mhc_sinkhorn
13	- */	13	+ */
14		14
15	-#include "arch35/mhc_sinkhorn.h"	15	+#include "arch35/mhc_sinkhorn_arch35.h"
16		16
17	using namespace AscendC;	17	using namespace AscendC;
18	using namespace MhcSinkhorn;	18	using namespace MhcSinkhorn;
19		19
20	template <int64_t TEMPLATE_MODE>	20	template <int64_t TEMPLATE_MODE>
21	-__global__ __aicore__ void mhc_sinkhorn(GM_ADDR h_res, GM_ADDR y, GM_ADDR norm_out,	21	+__global__ __aicore__ void mhc_sinkhorn(GM_ADDR h_res, GM_ADDR y, GM_ADDR norm_out,
22	- GM_ADDR sum_out, GM_ADDR workSpace, GM_ADDR tiling)	22	+ GM_ADDR sum_out, GM_ADDR workSpace, GM_ADDR tiling)
23	{	23	{
24	GM_ADDR user = GetUserWorkspace(workSpace);	24	GM_ADDR user = GetUserWorkspace(workSpace);
25	if (user == nullptr) {	25	if (user == nullptr) {
	@@ -39,4 +39,4 @@ __global__ __aicore__ void mhc_sinkhorn(GM_ADDR h_res, GM_ADDR y, GM_ADDR norm_o
39	op.Init(h_res, y, norm_out, sum_out, tiling);	39	op.Init(h_res, y, norm_out, sum_out, tiling);
40	op.Process();	40	op.Process();
41	}	41	}
42	-}	42	+}

Rposembedding/inplace_partial_rotary_mul/op_kernel/arch35/rotary_position_embedding_a_and_b.h→posembedding/inplace_partial_rotary_mul/op_kernel/arch35/inplace_partial_rotary_mul_a_and_b.h+39-38

	@@ -9,13 +9,13 @@
9	*/	9	*/
10		10
11	/*!	11	/*!
12	- * \file rotary_position_embedding_a_and_b.h	12	+ * \file inplace_partial_rotary_mul_a_and_b.h
13	* \brief	13	* \brief
14	*/	14	*/
15	-#ifndef ROTARY_POSITION_EMBEDDING_A_AND_B_H	15	+#ifndef INPLACE_PARTIAL_ROTARY_MUL_A_AND_B_H
16	-#define ROTARY_POSITION_EMBEDDING_A_AND_B_H	16	+#define INPLACE_PARTIAL_ROTARY_MUL_A_AND_B_H
17		17
18	-#include "apply_rotary_pos_emb_common.h"	18	+#include "inplace_partial_rotary_mul_arpe_common.h"
19		19
20	namespace InplacePartialRotaryMul {	20	namespace InplacePartialRotaryMul {
21	using namespace AscendC;	21	using namespace AscendC;
	@@ -29,8 +29,8 @@ public:
29		29
30	__aicore__ inline void Init(	30	__aicore__ inline void Init(
31	GM_ADDR q, GM_ADDR cos, GM_ADDR sin, GM_ADDR qOut, GM_ADDR workspace,	31	GM_ADDR q, GM_ADDR cos, GM_ADDR sin, GM_ADDR qOut, GM_ADDR workspace,
32	- const InplacePartialRopeRegbaseTilingData* tilingData,	32	+ const InplacePartialRopeRegbaseTilingData *tilingData,
33	- TPipe* pipe);	33	+ TPipe *pipe);
34		34
35	__aicore__ inline void Process();	35	__aicore__ inline void Process();
36		36
	@@ -40,19 +40,19 @@ private:
40	__aicore__ inline void InitAllBuffer();	40	__aicore__ inline void InitAllBuffer();
41	__aicore__ inline void InitLoopParams();	41	__aicore__ inline void InitLoopParams();
42	// 各个层级的Process函数	42	// 各个层级的Process函数
43	- __aicore__ inline void ProcessInLoop(LocalTensor<T>& cos, LocalTensor<T>& sin, int64_t bStart, int64_t bLength);	43	+ __aicore__ inline void ProcessInLoop(LocalTensor<T> &cos, LocalTensor<T> &sin, int64_t bStart, int64_t bLength);
44	// 拷入拷出函数	44	// 拷入拷出函数
45	__aicore__ inline void CopyInCosAndSin(int64_t bStart, int64_t bLength);	45	__aicore__ inline void CopyInCosAndSin(int64_t bStart, int64_t bLength);
46	- __aicore__ inline void CopyInQ(GlobalTensor<T>& source, int64_t bStart, int64_t bLength);	46	+ __aicore__ inline void CopyInQ(GlobalTensor<T> &source, int64_t bStart, int64_t bLength);
47	- __aicore__ inline void CopyOutQ(GlobalTensor<T>& target, int64_t bStart, int64_t bLength);	47	+ __aicore__ inline void CopyOutQ(GlobalTensor<T> &target, int64_t bStart, int64_t bLength);
48		48
49	// 计算函数	49	// 计算函数
50	- __aicore__ inline void Compute(LocalTensor<T>& cos, LocalTensor<T>& sin, int64_t bLength);	50	+ __aicore__ inline void Compute(LocalTensor<T> &cos, LocalTensor<T> &sin, int64_t bLength);
51		51
52	private:	52	private:
53	constexpr static uint32_t COS_DB_BUFFER = IsBoardCast ? 1 : DOUBLE_BUFFER;	53	constexpr static uint32_t COS_DB_BUFFER = IsBoardCast ? 1 : DOUBLE_BUFFER;
54		54
55	- TPipe* pipe_;	55	+ TPipe *pipe_;
56		56
57	// GlobalMemory	57	// GlobalMemory
58	GlobalTensor<T> qGm_;	58	GlobalTensor<T> qGm_;
	@@ -72,7 +72,7 @@ private:
72	int64_t bBlockLength_ = 0;	72	int64_t bBlockLength_ = 0;
73		73
74	// TilingData	74	// TilingData
75	- const InplacePartialRopeRegbaseTilingData* tilingData_;	75	+ const InplacePartialRopeRegbaseTilingData *tilingData_;
76	int64_t ubFactorB_ = 0;	76	int64_t ubFactorB_ = 0;
77	int64_t D_ = 0;	77	int64_t D_ = 0;
78	int64_t dAlign_ = 0;	78	int64_t dAlign_ = 0;
	@@ -86,8 +86,8 @@ private:
86	template <typename T, bool IsBoardCast>	86	template <typename T, bool IsBoardCast>
87	__aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::Init(	87	__aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::Init(
88	GM_ADDR q, GM_ADDR cos, GM_ADDR sin, GM_ADDR qOut, GM_ADDR workspace,	88	GM_ADDR q, GM_ADDR cos, GM_ADDR sin, GM_ADDR qOut, GM_ADDR workspace,
89	- const InplacePartialRopeRegbaseTilingData* tilingData,	89	+ const InplacePartialRopeRegbaseTilingData *tilingData,
90	- TPipe* pipe)	90	+ TPipe *pipe)
91	{	91	{
92	this->tilingData_ = tilingData;	92	this->tilingData_ = tilingData;
93	this->pipe_ = pipe;	93	this->pipe_ = pipe;
	@@ -101,10 +101,10 @@ template <typename T, bool IsBoardCast>
101	__aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::InitAllGlobalBuffer(	101	__aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::InitAllGlobalBuffer(
102	GM_ADDR q, GM_ADDR cos, GM_ADDR sin, GM_ADDR qOut)	102	GM_ADDR q, GM_ADDR cos, GM_ADDR sin, GM_ADDR qOut)
103	{	103	{
104	- this->qGm_.SetGlobalBuffer((__gm__ T*)q);	104	+ this->qGm_.SetGlobalBuffer((__gm__ T *)q);
105	- this->cosGm_.SetGlobalBuffer((__gm__ T*)cos);	105	+ this->cosGm_.SetGlobalBuffer((__gm__ T *)cos);
106	- this->sinGm_.SetGlobalBuffer((__gm__ T*)sin);	106	+ this->sinGm_.SetGlobalBuffer((__gm__ T *)sin);
107	- this->qOutGm_.SetGlobalBuffer((__gm__ T*)qOut);	107	+ this->qOutGm_.SetGlobalBuffer((__gm__ T *)qOut);
108	}	108	}
109		109
110	template <typename T, bool IsBoardCast>	110	template <typename T, bool IsBoardCast>
	@@ -120,14 +120,15 @@ __aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::InitAllBuff
120	}	120	}
121	this->copyInQSplitCoef_ = dSplitCoef_;	121	this->copyInQSplitCoef_ = dSplitCoef_;
122	this->dAlign_ = ops::CeilAlign<int64_t>(tilingData_->sliceLength / dSplitCoef_,	122	this->dAlign_ = ops::CeilAlign<int64_t>(tilingData_->sliceLength / dSplitCoef_,
123	- BLOCK_TYPE_SIZE / sizeof(T)) * dSplitCoef_;	123	+ BLOCK_TYPE_SIZE / sizeof(T)) *
		124	+ dSplitCoef_;
124	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::DEEPSEEK_INTERLEAVE)) {	125	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::DEEPSEEK_INTERLEAVE)) {
125	this->copyInQSplitCoef_ = 1;	126	this->copyInQSplitCoef_ = 1;
126	// 非boardcast时，使用批量计算API，需要拷贝时添加stride	127	// 非boardcast时，使用批量计算API，需要拷贝时添加stride
127	if constexpr (!IsBoardCast) {	128	if constexpr (!IsBoardCast) {
128	this->ubCopyInStride =	129	this->ubCopyInStride =
129	(this->dAlign_ * sizeof(T) - ops::CeilAlign<int64_t>(tilingData_->sliceLength * sizeof(T),	130	(this->dAlign_ * sizeof(T) - ops::CeilAlign<int64_t>(tilingData_->sliceLength * sizeof(T),
130	- BLOCK_TYPE_SIZE)) /	131	+ BLOCK_TYPE_SIZE)) /
131	BLOCK_TYPE_SIZE;	132	BLOCK_TYPE_SIZE;
132	}	133	}
133	}	134	}
	@@ -187,7 +188,7 @@ __aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::Process()
187		188
188	template <typename T, bool IsBoardCast>	189	template <typename T, bool IsBoardCast>
189	__aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::ProcessInLoop(	190	__aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::ProcessInLoop(
190	- LocalTensor<T>& cos, LocalTensor<T>& sin, int64_t bUbStart, int64_t bUbLength)	191	+ LocalTensor<T> &cos, LocalTensor<T> &sin, int64_t bUbStart, int64_t bUbLength)
191	{	192	{
192	CopyInQ(qGm_, bUbStart, bUbLength);	193	CopyInQ(qGm_, bUbStart, bUbLength);
193	Compute(cos, sin, bUbLength);	194	Compute(cos, sin, bUbLength);
	@@ -217,7 +218,7 @@ __aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::CopyInCosAn
217		218
218	template <typename T, bool IsBoardCast>	219	template <typename T, bool IsBoardCast>
219	__aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::CopyInQ(	220	__aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::CopyInQ(
220	- GlobalTensor<T>& source, int64_t bStart, int64_t bLength)	221	+ GlobalTensor<T> &source, int64_t bStart, int64_t bLength)
221	{	222	{
222	LocalTensor<T> target = this->qInQueue_.template AllocTensor<T>();	223	LocalTensor<T> target = this->qInQueue_.template AllocTensor<T>();
223	DataCopyExtParams copyExtParams;	224	DataCopyExtParams copyExtParams;
	@@ -236,7 +237,7 @@ __aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::CopyInQ(
236		237
237	template <typename T, bool IsBoardCast>	238	template <typename T, bool IsBoardCast>
238	__aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::CopyOutQ(	239	__aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::CopyOutQ(
239	- GlobalTensor<T>& target, int64_t bStart, int64_t bLength)	240	+ GlobalTensor<T> &target, int64_t bStart, int64_t bLength)
240	{	241	{
241	LocalTensor<T> source = this->qOutQueue_.template DeQue<T>();	242	LocalTensor<T> source = this->qOutQueue_.template DeQue<T>();
242	DataCopyExtParams copyExtParams;	243	DataCopyExtParams copyExtParams;
	@@ -244,13 +245,13 @@ __aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::CopyOutQ(
244	copyExtParams.blockLen = tilingData_->sliceLength * sizeof(T) / dSplitCoef_;	245	copyExtParams.blockLen = tilingData_->sliceLength * sizeof(T) / dSplitCoef_;
245	copyExtParams.srcStride = 0;	246	copyExtParams.srcStride = 0;
246	copyExtParams.dstStride = (tilingData_->D - tilingData_->sliceLength) * sizeof(T);	247	copyExtParams.dstStride = (tilingData_->D - tilingData_->sliceLength) * sizeof(T);
247	- DataCopyPad(target[bStart * D_+ tilingData_->sliceStart], source, copyExtParams);	248	+ DataCopyPad(target[bStart * D_ + tilingData_->sliceStart], source, copyExtParams);
248	this->qOutQueue_.FreeTensor(source);	249	this->qOutQueue_.FreeTensor(source);
249	}	250	}
250		251
251	template <typename T, bool IsBoardCast>	252	template <typename T, bool IsBoardCast>
252	__aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::Compute(	253	__aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::Compute(
253	- LocalTensor<T>& cos, LocalTensor<T>& sin, int64_t bLength)	254	+ LocalTensor<T> &cos, LocalTensor<T> &sin, int64_t bLength)
254	{	255	{
255	LocalTensor<T> inUb = this->qInQueue_.template DeQue<T>();	256	LocalTensor<T> inUb = this->qInQueue_.template DeQue<T>();
256	LocalTensor<T> outUb = this->qOutQueue_.template AllocTensor<T>();	257	LocalTensor<T> outUb = this->qOutQueue_.template AllocTensor<T>();
	@@ -267,27 +268,27 @@ __aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::Compute(
267	} else {	268	} else {
268	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::HALF)) {	269	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::HALF)) {
269	BatchHalfAlignVF<T, IsBoardCast>(	270	BatchHalfAlignVF<T, IsBoardCast>(
270	- (__ubuf__ T)inUb.GetPhyAddr(), (__ubuf__ T)cos.GetPhyAddr(),	271	+ (__ubuf__ T )inUb.GetPhyAddr(), (__ubuf__ T )cos.GetPhyAddr(),
271	- (__ubuf__ T)sin.GetPhyAddr(), (__ubuf__ T)outUb.GetPhyAddr(), bLength, 1, 1,	272	+ (__ubuf__ T )sin.GetPhyAddr(), (__ubuf__ T )outUb.GetPhyAddr(), bLength, 1, 1,
272	- tilingData_->sliceLength, dAlign_,	273	+ tilingData_->sliceLength, dAlign_,
273	ubFactorB_, 1);	274	ubFactorB_, 1);
274	} else if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::INTERLEAVE)) {	275	} else if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::INTERLEAVE)) {
275	BatchInterleaveModeVF<T, IsBoardCast>(	276	BatchInterleaveModeVF<T, IsBoardCast>(
276	- (__ubuf__ T)inUb.GetPhyAddr(), (__ubuf__ T)cos.GetPhyAddr(),	277	+ (__ubuf__ T )inUb.GetPhyAddr(), (__ubuf__ T )cos.GetPhyAddr(),
277	- (__ubuf__ T)sin.GetPhyAddr(), (__ubuf__ T)outUb.GetPhyAddr(), bLength, 1, 1,	278	+ (__ubuf__ T )sin.GetPhyAddr(), (__ubuf__ T )outUb.GetPhyAddr(), bLength, 1, 1,
278	- tilingData_->sliceLength, dAlign_,	279	+ tilingData_->sliceLength, dAlign_,
279	ubFactorB_, 1);	280	ubFactorB_, 1);
280	} else if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::QUARTER)) {	281	} else if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::QUARTER)) {
281	BatchQuarterAlignVF<T, IsBoardCast>(	282	BatchQuarterAlignVF<T, IsBoardCast>(
282	- (__ubuf__ T)inUb.GetPhyAddr(), (__ubuf__ T)cos.GetPhyAddr(),	283	+ (__ubuf__ T )inUb.GetPhyAddr(), (__ubuf__ T )cos.GetPhyAddr(),
283	- (__ubuf__ T)sin.GetPhyAddr(), (__ubuf__ T)outUb.GetPhyAddr(), bLength, 1, 1,	284	+ (__ubuf__ T )sin.GetPhyAddr(), (__ubuf__ T )outUb.GetPhyAddr(), bLength, 1, 1,
284	- tilingData_->sliceLength, dAlign_,	285	+ tilingData_->sliceLength, dAlign_,
285	ubFactorB_, 1);	286	ubFactorB_, 1);
286	} else {	287	} else {
287	BatchDeepSeekInterleaveModeVF<T, IsBoardCast>(	288	BatchDeepSeekInterleaveModeVF<T, IsBoardCast>(
288	- (__ubuf__ T)inUb.GetPhyAddr(), (__ubuf__ T)cos.GetPhyAddr(),	289	+ (__ubuf__ T )inUb.GetPhyAddr(), (__ubuf__ T )cos.GetPhyAddr(),
289	- (__ubuf__ T)sin.GetPhyAddr(), (__ubuf__ T)outUb.GetPhyAddr(), bLength, 1, 1,	290	+ (__ubuf__ T )sin.GetPhyAddr(), (__ubuf__ T )outUb.GetPhyAddr(), bLength, 1, 1,
290	- tilingData_->sliceLength, dAlign_,	291	+ tilingData_->sliceLength, dAlign_,
291	ubFactorB_, 1);	292	ubFactorB_, 1);
292	}	293	}
293	}	294	}
	@@ -297,4 +298,4 @@ __aicore__ inline void RotaryPositionEmbeddingAAndB<T, IsBoardCast>::Compute(
297	}	298	}
298	} // namespace InplacePartialRotaryMul	299	} // namespace InplacePartialRotaryMul
299		300
300	-#endif	301	+#endif

Rposembedding/inplace_partial_rotary_mul/op_kernel/arch35/rotary_position_embedding_ab.h→posembedding/inplace_partial_rotary_mul/op_kernel/arch35/inplace_partial_rotary_mul_ab.h+26-28

	@@ -9,13 +9,13 @@
9	*/	9	*/
10		10
11	/*!	11	/*!
12	- * \file rotary_position_embedding_ab.h	12	+ * \file inplace_partial_rotary_mul_ab.h
13	* \brief	13	* \brief
14	*/	14	*/
15	-#ifndef ROTARY_POSITION_EMBEDDING_AB_H	15	+#ifndef INPLACE_PARTIAL_ROTARY_MUL_AB_H
16	-#define ROTARY_POSITION_EMBEDDING_AB_H	16	+#define INPLACE_PARTIAL_ROTARY_MUL_AB_H
17		17
18	-#include "apply_rotary_pos_emb_common.h"	18	+#include "inplace_partial_rotary_mul_arpe_common.h"
19		19
20	namespace InplacePartialRotaryMul {	20	namespace InplacePartialRotaryMul {
21	using namespace AscendC;	21	using namespace AscendC;
	@@ -26,8 +26,8 @@ public:
26	__aicore__ inline RotaryPositionEmbeddingAB(){};	26	__aicore__ inline RotaryPositionEmbeddingAB(){};
27	__aicore__ inline void Init(	27	__aicore__ inline void Init(
28	GM_ADDR x, GM_ADDR cos, GM_ADDR sin, GM_ADDR y, GM_ADDR workspace,	28	GM_ADDR x, GM_ADDR cos, GM_ADDR sin, GM_ADDR y, GM_ADDR workspace,
29	- const InplacePartialRopeRegbaseTilingData* tilingData,	29	+ const InplacePartialRopeRegbaseTilingData *tilingData,
30	- TPipe* pipe);	30	+ TPipe *pipe);
31	__aicore__ inline void Process();	31	__aicore__ inline void Process();
32		32
33	private:	33	private:
	@@ -36,7 +36,7 @@ private:
36	int64_t nCount);	36	int64_t nCount);
37		37
38	private:	38	private:
39	- TPipe* pipe_;	39	+ TPipe *pipe_;
40	TQue<QuePosition::VECIN, 1> xInQueue_;	40	TQue<QuePosition::VECIN, 1> xInQueue_;
41	TQue<QuePosition::VECIN, 1> cosInQueue_;	41	TQue<QuePosition::VECIN, 1> cosInQueue_;
42	TQue<QuePosition::VECIN, 1> sinInQueue_;	42	TQue<QuePosition::VECIN, 1> sinInQueue_;
	@@ -46,7 +46,7 @@ private:
46	GlobalTensor<T> cosGm_;	46	GlobalTensor<T> cosGm_;
47	GlobalTensor<T> sinGm_;	47	GlobalTensor<T> sinGm_;
48	GlobalTensor<T> yGm_;	48	GlobalTensor<T> yGm_;
49	- const InplacePartialRopeRegbaseTilingData* tilingData_;	49	+ const InplacePartialRopeRegbaseTilingData *tilingData_;
50	DataCopyPadExtParams<T> padParams_ = {false, 0, 0, static_cast<T>(0)};	50	DataCopyPadExtParams<T> padParams_ = {false, 0, 0, static_cast<T>(0)};
51	uint8_t DB_FLAG = 2;	51	uint8_t DB_FLAG = 2;
52	uint32_t dSplitSize_ = 0;	52	uint32_t dSplitSize_ = 0;
	@@ -58,8 +58,8 @@ private:
58	template <typename T>	58	template <typename T>
59	__aicore__ inline void RotaryPositionEmbeddingAB<T>::Init(	59	__aicore__ inline void RotaryPositionEmbeddingAB<T>::Init(
60	GM_ADDR x, GM_ADDR cos, GM_ADDR sin, GM_ADDR y, GM_ADDR workspace,	60	GM_ADDR x, GM_ADDR cos, GM_ADDR sin, GM_ADDR y, GM_ADDR workspace,
61	- const InplacePartialRopeRegbaseTilingData* tilingData,	61	+ const InplacePartialRopeRegbaseTilingData *tilingData,
62	- TPipe* pipe)	62	+ TPipe *pipe)
63	{	63	{
64	pipe_ = pipe;	64	pipe_ = pipe;
65	tilingData_ = tilingData;	65	tilingData_ = tilingData;
	@@ -71,13 +71,11 @@ __aicore__ inline void RotaryPositionEmbeddingAB<T>::Init(
71		71
72	int64_t cosOffset = blockDimBS * tilingData_->blockFactorBS * tilingData_->sliceLength;	72	int64_t cosOffset = blockDimBS * tilingData_->blockFactorBS * tilingData_->sliceLength;
73	int64_t offset = blockDimBS * tilingData_->blockFactorBS * tilingData_->D;	73	int64_t offset = blockDimBS * tilingData_->blockFactorBS * tilingData_->D;
74	- int64_t xOffset = offset * tilingData_->N	74	+ int64_t xOffset = offset * tilingData_->N + blockDimN * tilingData_->blockFactorN * tilingData_->D + tilingData_->sliceStart;
75	- + blockDimN * tilingData_->blockFactorN * tilingData_->D	75	+ this->cosGm_.SetGlobalBuffer((__gm__ T *)cos + cosOffset);
76	- + tilingData_->sliceStart;	76	+ this->sinGm_.SetGlobalBuffer((__gm__ T *)sin + cosOffset);
77	- this->cosGm_.SetGlobalBuffer((__gm__ T*)cos + cosOffset);	77	+ this->xGm_.SetGlobalBuffer((__gm__ T *)x + xOffset);
78	- this->sinGm_.SetGlobalBuffer((__gm__ T*)sin + cosOffset);	78	+ this->yGm_.SetGlobalBuffer((__gm__ T *)y + xOffset);
79	- this->xGm_.SetGlobalBuffer((__gm__ T*)x + xOffset);
80	- this->yGm_.SetGlobalBuffer((__gm__ T*)y + xOffset);
81		79
82	sliceAlign_ = ops::CeilDiv(tilingData_->sliceLength * sizeof(T), GetUbBlockSize()) * GetUbBlockSize() / sizeof(T);	80	sliceAlign_ = ops::CeilDiv(tilingData_->sliceLength * sizeof(T), GetUbBlockSize()) * GetUbBlockSize() / sizeof(T);
83	int64_t bufferSize = sliceAlign_ * sizeof(T) * tilingData_->ubFactorBS;	81	int64_t bufferSize = sliceAlign_ * sizeof(T) * tilingData_->ubFactorBS;
	@@ -103,11 +101,11 @@ __aicore__ inline void RotaryPositionEmbeddingAB<T>::Process()
103	LocalTensor<T> cosBuffer = cosInQueue_.AllocTensor<T>();	101	LocalTensor<T> cosBuffer = cosInQueue_.AllocTensor<T>();
104	LocalTensor<T> sinBuffer = sinInQueue_.AllocTensor<T>();	102	LocalTensor<T> sinBuffer = sinInQueue_.AllocTensor<T>();
105	DataCopyPad(cosBuffer,	103	DataCopyPad(cosBuffer,
106	- cosGm_[bsLoopIdx * tilingData_->ubFactorBS * tilingData_->sliceLength], cosParams, padParams_);	104	+ cosGm_[bsLoopIdx * tilingData_->ubFactorBS * tilingData_->sliceLength], cosParams, padParams_);
107	cosInQueue_.EnQue(cosBuffer);	105	cosInQueue_.EnQue(cosBuffer);
108	cosBuffer = cosInQueue_.DeQue<T>();	106	cosBuffer = cosInQueue_.DeQue<T>();
109	DataCopyPad(sinBuffer,	107	DataCopyPad(sinBuffer,
110	- sinGm_[bsLoopIdx * tilingData_->ubFactorBS * tilingData_->sliceLength], cosParams, padParams_);	108	+ sinGm_[bsLoopIdx * tilingData_->ubFactorBS * tilingData_->sliceLength], cosParams, padParams_);
111	sinInQueue_.EnQue(sinBuffer);	109	sinInQueue_.EnQue(sinBuffer);
112	sinBuffer = sinInQueue_.DeQue<T>();	110	sinBuffer = sinInQueue_.DeQue<T>();
113		111
	@@ -129,12 +127,12 @@ __aicore__ inline void RotaryPositionEmbeddingAB<T>::ProcessLoop(
129	{	127	{
130	int64_t totalCount = bsCount * nCount;	128	int64_t totalCount = bsCount * nCount;
131	DataCopyExtParams inParams = {static_cast<uint16_t>(totalCount * tilingData_->dSplitCoef), dSplitSize_,	129	DataCopyExtParams inParams = {static_cast<uint16_t>(totalCount * tilingData_->dSplitCoef), dSplitSize_,
132	- static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(T)), 0, 0};	130	+ static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(T)), 0, 0};
133	DataCopyExtParams outParams = {static_cast<uint16_t>(totalCount * tilingData_->dSplitCoef), dSplitSize_, 0,	131	DataCopyExtParams outParams = {static_cast<uint16_t>(totalCount * tilingData_->dSplitCoef), dSplitSize_, 0,
134	- static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(T)), 0};	132	+ static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(T)), 0};
135	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::DEEPSEEK_INTERLEAVE)) {	133	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::DEEPSEEK_INTERLEAVE)) {
136	inParams = {static_cast<uint16_t>(totalCount), tilingData_->D * sizeof(T),	134	inParams = {static_cast<uint16_t>(totalCount), tilingData_->D * sizeof(T),
137	- static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(T)), 0, 0};	135	+ static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(T)), 0, 0};
138	}	136	}
139		137
140	LocalTensor<T> inBuffer = xInQueue_.AllocTensor<T>();	138	LocalTensor<T> inBuffer = xInQueue_.AllocTensor<T>();
	@@ -151,13 +149,13 @@ __aicore__ inline void RotaryPositionEmbeddingAB<T>::ProcessLoop(
151	InterleaveModeVF(sinBuffer, cosBuffer, inBuffer, outBuffer, tilingData_->sliceLength, bsCount, nCount);	149	InterleaveModeVF(sinBuffer, cosBuffer, inBuffer, outBuffer, tilingData_->sliceLength, bsCount, nCount);
152	} else if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::QUARTER)) {	150	} else if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::QUARTER)) {
153	QuarterAlignVF(sinBuffer,	151	QuarterAlignVF(sinBuffer,
154	- cosBuffer,	152	+ cosBuffer,
155	- inBuffer,	153	+ inBuffer,
156	- outBuffer,	154	+ outBuffer,
157	- tilingData_->sliceLength, sliceAlign_, bsCount, nCount);	155	+ tilingData_->sliceLength, sliceAlign_, bsCount, nCount);
158	} else {	156	} else {
159	DeepSeekInterleaveModeVF<T>(sinBuffer, cosBuffer, inBuffer, outBuffer, tilingData_->sliceLength, bsCount,	157	DeepSeekInterleaveModeVF<T>(sinBuffer, cosBuffer, inBuffer, outBuffer, tilingData_->sliceLength, bsCount,
160	- nCount);	158	+ nCount);
161	}	159	}
162		160
163	yOutQueue_.EnQue(outBuffer);	161	yOutQueue_.EnQue(outBuffer);
	@@ -171,4 +169,4 @@ __aicore__ inline void RotaryPositionEmbeddingAB<T>::ProcessLoop(
171		169
172	} // namespace InplacePartialRotaryMul	170	} // namespace InplacePartialRotaryMul
173		171
174	-#endif // ROTARY_POSITION_EMBEDDING_AB_H	172	+#endif // ROTARY_POSITION_EMBEDDING_AB_H

Rposembedding/inplace_partial_rotary_mul/op_kernel/arch35/rotary_position_embedding_aba_and_ba.h→posembedding/inplace_partial_rotary_mul/op_kernel/arch35/inplace_partial_rotary_mul_aba_and_ba.h+52-49

	@@ -9,13 +9,13 @@
9	*/	9	*/
10		10
11	/*!	11	/*!
12	- * \file rotary_position_embedding_aba_and_ba.h	12	+ * \file inplace_partial_rotary_mul_aba_and_ba.h
13	* \brief	13	* \brief
14	*/	14	*/
15	-#ifndef ROTARY_POSITION_EMBEDDING_ABA_AND_BA_H	15	+#ifndef INPLACE_PARTIAL_ROTARY_MUL_ABA_AND_BA_H
16	-#define ROTARY_POSITION_EMBEDDING_ABA_AND_BA_H	16	+#define INPLACE_PARTIAL_ROTARY_MUL_ABA_AND_BA_H
17		17
18	-#include "apply_rotary_pos_emb_common.h"	18	+#include "inplace_partial_rotary_mul_arpe_common.h"
19		19
20	namespace InplacePartialRotaryMul {	20	namespace InplacePartialRotaryMul {
21	using namespace AscendC;	21	using namespace AscendC;
	@@ -29,8 +29,8 @@ public:
29		29
30	__aicore__ inline void Init(	30	__aicore__ inline void Init(
31	GM_ADDR q, GM_ADDR cos, GM_ADDR sin, GM_ADDR qOut, GM_ADDR workspace,	31	GM_ADDR q, GM_ADDR cos, GM_ADDR sin, GM_ADDR qOut, GM_ADDR workspace,
32	- const InplacePartialRopeRegbaseTilingData* tilingData,	32	+ const InplacePartialRopeRegbaseTilingData *tilingData,
33	- TPipe* pipe);	33	+ TPipe *pipe);
34		34
35	__aicore__ inline void Process();	35	__aicore__ inline void Process();
36		36
	@@ -44,27 +44,27 @@ private:
44	int64_t sUbStart,	44	int64_t sUbStart,
45	int64_t sUbLength); // 第一重循环体，给定S范围，沿B轴进行遍历处理	45	int64_t sUbLength); // 第一重循环体，给定S范围，沿B轴进行遍历处理
46	__aicore__ inline void ProcessInSBLoop(	46	__aicore__ inline void ProcessInSBLoop(
47	- int64_t sUbStart, int64_t sUbLength, int64_t bUbStart, int64_t bUbLength, LocalTensor<T>& cos,	47	+ int64_t sUbStart, int64_t sUbLength, int64_t bUbStart, int64_t bUbLength, LocalTensor<T> &cos,
48	- LocalTensor<T>& sin); // 第二重循环体，给定BS范围，沿Q和K的N轴进行遍历处理	48	+ LocalTensor<T> &sin); // 第二重循环体，给定BS范围，沿Q和K的N轴进行遍历处理
49	__aicore__ inline void ProcessInSBNLoop(	49	__aicore__ inline void ProcessInSBNLoop(
50	int64_t sUbStart, int64_t sUbLength, int64_t bUbStart, int64_t bUbLength, int64_t nUbStart, int64_t nUbLength,	50	int64_t sUbStart, int64_t sUbLength, int64_t bUbStart, int64_t bUbLength, int64_t nUbStart, int64_t nUbLength,
51	- int64_t nTotalSize, LocalTensor<T>& cos, LocalTensor<T>& sin, GlobalTensor<T>& in,	51	+ int64_t nTotalSize, LocalTensor<T> &cos, LocalTensor<T> &sin, GlobalTensor<T> &in,
52	- GlobalTensor<T>& out); // 第三重循环体，给定BSN范围，计算其中数据的rope	52	+ GlobalTensor<T> &out); // 第三重循环体，给定BSN范围，计算其中数据的rope
53	// 拷入拷出函数	53	// 拷入拷出函数
54	__aicore__ inline void CopyInCosAndSin(int64_t sStart, int64_t sLength, int64_t bStart, int64_t bLength);	54	__aicore__ inline void CopyInCosAndSin(int64_t sStart, int64_t sLength, int64_t bStart, int64_t bLength);
55	__aicore__ inline void CopyInQ(	55	__aicore__ inline void CopyInQ(
56	- GlobalTensor<T>& source, int64_t sStart, int64_t sLength, int64_t bStart, int64_t bLength, int64_t nStart,	56	+ GlobalTensor<T> &source, int64_t sStart, int64_t sLength, int64_t bStart, int64_t bLength, int64_t nStart,
57	int64_t nLength, int64_t nTotalSize);	57	int64_t nLength, int64_t nTotalSize);
58	__aicore__ inline void CopyOutQ(	58	__aicore__ inline void CopyOutQ(
59	- GlobalTensor<T>& target, int64_t sStart, int64_t sLength, int64_t bStart, int64_t bLength, int64_t nStart,	59	+ GlobalTensor<T> &target, int64_t sStart, int64_t sLength, int64_t bStart, int64_t bLength, int64_t nStart,
60	int64_t nLength, int64_t nTotalSize);	60	int64_t nLength, int64_t nTotalSize);
61		61
62	// 计算函数	62	// 计算函数
63	__aicore__ inline void Compute(	63	__aicore__ inline void Compute(
64	- LocalTensor<T>& cos, LocalTensor<T>& sin, int64_t sLength, int64_t bLength, int64_t nLength);	64	+ LocalTensor<T> &cos, LocalTensor<T> &sin, int64_t sLength, int64_t bLength, int64_t nLength);
65		65
66	private:	66	private:
67	- TPipe* pipe_;	67	+ TPipe *pipe_;
68		68
69	// GlobalMemory	69	// GlobalMemory
70	GlobalTensor<T> qGm_;	70	GlobalTensor<T> qGm_;
	@@ -86,7 +86,7 @@ private:
86	int64_t sBlockLength_ = 0;	86	int64_t sBlockLength_ = 0;
87		87
88	// TilingData	88	// TilingData
89	- const InplacePartialRopeRegbaseTilingData* tilingData_;	89	+ const InplacePartialRopeRegbaseTilingData *tilingData_;
90	int64_t ubFactorB_ = 0;	90	int64_t ubFactorB_ = 0;
91	int64_t ubFactorS_ = 0;	91	int64_t ubFactorS_ = 0;
92	int64_t ubFactorN_ = 0;	92	int64_t ubFactorN_ = 0;
	@@ -102,8 +102,8 @@ private:
102	template <typename T, bool IsBBoardcast>	102	template <typename T, bool IsBBoardcast>
103	__aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::Init(	103	__aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::Init(
104	GM_ADDR q, GM_ADDR cos, GM_ADDR sin, GM_ADDR qOut, GM_ADDR workspace,	104	GM_ADDR q, GM_ADDR cos, GM_ADDR sin, GM_ADDR qOut, GM_ADDR workspace,
105	- const InplacePartialRopeRegbaseTilingData* tilingData,	105	+ const InplacePartialRopeRegbaseTilingData *tilingData,
106	- TPipe* pipe)	106	+ TPipe *pipe)
107	{	107	{
108	this->tilingData_ = tilingData;	108	this->tilingData_ = tilingData;
109	this->blockIdx_ = GetBlockIdx();	109	this->blockIdx_ = GetBlockIdx();
	@@ -117,10 +117,10 @@ template <typename T, bool IsBBoardcast>
117	__aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::InitAllGlobalBuffer(	117	__aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::InitAllGlobalBuffer(
118	GM_ADDR q, GM_ADDR cos, GM_ADDR sin, GM_ADDR qOut)	118	GM_ADDR q, GM_ADDR cos, GM_ADDR sin, GM_ADDR qOut)
119	{	119	{
120	- this->qGm_.SetGlobalBuffer((__gm__ T*)q);	120	+ this->qGm_.SetGlobalBuffer((__gm__ T *)q);
121	- this->cosGm_.SetGlobalBuffer((__gm__ T*)cos);	121	+ this->cosGm_.SetGlobalBuffer((__gm__ T *)cos);
122	- this->sinGm_.SetGlobalBuffer((__gm__ T*)sin);	122	+ this->sinGm_.SetGlobalBuffer((__gm__ T *)sin);
123	- this->qOutGm_.SetGlobalBuffer((__gm__ T*)qOut);	123	+ this->qOutGm_.SetGlobalBuffer((__gm__ T *)qOut);
124	}	124	}
125		125
126	template <typename T, bool IsBBoardcast>	126	template <typename T, bool IsBBoardcast>
	@@ -138,12 +138,14 @@ __aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::InitAll
138	}	138	}
139	this->copyInQSplitCoef_ = dSplitCoef_;	139	this->copyInQSplitCoef_ = dSplitCoef_;
140	this->dAlign_ = ops::CeilAlign<int64_t>(tilingData_->sliceLength / dSplitCoef_,	140	this->dAlign_ = ops::CeilAlign<int64_t>(tilingData_->sliceLength / dSplitCoef_,
141	- BLOCK_TYPE_SIZE / sizeof(T)) * dSplitCoef_;	141	+ BLOCK_TYPE_SIZE / sizeof(T)) *
		142	+ dSplitCoef_;
142	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::DEEPSEEK_INTERLEAVE)) {	143	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::DEEPSEEK_INTERLEAVE)) {
143	this->copyInQSplitCoef_ = 1;	144	this->copyInQSplitCoef_ = 1;
144	this->ubCopyInStride =	145	this->ubCopyInStride =
145	(this->dAlign_ * sizeof(T) - ops::CeilAlign<int64_t>(tilingData_->sliceLength * sizeof(T),	146	(this->dAlign_ * sizeof(T) - ops::CeilAlign<int64_t>(tilingData_->sliceLength * sizeof(T),
146	- BLOCK_TYPE_SIZE)) / BLOCK_TYPE_SIZE;	147	+ BLOCK_TYPE_SIZE)) /
		148	+ BLOCK_TYPE_SIZE;
147	}	149	}
148	this->pipe_->InitBuffer(this->qInQueue_, DOUBLE_BUFFER, ubFactorB_ * ubFactorS_ * ubFactorN_ * dAlign_ * sizeof(T));	150	this->pipe_->InitBuffer(this->qInQueue_, DOUBLE_BUFFER, ubFactorB_ * ubFactorS_ * ubFactorN_ * dAlign_ * sizeof(T));
149	this->pipe_->InitBuffer(	151	this->pipe_->InitBuffer(
	@@ -223,7 +225,7 @@ __aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::Process
223		225
224	template <typename T, bool IsBBoardcast>	226	template <typename T, bool IsBBoardcast>
225	__aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::ProcessInSBLoop(	227	__aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::ProcessInSBLoop(
226	- int64_t sUbStart, int64_t sUbLength, int64_t bUbStart, int64_t bUbLength, LocalTensor<T>& cos, LocalTensor<T>& sin)	228	+ int64_t sUbStart, int64_t sUbLength, int64_t bUbStart, int64_t bUbLength, LocalTensor<T> &cos, LocalTensor<T> &sin)
227	{	229	{
228	// 循环处理Q	230	// 循环处理Q
229	int64_t qUbLoopCount = ops::CeilDiv(tilingData_->N, ubFactorN_);	231	int64_t qUbLoopCount = ops::CeilDiv(tilingData_->N, ubFactorN_);
	@@ -238,7 +240,7 @@ __aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::Process
238	template <typename T, bool IsBBoardcast>	240	template <typename T, bool IsBBoardcast>
239	__aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::ProcessInSBNLoop(	241	__aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::ProcessInSBNLoop(
240	int64_t sUbStart, int64_t sUbLength, int64_t bUbStart, int64_t bUbLength, int64_t nUbStart, int64_t nUbLength,	242	int64_t sUbStart, int64_t sUbLength, int64_t bUbStart, int64_t bUbLength, int64_t nUbStart, int64_t nUbLength,
241	- int64_t nTotalSize, LocalTensor<T>& cos, LocalTensor<T>& sin, GlobalTensor<T>& in, GlobalTensor<T>& out)	243	+ int64_t nTotalSize, LocalTensor<T> &cos, LocalTensor<T> &sin, GlobalTensor<T> &in, GlobalTensor<T> &out)
242	{	244	{
243	CopyInQ(in, sUbStart, sUbLength, bUbStart, bUbLength, nUbStart, nUbLength, nTotalSize);	245	CopyInQ(in, sUbStart, sUbLength, bUbStart, bUbLength, nUbStart, nUbLength, nTotalSize);
244	Compute(cos, sin, sUbLength, bUbLength, nUbLength);	246	Compute(cos, sin, sUbLength, bUbLength, nUbLength);
	@@ -270,13 +272,13 @@ __aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::CopyInC
270	copyExtParams.srcStride = 0;	272	copyExtParams.srcStride = 0;
271	copyExtParams.dstStride = 0;	273	copyExtParams.dstStride = 0;
272	DataCopyPad(cosUb,	274	DataCopyPad(cosUb,
273	- this->cosGm_[bStart * tilingData_->S * tilingData_->sliceLength +	275	+ this->cosGm_[bStart * tilingData_->S * tilingData_->sliceLength +
274	- sStart * tilingData_->sliceLength],	276	+ sStart * tilingData_->sliceLength],
275	- copyExtParams, copyPadExtparams);	277	+ copyExtParams, copyPadExtparams);
276	DataCopyPad(sinUb,	278	DataCopyPad(sinUb,
277	- this->sinGm_[bStart * tilingData_->S * tilingData_->sliceLength +	279	+ this->sinGm_[bStart * tilingData_->S * tilingData_->sliceLength +
278	- sStart * tilingData_->sliceLength],	280	+ sStart * tilingData_->sliceLength],
279	- copyExtParams, copyPadExtparams);	281	+ copyExtParams, copyPadExtparams);
280	ResetLoopModePara(DataCopyMVType::OUT_TO_UB);	282	ResetLoopModePara(DataCopyMVType::OUT_TO_UB);
281	this->cosInQueue_.template EnQue(cosUb);	283	this->cosInQueue_.template EnQue(cosUb);
282	this->sinInQueue_.template EnQue(sinUb);	284	this->sinInQueue_.template EnQue(sinUb);
	@@ -284,7 +286,7 @@ __aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::CopyInC
284		286
285	template <typename T, bool IsBBoardcast>	287	template <typename T, bool IsBBoardcast>
286	__aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::CopyInQ(	288	__aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::CopyInQ(
287	- GlobalTensor<T>& source, int64_t sStart, int64_t sLength, int64_t bStart, int64_t bLength, int64_t nStart,	289	+ GlobalTensor<T> &source, int64_t sStart, int64_t sLength, int64_t bStart, int64_t bLength, int64_t nStart,
288	int64_t nLength, int64_t nTotalSize)	290	int64_t nLength, int64_t nTotalSize)
289	{	291	{
290	LocalTensor<T> target = this->qInQueue_.template AllocTensor<T>();	292	LocalTensor<T> target = this->qInQueue_.template AllocTensor<T>();
	@@ -310,7 +312,7 @@ __aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::CopyInQ
310	DataCopyPad(	312	DataCopyPad(
311	target,	313	target,
312	source[bStart * nTotalSize * tilingData_->S * D_ +	314	source[bStart * nTotalSize * tilingData_->S * D_ +
313	- nStart * tilingData_->S * D_ + sStart * D_ + tilingData_->sliceStart],	315	+ nStart * tilingData_->S * D_ + sStart * D_ + tilingData_->sliceStart],
314	copyExtParams, copyPadExtparams);	316	copyExtParams, copyPadExtparams);
315	ResetLoopModePara(DataCopyMVType::OUT_TO_UB);	317	ResetLoopModePara(DataCopyMVType::OUT_TO_UB);
316	this->qInQueue_.template EnQue(target);	318	this->qInQueue_.template EnQue(target);
	@@ -318,7 +320,7 @@ __aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::CopyInQ
318		320
319	template <typename T, bool IsBBoardcast>	321	template <typename T, bool IsBBoardcast>
320	__aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::CopyOutQ(	322	__aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::CopyOutQ(
321	- GlobalTensor<T>& target, int64_t sStart, int64_t sLength, int64_t bStart, int64_t bLength, int64_t nStart,	323	+ GlobalTensor<T> &target, int64_t sStart, int64_t sLength, int64_t bStart, int64_t bLength, int64_t nStart,
322	int64_t nLength, int64_t nTotalSize)	324	int64_t nLength, int64_t nTotalSize)
323	{	325	{
324	LocalTensor<T> source = this->qOutQueue_.template DeQue<T>();	326	LocalTensor<T> source = this->qOutQueue_.template DeQue<T>();
	@@ -338,7 +340,8 @@ __aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::CopyOut
338	copyExtParams.dstStride = (tilingData_->D - tilingData_->sliceLength) * sizeof(T);	340	copyExtParams.dstStride = (tilingData_->D - tilingData_->sliceLength) * sizeof(T);
339	DataCopyPad(	341	DataCopyPad(
340	target[bStart * nTotalSize * tilingData_->S * D_ + nStart * tilingData_->S * D_ + sStart * D_ +	342	target[bStart * nTotalSize * tilingData_->S * D_ + nStart * tilingData_->S * D_ + sStart * D_ +
341	- tilingData_->sliceStart], source,	343	+ tilingData_->sliceStart],
		344	+ source,
342	copyExtParams);	345	copyExtParams);
343	ResetLoopModePara(DataCopyMVType::UB_TO_OUT);	346	ResetLoopModePara(DataCopyMVType::UB_TO_OUT);
344	this->qOutQueue_.FreeTensor(source);	347	this->qOutQueue_.FreeTensor(source);
	@@ -346,34 +349,34 @@ __aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::CopyOut
346		349
347	template <typename T, bool IsBBoardcast>	350	template <typename T, bool IsBBoardcast>
348	__aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::Compute(	351	__aicore__ inline void RotaryPositionEmbeddingABAAndBA<T, IsBBoardcast>::Compute(
349	- LocalTensor<T>& cos, LocalTensor<T>& sin, int64_t sLength, int64_t bLength, int64_t nLength)	352	+ LocalTensor<T> &cos, LocalTensor<T> &sin, int64_t sLength, int64_t bLength, int64_t nLength)
350	{	353	{
351	LocalTensor<T> inUb = this->qInQueue_.template DeQue<T>();	354	LocalTensor<T> inUb = this->qInQueue_.template DeQue<T>();
352	LocalTensor<T> outUb = this->qOutQueue_.template AllocTensor<T>();	355	LocalTensor<T> outUb = this->qOutQueue_.template AllocTensor<T>();
353	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::HALF)) {	356	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::HALF)) {
354	BatchHalfAlignVF<T, IsBBoardcast>(	357	BatchHalfAlignVF<T, IsBBoardcast>(
355	- (__ubuf__ T)inUb.GetPhyAddr(), (__ubuf__ T)cos.GetPhyAddr(), (__ubuf__ T*)sin.GetPhyAddr(),	358	+ (__ubuf__ T )inUb.GetPhyAddr(), (__ubuf__ T )cos.GetPhyAddr(), (__ubuf__ T *)sin.GetPhyAddr(),
356	- (__ubuf__ T*)outUb.GetPhyAddr(), sLength, bLength, nLength, tilingData_->sliceLength, dAlign_,	359	+ (__ubuf__ T *)outUb.GetPhyAddr(), sLength, bLength, nLength, tilingData_->sliceLength, dAlign_,
357	- ubFactorS_, ubFactorN_);	360	+ ubFactorS_, ubFactorN_);
358	} else if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::INTERLEAVE)) {	361	} else if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::INTERLEAVE)) {
359	BatchInterleaveModeVF<T, IsBBoardcast>(	362	BatchInterleaveModeVF<T, IsBBoardcast>(
360	- (__ubuf__ T)inUb.GetPhyAddr(), (__ubuf__ T)cos.GetPhyAddr(), (__ubuf__ T*)sin.GetPhyAddr(),	363	+ (__ubuf__ T )inUb.GetPhyAddr(), (__ubuf__ T )cos.GetPhyAddr(), (__ubuf__ T *)sin.GetPhyAddr(),
361	- (__ubuf__ T*)outUb.GetPhyAddr(), sLength, bLength, nLength, tilingData_->sliceLength, dAlign_,	364	+ (__ubuf__ T *)outUb.GetPhyAddr(), sLength, bLength, nLength, tilingData_->sliceLength, dAlign_,
362	- ubFactorS_, ubFactorN_);	365	+ ubFactorS_, ubFactorN_);
363	} else if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::QUARTER)) {	366	} else if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::QUARTER)) {
364	BatchQuarterAlignVF<T, IsBBoardcast>(	367	BatchQuarterAlignVF<T, IsBBoardcast>(
365	- (__ubuf__ T)inUb.GetPhyAddr(), (__ubuf__ T)cos.GetPhyAddr(), (__ubuf__ T*)sin.GetPhyAddr(),	368	+ (__ubuf__ T )inUb.GetPhyAddr(), (__ubuf__ T )cos.GetPhyAddr(), (__ubuf__ T *)sin.GetPhyAddr(),
366	- (__ubuf__ T*)outUb.GetPhyAddr(), sLength, bLength, nLength, tilingData_->sliceLength, dAlign_,	369	+ (__ubuf__ T *)outUb.GetPhyAddr(), sLength, bLength, nLength, tilingData_->sliceLength, dAlign_,
367	- ubFactorS_, ubFactorN_);	370	+ ubFactorS_, ubFactorN_);
368	} else {	371	} else {
369	BatchDeepSeekInterleaveModeVF<T, IsBBoardcast>(	372	BatchDeepSeekInterleaveModeVF<T, IsBBoardcast>(
370	- (__ubuf__ T)inUb.GetPhyAddr(), (__ubuf__ T)cos.GetPhyAddr(), (__ubuf__ T*)sin.GetPhyAddr(),	373	+ (__ubuf__ T )inUb.GetPhyAddr(), (__ubuf__ T )cos.GetPhyAddr(), (__ubuf__ T *)sin.GetPhyAddr(),
371	- (__ubuf__ T*)outUb.GetPhyAddr(), sLength, bLength, nLength, tilingData_->sliceLength, dAlign_,	374	+ (__ubuf__ T *)outUb.GetPhyAddr(), sLength, bLength, nLength, tilingData_->sliceLength, dAlign_,
372	- ubFactorS_, ubFactorN_);	375	+ ubFactorS_, ubFactorN_);
373	}	376	}
374	this->qInQueue_.FreeTensor(inUb);	377	this->qInQueue_.FreeTensor(inUb);
375	this->qOutQueue_.template EnQue(outUb);	378	this->qOutQueue_.template EnQue(outUb);
376	}	379	}
377	} // namespace InplacePartialRotaryMul	380	} // namespace InplacePartialRotaryMul
378		381
379	-#endif	382	+#endif

Rposembedding/inplace_partial_rotary_mul/op_kernel/arch35/apply_rotary_pos_emb_common.h→posembedding/inplace_partial_rotary_mul/op_kernel/arch35/inplace_partial_rotary_mul_arpe_common.h+86-86

	@@ -9,11 +9,11 @@
9	*/	9	*/
10		10
11	/*!	11	/*!
12	- * \file apply_rotary_pos_emb_common.h	12	+ * \file inplace_partial_rotary_mul_arpe_common.h
13	* \brief	13	* \brief
14	*/	14	*/
15	-#ifndef APPLY_ROTARY_POS_EMB_COMMON_H	15	+#ifndef INPLACE_PARTIAL_ROTARY_MUL_ARPE_COMMON_H
16	-#define APPLY_ROTARY_POS_EMB_COMMON_H	16	+#define INPLACE_PARTIAL_ROTARY_MUL_ARPE_COMMON_H
17		17
18	#include "inplace_partial_rotary_mul_common.h"	18	#include "inplace_partial_rotary_mul_common.h"
19		19
	@@ -53,8 +53,8 @@ struct InplacePartialRotaryPosEmbeddingMode {
53	*/	53	*/
54	template <typename T>	54	template <typename T>
55	__aicore__ inline void HalfAlignVF(const LocalTensor<T> &sinTensor, const LocalTensor<T> &cosTensor,	55	__aicore__ inline void HalfAlignVF(const LocalTensor<T> &sinTensor, const LocalTensor<T> &cosTensor,
56	- const LocalTensor<T> &inTensor, const LocalTensor<T> &outTensor, uint32_t dLen, uint32_t dAlign, uint16_t currSNum,	56	+ const LocalTensor<T> &inTensor, const LocalTensor<T> &outTensor, uint32_t dLen, uint32_t dAlign, uint16_t currSNum,
57	- uint16_t currDNum)	57	+ uint16_t currDNum)
58	{	58	{
59	__ubuf__ T sinUb = (__ubuf__ T )sinTensor.GetPhyAddr();	59	__ubuf__ T sinUb = (__ubuf__ T )sinTensor.GetPhyAddr();
60	__ubuf__ T cosUb = (__ubuf__ T )cosTensor.GetPhyAddr();	60	__ubuf__ T cosUb = (__ubuf__ T )cosTensor.GetPhyAddr();
	@@ -120,8 +120,8 @@ __aicore__ inline void HalfAlignVF(const LocalTensor<T> &sinTensor, const LocalT
120	*/	120	*/
121	template <typename T>	121	template <typename T>
122	__aicore__ inline void QuarterAlignVF(const LocalTensor<T> &sinTensor, const LocalTensor<T> &cosTensor,	122	__aicore__ inline void QuarterAlignVF(const LocalTensor<T> &sinTensor, const LocalTensor<T> &cosTensor,
123	- const LocalTensor<T> &inTensor, const LocalTensor<T> &outTensor, uint32_t dLen, uint32_t dAlign, uint16_t currSNum,	123	+ const LocalTensor<T> &inTensor, const LocalTensor<T> &outTensor, uint32_t dLen, uint32_t dAlign, uint16_t currSNum,
124	- uint16_t currDNum)	124	+ uint16_t currDNum)
125	{	125	{
126	__ubuf__ T sinUb = (__ubuf__ T )sinTensor.GetPhyAddr();	126	__ubuf__ T sinUb = (__ubuf__ T )sinTensor.GetPhyAddr();
127	__ubuf__ T cosUb = (__ubuf__ T )cosTensor.GetPhyAddr();	127	__ubuf__ T cosUb = (__ubuf__ T )cosTensor.GetPhyAddr();
	@@ -204,8 +204,8 @@ __aicore__ inline void QuarterAlignVF(const LocalTensor<T> &sinTensor, const Loc
204		204
205	template <typename T>	205	template <typename T>
206	__aicore__ inline void InterleaveModeVF(const LocalTensor<T> &sinTensor, const LocalTensor<T> &cosTensor,	206	__aicore__ inline void InterleaveModeVF(const LocalTensor<T> &sinTensor, const LocalTensor<T> &cosTensor,
207	- const LocalTensor<T> &inTensor, const LocalTensor<T> &outTensor, uint32_t dLen, uint16_t currSNum,	207	+ const LocalTensor<T> &inTensor, const LocalTensor<T> &outTensor, uint32_t dLen, uint16_t currSNum,
208	- uint16_t currDNum)	208	+ uint16_t currDNum)
209	{	209	{
210	__ubuf__ T sinUb = (__ubuf__ T )sinTensor.GetPhyAddr();	210	__ubuf__ T sinUb = (__ubuf__ T )sinTensor.GetPhyAddr();
211	__ubuf__ T cosUb = (__ubuf__ T )cosTensor.GetPhyAddr();	211	__ubuf__ T cosUb = (__ubuf__ T )cosTensor.GetPhyAddr();
	@@ -318,8 +318,8 @@ __aicore__ inline void InterleaveModeVF(const LocalTensor<T> &sinTensor, const L
318		318
319	template <typename T>	319	template <typename T>
320	__aicore__ inline void DeepSeekInterleaveModeVF(const LocalTensor<T> &sinTensor, const LocalTensor<T> &cosTensor,	320	__aicore__ inline void DeepSeekInterleaveModeVF(const LocalTensor<T> &sinTensor, const LocalTensor<T> &cosTensor,
321	- const LocalTensor<T> &inTensor, const LocalTensor<T> &outTensor, uint32_t dLen, uint16_t currSNum,	321	+ const LocalTensor<T> &inTensor, const LocalTensor<T> &outTensor, uint32_t dLen, uint16_t currSNum,
322	- uint16_t currDNum)	322	+ uint16_t currDNum)
323	{	323	{
324	__ubuf__ T sinUb = (__ubuf__ T )sinTensor.GetPhyAddr();	324	__ubuf__ T sinUb = (__ubuf__ T )sinTensor.GetPhyAddr();
325	__ubuf__ T cosUb = (__ubuf__ T )cosTensor.GetPhyAddr();	325	__ubuf__ T cosUb = (__ubuf__ T )cosTensor.GetPhyAddr();
	@@ -367,13 +367,13 @@ __aicore__ inline void DeepSeekInterleaveModeVF(const LocalTensor<T> &sinTensor,
367	uint32_t halfOffset = offset + halfDAlign;	367	uint32_t halfOffset = offset + halfDAlign;
368	uint32_t inOffset = offset * HALF_INTERLEAVE_COEF;	368	uint32_t inOffset = offset * HALF_INTERLEAVE_COEF;
369	ops::LoadTwoTensorForDtypeT<T>(currInUb,	369	ops::LoadTwoTensorForDtypeT<T>(currInUb,
370	- currInUb,	370	+ currInUb,
371	- vregIn,	371	+ vregIn,
372	- vregHalfIn,	372	+ vregHalfIn,
373	- pregFull,	373	+ pregFull,
374	- pregFull,	374	+ pregFull,
375	- inOffset,	375	+ inOffset,
376	- inOffset + VL_FLOAT32_SIZE);	376	+ inOffset + VL_FLOAT32_SIZE);
377	ops::LoadTwoTensorForDtypeT<T>(	377	ops::LoadTwoTensorForDtypeT<T>(
378	currSinUb, currSinUb, vregSin, vregHalfSin, pregFull, pregFull, offset, halfOffset);	378	currSinUb, currSinUb, vregSin, vregHalfSin, pregFull, pregFull, offset, halfOffset);
379	ops::LoadTwoTensorForDtypeT<T>(	379	ops::LoadTwoTensorForDtypeT<T>(
	@@ -398,13 +398,13 @@ __aicore__ inline void DeepSeekInterleaveModeVF(const LocalTensor<T> &sinTensor,
398	uint32_t halfOffset = offset + halfDAlign;	398	uint32_t halfOffset = offset + halfDAlign;
399	uint32_t inOffset = offset * HALF_INTERLEAVE_COEF;	399	uint32_t inOffset = offset * HALF_INTERLEAVE_COEF;
400	ops::LoadTwoTensorForDtypeT<T>(currInUb,	400	ops::LoadTwoTensorForDtypeT<T>(currInUb,
401	- currInUb,	401	+ currInUb,
402	- vregIn,	402	+ vregIn,
403	- vregHalfIn,	403	+ vregHalfIn,
404	- pregFull,	404	+ pregFull,
405	- pregTail,	405	+ pregTail,
406	- inOffset,	406	+ inOffset,
407	- inOffset + VL_FLOAT32_SIZE);	407	+ inOffset + VL_FLOAT32_SIZE);
408	ops::LoadTwoTensorForDtypeT<T>(	408	ops::LoadTwoTensorForDtypeT<T>(
409	currSinUb, currSinUb, vregSin, vregHalfSin, pregHalfTail, pregHalfTail, offset, halfOffset);	409	currSinUb, currSinUb, vregSin, vregHalfSin, pregHalfTail, pregHalfTail, offset, halfOffset);
410	ops::LoadTwoTensorForDtypeT<T>(	410	ops::LoadTwoTensorForDtypeT<T>(
	@@ -452,8 +452,8 @@ __aicore__ inline void DeepSeekInterleaveModeVF(const LocalTensor<T> &sinTensor,
452		452
453	template <typename T, bool IsBBoardcast>	453	template <typename T, bool IsBBoardcast>
454	__aicore__ inline void BatchHalfAlignVF(__ubuf__ T in, __ubuf__ T cos, __ubuf__ T *sin,	454	__aicore__ inline void BatchHalfAlignVF(__ubuf__ T in, __ubuf__ T cos, __ubuf__ T *sin,
455	- __ubuf__ T *out, uint16_t sLength, uint16_t bLength, uint16_t nLength, int64_t d, int64_t dAlign,	455	+ __ubuf__ T *out, uint16_t sLength, uint16_t bLength, uint16_t nLength, int64_t d, int64_t dAlign,
456	- int64_t ubFactorS, int64_t ubFactorN)	456	+ int64_t ubFactorS, int64_t ubFactorN)
457	{	457	{
458	uint32_t dHalfSize = d / HALF_INTERLEAVE_COEF;	458	uint32_t dHalfSize = d / HALF_INTERLEAVE_COEF;
459	uint16_t dLoopCount = (dHalfSize + VL_FLOAT32_SIZE - 1) / VL_FLOAT32_SIZE;	459	uint16_t dLoopCount = (dHalfSize + VL_FLOAT32_SIZE - 1) / VL_FLOAT32_SIZE;
	@@ -519,8 +519,8 @@ __aicore__ inline void BatchHalfAlignVF(__ubuf__ T in, __ubuf__ T cos, __ubuf_
519		519
520	template <typename T, bool IsBBoardcast>	520	template <typename T, bool IsBBoardcast>
521	__aicore__ inline void BatchQuarterAlignVF(__ubuf__ T in, __ubuf__ T cos, __ubuf__ T *sin,	521	__aicore__ inline void BatchQuarterAlignVF(__ubuf__ T in, __ubuf__ T cos, __ubuf__ T *sin,
522	- __ubuf__ T *out, uint16_t sLength, uint16_t bLength, uint16_t nLength, int64_t d, int64_t dAlign,	522	+ __ubuf__ T *out, uint16_t sLength, uint16_t bLength, uint16_t nLength, int64_t d, int64_t dAlign,
523	- int64_t ubFactorS, int64_t ubFactorN)	523	+ int64_t ubFactorS, int64_t ubFactorN)
524	{	524	{
525	uint32_t dQuarterSize = d / QUARTER_MODE_COEF;	525	uint32_t dQuarterSize = d / QUARTER_MODE_COEF;
526	uint16_t dLoopCount = (dQuarterSize + VL_FLOAT32_SIZE - 1) / VL_FLOAT32_SIZE;	526	uint16_t dLoopCount = (dQuarterSize + VL_FLOAT32_SIZE - 1) / VL_FLOAT32_SIZE;
	@@ -566,53 +566,53 @@ __aicore__ inline void BatchQuarterAlignVF(__ubuf__ T in, __ubuf__ T cos, __ub
566	pregLoop = Reg::UpdateMask<float>(count);	566	pregLoop = Reg::UpdateMask<float>(count);
567	// 拷贝到RegBase内	567	// 拷贝到RegBase内
568	ops::LoadTwoTensorForDtypeT<T>(currInUb,	568	ops::LoadTwoTensorForDtypeT<T>(currInUb,
569	- currInUb,	569	+ currInUb,
570	- inPart1Reg,	570	+ inPart1Reg,
571	- inPart2Reg,	571	+ inPart2Reg,
572	- pregLoop,	572	+ pregLoop,
573	- pregLoop,	573	+ pregLoop,
574	- i * VL_FLOAT32_SIZE,	574	+ i * VL_FLOAT32_SIZE,
575	- i * VL_FLOAT32_SIZE + dQuarterOffset);	575	+ i * VL_FLOAT32_SIZE + dQuarterOffset);
576	ops::LoadTwoTensorForDtypeT<T>(currInUb,	576	ops::LoadTwoTensorForDtypeT<T>(currInUb,
577	- currInUb,	577	+ currInUb,
578	- inPart3Reg,	578	+ inPart3Reg,
579	- inPart4Reg,	579	+ inPart4Reg,
580	- pregLoop,	580	+ pregLoop,
581	- pregLoop,	581	+ pregLoop,
582	- i * VL_FLOAT32_SIZE + dHalfOffset,	582	+ i * VL_FLOAT32_SIZE + dHalfOffset,
583	- i * VL_FLOAT32_SIZE + dThreeQuarterOffset);	583	+ i * VL_FLOAT32_SIZE + dThreeQuarterOffset);
584	ops::LoadTwoTensorForDtypeT<T>(currCosUb,	584	ops::LoadTwoTensorForDtypeT<T>(currCosUb,
585	- currCosUb,	585	+ currCosUb,
586	- cosPart1Reg,	586	+ cosPart1Reg,
587	- cosPart2Reg,	587	+ cosPart2Reg,
588	- pregLoop,	588	+ pregLoop,
589	- pregLoop,	589	+ pregLoop,
590	- i * VL_FLOAT32_SIZE,	590	+ i * VL_FLOAT32_SIZE,
591	- i * VL_FLOAT32_SIZE + dQuarterOffset);	591	+ i * VL_FLOAT32_SIZE + dQuarterOffset);
592	ops::LoadTwoTensorForDtypeT<T>(currCosUb,	592	ops::LoadTwoTensorForDtypeT<T>(currCosUb,
593	- currCosUb,	593	+ currCosUb,
594	- cosPart3Reg,	594	+ cosPart3Reg,
595	- cosPart4Reg,	595	+ cosPart4Reg,
596	- pregLoop,	596	+ pregLoop,
597	- pregLoop,	597	+ pregLoop,
598	- i * VL_FLOAT32_SIZE + dHalfOffset,	598	+ i * VL_FLOAT32_SIZE + dHalfOffset,
599	- i * VL_FLOAT32_SIZE + dThreeQuarterOffset);	599	+ i * VL_FLOAT32_SIZE + dThreeQuarterOffset);
600	ops::LoadTwoTensorForDtypeT<T>(currSinUb,	600	ops::LoadTwoTensorForDtypeT<T>(currSinUb,
601	- currSinUb,	601	+ currSinUb,
602	- sinPart1Reg,	602	+ sinPart1Reg,
603	- sinPart2Reg,	603	+ sinPart2Reg,
604	- pregLoop,	604	+ pregLoop,
605	- pregLoop,	605	+ pregLoop,
606	- i * VL_FLOAT32_SIZE,	606	+ i * VL_FLOAT32_SIZE,
607	- i * VL_FLOAT32_SIZE + dQuarterOffset);	607	+ i * VL_FLOAT32_SIZE + dQuarterOffset);
608	ops::LoadTwoTensorForDtypeT<T>(currSinUb,	608	ops::LoadTwoTensorForDtypeT<T>(currSinUb,
609	- currSinUb,	609	+ currSinUb,
610	- sinPart3Reg,	610	+ sinPart3Reg,
611	- sinPart4Reg,	611	+ sinPart4Reg,
612	- pregLoop,	612	+ pregLoop,
613	- pregLoop,	613	+ pregLoop,
614	- i * VL_FLOAT32_SIZE + dHalfOffset,	614	+ i * VL_FLOAT32_SIZE + dHalfOffset,
615	- i * VL_FLOAT32_SIZE + dThreeQuarterOffset);	615	+ i * VL_FLOAT32_SIZE + dThreeQuarterOffset);
616	// 计算	616	// 计算
617	Reg::Mul(cosPart1Reg, inPart1Reg, cosPart1Reg, pregLoop);	617	Reg::Mul(cosPart1Reg, inPart1Reg, cosPart1Reg, pregLoop);
618	Reg::Mul(sinPart1Reg, inPart2Reg, sinPart1Reg, pregLoop);	618	Reg::Mul(sinPart1Reg, inPart2Reg, sinPart1Reg, pregLoop);
	@@ -643,8 +643,8 @@ __aicore__ inline void BatchQuarterAlignVF(__ubuf__ T in, __ubuf__ T cos, __ub
643		643
644	template <typename T, bool IsBBoardcast>	644	template <typename T, bool IsBBoardcast>
645	__aicore__ inline void BatchInterleaveModeVF(__ubuf__ T in, __ubuf__ T cos, __ubuf__ T *sin,	645	__aicore__ inline void BatchInterleaveModeVF(__ubuf__ T in, __ubuf__ T cos, __ubuf__ T *sin,
646	- __ubuf__ T *out, uint16_t sLength, uint16_t bLength, uint16_t nLength, int64_t d, int64_t dAlign,	646	+ __ubuf__ T *out, uint16_t sLength, uint16_t bLength, uint16_t nLength, int64_t d, int64_t dAlign,
647	- int64_t ubFactorS, int64_t ubFactorN)	647	+ int64_t ubFactorS, int64_t ubFactorN)
648	{	648	{
649	uint32_t loopSize = 2 * VL_FLOAT32_SIZE;	649	uint32_t loopSize = 2 * VL_FLOAT32_SIZE;
650	uint16_t dLoopCount = (d + loopSize - 1) / loopSize;	650	uint16_t dLoopCount = (d + loopSize - 1) / loopSize;
	@@ -726,8 +726,8 @@ __aicore__ inline void BatchInterleaveModeVF(__ubuf__ T in, __ubuf__ T cos, __
726		726
727	template <typename T, bool IsBBoardcast>	727	template <typename T, bool IsBBoardcast>
728	__aicore__ inline void BatchDeepSeekInterleaveModeVF(__ubuf__ T in, __ubuf__ T cos, __ubuf__ T *sin,	728	__aicore__ inline void BatchDeepSeekInterleaveModeVF(__ubuf__ T in, __ubuf__ T cos, __ubuf__ T *sin,
729	- __ubuf__ T *out, uint16_t sLength, uint16_t bLength, uint16_t nLength, int64_t d, int64_t dAlign,	729	+ __ubuf__ T *out, uint16_t sLength, uint16_t bLength, uint16_t nLength, int64_t d, int64_t dAlign,
730	- int64_t ubFactorS, int64_t ubFactorN)	730	+ int64_t ubFactorS, int64_t ubFactorN)
731	{	731	{
732	uint32_t loopSize = 2 * VL_FLOAT32_SIZE;	732	uint32_t loopSize = 2 * VL_FLOAT32_SIZE;
733	uint16_t dLoopCount = (d + loopSize - 1) / loopSize;	733	uint16_t dLoopCount = (d + loopSize - 1) / loopSize;
	@@ -810,8 +810,8 @@ __aicore__ inline void BatchDeepSeekInterleaveModeVF(__ubuf__ T *in, __ubuf__ T
810	// Mixed precision: TX is half/bfloat16 for input, cos/sin are float	810	// Mixed precision: TX is half/bfloat16 for input, cos/sin are float
811	template <typename TX>	811	template <typename TX>
812	__aicore__ inline void InterleaveModeVFMixed(const LocalTensor<TX> &inTensor, const LocalTensor<float> &cosTensor,	812	__aicore__ inline void InterleaveModeVFMixed(const LocalTensor<TX> &inTensor, const LocalTensor<float> &cosTensor,
813	- const LocalTensor<float> &sinTensor, const LocalTensor<TX> &outTensor, uint32_t dLen, uint16_t currSNum,	813	+ const LocalTensor<float> &sinTensor, const LocalTensor<TX> &outTensor, uint32_t dLen, uint16_t currSNum,
814	- uint16_t currDNum)	814	+ uint16_t currDNum)
815	{	815	{
816	__ubuf__ TX inUb = (__ubuf__ TX )inTensor.GetPhyAddr();	816	__ubuf__ TX inUb = (__ubuf__ TX )inTensor.GetPhyAddr();
817	__ubuf__ float cosUb = (__ubuf__ float )cosTensor.GetPhyAddr();	817	__ubuf__ float cosUb = (__ubuf__ float )cosTensor.GetPhyAddr();
	@@ -919,8 +919,8 @@ __aicore__ inline void InterleaveModeVFMixed(const LocalTensor<TX> &inTensor, co
919	// Mixed precision BatchInterleaveModeVF for ABA layout	919	// Mixed precision BatchInterleaveModeVF for ABA layout
920	template <typename TX, bool IsBBoardcast>	920	template <typename TX, bool IsBBoardcast>
921	__aicore__ inline void BatchInterleaveModeVFMixed(__ubuf__ TX in, __ubuf__ float cos,	921	__aicore__ inline void BatchInterleaveModeVFMixed(__ubuf__ TX in, __ubuf__ float cos,
922	- __ubuf__ float sin, __ubuf__ TX out, uint16_t sLength, uint16_t bLength, uint16_t nLength, int64_t d,	922	+ __ubuf__ float sin, __ubuf__ TX out, uint16_t sLength, uint16_t bLength, uint16_t nLength, int64_t d,
923	- int64_t dAlign, int64_t dAlignFloat, int64_t ubFactorS, int64_t ubFactorN)	923	+ int64_t dAlign, int64_t dAlignFloat, int64_t ubFactorS, int64_t ubFactorN)
924	{	924	{
925	uint32_t loopSize = 2 * VL_FLOAT32_SIZE;	925	uint32_t loopSize = 2 * VL_FLOAT32_SIZE;
926	uint16_t dLoopCount = (d + loopSize - 1) / loopSize;	926	uint16_t dLoopCount = (d + loopSize - 1) / loopSize;
	@@ -976,9 +976,9 @@ __aicore__ inline void BatchInterleaveModeVFMixed(__ubuf__ TX *in, __ubuf__ floa
976	// Load/Store的offset是TX元素偏移，需与cos/sin逻辑下标对齐，不能按dtype字节数换算。	976	// Load/Store的offset是TX元素偏移，需与cos/sin逻辑下标对齐，不能按dtype字节数换算。
977	ops::LoadOneTensorForDtypeT<TX>(currInUb, inPart1Reg, pregPart1, i * loopSize);	977	ops::LoadOneTensorForDtypeT<TX>(currInUb, inPart1Reg, pregPart1, i * loopSize);
978	ops::LoadOneTensorForDtypeT<TX>(currInUb,	978	ops::LoadOneTensorForDtypeT<TX>(currInUb,
979	- inPart2Reg,	979	+ inPart2Reg,
980	- pregPart2,	980	+ pregPart2,
981	- i * loopSize + VL_FLOAT32_SIZE);	981	+ i * loopSize + VL_FLOAT32_SIZE);
982	ops::LoadOneTensorForDtypeT<float>(currCosUb, cosPart1Reg, pregPart1, i * loopSize);	982	ops::LoadOneTensorForDtypeT<float>(currCosUb, cosPart1Reg, pregPart1, i * loopSize);
983	ops::LoadOneTensorForDtypeT<float>(	983	ops::LoadOneTensorForDtypeT<float>(
984	currCosUb, cosPart2Reg, pregPart2, i * loopSize + VL_FLOAT32_SIZE);	984	currCosUb, cosPart2Reg, pregPart2, i * loopSize + VL_FLOAT32_SIZE);
	@@ -996,9 +996,9 @@ __aicore__ inline void BatchInterleaveModeVFMixed(__ubuf__ TX *in, __ubuf__ floa
996	Reg::Add(cosPart2Reg, cosPart2Reg, sinPart2Reg, pregPart2);	996	Reg::Add(cosPart2Reg, cosPart2Reg, sinPart2Reg, pregPart2);
997	ops::StoreOneTensorForDtypeT<TX>(currOutUb, cosPart1Reg, pregPart1, i * loopSize);	997	ops::StoreOneTensorForDtypeT<TX>(currOutUb, cosPart1Reg, pregPart1, i * loopSize);
998	ops::StoreOneTensorForDtypeT<TX>(currOutUb,	998	ops::StoreOneTensorForDtypeT<TX>(currOutUb,
999	- cosPart2Reg,	999	+ cosPart2Reg,
1000	- pregPart2,	1000	+ pregPart2,
1001	- i * loopSize + VL_FLOAT32_SIZE);	1001	+ i * loopSize + VL_FLOAT32_SIZE);
1002	}	1002	}
1003	}	1003	}
1004	}	1004	}
	@@ -1006,4 +1006,4 @@ __aicore__ inline void BatchInterleaveModeVFMixed(__ubuf__ TX *in, __ubuf__ floa
1006	}	1006	}
1007	}	1007	}
1008		1008
1009	-#endif // APPLY_ROTARY_POS_EMB_COMMON_H	1009	+#endif // APPLY_ROTARY_POS_EMB_COMMON_H

Rposembedding/inplace_partial_rotary_mul/op_kernel/arch35/rotary_position_embedding_bab.h→posembedding/inplace_partial_rotary_mul/op_kernel/arch35/inplace_partial_rotary_mul_bab.h+28-27

	@@ -9,14 +9,14 @@
9	*/	9	*/
10		10
11	/*!	11	/*!
12	- * \file rotary_position_embedding_bab.h	12	+ * \file inplace_partial_rotary_mul_bab.h
13	* \brief	13	* \brief
14	*/	14	*/
15		15
16	-#ifndef ROTARY_POSITION_EMBEDDING_BAB_H	16	+#ifndef INPLACE_PARTIAL_ROTARY_MUL_BAB_H
17	-#define ROTARY_POSITION_EMBEDDING_BAB_H	17	+#define INPLACE_PARTIAL_ROTARY_MUL_BAB_H
18		18
19	-#include "apply_rotary_pos_emb_common.h"	19	+#include "inplace_partial_rotary_mul_arpe_common.h"
20		20
21	namespace InplacePartialRotaryMul {	21	namespace InplacePartialRotaryMul {
22	using namespace AscendC;	22	using namespace AscendC;
	@@ -24,15 +24,16 @@ using namespace AscendC;
24	template <typename T>	24	template <typename T>
25	class RotaryPositionEmbeddingBAB {	25	class RotaryPositionEmbeddingBAB {
26	public:	26	public:
27	- __aicore__ inline RotaryPositionEmbeddingBAB(TPipe* pipe, const InplacePartialRopeRegbaseTilingData* tiling)	27	+ __aicore__ inline RotaryPositionEmbeddingBAB(TPipe pipe, const InplacePartialRopeRegbaseTilingData tiling)
28	- : pipe_(pipe), tilingData_(tiling){};	28	+ : pipe_(pipe),
		29	+ tilingData_(tiling){};
29	__aicore__ inline void Init(GM_ADDR x, GM_ADDR cos, GM_ADDR sin, GM_ADDR y);	30	__aicore__ inline void Init(GM_ADDR x, GM_ADDR cos, GM_ADDR sin, GM_ADDR y);
30	__aicore__ inline void Process();	31	__aicore__ inline void Process();
31		32
32	private:	33	private:
33	constexpr static int32_t bufferNum = 2;	34	constexpr static int32_t bufferNum = 2;
34	- const InplacePartialRopeRegbaseTilingData* tilingData_;	35	+ const InplacePartialRopeRegbaseTilingData *tilingData_;
35	- TPipe* pipe_;	36	+ TPipe *pipe_;
36	int64_t blockIdx_ = 0;	37	int64_t blockIdx_ = 0;
37	int64_t dSplitCoef_ = 1; // 切分系数初始化为1	38	int64_t dSplitCoef_ = 1; // 切分系数初始化为1
38	uint32_t dSplitSize_ = 0;	39	uint32_t dSplitSize_ = 0;
	@@ -57,10 +58,10 @@ private:
57	__aicore__ inline void PrePareParams();	58	__aicore__ inline void PrePareParams();
58	__aicore__ inline void ProcessNLoop(const uint32_t bIdx, const uint32_t sIdx, const uint32_t currSNum);	59	__aicore__ inline void ProcessNLoop(const uint32_t bIdx, const uint32_t sIdx, const uint32_t currSNum);
59	__aicore__ inline void Compute(	60	__aicore__ inline void Compute(
60	- const LocalTensor<T>& sinTensor, const LocalTensor<T>& cosTensor, const LocalTensor<T>& inTensor,	61	+ const LocalTensor<T> &sinTensor, const LocalTensor<T> &cosTensor, const LocalTensor<T> &inTensor,
61	- const LocalTensor<T>& outTensor, const uint32_t currSNum, const uint32_t currDNum);	62	+ const LocalTensor<T> &outTensor, const uint32_t currSNum, const uint32_t currDNum);
62	__aicore__ inline void ProcessN(	63	__aicore__ inline void ProcessN(
63	- const LocalTensor<T>& sinTensor, const LocalTensor<T>& cosTensor, const uint32_t bIdx, const uint32_t sIdx,	64	+ const LocalTensor<T> &sinTensor, const LocalTensor<T> &cosTensor, const uint32_t bIdx, const uint32_t sIdx,
64	const uint32_t currSNum);	65	const uint32_t currSNum);
65	};	66	};
66		67
	@@ -76,13 +77,14 @@ __aicore__ inline void RotaryPositionEmbeddingBAB<T>::Init(GM_ADDR x, GM_ADDR co
76	}	77	}
77	this->dSplitSize_ = tilingData_->sliceLength / dSplitCoef_ * sizeof(T);	78	this->dSplitSize_ = tilingData_->sliceLength / dSplitCoef_ * sizeof(T);
78	this->dAlign_ = ops::CeilAlign<int64_t>(tilingData_->sliceLength / dSplitCoef_,	79	this->dAlign_ = ops::CeilAlign<int64_t>(tilingData_->sliceLength / dSplitCoef_,
79	- BLOCK_TYPE_SIZE / sizeof(T)) * dSplitCoef_;	80	+ BLOCK_TYPE_SIZE / sizeof(T)) *
		81	+ dSplitCoef_;
80	ubFactorN_ = tilingData_->ubFactorN;	82	ubFactorN_ = tilingData_->ubFactorN;
81	ubFactorS_ = tilingData_->ubFactorS;	83	ubFactorS_ = tilingData_->ubFactorS;
82	- this->xGm_.SetGlobalBuffer((__gm__ T*)x);	84	+ this->xGm_.SetGlobalBuffer((__gm__ T *)x);
83	- this->cosGm_.SetGlobalBuffer((__gm__ T*)cos);	85	+ this->cosGm_.SetGlobalBuffer((__gm__ T *)cos);
84	- this->sinGm_.SetGlobalBuffer((__gm__ T*)sin);	86	+ this->sinGm_.SetGlobalBuffer((__gm__ T *)sin);
85	- this->yOutGm_.SetGlobalBuffer((__gm__ T*)y);	87	+ this->yOutGm_.SetGlobalBuffer((__gm__ T *)y);
86	this->pipe_->InitBuffer(xInQue_, bufferNum, ubFactorS_ * ubFactorN_ * dAlign_ * sizeof(T));	88	this->pipe_->InitBuffer(xInQue_, bufferNum, ubFactorS_ * ubFactorN_ * dAlign_ * sizeof(T));
87	this->pipe_->InitBuffer(cosInQue_, bufferNum, ubFactorS_ * dAlign_ * sizeof(T));	89	this->pipe_->InitBuffer(cosInQue_, bufferNum, ubFactorS_ * dAlign_ * sizeof(T));
88	this->pipe_->InitBuffer(sinInQue_, bufferNum, ubFactorS_ * dAlign_ * sizeof(T));	90	this->pipe_->InitBuffer(sinInQue_, bufferNum, ubFactorS_ * dAlign_ * sizeof(T));
	@@ -141,7 +143,7 @@ __aicore__ inline void RotaryPositionEmbeddingBAB<T>::ProcessNLoop(
141		143
142	template <typename T>	144	template <typename T>
143	__aicore__ inline void RotaryPositionEmbeddingBAB<T>::ProcessN(	145	__aicore__ inline void RotaryPositionEmbeddingBAB<T>::ProcessN(
144	- const LocalTensor<T>& sinTensor, const LocalTensor<T>& cosTensor, const uint32_t bIdx, const uint32_t sIdx,	146	+ const LocalTensor<T> &sinTensor, const LocalTensor<T> &cosTensor, const uint32_t bIdx, const uint32_t sIdx,
145	const uint32_t currSNum)	147	const uint32_t currSNum)
146	{	148	{
147	LocalTensor<T> xTensor;	149	LocalTensor<T> xTensor;
	@@ -152,15 +154,14 @@ __aicore__ inline void RotaryPositionEmbeddingBAB<T>::ProcessN(
152	int64_t offset = baseOffset + idxN * ubFactorN_ * tilingData_->D;	154	int64_t offset = baseOffset + idxN * ubFactorN_ * tilingData_->D;
153	xTensor = xInQue_.AllocTensor<T>();	155	xTensor = xInQue_.AllocTensor<T>();
154	DataCopyExtParams copyInParams{static_cast<uint16_t>(currSNum * currDNum * dSplitCoef_), dSplitSize_,	156	DataCopyExtParams copyInParams{static_cast<uint16_t>(currSNum * currDNum * dSplitCoef_), dSplitSize_,
155	- static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(T)), 0, 0};	157	+ static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(T)), 0, 0};
156	DataCopyExtParams copyOutParams{static_cast<uint16_t>(currSNum * currDNum * dSplitCoef_), dSplitSize_, 0,	158	DataCopyExtParams copyOutParams{static_cast<uint16_t>(currSNum * currDNum * dSplitCoef_), dSplitSize_, 0,
157	- static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(T)), 0};	159	+ static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(T)), 0};
158	// deepseekInterleave 场景，copyIn时按照D对齐，copyOut 时，按照 d/2 对齐	160	// deepseekInterleave 场景，copyIn时按照D对齐，copyOut 时，按照 d/2 对齐
159	if (	161	if (
160	- tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::DEEPSEEK_INTERLEAVE))	162	+ tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::DEEPSEEK_INTERLEAVE)) {
161	- {
162	copyInParams = {static_cast<uint16_t>(currSNum * currDNum), tilingData_->sliceLength * sizeof(T),	163	copyInParams = {static_cast<uint16_t>(currSNum * currDNum), tilingData_->sliceLength * sizeof(T),
163	- static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(T)), 0, 0};	164	+ static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(T)), 0, 0};
164	}	165	}
165	DataCopyPadExtParams<T> padParams{false, 0, 0, 0};	166	DataCopyPadExtParams<T> padParams{false, 0, 0, 0};
166	DataCopyPad(xTensor, xGm_[offset], copyInParams, padParams);	167	DataCopyPad(xTensor, xGm_[offset], copyInParams, padParams);
	@@ -178,20 +179,20 @@ __aicore__ inline void RotaryPositionEmbeddingBAB<T>::ProcessN(
178		179
179	template <typename T>	180	template <typename T>
180	__aicore__ inline void RotaryPositionEmbeddingBAB<T>::Compute(	181	__aicore__ inline void RotaryPositionEmbeddingBAB<T>::Compute(
181	- const LocalTensor<T>& sinTensor, const LocalTensor<T>& cosTensor, const LocalTensor<T>& inTensor,	182	+ const LocalTensor<T> &sinTensor, const LocalTensor<T> &cosTensor, const LocalTensor<T> &inTensor,
182	- const LocalTensor<T>& outTensor, const uint32_t currSNum, const uint32_t currDNum)	183	+ const LocalTensor<T> &outTensor, const uint32_t currSNum, const uint32_t currDNum)
183	{	184	{
184	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::HALF)) {	185	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::HALF)) {
185	HalfAlignVF<T>(sinTensor, cosTensor, inTensor, outTensor, tilingData_->sliceLength, dAlign_, currSNum,	186	HalfAlignVF<T>(sinTensor, cosTensor, inTensor, outTensor, tilingData_->sliceLength, dAlign_, currSNum,
186	- currDNum);	187	+ currDNum);
187	} else if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::INTERLEAVE)) {	188	} else if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::INTERLEAVE)) {
188	InterleaveModeVF<T>(sinTensor, cosTensor, inTensor, outTensor, tilingData_->sliceLength, currSNum, currDNum);	189	InterleaveModeVF<T>(sinTensor, cosTensor, inTensor, outTensor, tilingData_->sliceLength, currSNum, currDNum);
189	} else if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::QUARTER)) {	190	} else if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::QUARTER)) {
190	QuarterAlignVF<T>(sinTensor, cosTensor, inTensor, outTensor, tilingData_->sliceLength, dAlign_, currSNum,	191	QuarterAlignVF<T>(sinTensor, cosTensor, inTensor, outTensor, tilingData_->sliceLength, dAlign_, currSNum,
191	- currDNum);	192	+ currDNum);
192	} else {	193	} else {
193	DeepSeekInterleaveModeVF<T>(sinTensor, cosTensor, inTensor, outTensor, tilingData_->sliceLength, currSNum,	194	DeepSeekInterleaveModeVF<T>(sinTensor, cosTensor, inTensor, outTensor, tilingData_->sliceLength, currSNum,
194	- currDNum);	195	+ currDNum);
195	}	196	}
196	}	197	}
197		198

Mposembedding/inplace_partial_rotary_mul/op_kernel/arch35/rotary_position_embedding_a_and_b_mixed.h+23-23

	@@ -15,7 +15,7 @@
15	#ifndef ROTARY_POSITION_EMBEDDING_A_AND_B_MIXED_H	15	#ifndef ROTARY_POSITION_EMBEDDING_A_AND_B_MIXED_H
16	#define ROTARY_POSITION_EMBEDDING_A_AND_B_MIXED_H	16	#define ROTARY_POSITION_EMBEDDING_A_AND_B_MIXED_H
17		17
18	-#include "apply_rotary_pos_emb_common.h"	18	+#include "inplace_partial_rotary_mul_arpe_common.h"
19		19
20	namespace InplacePartialRotaryMul {	20	namespace InplacePartialRotaryMul {
21	using namespace AscendC;	21	using namespace AscendC;
	@@ -28,7 +28,7 @@ public:
28	__aicore__ inline ~RotaryPositionEmbeddingAAndBMixed(){};	28	__aicore__ inline ~RotaryPositionEmbeddingAAndBMixed(){};
29		29
30	__aicore__ inline void Init(GM_ADDR q, GM_ADDR cos, GM_ADDR sin, GM_ADDR qOut, GM_ADDR workspace,	30	__aicore__ inline void Init(GM_ADDR q, GM_ADDR cos, GM_ADDR sin, GM_ADDR qOut, GM_ADDR workspace,
31	- const InplacePartialRopeRegbaseTilingData tilingData, TPipe pipe);	31	+ const InplacePartialRopeRegbaseTilingData tilingData, TPipe pipe);
32		32
33	__aicore__ inline void Process();	33	__aicore__ inline void Process();
34		34
	@@ -75,7 +75,7 @@ private:
75		75
76	template <typename TX, bool IsBoardCast>	76	template <typename TX, bool IsBoardCast>
77	__aicore__ inline void RotaryPositionEmbeddingAAndBMixed<TX, IsBoardCast>::Init(GM_ADDR q, GM_ADDR cos, GM_ADDR sin,	77	__aicore__ inline void RotaryPositionEmbeddingAAndBMixed<TX, IsBoardCast>::Init(GM_ADDR q, GM_ADDR cos, GM_ADDR sin,
78	- GM_ADDR qOut, GM_ADDR workspace, const InplacePartialRopeRegbaseTilingData tilingData, TPipe pipe)	78	+ GM_ADDR qOut, GM_ADDR workspace, const InplacePartialRopeRegbaseTilingData tilingData, TPipe pipe)
79	{	79	{
80	this->tilingData_ = tilingData;	80	this->tilingData_ = tilingData;
81	this->pipe_ = pipe;	81	this->pipe_ = pipe;
	@@ -116,7 +116,7 @@ __aicore__ inline void RotaryPositionEmbeddingAAndBMixed<TX, IsBoardCast>::InitA
116	if constexpr (!IsBoardCast) {	116	if constexpr (!IsBoardCast) {
117	this->ubCopyInStride =	117	this->ubCopyInStride =
118	(this->dAlign_ * sizeof(TX) -	118	(this->dAlign_ * sizeof(TX) -
119	- ops::CeilAlign<int64_t>(tilingData_->sliceLength * sizeof(TX), BLOCK_TYPE_SIZE)) /	119	+ ops::CeilAlign<int64_t>(tilingData_->sliceLength * sizeof(TX), BLOCK_TYPE_SIZE)) /
120	BLOCK_TYPE_SIZE;	120	BLOCK_TYPE_SIZE;
121	}	121	}
122	}	122	}
	@@ -152,22 +152,22 @@ __aicore__ inline void RotaryPositionEmbeddingAAndBMixed<TX, IsBoardCast>::Proce
152	LocalTensor<float> sinUb = this->sinInQueue_.template DeQue<float>();	152	LocalTensor<float> sinUb = this->sinInQueue_.template DeQue<float>();
153	for (int64_t ubLoopIdx = 0; ubLoopIdx < ubLoopCount; ubLoopIdx++) {	153	for (int64_t ubLoopIdx = 0; ubLoopIdx < ubLoopCount; ubLoopIdx++) {
154	this->ProcessInLoop(cosUb,	154	this->ProcessInLoop(cosUb,
155	- sinUb,	155	+ sinUb,
156	- bBlockStart_ + ubLoopIdx * ubFactorB_,	156	+ bBlockStart_ + ubLoopIdx * ubFactorB_,
157	- ubLoopIdx != ubLoopCount - 1 ? ubFactorB_ : bBlockLength_ - ubLoopIdx * ubFactorB_);	157	+ ubLoopIdx != ubLoopCount - 1 ? ubFactorB_ : bBlockLength_ - ubLoopIdx * ubFactorB_);
158	}	158	}
159	this->cosInQueue_.FreeTensor(cosUb);	159	this->cosInQueue_.FreeTensor(cosUb);
160	this->sinInQueue_.FreeTensor(sinUb);	160	this->sinInQueue_.FreeTensor(sinUb);
161	} else {	161	} else {
162	for (int64_t ubLoopIdx = 0; ubLoopIdx < ubLoopCount; ubLoopIdx++) {	162	for (int64_t ubLoopIdx = 0; ubLoopIdx < ubLoopCount; ubLoopIdx++) {
163	this->CopyInCosAndSin(bBlockStart_ + ubLoopIdx * ubFactorB_,	163	this->CopyInCosAndSin(bBlockStart_ + ubLoopIdx * ubFactorB_,
164	- ubLoopIdx != ubLoopCount - 1 ? ubFactorB_ : bBlockLength_ - ubLoopIdx * ubFactorB_);	164	+ ubLoopIdx != ubLoopCount - 1 ? ubFactorB_ : bBlockLength_ - ubLoopIdx * ubFactorB_);
165	LocalTensor<float> cosUb = this->cosInQueue_.template DeQue<float>();	165	LocalTensor<float> cosUb = this->cosInQueue_.template DeQue<float>();
166	LocalTensor<float> sinUb = this->sinInQueue_.template DeQue<float>();	166	LocalTensor<float> sinUb = this->sinInQueue_.template DeQue<float>();
167	this->ProcessInLoop(cosUb,	167	this->ProcessInLoop(cosUb,
168	- sinUb,	168	+ sinUb,
169	- bBlockStart_ + ubLoopIdx * ubFactorB_,	169	+ bBlockStart_ + ubLoopIdx * ubFactorB_,
170	- ubLoopIdx != ubLoopCount - 1 ? ubFactorB_ : bBlockLength_ - ubLoopIdx * ubFactorB_);	170	+ ubLoopIdx != ubLoopCount - 1 ? ubFactorB_ : bBlockLength_ - ubLoopIdx * ubFactorB_);
171	this->cosInQueue_.FreeTensor(cosUb);	171	this->cosInQueue_.FreeTensor(cosUb);
172	this->sinInQueue_.FreeTensor(sinUb);	172	this->sinInQueue_.FreeTensor(sinUb);
173	}	173	}
	@@ -252,22 +252,22 @@ __aicore__ inline void RotaryPositionEmbeddingAAndBMixed<TX, IsBoardCast>::Compu
252	InterleaveModeVFMixed<TX>(inUb, cos, sin, outUb, tilingData_->sliceLength, 1, bLength);	252	InterleaveModeVFMixed<TX>(inUb, cos, sin, outUb, tilingData_->sliceLength, 1, bLength);
253	} else {	253	} else {
254	BatchInterleaveModeVFMixed<TX, IsBoardCast>((__ubuf__ TX *)inUb.GetPhyAddr(),	254	BatchInterleaveModeVFMixed<TX, IsBoardCast>((__ubuf__ TX *)inUb.GetPhyAddr(),
255	- (__ubuf__ float *)cos.GetPhyAddr(),	255	+ (__ubuf__ float *)cos.GetPhyAddr(),
256	- (__ubuf__ float *)sin.GetPhyAddr(),	256	+ (__ubuf__ float *)sin.GetPhyAddr(),
257	- (__ubuf__ TX *)outUb.GetPhyAddr(),	257	+ (__ubuf__ TX *)outUb.GetPhyAddr(),
258	- bLength,	258	+ bLength,
259	- 1,	259	+ 1,
260	- 1,	260	+ 1,
261	- tilingData_->sliceLength,	261	+ tilingData_->sliceLength,
262	- dAlign_,	262	+ dAlign_,
263	- dAlignFloat_,	263	+ dAlignFloat_,
264	- ubFactorB_,	264	+ ubFactorB_,
265	- 1);	265	+ 1);
266	}	266	}
267		267
268	this->qInQueue_.FreeTensor(inUb);	268	this->qInQueue_.FreeTensor(inUb);
269	this->qOutQueue_.template EnQue(outUb);	269	this->qOutQueue_.template EnQue(outUb);
270	}	270	}
271	-} // namespace InplacePartialRotaryMul	271	+} // namespace InplacePartialRotaryMul
272		272
273	#endif	273	#endif

Mposembedding/inplace_partial_rotary_mul/op_kernel/arch35/rotary_position_embedding_ab_mixed.h+27-29

	@@ -15,7 +15,7 @@
15	#ifndef ROTARY_POSITION_EMBEDDING_AB_MIXED_H	15	#ifndef ROTARY_POSITION_EMBEDDING_AB_MIXED_H
16	#define ROTARY_POSITION_EMBEDDING_AB_MIXED_H	16	#define ROTARY_POSITION_EMBEDDING_AB_MIXED_H
17		17
18	-#include "apply_rotary_pos_emb_common.h"	18	+#include "inplace_partial_rotary_mul_arpe_common.h"
19		19
20	namespace InplacePartialRotaryMul {	20	namespace InplacePartialRotaryMul {
21	using namespace AscendC;	21	using namespace AscendC;
	@@ -25,12 +25,12 @@ class RotaryPositionEmbeddingABMixed {
25	public:	25	public:
26	__aicore__ inline RotaryPositionEmbeddingABMixed(){};	26	__aicore__ inline RotaryPositionEmbeddingABMixed(){};
27	__aicore__ inline void Init(GM_ADDR x, GM_ADDR cos, GM_ADDR sin, GM_ADDR y, GM_ADDR workspace,	27	__aicore__ inline void Init(GM_ADDR x, GM_ADDR cos, GM_ADDR sin, GM_ADDR y, GM_ADDR workspace,
28	- const InplacePartialRopeRegbaseTilingData tilingData, TPipe pipe);	28	+ const InplacePartialRopeRegbaseTilingData tilingData, TPipe pipe);
29	__aicore__ inline void Process();	29	__aicore__ inline void Process();
30		30
31	private:	31	private:
32	__aicore__ inline void ProcessLoop(int64_t xGmOffset, LocalTensor<float> cosBuffer, LocalTensor<float> sinBuffer,	32	__aicore__ inline void ProcessLoop(int64_t xGmOffset, LocalTensor<float> cosBuffer, LocalTensor<float> sinBuffer,
33	- int64_t ubIdx, int64_t bsCount, int64_t nCount);	33	+ int64_t ubIdx, int64_t bsCount, int64_t nCount);
34		34
35	private:	35	private:
36	TPipe *pipe_;	36	TPipe *pipe_;
	@@ -57,7 +57,7 @@ private:
57		57
58	template <typename TX>	58	template <typename TX>
59	__aicore__ inline void RotaryPositionEmbeddingABMixed<TX>::Init(GM_ADDR x, GM_ADDR cos, GM_ADDR sin, GM_ADDR y,	59	__aicore__ inline void RotaryPositionEmbeddingABMixed<TX>::Init(GM_ADDR x, GM_ADDR cos, GM_ADDR sin, GM_ADDR y,
60	- GM_ADDR workspace, const InplacePartialRopeRegbaseTilingData tilingData, TPipe pipe)	60	+ GM_ADDR workspace, const InplacePartialRopeRegbaseTilingData tilingData, TPipe pipe)
61	{	61	{
62	pipe_ = pipe;	62	pipe_ = pipe;
63	tilingData_ = tilingData;	63	tilingData_ = tilingData;
	@@ -96,8 +96,7 @@ __aicore__ inline void RotaryPositionEmbeddingABMixed<TX>::Process()
96	uint32_t nLoopCnt = ops::CeilDiv(nBlockCount_, tilingData_->ubFactorN);	96	uint32_t nLoopCnt = ops::CeilDiv(nBlockCount_, tilingData_->ubFactorN);
97	for (uint32_t bsLoopIdx = 0; bsLoopIdx < bsLoopCnt; bsLoopIdx++) {	97	for (uint32_t bsLoopIdx = 0; bsLoopIdx < bsLoopCnt; bsLoopIdx++) {
98	int64_t xGmOffset = bsLoopIdx * tilingData_->ubFactorBS * tilingData_->N * tilingData_->D;	98	int64_t xGmOffset = bsLoopIdx * tilingData_->ubFactorBS * tilingData_->N * tilingData_->D;
99	- uint32_t currBSNum = (bsLoopIdx != bsLoopCnt - 1) ? tilingData_->ubFactorBS	99	+ uint32_t currBSNum = (bsLoopIdx != bsLoopCnt - 1) ? tilingData_->ubFactorBS : bsBlockCount_ - (bsLoopIdx * tilingData_->ubFactorBS);
100	- : bsBlockCount_ - (bsLoopIdx * tilingData_->ubFactorBS);
101		100
102	DataCopyExtParams cosParams = {	101	DataCopyExtParams cosParams = {
103	static_cast<uint16_t>(currBSNum * tilingData_->dSplitCoef), dSplitSizeFloat_, 0, 0, 0};	102	static_cast<uint16_t>(currBSNum * tilingData_->dSplitCoef), dSplitSizeFloat_, 0, 0, 0};
	@@ -105,21 +104,20 @@ __aicore__ inline void RotaryPositionEmbeddingABMixed<TX>::Process()
105	LocalTensor<float> cosBuffer = cosInQueue_.AllocTensor<float>();	104	LocalTensor<float> cosBuffer = cosInQueue_.AllocTensor<float>();
106	LocalTensor<float> sinBuffer = sinInQueue_.AllocTensor<float>();	105	LocalTensor<float> sinBuffer = sinInQueue_.AllocTensor<float>();
107	DataCopyPad(cosBuffer,	106	DataCopyPad(cosBuffer,
108	- cosGm_[bsLoopIdx * tilingData_->ubFactorBS * tilingData_->sliceLength],	107	+ cosGm_[bsLoopIdx * tilingData_->ubFactorBS * tilingData_->sliceLength],
109	- cosParams,	108	+ cosParams,
110	- padParamsFloat_);	109	+ padParamsFloat_);
111	cosInQueue_.EnQue(cosBuffer);	110	cosInQueue_.EnQue(cosBuffer);
112	cosBuffer = cosInQueue_.DeQue<float>();	111	cosBuffer = cosInQueue_.DeQue<float>();
113	DataCopyPad(sinBuffer,	112	DataCopyPad(sinBuffer,
114	- sinGm_[bsLoopIdx * tilingData_->ubFactorBS * tilingData_->sliceLength],	113	+ sinGm_[bsLoopIdx * tilingData_->ubFactorBS * tilingData_->sliceLength],
115	- cosParams,	114	+ cosParams,
116	- padParamsFloat_);	115	+ padParamsFloat_);
117	sinInQueue_.EnQue(sinBuffer);	116	sinInQueue_.EnQue(sinBuffer);
118	sinBuffer = sinInQueue_.DeQue<float>();	117	sinBuffer = sinInQueue_.DeQue<float>();
119		118
120	for (int64_t nLoopIdx = 0; nLoopIdx < nLoopCnt; nLoopIdx++) {	119	for (int64_t nLoopIdx = 0; nLoopIdx < nLoopCnt; nLoopIdx++) {
121	- int64_t currNNum = (nLoopIdx != nLoopCnt - 1) ? tilingData_->ubFactorN	120	+ int64_t currNNum = (nLoopIdx != nLoopCnt - 1) ? tilingData_->ubFactorN : nBlockCount_ - (nLoopIdx * tilingData_->ubFactorN);
122	- : nBlockCount_ - (nLoopIdx * tilingData_->ubFactorN);
123	ProcessLoop(xGmOffset, cosBuffer, sinBuffer, nLoopIdx, currBSNum, currNNum);	121	ProcessLoop(xGmOffset, cosBuffer, sinBuffer, nLoopIdx, currBSNum, currNNum);
124	}	122	}
125		123
	@@ -130,25 +128,25 @@ __aicore__ inline void RotaryPositionEmbeddingABMixed<TX>::Process()
130		128
131	template <typename TX>	129	template <typename TX>
132	__aicore__ inline void RotaryPositionEmbeddingABMixed<TX>::ProcessLoop(int64_t xGmOffset, LocalTensor<float> cosBuffer,	130	__aicore__ inline void RotaryPositionEmbeddingABMixed<TX>::ProcessLoop(int64_t xGmOffset, LocalTensor<float> cosBuffer,
133	- LocalTensor<float> sinBuffer, int64_t ubIdx, int64_t bsCount, int64_t nCount)	131	+ LocalTensor<float> sinBuffer, int64_t ubIdx, int64_t bsCount, int64_t nCount)
134	{	132	{
135	int64_t totalCount = bsCount * nCount;	133	int64_t totalCount = bsCount * nCount;
136	DataCopyExtParams inParams = {static_cast<uint16_t>(totalCount * tilingData_->dSplitCoef),	134	DataCopyExtParams inParams = {static_cast<uint16_t>(totalCount * tilingData_->dSplitCoef),
137	- dSplitSizeTX_,	135	+ dSplitSizeTX_,
138	- static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(TX)),	136	+ static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(TX)),
139	- 0,	137	+ 0,
140	- 0};	138	+ 0};
141	DataCopyExtParams outParams = {static_cast<uint16_t>(totalCount * tilingData_->dSplitCoef),	139	DataCopyExtParams outParams = {static_cast<uint16_t>(totalCount * tilingData_->dSplitCoef),
142	- dSplitSizeTX_,	140	+ dSplitSizeTX_,
143	- 0,	141	+ 0,
144	- static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(TX)),	142	+ static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(TX)),
145	- 0};	143	+ 0};
146	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::DEEPSEEK_INTERLEAVE)) {	144	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::DEEPSEEK_INTERLEAVE)) {
147	inParams = {static_cast<uint16_t>(totalCount),	145	inParams = {static_cast<uint16_t>(totalCount),
148	- tilingData_->D * sizeof(TX),	146	+ tilingData_->D * sizeof(TX),
149	- static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(TX)),	147	+ static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(TX)),
150	- 0,	148	+ 0,
151	- 0};	149	+ 0};
152	}	150	}
153		151
154	LocalTensor<TX> inBuffer = xInQueue_.AllocTensor<TX>();	152	LocalTensor<TX> inBuffer = xInQueue_.AllocTensor<TX>();
	@@ -170,6 +168,6 @@ __aicore__ inline void RotaryPositionEmbeddingABMixed<TX>::ProcessLoop(int64_t x
170	yOutQueue_.FreeTensor(outBuffer);	168	yOutQueue_.FreeTensor(outBuffer);
171	}	169	}
172		170
173	-} // namespace InplacePartialRotaryMul	171	+} // namespace InplacePartialRotaryMul
174		172
175	-#endif // ROTARY_POSITION_EMBEDDING_AB_MIXED_H	173	+#endif // ROTARY_POSITION_EMBEDDING_AB_MIXED_H

Mposembedding/inplace_partial_rotary_mul/op_kernel/arch35/rotary_position_embedding_aba_and_ba_mixed.h+59-59

	@@ -16,7 +16,7 @@
16	#ifndef ROTARY_POSITION_EMBEDDING_ABA_AND_BA_MIXED_H	16	#ifndef ROTARY_POSITION_EMBEDDING_ABA_AND_BA_MIXED_H
17	#define ROTARY_POSITION_EMBEDDING_ABA_AND_BA_MIXED_H	17	#define ROTARY_POSITION_EMBEDDING_ABA_AND_BA_MIXED_H
18		18
19	-#include "apply_rotary_pos_emb_common.h"	19	+#include "inplace_partial_rotary_mul_arpe_common.h"
20		20
21	namespace InplacePartialRotaryMul {	21	namespace InplacePartialRotaryMul {
22	using namespace AscendC;	22	using namespace AscendC;
	@@ -28,7 +28,7 @@ public:
28	__aicore__ inline ~RotaryPositionEmbeddingABAAndBAMixed(){};	28	__aicore__ inline ~RotaryPositionEmbeddingABAAndBAMixed(){};
29		29
30	__aicore__ inline void Init(GM_ADDR q, GM_ADDR cos, GM_ADDR sin, GM_ADDR qOut, GM_ADDR workspace,	30	__aicore__ inline void Init(GM_ADDR q, GM_ADDR cos, GM_ADDR sin, GM_ADDR qOut, GM_ADDR workspace,
31	- const InplacePartialRopeRegbaseTilingData tilingData, TPipe pipe);	31	+ const InplacePartialRopeRegbaseTilingData tilingData, TPipe pipe);
32	__aicore__ inline void Process();	32	__aicore__ inline void Process();
33		33
34	private:	34	private:
	@@ -37,15 +37,15 @@ private:
37	__aicore__ inline void InitLoopParams();	37	__aicore__ inline void InitLoopParams();
38	__aicore__ inline void ProcessInSLoop(int64_t sUbStart, int64_t sUbLength);	38	__aicore__ inline void ProcessInSLoop(int64_t sUbStart, int64_t sUbLength);
39	__aicore__ inline void ProcessInSBLoop(int64_t sUbStart, int64_t sUbLength, int64_t bUbStart, int64_t bUbLength,	39	__aicore__ inline void ProcessInSBLoop(int64_t sUbStart, int64_t sUbLength, int64_t bUbStart, int64_t bUbLength,
40	- LocalTensor<TX> &cos, LocalTensor<TX> &sin);	40	+ LocalTensor<TX> &cos, LocalTensor<TX> &sin);
41	__aicore__ inline void ProcessInSBNLoop(int64_t sUbStart, int64_t sUbLength, int64_t bUbStart, int64_t bUbLength,	41	__aicore__ inline void ProcessInSBNLoop(int64_t sUbStart, int64_t sUbLength, int64_t bUbStart, int64_t bUbLength,
42	- int64_t nUbStart, int64_t nUbLength, int64_t nTotalSize, LocalTensor<float> &cosFloat,	42	+ int64_t nUbStart, int64_t nUbLength, int64_t nTotalSize, LocalTensor<float> &cosFloat,
43	- LocalTensor<float> &sinFloat, GlobalTensor<TX> &in, GlobalTensor<TX> &out);	43	+ LocalTensor<float> &sinFloat, GlobalTensor<TX> &in, GlobalTensor<TX> &out);
44	__aicore__ inline void CopyInCosAndSin(int64_t sStart, int64_t sLength, int64_t bStart, int64_t bLength);	44	__aicore__ inline void CopyInCosAndSin(int64_t sStart, int64_t sLength, int64_t bStart, int64_t bLength);
45	__aicore__ inline void CopyInQ(GlobalTensor<TX> &source, int64_t sStart, int64_t sLength, int64_t bStart,	45	__aicore__ inline void CopyInQ(GlobalTensor<TX> &source, int64_t sStart, int64_t sLength, int64_t bStart,
46	- int64_t bLength, int64_t nStart, int64_t nLength, int64_t nTotalSize);	46	+ int64_t bLength, int64_t nStart, int64_t nLength, int64_t nTotalSize);
47	__aicore__ inline void CopyOutQ(GlobalTensor<TX> &target, int64_t sStart, int64_t sLength, int64_t bStart,	47	__aicore__ inline void CopyOutQ(GlobalTensor<TX> &target, int64_t sStart, int64_t sLength, int64_t bStart,
48	- int64_t bLength, int64_t nStart, int64_t nLength, int64_t nTotalSize);	48	+ int64_t bLength, int64_t nStart, int64_t nLength, int64_t nTotalSize);
49	__aicore__ inline void Compute(	49	__aicore__ inline void Compute(
50	LocalTensor<float> &cos, LocalTensor<float> &sin, int64_t sLength, int64_t bLength, int64_t nLength);	50	LocalTensor<float> &cos, LocalTensor<float> &sin, int64_t sLength, int64_t bLength, int64_t nLength);
51		51
	@@ -82,7 +82,7 @@ private:
82		82
83	template <typename TX, bool IsBBoardcast>	83	template <typename TX, bool IsBBoardcast>
84	__aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::Init(GM_ADDR q, GM_ADDR cos, GM_ADDR sin,	84	__aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::Init(GM_ADDR q, GM_ADDR cos, GM_ADDR sin,
85	- GM_ADDR qOut, GM_ADDR workspace, const InplacePartialRopeRegbaseTilingData tilingData, TPipe pipe)	85	+ GM_ADDR qOut, GM_ADDR workspace, const InplacePartialRopeRegbaseTilingData tilingData, TPipe pipe)
86	{	86	{
87	this->tilingData_ = tilingData;	87	this->tilingData_ = tilingData;
88	this->blockIdx_ = GetBlockIdx();	88	this->blockIdx_ = GetBlockIdx();
	@@ -123,7 +123,7 @@ __aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::I
123	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::DEEPSEEK_INTERLEAVE)) {	123	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::DEEPSEEK_INTERLEAVE)) {
124	this->copyInQSplitCoef_ = 1;	124	this->copyInQSplitCoef_ = 1;
125	this->ubCopyInStride = (this->dAlign_ * sizeof(TX) -	125	this->ubCopyInStride = (this->dAlign_ * sizeof(TX) -
126	- ops::CeilAlign<int64_t>(tilingData_->sliceLength * sizeof(TX), BLOCK_TYPE_SIZE)) /	126	+ ops::CeilAlign<int64_t>(tilingData_->sliceLength * sizeof(TX), BLOCK_TYPE_SIZE)) /
127	BLOCK_TYPE_SIZE;	127	BLOCK_TYPE_SIZE;
128	}	128	}
129	this->pipe_->InitBuffer(	129	this->pipe_->InitBuffer(
	@@ -164,7 +164,7 @@ __aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::P
164	int64_t ubLoopCount = ops::CeilDiv(sBlockLength_, ubFactorS_);	164	int64_t ubLoopCount = ops::CeilDiv(sBlockLength_, ubFactorS_);
165	for (int64_t ubLoopIdx = 0; ubLoopIdx < ubLoopCount; ubLoopIdx++) {	165	for (int64_t ubLoopIdx = 0; ubLoopIdx < ubLoopCount; ubLoopIdx++) {
166	this->ProcessInSLoop(sBlockStart_ + ubLoopIdx * ubFactorS_,	166	this->ProcessInSLoop(sBlockStart_ + ubLoopIdx * ubFactorS_,
167	- ubLoopIdx != ubLoopCount - 1 ? ubFactorS_ : sBlockLength_ - ubLoopIdx * ubFactorS_);	167	+ ubLoopIdx != ubLoopCount - 1 ? ubFactorS_ : sBlockLength_ - ubLoopIdx * ubFactorS_);
168	}	168	}
169	}	169	}
170		170
	@@ -181,30 +181,30 @@ __aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::P
181	LocalTensor<TX> sinUb = sinUbFloat.template ReinterpretCast<TX>();	181	LocalTensor<TX> sinUb = sinUbFloat.template ReinterpretCast<TX>();
182	for (int64_t ubLoopIdx = 0; ubLoopIdx < ubLoopCount; ubLoopIdx++) {	182	for (int64_t ubLoopIdx = 0; ubLoopIdx < ubLoopCount; ubLoopIdx++) {
183	this->ProcessInSBLoop(sUbStart,	183	this->ProcessInSBLoop(sUbStart,
184	- sUbLength,	184	+ sUbLength,
185	- bBlockStart_ + ubLoopIdx * ubFactorB_,	185	+ bBlockStart_ + ubLoopIdx * ubFactorB_,
186	- ubLoopIdx != ubLoopCount - 1 ? ubFactorB_ : bBlockLength_ - ubLoopIdx * ubFactorB_,	186	+ ubLoopIdx != ubLoopCount - 1 ? ubFactorB_ : bBlockLength_ - ubLoopIdx * ubFactorB_,
187	- cosUb,	187	+ cosUb,
188	- sinUb);	188	+ sinUb);
189	}	189	}
190	this->sinInQueue_.FreeTensor(sinUbFloat);	190	this->sinInQueue_.FreeTensor(sinUbFloat);
191	this->cosInQueue_.FreeTensor(cosUbFloat);	191	this->cosInQueue_.FreeTensor(cosUbFloat);
192	} else {	192	} else {
193	for (int64_t ubLoopIdx = 0; ubLoopIdx < ubLoopCount; ubLoopIdx++) {	193	for (int64_t ubLoopIdx = 0; ubLoopIdx < ubLoopCount; ubLoopIdx++) {
194	this->CopyInCosAndSin(sUbStart,	194	this->CopyInCosAndSin(sUbStart,
195	- sUbLength,	195	+ sUbLength,
196	- bBlockStart_ + ubLoopIdx * ubFactorB_,	196	+ bBlockStart_ + ubLoopIdx * ubFactorB_,
197	- ubLoopIdx != ubLoopCount - 1 ? ubFactorB_ : bBlockLength_ - ubLoopIdx * ubFactorB_);	197	+ ubLoopIdx != ubLoopCount - 1 ? ubFactorB_ : bBlockLength_ - ubLoopIdx * ubFactorB_);
198	LocalTensor<float> cosUbFloat = this->cosInQueue_.template DeQue<float>();	198	LocalTensor<float> cosUbFloat = this->cosInQueue_.template DeQue<float>();
199	LocalTensor<float> sinUbFloat = this->sinInQueue_.template DeQue<float>();	199	LocalTensor<float> sinUbFloat = this->sinInQueue_.template DeQue<float>();
200	LocalTensor<TX> cosUb = cosUbFloat.template ReinterpretCast<TX>();	200	LocalTensor<TX> cosUb = cosUbFloat.template ReinterpretCast<TX>();
201	LocalTensor<TX> sinUb = sinUbFloat.template ReinterpretCast<TX>();	201	LocalTensor<TX> sinUb = sinUbFloat.template ReinterpretCast<TX>();
202	this->ProcessInSBLoop(sUbStart,	202	this->ProcessInSBLoop(sUbStart,
203	- sUbLength,	203	+ sUbLength,
204	- bBlockStart_ + ubLoopIdx * ubFactorB_,	204	+ bBlockStart_ + ubLoopIdx * ubFactorB_,
205	- ubLoopIdx != ubLoopCount - 1 ? ubFactorB_ : bBlockLength_ - ubLoopIdx * ubFactorB_,	205	+ ubLoopIdx != ubLoopCount - 1 ? ubFactorB_ : bBlockLength_ - ubLoopIdx * ubFactorB_,
206	- cosUb,	206	+ cosUb,
207	- sinUb);	207	+ sinUb);
208	this->cosInQueue_.FreeTensor(cosUbFloat);	208	this->cosInQueue_.FreeTensor(cosUbFloat);
209	this->sinInQueue_.FreeTensor(sinUbFloat);	209	this->sinInQueue_.FreeTensor(sinUbFloat);
210	}	210	}
	@@ -213,30 +213,30 @@ __aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::P
213		213
214	template <typename TX, bool IsBBoardcast>	214	template <typename TX, bool IsBBoardcast>
215	__aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::ProcessInSBLoop(int64_t sUbStart,	215	__aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::ProcessInSBLoop(int64_t sUbStart,
216	- int64_t sUbLength, int64_t bUbStart, int64_t bUbLength, LocalTensor<TX> &cos, LocalTensor<TX> &sin)	216	+ int64_t sUbLength, int64_t bUbStart, int64_t bUbLength, LocalTensor<TX> &cos, LocalTensor<TX> &sin)
217	{	217	{
218	int64_t qUbLoopCount = ops::CeilDiv(tilingData_->N, ubFactorN_);	218	int64_t qUbLoopCount = ops::CeilDiv(tilingData_->N, ubFactorN_);
219	LocalTensor<float> cosFloat = cos.template ReinterpretCast<float>();	219	LocalTensor<float> cosFloat = cos.template ReinterpretCast<float>();
220	LocalTensor<float> sinFloat = sin.template ReinterpretCast<float>();	220	LocalTensor<float> sinFloat = sin.template ReinterpretCast<float>();
221	for (int64_t ubLoopIdx = 0; ubLoopIdx < qUbLoopCount; ubLoopIdx++) {	221	for (int64_t ubLoopIdx = 0; ubLoopIdx < qUbLoopCount; ubLoopIdx++) {
222	this->ProcessInSBNLoop(sUbStart,	222	this->ProcessInSBNLoop(sUbStart,
223	- sUbLength,	223	+ sUbLength,
224	- bUbStart,	224	+ bUbStart,
225	- bUbLength,	225	+ bUbLength,
226	- ubLoopIdx * ubFactorN_,	226	+ ubLoopIdx * ubFactorN_,
227	- ubLoopIdx != qUbLoopCount - 1 ? ubFactorN_ : tilingData_->N - ubLoopIdx * ubFactorN_,	227	+ ubLoopIdx != qUbLoopCount - 1 ? ubFactorN_ : tilingData_->N - ubLoopIdx * ubFactorN_,
228	- tilingData_->N,	228	+ tilingData_->N,
229	- cosFloat,	229	+ cosFloat,
230	- sinFloat,	230	+ sinFloat,
231	- qGm_,	231	+ qGm_,
232	- qOutGm_);	232	+ qOutGm_);
233	}	233	}
234	}	234	}
235		235
236	template <typename TX, bool IsBBoardcast>	236	template <typename TX, bool IsBBoardcast>
237	__aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::ProcessInSBNLoop(int64_t sUbStart,	237	__aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::ProcessInSBNLoop(int64_t sUbStart,
238	- int64_t sUbLength, int64_t bUbStart, int64_t bUbLength, int64_t nUbStart, int64_t nUbLength, int64_t nTotalSize,	238	+ int64_t sUbLength, int64_t bUbStart, int64_t bUbLength, int64_t nUbStart, int64_t nUbLength, int64_t nTotalSize,
239	- LocalTensor<float> &cosFloat, LocalTensor<float> &sinFloat, GlobalTensor<TX> &in, GlobalTensor<TX> &out)	239	+ LocalTensor<float> &cosFloat, LocalTensor<float> &sinFloat, GlobalTensor<TX> &in, GlobalTensor<TX> &out)
240	{	240	{
241	CopyInQ(in, sUbStart, sUbLength, bUbStart, bUbLength, nUbStart, nUbLength, nTotalSize);	241	CopyInQ(in, sUbStart, sUbLength, bUbStart, bUbLength, nUbStart, nUbLength, nTotalSize);
242	Compute(cosFloat, sinFloat, sUbLength, bUbLength, nUbLength);	242	Compute(cosFloat, sinFloat, sUbLength, bUbLength, nUbLength);
	@@ -268,13 +268,13 @@ __aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::C
268	copyExtParams.srcStride = 0;	268	copyExtParams.srcStride = 0;
269	copyExtParams.dstStride = 0;	269	copyExtParams.dstStride = 0;
270	DataCopyPad(cosUb,	270	DataCopyPad(cosUb,
271	- this->cosGm_[bStart * tilingData_->S * tilingData_->sliceLength + sStart * tilingData_->sliceLength],	271	+ this->cosGm_[bStart * tilingData_->S * tilingData_->sliceLength + sStart * tilingData_->sliceLength],
272	- copyExtParams,	272	+ copyExtParams,
273	- copyPadExtparams);	273	+ copyPadExtparams);
274	DataCopyPad(sinUb,	274	DataCopyPad(sinUb,
275	- this->sinGm_[bStart * tilingData_->S * tilingData_->sliceLength + sStart * tilingData_->sliceLength],	275	+ this->sinGm_[bStart * tilingData_->S * tilingData_->sliceLength + sStart * tilingData_->sliceLength],
276	- copyExtParams,	276	+ copyExtParams,
277	- copyPadExtparams);	277	+ copyPadExtparams);
278	ResetLoopModePara(DataCopyMVType::OUT_TO_UB);	278	ResetLoopModePara(DataCopyMVType::OUT_TO_UB);
279	this->cosInQueue_.template EnQue(cosUb);	279	this->cosInQueue_.template EnQue(cosUb);
280	this->sinInQueue_.template EnQue(sinUb);	280	this->sinInQueue_.template EnQue(sinUb);
	@@ -282,8 +282,8 @@ __aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::C
282		282
283	template <typename TX, bool IsBBoardcast>	283	template <typename TX, bool IsBBoardcast>
284	__aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::CopyInQ(GlobalTensor<TX> &source,	284	__aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::CopyInQ(GlobalTensor<TX> &source,
285	- int64_t sStart, int64_t sLength, int64_t bStart, int64_t bLength, int64_t nStart, int64_t nLength,	285	+ int64_t sStart, int64_t sLength, int64_t bStart, int64_t bLength, int64_t nStart, int64_t nLength,
286	- int64_t nTotalSize)	286	+ int64_t nTotalSize)
287	{	287	{
288	LocalTensor<TX> target = this->qInQueue_.template AllocTensor<TX>();	288	LocalTensor<TX> target = this->qInQueue_.template AllocTensor<TX>();
289	LoopModeParams loopParams;	289	LoopModeParams loopParams;
	@@ -310,8 +310,8 @@ __aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::C
310		310
311	template <typename TX, bool IsBBoardcast>	311	template <typename TX, bool IsBBoardcast>
312	__aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::CopyOutQ(GlobalTensor<TX> &target,	312	__aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::CopyOutQ(GlobalTensor<TX> &target,
313	- int64_t sStart, int64_t sLength, int64_t bStart, int64_t bLength, int64_t nStart, int64_t nLength,	313	+ int64_t sStart, int64_t sLength, int64_t bStart, int64_t bLength, int64_t nStart, int64_t nLength,
314	- int64_t nTotalSize)	314	+ int64_t nTotalSize)
315	{	315	{
316	LocalTensor<TX> source = this->qOutQueue_.template DeQue<TX>();	316	LocalTensor<TX> source = this->qOutQueue_.template DeQue<TX>();
317	LoopModeParams loopParams;	317	LoopModeParams loopParams;
	@@ -343,22 +343,22 @@ __aicore__ inline void RotaryPositionEmbeddingABAAndBAMixed<TX, IsBBoardcast>::C
343	int64_t totalLength = sLength * bLength * nLength * tilingData_->sliceLength;	343	int64_t totalLength = sLength * bLength * nLength * tilingData_->sliceLength;
344		344
345	BatchInterleaveModeVFMixed<TX, IsBBoardcast>((__ubuf__ TX *)inUb.GetPhyAddr(),	345	BatchInterleaveModeVFMixed<TX, IsBBoardcast>((__ubuf__ TX *)inUb.GetPhyAddr(),
346	- (__ubuf__ float *)cos.GetPhyAddr(),	346	+ (__ubuf__ float *)cos.GetPhyAddr(),
347	- (__ubuf__ float *)sin.GetPhyAddr(),	347	+ (__ubuf__ float *)sin.GetPhyAddr(),
348	- (__ubuf__ TX *)outUb.GetPhyAddr(),	348	+ (__ubuf__ TX *)outUb.GetPhyAddr(),
349	- sLength,	349	+ sLength,
350	- bLength,	350	+ bLength,
351	- nLength,	351	+ nLength,
352	- tilingData_->sliceLength,	352	+ tilingData_->sliceLength,
353	- dAlign_,	353	+ dAlign_,
354	- dAlignFloat_,	354	+ dAlignFloat_,
355	- ubFactorS_,	355	+ ubFactorS_,
356	- ubFactorN_);	356	+ ubFactorN_);
357		357
358	this->qInQueue_.FreeTensor(inUb);	358	this->qInQueue_.FreeTensor(inUb);
359	this->qOutQueue_.template EnQue(outUb);	359	this->qOutQueue_.template EnQue(outUb);
360	}	360	}
361		361
362	-} // namespace InplacePartialRotaryMul	362	+} // namespace InplacePartialRotaryMul
363		363
364	-#endif // ROTARY_POSITION_EMBEDDING_ABA_AND_BA_MIXED_H	364	+#endif // ROTARY_POSITION_EMBEDDING_ABA_AND_BA_MIXED_H

Mposembedding/inplace_partial_rotary_mul/op_kernel/arch35/rotary_position_embedding_bab_mixed.h+24-24

	@@ -16,7 +16,7 @@
16	#ifndef ROTARY_POSITION_EMBEDDING_BAB_MIXED_H	16	#ifndef ROTARY_POSITION_EMBEDDING_BAB_MIXED_H
17	#define ROTARY_POSITION_EMBEDDING_BAB_MIXED_H	17	#define ROTARY_POSITION_EMBEDDING_BAB_MIXED_H
18		18
19	-#include "apply_rotary_pos_emb_common.h"	19	+#include "inplace_partial_rotary_mul_arpe_common.h"
20		20
21	namespace InplacePartialRotaryMul {	21	namespace InplacePartialRotaryMul {
22	using namespace AscendC;	22	using namespace AscendC;
	@@ -25,7 +25,8 @@ template <typename TX>
25	class RotaryPositionEmbeddingBABMixed {	25	class RotaryPositionEmbeddingBABMixed {
26	public:	26	public:
27	__aicore__ inline RotaryPositionEmbeddingBABMixed(TPipe pipe, const InplacePartialRopeRegbaseTilingData tiling)	27	__aicore__ inline RotaryPositionEmbeddingBABMixed(TPipe pipe, const InplacePartialRopeRegbaseTilingData tiling)
28	- : pipe_(pipe), tilingData_(tiling){};	28	+ : pipe_(pipe),
		29	+ tilingData_(tiling){};
29	__aicore__ inline void Init(GM_ADDR x, GM_ADDR cos, GM_ADDR sin, GM_ADDR y);	30	__aicore__ inline void Init(GM_ADDR x, GM_ADDR cos, GM_ADDR sin, GM_ADDR y);
30	__aicore__ inline void Process();	31	__aicore__ inline void Process();
31		32
	@@ -58,10 +59,10 @@ private:
58	__aicore__ inline void PrePareParams();	59	__aicore__ inline void PrePareParams();
59	__aicore__ inline void ProcessNLoop(const uint32_t bIdx, const uint32_t sIdx, const uint32_t currSNum);	60	__aicore__ inline void ProcessNLoop(const uint32_t bIdx, const uint32_t sIdx, const uint32_t currSNum);
60	__aicore__ inline void Compute(const LocalTensor<float> &sinTensor, const LocalTensor<float> &cosTensor,	61	__aicore__ inline void Compute(const LocalTensor<float> &sinTensor, const LocalTensor<float> &cosTensor,
61	- const LocalTensor<TX> &inTensor, const LocalTensor<TX> &outTensor, const uint32_t currSNum,	62	+ const LocalTensor<TX> &inTensor, const LocalTensor<TX> &outTensor, const uint32_t currSNum,
62	- const uint32_t currDNum);	63	+ const uint32_t currDNum);
63	__aicore__ inline void ProcessN(const LocalTensor<float> &sinTensor, const LocalTensor<float> &cosTensor,	64	__aicore__ inline void ProcessN(const LocalTensor<float> &sinTensor, const LocalTensor<float> &cosTensor,
64	- const uint32_t bIdx, const uint32_t sIdx, const uint32_t currSNum);	65	+ const uint32_t bIdx, const uint32_t sIdx, const uint32_t currSNum);
65	};	66	};
66		67
67	template <typename TX>	68	template <typename TX>
	@@ -144,7 +145,7 @@ __aicore__ inline void RotaryPositionEmbeddingBABMixed<TX>::ProcessNLoop(
144		145
145	template <typename TX>	146	template <typename TX>
146	__aicore__ inline void RotaryPositionEmbeddingBABMixed<TX>::ProcessN(const LocalTensor<float> &sinTensor,	147	__aicore__ inline void RotaryPositionEmbeddingBABMixed<TX>::ProcessN(const LocalTensor<float> &sinTensor,
147	- const LocalTensor<float> &cosTensor, const uint32_t bIdx, const uint32_t sIdx, const uint32_t currSNum)	148	+ const LocalTensor<float> &cosTensor, const uint32_t bIdx, const uint32_t sIdx, const uint32_t currSNum)
148	{	149	{
149	LocalTensor<TX> xTensor;	150	LocalTensor<TX> xTensor;
150	LocalTensor<TX> yTensor;	151	LocalTensor<TX> yTensor;
	@@ -154,23 +155,22 @@ __aicore__ inline void RotaryPositionEmbeddingBABMixed<TX>::ProcessN(const Local
154	int64_t offset = baseOffset + idxN * ubFactorN_ * tilingData_->D;	155	int64_t offset = baseOffset + idxN * ubFactorN_ * tilingData_->D;
155	xTensor = xInQue_.AllocTensor<TX>();	156	xTensor = xInQue_.AllocTensor<TX>();
156	DataCopyExtParams copyInParams{static_cast<uint16_t>(currSNum * currDNum * dSplitCoef_),	157	DataCopyExtParams copyInParams{static_cast<uint16_t>(currSNum * currDNum * dSplitCoef_),
157	- dSplitSize_,	158	+ dSplitSize_,
158	- static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(TX)),	159	+ static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(TX)),
159	- 0,	160	+ 0,
160	- 0};	161	+ 0};
161	DataCopyExtParams copyOutParams{static_cast<uint16_t>(currSNum * currDNum * dSplitCoef_),	162	DataCopyExtParams copyOutParams{static_cast<uint16_t>(currSNum * currDNum * dSplitCoef_),
162	- dSplitSize_,	163	+ dSplitSize_,
163	- 0,	164	+ 0,
164	- static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(TX)),	165	+ static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(TX)),
165	- 0};	166	+ 0};
166	if (	167	if (
167	- tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::DEEPSEEK_INTERLEAVE))	168	+ tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::DEEPSEEK_INTERLEAVE)) {
168	- {
169	copyInParams = {static_cast<uint16_t>(currSNum * currDNum),	169	copyInParams = {static_cast<uint16_t>(currSNum * currDNum),
170	- tilingData_->sliceLength * sizeof(TX),	170	+ tilingData_->sliceLength * sizeof(TX),
171	- static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(TX)),	171	+ static_cast<uint32_t>((tilingData_->D - tilingData_->sliceLength) * sizeof(TX)),
172	- 0,	172	+ 0,
173	- 0};	173	+ 0};
174	}	174	}
175	DataCopyPadExtParams<TX> padParams{false, 0, 0, 0};	175	DataCopyPadExtParams<TX> padParams{false, 0, 0, 0};
176	DataCopyPad(xTensor, xGm_[offset], copyInParams, padParams);	176	DataCopyPad(xTensor, xGm_[offset], copyInParams, padParams);
	@@ -188,8 +188,8 @@ __aicore__ inline void RotaryPositionEmbeddingBABMixed<TX>::ProcessN(const Local
188		188
189	template <typename TX>	189	template <typename TX>
190	__aicore__ inline void RotaryPositionEmbeddingBABMixed<TX>::Compute(const LocalTensor<float> &sinTensor,	190	__aicore__ inline void RotaryPositionEmbeddingBABMixed<TX>::Compute(const LocalTensor<float> &sinTensor,
191	- const LocalTensor<float> &cosTensor, const LocalTensor<TX> &inTensor, const LocalTensor<TX> &outTensor,	191	+ const LocalTensor<float> &cosTensor, const LocalTensor<TX> &inTensor, const LocalTensor<TX> &outTensor,
192	- const uint32_t currSNum, const uint32_t currDNum)	192	+ const uint32_t currSNum, const uint32_t currDNum)
193	{	193	{
194	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::INTERLEAVE)) {	194	if (tilingData_->rotaryMode == static_cast<int64_t>(InplacePartialRotaryPosEmbeddingMode::INTERLEAVE)) {
195	InterleaveModeVFMixed<TX>(	195	InterleaveModeVFMixed<TX>(
	@@ -205,5 +205,5 @@ __aicore__ inline void RotaryPositionEmbeddingBABMixed<TX>::Compute(const LocalT
205	}	205	}
206	}	206	}
207		207
208	-} // namespace InplacePartialRotaryMul	208	+} // namespace InplacePartialRotaryMul
209	-#endif // ROTARY_POSITION_EMBEDDING_BAB_MIXED_H	209	+#endif // ROTARY_POSITION_EMBEDDING_BAB_MIXED_H

Mposembedding/inplace_partial_rotary_mul/op_kernel/inplace_partial_rotary_mul_apt.cpp+22-53

	@@ -13,10 +13,10 @@
13	* \brief	13	* \brief
14	*/	14	*/
15	#include "kernel_operator.h"	15	#include "kernel_operator.h"
16	-#include "arch35/rotary_position_embedding_bab.h"	16	+#include "arch35/inplace_partial_rotary_mul_bab.h"
17	-#include "arch35/rotary_position_embedding_ab.h"	17	+#include "arch35/inplace_partial_rotary_mul_ab.h"
18	-#include "arch35/rotary_position_embedding_aba_and_ba.h"	18	+#include "arch35/inplace_partial_rotary_mul_aba_and_ba.h"
19	-#include "arch35/rotary_position_embedding_a_and_b.h"	19	+#include "arch35/inplace_partial_rotary_mul_a_and_b.h"
20	#include "arch35/rotary_position_embedding_bab_mixed.h"	20	#include "arch35/rotary_position_embedding_bab_mixed.h"
21	#include "arch35/rotary_position_embedding_aba_and_ba_mixed.h"	21	#include "arch35/rotary_position_embedding_aba_and_ba_mixed.h"
22	#include "arch35/rotary_position_embedding_a_and_b_mixed.h"	22	#include "arch35/rotary_position_embedding_a_and_b_mixed.h"
	@@ -59,48 +59,37 @@ extern "C" __global__ __aicore__ void inplace_partial_rotary_mul(GM_ADDR x, GM_A
59	return;	59	return;
60	}	60	}
61	}	61	}
62	- if (TILING_KEY_IS(TILING_KEY_ABA))	62	+ if (TILING_KEY_IS(TILING_KEY_ABA)) {
63	- {
64	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	63	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
65	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	64	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
66	InplacePartialRotaryMul::RotaryPositionEmbeddingABAAndBA<DTYPE_X, false> op;	65	InplacePartialRotaryMul::RotaryPositionEmbeddingABAAndBA<DTYPE_X, false> op;
67	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);	66	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);
68	op.Process();	67	op.Process();
69	- }	68	+ } else if (TILING_KEY_IS(TILING_KEY_BA)) {
70	- else if (TILING_KEY_IS(TILING_KEY_BA))
71	- {
72	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	69	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
73	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	70	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
74	InplacePartialRotaryMul::RotaryPositionEmbeddingABAAndBA<DTYPE_X, true> op;	71	InplacePartialRotaryMul::RotaryPositionEmbeddingABAAndBA<DTYPE_X, true> op;
75	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);	72	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);
76	op.Process();	73	op.Process();
77	- }	74	+ } else if (TILING_KEY_IS(TILING_KEY_BAB)) {
78	- else if (TILING_KEY_IS(TILING_KEY_BAB))
79	- {
80	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	75	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
81	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	76	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
82	InplacePartialRotaryMul::RotaryPositionEmbeddingBAB<DTYPE_X> op(&pipe, tilingData);	77	InplacePartialRotaryMul::RotaryPositionEmbeddingBAB<DTYPE_X> op(&pipe, tilingData);
83	op.Init(x, cos, sin, y);	78	op.Init(x, cos, sin, y);
84	op.Process();	79	op.Process();
85	- }	80	+ } else if (TILING_KEY_IS(TILING_KEY_AB)) {
86	- else if (TILING_KEY_IS(TILING_KEY_AB))
87	- {
88	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	81	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
89	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	82	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
90	InplacePartialRotaryMul::RotaryPositionEmbeddingAB<DTYPE_X> op;	83	InplacePartialRotaryMul::RotaryPositionEmbeddingAB<DTYPE_X> op;
91	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);	84	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);
92	op.Process();	85	op.Process();
93	- }	86	+ } else if (TILING_KEY_IS(TILING_KEY_A)) {
94	- else if (TILING_KEY_IS(TILING_KEY_A))
95	- {
96	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	87	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
97	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	88	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
98	InplacePartialRotaryMul::RotaryPositionEmbeddingAAndB<DTYPE_X, false> op;	89	InplacePartialRotaryMul::RotaryPositionEmbeddingAAndB<DTYPE_X, false> op;
99	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);	90	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);
100	op.Process();	91	op.Process();
101	- }	92	+ } else if (TILING_KEY_IS(TILING_KEY_B)) {
102	- else if (TILING_KEY_IS(TILING_KEY_B))
103	- {
104	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	93	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
105	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	94	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
106	InplacePartialRotaryMul::RotaryPositionEmbeddingAAndB<DTYPE_X, true> op;	95	InplacePartialRotaryMul::RotaryPositionEmbeddingAAndB<DTYPE_X, true> op;
	@@ -108,16 +97,13 @@ extern "C" __global__ __aicore__ void inplace_partial_rotary_mul(GM_ADDR x, GM_A
108	op.Process();	97	op.Process();
109	}	98	}
110	// Mixed precision: x is half/bfloat16, cos/sin are float32	99	// Mixed precision: x is half/bfloat16, cos/sin are float32
111	- else if (TILING_KEY_IS(TILING_KEY_BAB_FP16_FP32_MIXED))	100	+ else if (TILING_KEY_IS(TILING_KEY_BAB_FP16_FP32_MIXED)) {
112	- {
113	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	101	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
114	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	102	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
115	InplacePartialRotaryMul::RotaryPositionEmbeddingBABMixed<half> op(&pipe, tilingData);	103	InplacePartialRotaryMul::RotaryPositionEmbeddingBABMixed<half> op(&pipe, tilingData);
116	op.Init(x, cos, sin, y);	104	op.Init(x, cos, sin, y);
117	op.Process();	105	op.Process();
118	- }	106	+ } else if (TILING_KEY_IS(TILING_KEY_BAB_BF16_FP32_MIXED)) {
119	- else if (TILING_KEY_IS(TILING_KEY_BAB_BF16_FP32_MIXED))
120	- {
121	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	107	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
122	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	108	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
123	InplacePartialRotaryMul::RotaryPositionEmbeddingBABMixed<bfloat16_t> op(&pipe, tilingData);	109	InplacePartialRotaryMul::RotaryPositionEmbeddingBABMixed<bfloat16_t> op(&pipe, tilingData);
	@@ -125,32 +111,25 @@ extern "C" __global__ __aicore__ void inplace_partial_rotary_mul(GM_ADDR x, GM_A
125	op.Process();	111	op.Process();
126	}	112	}
127	// Mixed precision ABA/BA kernels	113	// Mixed precision ABA/BA kernels
128	- else if (TILING_KEY_IS(TILING_KEY_ABA_FP16_FP32_MIXED))	114	+ else if (TILING_KEY_IS(TILING_KEY_ABA_FP16_FP32_MIXED)) {
129	- {
130	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	115	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
131	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	116	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
132	InplacePartialRotaryMul::RotaryPositionEmbeddingABAAndBAMixed<half, false> op;	117	InplacePartialRotaryMul::RotaryPositionEmbeddingABAAndBAMixed<half, false> op;
133	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);	118	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);
134	op.Process();	119	op.Process();
135	- }	120	+ } else if (TILING_KEY_IS(TILING_KEY_ABA_BF16_FP32_MIXED)) {
136	- else if (TILING_KEY_IS(TILING_KEY_ABA_BF16_FP32_MIXED))
137	- {
138	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	121	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
139	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	122	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
140	InplacePartialRotaryMul::RotaryPositionEmbeddingABAAndBAMixed<bfloat16_t, false> op;	123	InplacePartialRotaryMul::RotaryPositionEmbeddingABAAndBAMixed<bfloat16_t, false> op;
141	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);	124	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);
142	op.Process();	125	op.Process();
143	- }	126	+ } else if (TILING_KEY_IS(TILING_KEY_BA_FP16_FP32_MIXED)) {
144	- else if (TILING_KEY_IS(TILING_KEY_BA_FP16_FP32_MIXED))
145	- {
146	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	127	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
147	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	128	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
148	InplacePartialRotaryMul::RotaryPositionEmbeddingABAAndBAMixed<half, true> op;	129	InplacePartialRotaryMul::RotaryPositionEmbeddingABAAndBAMixed<half, true> op;
149	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);	130	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);
150	op.Process();	131	op.Process();
151	- }	132	+ } else if (TILING_KEY_IS(TILING_KEY_BA_BF16_FP32_MIXED)) {
152	- else if (TILING_KEY_IS(TILING_KEY_BA_BF16_FP32_MIXED))
153	- {
154	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	133	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
155	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	134	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
156	InplacePartialRotaryMul::RotaryPositionEmbeddingABAAndBAMixed<bfloat16_t, true> op;	135	InplacePartialRotaryMul::RotaryPositionEmbeddingABAAndBAMixed<bfloat16_t, true> op;
	@@ -158,32 +137,25 @@ extern "C" __global__ __aicore__ void inplace_partial_rotary_mul(GM_ADDR x, GM_A
158	op.Process();	137	op.Process();
159	}	138	}
160	// Mixed precision AAndB kernels	139	// Mixed precision AAndB kernels
161	- else if (TILING_KEY_IS(TILING_KEY_A_FP16_FP32_MIXED))	140	+ else if (TILING_KEY_IS(TILING_KEY_A_FP16_FP32_MIXED)) {
162	- {
163	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	141	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
164	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	142	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
165	InplacePartialRotaryMul::RotaryPositionEmbeddingAAndBMixed<half, false> op;	143	InplacePartialRotaryMul::RotaryPositionEmbeddingAAndBMixed<half, false> op;
166	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);	144	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);
167	op.Process();	145	op.Process();
168	- }	146	+ } else if (TILING_KEY_IS(TILING_KEY_A_BF16_FP32_MIXED)) {
169	- else if (TILING_KEY_IS(TILING_KEY_A_BF16_FP32_MIXED))
170	- {
171	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	147	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
172	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	148	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
173	InplacePartialRotaryMul::RotaryPositionEmbeddingAAndBMixed<bfloat16_t, false> op;	149	InplacePartialRotaryMul::RotaryPositionEmbeddingAAndBMixed<bfloat16_t, false> op;
174	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);	150	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);
175	op.Process();	151	op.Process();
176	- }	152	+ } else if (TILING_KEY_IS(TILING_KEY_B_FP16_FP32_MIXED)) {
177	- else if (TILING_KEY_IS(TILING_KEY_B_FP16_FP32_MIXED))
178	- {
179	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	153	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
180	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	154	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
181	InplacePartialRotaryMul::RotaryPositionEmbeddingAAndBMixed<half, true> op;	155	InplacePartialRotaryMul::RotaryPositionEmbeddingAAndBMixed<half, true> op;
182	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);	156	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);
183	op.Process();	157	op.Process();
184	- }	158	+ } else if (TILING_KEY_IS(TILING_KEY_B_BF16_FP32_MIXED)) {
185	- else if (TILING_KEY_IS(TILING_KEY_B_BF16_FP32_MIXED))
186	- {
187	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	159	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
188	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	160	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
189	InplacePartialRotaryMul::RotaryPositionEmbeddingAAndBMixed<bfloat16_t, true> op;	161	InplacePartialRotaryMul::RotaryPositionEmbeddingAAndBMixed<bfloat16_t, true> op;
	@@ -191,16 +163,13 @@ extern "C" __global__ __aicore__ void inplace_partial_rotary_mul(GM_ADDR x, GM_A
191	op.Process();	163	op.Process();
192	}	164	}
193	// Mixed precision AB kernels	165	// Mixed precision AB kernels
194	- else if (TILING_KEY_IS(TILING_KEY_AB_FP16_FP32_MIXED))	166	+ else if (TILING_KEY_IS(TILING_KEY_AB_FP16_FP32_MIXED)) {
195	- {
196	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	167	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
197	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	168	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
198	InplacePartialRotaryMul::RotaryPositionEmbeddingABMixed<half> op;	169	InplacePartialRotaryMul::RotaryPositionEmbeddingABMixed<half> op;
199	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);	170	op.Init(x, cos, sin, y, workspace, tilingData, &pipe);
200	op.Process();	171	op.Process();
201	- }	172	+ } else if (TILING_KEY_IS(TILING_KEY_AB_BF16_FP32_MIXED)) {
202	- else if (TILING_KEY_IS(TILING_KEY_AB_BF16_FP32_MIXED))
203	- {
204	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);	173	GET_TILING_DATA_WITH_STRUCT(InplacePartialRopeRegbaseTilingData, tiling_data_in, tiling);
205	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;	174	const InplacePartialRopeRegbaseTilingData *__restrict tilingData = &tiling_data_in;
206	InplacePartialRotaryMul::RotaryPositionEmbeddingABMixed<bfloat16_t> op;	175	InplacePartialRotaryMul::RotaryPositionEmbeddingABMixed<bfloat16_t> op;

Rposembedding/norm_rope_concat/op_host/norm_rope_concat_base.h→posembedding/norm_rope_concat/op_host/norm_rope_concat_base_host.h+5-5

	@@ -9,11 +9,11 @@
9	*/	9	*/
10		10
11	/* !	11	/* !
12	- * \file norm_rope_concat_base.h	12	+ * \file norm_rope_concat_base_host.h
13	* \brief	13	* \brief
14	*/	14	*/
15	-#ifndef _NORM_ROPE_CONCAT_BASE_H_	15	+#ifndef _NORM_ROPE_CONCAT_BASE_HOST_H_
16	-#define _NORM_ROPE_CONCAT_BASE_H_	16	+#define _NORM_ROPE_CONCAT_BASE_HOST_H_
17	#include <cstddef>	17	#include <cstddef>
18	#include <numeric>	18	#include <numeric>
19	#include <algorithm>	19	#include <algorithm>
	@@ -56,7 +56,7 @@ enum class NormType : int64_t {
56	inline bool IsNormTypeValid(int64_t normType)	56	inline bool IsNormTypeValid(int64_t normType)
57	{	57	{
58	return normType >= static_cast<int64_t>(NormType::NONE) &&	58	return normType >= static_cast<int64_t>(NormType::NONE) &&
59	- normType <= static_cast<int64_t>(NormType::RMS_NORM_AFFINE);	59	+ normType <= static_cast<int64_t>(NormType::RMS_NORM_AFFINE);
60	}	60	}
61		61
62	// read_store	62	// read_store
	@@ -173,4 +173,4 @@ enum class AttrIndexBackward : size_t {
173	};	173	};
174	} // namespace nrc	174	} // namespace nrc
175		175
176	-#endif // _NORM_ROPE_CONCAT_BASE_H_	176	+#endif // _NORM_ROPE_CONCAT_BASE_H_

Mposembedding/norm_rope_concat/op_host/norm_rope_concat_infershape.cpp+4-4

	@@ -15,13 +15,13 @@
15		15
16	#include <register/op_impl_registry.h>	16	#include <register/op_impl_registry.h>
17	#include "log/log.h"	17	#include "log/log.h"
18	-#include "norm_rope_concat_base.h"	18	+#include "norm_rope_concat_base_host.h"
19		19
20	using namespace ge;	20	using namespace ge;
21	using namespace nrc;	21	using namespace nrc;
22	namespace ops {	22	namespace ops {
23	static ge::graphStatus CheckShape(gert::InferShapeContext context, const gert::Shape shape, int64_t batch, int64_t head,	23	static ge::graphStatus CheckShape(gert::InferShapeContext context, const gert::Shape shape, int64_t batch, int64_t head,
24	- int64_t dim, int64_t &seq)	24	+ int64_t dim, int64_t &seq)
25	{	25	{
26	if (shape->GetDimNum() != INPUT_DIM_NUM) {	26	if (shape->GetDimNum() != INPUT_DIM_NUM) {
27	OP_LOGE(context->GetNodeName(), "Input must be 4D tensors(B, S, H, D).");	27	OP_LOGE(context->GetNodeName(), "Input must be 4D tensors(B, S, H, D).");
	@@ -35,7 +35,7 @@ static ge::graphStatus CheckShape(gert::InferShapeContext *context, const gert::
35	return ge::GRAPH_SUCCESS;	35	return ge::GRAPH_SUCCESS;
36	}	36	}
37		37
38	-static ge::graphStatus InferShape4NormRopeConcat(gert::InferShapeContext *context)	38	+static ge::graphStatus InferShape4NormRopeConcat(gert::InferShapeContext *context)
39	{	39	{
40	OP_CHECK_IF(context == nullptr, OP_LOGE("NormRopeConcat", "context is nullptr"), return ge::GRAPH_FAILED);	40	OP_CHECK_IF(context == nullptr, OP_LOGE("NormRopeConcat", "context is nullptr"), return ge::GRAPH_FAILED);
41	OP_LOGD(context, "Enter InferShape4NormRopeConcat.");	41	OP_LOGD(context, "Enter InferShape4NormRopeConcat.");
	@@ -182,4 +182,4 @@ static ge::graphStatus InferDataType4NormRopeConcat(gert::InferDataTypeContext *
182	}	182	}
183		183
184	IMPL_OP_INFERSHAPE(NormRopeConcat).InferShape(InferShape4NormRopeConcat).InferDataType(InferDataType4NormRopeConcat);	184	IMPL_OP_INFERSHAPE(NormRopeConcat).InferShape(InferShape4NormRopeConcat).InferDataType(InferDataType4NormRopeConcat);
185	-} // namespace ops	185	+} // namespace ops

Mposembedding/norm_rope_concat/op_host/norm_rope_concat_tiling.h+1-4

	@@ -15,13 +15,12 @@
15	#ifndef _NORM_ROPE_CONCAT_TILING_H_	15	#ifndef _NORM_ROPE_CONCAT_TILING_H_
16	#define _NORM_ROPE_CONCAT_TILING_H_	16	#define _NORM_ROPE_CONCAT_TILING_H_
17		17
18	-
19	#include "register/tilingdata_base.h"	18	#include "register/tilingdata_base.h"
20	#include "tiling/tiling_api.h"	19	#include "tiling/tiling_api.h"
21	#include "op_host/tiling_base.h"	20	#include "op_host/tiling_base.h"
22	#include "exe_graph/runtime/tiling_context.h"	21	#include "exe_graph/runtime/tiling_context.h"
23	#include "register/op_def_registry.h"	22	#include "register/op_def_registry.h"
24	-#include "norm_rope_concat_base.h"	23	+#include "norm_rope_concat_base_host.h"
25	#include "../op_kernel/norm_rope_concat_tiling_key.h"	24	#include "../op_kernel/norm_rope_concat_tiling_key.h"
26		25
27	template <typename T>	26	template <typename T>
	@@ -137,7 +136,6 @@ struct NormRopeConcatContext {
137	const bool *isTraining;	136	const bool *isTraining;
138	};	137	};
139		138
140	-
141	class NormRopeConcatTiling {	139	class NormRopeConcatTiling {
142	public:	140	public:
143	NormRopeConcatTiling() = default;	141	NormRopeConcatTiling() = default;
	@@ -195,5 +193,4 @@ private:
195		193
196	} // namespace optiling	194	} // namespace optiling
197		195
198	-
199	#endif // _NORM_ROPE_CONCAT_TILING_H_	196	#endif // _NORM_ROPE_CONCAT_TILING_H_

Mposembedding/norm_rope_concat/op_kernel/norm_rope_concat.h+79-62

	@@ -16,11 +16,11 @@
16	#ifndef _NORM_ROPE_CONCAT_H_	16	#ifndef _NORM_ROPE_CONCAT_H_
17	#define _NORM_ROPE_CONCAT_H_	17	#define _NORM_ROPE_CONCAT_H_
18		18
19	-#include "norm_rope_concat_base.h"	19	+#include "norm_rope_concat_base_kernel.h"
20		20
21	namespace nrc {	21	namespace nrc {
22	#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3510)	22	#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3510)
23	- using namespace AscendC::Reg;	23	+using namespace AscendC::Reg;
24	#endif	24	#endif
25	template <bool isTraining>	25	template <bool isTraining>
26	class NormOperationForward : public NormOperation {	26	class NormOperationForward : public NormOperation {
	@@ -70,15 +70,15 @@ private:
70		70
71	template <NormType normType>	71	template <NormType normType>
72	__aicore__ inline void DoRMSNorm(const LocalTensor<float> &x, uint32_t heads);	72	__aicore__ inline void DoRMSNorm(const LocalTensor<float> &x, uint32_t heads);
73	-	73	+
74	__aicore__ inline void DoMulAdd(const LocalTensor<float> &x, uint32_t heads);	74	__aicore__ inline void DoMulAdd(const LocalTensor<float> &x, uint32_t heads);
75		75
76	__aicore__ inline void DoMul(const LocalTensor<float> &x, uint32_t heads);	76	__aicore__ inline void DoMul(const LocalTensor<float> &x, uint32_t heads);
77		77
78	- #if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3510)	78	+#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3510)
79	- __aicore__ inline void DoMulAddVfCall(const LocalTensor<float> &x, uint32_t heads);	79	+ __aicore__ inline void DoMulAddVfCall(const LocalTensor<float> &x, uint32_t heads);
80	- __simd_vf__ inline void DoMulAddVf(__ubuf__ float* xBuf, __ubuf__ float* weightBuf, __ubuf__ float* biasBuf, uint32_t heads);	80	+ __simd_vf__ inline void DoMulAddVf(__ubuf__ float xBuf, __ubuf__ float weightBuf, __ubuf__ float *biasBuf, uint32_t heads);
81	- #endif	81	+#endif
82	private:	82	private:
83	TQue<QuePosition::VECIN, DOUBLE_BUFFER> inQue_;	83	TQue<QuePosition::VECIN, DOUBLE_BUFFER> inQue_;
84	TQue<QuePosition::VECIN, SINGLE_BUFFER> normQue_;	84	TQue<QuePosition::VECIN, SINGLE_BUFFER> normQue_;
	@@ -126,39 +126,40 @@ __aicore__ inline void NormOperationForward<isTraining>::DoMulAdd(const LocalTen
126	}	126	}
127		127
128	#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3510)	128	#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3510)
129	- template <bool isTraining>	129	+template <bool isTraining>
130	- __simd_vf__ inline void NormOperationForward<isTraining>::DoMulAddVf(	130	+__simd_vf__ inline void NormOperationForward<isTraining>::DoMulAddVf(
131	- __ubuf__ float* xBuf, __ubuf__ float* weightBuf, __ubuf__ float* biasBuf, uint32_t heads) {	131	+ __ubuf__ float xBuf, __ubuf__ float weightBuf, __ubuf__ float *biasBuf, uint32_t heads)
132	- RegTensor<float> xRegTensor;	132	+{
133	- RegTensor<float> weightRegTensor;	133	+ RegTensor<float> xRegTensor;
134	- RegTensor<float> biasRegTensor;	134	+ RegTensor<float> weightRegTensor;
		135	+ RegTensor<float> biasRegTensor;
135		136
136	- uint32_t oneRepeatSize = AscendC::GetVecLen() / sizeof(float);	137	+ uint32_t oneRepeatSize = AscendC::GetVecLen() / sizeof(float);
137	- uint32_t repeatTimes = (this->alignedNormDim_ + oneRepeatSize - 1) / oneRepeatSize;	138	+ uint32_t repeatTimes = (this->alignedNormDim_ + oneRepeatSize - 1) / oneRepeatSize;
138	- for (uint32_t i = 0; i < heads; ++i) {	139	+ for (uint32_t i = 0; i < heads; ++i) {
139	- uint32_t len = this->alignedNormDim_;	140	+ uint32_t len = this->alignedNormDim_;
140	- __ubuf__ float* tempXBuf = xBuf + i * len;	141	+ __ubuf__ float tempXBuf = xBuf + i len;
141	- __ubuf__ float* tempWeightBuf = weightBuf;	142	+ __ubuf__ float *tempWeightBuf = weightBuf;
142	- __ubuf__ float* tempBiasBuf = biasBuf;	143	+ __ubuf__ float *tempBiasBuf = biasBuf;
143	- for (uint32_t j = 0; j < repeatTimes; ++j) {	144	+ for (uint32_t j = 0; j < repeatTimes; ++j) {
144	- MaskReg maskReg = UpdateMask<float>(len);	145	+ MaskReg maskReg = UpdateMask<float>(len);
145	- LoadAlign(xRegTensor, tempXBuf + j * oneRepeatSize);	146	+ LoadAlign(xRegTensor, tempXBuf + j * oneRepeatSize);
146	- LoadAlign(weightRegTensor, tempWeightBuf + j * oneRepeatSize);	147	+ LoadAlign(weightRegTensor, tempWeightBuf + j * oneRepeatSize);
147	- LoadAlign(biasRegTensor, tempBiasBuf + j * oneRepeatSize);	148	+ LoadAlign(biasRegTensor, tempBiasBuf + j * oneRepeatSize);
148	- MulDstAdd(xRegTensor, weightRegTensor, biasRegTensor, maskReg);	149	+ MulDstAdd(xRegTensor, weightRegTensor, biasRegTensor, maskReg);
149	- StoreAlign(tempXBuf + j * oneRepeatSize, xRegTensor, maskReg);	150	+ StoreAlign(tempXBuf + j * oneRepeatSize, xRegTensor, maskReg);
150	- }
151	}	151	}
152	}	152	}
		153	+}
153		154
154	- template <bool isTraining>	155	+template <bool isTraining>
155	- __aicore__ inline void NormOperationForward<isTraining>::DoMulAddVfCall(const LocalTensor<float> &x, uint32_t heads)	156	+__aicore__ inline void NormOperationForward<isTraining>::DoMulAddVfCall(const LocalTensor<float> &x, uint32_t heads)
156	- {	157	+{
157	- __ubuf__ float* xBuf = (__ubuf__ float*)x.GetPhyAddr();	158	+ __ubuf__ float xBuf = (__ubuf__ float )x.GetPhyAddr();
158	- __ubuf__ float* weightBuf = (__ubuf__ float*)weight_.GetPhyAddr();	159	+ __ubuf__ float weightBuf = (__ubuf__ float )weight_.GetPhyAddr();
159	- __ubuf__ float* biasBuf = (__ubuf__ float*)bias_.GetPhyAddr();	160	+ __ubuf__ float biasBuf = (__ubuf__ float )bias_.GetPhyAddr();
160	- DoMulAddVf(xBuf, weightBuf, biasBuf, heads);	161	+ DoMulAddVf(xBuf, weightBuf, biasBuf, heads);
161	- }	162	+}
162	#endif	163	#endif
163		164
164	template <bool isTraining>	165	template <bool isTraining>
	@@ -166,9 +167,8 @@ __aicore__ inline void NormOperationForward<isTraining>::DoMul(const LocalTensor
166	{	167	{
167	if (isAligned64_) {	168	if (isAligned64_) {
168	for (uint32_t i = 0; i < rptTimes_; ++i) {	169	for (uint32_t i = 0; i < rptTimes_; ++i) {
169	- Mul(x[i * B32_DATA_NUM_PER_REPEAT], x[i * B32_DATA_NUM_PER_REPEAT], weight_[i * B32_DATA_NUM_PER_REPEAT],	170	+ Mul(x[i * B32_DATA_NUM_PER_REPEAT], x[i * B32_DATA_NUM_PER_REPEAT], weight_[i * B32_DATA_NUM_PER_REPEAT],
170	- B32_DATA_NUM_PER_REPEAT, heads, {1, 1, 1, static_cast<uint8_t>(this->normDim_ / 8),	171	+ B32_DATA_NUM_PER_REPEAT, heads, {1, 1, 1, static_cast<uint8_t>(this->normDim_ / 8), static_cast<uint8_t>(this->normDim_ / 8), 0});
171	- static_cast<uint8_t>(this->normDim_ / 8), 0});
172	}	172	}
173	} else {	173	} else {
174	for (uint32_t i = 0; i < heads; ++i) {	174	for (uint32_t i = 0; i < heads; ++i) {
	@@ -180,7 +180,7 @@ __aicore__ inline void NormOperationForward<isTraining>::DoMul(const LocalTensor
180	template <bool isTraining>	180	template <bool isTraining>
181	template <NormType normType>	181	template <NormType normType>
182	__aicore__ inline void NormOperationForward<isTraining>::Prepare(GM_ADDR x, GM_ADDR weight, GM_ADDR bias,	182	__aicore__ inline void NormOperationForward<isTraining>::Prepare(GM_ADDR x, GM_ADDR weight, GM_ADDR bias,
183	- GM_ADDR mean, GM_ADDR rstd)	183	+ GM_ADDR mean, GM_ADDR rstd)
184	{	184	{
185	this->xGm_.SetGlobalBuffer((__gm__ DTYPE_QUERY *)x);	185	this->xGm_.SetGlobalBuffer((__gm__ DTYPE_QUERY *)x);
186	this->weightGm_.SetGlobalBuffer((__gm__ DTYPE_QUERY *)weight);	186	this->weightGm_.SetGlobalBuffer((__gm__ DTYPE_QUERY *)weight);
	@@ -193,7 +193,7 @@ __aicore__ inline void NormOperationForward<isTraining>::Prepare(GM_ADDR x, GM_A
193	// if affine, copy weight & bias	193	// if affine, copy weight & bias
194	DataCopyExtParams copyParams{1, static_cast<uint32_t>(this->normDim_ * sizeof(DTYPE_QUERY)), 0, 0, 0};	194	DataCopyExtParams copyParams{1, static_cast<uint32_t>(this->normDim_ * sizeof(DTYPE_QUERY)), 0, 0, 0};
195	DataCopyPadExtParams<DTYPE_QUERY> padParams{true, 0,	195	DataCopyPadExtParams<DTYPE_QUERY> padParams{true, 0,
196	- static_cast<uint8_t>(this->alignedNormDim_ - this->normDim_), 0};	196	+ static_cast<uint8_t>(this->alignedNormDim_ - this->normDim_), 0};
197	if constexpr (normType == NormType::LAYER_NORM_AFFINE \|\| normType == NormType::LAYER_NORM_AFFINE_ACROSS_HEADS) {	197	if constexpr (normType == NormType::LAYER_NORM_AFFINE \|\| normType == NormType::LAYER_NORM_AFFINE_ACROSS_HEADS) {
198	LocalTensor<DTYPE_QUERY> norm = normQue_.AllocTensor<DTYPE_QUERY>();	198	LocalTensor<DTYPE_QUERY> norm = normQue_.AllocTensor<DTYPE_QUERY>();
199	DataCopyPad(norm, this->weightGm_, copyParams, padParams);	199	DataCopyPad(norm, this->weightGm_, copyParams, padParams);
	@@ -207,7 +207,7 @@ __aicore__ inline void NormOperationForward<isTraining>::Prepare(GM_ADDR x, GM_A
207	}	207	}
208	normQue_.FreeTensor(deQue);	208	normQue_.FreeTensor(deQue);
209	}	209	}
210	-	210	+
211	if constexpr (normType == NormType::RMS_NORM_AFFINE) {	211	if constexpr (normType == NormType::RMS_NORM_AFFINE) {
212	LocalTensor<DTYPE_QUERY> norm = normQue_.AllocTensor<DTYPE_QUERY>();	212	LocalTensor<DTYPE_QUERY> norm = normQue_.AllocTensor<DTYPE_QUERY>();
213	DataCopyPad(norm, this->weightGm_, copyParams, padParams);	213	DataCopyPad(norm, this->weightGm_, copyParams, padParams);
	@@ -225,8 +225,8 @@ __aicore__ inline void NormOperationForward<isTraining>::Prepare(GM_ADDR x, GM_A
225	template <bool isTraining>	225	template <bool isTraining>
226	template <NormType normType>	226	template <NormType normType>
227	__aicore__ inline void NormOperationForward<isTraining>::Process(const LocalTensor<float> &x,	227	__aicore__ inline void NormOperationForward<isTraining>::Process(const LocalTensor<float> &x,
228	- int64_t inOffset, int64_t normOffset,	228	+ int64_t inOffset, int64_t normOffset,
229	- uint32_t heads)	229	+ uint32_t heads)
230	{	230	{
231	CopyIn(inOffset, heads);	231	CopyIn(inOffset, heads);
232	Compute<normType>(x, heads);	232	Compute<normType>(x, heads);
	@@ -295,11 +295,11 @@ __aicore__ inline void NormOperationForward<isTraining>::DoLayerNorm(const Local
295	Div(x, x, tmp0, size);	295	Div(x, x, tmp0, size);
296	PipeBarrier<PIPE_V>();	296	PipeBarrier<PIPE_V>();
297	if constexpr (normType == NormType::LAYER_NORM_AFFINE \|\| normType == NormType::LAYER_NORM_AFFINE_ACROSS_HEADS) {	297	if constexpr (normType == NormType::LAYER_NORM_AFFINE \|\| normType == NormType::LAYER_NORM_AFFINE_ACROSS_HEADS) {
298	- #if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3510)	298	+#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3510)
299	- DoMulAddVfCall(x, heads);	299	+ DoMulAddVfCall(x, heads);
300	- #else	300	+#else
301	- DoMulAdd(x, heads);	301	+ DoMulAdd(x, heads);
302	- #endif	302	+#endif
303	}	303	}
304	PipeBarrier<PIPE_V>();	304	PipeBarrier<PIPE_V>();
305	}	305	}
	@@ -382,7 +382,6 @@ __aicore__ inline void NormOperationForward<isTraining>::CopyOutMeanAndRstd(int6
382	rstdQue_.FreeTensor(rstd);	382	rstdQue_.FreeTensor(rstd);
383	}	383	}
384		384
385	-
386	template <RopeType ropeType>	385	template <RopeType ropeType>
387	class RopeOperationForward : public RopeOperation<ropeType> {	386	class RopeOperationForward : public RopeOperation<ropeType> {
388	public:	387	public:
	@@ -418,7 +417,7 @@ private:
418	template <RopeType ropeType>	417	template <RopeType ropeType>
419	template <RopeType actualRopeType>	418	template <RopeType actualRopeType>
420	__aicore__ inline void RopeOperationForward<ropeType>::Process(const LocalTensor<float> &x,	419	__aicore__ inline void RopeOperationForward<ropeType>::Process(const LocalTensor<float> &x,
421	- int64_t outOffset, uint32_t heads)	420	+ int64_t outOffset, uint32_t heads)
422	{	421	{
423	Compute<actualRopeType>(x, heads);	422	Compute<actualRopeType>(x, heads);
424	CopyOut(outOffset, heads);	423	CopyOut(outOffset, heads);
	@@ -427,7 +426,7 @@ __aicore__ inline void RopeOperationForward<ropeType>::Process(const LocalTensor
427	template <RopeType ropeType>	426	template <RopeType ropeType>
428	template <RopeType actualRopeType>	427	template <RopeType actualRopeType>
429	__aicore__ inline void RopeOperationForward<ropeType>::Compute(const LocalTensor<float> &x,	428	__aicore__ inline void RopeOperationForward<ropeType>::Compute(const LocalTensor<float> &x,
430	- uint32_t heads)	429	+ uint32_t heads)
431	{	430	{
432	LocalTensor<DTYPE_QUERY> output = outQue_.AllocTensor<DTYPE_QUERY>();	431	LocalTensor<DTYPE_QUERY> output = outQue_.AllocTensor<DTYPE_QUERY>();
433	uint32_t size = this->alignedRopeDim_ * heads;	432	uint32_t size = this->alignedRopeDim_ * heads;
	@@ -473,15 +472,33 @@ public:
473	GM_ADDR norm_key_rstd, GM_ADDR norm_added_query_mean,	472	GM_ADDR norm_key_rstd, GM_ADDR norm_added_query_mean,
474	GM_ADDR norm_added_query_rstd, GM_ADDR norm_added_key_mean,	473	GM_ADDR norm_added_query_rstd, GM_ADDR norm_added_key_mean,
475	GM_ADDR norm_added_key_rstd)	474	GM_ADDR norm_added_key_rstd)
476	- : query_(query), key_(key), value_(value), encoderQuery_(encoder_query), encoderKey_(encoder_key),	475	+ : query_(query),
477	- encoderValue_(encoder_value), normQueryWeight_(norm_query_weight), normQueryBias_(norm_query_bias),	476	+ key_(key),
478	- normKeyWeight_(norm_key_weight), normKeyBias_(norm_key_bias), normAddedQueryWeight_(norm_added_query_weight),	477	+ value_(value),
479	- normAddedQueryBias_(norm_added_query_bias), normAddedKeyWeight_(norm_added_key_weight),	478	+ encoderQuery_(encoder_query),
480	- normAddedKeyBias_(norm_added_key_bias), ropeSin_(rope_sin), ropeCos_(rope_cos), queryOutput_(query_output),	479	+ encoderKey_(encoder_key),
481	- keyOutput_(key_output), valueOutput_(value_output), normQueryMean_(norm_query_mean),	480	+ encoderValue_(encoder_value),
482	- normQueryRstd_(norm_query_rstd), normKeyMean_(norm_key_mean), normKeyRstd_(norm_key_rstd),	481	+ normQueryWeight_(norm_query_weight),
483	- normAddedQueryMean_(norm_added_query_mean), normAddedQueryRstd_(norm_added_query_rstd),	482	+ normQueryBias_(norm_query_bias),
484	- normAddedKeyMean_(norm_added_key_mean), normAddedKeyRstd_(norm_added_key_rstd)	483	+ normKeyWeight_(norm_key_weight),
		484	+ normKeyBias_(norm_key_bias),
		485	+ normAddedQueryWeight_(norm_added_query_weight),
		486	+ normAddedQueryBias_(norm_added_query_bias),
		487	+ normAddedKeyWeight_(norm_added_key_weight),
		488	+ normAddedKeyBias_(norm_added_key_bias),
		489	+ ropeSin_(rope_sin),
		490	+ ropeCos_(rope_cos),
		491	+ queryOutput_(query_output),
		492	+ keyOutput_(key_output),
		493	+ valueOutput_(value_output),
		494	+ normQueryMean_(norm_query_mean),
		495	+ normQueryRstd_(norm_query_rstd),
		496	+ normKeyMean_(norm_key_mean),
		497	+ normKeyRstd_(norm_key_rstd),
		498	+ normAddedQueryMean_(norm_added_query_mean),
		499	+ normAddedQueryRstd_(norm_added_query_rstd),
		500	+ normAddedKeyMean_(norm_added_key_mean),
		501	+ normAddedKeyRstd_(norm_added_key_rstd)
485	{	502	{
486	}	503	}
487		504
	@@ -672,4 +689,4 @@ __aicore__ inline void NormRopeConcat<normType, addedNormType, ropeType, concatO
672	}	689	}
673	}	690	}
674	} // namespace nrc	691	} // namespace nrc
675	-#endif // _NORM_ROPE_CONCAT_H_	692	+#endif // _NORM_ROPE_CONCAT_H_

Rposembedding/norm_rope_concat/op_kernel/norm_rope_concat_base.h→posembedding/norm_rope_concat/op_kernel/norm_rope_concat_base_kernel.h+15-9

	@@ -9,13 +9,12 @@
9	*/	9	*/
10		10
11	/* !	11	/* !
12	- * \file norm_rope_concat_base.h	12	+ * \file norm_rope_concat_base_kernel.h
13	* \brief	13	* \brief
14	*/	14	*/
15		15
16	-	16	+#ifndef _NORM_ROPE_CONCAT_BASE_KERNEL_H_
17	-#ifndef _NORM_ROPE_CONCAT_BASE_H_	17	+#define _NORM_ROPE_CONCAT_BASE_KERNEL_H_
18	-#define _NORM_ROPE_CONCAT_BASE_H_
19		18
20	#include "kernel_operator.h"	19	#include "kernel_operator.h"
21		20
	@@ -92,13 +91,16 @@ public:
92	__aicore__ inline RopeOperation(TPipe *pipe, GM_ADDR sin, GM_ADDR cos, uint32_t actualSeq,	91	__aicore__ inline RopeOperation(TPipe *pipe, GM_ADDR sin, GM_ADDR cos, uint32_t actualSeq,
93	uint32_t ropeDim, uint32_t ropeNum,	92	uint32_t ropeDim, uint32_t ropeNum,
94	uint32_t alignedRopeDim)	93	uint32_t alignedRopeDim)
95	- : ropeDim_(ropeDim), ropeNum_(ropeNum), alignedRopeDim_(alignedRopeDim),	94	+ : ropeDim_(ropeDim),
96	- totalRopeDim_(ropeNum * alignedRopeDim), actualSeq_(actualSeq)	95	+ ropeNum_(ropeNum),
		96	+ alignedRopeDim_(alignedRopeDim),
		97	+ totalRopeDim_(ropeNum * alignedRopeDim),
		98	+ actualSeq_(actualSeq)
97	{	99	{
98	if constexpr (ropeType == RopeType::NONE) {	100	if constexpr (ropeType == RopeType::NONE) {
99	return;	101	return;
100	}	102	}
101	-	103	+
102	sinGm_.SetGlobalBuffer((__gm__ DTYPE_ROPE_SIN *)sin);	104	sinGm_.SetGlobalBuffer((__gm__ DTYPE_ROPE_SIN *)sin);
103	cosGm_.SetGlobalBuffer((__gm__ DTYPE_ROPE_SIN *)cos);	105	cosGm_.SetGlobalBuffer((__gm__ DTYPE_ROPE_SIN *)cos);
104	pipe->InitBuffer(ropeQueue_, SINGLE_BUFFER, totalRopeDim_ * NUM_TWO * sizeof(DTYPE_ROPE_SIN));	106	pipe->InitBuffer(ropeQueue_, SINGLE_BUFFER, totalRopeDim_ * NUM_TWO * sizeof(DTYPE_ROPE_SIN));
	@@ -212,7 +214,11 @@ class NormOperation {
212	public:	214	public:
213	__aicore__ inline NormOperation(float eps, float scale, uint32_t normDim, uint32_t normNum, uint32_t alignedNormDim,	215	__aicore__ inline NormOperation(float eps, float scale, uint32_t normDim, uint32_t normNum, uint32_t alignedNormDim,
214	uint32_t alignedNormNum)	216	uint32_t alignedNormNum)
215	- : eps_(eps), scale_(scale), normDim_(normDim), normNum_(normNum), alignedNormDim_(alignedNormDim),	217	+ : eps_(eps),
		218	+ scale_(scale),
		219	+ normDim_(normDim),
		220	+ normNum_(normNum),
		221	+ alignedNormDim_(alignedNormDim),
216	alignedNormNum_(alignedNormNum)	222	alignedNormNum_(alignedNormNum)
217	{	223	{
218	}	224	}
	@@ -232,4 +238,4 @@ protected:
232	};	238	};
233	} // namespace nrc	239	} // namespace nrc
234		240
235	-#endif // _NORM_ROPE_CONCAT_BASE_H_	241	+#endif // _NORM_ROPE_CONCAT_BASE_H_

Rposembedding/norm_rope_concat_grad/op_host/norm_rope_concat_grad_base.h→posembedding/norm_rope_concat_grad/op_host/norm_rope_concat_grad_base_host.h+1-1

	@@ -9,7 +9,7 @@
9	*/	9	*/
10		10
11	/* !	11	/* !
12	- * \file norm_rope_concat_grad_base.h	12	+ * \file norm_rope_concat_grad_base_host.h
13	* \brief	13	* \brief
14	*/	14	*/
15		15

Mposembedding/norm_rope_concat_grad/op_host/norm_rope_concat_grad_infershape.cpp+4-4

	@@ -15,13 +15,13 @@
15		15
16	#include <register/op_impl_registry.h>	16	#include <register/op_impl_registry.h>
17	#include "log/log.h"	17	#include "log/log.h"
18	-#include "norm_rope_concat_grad_base.h"	18	+#include "norm_rope_concat_grad_base_host.h"
19		19
20	using namespace ge;	20	using namespace ge;
21	using namespace NormRopeConcatGrad;	21	using namespace NormRopeConcatGrad;
22	namespace ops {	22	namespace ops {
23	static ge::graphStatus CheckShape(gert::InferShapeContext context, const gert::Shape shape, int64_t batch, int64_t head,	23	static ge::graphStatus CheckShape(gert::InferShapeContext context, const gert::Shape shape, int64_t batch, int64_t head,
24	- int64_t dim, int64_t &seq)	24	+ int64_t dim, int64_t &seq)
25	{	25	{
26	if (shape->GetDimNum() != INPUT_DIM_NUM) {	26	if (shape->GetDimNum() != INPUT_DIM_NUM) {
27	OP_LOGE(context, "Input must be 4D tensors(B, S, H, D).");	27	OP_LOGE(context, "Input must be 4D tensors(B, S, H, D).");
	@@ -36,7 +36,7 @@ static ge::graphStatus CheckShape(gert::InferShapeContext *context, const gert::
36	}	36	}
37		37
38	static ge::graphStatus CheckTransposeShape(gert::InferShapeContext context, const gert::Shape shape, int64_t batch,	38	static ge::graphStatus CheckTransposeShape(gert::InferShapeContext context, const gert::Shape shape, int64_t batch,
39	- int64_t head, int64_t dim, int64_t &seq)	39	+ int64_t head, int64_t dim, int64_t &seq)
40	{	40	{
41	if (shape->GetDimNum() != INPUT_DIM_NUM) {	41	if (shape->GetDimNum() != INPUT_DIM_NUM) {
42	OP_LOGE(context, "Input must be 4D tensors(B, S, H, D).");	42	OP_LOGE(context, "Input must be 4D tensors(B, S, H, D).");
	@@ -234,4 +234,4 @@ static ge::graphStatus InferDataType4NormRopeConcatGrad(gert::InferDataTypeConte
234	}	234	}
235		235
236	IMPL_OP_INFERSHAPE(NormRopeConcatGrad).InferShape(InferShape4NormRopeConcatGrad).InferDataType(InferDataType4NormRopeConcatGrad);	236	IMPL_OP_INFERSHAPE(NormRopeConcatGrad).InferShape(InferShape4NormRopeConcatGrad).InferDataType(InferDataType4NormRopeConcatGrad);
237	-} // namespace ops	237	+} // namespace ops

Mposembedding/norm_rope_concat_grad/op_host/norm_rope_concat_grad_tiling.h+1-1

	@@ -24,7 +24,7 @@
24	#include "tiling/tiling_api.h"	24	#include "tiling/tiling_api.h"
25	#include "exe_graph/runtime/tiling_context.h"	25	#include "exe_graph/runtime/tiling_context.h"
26	#include "register/op_def_registry.h"	26	#include "register/op_def_registry.h"
27	-#include "norm_rope_concat_grad_base.h"	27	+#include "norm_rope_concat_grad_base_host.h"
28		28
29	using namespace NormRopeConcatGrad;	29	using namespace NormRopeConcatGrad;
30		30

Mposembedding/norm_rope_concat_grad/op_host/op_api/aclnn_norm_rope_concat_grad.cpp+29-44

	@@ -15,7 +15,7 @@
15		15
16	#include "acl/acl.h"	16	#include "acl/acl.h"
17	#include "norm_rope_concat_grad.h"	17	#include "norm_rope_concat_grad.h"
18	-#include "../norm_rope_concat_grad_base.h"	18	+#include "../norm_rope_concat_grad_base_host.h"
19	#include "aclnn/aclnn_base.h"	19	#include "aclnn/aclnn_base.h"
20	#include "aclnn_kernels/common/op_error_check.h"	20	#include "aclnn_kernels/common/op_error_check.h"
21	#include "aclnn_kernels/contiguous.h"	21	#include "aclnn_kernels/contiguous.h"
	@@ -31,7 +31,6 @@
31	#include "opdev/make_op_executor.h"	31	#include "opdev/make_op_executor.h"
32	#include "aclnn_norm_rope_concat_grad.h"	32	#include "aclnn_norm_rope_concat_grad.h"
33		33
34	-
35	using namespace op;	34	using namespace op;
36	using namespace NormRopeConcatGrad;	35	using namespace NormRopeConcatGrad;
37		36
	@@ -180,18 +179,18 @@ static bool CheckNormDtypeValid(const aclTensor *normQueryMean, const aclTensor
180	}	179	}
181		180
182	inline static aclnnStatus CheckParam(const aclTensor gradQueryOutput, const aclTensor gradKeyOutput, const aclTensor *gradValueOutput,	181	inline static aclnnStatus CheckParam(const aclTensor gradQueryOutput, const aclTensor gradKeyOutput, const aclTensor *gradValueOutput,
183	- const aclTensor query, const aclTensor key, const aclTensor encoderQuery, const aclTensor encoderKey,	182	+ const aclTensor query, const aclTensor key, const aclTensor encoderQuery, const aclTensor encoderKey,
184	- const aclTensor normQueryWeight, const aclTensor normQueryMean, const aclTensor *normQueryRstd,	183	+ const aclTensor normQueryWeight, const aclTensor normQueryMean, const aclTensor *normQueryRstd,
185	- const aclTensor normKeyWeight, const aclTensor normKeyMean, const aclTensor *normKeyRstd,	184	+ const aclTensor normKeyWeight, const aclTensor normKeyMean, const aclTensor *normKeyRstd,
186	- const aclTensor normAddedQueryWeight, const aclTensor normAddedQueryMean,	185	+ const aclTensor normAddedQueryWeight, const aclTensor normAddedQueryMean,
187	- const aclTensor normAddedQueryRstd, const aclTensor normAddedKeyWeight, const aclTensor *normAddedKeyMean,	186	+ const aclTensor normAddedQueryRstd, const aclTensor normAddedKeyWeight, const aclTensor *normAddedKeyMean,
188	- const aclTensor normAddedKeyRstd, const aclTensor ropeSin, const aclTensor *ropeCos,	187	+ const aclTensor normAddedKeyRstd, const aclTensor ropeSin, const aclTensor *ropeCos,
189	- const aclTensor gradQuery, const aclTensor gradKey, const aclTensor *gradValue,	188	+ const aclTensor gradQuery, const aclTensor gradKey, const aclTensor *gradValue,
190	- const aclTensor gradEncoderQuery, const aclTensor gradEncoderKey, const aclTensor *gradEncoderValue,	189	+ const aclTensor gradEncoderQuery, const aclTensor gradEncoderKey, const aclTensor *gradEncoderValue,
191	- const aclTensor gradNormQueryWeight, const aclTensor gradNormQueryBias, const aclTensor *gradNormKeyWeight,	190	+ const aclTensor gradNormQueryWeight, const aclTensor gradNormQueryBias, const aclTensor *gradNormKeyWeight,
192	- const aclTensor gradNormKeyBias, const aclTensor gradNormAddedQueryWeight,	191	+ const aclTensor gradNormKeyBias, const aclTensor gradNormAddedQueryWeight,
193	- const aclTensor gradNormAddedQueryBias, const aclTensor gradNormAddedKeyWeight,	192	+ const aclTensor gradNormAddedQueryBias, const aclTensor gradNormAddedKeyWeight,
194	- const aclTensor *gradNormAddedKeyBias)	193	+ const aclTensor *gradNormAddedKeyBias)
195	{	194	{
196	// check nullptr	195	// check nullptr
197	CHECK_RET(CheckNotNull(gradQueryOutput, gradKeyOutput, gradValueOutput, query, key, gradQuery, gradKey, gradValue),	196	CHECK_RET(CheckNotNull(gradQueryOutput, gradKeyOutput, gradValueOutput, query, key, gradQuery, gradKey, gradValue),
	@@ -271,35 +270,21 @@ aclnnStatus aclnnNormRopeConcatBackwardGetWorkspaceSize(
271	CHECK_RET(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_QUERY_INDEX)] != nullptr, ACLNN_ERR_INNER_NULLPTR);	270	CHECK_RET(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_QUERY_INDEX)] != nullptr, ACLNN_ERR_INNER_NULLPTR);
272	CHECK_RET(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_KEY_INDEX)] != nullptr, ACLNN_ERR_INNER_NULLPTR);	271	CHECK_RET(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_KEY_INDEX)] != nullptr, ACLNN_ERR_INNER_NULLPTR);
273	CHECK_RET(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_VALUE_INDEX)] != nullptr, ACLNN_ERR_INNER_NULLPTR);	272	CHECK_RET(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_VALUE_INDEX)] != nullptr, ACLNN_ERR_INNER_NULLPTR);
274	- CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[	273	+ CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_QUERY_INDEX)], gradQuery, l0Executor), ACLNN_ERR_INNER_NULLPTR);
275	- static_cast<size_t>(OutputIndexBackward::GRAD_QUERY_INDEX)], gradQuery, l0Executor), ACLNN_ERR_INNER_NULLPTR);	274	+ CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_KEY_INDEX)], gradKey, l0Executor), ACLNN_ERR_INNER_NULLPTR);
276	- CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[	275	+ CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_VALUE_INDEX)], gradValue, l0Executor), ACLNN_ERR_INNER_NULLPTR);
277	- static_cast<size_t>(OutputIndexBackward::GRAD_KEY_INDEX)], gradKey, l0Executor), ACLNN_ERR_INNER_NULLPTR);	276	+ // optional tensor
278	- CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[	277	+ CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_ENCODER_QUERY_INDEX)], gradEncoderQuery, l0Executor), ACLNN_ERR_INNER_NULLPTR);
279	- static_cast<size_t>(OutputIndexBackward::GRAD_VALUE_INDEX)], gradValue, l0Executor), ACLNN_ERR_INNER_NULLPTR);	278	+ CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_ENCODER_KEY_INDEX)], gradEncoderKey, l0Executor), ACLNN_ERR_INNER_NULLPTR);
280	- //optional tensor	279	+ CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_ENCODER_VALUE_INDEX)], gradEncoderValue, l0Executor), ACLNN_ERR_INNER_NULLPTR);
281	- CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[	280	+ CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_NORM_QUERY_WEIGHT_INDEX)], gradNormQueryWeight, l0Executor), ACLNN_ERR_INNER_NULLPTR);
282	- static_cast<size_t>(OutputIndexBackward::GRAD_ENCODER_QUERY_INDEX)], gradEncoderQuery, l0Executor), ACLNN_ERR_INNER_NULLPTR);	281	+ CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_NORM_QUERY_BIAS_INDEX)], gradNormQueryBias, l0Executor), ACLNN_ERR_INNER_NULLPTR);
283	- CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[	282	+ CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_NORM_KEY_WEIGHT_INDEX)], gradNormKeyWeight, l0Executor), ACLNN_ERR_INNER_NULLPTR);
284	- static_cast<size_t>(OutputIndexBackward::GRAD_ENCODER_KEY_INDEX)], gradEncoderKey, l0Executor), ACLNN_ERR_INNER_NULLPTR);	283	+ CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_NORM_KEY_BIAS_INDEX)], gradNormKeyBias, l0Executor), ACLNN_ERR_INNER_NULLPTR);
285	- CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[	284	+ CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_NORM_ADDED_QUERY_WEIGHT_INDEX)], gradNormAddedQueryWeight, l0Executor), ACLNN_ERR_INNER_NULLPTR);
286	- static_cast<size_t>(OutputIndexBackward::GRAD_ENCODER_VALUE_INDEX)], gradEncoderValue, l0Executor), ACLNN_ERR_INNER_NULLPTR);	285	+ CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_NORM_ADDED_QUERY_BIAS_INDEX)], gradNormAddedQueryBias, l0Executor), ACLNN_ERR_INNER_NULLPTR);
287	- CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[	286	+ CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_NORM_ADDED_KEY_WEIGHT_INDEX)], gradNormAddedKeyWeight, l0Executor), ACLNN_ERR_INNER_NULLPTR);
288	- static_cast<size_t>(OutputIndexBackward::GRAD_NORM_QUERY_WEIGHT_INDEX)], gradNormQueryWeight, l0Executor), ACLNN_ERR_INNER_NULLPTR);	287	+ CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[static_cast<size_t>(OutputIndexBackward::GRAD_NORM_ADDED_KEY_BIAS_INDEX)], gradNormAddedKeyBias, l0Executor), ACLNN_ERR_INNER_NULLPTR);
289	- CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[
290	- static_cast<size_t>(OutputIndexBackward::GRAD_NORM_QUERY_BIAS_INDEX)], gradNormQueryBias, l0Executor), ACLNN_ERR_INNER_NULLPTR);
291	- CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[
292	- static_cast<size_t>(OutputIndexBackward::GRAD_NORM_KEY_WEIGHT_INDEX)], gradNormKeyWeight, l0Executor), ACLNN_ERR_INNER_NULLPTR);
293	- CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[
294	- static_cast<size_t>(OutputIndexBackward::GRAD_NORM_KEY_BIAS_INDEX)], gradNormKeyBias, l0Executor), ACLNN_ERR_INNER_NULLPTR);
295	- CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[
296	- static_cast<size_t>(OutputIndexBackward::GRAD_NORM_ADDED_QUERY_WEIGHT_INDEX)], gradNormAddedQueryWeight, l0Executor), ACLNN_ERR_INNER_NULLPTR);
297	- CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[
298	- static_cast<size_t>(OutputIndexBackward::GRAD_NORM_ADDED_QUERY_BIAS_INDEX)], gradNormAddedQueryBias, l0Executor), ACLNN_ERR_INNER_NULLPTR);
299	- CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[
300	- static_cast<size_t>(OutputIndexBackward::GRAD_NORM_ADDED_KEY_WEIGHT_INDEX)], gradNormAddedKeyWeight, l0Executor), ACLNN_ERR_INNER_NULLPTR);
301	- CHECK_RET(checkViewCopyResult(l0NormRopeConcatGradOuts[
302	- static_cast<size_t>(OutputIndexBackward::GRAD_NORM_ADDED_KEY_BIAS_INDEX)], gradNormAddedKeyBias, l0Executor), ACLNN_ERR_INNER_NULLPTR);
303		288
304	*workspaceSize = uniqueExecutor->GetWorkspaceSize();	289	*workspaceSize = uniqueExecutor->GetWorkspaceSize();
305	uniqueExecutor.ReleaseTo(executor);	290	uniqueExecutor.ReleaseTo(executor);
	@@ -315,4 +300,4 @@ aclnnStatus aclnnNormRopeConcatBackward(void *workspace, uint64_t workspaceSize,
315		300
316	#ifdef __cplusplus	301	#ifdef __cplusplus
317	}	302	}
318	-#endif	303	+#endif

Mposembedding/norm_rope_concat_grad/op_kernel/norm_rope_concat_grad.cpp+70-71

Rposembedding/norm_rope_concat_grad/op_kernel/norm_rope_concat_grad_base.h→posembedding/norm_rope_concat_grad/op_kernel/norm_rope_concat_grad_base_kernel.h+14-7

	@@ -9,12 +9,12 @@
9	*/	9	*/
10		10
11	/* !	11	/* !
12	- * \file norm_rope_concat_grad_base.h	12	+ * \file norm_rope_concat_grad_base_kernel.h
13	* \brief	13	* \brief
14	*/	14	*/
15		15
16	-#ifndef NORM_ROPE_CONCAT_GRAD_BASE_H	16	+#ifndef NORM_ROPE_CONCAT_GRAD_BASE_KERNEL_H
17	-#define NORM_ROPE_CONCAT_GRAD_BASE_H	17	+#define NORM_ROPE_CONCAT_GRAD_BASE_KERNEL_H
18		18
19	#include "kernel_operator.h"	19	#include "kernel_operator.h"
20		20
	@@ -112,7 +112,10 @@ public:
112	__aicore__ inline RopeOperation(TBufPool<TPosition::VECCALC, BUF_ID_SIZE> &bufPool, GM_ADDR sin, GM_ADDR cos,	112	__aicore__ inline RopeOperation(TBufPool<TPosition::VECCALC, BUF_ID_SIZE> &bufPool, GM_ADDR sin, GM_ADDR cos,
113	GM_ADDR y, uint32_t actualSeq, uint32_t curSeq, uint32_t ropeDim,	113	GM_ADDR y, uint32_t actualSeq, uint32_t curSeq, uint32_t ropeDim,
114	uint32_t alignedRopeDim)	114	uint32_t alignedRopeDim)
115	- : ropeDim_(ropeDim), alignedRopeDim_(alignedRopeDim), isActive_(curSeq < actualSeq), actualSeq_(actualSeq)	115	+ : ropeDim_(ropeDim),
		116	+ alignedRopeDim_(alignedRopeDim),
		117	+ isActive_(curSeq < actualSeq),
		118	+ actualSeq_(actualSeq)
116	{	119	{
117	yGm_.SetGlobalBuffer((__gm__ DTYPE_QUERY *)y);	120	yGm_.SetGlobalBuffer((__gm__ DTYPE_QUERY *)y);
118	if constexpr (ropeType == RopeType::NONE) {	121	if constexpr (ropeType == RopeType::NONE) {
	@@ -191,7 +194,7 @@ public:
191	DataCopyPad(rope, sinGm_[curSeq * ropeDim_], params, padParams);	194	DataCopyPad(rope, sinGm_[curSeq * ropeDim_], params, padParams);
192	DataCopyPad(rope[alignedRopeDim_], cosGm_[curSeq * ropeDim_], params, padParams);	195	DataCopyPad(rope[alignedRopeDim_], cosGm_[curSeq * ropeDim_], params, padParams);
193	}	196	}
194	-	197	+
195	ropeQueue_.EnQue(rope);	198	ropeQueue_.EnQue(rope);
196	LocalTensor<DTYPE_ROPE_SIN> deQue = ropeQueue_.DeQue<DTYPE_ROPE_SIN>();	199	LocalTensor<DTYPE_ROPE_SIN> deQue = ropeQueue_.DeQue<DTYPE_ROPE_SIN>();
197	if (sizeof(DTYPE_ROPE_SIN) < sizeof(float)) {	200	if (sizeof(DTYPE_ROPE_SIN) < sizeof(float)) {
	@@ -262,7 +265,11 @@ public:
262	__aicore__ inline NormOperation(GM_ADDR x, GM_ADDR weight, GM_ADDR bias, GM_ADDR mean, GM_ADDR rstd, float eps,	265	__aicore__ inline NormOperation(GM_ADDR x, GM_ADDR weight, GM_ADDR bias, GM_ADDR mean, GM_ADDR rstd, float eps,
263	float scale, uint32_t normDim, uint32_t normNum, uint32_t alignedNormDim,	266	float scale, uint32_t normDim, uint32_t normNum, uint32_t alignedNormDim,
264	uint32_t alignedNormNum)	267	uint32_t alignedNormNum)
265	- : eps_(eps), scale_(scale), normDim_(normDim), normNum_(normNum), alignedNormDim_(alignedNormDim),	268	+ : eps_(eps),
		269	+ scale_(scale),
		270	+ normDim_(normDim),
		271	+ normNum_(normNum),
		272	+ alignedNormDim_(alignedNormDim),
266	alignedNormNum_(alignedNormNum)	273	alignedNormNum_(alignedNormNum)
267	{	274	{
268	xGm_.SetGlobalBuffer((__gm__ DTYPE_QUERY *)x);	275	xGm_.SetGlobalBuffer((__gm__ DTYPE_QUERY *)x);
	@@ -287,4 +294,4 @@ protected:
287	};	294	};
288	} // namespace nrcg	295	} // namespace nrcg
289		296
290	-#endif	297	+#endif

Rposembedding/norm_rope_concat_grad/op_kernel/norm_rope_concat_grad.h→posembedding/norm_rope_concat_grad/op_kernel/norm_rope_concat_grad_kernel.h+43-20

	@@ -9,13 +9,13 @@
9	*/	9	*/
10		10
11	/* !	11	/* !
12	- * \file norm_rope_concat_grad.h	12	+ * \file norm_rope_concat_grad_kernel.h
13	* \brief	13	* \brief
14	*/	14	*/
15	-#ifndef NORM_ROPE_CONCAT_GRAD_H	15	+#ifndef NORM_ROPE_CONCAT_GRAD_KERNEL_H
16	-#define NORM_ROPE_CONCAT_GRAD_H	16	+#define NORM_ROPE_CONCAT_GRAD_KERNEL_H
17		17
18	-#include "norm_rope_concat_grad_base.h"	18	+#include "norm_rope_concat_grad_base_kernel.h"
19	using namespace AscendC;	19	using namespace AscendC;
20		20
21	namespace nrcg {	21	namespace nrcg {
	@@ -29,7 +29,8 @@ public:
29	uint32_t alignedNormNum, uint32_t affineBlockNums, uint32_t avgHeadsLow)	29	uint32_t alignedNormNum, uint32_t affineBlockNums, uint32_t avgHeadsLow)
30	: NormOperation<normType>(x, weight, bias, mean, rstd, eps, scale, normDim, normNum, alignedNormDim,	30	: NormOperation<normType>(x, weight, bias, mean, rstd, eps, scale, normDim, normNum, alignedNormDim,
31	alignedNormNum),	31	alignedNormNum),
32	- affineBlockNums_(affineBlockNums), avgHeadsLow_(avgHeadsLow)	32	+ affineBlockNums_(affineBlockNums),
		33	+ avgHeadsLow_(avgHeadsLow)
33	{	34	{
34	xGradGM_.SetGlobalBuffer((__gm__ DTYPE_QUERY *)xGrad);	35	xGradGM_.SetGlobalBuffer((__gm__ DTYPE_QUERY *)xGrad);
35	outParams_.blockLen = normDim * sizeof(DTYPE_QUERY);	36	outParams_.blockLen = normDim * sizeof(DTYPE_QUERY);
	@@ -500,19 +501,41 @@ public:
500	GM_ADDR grad_norm_query_bias, GM_ADDR grad_norm_key_weight, GM_ADDR grad_norm_key_bias,	501	GM_ADDR grad_norm_query_bias, GM_ADDR grad_norm_key_weight, GM_ADDR grad_norm_key_bias,
501	GM_ADDR grad_norm_added_query_weight, GM_ADDR grad_norm_added_query_bias, GM_ADDR grad_norm_added_key_weight,	502	GM_ADDR grad_norm_added_query_weight, GM_ADDR grad_norm_added_query_bias, GM_ADDR grad_norm_added_key_weight,
502	GM_ADDR grad_norm_added_key_bias, GM_ADDR workspace)	503	GM_ADDR grad_norm_added_key_bias, GM_ADDR workspace)
503	- : gradQueryOutput_(grad_query_output), gradKeyOutput_(grad_key_output), gradValueOutput_(grad_value_output),	504	+ : gradQueryOutput_(grad_query_output),
504	- query_(query), key_(key), encoderQuery_(encoder_query), encoderKey_(encoder_key),	505	+ gradKeyOutput_(grad_key_output),
505	- normQueryWeight_(norm_query_weight), normQueryMean_(norm_query_mean), normQueryRstd_(norm_query_rstd),	506	+ gradValueOutput_(grad_value_output),
506	- normKeyWeight_(norm_key_weight), normKeyMean_(norm_key_mean), normKeyRstd_(norm_key_rstd),	507	+ query_(query),
507	- normAddedQueryWeight_(norm_added_query_weight), normAddedQueryMean_(norm_added_query_mean),	508	+ key_(key),
508	- normAddedQueryRstd_(norm_added_query_rstd), normAddedKeyWeight_(norm_added_key_weight),	509	+ encoderQuery_(encoder_query),
509	- normAddedKeyMean_(norm_added_key_mean), normAddedKeyRstd_(norm_added_key_rstd), ropeSin_(rope_sin),	510	+ encoderKey_(encoder_key),
510	- ropeCos_(rope_cos), gradQuery_(grad_query), gradKey_(grad_key), gradValue_(grad_value),	511	+ normQueryWeight_(norm_query_weight),
511	- gradEncoderQuery_(grad_encoderquery), gradEncoderKey_(grad_encoderkey), gradEncoderValue_(grad_encodervalue),	512	+ normQueryMean_(norm_query_mean),
512	- gradNormQueryWeight_(grad_norm_query_weight), gradNormQueryBias_(grad_norm_query_bias),	513	+ normQueryRstd_(norm_query_rstd),
513	- gradNormKeyWeight_(grad_norm_key_weight), gradNormKeyBias_(grad_norm_key_bias),	514	+ normKeyWeight_(norm_key_weight),
514	- gradNormAddedQueryWeight_(grad_norm_added_query_weight), gradNormAddedQueryBias_(grad_norm_added_query_bias),	515	+ normKeyMean_(norm_key_mean),
515	- gradNormAddedKeyWeight_(grad_norm_added_key_weight), gradNormAddedKeyBias_(grad_norm_added_key_bias),	516	+ normKeyRstd_(norm_key_rstd),
		517	+ normAddedQueryWeight_(norm_added_query_weight),
		518	+ normAddedQueryMean_(norm_added_query_mean),
		519	+ normAddedQueryRstd_(norm_added_query_rstd),
		520	+ normAddedKeyWeight_(norm_added_key_weight),
		521	+ normAddedKeyMean_(norm_added_key_mean),
		522	+ normAddedKeyRstd_(norm_added_key_rstd),
		523	+ ropeSin_(rope_sin),
		524	+ ropeCos_(rope_cos),
		525	+ gradQuery_(grad_query),
		526	+ gradKey_(grad_key),
		527	+ gradValue_(grad_value),
		528	+ gradEncoderQuery_(grad_encoderquery),
		529	+ gradEncoderKey_(grad_encoderkey),
		530	+ gradEncoderValue_(grad_encodervalue),
		531	+ gradNormQueryWeight_(grad_norm_query_weight),
		532	+ gradNormQueryBias_(grad_norm_query_bias),
		533	+ gradNormKeyWeight_(grad_norm_key_weight),
		534	+ gradNormKeyBias_(grad_norm_key_bias),
		535	+ gradNormAddedQueryWeight_(grad_norm_added_query_weight),
		536	+ gradNormAddedQueryBias_(grad_norm_added_query_bias),
		537	+ gradNormAddedKeyWeight_(grad_norm_added_key_weight),
		538	+ gradNormAddedKeyBias_(grad_norm_added_key_bias),
516	workspace_(workspace)	539	workspace_(workspace)
517	{	540	{
518	}	541	}
	@@ -984,7 +1007,7 @@ __aicore__ inline void NormRopeConcatGrad<normType, addedNormType, ropeType, con
984	ropeOp.PreProcess(s);	1007	ropeOp.PreProcess(s);
985	for (uint32_t b = 0; b < batch_; ++b) {	1008	for (uint32_t b = 0; b < batch_; ++b) {
986	uint64_t outGradOffset = static_cast<uint64_t>(b) * headNum_ * totalSeq * headDim_ +	1009	uint64_t outGradOffset = static_cast<uint64_t>(b) * headNum_ * totalSeq * headDim_ +
987	- static_cast<uint64_t>(outputSeq + s) * headDim_;	1010	+ static_cast<uint64_t>(outputSeq + s) * headDim_;
988	uint64_t inOffset = static_cast<uint64_t>(b * xSeq + s) * headNum_ * headDim_;	1011	uint64_t inOffset = static_cast<uint64_t>(b * xSeq + s) * headNum_ * headDim_;
989	uint64_t normOffset = static_cast<uint64_t>(b * xSeq + s) * headNum_;	1012	uint64_t normOffset = static_cast<uint64_t>(b * xSeq + s) * headNum_;
990	for (uint32_t n = 0; n < splitHeadNum_ - 1; ++n) {	1013	for (uint32_t n = 0; n < splitHeadNum_ - 1; ++n) {
	@@ -1072,4 +1095,4 @@ __aicore__ inline void NormRopeConcatGrad<normType, addedNormType, ropeType, con
1072	workspacePushQue_.FreeTensor(workspacePushBuf2);	1095	workspacePushQue_.FreeTensor(workspacePushBuf2);
1073	}	1096	}
1074	} // namespace nrcg	1097	} // namespace nrcg
1075	-#endif // NORM_ROPE_CONCAT_GRAD_H	1098	+#endif // NORM_ROPE_CONCAT_GRAD_H

Rposembedding/rope_with_sin_cos_cache/op_kernel/arch35/rope_with_sin_cos_cache_base.h→posembedding/rope_with_sin_cos_cache/op_kernel/arch35/rope_with_sin_cos_cache_base_arch35.h+4-6

	@@ -9,7 +9,7 @@
9	*/	9	*/
10		10
11	/*!	11	/*!
12	- * \file rope_with_sin_cos_cache_base.h	12	+ * \file rope_with_sin_cos_cache_base_arch35.h
13	* \brief rope_with_sin_cos_cache_base.h	13	* \brief rope_with_sin_cos_cache_base.h
14	*/	14	*/
15	#ifndef ROPE_WITH_SIN_COS_CACHE_ARCH35_BASE_H	15	#ifndef ROPE_WITH_SIN_COS_CACHE_ARCH35_BASE_H
	@@ -23,12 +23,11 @@ using AscendC::Duplicate;
23	using AscendC::HardEvent;	23	using AscendC::HardEvent;
24		24
25	template <typename T>	25	template <typename T>
26	-class RopeWithSinCosCacheBase	26	+class RopeWithSinCosCacheBase {
27	-{
28	public:	27	public:
29	// 构造函数	28	// 构造函数
30	__aicore__ inline RopeWithSinCosCacheBase(){};	29	__aicore__ inline RopeWithSinCosCacheBase(){};
31	- __aicore__ inline void InitData(const RopeWithSinCosCacheTilingData& tilingData);	30	+ __aicore__ inline void InitData(const RopeWithSinCosCacheTilingData &tilingData);
32	__aicore__ inline void SToMTE2Sync();	31	__aicore__ inline void SToMTE2Sync();
33	__aicore__ inline void MTE2ToSSync();	32	__aicore__ inline void MTE2ToSSync();
34	__aicore__ inline void SToMTE3Sync();	33	__aicore__ inline void SToMTE3Sync();
	@@ -72,7 +71,7 @@ protected:
72	};	71	};
73		72
74	template <typename T>	73	template <typename T>
75	-__aicore__ inline void RopeWithSinCosCacheBase<T>::InitData(const RopeWithSinCosCacheTilingData& tilingData)	74	+__aicore__ inline void RopeWithSinCosCacheBase<T>::InitData(const RopeWithSinCosCacheTilingData &tilingData)
76	{	75	{
77	blockIdx_ = AscendC::GetBlockIdx();	76	blockIdx_ = AscendC::GetBlockIdx();
78		77
	@@ -170,4 +169,3 @@ __aicore__ inline void RopeWithSinCosCacheBase<T>::MTE2ToVSync()
170	} // namespace RopeWithSinCosCache	169	} // namespace RopeWithSinCosCache
171		170
172	#endif // namespace RopeWithSinCosCache	171	#endif // namespace RopeWithSinCosCache
173	-

Rposembedding/rope_with_sin_cos_cache/op_kernel/arch35/rope_with_sin_cos_cache_f_bf16.h→posembedding/rope_with_sin_cos_cache/op_kernel/arch35/rope_with_sin_cos_cache_f_bf16_arch35.h+22-22

	@@ -9,26 +9,25 @@
9	*/	9	*/
10		10
11	/*!	11	/*!
12	- * \file rope_with_sin_cos_cache_f_bf16.h	12	+ * \file rope_with_sin_cos_cache_f_bf16_arch35.h
13	* \brief rope_with_sin_cos_cache_f_bf16.h	13	* \brief rope_with_sin_cos_cache_f_bf16.h
14	*/	14	*/
15		15
16	#ifndef ROPE_WITH_SIN_COS_CACHE_ARCH35_BF16_H	16	#ifndef ROPE_WITH_SIN_COS_CACHE_ARCH35_BF16_H
17	#define ROPE_WITH_SIN_COS_CACHE_ARCH35_BF16_H	17	#define ROPE_WITH_SIN_COS_CACHE_ARCH35_BF16_H
18		18
19	-#include "rope_with_sin_cos_cache_base.h"	19	+#include "rope_with_sin_cos_cache_base_arch35.h"
20		20
21	namespace RopeWithSinCosCache {	21	namespace RopeWithSinCosCache {
22	using namespace AscendC;	22	using namespace AscendC;
23		23
24	template <typename T>	24	template <typename T>
25	-class RopeWithSinCosCacheFP16 : public RopeWithSinCosCacheBase<T>	25	+class RopeWithSinCosCacheFP16 : public RopeWithSinCosCacheBase<T> {
26	-{
27	public:	26	public:
28	__aicore__ inline RopeWithSinCosCacheFP16(){};	27	__aicore__ inline RopeWithSinCosCacheFP16(){};
29	__aicore__ inline void Init(	28	__aicore__ inline void Init(
30	GM_ADDR position_id, GM_ADDR query_in, GM_ADDR key_in, GM_ADDR cos_sin_cache, GM_ADDR query_out,	29	GM_ADDR position_id, GM_ADDR query_in, GM_ADDR key_in, GM_ADDR cos_sin_cache, GM_ADDR query_out,
31	- GM_ADDR key_out, const RopeWithSinCosCacheTilingData& tiling_data, TPipe* pipe);	30	+ GM_ADDR key_out, const RopeWithSinCosCacheTilingData &tiling_data, TPipe *pipe);
32	__aicore__ inline void Process();	31	__aicore__ inline void Process();
33	__aicore__ inline void Compute(uint64_t index, uint64_t loopN);	32	__aicore__ inline void Compute(uint64_t index, uint64_t loopN);
34	__aicore__ inline void ComputeAlongHeads(uint64_t indexToken, uint64_t indexHeads);	33	__aicore__ inline void ComputeAlongHeads(uint64_t indexToken, uint64_t indexHeads);
	@@ -72,7 +71,7 @@ protected:
72	template <typename T>	71	template <typename T>
73	__aicore__ inline void RopeWithSinCosCacheFP16<T>::Init(	72	__aicore__ inline void RopeWithSinCosCacheFP16<T>::Init(
74	GM_ADDR position_id, GM_ADDR query_in, GM_ADDR key_in, GM_ADDR cos_sin_cache, GM_ADDR query_out, GM_ADDR key_out,	73	GM_ADDR position_id, GM_ADDR query_in, GM_ADDR key_in, GM_ADDR cos_sin_cache, GM_ADDR query_out, GM_ADDR key_out,
75	- const RopeWithSinCosCacheTilingData& tiling_data, TPipe* pipe)	74	+ const RopeWithSinCosCacheTilingData &tiling_data, TPipe *pipe)
76	{	75	{
77	this->InitData(tiling_data);	76	this->InitData(tiling_data);
78	headBlockLen = static_cast<uint16_t>(this->head_size / ELE_NUM_FP32);	77	headBlockLen = static_cast<uint16_t>(this->head_size / ELE_NUM_FP32);
	@@ -87,13 +86,13 @@ __aicore__ inline void RopeWithSinCosCacheFP16<T>::Init(
87	(this->blockIdx_ - this->front_core) * this->num_tokens_each_tail_core;	86	(this->blockIdx_ - this->front_core) * this->num_tokens_each_tail_core;
88	}	87	}
89		88
90	- position_id_GM.SetGlobalBuffer((__gm__ uint64_t*)position_id + blockOffset);	89	+ position_id_GM.SetGlobalBuffer((__gm__ uint64_t *)position_id + blockOffset);
91	- query_in_GM.SetGlobalBuffer((__gm__ T)query_in + blockOffset this->q_leading_dimension);	90	+ query_in_GM.SetGlobalBuffer((__gm__ T )query_in + blockOffset this->q_leading_dimension);
92	- key_in_GM.SetGlobalBuffer((__gm__ T)key_in + blockOffset this->k_leading_dimension);	91	+ key_in_GM.SetGlobalBuffer((__gm__ T )key_in + blockOffset this->k_leading_dimension);
93	- cos_sin_cache_GM.SetGlobalBuffer((__gm__ T*)cos_sin_cache);	92	+ cos_sin_cache_GM.SetGlobalBuffer((__gm__ T *)cos_sin_cache);
94		93
95	- queryGM.SetGlobalBuffer((__gm__ T)query_out + blockOffset this->num_q_heads * this->head_size);	94	+ queryGM.SetGlobalBuffer((__gm__ T )query_out + blockOffset this->num_q_heads * this->head_size);
96	- keyGM.SetGlobalBuffer((__gm__ T)key_out + blockOffset this->num_kv_heads * this->head_size);	95	+ keyGM.SetGlobalBuffer((__gm__ T )key_out + blockOffset this->num_kv_heads * this->head_size);
97		96
98	num_heads_max = (this->num_q_heads > this->num_kv_heads) ? this->num_q_heads : this->num_kv_heads;	97	num_heads_max = (this->num_q_heads > this->num_kv_heads) ? this->num_q_heads : this->num_kv_heads;
99	if (this->loop_for_one_token == 0) {	98	if (this->loop_for_one_token == 0) {
	@@ -440,12 +439,13 @@ __aicore__ inline void RopeWithSinCosCacheFP16<T>::Compute(uint64_t index, uint6
440	}	439	}
441		440
442	template <typename T>	441	template <typename T>
443	-__aicore__ inline void RopeWithSinCosCacheFP16<T>::ComputeAlongHeads(uint64_t indexToken, uint64_t indexHeads){	442	+__aicore__ inline void RopeWithSinCosCacheFP16<T>::ComputeAlongHeads(uint64_t indexToken, uint64_t indexHeads)
		443	+{
444	q_size = this->num_q_heads * this->head_size;	444	q_size = this->num_q_heads * this->head_size;
445	k_size = this->num_kv_heads * this->head_size;	445	k_size = this->num_kv_heads * this->head_size;
446	uint64_t offsetQHead = indexHeads * this->num_qheads_each_loop * this->head_size;	446	uint64_t offsetQHead = indexHeads * this->num_qheads_each_loop * this->head_size;
447	uint64_t offsetKHead = indexHeads * this->num_kheads_each_loop * this->head_size;	447	uint64_t offsetKHead = indexHeads * this->num_kheads_each_loop * this->head_size;
448	- uint64_t offset = indexToken * q_size + offsetQHead;//非连续?	448	+ uint64_t offset = indexToken * q_size + offsetQHead; // 非连续?
449	uint64_t offsetk = indexToken * k_size + offsetKHead;	449	uint64_t offsetk = indexToken * k_size + offsetKHead;
450	uint64_t query_in_offset = indexToken * this->q_leading_dimension + offsetQHead;	450	uint64_t query_in_offset = indexToken * this->q_leading_dimension + offsetQHead;
451	uint64_t key_in_offset = indexToken * this->k_leading_dimension + offsetKHead;	451	uint64_t key_in_offset = indexToken * this->k_leading_dimension + offsetKHead;
	@@ -560,23 +560,23 @@ __aicore__ inline void RopeWithSinCosCacheFP16<T>::ComputeAlongHeads(uint64_t in
560		560
561	if (this->head_size != this->rotary_dim) {	561	if (this->head_size != this->rotary_dim) {
562	DataCopy(outQueAfterCastLocal, inQueCalLocal,	562	DataCopy(outQueAfterCastLocal, inQueCalLocal,
563	- {static_cast<uint16_t>(loopNQhead), static_cast<uint16_t>(rotaryBlockLen/2), 0,	563	+ {static_cast<uint16_t>(loopNQhead), static_cast<uint16_t>(rotaryBlockLen / 2), 0,
564	- static_cast<uint16_t>(headBlockLen/2 - rotaryBlockLen/2)});	564	+ static_cast<uint16_t>(headBlockLen / 2 - rotaryBlockLen / 2)});
565	PipeBarrier<PIPE_ALL>();	565	PipeBarrier<PIPE_ALL>();
566	DataCopy(outQueAfterCastLocal[this->rotary_dim], inQQueBeforeCastLocal[this->rotary_dim],	566	DataCopy(outQueAfterCastLocal[this->rotary_dim], inQQueBeforeCastLocal[this->rotary_dim],
567	- {static_cast<uint16_t>(loopNQhead), static_cast<uint16_t>(headBlockLen/2 - rotaryBlockLen/2),	567	+ {static_cast<uint16_t>(loopNQhead), static_cast<uint16_t>(headBlockLen / 2 - rotaryBlockLen / 2),
568	- static_cast<uint16_t>(rotaryBlockLen/2), static_cast<uint16_t>(rotaryBlockLen/2)});	568	+ static_cast<uint16_t>(rotaryBlockLen / 2), static_cast<uint16_t>(rotaryBlockLen / 2)});
569	PipeBarrier<PIPE_ALL>();	569	PipeBarrier<PIPE_ALL>();
570	} else {	570	} else {
571	DataCopy(outQueAfterCastLocal, inQueCalLocal,	571	DataCopy(outQueAfterCastLocal, inQueCalLocal,
572	- {static_cast<uint16_t>(loopNQhead), static_cast<uint16_t>(headBlockLen/2), 0, 0});	572	+ {static_cast<uint16_t>(loopNQhead), static_cast<uint16_t>(headBlockLen / 2), 0, 0});
573	PipeBarrier<PIPE_ALL>();	573	PipeBarrier<PIPE_ALL>();
574	}	574	}
575	DataCopy(queryGM[offset], outQueAfterCastLocal,	575	DataCopy(queryGM[offset], outQueAfterCastLocal,
576	- {static_cast<uint16_t>(loopNQhead), static_cast<uint16_t>(headBlockLen/2), 0, 0});	576	+ {static_cast<uint16_t>(loopNQhead), static_cast<uint16_t>(headBlockLen / 2), 0, 0});
577	PipeBarrier<PIPE_ALL>();	577	PipeBarrier<PIPE_ALL>();
578	}	578	}
579	- //key	579	+ // key
580	if (indexHeads < this->loop_along_kheads) {	580	if (indexHeads < this->loop_along_kheads) {
581	uint64_t loopNKhead = (indexHeads == this->loop_along_kheads - 1 && this->num_kheads_last_loop != 0) ?	581	uint64_t loopNKhead = (indexHeads == this->loop_along_kheads - 1 && this->num_kheads_last_loop != 0) ?
582	this->num_kheads_last_loop :	582	this->num_kheads_last_loop :
	@@ -696,4 +696,4 @@ __aicore__ inline void RopeWithSinCosCacheFP16<T>::Process()
696	}	696	}
697	} // namespace RopeWithSinCosCache	697	} // namespace RopeWithSinCosCache
698		698
699	-#endif	699	+#endif

Rposembedding/rope_with_sin_cos_cache/op_kernel/arch35/rope_with_sin_cos_cache_fp32.h→posembedding/rope_with_sin_cos_cache/op_kernel/arch35/rope_with_sin_cos_cache_fp32_arch35.h+30-31

	@@ -9,33 +9,32 @@
9	*/	9	*/
10		10
11	/*!	11	/*!
12	- * \file rope_with_sin_cos_cache_fp32.h	12	+ * \file rope_with_sin_cos_cache_fp32_arch35.h
13	* \brief rope_with_sin_cos_cache_fp32.h	13	* \brief rope_with_sin_cos_cache_fp32.h
14	*/	14	*/
15		15
16	#ifndef ROPE_WITH_SIN_COS_CACHE_ARCH35_FP32_H	16	#ifndef ROPE_WITH_SIN_COS_CACHE_ARCH35_FP32_H
17	#define ROPE_WITH_SIN_COS_CACHE_ARCH35_FP32_H	17	#define ROPE_WITH_SIN_COS_CACHE_ARCH35_FP32_H
18		18
19	-#include "rope_with_sin_cos_cache_base.h"	19	+#include "rope_with_sin_cos_cache_base_arch35.h"
20		20
21	namespace RopeWithSinCosCache {	21	namespace RopeWithSinCosCache {
22	using namespace AscendC;	22	using namespace AscendC;
23		23
24	template <typename T>	24	template <typename T>
25	-class RopeWithSinCosCacheF32 : public RopeWithSinCosCacheBase<T>	25	+class RopeWithSinCosCacheF32 : public RopeWithSinCosCacheBase<T> {
26	-{
27	public:	26	public:
28	__aicore__ inline RopeWithSinCosCacheF32(){};	27	__aicore__ inline RopeWithSinCosCacheF32(){};
29	__aicore__ inline void Init(	28	__aicore__ inline void Init(
30	GM_ADDR position_id, GM_ADDR query_in, GM_ADDR key_in, GM_ADDR cos_sin_cache, GM_ADDR query_out,	29	GM_ADDR position_id, GM_ADDR query_in, GM_ADDR key_in, GM_ADDR cos_sin_cache, GM_ADDR query_out,
31	- GM_ADDR key_out, const RopeWithSinCosCacheTilingData& tiling_data, TPipe* pipe);	30	+ GM_ADDR key_out, const RopeWithSinCosCacheTilingData &tiling_data, TPipe *pipe);
32	__aicore__ inline void Process();	31	__aicore__ inline void Process();
33	__aicore__ inline void Compute(uint64_t index, uint64_t loopN);	32	__aicore__ inline void Compute(uint64_t index, uint64_t loopN);
34	__aicore__ inline void ComputeAlongHeads(uint64_t indexToken, uint64_t indexHeads);	33	__aicore__ inline void ComputeAlongHeads(uint64_t indexToken, uint64_t indexHeads);
35	__aicore__ inline void GetCosSinCache(	34	__aicore__ inline void GetCosSinCache(
36	- LocalTensor<T> copyBuf0Local, LocalTensor<float> inCosSin,	35	+ LocalTensor<T> copyBuf0Local, LocalTensor<float> inCosSin,
37	- LocalTensor<float> CosSin, uint64_t offsetPos, uint64_t cosSinOffset, uint64_t localStartAddr,	36	+ LocalTensor<float> CosSin, uint64_t offsetPos, uint64_t cosSinOffset, uint64_t localStartAddr,
38	- uint32_t (&rcShape_)[2], uint32_t (&dstShape_)[2]);	37	+ uint32_t (&rcShape_)[2], uint32_t (&dstShape_)[2]);
39		38
40	protected:	39	protected:
41	static constexpr uint64_t BLOCK_SIZE = 32;	40	static constexpr uint64_t BLOCK_SIZE = 32;
	@@ -68,7 +67,7 @@ protected:
68	template <typename T>	67	template <typename T>
69	__aicore__ inline void RopeWithSinCosCacheF32<T>::Init(	68	__aicore__ inline void RopeWithSinCosCacheF32<T>::Init(
70	GM_ADDR position_id, GM_ADDR query_in, GM_ADDR key_in, GM_ADDR cos_sin_cache, GM_ADDR query_out, GM_ADDR key_out,	69	GM_ADDR position_id, GM_ADDR query_in, GM_ADDR key_in, GM_ADDR cos_sin_cache, GM_ADDR query_out, GM_ADDR key_out,
71	- const RopeWithSinCosCacheTilingData& tiling_data, TPipe* pipe)	70	+ const RopeWithSinCosCacheTilingData &tiling_data, TPipe *pipe)
72	{	71	{
73	this->InitData(tiling_data);	72	this->InitData(tiling_data);
74	headBlockLen = static_cast<uint16_t>(this->head_size / ELE_NUM_FP32);	73	headBlockLen = static_cast<uint16_t>(this->head_size / ELE_NUM_FP32);
	@@ -83,16 +82,16 @@ __aicore__ inline void RopeWithSinCosCacheF32<T>::Init(
83	(this->blockIdx_ - this->front_core) * this->num_tokens_each_tail_core;	82	(this->blockIdx_ - this->front_core) * this->num_tokens_each_tail_core;
84	}	83	}
85		84
86	- position_id_GM.SetGlobalBuffer((__gm__ uint64_t*)position_id + blockOffset);	85	+ position_id_GM.SetGlobalBuffer((__gm__ uint64_t *)position_id + blockOffset);
87	- query_in_GM.SetGlobalBuffer((__gm__ T)query_in + blockOffset this->num_q_heads * this->head_size);	86	+ query_in_GM.SetGlobalBuffer((__gm__ T )query_in + blockOffset this->num_q_heads * this->head_size);
88	- key_in_GM.SetGlobalBuffer((__gm__ T)key_in + blockOffset this->num_kv_heads * this->head_size);	87	+ key_in_GM.SetGlobalBuffer((__gm__ T )key_in + blockOffset this->num_kv_heads * this->head_size);
89	- cos_sin_cache_GM.SetGlobalBuffer((__gm__ T*)cos_sin_cache);	88	+ cos_sin_cache_GM.SetGlobalBuffer((__gm__ T *)cos_sin_cache);
90		89
91	- queryGM.SetGlobalBuffer((__gm__ T)query_out + blockOffset this->num_q_heads * this->head_size);	90	+ queryGM.SetGlobalBuffer((__gm__ T )query_out + blockOffset this->num_q_heads * this->head_size);
92	- keyGM.SetGlobalBuffer((__gm__ T)key_out + blockOffset this->num_kv_heads * this->head_size);	91	+ keyGM.SetGlobalBuffer((__gm__ T )key_out + blockOffset this->num_kv_heads * this->head_size);
93		92
94	- pipe->InitBuffer(copyBuf0, this->rotary_dim / 2 * sizeof(T));	93	+ pipe->InitBuffer(copyBuf0, this->rotary_dim / 2 * sizeof(T));
95	- if(this->loop_for_one_token==0){	94	+ if (this->loop_for_one_token == 0) {
96	pipe->InitBuffer(	95	pipe->InitBuffer(
97	inQQue, BUFFER_NUM, this->num_tokens_each_loop_current_core * num_heads_max * this->head_size * sizeof(T));	96	inQQue, BUFFER_NUM, this->num_tokens_each_loop_current_core * num_heads_max * this->head_size * sizeof(T));
98	pipe->InitBuffer(	97	pipe->InitBuffer(
	@@ -105,7 +104,7 @@ __aicore__ inline void RopeWithSinCosCacheF32<T>::Init(
105	pipe->InitBuffer(cosSinBuf, this->num_tokens_each_loop_current_core * num_heads_max * this->rotary_dim * sizeof(T));	104	pipe->InitBuffer(cosSinBuf, this->num_tokens_each_loop_current_core * num_heads_max * this->rotary_dim * sizeof(T));
106	pipe->InitBuffer(	105	pipe->InitBuffer(
107	inQueCalBuf, this->num_tokens_each_loop_current_core * num_heads_max * this->rotary_dim * sizeof(T));	106	inQueCalBuf, this->num_tokens_each_loop_current_core * num_heads_max * this->rotary_dim * sizeof(T));
108	-	107	+
109	if (this->is_neox_style == 0) {	108	if (this->is_neox_style == 0) {
110	pipe->InitBuffer(temp1, this->num_tokens_each_loop_current_core * num_heads_max * this->rotary_dim * sizeof(T));	109	pipe->InitBuffer(temp1, this->num_tokens_each_loop_current_core * num_heads_max * this->rotary_dim * sizeof(T));
111	pipe->InitBuffer(offsetBuf, this->rotary_dim * sizeof(uint32_t));	110	pipe->InitBuffer(offsetBuf, this->rotary_dim * sizeof(uint32_t));
	@@ -159,7 +158,7 @@ RopeWithSinCosCacheF32<T>::GetCosSinCache(LocalTensor<T> copyBuf0Local, LocalTen
159	static_cast<uint16_t>(this->mrope_section0 + this->mrope_section1) + cosSinOffset],	158	static_cast<uint16_t>(this->mrope_section0 + this->mrope_section1) + cosSinOffset],
160	{1, static_cast<uint16_t>(this->mrope_section2 * sizeof(T)), 0, 0}, {true, padding2, 0, 0});	159	{1, static_cast<uint16_t>(this->mrope_section2 * sizeof(T)), 0, 0}, {true, padding2, 0, 0});
161	PipeBarrier<PIPE_ALL>();	160	PipeBarrier<PIPE_ALL>();
162	-	161	+
163	// 如果mrope_section不是32B对齐，DataCopyPad接口会自动填充，同时为了兼容旧版16 24 24做法，避免该场景性能劣化	162	// 如果mrope_section不是32B对齐，DataCopyPad接口会自动填充，同时为了兼容旧版16 24 24做法，避免该场景性能劣化
164	uint8_t padding1 = (this->mrope_section0 * sizeof(T) % BLOCK_SIZE) / sizeof(T);	163	uint8_t padding1 = (this->mrope_section0 * sizeof(T) % BLOCK_SIZE) / sizeof(T);
165	if (this->mrope_section1 % MROPE_SECTION_BASE_SIZE == 0) {	164	if (this->mrope_section1 % MROPE_SECTION_BASE_SIZE == 0) {
	@@ -319,7 +318,7 @@ __aicore__ inline void RopeWithSinCosCacheF32<T>::Compute(uint64_t index, uint64
319	} else {	318	} else {
320	DataCopy(	319	DataCopy(
321	outLocal, inQueCalLocal,	320	outLocal, inQueCalLocal,
322	- {static_cast<uint16_t>(loopN), static_cast<uint16_t>(this->num_q_heads * headBlockLen), 0, 0});	321	+ {static_cast<uint16_t>(loopN), static_cast<uint16_t>(this->num_q_heads * headBlockLen), 0, 0});
323	}	322	}
324	PipeBarrier<PIPE_ALL>();	323	PipeBarrier<PIPE_ALL>();
325	DataCopy(	324	DataCopy(
	@@ -423,12 +422,13 @@ __aicore__ inline void RopeWithSinCosCacheF32<T>::Compute(uint64_t index, uint64
423	}	422	}
424		423
425	template <typename T>	424	template <typename T>
426	-__aicore__ inline void RopeWithSinCosCacheF32<T>::ComputeAlongHeads(uint64_t indexToken, uint64_t indexHeads){	425	+__aicore__ inline void RopeWithSinCosCacheF32<T>::ComputeAlongHeads(uint64_t indexToken, uint64_t indexHeads)
		426	+{
427	q_size = this->num_q_heads * this->head_size;	427	q_size = this->num_q_heads * this->head_size;
428	k_size = this->num_kv_heads * this->head_size;	428	k_size = this->num_kv_heads * this->head_size;
429	uint64_t offsetQHead = indexHeads * this->num_qheads_each_loop * this->head_size;	429	uint64_t offsetQHead = indexHeads * this->num_qheads_each_loop * this->head_size;
430	uint64_t offsetKHead = indexHeads * this->num_kheads_each_loop * this->head_size;	430	uint64_t offsetKHead = indexHeads * this->num_kheads_each_loop * this->head_size;
431	- uint64_t offset = indexToken * q_size + offsetQHead;//非连续?	431	+ uint64_t offset = indexToken * q_size + offsetQHead; // 非连续?
432	uint64_t offsetk = indexToken * k_size + offsetKHead;	432	uint64_t offsetk = indexToken * k_size + offsetKHead;
433		433
434	uint32_t dstShape_[2] = {static_cast<uint32_t>(this->num_heads_each_loop), static_cast<uint32_t>(this->rotary_dim)};	434	uint32_t dstShape_[2] = {static_cast<uint32_t>(this->num_heads_each_loop), static_cast<uint32_t>(this->rotary_dim)};
	@@ -546,7 +546,7 @@ __aicore__ inline void RopeWithSinCosCacheF32<T>::ComputeAlongHeads(uint64_t ind
546	{static_cast<uint16_t>(loopNQhead), static_cast<uint16_t>(headBlockLen), 0, 0});	546	{static_cast<uint16_t>(loopNQhead), static_cast<uint16_t>(headBlockLen), 0, 0});
547	PipeBarrier<PIPE_ALL>();	547	PipeBarrier<PIPE_ALL>();
548	}	548	}
549	- //key	549	+ // key
550	if (indexHeads < this->loop_along_kheads) {	550	if (indexHeads < this->loop_along_kheads) {
551	uint64_t loopNKhead = (indexHeads == this->loop_along_kheads - 1 && this->num_kheads_last_loop != 0) ?	551	uint64_t loopNKhead = (indexHeads == this->loop_along_kheads - 1 && this->num_kheads_last_loop != 0) ?
552	this->num_kheads_last_loop :	552	this->num_kheads_last_loop :
	@@ -631,7 +631,7 @@ __aicore__ inline void RopeWithSinCosCacheF32<T>::ComputeAlongHeads(uint64_t ind
631	{static_cast<uint16_t>(loopNKhead), static_cast<uint16_t>(headBlockLen), 0, 0});	631	{static_cast<uint16_t>(loopNKhead), static_cast<uint16_t>(headBlockLen), 0, 0});
632	PipeBarrier<PIPE_ALL>();	632	PipeBarrier<PIPE_ALL>();
633	}	633	}
634	-	634	+
635	inQueueCosSinCache.FreeTensor(inCosSin);	635	inQueueCosSinCache.FreeTensor(inCosSin);
636	inQQue.FreeTensor(inLocal);	636	inQQue.FreeTensor(inLocal);
637	outQue.FreeTensor(outLocal);	637	outQue.FreeTensor(outLocal);
	@@ -640,7 +640,7 @@ __aicore__ inline void RopeWithSinCosCacheF32<T>::ComputeAlongHeads(uint64_t ind
640	template <typename T>	640	template <typename T>
641	__aicore__ inline void RopeWithSinCosCacheF32<T>::Process()	641	__aicore__ inline void RopeWithSinCosCacheF32<T>::Process()
642	{	642	{
643	- if(this->loop_for_one_token==0){	643	+ if (this->loop_for_one_token == 0) {
644	for (uint64_t n = 0; n < this->loop_time_current_core - 1; n++) {	644	for (uint64_t n = 0; n < this->loop_time_current_core - 1; n++) {
645	Compute(n, this->num_tokens_each_loop_current_core);	645	Compute(n, this->num_tokens_each_loop_current_core);
646	}	646	}
	@@ -650,17 +650,16 @@ __aicore__ inline void RopeWithSinCosCacheF32<T>::Process()
650	} else {	650	} else {
651	Compute(this->loop_time_current_core - 1, this->num_tokens_last_loop_current_core);	651	Compute(this->loop_time_current_core - 1, this->num_tokens_last_loop_current_core);
652	}	652	}
653	- }	653	+ } else {
654	- else{
655	uint64_t loop_along_heads =	654	uint64_t loop_along_heads =
656	this->loop_along_qheads > this->loop_along_kheads ? this->loop_along_qheads : this->loop_along_kheads;	655	this->loop_along_qheads > this->loop_along_kheads ? this->loop_along_qheads : this->loop_along_kheads;
657	- for(uint64_t n=0;n<this->num_tokens_current_core;n++){	656	+ for (uint64_t n = 0; n < this->num_tokens_current_core; n++) {
658	- for(uint64_t m=0;m<loop_along_heads;m++){	657	+ for (uint64_t m = 0; m < loop_along_heads; m++) {
659	- ComputeAlongHeads(n,m);	658	+ ComputeAlongHeads(n, m);
660	}	659	}
661	}	660	}
662	}	661	}
663	}	662	}
664	} // namespace RopeWithSinCosCache	663	} // namespace RopeWithSinCosCache
665		664
666	-#endif	665	+#endif

Mposembedding/rope_with_sin_cos_cache/op_kernel/rope_with_sin_cos_cache_apt.cpp+5-6

	@@ -13,8 +13,8 @@
13	* \brief 950 (A5) 平台入口文件，复用 membase 实现逻辑	13	* \brief 950 (A5) 平台入口文件，复用 membase 实现逻辑
14	*/	14	*/
15	#include "kernel_operator.h"	15	#include "kernel_operator.h"
16	-#include "arch35/rope_with_sin_cos_cache_fp32.h"	16	+#include "arch35/rope_with_sin_cos_cache_fp32_arch35.h"
17	-#include "arch35/rope_with_sin_cos_cache_f_bf16.h"	17	+#include "arch35/rope_with_sin_cos_cache_f_bf16_arch35.h"
18		18
19	using namespace AscendC;	19	using namespace AscendC;
20	using namespace RopeWithSinCosCache;	20	using namespace RopeWithSinCosCache;
	@@ -28,7 +28,7 @@ extern "C" __global__ __aicore__ void rope_with_sin_cos_cache(
28	TPipe pipe;	28	TPipe pipe;
29	#if ORIG_DTYPE_QUERYIN == DT_BF16	29	#if ORIG_DTYPE_QUERYIN == DT_BF16
30	if (TILING_KEY_IS(20)) {	30	if (TILING_KEY_IS(20)) {
31	- TPipe* ptr = &pipe;	31	+ TPipe *ptr = &pipe;
32	if (ptr != nullptr) {	32	if (ptr != nullptr) {
33	RopeWithSinCosCacheFP16<bfloat16_t> op;	33	RopeWithSinCosCacheFP16<bfloat16_t> op;
34	op.Init(position_id, query_in, key_in, cos_sin_cache, query_out, key_out, tilingData, ptr);	34	op.Init(position_id, query_in, key_in, cos_sin_cache, query_out, key_out, tilingData, ptr);
	@@ -37,7 +37,7 @@ extern "C" __global__ __aicore__ void rope_with_sin_cos_cache(
37	}	37	}
38	#elif ORIG_DTYPE_QUERYIN == DT_FLOAT16	38	#elif ORIG_DTYPE_QUERYIN == DT_FLOAT16
39	if (TILING_KEY_IS(21)) {	39	if (TILING_KEY_IS(21)) {
40	- TPipe* ptr = &pipe;	40	+ TPipe *ptr = &pipe;
41	if (ptr != nullptr) {	41	if (ptr != nullptr) {
42	RopeWithSinCosCacheFP16<half> op;	42	RopeWithSinCosCacheFP16<half> op;
43	op.Init(position_id, query_in, key_in, cos_sin_cache, query_out, key_out, tilingData, ptr);	43	op.Init(position_id, query_in, key_in, cos_sin_cache, query_out, key_out, tilingData, ptr);
	@@ -46,7 +46,7 @@ extern "C" __global__ __aicore__ void rope_with_sin_cos_cache(
46	}	46	}
47	#elif ORIG_DTYPE_QUERYIN == DT_FLOAT	47	#elif ORIG_DTYPE_QUERYIN == DT_FLOAT
48	if (TILING_KEY_IS(22)) {	48	if (TILING_KEY_IS(22)) {
49	- TPipe* ptr = &pipe;	49	+ TPipe *ptr = &pipe;
50	if (ptr != nullptr) {	50	if (ptr != nullptr) {
51	RopeWithSinCosCacheF32<float> op;	51	RopeWithSinCosCacheF32<float> op;
52	op.Init(position_id, query_in, key_in, cos_sin_cache, query_out, key_out, tilingData, ptr);	52	op.Init(position_id, query_in, key_in, cos_sin_cache, query_out, key_out, tilingData, ptr);
	@@ -55,4 +55,3 @@ extern "C" __global__ __aicore__ void rope_with_sin_cos_cache(
55	}	55	}
56	#endif	56	#endif
57	}	57	}
58	-

	@@ -13,28 +13,27 @@
13	* \brief	13	* \brief
14	*/	14	*/
15	#include "kernel_operator.h"	15	#include "kernel_operator.h"
16	-#include "norm_rope_concat_grad.h"	16	+#include "norm_rope_concat_grad_kernel.h"
17		17
18	using namespace AscendC;	18	using namespace AscendC;
19	using namespace nrcg;	19	using namespace nrcg;
20		20
21	-#define Impl(tilingKey, normType, addedNormType, ropeType, concatOrder) \	21	+#define Impl(tilingKey, normType, addedNormType, ropeType, concatOrder) \
22	- if (TILING_KEY_IS(tilingKey)) { \	22	+ if (TILING_KEY_IS(tilingKey)) { \
23	- NormRopeConcatGrad<normType, addedNormType, ropeType, concatOrder> op( \	23	+ NormRopeConcatGrad<normType, addedNormType, ropeType, concatOrder> op( \
24	- gm_grad_query_output, gm_grad_key_output, gm_grad_value_output, gm_query, gm_key, gm_encoder_query, \	24	+ gm_grad_query_output, gm_grad_key_output, gm_grad_value_output, gm_query, gm_key, gm_encoder_query, \
25	- gm_encoder_key, gm_norm_query_weight, gm_norm_query_mean, gm_norm_query_rstd, gm_norm_key_weight, \	25	+ gm_encoder_key, gm_norm_query_weight, gm_norm_query_mean, gm_norm_query_rstd, gm_norm_key_weight, \
26	- gm_norm_key_mean, gm_norm_key_rstd, gm_norm_added_query_weight, gm_norm_added_query_mean, \	26	+ gm_norm_key_mean, gm_norm_key_rstd, gm_norm_added_query_weight, gm_norm_added_query_mean, \
27	- gm_norm_added_query_rstd, gm_norm_added_key_weight, gm_norm_added_key_mean, gm_norm_added_key_rstd, \	27	+ gm_norm_added_query_rstd, gm_norm_added_key_weight, gm_norm_added_key_mean, gm_norm_added_key_rstd, \
28	- gm_rope_sin, gm_rope_cos, gm_grad_query, gm_grad_key, gm_grad_value, gm_grad_encoder_query, \	28	+ gm_rope_sin, gm_rope_cos, gm_grad_query, gm_grad_key, gm_grad_value, gm_grad_encoder_query, \
29	- gm_grad_encoder_key, gm_grad_encoder_value, gm_grad_norm_query_weight, gm_grad_norm_query_bias, \	29	+ gm_grad_encoder_key, gm_grad_encoder_value, gm_grad_norm_query_weight, gm_grad_norm_query_bias, \
30	- gm_grad_norm_key_weight, gm_grad_norm_key_bias, gm_grad_norm_added_query_weight, \	30	+ gm_grad_norm_key_weight, gm_grad_norm_key_bias, gm_grad_norm_added_query_weight, \
31	- gm_grad_norm_added_query_bias, gm_grad_norm_added_key_weight, gm_grad_norm_added_key_bias, workspaceGM); \	31	+ gm_grad_norm_added_query_bias, gm_grad_norm_added_key_weight, gm_grad_norm_added_key_bias, workspaceGM); \
32	- op.Init(&pipe, tilingData); \	32	+ op.Init(&pipe, tilingData); \
33	- op.Process(); \	33	+ op.Process(); \
34	- return; \	34	+ return; \
35	}	35	}
36		36
37	-
38	extern "C" __global__ __aicore__ void norm_rope_concat_grad(	37	extern "C" __global__ __aicore__ void norm_rope_concat_grad(
39	GM_ADDR gm_grad_query_output, GM_ADDR gm_grad_key_output, GM_ADDR gm_grad_value_output, GM_ADDR gm_query,	38	GM_ADDR gm_grad_query_output, GM_ADDR gm_grad_key_output, GM_ADDR gm_grad_value_output, GM_ADDR gm_query,
40	GM_ADDR gm_key, GM_ADDR gm_encoder_query, GM_ADDR gm_encoder_key, GM_ADDR gm_norm_query_weight,	39	GM_ADDR gm_key, GM_ADDR gm_encoder_query, GM_ADDR gm_encoder_key, GM_ADDR gm_norm_query_weight,
	@@ -50,61 +49,61 @@ extern "C" __global__ __aicore__ void norm_rope_concat_grad(
50	{	49	{
51	GET_TILING_DATA(tilingData, tiling);	50	GET_TILING_DATA(tilingData, tiling);
52	AscendC::TPipe pipe;	51	AscendC::TPipe pipe;
53	- Impl(0, NormType::NONE,NormType::NONE, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)	52	+ Impl(0, NormType::NONE, NormType::NONE, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)
54	- Impl(10, NormType::NONE,NormType::NONE, RopeType::NONE, ConcatOrder::AFTER_ENCODER)	53	+ Impl(10, NormType::NONE, NormType::NONE, RopeType::NONE, ConcatOrder::AFTER_ENCODER)
55	- Impl(100, NormType::NONE,NormType::NONE, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)	54	+ Impl(100, NormType::NONE, NormType::NONE, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)
56	- Impl(110, NormType::NONE,NormType::NONE, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)	55	+ Impl(110, NormType::NONE, NormType::NONE, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)
57	- Impl(200, NormType::NONE,NormType::NONE, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)	56	+ Impl(200, NormType::NONE, NormType::NONE, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)
58	- Impl(210, NormType::NONE,NormType::NONE, RopeType::HALF, ConcatOrder::AFTER_ENCODER)	57	+ Impl(210, NormType::NONE, NormType::NONE, RopeType::HALF, ConcatOrder::AFTER_ENCODER)
59		58
60	- Impl(100000, NormType::NONE,NormType::LAYER_NORM, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)	59	+ Impl(100000, NormType::NONE, NormType::LAYER_NORM, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)
61	- Impl(100010, NormType::NONE,NormType::LAYER_NORM, RopeType::NONE, ConcatOrder::AFTER_ENCODER)	60	+ Impl(100010, NormType::NONE, NormType::LAYER_NORM, RopeType::NONE, ConcatOrder::AFTER_ENCODER)
62	- Impl(100100, NormType::NONE,NormType::LAYER_NORM, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)	61	+ Impl(100100, NormType::NONE, NormType::LAYER_NORM, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)
63	- Impl(100110, NormType::NONE,NormType::LAYER_NORM, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)	62	+ Impl(100110, NormType::NONE, NormType::LAYER_NORM, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)
64	- Impl(100200, NormType::NONE,NormType::LAYER_NORM, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)	63	+ Impl(100200, NormType::NONE, NormType::LAYER_NORM, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)
65	- Impl(100210, NormType::NONE,NormType::LAYER_NORM, RopeType::HALF, ConcatOrder::AFTER_ENCODER)	64	+ Impl(100210, NormType::NONE, NormType::LAYER_NORM, RopeType::HALF, ConcatOrder::AFTER_ENCODER)
66	- Impl(200000, NormType::NONE,NormType::LAYER_NORM_AFFINE, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)	65	+ Impl(200000, NormType::NONE, NormType::LAYER_NORM_AFFINE, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)
67	- Impl(200010, NormType::NONE,NormType::LAYER_NORM_AFFINE, RopeType::NONE, ConcatOrder::AFTER_ENCODER)	66	+ Impl(200010, NormType::NONE, NormType::LAYER_NORM_AFFINE, RopeType::NONE, ConcatOrder::AFTER_ENCODER)
68	- Impl(200100, NormType::NONE,NormType::LAYER_NORM_AFFINE, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)	67	+ Impl(200100, NormType::NONE, NormType::LAYER_NORM_AFFINE, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)
69	- Impl(200110, NormType::NONE,NormType::LAYER_NORM_AFFINE, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)	68	+ Impl(200110, NormType::NONE, NormType::LAYER_NORM_AFFINE, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)
70	- Impl(200200, NormType::NONE,NormType::LAYER_NORM_AFFINE, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)	69	+ Impl(200200, NormType::NONE, NormType::LAYER_NORM_AFFINE, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)
71	- Impl(200210, NormType::NONE,NormType::LAYER_NORM_AFFINE, RopeType::HALF, ConcatOrder::AFTER_ENCODER)	70	+ Impl(200210, NormType::NONE, NormType::LAYER_NORM_AFFINE, RopeType::HALF, ConcatOrder::AFTER_ENCODER)
72		71
73	- Impl(100000000, NormType::LAYER_NORM,NormType::NONE, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)	72	+ Impl(100000000, NormType::LAYER_NORM, NormType::NONE, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)
74	- Impl(100000010, NormType::LAYER_NORM,NormType::NONE, RopeType::NONE, ConcatOrder::AFTER_ENCODER)	73	+ Impl(100000010, NormType::LAYER_NORM, NormType::NONE, RopeType::NONE, ConcatOrder::AFTER_ENCODER)
75	- Impl(100000100, NormType::LAYER_NORM,NormType::NONE, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)	74	+ Impl(100000100, NormType::LAYER_NORM, NormType::NONE, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)
76	- Impl(100000110, NormType::LAYER_NORM,NormType::NONE, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)	75	+ Impl(100000110, NormType::LAYER_NORM, NormType::NONE, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)
77	- Impl(100000200, NormType::LAYER_NORM,NormType::NONE, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)	76	+ Impl(100000200, NormType::LAYER_NORM, NormType::NONE, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)
78	- Impl(100000210, NormType::LAYER_NORM,NormType::NONE, RopeType::HALF, ConcatOrder::AFTER_ENCODER)	77	+ Impl(100000210, NormType::LAYER_NORM, NormType::NONE, RopeType::HALF, ConcatOrder::AFTER_ENCODER)
79	- Impl(100100000, NormType::LAYER_NORM,NormType::LAYER_NORM, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)	78	+ Impl(100100000, NormType::LAYER_NORM, NormType::LAYER_NORM, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)
80	- Impl(100100010, NormType::LAYER_NORM,NormType::LAYER_NORM, RopeType::NONE, ConcatOrder::AFTER_ENCODER)	79	+ Impl(100100010, NormType::LAYER_NORM, NormType::LAYER_NORM, RopeType::NONE, ConcatOrder::AFTER_ENCODER)
81	- Impl(100100100, NormType::LAYER_NORM,NormType::LAYER_NORM, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)	80	+ Impl(100100100, NormType::LAYER_NORM, NormType::LAYER_NORM, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)
82	- Impl(100100110, NormType::LAYER_NORM,NormType::LAYER_NORM, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)	81	+ Impl(100100110, NormType::LAYER_NORM, NormType::LAYER_NORM, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)
83	- Impl(100100200, NormType::LAYER_NORM,NormType::LAYER_NORM, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)	82	+ Impl(100100200, NormType::LAYER_NORM, NormType::LAYER_NORM, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)
84	- Impl(100100210, NormType::LAYER_NORM,NormType::LAYER_NORM, RopeType::HALF, ConcatOrder::AFTER_ENCODER)	83	+ Impl(100100210, NormType::LAYER_NORM, NormType::LAYER_NORM, RopeType::HALF, ConcatOrder::AFTER_ENCODER)
85	- Impl(100200000, NormType::LAYER_NORM,NormType::LAYER_NORM_AFFINE, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)	84	+ Impl(100200000, NormType::LAYER_NORM, NormType::LAYER_NORM_AFFINE, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)
86	- Impl(100200010, NormType::LAYER_NORM,NormType::LAYER_NORM_AFFINE, RopeType::NONE, ConcatOrder::AFTER_ENCODER)	85	+ Impl(100200010, NormType::LAYER_NORM, NormType::LAYER_NORM_AFFINE, RopeType::NONE, ConcatOrder::AFTER_ENCODER)
87	- Impl(100200100, NormType::LAYER_NORM,NormType::LAYER_NORM_AFFINE, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)	86	+ Impl(100200100, NormType::LAYER_NORM, NormType::LAYER_NORM_AFFINE, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)
88	- Impl(100200110, NormType::LAYER_NORM,NormType::LAYER_NORM_AFFINE, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)	87	+ Impl(100200110, NormType::LAYER_NORM, NormType::LAYER_NORM_AFFINE, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)
89	- Impl(100200200, NormType::LAYER_NORM,NormType::LAYER_NORM_AFFINE, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)	88	+ Impl(100200200, NormType::LAYER_NORM, NormType::LAYER_NORM_AFFINE, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)
90	- Impl(100200210, NormType::LAYER_NORM,NormType::LAYER_NORM_AFFINE, RopeType::HALF, ConcatOrder::AFTER_ENCODER)	89	+ Impl(100200210, NormType::LAYER_NORM, NormType::LAYER_NORM_AFFINE, RopeType::HALF, ConcatOrder::AFTER_ENCODER)
91		90
92	- Impl(200000000, NormType::LAYER_NORM_AFFINE,NormType::NONE, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)	91	+ Impl(200000000, NormType::LAYER_NORM_AFFINE, NormType::NONE, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)
93	- Impl(200000010, NormType::LAYER_NORM_AFFINE,NormType::NONE, RopeType::NONE, ConcatOrder::AFTER_ENCODER)	92	+ Impl(200000010, NormType::LAYER_NORM_AFFINE, NormType::NONE, RopeType::NONE, ConcatOrder::AFTER_ENCODER)
94	- Impl(200000100, NormType::LAYER_NORM_AFFINE,NormType::NONE, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)	93	+ Impl(200000100, NormType::LAYER_NORM_AFFINE, NormType::NONE, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)
95	- Impl(200000110, NormType::LAYER_NORM_AFFINE,NormType::NONE, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)	94	+ Impl(200000110, NormType::LAYER_NORM_AFFINE, NormType::NONE, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)
96	- Impl(200000200, NormType::LAYER_NORM_AFFINE,NormType::NONE, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)	95	+ Impl(200000200, NormType::LAYER_NORM_AFFINE, NormType::NONE, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)
97	- Impl(200000210, NormType::LAYER_NORM_AFFINE,NormType::NONE, RopeType::HALF, ConcatOrder::AFTER_ENCODER)	96	+ Impl(200000210, NormType::LAYER_NORM_AFFINE, NormType::NONE, RopeType::HALF, ConcatOrder::AFTER_ENCODER)
98	- Impl(200100000, NormType::LAYER_NORM_AFFINE,NormType::LAYER_NORM, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)	97	+ Impl(200100000, NormType::LAYER_NORM_AFFINE, NormType::LAYER_NORM, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)
99	- Impl(200100010, NormType::LAYER_NORM_AFFINE,NormType::LAYER_NORM, RopeType::NONE, ConcatOrder::AFTER_ENCODER)	98	+ Impl(200100010, NormType::LAYER_NORM_AFFINE, NormType::LAYER_NORM, RopeType::NONE, ConcatOrder::AFTER_ENCODER)
100	- Impl(200100100, NormType::LAYER_NORM_AFFINE,NormType::LAYER_NORM, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)	99	+ Impl(200100100, NormType::LAYER_NORM_AFFINE, NormType::LAYER_NORM, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)
101	- Impl(200100110, NormType::LAYER_NORM_AFFINE,NormType::LAYER_NORM, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)	100	+ Impl(200100110, NormType::LAYER_NORM_AFFINE, NormType::LAYER_NORM, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)
102	- Impl(200100200, NormType::LAYER_NORM_AFFINE,NormType::LAYER_NORM, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)	101	+ Impl(200100200, NormType::LAYER_NORM_AFFINE, NormType::LAYER_NORM, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)
103	- Impl(200100210, NormType::LAYER_NORM_AFFINE,NormType::LAYER_NORM, RopeType::HALF, ConcatOrder::AFTER_ENCODER)	102	+ Impl(200100210, NormType::LAYER_NORM_AFFINE, NormType::LAYER_NORM, RopeType::HALF, ConcatOrder::AFTER_ENCODER)
104	- Impl(200200000, NormType::LAYER_NORM_AFFINE,NormType::LAYER_NORM_AFFINE, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)	103	+ Impl(200200000, NormType::LAYER_NORM_AFFINE, NormType::LAYER_NORM_AFFINE, RopeType::NONE, ConcatOrder::BEFORE_ENCODER)
105	- Impl(200200010, NormType::LAYER_NORM_AFFINE,NormType::LAYER_NORM_AFFINE, RopeType::NONE, ConcatOrder::AFTER_ENCODER)	104	+ Impl(200200010, NormType::LAYER_NORM_AFFINE, NormType::LAYER_NORM_AFFINE, RopeType::NONE, ConcatOrder::AFTER_ENCODER)
106	- Impl(200200100, NormType::LAYER_NORM_AFFINE,NormType::LAYER_NORM_AFFINE, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)	105	+ Impl(200200100, NormType::LAYER_NORM_AFFINE, NormType::LAYER_NORM_AFFINE, RopeType::INTERLEAVE, ConcatOrder::BEFORE_ENCODER)
107	- Impl(200200110, NormType::LAYER_NORM_AFFINE,NormType::LAYER_NORM_AFFINE, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)	106	+ Impl(200200110, NormType::LAYER_NORM_AFFINE, NormType::LAYER_NORM_AFFINE, RopeType::INTERLEAVE, ConcatOrder::AFTER_ENCODER)
108	- Impl(200200200, NormType::LAYER_NORM_AFFINE,NormType::LAYER_NORM_AFFINE, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)	107	+ Impl(200200200, NormType::LAYER_NORM_AFFINE, NormType::LAYER_NORM_AFFINE, RopeType::HALF, ConcatOrder::BEFORE_ENCODER)
109	- Impl(200200210, NormType::LAYER_NORM_AFFINE,NormType::LAYER_NORM_AFFINE, RopeType::HALF, ConcatOrder::AFTER_ENCODER)	108	+ Impl(200200210, NormType::LAYER_NORM_AFFINE, NormType::LAYER_NORM_AFFINE, RopeType::HALF, ConcatOrder::AFTER_ENCODER)
110	-}	109	+}