pre-commit修复posembedding、mhc、ffn等目录.h/.hpp历史格式问题-ops-transformer-AtomGit

Mcommon/include/err/ops_err.h+4-5

	@@ -20,13 +20,12 @@
20		20
21	constexpr int32_t OP_TRANSFORMER_MODULE_ID = 63;	21	constexpr int32_t OP_TRANSFORMER_MODULE_ID = 63;
22		22
23	-#define OPS_INNER_ERR_STUB(ERR_CODE_STR, OPS_DESC, FMT, ...) \	23	+#define OPS_INNER_ERR_STUB(ERR_CODE_STR, OPS_DESC, FMT, ...) \
24	- do { \	24	+ do { \
25	- OpLogSub(OP_TRANSFORMER_MODULE_ID, DLOG_ERROR, OPS_DESC, FMT, ##__VA_ARGS__); \	25	+ OpLogSub(OP_TRANSFORMER_MODULE_ID, DLOG_ERROR, OPS_DESC, FMT, ##__VA_ARGS__); \
26	- REPORT_INNER_ERR_MSG(ERR_CODE_STR, FMT, ##__VA_ARGS__); \	26	+ REPORT_INNER_ERR_MSG(ERR_CODE_STR, FMT, ##__VA_ARGS__); \
27	} while (0)	27	} while (0)
28		28
29	-
30	/* 基础报错 */	29	/* 基础报错 */
31	#define OPS_REPORT_VECTOR_INNER_ERR(OPS_DESC, ...) OPS_INNER_ERR_STUB("E89999", OPS_DESC, __VA_ARGS__)	30	#define OPS_REPORT_VECTOR_INNER_ERR(OPS_DESC, ...) OPS_INNER_ERR_STUB("E89999", OPS_DESC, __VA_ARGS__)
32	#define OPS_REPORT_CUBE_INNER_ERR(OPS_DESC, ...) OPS_INNER_ERR_STUB("E69999", OPS_DESC, __VA_ARGS__)	31	#define OPS_REPORT_CUBE_INNER_ERR(OPS_DESC, ...) OPS_INNER_ERR_STUB("E69999", OPS_DESC, __VA_ARGS__)

Mcommon/include/external/aclnn_kernels/common/op_error_check.h+88-89

Mcommon/include/fallback/fallback.h+61-59

	@@ -419,8 +419,10 @@ auto ConvertToOpApiFunc(const Tuple &params, void *opApiAddr) ->
419	template <typename Tuple>	419	template <typename Tuple>
420	class ConvertedParams {	420	class ConvertedParams {
421	public:	421	public:
422	- ConvertedParams(Tuple &&convertedParams) : convertedParams_(std::move(convertedParams)) {};	422	+ ConvertedParams(Tuple &&convertedParams)
423	- ConvertedParams(ConvertedParams &&other) : convertedParams_(std::move(other.convertedParams_))	423	+ : convertedParams_(std::move(convertedParams)) {};
		424	+ ConvertedParams(ConvertedParams &&other)
		425	+ : convertedParams_(std::move(other.convertedParams_))
424	{	426	{
425	other.validParams_ = false;	427	other.validParams_ = false;
426	};	428	};
	@@ -467,63 +469,63 @@ using CanUsePTACache = bool ()(const char );
467		469
468	using ResetCacheThreadLocal = void (*)();	470	using ResetCacheThreadLocal = void (*)();
469		471
470	-#define EXEC_OPAPI_CMD(aclnn_api, ...) \	472	+#define EXEC_OPAPI_CMD(aclnn_api, ...) \
471	- ({ \	473	+ ({ \
472	- static auto ret = GRAPH_SUCCESS; \	474	+ static auto ret = GRAPH_SUCCESS; \
473	- do { \	475	+ do { \
474	- static const auto ResetCacheThreadLocalAddr = GetOpApiFuncAddr("ResetCacheThreadLocal"); \	476	+ static const auto ResetCacheThreadLocalAddr = GetOpApiFuncAddr("ResetCacheThreadLocal"); \
475	- static const auto getWorkspaceSizeFuncAddr = GetOpApiFuncAddr(#aclnn_api "GetWorkspaceSize"); \	477	+ static const auto getWorkspaceSizeFuncAddr = GetOpApiFuncAddr(#aclnn_api "GetWorkspaceSize"); \
476	- static const auto opApiFuncAddr = GetOpApiFuncAddr(#aclnn_api); \	478	+ static const auto opApiFuncAddr = GetOpApiFuncAddr(#aclnn_api); \
477	- if (getWorkspaceSizeFuncAddr == nullptr \|\| opApiFuncAddr == nullptr \|\| \	479	+ if (getWorkspaceSizeFuncAddr == nullptr \|\| opApiFuncAddr == nullptr \|\| \
478	- ResetCacheThreadLocalAddr == nullptr) { \	480	+ ResetCacheThreadLocalAddr == nullptr) { \
479	- OP_LOGE("aclnnfallback", "%s or %s not in %s or %s or %s or ResetCacheThreadLocal not found.", \	481	+ OP_LOGE("aclnnfallback", "%s or %s not in %s or %s or %s or ResetCacheThreadLocal not found.", \
480	- #aclnn_api "GetWorkspaceSize", #aclnn_api, GetCustOpApiLibName(), \	482	+ #aclnn_api "GetWorkspaceSize", #aclnn_api, GetCustOpApiLibName(), \
481	- GetTransformerOpApiLibName(), GetOpApiLibName()); \	483	+ GetTransformerOpApiLibName(), GetOpApiLibName()); \
482	- ret = GRAPH_FAILED; \	484	+ ret = GRAPH_FAILED; \
483	- break; \	485	+ break; \
484	- } \	486	+ } \
485	- auto ResetCacheThreadLocalFunc = reinterpret_cast<ResetCacheThreadLocal>(ResetCacheThreadLocalAddr); \	487	+ auto ResetCacheThreadLocalFunc = reinterpret_cast<ResetCacheThreadLocal>(ResetCacheThreadLocalAddr); \
486	- ResetCacheThreadLocalFunc(); \	488	+ ResetCacheThreadLocalFunc(); \
487	- uint64_t workspace_size = 0; \	489	+ uint64_t workspace_size = 0; \
488	- uint64_t *workspace_size_addr = &workspace_size; \	490	+ uint64_t *workspace_size_addr = &workspace_size; \
489	- aclOpExecutor *executor = nullptr; \	491	+ aclOpExecutor *executor = nullptr; \
490	- aclOpExecutor **executor_addr = &executor; \	492	+ aclOpExecutor **executor_addr = &executor; \
491	- auto converted_params = ConvertTypes(__VA_ARGS__, workspace_size_addr, executor_addr); \	493	+ auto converted_params = ConvertTypes(__VA_ARGS__, workspace_size_addr, executor_addr); \
492	- static auto getWorkspaceSizeFunc = ConvertToOpApiFunc(converted_params, getWorkspaceSizeFuncAddr); \	494	+ static auto getWorkspaceSizeFunc = ConvertToOpApiFunc(converted_params, getWorkspaceSizeFuncAddr); \
493	- auto workspace_status = call(getWorkspaceSizeFunc, converted_params); \	495	+ auto workspace_status = call(getWorkspaceSizeFunc, converted_params); \
494	- if (workspace_status != 0) { \	496	+ if (workspace_status != 0) { \
495	- OP_LOGE("aclnnfallback", "call %s failed:", #aclnn_api); \	497	+ OP_LOGE("aclnnfallback", "call %s failed:", #aclnn_api); \
496	- ret = GRAPH_FAILED; \	498	+ ret = GRAPH_FAILED; \
497	- break; \	499	+ break; \
498	- } \	500	+ } \
499	- void *workspace_addr = nullptr; \	501	+ void *workspace_addr = nullptr; \
500	- if (workspace_size > 0) { \	502	+ if (workspace_size > 0) { \
501	- workspace_addr = host_api_ctx->MallocWorkspace(workspace_size); \	503	+ workspace_addr = host_api_ctx->MallocWorkspace(workspace_size); \
502	- if (workspace_addr == nullptr) { \	504	+ if (workspace_addr == nullptr) { \
503	- OP_LOGE("aclnnfallback", "call %s allocate workspace failed", #aclnn_api); \	505	+ OP_LOGE("aclnnfallback", "call %s allocate workspace failed", #aclnn_api); \
504	- ret = GRAPH_FAILED; \	506	+ ret = GRAPH_FAILED; \
505	- break; \	507	+ break; \
506	- } \	508	+ } \
507	- } \	509	+ } \
508	- auto acl_stream = host_api_ctx->GetStream(); \	510	+ auto acl_stream = host_api_ctx->GetStream(); \
509	- auto acl_call = [converted_params, workspace_addr, workspace_size, host_api_ctx, acl_stream, \	511	+ auto acl_call = [converted_params, workspace_addr, workspace_size, host_api_ctx, acl_stream, \
510	- executor]() -> int { \	512	+ executor]() -> int { \
511	- using OpApiFunc = int ()(void , uint64_t, aclOpExecutor *, const aclrtStream); \	513	+ using OpApiFunc = int ()(void , uint64_t, aclOpExecutor *, const aclrtStream); \
512	- OpApiFunc opApiFunc = reinterpret_cast<OpApiFunc>(opApiFuncAddr); \	514	+ OpApiFunc opApiFunc = reinterpret_cast<OpApiFunc>(opApiFuncAddr); \
513	- auto api_ret_inner = opApiFunc(workspace_addr, workspace_size, executor, acl_stream); \	515	+ auto api_ret_inner = opApiFunc(workspace_addr, workspace_size, executor, acl_stream); \
514	- ReleaseConvertTypes(converted_params); \	516	+ ReleaseConvertTypes(converted_params); \
515	- host_api_ctx->FreeWorkspace(); \	517	+ host_api_ctx->FreeWorkspace(); \
516	- if (api_ret_inner != 0) { \	518	+ if (api_ret_inner != 0) { \
517	- OP_LOGE("aclnnfallback", "call %s allocate workspace failed api_ret_inner: %d", #aclnn_api, \	519	+ OP_LOGE("aclnnfallback", "call %s allocate workspace failed api_ret_inner: %d", #aclnn_api, \
518	- api_ret_inner); \	520	+ api_ret_inner); \
519	- return GRAPH_FAILED; \	521	+ return GRAPH_FAILED; \
520	- } \	522	+ } \
521	- return api_ret_inner; \	523	+ return api_ret_inner; \
522	- }; \	524	+ }; \
523	- \	525	+\
524	- ret = acl_call(); \	526	+ ret = acl_call(); \
525	- } while (false); \	527	+ } while (false); \
526	- (ret); \	528	+ (ret); \
527	})	529	})
528		530
529	} // namespace fallback	531	} // namespace fallback

Mcommon/include/fallback/fallback_2stages.h+37-37

Mcommon/include/op_host/tiling_base.h+3-3

	@@ -87,9 +87,9 @@ struct FACompileInfoCommon {
87		87
88	class TilingBaseClass {	88	class TilingBaseClass {
89	public:	89	public:
90	- explicit TilingBaseClass(gert::TilingContext *context) : context_(context)	90	+ explicit TilingBaseClass(gert::TilingContext *context)
91	- {	91	+ : context_(context)
92	- }	92	+ {}
93		93
94	virtual ~TilingBaseClass() = default;	94	virtual ~TilingBaseClass() = default;
95		95

Mcommon/include/op_host/tiling_key.h+2-2

	@@ -56,8 +56,8 @@ constexpr uint64_t GET_TILINGKEY(Args... templateIds)
56	// usage: get tilingKey from inputed types	56	// usage: get tilingKey from inputed types
57	// uint64_t tilingKey = TILINGKEY(S2, S1, N2, FLOAT32, BSND, ALL)	57	// uint64_t tilingKey = TILINGKEY(S2, S1, N2, FLOAT32, BSND, ALL)
58		58
59	-#define TILINGKEY(ub2, ub1, block, dtype, layout, sparse) \	59	+#define TILINGKEY(ub2, ub1, block, dtype, layout, sparse) \
60	- (GET_TILINGKEY(AxisEnum::ub2, AxisEnum::ub1, AxisEnum::block, DtypeEnum::dtype, LayoutEnum::layout, \	60	+ (GET_TILINGKEY(AxisEnum::ub2, AxisEnum::ub1, AxisEnum::block, DtypeEnum::dtype, LayoutEnum::layout, \
61	SparseEnum::sparse))	61	SparseEnum::sparse))
62		62
63	} // namespace OpTiling	63	} // namespace OpTiling

Mcommon/include/op_host/tiling_templates_registry.h+28-28

Mcommon/include/op_host/tiling_type.h+2-2

	@@ -134,8 +134,8 @@ constexpr uint64_t GET_TILINGKEY(Args... templateIds)
134	// usage: get tilingKey from inputed types	134	// usage: get tilingKey from inputed types
135	// uint64_t tilingKey = TILINGKEY(S2, S1, N2, FLOAT32, BSND, ALL)	135	// uint64_t tilingKey = TILINGKEY(S2, S1, N2, FLOAT32, BSND, ALL)
136		136
137	-#define TILINGKEY(ub2, ub1, block, dtype, layout, sparse) \	137	+#define TILINGKEY(ub2, ub1, block, dtype, layout, sparse) \
138	- (GET_TILINGKEY(AxisEnum::ub2, AxisEnum::ub1, AxisEnum::block, DtypeEnum::dtype, LayoutEnum::layout, \	138	+ (GET_TILINGKEY(AxisEnum::ub2, AxisEnum::ub1, AxisEnum::block, DtypeEnum::dtype, LayoutEnum::layout, \
139	SparseEnum::sparse))	139	SparseEnum::sparse))
140		140
141	} // namespace optiling	141	} // namespace optiling

Mcommon/include/static/op_resource.h+18-18

Mcommon/stub/inc/framework/stub_ops.h+0-1

	@@ -136,7 +136,6 @@ REG_OP(Data)
136	DT_UINT16, DT_COMPLEX128, DT_FLOAT16, DT_BF16, DT_UINT32, DT_UINT64, DT_STRING}))	136	DT_UINT16, DT_COMPLEX128, DT_FLOAT16, DT_BF16, DT_UINT32, DT_UINT64, DT_STRING}))
137	.OP_END_FACTORY_REG(Fill)	137	.OP_END_FACTORY_REG(Fill)
138		138
139	-
140	} // namespace ge	139	} // namespace ge
141		140
142	#endif // MATH_COMMON_STUB_OPS_H	141	#endif // MATH_COMMON_STUB_OPS_H

Mcommon/stub/op_api/aclnn_kernels/common/op_error_check.h+88-89

Mcommon/stub/op_api/level0/batch_norm_backward.h+4-4

	@@ -31,10 +31,10 @@ const aclTensor BNInferGrad(const aclTensor gradOut, const aclTensor *weight,
31	aclOpExecutor *executor);	31	aclOpExecutor *executor);
32		32
33	constexpr size_t BN_GRAD_V3_OUTPUT_NUM = 3;	33	constexpr size_t BN_GRAD_V3_OUTPUT_NUM = 3;
34	-const std::array<aclTensor *, BN_GRAD_V3_OUTPUT_NUM>	34	+const std::array<aclTensor *, BN_GRAD_V3_OUTPUT_NUM> BatchNormGradV3(
35	-BatchNormGradV3(const aclTensor gradOut, const aclTensor input, const aclTensor weight, const aclTensor runningMean,	35	+ const aclTensor gradOut, const aclTensor input, const aclTensor weight, const aclTensor runningMean,
36	- const aclTensor runningVar, const aclTensor saveMean, const aclTensor *saveInvstd, bool training,	36	+ const aclTensor runningVar, const aclTensor saveMean, const aclTensor *saveInvstd, bool training, float eps,
37	- float eps, aclOpExecutor *executor);	37	+ aclOpExecutor *executor);
38	} // namespace l0op	38	} // namespace l0op
39		39
40	#endif // PTA_NPU_OP_API_INC_LEVEL0_OP_BATCH_NORM_GRAD_OP_H_	40	#endif // PTA_NPU_OP_API_INC_LEVEL0_OP_BATCH_NORM_GRAD_OP_H_

Mcommon/stub/op_tiling/register/tuning_bank_key_registry.h+70-72

	@@ -23,16 +23,16 @@
23		23
24	#define REGISTER_OP_BANK_KEY_CONVERT_FUN_UNIQ_HELPER(optype, opfunc) REGISTER_OP_BANK_KEY_UNIQ(optype, (opfunc))	24	#define REGISTER_OP_BANK_KEY_CONVERT_FUN_UNIQ_HELPER(optype, opfunc) REGISTER_OP_BANK_KEY_UNIQ(optype, (opfunc))
25		25
26	-#define REGISTER_OP_BANK_KEY_UNIQ(optype, opfunc) \	26	+#define REGISTER_OP_BANK_KEY_UNIQ(optype, opfunc) \
27	static tuningtiling::OpBankKeyFuncRegistry g_##optype##BankKeyRegistryInterf(#optype, (opfunc))	27	static tuningtiling::OpBankKeyFuncRegistry g_##optype##BankKeyRegistryInterf(#optype, (opfunc))
28		28
29	-#define REGISTER_OP_BANK_KEY_PARSE_FUN(op, parse_func, load_func) \	29	+#define REGISTER_OP_BANK_KEY_PARSE_FUN(op, parse_func, load_func) \
30	REGISTER_OP_BANK_KEY_PARSE_FUN_UNIQ_HELPER(op, (parse_func), (load_func))	30	REGISTER_OP_BANK_KEY_PARSE_FUN_UNIQ_HELPER(op, (parse_func), (load_func))
31		31
32	-#define REGISTER_OP_BANK_KEY_PARSE_FUN_UNIQ_HELPER(optype, parse_func, load_func) \	32	+#define REGISTER_OP_BANK_KEY_PARSE_FUN_UNIQ_HELPER(optype, parse_func, load_func) \
33	REGISTER_OP_BANK_KEY_PARSE_UNIQ(optype, (parse_func), (load_func))	33	REGISTER_OP_BANK_KEY_PARSE_UNIQ(optype, (parse_func), (load_func))
34		34
35	-#define REGISTER_OP_BANK_KEY_PARSE_UNIQ(optype, parse_func, load_func) \	35	+#define REGISTER_OP_BANK_KEY_PARSE_UNIQ(optype, parse_func, load_func) \
36	static tuningtiling::OpBankKeyFuncRegistry g_##optype##BankParseInterf(#optype, (parse_func), (load_func))	36	static tuningtiling::OpBankKeyFuncRegistry g_##optype##BankParseInterf(#optype, (parse_func), (load_func))
37		37
38	// v2	38	// v2
	@@ -40,90 +40,88 @@
40		40
41	#define REGISTER_OP_BANK_KEY_CONVERT_FUN_UNIQ_HELPER_V2(optype, opfunc) REGISTER_OP_BANK_KEY_UNIQ_V2(optype, (opfunc))	41	#define REGISTER_OP_BANK_KEY_CONVERT_FUN_UNIQ_HELPER_V2(optype, opfunc) REGISTER_OP_BANK_KEY_UNIQ_V2(optype, (opfunc))
42		42
43	-#define REGISTER_OP_BANK_KEY_UNIQ_V2(optype, opfunc) \	43	+#define REGISTER_OP_BANK_KEY_UNIQ_V2(optype, opfunc) \
44	static tuningtiling::OpBankKeyFuncRegistryV2 g_##optype##BankKeyRegistryInterf(#optype, (opfunc))	44	static tuningtiling::OpBankKeyFuncRegistryV2 g_##optype##BankKeyRegistryInterf(#optype, (opfunc))
45		45
46	-#define REGISTER_OP_BANK_KEY_PARSE_FUN_V2(op, parse_func, load_func) \	46	+#define REGISTER_OP_BANK_KEY_PARSE_FUN_V2(op, parse_func, load_func) \
47	REGISTER_OP_BANK_KEY_PARSE_FUN_UNIQ_HELPER_V2(op, (parse_func), (load_func))	47	REGISTER_OP_BANK_KEY_PARSE_FUN_UNIQ_HELPER_V2(op, (parse_func), (load_func))
48		48
49	-#define REGISTER_OP_BANK_KEY_PARSE_FUN_UNIQ_HELPER_V2(optype, parse_func, load_func) \	49	+#define REGISTER_OP_BANK_KEY_PARSE_FUN_UNIQ_HELPER_V2(optype, parse_func, load_func) \
50	REGISTER_OP_BANK_KEY_PARSE_UNIQ_V2(optype, (parse_func), (load_func))	50	REGISTER_OP_BANK_KEY_PARSE_UNIQ_V2(optype, (parse_func), (load_func))
51		51
52	-#define REGISTER_OP_BANK_KEY_PARSE_UNIQ_V2(optype, parse_func, load_func) \	52	+#define REGISTER_OP_BANK_KEY_PARSE_UNIQ_V2(optype, parse_func, load_func) \
53	static tuningtiling::OpBankKeyFuncRegistryV2 g_##optype##BankParseInterf(#optype, (parse_func), (load_func))	53	static tuningtiling::OpBankKeyFuncRegistryV2 g_##optype##BankParseInterf(#optype, (parse_func), (load_func))
54		54
55	-#define TUNING_TILING_MAKE_SHARED(exec_expr0, exec_expr1) \	55	+#define TUNING_TILING_MAKE_SHARED(exec_expr0, exec_expr1) \
56	- do { \	56	+ do { \
57	- try { \	57	+ try { \
58	- exec_expr0; \	58	+ exec_expr0; \
59	- } catch (...) { \	59	+ } catch (...) { \
60	- exec_expr1; \	60	+ exec_expr1; \
61	- } \	61	+ } \
62	} while (0)	62	} while (0)
63		63
64	// v1 stub	64	// v1 stub
65	-#define DECLARE_STRUCT_RELATE_WITH_OP(op, bank_key, ...) \	65	+#define DECLARE_STRUCT_RELATE_WITH_OP(op, bank_key, ...) \
66	- do { \	66	+ do { \
67	- NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(bank_key, __VA_ARGS__); \	67	+ NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(bank_key, __VA_ARGS__); \
68	- static bool ParseFunc##op##bank_key(const std::shared_ptr<void> &in_args, size_t len, \	68	+ static bool ParseFunc##op##bank_key(const std::shared_ptr<void> &in_args, size_t len, \
69	- ge::AscendString &bank_key_str) \	69	+ ge::AscendString &bank_key_str) \
70	- { \	70	+ { \
71	- if (sizeof(bank_key_str) != len \|\| in_args == nullptr) { \	71	+ if (sizeof(bank_key_str) != len \|\| in_args == nullptr) { \
72	- return false; \	72	+ return false; \
73	- } \	73	+ } \
74	- return false; \	74	+ return false; \
75	- } \	75	+ } \
76	- static bool LoadFunc##op##bank_key(std::shared_ptr<void> &in_args, size_t &len, \	76	+ static bool LoadFunc##op##bank_key(std::shared_ptr<void> &in_args, size_t &len, \
77	- const ge::AscendString &bank_key_str) \	77	+ const ge::AscendString &bank_key_str) \
78	- { \	78	+ { \
79	- len = sizeof(bank_key_str); \	79	+ len = sizeof(bank_key_str); \
80	- TUNING_TILING_MAKE_SHARED(in_args = std::make_shared<bank_key>(), return false); \	80	+ TUNING_TILING_MAKE_SHARED(in_args = std::make_shared<bank_key>(), return false); \
81	- auto op_ky = std::static_pointer_cast<bank_key>(in_args); \	81	+ auto op_ky = std::static_pointer_cast<bank_key>(in_args); \
82	- return false; \	82	+ return false; \
83	- } \	83	+ } \
84	- REGISTER_OP_BANK_KEY_PARSE_FUN(op, ParseFunc##op##bank_key, LoadFunc##op##bank_key) \	84	+ REGISTER_OP_BANK_KEY_PARSE_FUN(op, ParseFunc##op##bank_key, LoadFunc##op##bank_key) \
85	} while (0)	85	} while (0)
86		86
87	-
88	// v2	87	// v2
89	-#define DECLARE_STRUCT_RELATE_WITH_OP_V2(op, bank_key, ...) \	88	+#define DECLARE_STRUCT_RELATE_WITH_OP_V2(op, bank_key, ...) \
90	- do { \	89	+ do { \
91	- NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(bank_key, __VA_ARGS__); \	90	+ NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(bank_key, __VA_ARGS__); \
92	- static bool ParseFuncV2##op##bank_key(const std::shared_ptr<void> &in_args, size_t len, \	91	+ static bool ParseFuncV2##op##bank_key(const std::shared_ptr<void> &in_args, size_t len, \
93	- ge::AscendString &bank_key_json_str) \	92	+ ge::AscendString &bank_key_json_str) \
94	- { \	93	+ { \
95	- if (sizeof(bank_key) != len \|\| in_args == nullptr) { \	94	+ if (sizeof(bank_key) != len \|\| in_args == nullptr) { \
96	- return false; \	95	+ return false; \
97	- } \	96	+ } \
98	- nlohmann::json bank_key_json; \	97	+ nlohmann::json bank_key_json; \
99	- bank_key_json = *(std::static_pointer_cast<bank_key>(in_args)); \	98	+ bank_key_json = *(std::static_pointer_cast<bank_key>(in_args)); \
100	- try { \	99	+ try { \
101	- std::string json_dump_str = bank_key_json.dump(); \	100	+ std::string json_dump_str = bank_key_json.dump(); \
102	- bank_key_json_str = ge::AscendString(json_dump_str.c_str()); \	101	+ bank_key_json_str = ge::AscendString(json_dump_str.c_str()); \
103	- } catch (std::exception & e) { \	102	+ } catch (std::exception & e) { \
104	- return false; \	103	+ return false; \
105	- } \	104	+ } \
106	- return true; \	105	+ return true; \
107	- } \	106	+ } \
108	- static bool LoadFuncV2##op##bank_key(std::shared_ptr<void> &in_args, size_t &len, \	107	+ static bool LoadFuncV2##op##bank_key(std::shared_ptr<void> &in_args, size_t &len, \
109	- const ge::AscendString &bank_key_json_str) \	108	+ const ge::AscendString &bank_key_json_str) \
110	- { \	109	+ { \
111	- len = sizeof(bank_key); \	110	+ len = sizeof(bank_key); \
112	- TUNING_TILING_MAKE_SHARED(in_args = std::make_shared<bank_key>(), return false); \	111	+ TUNING_TILING_MAKE_SHARED(in_args = std::make_shared<bank_key>(), return false); \
113	- nlohmann::json bank_key_json; \	112	+ nlohmann::json bank_key_json; \
114	- try { \	113	+ try { \
115	- bank_key_json = nlohmann::json::parse(bank_key_json_str.GetString()); \	114	+ bank_key_json = nlohmann::json::parse(bank_key_json_str.GetString()); \
116	- auto op_ky = std::static_pointer_cast<bank_key>(in_args); \	115	+ auto op_ky = std::static_pointer_cast<bank_key>(in_args); \
117	- *op_ky = bank_key_json.get<bank_key>(); \	116	+ *op_ky = bank_key_json.get<bank_key>(); \
118	- } catch (std::exception & e) { \	117	+ } catch (std::exception & e) { \
119	- return false; \	118	+ return false; \
120	- } \	119	+ } \
121	- return true; \	120	+ return true; \
122	- } \	121	+ } \
123	- REGISTER_OP_BANK_KEY_PARSE_FUN_V2(op, ParseFuncV2##op##bank_key, LoadFuncV2##op##bank_key) \	122	+ REGISTER_OP_BANK_KEY_PARSE_FUN_V2(op, ParseFuncV2##op##bank_key, LoadFuncV2##op##bank_key) \
124	} while (0)	123	} while (0)
125		124
126	-
127	namespace tuningtiling {	125	namespace tuningtiling {
128	// v1兼容老版本om	126	// v1兼容老版本om
129	using OpBankKeyConvertFun = std::function<bool(const gert::TilingContext *, std::shared_ptr<void> &, size_t &)>;	127	using OpBankKeyConvertFun = std::function<bool(const gert::TilingContext *, std::shared_ptr<void> &, size_t &)>;

Mcommon/stub/op_tiling/register/tuning_tiling_reflection_utils.h+13-13

	@@ -52,13 +52,13 @@ struct StructInfo {
52	}	52	}
53	};	53	};
54		54
55	-#define DECLARE_SCHEMA(Struct, ...) \	55	+#define DECLARE_SCHEMA(Struct, ...) \
56	- template <> \	56	+ template <> \
57	- struct StructInfo<Struct> { \	57	+ struct StructInfo<Struct> { \
58	- static decltype(std::make_tuple(__VA_ARGS__)) Info() \	58	+ static decltype(std::make_tuple(__VA_ARGS__)) Info() \
59	- { \	59	+ { \
60	- return std::make_tuple(__VA_ARGS__); \	60	+ return std::make_tuple(__VA_ARGS__); \
61	- } \	61	+ } \
62	};	62	};
63		63
64	#define FIELD(class, FieldName) std::make_tuple(#FieldName, &class ::FieldName)	64	#define FIELD(class, FieldName) std::make_tuple(#FieldName, &class ::FieldName)
	@@ -135,9 +135,9 @@ void DumpObj(T &&obj, const std::string &field_name, Js &j)
135		135
136	template <typename T>	136	template <typename T>
137	struct DumpFunctor {	137	struct DumpFunctor {
138	- explicit DumpFunctor(T &j) : js(j)	138	+ explicit DumpFunctor(T &j)
139	- {	139	+ : js(j)
140	- }	140	+ {}
141	template <typename Name, typename Field>	141	template <typename Name, typename Field>
142	void operator()(Name &&name, Field &&field) const	142	void operator()(Name &&name, Field &&field) const
143	{	143	{
	@@ -174,9 +174,9 @@ void FromJsonImpl(T &&obj, const std::string &field_name, const Js &j)
174		174
175	template <typename Js>	175	template <typename Js>
176	struct FromJsonFunctor {	176	struct FromJsonFunctor {
177	- explicit FromJsonFunctor(const Js &j) : js(j)	177	+ explicit FromJsonFunctor(const Js &j)
178	- {	178	+ : js(j)
179	- }	179	+ {}
180	template <typename Name, typename Field>	180	template <typename Name, typename Field>
181	void operator()(Name &&name, Field &&field) const	181	void operator()(Name &&name, Field &&field) const
182	{	182	{

Mcommon/stub/op_tiling/register/tuning_tiling_registry.h+48-48

	@@ -37,46 +37,46 @@ protected:
37	ge::AscendString class_name_;	37	ge::AscendString class_name_;
38	};	38	};
39		39
40	-#define BEGIN_TUNING_TILING_DEF(class_name) \	40	+#define BEGIN_TUNING_TILING_DEF(class_name) \
41	- class class_name : public TuningTilingDef { \	41	+ class class_name : public TuningTilingDef { \
42	- public: \	42	+ public: \
43	- virtual void FromJson(const nlohmann::json &j) \	43	+ virtual void FromJson(const nlohmann::json &j) \
44	- { \	44	+ { \
45	- FromJsonImpl(*this, "", j); \	45	+ FromJsonImpl(*this, "", j); \
46	- } \	46	+ } \
47	- \	47	+\
48	- virtual void ToJson(nlohmann::json &j) \	48	+ virtual void ToJson(nlohmann::json &j) \
49	- { \	49	+ { \
50	- DumpObj(*this, "", j); \	50	+ DumpObj(*this, "", j); \
51	- } \	51	+ } \
52	- \	52	+\
53	- std::vector<TilingItem> GetItemInfo() const \	53	+ std::vector<TilingItem> GetItemInfo() const \
54	- { \	54	+ { \
55	- return field_info_; \	55	+ return field_info_; \
56	- } \	56	+ } \
57	- \	57	+\
58	- class FieldHandler { \	58	+ class FieldHandler { \
59	- public: \	59	+ public: \
60	- FieldHandler(class_name *pinstance, const ge::AscendString &dtype, const ge::AscendString &name) \	60	+ FieldHandler(class_name *pinstance, const ge::AscendString &dtype, const ge::AscendString &name) \
61	- { \	61	+ { \
62	- pinstance->field_info_.push_back({dtype, name}); \	62	+ pinstance->field_info_.push_back({dtype, name}); \
63	- } \	63	+ } \
64	- }; \	64	+ }; \
65	- friend class FieldHandler; \	65	+ friend class FieldHandler; \
66	- \	66	+\
67	- public: \	67	+ public: \
68	- class_name() \	68	+ class_name() \
69	- { \	69	+ { \
70	- class_name_ = #class_name; \	70	+ class_name_ = #class_name; \
71	};	71	};
72		72
73	-#define TUNING_TILING_DATA_FIELD_DEF(data_type, field_name) \	73	+#define TUNING_TILING_DATA_FIELD_DEF(data_type, field_name) \
74	-public: \	74	+public: \
75	- data_type field_name; \	75	+ data_type field_name; \
76	FieldHandler field_name##_handler_ = FieldHandler(this, #data_type, #field_name);	76	FieldHandler field_name##_handler_ = FieldHandler(this, #data_type, #field_name);
77		77
78	-#define END_TUNING_TILING_DEF \	78	+#define END_TUNING_TILING_DEF \
79	- } \	79	+ } \
80	;	80	;
81		81
82	using TuningTilingDefConstructor = std::shared_ptr<TuningTilingDef> (*)();	82	using TuningTilingDefConstructor = std::shared_ptr<TuningTilingDef> (*)();
	@@ -87,18 +87,18 @@ public:
87	static std::shared_ptr<TuningTilingDef> CreateTilingDataInstance(const ge::AscendString &optype);	87	static std::shared_ptr<TuningTilingDef> CreateTilingDataInstance(const ge::AscendString &optype);
88	};	88	};
89		89
90	-#define REGISTER_TUNING_TILING_CLASS(optype, class_name) \	90	+#define REGISTER_TUNING_TILING_CLASS(optype, class_name) \
91	- class optype##Helper { \	91	+ class optype##Helper { \
92	- public: \	92	+ public: \
93	- optype##Helper() \	93	+ optype##Helper() \
94	- { \	94	+ { \
95	- TuningTilingClassFactory::RegisterTilingData(#optype, optype##Helper::CreateTilingDataInstance); \	95	+ TuningTilingClassFactory::RegisterTilingData(#optype, optype##Helper::CreateTilingDataInstance); \
96	- } \	96	+ } \
97	- static std::shared_ptr<TuningTilingDef> CreateTilingDataInstance() \	97	+ static std::shared_ptr<TuningTilingDef> CreateTilingDataInstance() \
98	- { \	98	+ { \
99	- return std::make_shared<class_name>(); \	99	+ return std::make_shared<class_name>(); \
100	- } \	100	+ } \
101	- }; \	101	+ }; \
102	optype##Helper g_tuning_tiling_##optype##Helper;	102	optype##Helper g_tuning_tiling_##optype##Helper;
103	using TuningTilingDefPtr = std::shared_ptr<TuningTilingDef>;	103	using TuningTilingDefPtr = std::shared_ptr<TuningTilingDef>;
104	} // namespace tuningtiling	104	} // namespace tuningtiling

Mffn/ffn/op_host/ffn_tiling.h+1-1

	@@ -60,4 +60,4 @@ END_TILING_DATA_DEF;
60	REGISTER_TILING_DATA_CLASS(FFN, FFNTilingData)	60	REGISTER_TILING_DATA_CLASS(FFN, FFNTilingData)
61	} // namespace optiling	61	} // namespace optiling
62		62
63	-#endif // AIR_CXX_RUNTIME_V2_OP_IMPL_FFN_H_	63	+#endif // AIR_CXX_RUNTIME_V2_OP_IMPL_FFN_H_

Mffn/ffn/op_host/op_api/ffn.h+1-1

	@@ -29,4 +29,4 @@ const aclTensor FFN(const aclTensor x, const aclTensor *weight1, const aclTens
29	aclOpExecutor *executor);	29	aclOpExecutor *executor);
30	}	30	}
31		31
32	-#endif	32	+#endif

Mffn/ffn/op_kernel/ffn_antiquant.h+3-4

	@@ -44,8 +44,7 @@ public:
44	__aicore__ inline FFNAntiQuant(mm1Type &mm1_, mm2Type &mm2_)	44	__aicore__ inline FFNAntiQuant(mm1Type &mm1_, mm2Type &mm2_)
45	: mm1(mm1_),	45	: mm1(mm1_),
46	mm2(mm2_)	46	mm2(mm2_)
47	- {	47	+ {}
48	- }
49	__aicore__ inline void Init(__gm__ uint8_t x, __gm__ uint8_t weight1, __gm__ uint8_t *weight2,	48	__aicore__ inline void Init(__gm__ uint8_t x, __gm__ uint8_t weight1, __gm__ uint8_t *weight2,
50	__gm__ uint8_t expertTokens, __gm__ uint8_t bias1, __gm__ uint8_t *bias2,	49	__gm__ uint8_t expertTokens, __gm__ uint8_t bias1, __gm__ uint8_t *bias2,
51	__gm__ uint8_t antiQuantScale1, __gm__ uint8_t antiQuantScale2,	50	__gm__ uint8_t antiQuantScale1, __gm__ uint8_t antiQuantScale2,
	@@ -318,8 +317,8 @@ protected:
318	if (isMatMul1) {	317	if (isMatMul1) {
319	n1Loops = nLoops;	318	n1Loops = nLoops;
320	m1Loops = mLoops;	319	m1Loops = mLoops;
321	- singleM1 = singleM; // compute C matrix block length along m direction for each cube	320	+ singleM1 = singleM; // compute C matrix block length along m direction for each cube
322	- singleN1 = singleN; // compute C matrix block length along n direction for each cube	321	+ singleN1 = singleN; // compute C matrix block length along n direction for each cube
323	singleM1Tail = tokens - (m1Loops - 1) * singleM1; // recompute last block length along m direction	322	singleM1Tail = tokens - (m1Loops - 1) * singleM1; // recompute last block length along m direction
324	singleN1Tail = n1 - (n1Loops - 1) * singleN1; // recompute last block length along n direction	323	singleN1Tail = n1 - (n1Loops - 1) * singleN1; // recompute last block length along n direction
325	castWeightSingleN1 = n1;	324	castWeightSingleN1 = n1;

Mffn/ffn/op_kernel/ffn_antiquant_msd.h+34-44

	@@ -98,8 +98,7 @@ public:
98	__aicore__ inline FFNAntiQuantMSD(mm1Type &mm1_, mm2Type &mm2_)	98	__aicore__ inline FFNAntiQuantMSD(mm1Type &mm1_, mm2Type &mm2_)
99	: mm1(mm1_),	99	: mm1(mm1_),
100	mm2(mm2_)	100	mm2(mm2_)
101	- {	101	+ {}
102	- }
103	__aicore__ inline void Init(__gm__ uint8_t x, __gm__ uint8_t weight1, __gm__ uint8_t *weight2,	102	__aicore__ inline void Init(__gm__ uint8_t x, __gm__ uint8_t weight1, __gm__ uint8_t *weight2,
104	__gm__ uint8_t expertTokens, __gm__ uint8_t bias1, __gm__ uint8_t *bias2,	103	__gm__ uint8_t expertTokens, __gm__ uint8_t bias1, __gm__ uint8_t *bias2,
105	__gm__ uint8_t antiQuantScale1, __gm__ uint8_t antiQuantScale2,	104	__gm__ uint8_t antiQuantScale1, __gm__ uint8_t antiQuantScale2,
	@@ -323,8 +322,8 @@ __aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>
323	}	322	}
324		323
325	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>	324	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>
326	-__aicore__ inline void	325	+__aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::GetMaxToken(
327	-FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::GetMaxToken(__gm__ uint8_t *expertTokens)	326	+ __gm__ uint8_t *expertTokens)
328	{	327	{
329	uint32_t expertTokensUbSize = AlignUp<UB_BLOCK_UNIT_SIZE>(expertNum * sizeof(int64_t));	328	uint32_t expertTokensUbSize = AlignUp<UB_BLOCK_UNIT_SIZE>(expertNum * sizeof(int64_t));
330	pipe->InitBuffer(eTokens64Buf, expertTokensUbSize); // 32Byte alignment	329	pipe->InitBuffer(eTokens64Buf, expertTokensUbSize); // 32Byte alignment
	@@ -350,8 +349,8 @@ FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::GetMaxToken(__gm__ ui
350	}	349	}
351		350
352	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>	351	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>
353	-__aicore__ inline void	352	+__aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::InitWorkspace(
354	-FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::InitWorkspace(__gm__ uint8_t *workSpace)	353	+ __gm__ uint8_t *workSpace)
355	{	354	{
356	// init global buffer	355	// init global buffer
357	uint32_t maxParallelExpertNum1 = Max<uint32_t>(cubeCoreNum / Ceil(n1, tilingData->mm1TilingData.baseN), 1);	356	uint32_t maxParallelExpertNum1 = Max<uint32_t>(cubeCoreNum / Ceil(n1, tilingData->mm1TilingData.baseN), 1);
	@@ -413,23 +412,22 @@ __aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>
413	countOfFloatUbCalcShape += 1;	412	countOfFloatUbCalcShape += 1;
414	middleResultFP16 =	413	middleResultFP16 =
415	tmpBuff_.GetWithOffset<half>(ubCalcShape_, countOfFloatUbCalcShape * ubCalcShape_ * sizeof(float));	414	tmpBuff_.GetWithOffset<half>(ubCalcShape_, countOfFloatUbCalcShape * ubCalcShape_ * sizeof(float));
416	- aMax = tmpBuff_.GetWithOffset<float>(maxTokens * FACTOR_FOR_FLOAT_ALIGN_TO_32BYTE,	415	+ aMax = tmpBuff_.GetWithOffset<float>(
417	- countOfFloatUbCalcShape * ubCalcShape_ * sizeof(float) +	416	+ maxTokens * FACTOR_FOR_FLOAT_ALIGN_TO_32BYTE,
418	- ubCalcShape_ * sizeof(half));	417	+ countOfFloatUbCalcShape * ubCalcShape_ * sizeof(float) + ubCalcShape_ * sizeof(half));
419	}	418	}
420		419
421	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>	420	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>
422	-__aicore__ inline void	421	+__aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::MM1VectorTiling(
423	-FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::MM1VectorTiling(TilingConfig &tilingParams)	422	+ TilingConfig &tilingParams)
424	{	423	{
425	uint32_t vecBlockDimK_ = tilingParams.aivNumPerExpert;	424	uint32_t vecBlockDimK_ = tilingParams.aivNumPerExpert;
426	uint32_t vecBaseK_ = Ceil(tilingParams.k, vecBlockDimK_);	425	uint32_t vecBaseK_ = Ceil(tilingParams.k, vecBlockDimK_);
427	// baseK 128 align up	426	// baseK 128 align up
428	vecBaseK_ = (vecBaseK_ + NUM_ALIGN_TO_ONE_HUNDRED_TWEENTY_EIGHT) & (~NUM_ALIGN_TO_ONE_HUNDRED_TWEENTY_EIGHT);	427	vecBaseK_ = (vecBaseK_ + NUM_ALIGN_TO_ONE_HUNDRED_TWEENTY_EIGHT) & (~NUM_ALIGN_TO_ONE_HUNDRED_TWEENTY_EIGHT);
429	- vecBlockDimK_ = Ceil(tilingParams.k, vecBaseK_); // recompute coreNum in K-axis	428	+ vecBlockDimK_ = Ceil(tilingParams.k, vecBaseK_); // recompute coreNum in K-axis
430	- uint32_t vecBlockDimM_ =	429	+ uint32_t vecBlockDimM_ = tilingParams.aivNumPerExpert / vecBlockDimK_; // recompute coreNum in M-axis
431	- tilingParams.aivNumPerExpert / vecBlockDimK_; // recompute coreNum in M-axis	430	+ // recompute singleM and M-axis coreNum
432	- // recompute singleM and M-axis coreNum
433	uint32_t vecSingleM_ = Ceil(tilingParams.mVec, vecBlockDimM_);	431	uint32_t vecSingleM_ = Ceil(tilingParams.mVec, vecBlockDimM_);
434	vecBlockDimM_ = Ceil(tilingParams.mVec, vecSingleM_);	432	vecBlockDimM_ = Ceil(tilingParams.mVec, vecSingleM_);
435	uint32_t vecSingleMTail_ = tilingParams.mVec - (vecBlockDimM_ - 1) * vecSingleM_;	433	uint32_t vecSingleMTail_ = tilingParams.mVec - (vecBlockDimM_ - 1) * vecSingleM_;
	@@ -444,9 +442,8 @@ FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::MM1VectorTiling(Tilin
444	}	442	}
445		443
446	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>	444	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>
447	-__aicore__ inline void	445	+__aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::MM2VectorTiling(
448	-FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::MM2VectorTiling(uint32_t curV2BaseN, uint32_t blockDimK,	446	+ uint32_t curV2BaseN, uint32_t blockDimK, uint32_t v2BaseM, TilingConfig &tilingParams)
449	- uint32_t v2BaseM, TilingConfig &tilingParams)
450	{	447	{
451	tilingParams.vecBaseK = curV2BaseN;	448	tilingParams.vecBaseK = curV2BaseN;
452	tilingParams.vecBaseM = v2BaseM;	449	tilingParams.vecBaseM = v2BaseM;
	@@ -494,9 +491,8 @@ __aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>
494	}	491	}
495		492
496	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>	493	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>
497	-__aicore__ inline void	494	+__aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::CalcTailBlock(
498	-FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::CalcTailBlock(TilingConfig tilingParams,	495	+ TilingConfig tilingParams, uint32_t &curSingleM, uint32_t &curBaseK)
499	- uint32_t &curSingleM, uint32_t &curBaseK)
500	{	496	{
501	if (vec1BlockMIdx_ == tilingParams.vecBlockDimM - 1) {	497	if (vec1BlockMIdx_ == tilingParams.vecBlockDimM - 1) {
502	curSingleM = tilingParams.vecSingleMTail;	498	curSingleM = tilingParams.vecSingleMTail;
	@@ -507,9 +503,8 @@ FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::CalcTailBlock(TilingC
507	}	503	}
508		504
509	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>	505	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>
510	-__aicore__ inline void	506	+__aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::PreProcessMM1(
511	-FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::PreProcessMM1(TilingConfig tilingParams,	507	+ TilingConfig tilingParams, ExpertParallInfo mmExpertParallInfo)
512	- ExpertParallInfo mmExpertParallInfo)
513	{	508	{
514	MM1VectorTiling(tilingParams);	509	MM1VectorTiling(tilingParams);
515	vec1BlockKIdx_ = (coreIdx - expertIdxInParaGroupMM1Pre * tilingParams.aivNumPerExpert) % tilingParams.vecBlockDimK;	510	vec1BlockKIdx_ = (coreIdx - expertIdxInParaGroupMM1Pre * tilingParams.aivNumPerExpert) % tilingParams.vecBlockDimK;
	@@ -745,10 +740,9 @@ __aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>
745	}	740	}
746		741
747	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>	742	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>
748	-__aicore__ inline void	743	+__aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::MM1Compute(
749	-FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::MM1Compute(uint32_t curCubeSingleCoreN, uint32_t offsetN,	744	+ uint32_t curCubeSingleCoreN, uint32_t offsetN, const TilingConfig &tilingParams,
750	- const TilingConfig &tilingParams,	745	+ ExpertParallInfo mmExpertParallInfo)
751	- ExpertParallInfo mmExpertParallInfo)
752	{	746	{
753	uint32_t AMatrixMM1Offset =	747	uint32_t AMatrixMM1Offset =
754	mmExpertParallInfo.LocalOffset[expertIdxInParaGroupMM1] * ANTIQUANT_MSD_STEP * tilingParams.k;	748	mmExpertParallInfo.LocalOffset[expertIdxInParaGroupMM1] * ANTIQUANT_MSD_STEP * tilingParams.k;
	@@ -969,9 +963,8 @@ __aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>
969	}	963	}
970		964
971	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>	965	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>
972	-__aicore__ inline void	966	+__aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::CopyOutFinalResult(
973	-FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::CopyOutFinalResult(uint32_t curV2BaseM, uint32_t curV2BaseN,	967	+ uint32_t curV2BaseM, uint32_t curV2BaseN, uint32_t yOffset, uint32_t n)
974	- uint32_t yOffset, uint32_t n)
975	{	968	{
976	uint32_t curBaseNAligned = (curV2BaseN + NUM_ALIGN_TO_THIRTYTWO) & (~NUM_ALIGN_TO_THIRTYTWO);	969	uint32_t curBaseNAligned = (curV2BaseN + NUM_ALIGN_TO_THIRTYTWO) & (~NUM_ALIGN_TO_THIRTYTWO);
977	LocalTensor<yT> outputInUb = outQueueY_.AllocTensor<yT>();	970	LocalTensor<yT> outputInUb = outQueueY_.AllocTensor<yT>();
	@@ -989,8 +982,8 @@ FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::CopyOutFinalResult(ui
989	}	982	}
990		983
991	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>	984	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>
992	-__aicore__ inline void	985	+__aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::ActivationCompute(
993	-FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::ActivationCompute(uint32_t computeSize)	986	+ uint32_t computeSize)
994	{	987	{
995	uint32_t usedTmpBufferSize =	988	uint32_t usedTmpBufferSize =
996	(countOfFloatUbCalcShape * ubCalcShape_ + maxTokens * FACTOR_FOR_FLOAT_ALIGN_TO_32BYTE) * sizeof(float) +	989	(countOfFloatUbCalcShape * ubCalcShape_ + maxTokens * FACTOR_FOR_FLOAT_ALIGN_TO_32BYTE) * sizeof(float) +
	@@ -1159,15 +1152,14 @@ __aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>
1159	}	1152	}
1160		1153
1161	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>	1154	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>
1162	-__aicore__ inline void	1155	+__aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::ComputeExpertMM2(
1163	-FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::ComputeExpertMM2(TilingConfig &tilingParams2,	1156	+ TilingConfig &tilingParams2, ExpertParallInfo mm2ExpertParallInfo)
1164	- ExpertParallInfo mm2ExpertParallInfo)
1165	{	1157	{
1166	uint32_t curIterCount = 0;	1158	uint32_t curIterCount = 0;
1167	for (uint32_t i = mm2ExpertParallInfo.start; i < mm2ExpertParallInfo.size;	1159	for (uint32_t i = mm2ExpertParallInfo.start; i < mm2ExpertParallInfo.size;
1168	i += mm2ExpertParallInfo.expertParallelism) {	1160	i += mm2ExpertParallInfo.expertParallelism) {
1169	- expertIdxInParaGroupMM2 = Min<uint32_t>(coreIdx / tilingParams2.aicNumPerExpert,	1161	+ expertIdxInParaGroupMM2 =
1170	- mm2ExpertParallInfo.expertParallelism - 1);	1162	+ Min<uint32_t>(coreIdx / tilingParams2.aicNumPerExpert, mm2ExpertParallInfo.expertParallelism - 1);
1171	expertIdxInParaGroupMM2 += curIterCount * mm2ExpertParallInfo.expertParallelism;	1163	expertIdxInParaGroupMM2 += curIterCount * mm2ExpertParallInfo.expertParallelism;
1172	expertIdxInParaGroupMM2 = Min<uint32_t>(expertIdxInParaGroupMM2, mm2ExpertParallInfo.size - 1);	1164	expertIdxInParaGroupMM2 = Min<uint32_t>(expertIdxInParaGroupMM2, mm2ExpertParallInfo.size - 1);
1173	currentExpertMM2 = mm2ExpertParallInfo.expertIdxBuf[expertIdxInParaGroupMM2];	1165	currentExpertMM2 = mm2ExpertParallInfo.expertIdxBuf[expertIdxInParaGroupMM2];
	@@ -1242,9 +1234,8 @@ __aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>
1242	}	1234	}
1243		1235
1244	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>	1236	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>
1245	-__aicore__ inline void	1237	+__aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::ComputeExpert(
1246	-FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::ComputeExpert(ExpertParallInfo mm1ExpertParallInfo,	1238	+ ExpertParallInfo mm1ExpertParallInfo, ExpertParallInfo mm2ExpertParallInfo)
1247	- ExpertParallInfo mm2ExpertParallInfo)
1248	{	1239	{
1249	TilingConfig tilingParams1;	1240	TilingConfig tilingParams1;
1250	TilingConfig tilingParams2;	1241	TilingConfig tilingParams2;
	@@ -1276,9 +1267,8 @@ FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::ComputeExpert(ExpertP
1276	}	1267	}
1277		1268
1278	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>	1269	template <typename xT, typename wT, typename mm1Type, typename mm2Type, typename c1T, typename yT, typename biasT>
1279	-__aicore__ inline void	1270	+__aicore__ inline void FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::ComputeExpertParallNum(
1280	-FFNAntiQuantMSD<xT, wT, mm1Type, mm2Type, c1T, yT, biasT>::ComputeExpertParallNum(const uint32_t expertI,	1271	+ const uint32_t expertI, ExpertParallInfo &expertParallInfo)
1281	- ExpertParallInfo &expertParallInfo)
1282	{	1272	{
1283	if (expertI == expertNum) {	1273	if (expertI == expertNum) {
1284	expertParallInfo.expertParallelism = Min(expertParallInfo.size, expertParallInfo.maxExpertParallelism);	1274	expertParallInfo.expertParallelism = Min(expertParallInfo.size, expertParallInfo.maxExpertParallelism);

Mffn/ffn/op_kernel/ffn_base.h+1-2

	@@ -70,8 +70,7 @@ public:
70	__aicore__ inline FFNBase(typename mm1Type::MT &mm1_, typename mm2Type::MT &mm2_)	70	__aicore__ inline FFNBase(typename mm1Type::MT &mm1_, typename mm2Type::MT &mm2_)
71	: mm1(mm1_),	71	: mm1(mm1_),
72	mm2(mm2_)	72	mm2(mm2_)
73	- {	73	+ {}
74	- }
75		74
76	/** Init function before process function	75	/** Init function before process function
77	* @param x: input 2D matrix.	76	* @param x: input 2D matrix.

Mffn/ffn/op_kernel/ffn_glu.h+1-3

	@@ -24,9 +24,7 @@ using namespace matmul;
24	template <typename T>	24	template <typename T>
25	class FFNGlu {	25	class FFNGlu {
26	public:	26	public:
27	- __aicore__ inline FFNGlu()	27	+ __aicore__ inline FFNGlu() {}
28	- {
29	- }
30	__aicore__ inline void Init(__gm__ uint8_t x, __gm__ uint8_t weight1, __gm__ uint8_t *weight2,	28	__aicore__ inline void Init(__gm__ uint8_t x, __gm__ uint8_t weight1, __gm__ uint8_t *weight2,
31	__gm__ uint8_t bias1, __gm__ uint8_t bias2, __gm__ uint8_t *y,	29	__gm__ uint8_t bias1, __gm__ uint8_t bias2, __gm__ uint8_t *y,
32	__gm__ uint8_t workSpace, const FFNTilingData __restrict tiling, TPipe *tPipe);	30	__gm__ uint8_t workSpace, const FFNTilingData __restrict tiling, TPipe *tPipe);

Mffn/ffn/op_kernel/ffn_high_performence.h+2-3

	@@ -24,8 +24,7 @@ class FFNHighPerformence : public FFNBase<T, mm1Type, mm2Type, c1T, c2T, BiasT>
24	public:	24	public:
25	__aicore__ inline FFNHighPerformence(typename mm1Type::MT &mm1_, typename mm2Type::MT &mm2_)	25	__aicore__ inline FFNHighPerformence(typename mm1Type::MT &mm1_, typename mm2Type::MT &mm2_)
26	: FFNBase<T, mm1Type, mm2Type, c1T, c2T, BiasT>(mm1_, mm2_)	26	: FFNBase<T, mm1Type, mm2Type, c1T, c2T, BiasT>(mm1_, mm2_)
27	- {	27	+ {}
28	- }
29		28
30	/** @brief main entry function.	29	/** @brief main entry function.
31	*/	30	*/
	@@ -41,4 +40,4 @@ public:
41		40
42	} // namespace FFN	41	} // namespace FFN
43		42
44	-#endif // ASCENDC_FFN_HIGH_PERFORMENCE_H	43	+#endif // ASCENDC_FFN_HIGH_PERFORMENCE_H

Mffn/ffn/op_kernel/ffn_high_precision.h+1-2

	@@ -24,8 +24,7 @@ class FFNHighPrecision : public FFNBase<T, mm1Type, mm2Type, c1T, c2T, BiasT> {
24	public:	24	public:
25	__aicore__ inline FFNHighPrecision(typename mm1Type::MT &mm1_, typename mm2Type::MT &mm2_)	25	__aicore__ inline FFNHighPrecision(typename mm1Type::MT &mm1_, typename mm2Type::MT &mm2_)
26	: FFNBase<T, mm1Type, mm2Type, c1T, c2T, BiasT>(mm1_, mm2_)	26	: FFNBase<T, mm1Type, mm2Type, c1T, c2T, BiasT>(mm1_, mm2_)
27	- {	27	+ {}
28	- }
29		28
30	/** @brief main entry function.	29	/** @brief main entry function.
31	*/	30	*/

Mffn/ffn/op_kernel/ffn_kernel.h+11-11

Mffn/ffn/op_kernel/ffn_nonquant_nz.h+8-10

	@@ -41,8 +41,7 @@ public:
41	/** @brief constructor */	41	/** @brief constructor */
42	__aicore__ inline FFNProcess(ComputeType &computeOp_)	42	__aicore__ inline FFNProcess(ComputeType &computeOp_)
43	: computeOp(computeOp_)	43	: computeOp(computeOp_)
44	- {	44	+ {}
45	- }
46		45
47	__aicore__ inline void Init(const FFNTilingData *__restrict tiling);	46	__aicore__ inline void Init(const FFNTilingData *__restrict tiling);
48		47
	@@ -193,8 +192,7 @@ public:
193	__aicore__ inline FFNCompute(typename mm1Type::MT &mm1_, typename mm2Type::MT &mm2_)	192	__aicore__ inline FFNCompute(typename mm1Type::MT &mm1_, typename mm2Type::MT &mm2_)
194	: mm1(mm1_),	193	: mm1(mm1_),
195	mm2(mm2_)	194	mm2(mm2_)
196	- {	195	+ {}
197	- }
198		196
199	/** Init function before process function	197	/** Init function before process function
200	* @param x: input 2D matrix.	198	* @param x: input 2D matrix.
	@@ -288,9 +286,10 @@ __aicore__ inline void FFNCompute<T, mm1Type, mm2Type, c1T, c2T, BiasT>::AllCore
288	}	286	}
289		287
290	template <typename T, class mm1Type, class mm2Type, typename c1T, typename c2T, typename BiasT>	288	template <typename T, class mm1Type, class mm2Type, typename c1T, typename c2T, typename BiasT>
291	-__aicore__ inline void	289	+__aicore__ inline void FFNCompute<T, mm1Type, mm2Type, c1T, c2T, BiasT>::CalcOffset(MNConfig &mnConfig, uint32_t &posN,
292	-FFNCompute<T, mm1Type, mm2Type, c1T, c2T, BiasT>::CalcOffset(MNConfig &mnConfig, uint32_t &posN, uint64_t &xOffset,	290	+ uint64_t &xOffset,
293	- uint64_t &wOffset, uint64_t &outOffset)	291	+ uint64_t &wOffset,
		292	+ uint64_t &outOffset)
294	{	293	{
295	uint32_t posM = mnConfig.mIdx * mnConfig.baseM;	294	uint32_t posM = mnConfig.mIdx * mnConfig.baseM;
296	posN = mnConfig.nIdx * mnConfig.baseN;	295	posN = mnConfig.nIdx * mnConfig.baseN;
	@@ -428,9 +427,8 @@ __aicore__ inline uint32_t FFNCompute<T, mm1Type, mm2Type, c1T, c2T, BiasT>::Cur
428	}	427	}
429		428
430	template <typename T, class mm1Type, class mm2Type, typename c1T, typename c2T, typename BiasT>	429	template <typename T, class mm1Type, class mm2Type, typename c1T, typename c2T, typename BiasT>
431	-__aicore__ inline void	430	+__aicore__ inline void FFNCompute<T, mm1Type, mm2Type, c1T, c2T, BiasT>::ZeroN1WithoutBiasCompute(
432	-FFNCompute<T, mm1Type, mm2Type, c1T, c2T, BiasT>::ZeroN1WithoutBiasCompute(const MNConfig &mnConfig,	431	+ const MNConfig &mnConfig, const uint32_t coreIdx)
433	- const uint32_t coreIdx)
434	{	432	{
435	uint32_t offset = mnConfig.singleN * coreIdx;	433	uint32_t offset = mnConfig.singleN * coreIdx;
436	uint32_t cursingleN = CursingleMNCompute(mnConfig.singleN, mnConfig.n, offset);	434	uint32_t cursingleN = CursingleMNCompute(mnConfig.singleN, mnConfig.n, offset);

Mffn/ffn/tests/comm/inc/ffn_case.h+1-2

	@@ -51,9 +51,8 @@ protected:
51	bool InitCurrentCasePtr() override;	51	bool InitCurrentCasePtr() override;
52	};	52	};
53		53
54	-
55	Tensor GenTensor(const char *name, const std::initializer_list<int64_t> &shape, ge::DataType dType,	54	Tensor GenTensor(const char *name, const std::initializer_list<int64_t> &shape, ge::DataType dType,
56	ge::Format format = ge::FORMAT_ND);	55	ge::Format format = ge::FORMAT_ND);
57		56
58	} // namespace ops::adv::tests::ffn	57	} // namespace ops::adv::tests::ffn
59	-#endif // UTEST_FFN_CASE_H	58	+#endif // UTEST_FFN_CASE_H

Mffn/ffn/tests/utest/ts_ffn.h+1-1

	@@ -37,4 +37,4 @@ class Ts_FFN_WithParam_Ascend910B3 : public Ts_WithParam_Ascend910B3<FFNCase> {}
37		37
38	class Ts_FFN_WithParam_Ascend310P3 : public Ts_WithParam_Ascend310P3<FFNCase> {};	38	class Ts_FFN_WithParam_Ascend310P3 : public Ts_WithParam_Ascend310P3<FFNCase> {};
39		39
40	-#endif // UTEST_TS_FFN_H	40	+#endif // UTEST_TS_FFN_H

Mffn/ffn/tests/utest_aclnn/ts_aclnn_ffn.h+1-1

	@@ -31,4 +31,4 @@ class Ts_Aclnn_FFN_WithParam_Ascend910B1 : public Ts_WithParam_Ascend910B1<Aclnn
31	class Ts_Aclnn_FFN_WithParam_Ascend910B2 : public Ts_WithParam_Ascend910B2<AclnnFFNCase> {};	31	class Ts_Aclnn_FFN_WithParam_Ascend910B2 : public Ts_WithParam_Ascend910B2<AclnnFFNCase> {};
32	class Ts_Aclnn_FFN_WithParam_Ascend910B3 : public Ts_WithParam_Ascend910B3<AclnnFFNCase> {};	32	class Ts_Aclnn_FFN_WithParam_Ascend910B3 : public Ts_WithParam_Ascend910B3<AclnnFFNCase> {};
33		33
34	-#endif // UTEST_TS_FFN_H	34	+#endif // UTEST_TS_FFN_H

Mffn/ffn_worker_batching/op_host/ffn_worker_batching_tiling.h+5-6

	@@ -40,13 +40,12 @@ END_TILING_DATA_DEF;
40		40
41	REGISTER_TILING_DATA_CLASS(FfnWorkerBatching, FfnWorkerBatchingTilingData)	41	REGISTER_TILING_DATA_CLASS(FfnWorkerBatching, FfnWorkerBatchingTilingData)
42		42
43	-struct FfnWorkerBatchingCompileInfo {	43	+struct FfnWorkerBatchingCompileInfo {};
44	-};
45		44
46	-class FfnWorkerBatchingTiling	45	+class FfnWorkerBatchingTiling {
47	-{
48	public:	46	public:
49	- explicit FfnWorkerBatchingTiling(gert::TilingContext* context) : context_(context){};	47	+ explicit FfnWorkerBatchingTiling(gert::TilingContext *context)
		48	+ : context_(context) {};
50	ge::graphStatus RunFfnWorkerBatchingTiling();	49	ge::graphStatus RunFfnWorkerBatchingTiling();
51		50
52	private:	51	private:
	@@ -70,4 +69,4 @@ private:
70	};	69	};
71		70
72	} // namespace optiling	71	} // namespace optiling
73	-#endif // OP_HOST_FFN_WORKER_BATCHING_TILING_H	72	+#endif // OP_HOST_FFN_WORKER_BATCHING_TILING_H

Mffn/ffn_worker_batching/op_kernel/arch35/ffn_wb_a5_gather.h+1-3

	@@ -48,9 +48,7 @@ constexpr int64_t IDX_I_LANES = 3; // aIdx/bsIdx/kIdx
48	template <bool isScanFlag = false>	48	template <bool isScanFlag = false>
49	class FfnWbA5Gather {	49	class FfnWbA5Gather {
50	public:	50	public:
51	- __aicore__ inline FfnWbA5Gather()	51	+ __aicore__ inline FfnWbA5Gather() {}
52	- {
53	- }
54	__aicore__ inline void Init(GM_ADDR expertid_idx, GM_ADDR y, GM_ADDR session_ids, GM_ADDR micro_batch_ids,	52	__aicore__ inline void Init(GM_ADDR expertid_idx, GM_ADDR y, GM_ADDR session_ids, GM_ADDR micro_batch_ids,
55	GM_ADDR token_ids, GM_ADDR expert_offsets, GM_ADDR dynamic_scale,	53	GM_ADDR token_ids, GM_ADDR expert_offsets, GM_ADDR dynamic_scale,
56	const ScheduleContextInfo contextInfo, TPipe pipe, uint32_t usedCoreNum)	54	const ScheduleContextInfo contextInfo, TPipe pipe, uint32_t usedCoreNum)

Mffn/ffn_worker_batching/op_kernel/arch35/ffn_wb_a5_sort.h+3-3

	@@ -259,9 +259,9 @@ public:
259	const int64_t loopFloats = GetSortLen<float>(param_.oneLoopMaxElements);	259	const int64_t loopFloats = GetSortLen<float>(param_.oneLoopMaxElements);
260	pipe_->InitBuffer(inQue_, 1, loopFloats * MRG_LIST_NUM * sizeof(float) + ONE_BLK_SIZE);	260	pipe_->InitBuffer(inQue_, 1, loopFloats * MRG_LIST_NUM * sizeof(float) + ONE_BLK_SIZE);
261	pipe_->InitBuffer(outQue_, 1, loopFloats * MRG_LIST_NUM * sizeof(float) + ONE_BLK_SIZE);	261	pipe_->InitBuffer(outQue_, 1, loopFloats * MRG_LIST_NUM * sizeof(float) + ONE_BLK_SIZE);
262	- pipe_->InitBuffer(cntBuf_,	262	+ pipe_->InitBuffer(
263	- Align(param_.segNum * (ONE_BLK_SIZE / sizeof(int32_t)), sizeof(int32_t)) * sizeof(int32_t) +	263	+ cntBuf_,
264	- ONE_BLK_SIZE);	264	+ Align(param_.segNum * (ONE_BLK_SIZE / sizeof(int32_t)), sizeof(int32_t)) * sizeof(int32_t) + ONE_BLK_SIZE);
265	}	265	}
266		266
267	// 跑完全部归并轮次;返回最终结果所在的工作区序号(0=wsA,1=wsB)。	267	// 跑完全部归并轮次;返回最终结果所在的工作区序号(0=wsA,1=wsB)。

Mffn/ffn_worker_batching/op_kernel/ffn_wb_common.h+14-15

	@@ -189,7 +189,7 @@ __aicore__ inline T PowerOfFourCeil(T x)
189	return result;	189	return result;
190	}	190	}
191		191
192	-__aicore__ inline void TilingOneCoreSort(SortCustomTilingDataKernel* tilingData, const int64_t totalLength)	192	+__aicore__ inline void TilingOneCoreSort(SortCustomTilingDataKernel *tilingData, const int64_t totalLength)
193	{	193	{
194	tilingData->needCoreNum = totalLength == 0 ? 0 : 1;	194	tilingData->needCoreNum = totalLength == 0 ? 0 : 1;
195	tilingData->perCoreElements = totalLength;	195	tilingData->perCoreElements = totalLength;
	@@ -202,8 +202,8 @@ __aicore__ inline void TilingOneCoreSort(SortCustomTilingDataKernel* tilingData,
202	tilingData->lastCoreLastLoopElements = totalLength;	202	tilingData->lastCoreLastLoopElements = totalLength;
203	}	203	}
204		204
205	-__aicore__ inline void TilingMultiCoreSort(	205	+__aicore__ inline void TilingMultiCoreSort(SortCustomTilingDataKernel *tilingData, const int64_t totalLength,
206	- SortCustomTilingDataKernel* tilingData, const int64_t totalLength, const ScheduleContextInfo* contextInfo)	206	+ const ScheduleContextInfo *contextInfo)
207	{	207	{
208	int64_t needCoreNum = CeilDiv(totalLength, contextInfo->sortLoopMaxElement); // 向上取整	208	int64_t needCoreNum = CeilDiv(totalLength, contextInfo->sortLoopMaxElement); // 向上取整
209	needCoreNum = PowerOfFourCeil(needCoreNum); // 用到多核时，核数最多是4^x, 计算不小于	209	needCoreNum = PowerOfFourCeil(needCoreNum); // 用到多核时，核数最多是4^x, 计算不小于
	@@ -254,9 +254,8 @@ __aicore__ inline void TilingMultiCoreSort(
254	tilingData->oneLoopMaxElementsMrg = MGR_SORT_MAX_ELEMENT;	254	tilingData->oneLoopMaxElementsMrg = MGR_SORT_MAX_ELEMENT;
255	}	255	}
256		256
257	-__aicore__ inline void TilingScanMultiCoreSort(	257	+__aicore__ inline void TilingScanMultiCoreSort(SortCustomTilingDataKernel *tilingdataSort,
258	- SortCustomTilingDataKernel* tilingdataSort,	258	+ const ScheduleContextInfo *contextInfo)
259	- const ScheduleContextInfo* contextInfo)
260	{	259	{
261	// 按A分核	260	// 按A分核
262	// 1. 计算每次排序可以放下多少个 session_num	261	// 1. 计算每次排序可以放下多少个 session_num
	@@ -283,7 +282,7 @@ __aicore__ inline void TilingScanMultiCoreSort(
283	tilingdataSort->oneLoopMaxElementsMrg = MGR_SORT_MAX_ELEMENT;	282	tilingdataSort->oneLoopMaxElementsMrg = MGR_SORT_MAX_ELEMENT;
284	}	283	}
285		284
286	-__aicore__ inline void TilingSort(SortCustomTilingDataKernel* tilingdataSort, const ScheduleContextInfo* contextInfo)	285	+__aicore__ inline void TilingSort(SortCustomTilingDataKernel tilingdataSort, const ScheduleContextInfo contextInfo)
287	{	286	{
288	auto totalLength = contextInfo->outNum * contextInfo->BS * contextInfo->K;	287	auto totalLength = contextInfo->outNum * contextInfo->BS * contextInfo->K;
289	tilingdataSort->totalLength = totalLength;	288	tilingdataSort->totalLength = totalLength;
	@@ -297,8 +296,8 @@ __aicore__ inline void TilingSort(SortCustomTilingDataKernel* tilingdataSort, co
297	}	296	}
298	}	297	}
299		298
300	-__aicore__ inline void TilingScanSort(	299	+__aicore__ inline void TilingScanSort(SortCustomTilingDataKernel *tilingdataSort,
301	- SortCustomTilingDataKernel* tilingdataSort, const ScheduleContextInfo* contextInfo)	300	+ const ScheduleContextInfo *contextInfo)
302	{	301	{
303	tilingdataSort->totalLengthWithPad =	302	tilingdataSort->totalLengthWithPad =
304	contextInfo->A * (contextInfo->BS * contextInfo->K + contextInfo->BsKPaddingCount);	303	contextInfo->A * (contextInfo->BS * contextInfo->K + contextInfo->BsKPaddingCount);
	@@ -316,14 +315,14 @@ __aicore__ inline void TilingScanSort(
316	}	315	}
317	}	316	}
318		317
319	-__aicore__ inline void ValidGatherIdxLengthCompute(	318	+__aicore__ inline void ValidGatherIdxLengthCompute(GM_ADDR work_space, ScheduleContextInfo &contextInfo,
320	- GM_ADDR work_space, ScheduleContextInfo& contextInfo, GM_ADDR actual_token_num)	319	+ GM_ADDR actual_token_num)
321	{	320	{
322	GlobalTensor<int32_t> workSpace;	321	GlobalTensor<int32_t> workSpace;
323	GlobalTensor<int64_t> actualTokenNumGm;	322	GlobalTensor<int64_t> actualTokenNumGm;
324		323
325	- workSpace.SetGlobalBuffer((__gm__ int32_t*)work_space, OFFSET_SORTED_EXPERT_IDS);	324	+ workSpace.SetGlobalBuffer((__gm__ int32_t *)work_space, OFFSET_SORTED_EXPERT_IDS);
326	- actualTokenNumGm.SetGlobalBuffer((__gm__ int64_t*)actual_token_num);	325	+ actualTokenNumGm.SetGlobalBuffer((__gm__ int64_t *)actual_token_num);
327		326
328	DataCacheCleanAndInvalid<int32_t, AscendC::CacheLine::SINGLE_CACHE_LINE, AscendC::DcciDst::CACHELINE_ALL>(	327	DataCacheCleanAndInvalid<int32_t, AscendC::CacheLine::SINGLE_CACHE_LINE, AscendC::DcciDst::CACHELINE_ALL>(
329	workSpace);	328	workSpace);
	@@ -336,8 +335,8 @@ __aicore__ inline void ValidGatherIdxLengthCompute(
336	}	335	}
337	}	336	}
338		337
339	-__aicore__ inline void Tiling4SrcToDstCompute(	338	+__aicore__ inline void Tiling4SrcToDstCompute(const ScheduleContextInfo *tilingData,
340	- const ScheduleContextInfo* tilingData, FfnWBGroupListingTileInfo& tilingInfo)	339	+ FfnWBGroupListingTileInfo &tilingInfo)
341	{	340	{
342	int64_t ubSizePlatForm = tilingData->ubSize;	341	int64_t ubSizePlatForm = tilingData->ubSize;
343	int64_t validGatherIdxLength = tilingData->validGatherIdxLength;	342	int64_t validGatherIdxLength = tilingData->validGatherIdxLength;

Mffn/ffn_worker_batching/op_kernel/ffn_wb_group_listing.h+47-37

Mffn/ffn_worker_batching/op_kernel/ffn_wb_scan_get_valid_experts.h+45-53

	@@ -27,13 +27,11 @@
27	namespace FfnWbBatching {	27	namespace FfnWbBatching {
28	using namespace AscendC;	28	using namespace AscendC;
29		29
30	-class KernelScanGetValidExperts : public SortMaskBase	30	+class KernelScanGetValidExperts : public SortMaskBase {
31	-{
32	public:	31	public:
33	__aicore__ inline KernelScanGetValidExperts(){};	32	__aicore__ inline KernelScanGetValidExperts(){};
34	- __aicore__ inline void Init(	33	+ __aicore__ inline void Init(GM_ADDR tokenInfoGm, GM_ADDR workspace, GM_ADDR groupList,
35	- GM_ADDR tokenInfoGm, GM_ADDR workspace, GM_ADDR groupList, const ScheduleContextInfo* contextInfo,	34	+ const ScheduleContextInfo contextInfo, TPipe tPipe);
36	- TPipe* tPipe);
37	__aicore__ inline void Process();	35	__aicore__ inline void Process();
38		36
39	private:	37	private:
	@@ -55,7 +53,7 @@ private:
55		53
56	GlobalTensor<int64_t> groupListGm_;	54	GlobalTensor<int64_t> groupListGm_;
57		55
58	- const ScheduleContextInfo* contextInfo_ = nullptr;	56	+ const ScheduleContextInfo *contextInfo_ = nullptr;
59		57
60	int32_t totalValidCnt_ = 0;	58	int32_t totalValidCnt_ = 0;
61	int32_t curValidCnt_ = 0;	59	int32_t curValidCnt_ = 0;
	@@ -64,7 +62,7 @@ private:
64	int64_t BsKLenWithPading_ = 0; // BS*K_plus_1按block对齐后的个数	62	int64_t BsKLenWithPading_ = 0; // BS*K_plus_1按block对齐后的个数
65		63
66	int64_t perCoreSortNum_ = 0; // perCoreSessionNum * BsKLenWithPading_ 主核每个核排序的元素总个数; 不保证32个数对齐	64	int64_t perCoreSortNum_ = 0; // perCoreSessionNum * BsKLenWithPading_ 主核每个核排序的元素总个数; 不保证32个数对齐
67	- int64_t sessionLoops_ = 0; // 当前核 ub循环次数	65	+ int64_t sessionLoops_ = 0; // 当前核 ub循环次数
68		66
69	int64_t perLoopSessionNum_ = 0; // 当前核一次ub处理A中的几个	67	int64_t perLoopSessionNum_ = 0; // 当前核一次ub处理A中的几个
70	int64_t lastLoopSessionNum_ = 0; // 当前核尾ub处理A中的几个	68	int64_t lastLoopSessionNum_ = 0; // 当前核尾ub处理A中的几个
	@@ -80,8 +78,8 @@ private:
80	int64_t curCoreSessionNum_ = 0;	78	int64_t curCoreSessionNum_ = 0;
81	};	79	};
82		80
83	-__aicore__ inline void KernelScanGetValidExperts::VBSCopyInAndClear(	81	+__aicore__ inline void KernelScanGetValidExperts::VBSCopyInAndClear(int64_t progress, int64_t size,
84	- int64_t progress, int64_t size, int64_t loopSessionCnt)	82	+ int64_t loopSessionCnt)
85	{	83	{
86	LocalTensor<int32_t> inLocal = sortDataCopyInQueue.AllocTensor<int32_t>();	84	LocalTensor<int32_t> inLocal = sortDataCopyInQueue.AllocTensor<int32_t>();
87	int64_t inOffset = progress * perLoopSessionNum_ * contextInfo_->M * F_;	85	int64_t inOffset = progress * perLoopSessionNum_ * contextInfo_->M * F_;
	@@ -89,8 +87,8 @@ __aicore__ inline void KernelScanGetValidExperts::VBSCopyInAndClear(
89	static_cast<uint16_t>(loopSessionCnt),	87	static_cast<uint16_t>(loopSessionCnt),
90	static_cast<uint32_t>(contextInfo_->BS * contextInfo_->K * sizeof(int32_t)),	88	static_cast<uint32_t>(contextInfo_->BS * contextInfo_->K * sizeof(int32_t)),
91	static_cast<uint32_t>((contextInfo_->M * F_ - contextInfo_->BS * contextInfo_->K) * sizeof(int32_t)), 0, 0};	89	static_cast<uint32_t>((contextInfo_->M * F_ - contextInfo_->BS * contextInfo_->K) * sizeof(int32_t)), 0, 0};
92	- DataCopyPadExtParams<int32_t> dataCopyPadParams{	90	+ DataCopyPadExtParams<int32_t> dataCopyPadParams{true, 0, static_cast<uint8_t>(contextInfo_->BsKPaddingCount),
93	- true, 0, static_cast<uint8_t>(contextInfo_->BsKPaddingCount), INT_MAX};	91	+ INT_MAX};
94	DataCopyPad(inLocal[0], expertIdsGm[inOffset], dataCopyParams, dataCopyPadParams);	92	DataCopyPad(inLocal[0], expertIdsGm[inOffset], dataCopyParams, dataCopyPadParams);
95		93
96	int64_t interVal = bufferSize_ / NUM_TWO / sizeof(int32_t);	94	int64_t interVal = bufferSize_ / NUM_TWO / sizeof(int32_t);
	@@ -132,9 +130,9 @@ __aicore__ inline void KernelScanGetValidExperts::UBSortCompute(int64_t progress
132	PipeBarrier<PIPE_V>();	130	PipeBarrier<PIPE_V>();
133		131
134	LocalTensor<uint8_t> maskLocalTensorUInt8 = maskLocalTensor.ReinterpretCast<uint8_t>();	132	LocalTensor<uint8_t> maskLocalTensorUInt8 = maskLocalTensor.ReinterpretCast<uint8_t>();
135	- AscendC::CompareScalar(	133	+ AscendC::CompareScalar(maskLocalTensorUInt8, expertIdsLocalFp32, static_cast<float>(-expertStart_),
136	- maskLocalTensorUInt8, expertIdsLocalFp32, static_cast<float>(-expertStart_), AscendC::CMPMODE::GT,	134	+ AscendC::CMPMODE::GT,
137	- (size + ONE_REPEAT_COMPARE_NUM - 1) / ONE_REPEAT_COMPARE_NUM * ONE_REPEAT_COMPARE_NUM);	135	+ (size + ONE_REPEAT_COMPARE_NUM - 1) / ONE_REPEAT_COMPARE_NUM * ONE_REPEAT_COMPARE_NUM);
138	PipeBarrier<PIPE_V>();	136	PipeBarrier<PIPE_V>();
139		137
140	GatherMaskParams gatherMaskParams;	138	GatherMaskParams gatherMaskParams;
	@@ -218,19 +216,17 @@ __aicore__ inline void KernelScanGetValidExperts::GatherOutProcess()
218	srcSortNumGm[0]);	216	srcSortNumGm[0]);
219	int32_t vNum = srcSortNumGm.GetValue(0);	217	int32_t vNum = srcSortNumGm.GetValue(0);
220	if (vNum > 0) {	218	if (vNum > 0) {
221	- DataCopyExtParams dataCopyParams{	219	+ DataCopyExtParams dataCopyParams{static_cast<uint16_t>(1),
222	- static_cast<uint16_t>(1), static_cast<uint32_t>(vNum * sizeof(int32_t)), 0, 0, 0};	220	+ static_cast<uint32_t>(vNum * sizeof(int32_t)), 0, 0, 0};
223	int32_t padCnt = Align(vNum, sizeof(int32_t)) - vNum;	221	int32_t padCnt = Align(vNum, sizeof(int32_t)) - vNum;
224	// 太大，不支持，走排序流程	222	// 太大，不支持，走排序流程
225	ASSERT_MSG((inOffset + vNum + padCnt) <= interVal, "gather valid num too big");	223	ASSERT_MSG((inOffset + vNum + padCnt) <= interVal, "gather valid num too big");
226	DataCopyPadExtParams<int32_t> dataCopyPadParams{true, 0, static_cast<uint8_t>(padCnt), INT_MAX};	224	DataCopyPadExtParams<int32_t> dataCopyPadParams{true, 0, static_cast<uint8_t>(padCnt), INT_MAX};
227		225
228	- DataCopyPad(	226	+ DataCopyPad(inLocal[inOffset], workspaceGms[0][i * GetSortLen<float>(perCoreSortNum_)], dataCopyParams,
229	- inLocal[inOffset], workspaceGms[0][i * GetSortLen<float>(perCoreSortNum_)], dataCopyParams,	227	+ dataCopyPadParams);
230	- dataCopyPadParams);	228	+ DataCopyPad(inLocal[interVal + inOffset], workspaceGms[1][i * GetSortLen<float>(perCoreSortNum_)],
231	- DataCopyPad(	229	+ dataCopyParams, dataCopyPadParams);
232	- inLocal[interVal + inOffset], workspaceGms[1][i * GetSortLen<float>(perCoreSortNum_)],
233	- dataCopyParams, dataCopyPadParams);
234		230
235	inOffset += vNum + padCnt;	231	inOffset += vNum + padCnt;
236	}	232	}
	@@ -292,8 +288,8 @@ __aicore__ inline void KernelScanGetValidExperts::GatherOutProcess()
292	(rsvdCnt + ONE_REPEAT_COMPARE_NUM - 1) / ONE_REPEAT_COMPARE_NUM * ONE_REPEAT_COMPARE_NUM);	288	(rsvdCnt + ONE_REPEAT_COMPARE_NUM - 1) / ONE_REPEAT_COMPARE_NUM * ONE_REPEAT_COMPARE_NUM);
293	PipeBarrier<PIPE_V>();	289	PipeBarrier<PIPE_V>();
294	GatherMask(outLocal, inLocal, maskLocalUInt32, true, rsvdCnt, gatherMaskParams, rsvdCnt2);	290	GatherMask(outLocal, inLocal, maskLocalUInt32, true, rsvdCnt, gatherMaskParams, rsvdCnt2);
295	- GatherMask(	291	+ GatherMask(outLocal[interVal], inLocal[interVal], maskLocalUInt32, true, rsvdCnt, gatherMaskParams,
296	- outLocal[interVal], inLocal[interVal], maskLocalUInt32, true, rsvdCnt, gatherMaskParams, rsvdCnt2);	292	+ rsvdCnt2);
297	PipeBarrier<PIPE_V>();	293	PipeBarrier<PIPE_V>();
298	SetWaitFlag<HardEvent::V_S>(HardEvent::V_S);	294	SetWaitFlag<HardEvent::V_S>(HardEvent::V_S);
299	groupListOutLocal.SetValue(curExpertIdOffset, i);	295	groupListOutLocal.SetValue(curExpertIdOffset, i);
	@@ -303,8 +299,8 @@ __aicore__ inline void KernelScanGetValidExperts::GatherOutProcess()
303	sortDataCopyOutQueue.EnQue(outLocal);	299	sortDataCopyOutQueue.EnQue(outLocal);
304	outLocal = sortDataCopyOutQueue.DeQue<int32_t>();	300	outLocal = sortDataCopyOutQueue.DeQue<int32_t>();
305		301
306	- DataCopyExtParams dataCopyParams{	302	+ DataCopyExtParams dataCopyParams{static_cast<uint16_t>(1),
307	- static_cast<uint16_t>(1), static_cast<uint32_t>(rsvdCnt2 * sizeof(int32_t)), 0, 0, 0};	303	+ static_cast<uint32_t>(rsvdCnt2 * sizeof(int32_t)), 0, 0, 0};
308	DataCopyPad(sortedexpertIdsGm[outOffset], outLocal, dataCopyParams);	304	DataCopyPad(sortedexpertIdsGm[outOffset], outLocal, dataCopyParams);
309	DataCopyPad(sortedRowIdsGm[outOffset], outLocal[interVal], dataCopyParams);	305	DataCopyPad(sortedRowIdsGm[outOffset], outLocal[interVal], dataCopyParams);
310	outOffset += rsvdCnt2;	306	outOffset += rsvdCnt2;
	@@ -317,8 +313,8 @@ __aicore__ inline void KernelScanGetValidExperts::GatherOutProcess()
317	curExpertIdOffset += NUM_TWO;	313	curExpertIdOffset += NUM_TWO;
318	}	314	}
319		315
320	- DataCopyExtParams copyParams{	316	+ DataCopyExtParams copyParams{static_cast<uint16_t>(1),
321	- static_cast<uint16_t>(1), static_cast<uint32_t>(curExpertIdOffset * sizeof(int64_t)), 0, 0, 0};	317	+ static_cast<uint32_t>(curExpertIdOffset * sizeof(int64_t)), 0, 0, 0};
322	SetWaitFlag<HardEvent::S_MTE3>(HardEvent::S_MTE3);	318	SetWaitFlag<HardEvent::S_MTE3>(HardEvent::S_MTE3);
323	DataCopyPad(groupListGm_, groupListOutLocal, copyParams);	319	DataCopyPad(groupListGm_, groupListOutLocal, copyParams);
324		320
	@@ -348,8 +344,8 @@ __aicore__ inline void KernelScanGetValidExperts::CopyOutValidCount()
348	sortDataCopyOutQueue.FreeTensor(outLocal);	344	sortDataCopyOutQueue.FreeTensor(outLocal);
349	}	345	}
350		346
351	-__aicore__ inline void KernelScanGetValidExperts::Init(	347	+__aicore__ inline void KernelScanGetValidExperts::Init(GM_ADDR tokenInfoGm, GM_ADDR workspace, GM_ADDR groupList,
352	- GM_ADDR tokenInfoGm, GM_ADDR workspace, GM_ADDR groupList, const ScheduleContextInfo* contextInfo, TPipe* tPipe)	348	+ const ScheduleContextInfo contextInfo, TPipe tPipe)
353	{	349	{
354	this->pipe = tPipe;	350	this->pipe = tPipe;
355	contextInfo_ = contextInfo;	351	contextInfo_ = contextInfo;
	@@ -390,33 +386,30 @@ __aicore__ inline void KernelScanGetValidExperts::Init(
390	}	386	}
391		387
392	int64_t expertIdStartPos = contextInfo_->curMicroBatchID * F_ + 1 + 1;	388	int64_t expertIdStartPos = contextInfo_->curMicroBatchID * F_ + 1 + 1;
393	- expertIdsGmFStart_.SetGlobalBuffer((__gm__ int32_t)tokenInfoGm + contextInfo_->curMicroBatchID F_);	389	+ expertIdsGmFStart_.SetGlobalBuffer((__gm__ int32_t )tokenInfoGm + contextInfo_->curMicroBatchID F_);
394	- expertIdsGm.SetGlobalBuffer(	390	+ expertIdsGm.SetGlobalBuffer((__gm__ int32_t *)tokenInfoGm + expertIdStartPos +
395	- (__gm__ int32_t)tokenInfoGm + expertIdStartPos + blockIdx_ perCoreSessionNum_ * contextInfo_->M * F_);	391	+ blockIdx_ * perCoreSessionNum_ * contextInfo_->M * F_);
396	// rsvdCntGm 在scan阶段已经清零	392	// rsvdCntGm 在scan阶段已经清零
397	- rsvdCntGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t*>(workspace), SCAN_BATCHID_GM_OFFSET);	393	+ rsvdCntGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace), SCAN_BATCHID_GM_OFFSET);
398	- workspaceSortNumGm_.SetGlobalBuffer(	394	+ workspaceSortNumGm_.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS,
399	- reinterpret_cast<__gm__ int32_t*>(workspace) + OFFSET_SORTED_EXPERT_IDS, contextInfo_->sortNumWorkSpace);	395	+ contextInfo_->sortNumWorkSpace);
400		396
401	sortedexpertIdsGm.SetGlobalBuffer(	397	sortedexpertIdsGm.SetGlobalBuffer(
402	- reinterpret_cast<__gm__ int32_t*>(workspace) + OFFSET_SORTED_EXPERT_IDS + contextInfo_->sortNumWorkSpace,	398	+ reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS + contextInfo_->sortNumWorkSpace,
403	- this->totalLength);
404	- sortedRowIdsGm.SetGlobalBuffer(
405	- reinterpret_cast<__gm__ int32_t*>(workspace) + OFFSET_SORTED_EXPERT_IDS + contextInfo_->sortNumWorkSpace +
406	- this->totalLength,
407	this->totalLength);	399	this->totalLength);
		400	+ sortedRowIdsGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS +
		401	+ contextInfo_->sortNumWorkSpace + this->totalLength,
		402	+ this->totalLength);
408		403
409	- groupListGm_.SetGlobalBuffer((__gm__ int64_t)groupList, contextInfo_->expertNum NUM_TWO);	404	+ groupListGm_.SetGlobalBuffer((__gm__ int64_t )groupList, contextInfo_->expertNum NUM_TWO);
410		405
411	// key and value	406	// key and value
412	- workspaceGms[0].SetGlobalBuffer(	407	+ workspaceGms[0].SetGlobalBuffer((__gm__ int32_t *)workspace + OFFSET_SORTED_EXPERT_IDS +
413	- (__gm__ int32_t*)workspace + OFFSET_SORTED_EXPERT_IDS + contextInfo_->sortNumWorkSpace +	408	+ contextInfo_->sortNumWorkSpace + this->totalLength * NUM_TWO,
414	- this->totalLength * NUM_TWO,	409	+ this->totalLength * NUM_TWO);
415	- this->totalLength * NUM_TWO);	410	+ workspaceGms[1].SetGlobalBuffer((__gm__ int32_t *)workspace + OFFSET_SORTED_EXPERT_IDS +
416	- workspaceGms[1].SetGlobalBuffer(	411	+ contextInfo_->sortNumWorkSpace + this->totalLength * (NUM_TWO + NUM_TWO),
417	- (__gm__ int32_t*)workspace + OFFSET_SORTED_EXPERT_IDS + contextInfo_->sortNumWorkSpace +	412	+ this->totalLength * NUM_TWO);
418	- this->totalLength * (NUM_TWO + NUM_TWO),
419	- this->totalLength * NUM_TWO);
420		413
421	bufferSize_ =	414	bufferSize_ =
422	Ceil(contextInfo_->sortLoopMaxElement, ONE_REPEAT_SORT_NUM) * ONE_REPEAT_SORT_NUM * sizeof(int32_t) * NUM_TWO;	415	Ceil(contextInfo_->sortLoopMaxElement, ONE_REPEAT_SORT_NUM) * ONE_REPEAT_SORT_NUM * sizeof(int32_t) * NUM_TWO;
	@@ -446,9 +439,8 @@ __aicore__ inline void KernelScanGetValidExperts::ClearTokenInfoFlag()
446	curElementA = lastLoopANum;	439	curElementA = lastLoopANum;
447	}	440	}
448		441
449	- DataCopyExtParams copyOutParams{	442	+ DataCopyExtParams copyOutParams{static_cast<uint16_t>(curElementA), static_cast<uint32_t>(sizeof(int32_t)),
450	- static_cast<uint16_t>(curElementA), static_cast<uint32_t>(sizeof(int32_t)), 0,	443	+ 0, static_cast<uint32_t>((contextInfo_->M * F_ - 1) * sizeof(int32_t)), 0};
451	- static_cast<uint32_t>((contextInfo_->M * F_ - 1) * sizeof(int32_t)), 0};
452	DataCopyPad(expertIdsGmFStart_[idx * perLoopANum * contextInfo_->M * F_], clearLocal, copyOutParams);	444	DataCopyPad(expertIdsGmFStart_[idx * perLoopANum * contextInfo_->M * F_], clearLocal, copyOutParams);
453	}	445	}
454	}	446	}

Mffn/ffn_worker_batching/op_kernel/ffn_wb_scan_group_listing_one_core.h+27-25

	@@ -15,9 +15,9 @@
15	*/	15	*/
16		16
17	/*!	17	/*!
18	-* \file ffn_wb_scan_group_listing_one_core.h	18	+ * \file ffn_wb_scan_group_listing_one_core.h
19	-* \brief	19	+ * \brief
20	-*/	20	+ */
21		21
22	#ifndef OP_KERNEL_FFN_WB_SCAN_GROUP_LISTING_ONE_CORE_H	22	#ifndef OP_KERNEL_FFN_WB_SCAN_GROUP_LISTING_ONE_CORE_H
23	#define OP_KERNEL_FFN_WB_SCAN_GROUP_LISTING_ONE_CORE_H	23	#define OP_KERNEL_FFN_WB_SCAN_GROUP_LISTING_ONE_CORE_H
	@@ -25,16 +25,16 @@
25	#include "ffn_wb_common.h"	25	#include "ffn_wb_common.h"
26	#include "kernel_operator.h"	26	#include "kernel_operator.h"
27		27
28	-namespace FfnWbBatching{	28	+namespace FfnWbBatching {
29	using namespace AscendC;	29	using namespace AscendC;
30		30
31	class KernelFfnWBScanGroupListingOneCore {	31	class KernelFfnWBScanGroupListingOneCore {
32	public:	32	public:
33	- __aicore__ inline KernelFfnWBScanGroupListingOneCore() {};	33	+ __aicore__ inline KernelFfnWBScanGroupListingOneCore(){};
34	- __aicore__ inline void Init(GM_ADDR x, GM_ADDR groupList, ScheduleContextInfo scheduleContext, TPipe tPipe) {	34	+ __aicore__ inline void Init(GM_ADDR x, GM_ADDR groupList, ScheduleContextInfo scheduleContext, TPipe tPipe)
		35	+ {
35	int64_t outQueSize = 1024 * NUM_TWO * sizeof(int64_t) * NUM_TWO; // 1024个数, key & value, doblebuffer	36	int64_t outQueSize = 1024 * NUM_TWO * sizeof(int64_t) * NUM_TWO; // 1024个数, key & value, doblebuffer
36	- int64_t ubMaxRows = Align((scheduleContext->ubSize - outQueSize) / sizeof(int32_t) / NUM_TWO,	37	+ int64_t ubMaxRows = Align((scheduleContext->ubSize - outQueSize) / sizeof(int32_t) / NUM_TWO, sizeof(int32_t));
37	- sizeof(int32_t));
38	int64_t validGatherIdxLength = scheduleContext->validGatherIdxLength;	38	int64_t validGatherIdxLength = scheduleContext->validGatherIdxLength;
39		39
40	perLoopRows = Min(8192L, ubMaxRows); // max 8192	40	perLoopRows = Min(8192L, ubMaxRows); // max 8192
	@@ -50,12 +50,14 @@ public:
50	groupListGm.SetGlobalBuffer((__gm__ int64_t )groupList, expertNum NUM_TWO);	50	groupListGm.SetGlobalBuffer((__gm__ int64_t )groupList, expertNum NUM_TWO);
51		51
52	pipe->InitBuffer(copyInQueue, 1, perLoopRows * sizeof(int32_t));	52	pipe->InitBuffer(copyInQueue, 1, perLoopRows * sizeof(int32_t));
53	- pipe->InitBuffer(copyOutQueue, 1, 1024 * NUM_TWO * sizeof(int64_t) + 32); // 1024个数, 多留32Bytes给最后的输出加上[0, 0]	53	+ pipe->InitBuffer(copyOutQueue, 1,
		54	+ 1024 * NUM_TWO * sizeof(int64_t) + 32); // 1024个数, 多留32Bytes给最后的输出加上[0, 0]
54	pipe->InitBuffer(maskBuf, perLoopRows * sizeof(int32_t));	55	pipe->InitBuffer(maskBuf, perLoopRows * sizeof(int32_t));
55	pipe->InitBuffer(expertIdsBuf, perLoopRows * sizeof(int32_t));	56	pipe->InitBuffer(expertIdsBuf, perLoopRows * sizeof(int32_t));
56	}	57	}
57		58
58	- __aicore__ inline void Process() {	59	+ __aicore__ inline void Process()
		60	+ {
59	if (blockIdx == 0) {	61	if (blockIdx == 0) {
60	groupListOutLocal = copyOutQueue.AllocTensor<int64_t>();	62	groupListOutLocal = copyOutQueue.AllocTensor<int64_t>();
61	curExpertIdOffset = 0;	63	curExpertIdOffset = 0;
	@@ -82,8 +84,7 @@ public:
82	}	84	}
83		85
84	DataCopyExtParams copyParams{static_cast<uint16_t>(1),	86	DataCopyExtParams copyParams{static_cast<uint16_t>(1),
85	- static_cast<uint32_t>(curExpertIdOffset * NUM_TWO * sizeof(int64_t)),	87	+ static_cast<uint32_t>(curExpertIdOffset * NUM_TWO * sizeof(int64_t)), 0, 0, 0};
86	- 0, 0, 0};
87	SetWaitFlag<HardEvent::S_MTE3>(HardEvent::S_MTE3);	88	SetWaitFlag<HardEvent::S_MTE3>(HardEvent::S_MTE3);
88	DataCopyPad(groupListGm[totalOutExpertIdNum * NUM_TWO], groupListOutLocal, copyParams);	89	DataCopyPad(groupListGm[totalOutExpertIdNum * NUM_TWO], groupListOutLocal, copyParams);
89	totalOutExpertIdNum += curExpertIdOffset;	90	totalOutExpertIdNum += curExpertIdOffset;
	@@ -93,15 +94,18 @@ public:
93	}	94	}
94		95
95	private:	96	private:
96	- __aicore__ inline void CopyIn(int64_t progress, int64_t inputNum) {	97	+ __aicore__ inline void CopyIn(int64_t progress, int64_t inputNum)
		98	+ {
97	LocalTensor<int32_t> inLocal = copyInQueue.AllocTensor<int32_t>();	99	LocalTensor<int32_t> inLocal = copyInQueue.AllocTensor<int32_t>();
98	- DataCopyExtParams dataCopyParams{static_cast<uint16_t>(1), static_cast<uint32_t>(inputNum * sizeof(int32_t)), 0, 0, 0};	100	+ DataCopyExtParams dataCopyParams{static_cast<uint16_t>(1), static_cast<uint32_t>(inputNum * sizeof(int32_t)), 0,
		101	+ 0, 0};
99	DataCopyPadExtParams<int32_t> dataCopyPadParams{false, 0, 0, 0};	102	DataCopyPadExtParams<int32_t> dataCopyPadParams{false, 0, 0, 0};
100	DataCopyPad(inLocal, expandedExpertIdsGm[progress * perLoopRows], dataCopyParams, dataCopyPadParams);	103	DataCopyPad(inLocal, expandedExpertIdsGm[progress * perLoopRows], dataCopyParams, dataCopyPadParams);
101	copyInQueue.EnQue<int32_t>(inLocal);	104	copyInQueue.EnQue<int32_t>(inLocal);
102	}	105	}
103		106
104	- __aicore__ inline void Compute(int64_t inputNum, int64_t curLoop) {	107	+ __aicore__ inline void Compute(int64_t inputNum, int64_t curLoop)
		108	+ {
105	uint64_t rsvdCnt = 0;	109	uint64_t rsvdCnt = 0;
106	uint32_t inLocalIndex = 0;	110	uint32_t inLocalIndex = 0;
107	int32_t curExpertId = -1;	111	int32_t curExpertId = -1;
	@@ -112,7 +116,7 @@ private:
112	SetWaitFlag<HardEvent::MTE2_S>(HardEvent::MTE2_S);	116	SetWaitFlag<HardEvent::MTE2_S>(HardEvent::MTE2_S);
113	int32_t firstExpertId = inLocal.GetValue(inLocalIndex);	117	int32_t firstExpertId = inLocal.GetValue(inLocalIndex);
114	// 专家在上一loop结尾处截止的场景	118	// 专家在上一loop结尾处截止的场景
115	- if(firstExpertId != lastExpertId && lastExpertId != -1 && curExpertIdOffset < expertNum) {	119	+ if (firstExpertId != lastExpertId && lastExpertId != -1 && curExpertIdOffset < expertNum) {
116	groupListOutLocal.SetValue(curExpertIdOffset * NUM_TWO, lastExpertId);	120	groupListOutLocal.SetValue(curExpertIdOffset * NUM_TWO, lastExpertId);
117	groupListOutLocal.SetValue(curExpertIdOffset * NUM_TWO + 1, tokenCount);	121	groupListOutLocal.SetValue(curExpertIdOffset * NUM_TWO + 1, tokenCount);
118	curExpertIdOffset += 1;	122	curExpertIdOffset += 1;
	@@ -124,13 +128,11 @@ private:
124	SetWaitFlag<HardEvent::S_V>(HardEvent::S_V);	128	SetWaitFlag<HardEvent::S_V>(HardEvent::S_V);
125	LocalTensor<uint32_t> maskLocalUInt32 = maskBuf.Get<uint32_t>();	129	LocalTensor<uint32_t> maskLocalUInt32 = maskBuf.Get<uint32_t>();
126	LocalTensor<uint8_t> maskLocalTensorUInt8 = maskLocalUInt32.ReinterpretCast<uint8_t>();	130	LocalTensor<uint8_t> maskLocalTensorUInt8 = maskLocalUInt32.ReinterpretCast<uint8_t>();
127	- AscendC::CompareScalar(maskLocalTensorUInt8,	131	+ AscendC::CompareScalar(
128	- inLocal,	132	+ maskLocalTensorUInt8, inLocal, curExpertId, AscendC::CMPMODE::EQ,
129	- curExpertId,
130	- AscendC::CMPMODE::EQ,
131	(inputNum + ONE_REPEAT_COMPARE_NUM - 1) / ONE_REPEAT_COMPARE_NUM * ONE_REPEAT_COMPARE_NUM);	133	(inputNum + ONE_REPEAT_COMPARE_NUM - 1) / ONE_REPEAT_COMPARE_NUM * ONE_REPEAT_COMPARE_NUM);
132	PipeBarrier<PIPE_V>();	134	PipeBarrier<PIPE_V>();
133	-	135	+
134	GatherMaskParams gatherMaskParams;	136	GatherMaskParams gatherMaskParams;
135	gatherMaskParams.repeatTimes = 1;	137	gatherMaskParams.repeatTimes = 1;
136	gatherMaskParams.src0BlockStride = 1;	138	gatherMaskParams.src0BlockStride = 1;
	@@ -138,10 +140,10 @@ private:
138	gatherMaskParams.src1RepeatStride = 0;	140	gatherMaskParams.src1RepeatStride = 0;
139	GatherMask(expertIdsBufFp32, inLocal, maskLocalUInt32, true, inputNum, gatherMaskParams, rsvdCnt);	141	GatherMask(expertIdsBufFp32, inLocal, maskLocalUInt32, true, inputNum, gatherMaskParams, rsvdCnt);
140	tokenCount += rsvdCnt;	142	tokenCount += rsvdCnt;
141	- if(inLocalIndex + rsvdCnt >= inputNum) {	143	+ if (inLocalIndex + rsvdCnt >= inputNum) {
142	lastExpertId = curExpertId;	144	lastExpertId = curExpertId;
143	break;	145	break;
144	- } else if(inLocalIndex + rsvdCnt < inputNum && curExpertIdOffset < expertNum) {	146	+ } else if (inLocalIndex + rsvdCnt < inputNum && curExpertIdOffset < expertNum) {
145	groupListOutLocal.SetValue(curExpertIdOffset * NUM_TWO, curExpertId);	147	groupListOutLocal.SetValue(curExpertIdOffset * NUM_TWO, curExpertId);
146	groupListOutLocal.SetValue(curExpertIdOffset * NUM_TWO + 1, tokenCount);	148	groupListOutLocal.SetValue(curExpertIdOffset * NUM_TWO + 1, tokenCount);
147	tokenCount = 0;	149	tokenCount = 0;
	@@ -182,5 +184,5 @@ private:
182	int32_t firstExpertId = -1;	184	int32_t firstExpertId = -1;
183	};	185	};
184		186
185	-} // namespace FfnWbBatching	187	+} // namespace FfnWbBatching
186	-#endif // OP_KERNEL_FFN_WB_SCAN_GROUP_LISTING_ONE_CORE_H	188	+#endif // OP_KERNEL_FFN_WB_SCAN_GROUP_LISTING_ONE_CORE_H

Mffn/ffn_worker_batching/op_kernel/ffn_wb_scan_sort_multi_core.h+68-66

Mffn/ffn_worker_batching/op_kernel/ffn_wb_scan_sort_multi_core_bsk.h+98-93

	@@ -32,23 +32,23 @@ class KernelScanSortMaskMultiCoreBsK : public SortMaskBase {
32	public:	32	public:
33	__aicore__ inline KernelScanSortMaskMultiCoreBsK(){};	33	__aicore__ inline KernelScanSortMaskMultiCoreBsK(){};
34	__aicore__ inline void Init(GM_ADDR tokenInfoGm, GM_ADDR workspace, SortCustomTilingDataKernel *tilingData,	34	__aicore__ inline void Init(GM_ADDR tokenInfoGm, GM_ADDR workspace, SortCustomTilingDataKernel *tilingData,
35	- const ScheduleContextInfo contextInfo, TPipe tPipe);	35	+ const ScheduleContextInfo contextInfo, TPipe tPipe);
36	- __aicore__ inline void Process();	36	+ __aicore__ inline void Process();
37		37
38	private:	38	private:
39	- __aicore__ inline void VBSProcess();	39	+ __aicore__ inline void VBSProcess();
40	- __aicore__ inline void UBSortProcess(int64_t progress, int64_t size,int64_t sortNum, int64_t loopSessionCnt);	40	+ __aicore__ inline void UBSortProcess(int64_t progress, int64_t size, int64_t sortNum, int64_t loopSessionCnt);
41	- __aicore__ inline void OneCoreVMSProcess(int64_t listNum, int64_t perListElements, int64_t lastListElements);	41	+ __aicore__ inline void OneCoreVMSProcess(int64_t listNum, int64_t perListElements, int64_t lastListElements);
42	- __aicore__ inline void VMSProcess();	42	+ __aicore__ inline void VMSProcess();
43	- __aicore__ inline void SortOutProcess();	43	+ __aicore__ inline void SortOutProcess();
44	- __aicore__ inline void VBSCopyInAndClear(int64_t progress, int64_t size, int64_t sortNum, int64_t loopSessionCnt);	44	+ __aicore__ inline void VBSCopyInAndClear(int64_t progress, int64_t size, int64_t sortNum, int64_t loopSessionCnt);
45	- __aicore__ inline void UBSortCompute(int64_t progress, int64_t size, int64_t sortNum);	45	+ __aicore__ inline void UBSortCompute(int64_t progress, int64_t size, int64_t sortNum);
46	- __aicore__ inline void VBSCopyOut(int64_t progress, int64_t size, int64_t sortNum);	46	+ __aicore__ inline void VBSCopyOut(int64_t progress, int64_t size, int64_t sortNum);
47	- __aicore__ inline void InitSortMaskMrgSort(SortCustomMrgsort *sorter, int64_t listNum,	47	+ __aicore__ inline void InitSortMaskMrgSort(SortCustomMrgsort *sorter, int64_t listNum, int64_t coreOffset,
48	- int64_t coreOffset, int64_t sortNumCoreOffset, int64_t loopOffset, int64_t loopIdxOffset);	48	+ int64_t sortNumCoreOffset, int64_t loopOffset, int64_t loopIdxOffset);
49	- __aicore__ inline void InitSortMaskMrgSortOut(SortCustomMrgsortOut *sorter, int64_t listNum, int64_t coreOffset);	49	+ __aicore__ inline void InitSortMaskMrgSortOut(SortCustomMrgsortOut *sorter, int64_t listNum, int64_t coreOffset);
50	- __aicore__ inline void CopyOutValidCount();	50	+ __aicore__ inline void CopyOutValidCount();
51	- __aicore__ inline void ClearTokenInfoFlag();	51	+ __aicore__ inline void ClearTokenInfoFlag();
52		52
53	private:	53	private:
54	GlobalTensor<float> workspaceGms[NUM_TWO];	54	GlobalTensor<float> workspaceGms[NUM_TWO];
	@@ -66,10 +66,10 @@ private:
66		66
67	int64_t perCoreSortNum = 0; // perCoreSessionNum * BsKLenWithPading 主核每个核排序的元素总个数; 不保证32个数对齐	67	int64_t perCoreSortNum = 0; // perCoreSessionNum * BsKLenWithPading 主核每个核排序的元素总个数; 不保证32个数对齐
68	int64_t lastCoreSortNum = 0; // lastCoreSessionNum * BsKLenWithPading;	68	int64_t lastCoreSortNum = 0; // lastCoreSessionNum * BsKLenWithPading;
69	- int64_t sessionLoops = 0; // 当前核 ub循环次数	69	+ int64_t sessionLoops = 0; // 当前核 ub循环次数
70		70
71	- int64_t perLoopElements = 0;	71	+ int64_t perLoopElements = 0;
72	- int64_t lastLoopElement = 0;	72	+ int64_t lastLoopElement = 0;
73		73
74	// for MoeMrgsort	74	// for MoeMrgsort
75	SortCustomMrgsort mrgsorter;	75	SortCustomMrgsort mrgsorter;
	@@ -84,45 +84,48 @@ private:
84	int64_t lastListElements = 0;	84	int64_t lastListElements = 0;
85	int64_t vmsSortNumStride_ = 0; // 核间	85	int64_t vmsSortNumStride_ = 0; // 核间
86		86
87	- int64_t perCorePad_ = 0; // 记录pad	87	+ int64_t perCorePad_ = 0; // 记录pad
88	int64_t lastCorePad_ = 0; // 记录pad	88	int64_t lastCorePad_ = 0; // 记录pad
89	- int64_t splitNum_ = 1; // 原始块被切分后的块数	89	+ int64_t splitNum_ = 1; // 原始块被切分后的块数
90	- int64_t recordCnt_ = 0; // 记录1个原始块的有效专家数	90	+ int64_t recordCnt_ = 0; // 记录1个原始块的有效专家数
91	-	91	+
92	static constexpr int64_t MAX_MRGSORT_LIST = 4;	92	static constexpr int64_t MAX_MRGSORT_LIST = 4;
93	};	93	};
94		94
95	__aicore__ inline void KernelScanSortMaskMultiCoreBsK::VBSCopyInAndClear(int64_t progress, int64_t size,	95	__aicore__ inline void KernelScanSortMaskMultiCoreBsK::VBSCopyInAndClear(int64_t progress, int64_t size,
96	- int64_t sortNum, int64_t loopSessionCnt)	96	+ int64_t sortNum, int64_t loopSessionCnt)
97	{	97	{
98	int64_t curCorePad = 0;	98	int64_t curCorePad = 0;
99	-	99	+
100	// 根据当前块位置计算参数	100	// 根据当前块位置计算参数
101	if ((progress % splitNum_) != (splitNum_ - 1)) {	101	if ((progress % splitNum_) != (splitNum_ - 1)) {
102	curCorePad = perCorePad_;	102	curCorePad = perCorePad_;
103	} else {	103	} else {
104	curCorePad = lastCorePad_;	104	curCorePad = lastCorePad_;
105	}	105	}
106	-	106	+
107	LocalTensor<int32_t> inLocal = sortDataCopyInQueue.AllocTensor<int32_t>();	107	LocalTensor<int32_t> inLocal = sortDataCopyInQueue.AllocTensor<int32_t>();
108	int64_t inOffset = 0;	108	int64_t inOffset = 0;
109	-
110	- // 偏移量应该是上一次实际参与排序的元素个数（不包含pad）
111	- inOffset = (progress % splitNum_) * (perLoopElements - perCorePad_) + (progress / splitNum_) * (contextInfo_->M * this->F); // 单循环处理元素个数
112		109
113	- DataCopyExtParams dataCopyParams{static_cast<uint16_t>(loopSessionCnt), // 搬运块数 = 1	110	+ // 偏移量应该是上一次实际参与排序的元素个数（不包含pad）
114	- static_cast<uint32_t>((size - curCorePad) * sizeof(int32_t)), // 实际元素个数	111	+ inOffset = (progress % splitNum_) * (perLoopElements - perCorePad_) +
115	- 0, 0, 0};	112	+ (progress / splitNum_) * (contextInfo_->M * this->F); // 单循环处理元素个数
116	- DataCopyPadExtParams<int32_t> dataCopyPadParams{true, 0, static_cast<uint8_t>(curCorePad), INT_MAX}; // 当前pad长度=perCorePad_	113	+
117	-	114	+ DataCopyExtParams dataCopyParams{static_cast<uint16_t>(loopSessionCnt), // 搬运块数 = 1
		115	+ static_cast<uint32_t>((size - curCorePad) * sizeof(int32_t)), // 实际元素个数
		116	+ 0, 0, 0};
		117	+ DataCopyPadExtParams<int32_t> dataCopyPadParams{true, 0, static_cast<uint8_t>(curCorePad),
		118	+ INT_MAX}; // 当前pad长度=perCorePad_
		119	+
118	DataCopyPad(inLocal[0], expertIdsGm[inOffset], dataCopyParams, dataCopyPadParams);	120	DataCopyPad(inLocal[0], expertIdsGm[inOffset], dataCopyParams, dataCopyPadParams);
119	-	121	+
120	LocalTensor<int32_t> rowIdsLocal = inLocal[sortNum];	122	LocalTensor<int32_t> rowIdsLocal = inLocal[sortNum];
121	// 索引生成只有(A, BsKPad)	123	// 索引生成只有(A, BsKPad)
122	-	124	+
123	int64_t startValue = blockIdx_ * tilingData_->perCoreSessionNum * BsKLenWithPading +	125	int64_t startValue = blockIdx_ * tilingData_->perCoreSessionNum * BsKLenWithPading +
124	- (progress % splitNum_) * perLoopElements + (progress / splitNum_) * BsKLenWithPading; // 起始值为上一次参与排序的元素个数（包括pad）	126	+ (progress % splitNum_) * perLoopElements +
125	-	127	+ (progress / splitNum_) * BsKLenWithPading; // 起始值为上一次参与排序的元素个数（包括pad）
		128	+
126	// size是向上32位补齐后的元素个数	129	// size是向上32位补齐后的元素个数
127	ArithProgression<int32_t>(rowIdsLocal, startValue, 1, size); // size包括: BsKLenWithPading	130	ArithProgression<int32_t>(rowIdsLocal, startValue, 1, size); // size包括: BsKLenWithPading
128		131
	@@ -135,13 +138,13 @@ __aicore__ inline void KernelScanSortMaskMultiCoreBsK::VBSCopyInAndClear(int64_t
135	sortDataCopyOutQueue.EnQue(clearLocal);	138	sortDataCopyOutQueue.EnQue(clearLocal);
136		139
137	clearLocal = sortDataCopyOutQueue.DeQue<int32_t>();	140	clearLocal = sortDataCopyOutQueue.DeQue<int32_t>();
138	-	141	+
139	DataCopyExtParams copyoutParams{static_cast<uint16_t>(loopSessionCnt),	142	DataCopyExtParams copyoutParams{static_cast<uint16_t>(loopSessionCnt),
140	- static_cast<uint32_t>((size - curCorePad) * sizeof(int32_t)), // 实际元素个数	143	+ static_cast<uint32_t>((size - curCorePad) * sizeof(int32_t)), // 实际元素个数
141	- 0, 0, 0};	144	+ 0, 0, 0};
142	SetWaitFlag<HardEvent::MTE2_MTE3>(HardEvent::MTE2_MTE3);	145	SetWaitFlag<HardEvent::MTE2_MTE3>(HardEvent::MTE2_MTE3);
143	DataCopyPad(expertIdsGm[inOffset], clearLocal, copyoutParams);	146	DataCopyPad(expertIdsGm[inOffset], clearLocal, copyoutParams);
144	-	147	+
145	sortDataCopyOutQueue.FreeTensor(clearLocal);	148	sortDataCopyOutQueue.FreeTensor(clearLocal);
146	}	149	}
147		150
	@@ -162,21 +165,18 @@ __aicore__ inline void KernelScanSortMaskMultiCoreBsK::UBSortCompute(int64_t pro
162		165
163	LocalTensor<uint8_t> maskLocalTensorUInt8 = maskLocalTensor.ReinterpretCast<uint8_t>();	166	LocalTensor<uint8_t> maskLocalTensorUInt8 = maskLocalTensor.ReinterpretCast<uint8_t>();
164		167
165	- AscendC::CompareScalar(maskLocalTensorUInt8,	168	+ AscendC::CompareScalar(maskLocalTensorUInt8, expertIdsLocalFp32, static_cast<float>(-expertStart_),
166	- expertIdsLocalFp32,	169	+ AscendC::CMPMODE::GT,
167	- static_cast<float>(-expertStart_),	170	+ (size + ONE_REPEAT_COMPARE_NUM - 1) / ONE_REPEAT_COMPARE_NUM * ONE_REPEAT_COMPARE_NUM);
168	- AscendC::CMPMODE::GT,	171	+
169	- (size + ONE_REPEAT_COMPARE_NUM - 1) / ONE_REPEAT_COMPARE_NUM * ONE_REPEAT_COMPARE_NUM);
170	-
171	GatherMaskParams gatherMaskParams;	172	GatherMaskParams gatherMaskParams;
172	gatherMaskParams.repeatTimes = 1;	173	gatherMaskParams.repeatTimes = 1;
173	gatherMaskParams.src0BlockStride = 1;	174	gatherMaskParams.src0BlockStride = 1;
174	gatherMaskParams.src0RepeatStride = 8; // 8 blocks	175	gatherMaskParams.src0RepeatStride = 8; // 8 blocks
175	gatherMaskParams.src1RepeatStride = 0;	176	gatherMaskParams.src1RepeatStride = 0;
176		177
177	- GatherMask(	178	+ GatherMask(expertIdsLocalFp32, expertIdsLocalFp32, maskLocalTensor, true, size, gatherMaskParams, rsvdCnt);
178	- expertIdsLocalFp32, expertIdsLocalFp32, maskLocalTensor, true, size, gatherMaskParams, rsvdCnt);	179	+
179	-
180	curValidCnt_ = rsvdCnt;	180	curValidCnt_ = rsvdCnt;
181		181
182	if (rsvdCnt == 0) {	182	if (rsvdCnt == 0) {
	@@ -213,11 +213,11 @@ __aicore__ inline void KernelScanSortMaskMultiCoreBsK::VBSCopyOut(int64_t progre
213	{	213	{
214	if (curValidCnt_ > 0) {	214	if (curValidCnt_ > 0) {
215	LocalTensor<float> outLocal = sortDataCopyOutQueue.DeQue<float>();	215	LocalTensor<float> outLocal = sortDataCopyOutQueue.DeQue<float>();
216	- DataCopyExtParams copyParams{1, static_cast<uint32_t>(GetSortLen<float>(curValidCnt_) * sizeof(int32_t)),	216	+ DataCopyExtParams copyParams{1, static_cast<uint32_t>(GetSortLen<float>(curValidCnt_) * sizeof(int32_t)), 0, 0,
217	- 0, 0, 0};	217	+ 0};
218	- int64_t wkOffset = blockIdx_ * GetSortLen<float>(perCoreSortNum) +	218	+ int64_t wkOffset =
219	- GetSortLen<float>(progress * (perLoopElements));	219	+ blockIdx_ * GetSortLen<float>(perCoreSortNum) + GetSortLen<float>(progress * (perLoopElements));
220	-	220	+
221	DataCopyPad(workspaceGms[0][wkOffset], outLocal, copyParams);	221	DataCopyPad(workspaceGms[0][wkOffset], outLocal, copyParams);
222		222
223	sortDataCopyOutQueue.FreeTensor(outLocal);	223	sortDataCopyOutQueue.FreeTensor(outLocal);
	@@ -229,16 +229,17 @@ __aicore__ inline void KernelScanSortMaskMultiCoreBsK::VBSCopyOut(int64_t progre
229	tempTensor.SetValue(0, curValidCnt_);	229	tempTensor.SetValue(0, curValidCnt_);
230	SetWaitFlag<HardEvent::S_MTE3>(HardEvent::S_MTE3);	230	SetWaitFlag<HardEvent::S_MTE3>(HardEvent::S_MTE3);
231	DataCopyExtParams copyParams1{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(int32_t)), 0, 0, 0};	231	DataCopyExtParams copyParams1{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(int32_t)), 0, 0, 0};
232	- DataCopyPad(workspaceSortNumGm_[blockIdx_ * tilingData_->sortNumWorkSpacePerCore + progress],	232	+ DataCopyPad(workspaceSortNumGm_[blockIdx_ * tilingData_->sortNumWorkSpacePerCore + progress], tempTensor,
233	- tempTensor, copyParams1);	233	+ copyParams1);
234	}	234	}
235		235
236	__aicore__ inline void KernelScanSortMaskMultiCoreBsK::InitSortMaskMrgSort(SortCustomMrgsort *sorter, int64_t listNum,	236	__aicore__ inline void KernelScanSortMaskMultiCoreBsK::InitSortMaskMrgSort(SortCustomMrgsort *sorter, int64_t listNum,
237	- int64_t coreOffset, int64_t sortNumCoreOffset, int64_t loopOffset, int64_t loopIdxOffset)	237	+ int64_t coreOffset,
		238	+ int64_t sortNumCoreOffset,
		239	+ int64_t loopOffset, int64_t loopIdxOffset)
238	{	240	{
239	GlobalTensor<float> srcWsGm = workspaceGms[srcWsIndex][blockIdx_ * coreOffset + loopOffset];	241	GlobalTensor<float> srcWsGm = workspaceGms[srcWsIndex][blockIdx_ * coreOffset + loopOffset];
240	- GlobalTensor<int32_t> srcSortNumGm = workspaceSortNumGm_[blockIdx_ * sortNumCoreOffset +	242	+ GlobalTensor<int32_t> srcSortNumGm = workspaceSortNumGm_[blockIdx_ * sortNumCoreOffset + loopIdxOffset];
241	- loopIdxOffset];
242	LocalTensor<float> inLocal = sortDataCopyInQueue.AllocTensor<float>();	243	LocalTensor<float> inLocal = sortDataCopyInQueue.AllocTensor<float>();
243	LocalTensor<float> outLocal = sortDataCopyOutQueue.AllocTensor<float>();	244	LocalTensor<float> outLocal = sortDataCopyOutQueue.AllocTensor<float>();
244	for (int64_t i = 0; i < listNum; i++) {	245	for (int64_t i = 0; i < listNum; i++) {
	@@ -253,8 +254,8 @@ __aicore__ inline void KernelScanSortMaskMultiCoreBsK::InitSortMaskMrgSort(SortC
253	tempBuffer.FreeTensor(outSortNumLocal);	254	tempBuffer.FreeTensor(outSortNumLocal);
254	}	255	}
255		256
256	-__aicore__ inline void KernelScanSortMaskMultiCoreBsK::InitSortMaskMrgSortOut(	257	+__aicore__ inline void KernelScanSortMaskMultiCoreBsK::InitSortMaskMrgSortOut(SortCustomMrgsortOut *sorter,
257	- SortCustomMrgsortOut *sorter, int64_t listNum, int64_t coreOffset)	258	+ int64_t listNum, int64_t coreOffset)
258	{	259	{
259	GlobalTensor<float> srcWsGm = workspaceGms[srcWsIndex];	260	GlobalTensor<float> srcWsGm = workspaceGms[srcWsIndex];
260	GlobalTensor<int32_t> srcSortNumGm = workspaceSortNumGm_;	261	GlobalTensor<int32_t> srcSortNumGm = workspaceSortNumGm_;
	@@ -276,8 +277,8 @@ __aicore__ inline void KernelScanSortMaskMultiCoreBsK::InitSortMaskMrgSortOut(
276	sortDataCopyOutQueue.FreeTensor(outLocal);	277	sortDataCopyOutQueue.FreeTensor(outLocal);
277	}	278	}
278		279
279	-__aicore__ inline void KernelScanSortMaskMultiCoreBsK::OneCoreVMSProcess(	280	+__aicore__ inline void KernelScanSortMaskMultiCoreBsK::OneCoreVMSProcess(int64_t listNum, int64_t perListElements,
280	- int64_t listNum, int64_t perListElements, int64_t lastListElements)	281	+ int64_t lastListElements)
281	{	282	{
282	int64_t coreOffset = GetSortLen<float>(perCoreSortNum);	283	int64_t coreOffset = GetSortLen<float>(perCoreSortNum);
283	int64_t sortNumCoreOffset = tilingData_->sortNumWorkSpacePerCore;	284	int64_t sortNumCoreOffset = tilingData_->sortNumWorkSpacePerCore;
	@@ -296,14 +297,14 @@ __aicore__ inline void KernelScanSortMaskMultiCoreBsK::OneCoreVMSProcess(
296	int64_t loopOffset = GetSortLen<float>(mrgsortParam.perListElements * MAX_MRGSORT_LIST);	297	int64_t loopOffset = GetSortLen<float>(mrgsortParam.perListElements * MAX_MRGSORT_LIST);
297	int64_t loopIdxOffset = mrgsortParam.sortNumStride * MAX_MRGSORT_LIST;	298	int64_t loopIdxOffset = mrgsortParam.sortNumStride * MAX_MRGSORT_LIST;
298	for (int64_t loop = 0; loop < loops - 1; loop++) {	299	for (int64_t loop = 0; loop < loops - 1; loop++) {
299	- InitSortMaskMrgSort(&mrgsorter, MAX_MRGSORT_LIST, coreOffset, sortNumCoreOffset,	300	+ InitSortMaskMrgSort(&mrgsorter, MAX_MRGSORT_LIST, coreOffset, sortNumCoreOffset, loop * loopOffset,
300	- loop * loopOffset, loop * loopIdxOffset);	301	+ loop * loopIdxOffset);
301	mrgsorter.Init(&mrgsortParam);	302	mrgsorter.Init(&mrgsortParam);
302	mrgsorter.Process();	303	mrgsorter.Process();
303	}	304	}
304		305
305	- InitSortMaskMrgSort(&mrgsorter, remainListNum, coreOffset, sortNumCoreOffset,	306	+ InitSortMaskMrgSort(&mrgsorter, remainListNum, coreOffset, sortNumCoreOffset, (loops - 1) * loopOffset,
306	- (loops - 1) * loopOffset, (loops - 1) * loopIdxOffset);	307	+ (loops - 1) * loopIdxOffset);
307	mrgsorter.Init(&mrgsortParam);	308	mrgsorter.Init(&mrgsortParam);
308	mrgsorter.Process();	309	mrgsorter.Process();
309		310
	@@ -340,14 +341,14 @@ __aicore__ inline void KernelScanSortMaskMultiCoreBsK::OneCoreVMSProcess(
340	intriParamsOut.blockCount = 1;	341	intriParamsOut.blockCount = 1;
341	intriParamsOut.blockLen = GetSortLen<float>(this->totalValidCnt_ * NUM_TWO) * sizeof(float);	342	intriParamsOut.blockLen = GetSortLen<float>(this->totalValidCnt_ * NUM_TWO) * sizeof(float);
342	DataCopyPad(dstWsGm, inLocal, intriParamsOut);	343	DataCopyPad(dstWsGm, inLocal, intriParamsOut);
343	-	344	+
344	sortDataCopyInQueue.FreeTensor(inLocal);	345	sortDataCopyInQueue.FreeTensor(inLocal);
345	}	346	}
346	}	347	}
347	}	348	}
348		349
349	-__aicore__ inline void KernelScanSortMaskMultiCoreBsK::UBSortProcess(int64_t progress, int64_t size,	350	+__aicore__ inline void KernelScanSortMaskMultiCoreBsK::UBSortProcess(int64_t progress, int64_t size, int64_t sortNum,
350	- int64_t sortNum, int64_t loopSessionCnt)	351	+ int64_t loopSessionCnt)
351	{	352	{
352	VBSCopyInAndClear(progress, size, sortNum, loopSessionCnt);	353	VBSCopyInAndClear(progress, size, sortNum, loopSessionCnt);
353	UBSortCompute(progress, size, sortNum);	354	UBSortCompute(progress, size, sortNum);
	@@ -448,7 +449,8 @@ __aicore__ inline void KernelScanSortMaskMultiCoreBsK::CopyOutValidCount()
448	}	449	}
449		450
450	__aicore__ inline void KernelScanSortMaskMultiCoreBsK::Init(GM_ADDR tokenInfoGm, GM_ADDR workspace,	451	__aicore__ inline void KernelScanSortMaskMultiCoreBsK::Init(GM_ADDR tokenInfoGm, GM_ADDR workspace,
451	- SortCustomTilingDataKernel tilingData, const ScheduleContextInfo contextInfo, TPipe *tPipe)	452	+ SortCustomTilingDataKernel *tilingData,
		453	+ const ScheduleContextInfo contextInfo, TPipe tPipe)
452	{	454	{
453	this->pipe = tPipe;	455	this->pipe = tPipe;
454	tilingData_ = tilingData;	456	tilingData_ = tilingData;
	@@ -470,8 +472,10 @@ __aicore__ inline void KernelScanSortMaskMultiCoreBsK::Init(GM_ADDR tokenInfoGm,
470		472
471	int64_t perCoreBsKLen = contextInfo_->sortLoopMaxElement;	473	int64_t perCoreBsKLen = contextInfo_->sortLoopMaxElement;
472	// 尾块也单独占一个UB	474	// 尾块也单独占一个UB
473	- perCoreSortNum = tilingData_->perCoreSessionNum * CeilDiv(contextInfo_->BS * contextInfo_->K, perCoreBsKLen) * perCoreBsKLen;	475	+ perCoreSortNum =
474	- lastCoreSortNum = tilingData_->lastCoreSessionNum * CeilDiv(contextInfo_->BS * contextInfo_->K, perCoreBsKLen) * perCoreBsKLen;	476	+ tilingData_->perCoreSessionNum * CeilDiv(contextInfo_->BS * contextInfo_->K, perCoreBsKLen) * perCoreBsKLen;
		477	+ lastCoreSortNum =
		478	+ tilingData_->lastCoreSessionNum * CeilDiv(contextInfo_->BS * contextInfo_->K, perCoreBsKLen) * perCoreBsKLen;
475		479
476	// 切BSK，计算每个循环可容纳BSK大小	480	// 切BSK，计算每个循环可容纳BSK大小
477	// 非尾块元素个数均为UB最大元素个数	481	// 非尾块元素个数均为UB最大元素个数
	@@ -490,34 +494,36 @@ __aicore__ inline void KernelScanSortMaskMultiCoreBsK::Init(GM_ADDR tokenInfoGm,
490		494
491	// 原始块被切分后的块数	495	// 原始块被切分后的块数
492	splitNum_ = CeilDiv(contextInfo_->BS * contextInfo_->K, perCoreBsKLen);	496	splitNum_ = CeilDiv(contextInfo_->BS * contextInfo_->K, perCoreBsKLen);
493	-	497	+
494	// 重新计算单核排序所需变量	498	// 重新计算单核排序所需变量
495	// 需要循环多少次，ub总数 = 原始session数 * 原始块被切分后的块数	499	// 需要循环多少次，ub总数 = 原始session数 * 原始块被切分后的块数
496	- sessionLoops = curCoreSessionNum * splitNum_;	500	+ sessionLoops = curCoreSessionNum * splitNum_;
497	-	501	+
498	int64_t expertIdStartPos = contextInfo_->curMicroBatchID * F + 1 + 1;	502	int64_t expertIdStartPos = contextInfo_->curMicroBatchID * F + 1 + 1;
499	expertIdsGmFStart_.SetGlobalBuffer((__gm__ int32_t )tokenInfoGm + contextInfo_->curMicroBatchID F);	503	expertIdsGmFStart_.SetGlobalBuffer((__gm__ int32_t )tokenInfoGm + contextInfo_->curMicroBatchID F);
500	expertIdsGm.SetGlobalBuffer((__gm__ int32_t *)tokenInfoGm + expertIdStartPos +	504	expertIdsGm.SetGlobalBuffer((__gm__ int32_t *)tokenInfoGm + expertIdStartPos +
501	- blockIdx_ * tilingData_->perCoreSessionNum * contextInfo_->M * F);	505	+ blockIdx_ * tilingData_->perCoreSessionNum * contextInfo_->M * F);
502	// rsvdCntGm 在scan阶段已经清零	506	// rsvdCntGm 在scan阶段已经清零
503	rsvdCntGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace), SCAN_BATCHID_GM_OFFSET);	507	rsvdCntGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace), SCAN_BATCHID_GM_OFFSET);
504	workspaceSortNumGm_.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS,	508	workspaceSortNumGm_.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS,
505	contextInfo_->sortNumWorkSpace);	509	contextInfo_->sortNumWorkSpace);
506		510
507	- sortedexpertIdsGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS +	511	+ sortedexpertIdsGm.SetGlobalBuffer(
508	- contextInfo_->sortNumWorkSpace, this->totalLength);	512	+ reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS + contextInfo_->sortNumWorkSpace,
		513	+ this->totalLength);
509	sortedRowIdsGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS +	514	sortedRowIdsGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS +
510	- contextInfo_->sortNumWorkSpace + this->totalLength, this->totalLength);	515	+ contextInfo_->sortNumWorkSpace + this->totalLength,
		516	+ this->totalLength);
511		517
512	workspaceGms[0].SetGlobalBuffer((__gm__ float *)workspace + OFFSET_SORTED_EXPERT_IDS +	518	workspaceGms[0].SetGlobalBuffer((__gm__ float *)workspace + OFFSET_SORTED_EXPERT_IDS +
513	- contextInfo_->sortNumWorkSpace + this->totalLength * NUM_TWO, this->totalLength * NUM_TWO);	519	+ contextInfo_->sortNumWorkSpace + this->totalLength * NUM_TWO,
		520	+ this->totalLength * NUM_TWO);
514	workspaceGms[1].SetGlobalBuffer((__gm__ float *)workspace + OFFSET_SORTED_EXPERT_IDS +	521	workspaceGms[1].SetGlobalBuffer((__gm__ float *)workspace + OFFSET_SORTED_EXPERT_IDS +
515	- contextInfo_->sortNumWorkSpace + this->totalLength * (NUM_TWO + NUM_TWO),	522	+ contextInfo_->sortNumWorkSpace + this->totalLength * (NUM_TWO + NUM_TWO),
516	- this->totalLength * NUM_TWO);	523	+ this->totalLength * NUM_TWO);
517		524
518	- bufferSize_ = Ceil(Max(tilingData_->oneLoopMaxElementsMrg * MAX_MRGSORT_LIST, perCoreBsKLen),	525	+ bufferSize_ = Ceil(Max(tilingData_->oneLoopMaxElementsMrg * MAX_MRGSORT_LIST, perCoreBsKLen), ONE_REPEAT_SORT_NUM) *
519	- ONE_REPEAT_SORT_NUM) *	526	+ ONE_REPEAT_SORT_NUM * sizeof(int32_t) * NUM_TWO;
520	- ONE_REPEAT_SORT_NUM * sizeof(int32_t) * NUM_TWO;
521	pipe->InitBuffer(sortDataCopyInQueue, bufferNum, bufferSize_);	527	pipe->InitBuffer(sortDataCopyInQueue, bufferNum, bufferSize_);
522	pipe->InitBuffer(sortDataCopyOutQueue, bufferNum, bufferSize_);	528	pipe->InitBuffer(sortDataCopyOutQueue, bufferNum, bufferSize_);
523	pipe->InitBuffer(sortedBuffer, bufferSize_);	529	pipe->InitBuffer(sortedBuffer, bufferSize_);
	@@ -528,7 +534,7 @@ __aicore__ inline void KernelScanSortMaskMultiCoreBsK::ClearTokenInfoFlag()
528	{	534	{
529	// 用最后一个核清理flag. 一个block一个有效数(0, int32_t). 总共需要A个block.	535	// 用最后一个核清理flag. 一个block一个有效数(0, int32_t). 总共需要A个block.
530	if (blockIdx_ == contextInfo_->coreNum - 1) {	536	if (blockIdx_ == contextInfo_->coreNum - 1) {
531	- int64_t perLoopElement = bufferSize_ / BLOCK_SIZE; // buffer总共可以支持的block个数	537	+ int64_t perLoopElement = bufferSize_ / BLOCK_SIZE; // buffer总共可以支持的block个数
532	int64_t loops = Ceil(contextInfo_->A, perLoopElement);	538	int64_t loops = Ceil(contextInfo_->A, perLoopElement);
533	int64_t lastLoopElement = contextInfo_->A - (loops - 1) * perLoopElement;	539	int64_t lastLoopElement = contextInfo_->A - (loops - 1) * perLoopElement;
534	int64_t duplicateNum = Min(static_cast<int64_t>(contextInfo_->A), perLoopElement) * 8; // 8: block num	540	int64_t duplicateNum = Min(static_cast<int64_t>(contextInfo_->A), perLoopElement) * 8; // 8: block num
	@@ -545,9 +551,8 @@ __aicore__ inline void KernelScanSortMaskMultiCoreBsK::ClearTokenInfoFlag()
545	}	551	}
546		552
547	DataCopyExtParams copyOutParams{static_cast<uint16_t>(curElementA), static_cast<uint32_t>(sizeof(int32_t)),	553	DataCopyExtParams copyOutParams{static_cast<uint16_t>(curElementA), static_cast<uint32_t>(sizeof(int32_t)),
548	- 0, static_cast<uint32_t>((contextInfo_->M * F - 1) * sizeof(int32_t)), 0};	554	+ 0, static_cast<uint32_t>((contextInfo_->M * F - 1) * sizeof(int32_t)), 0};
549	- DataCopyPad(expertIdsGmFStart_[idx * perLoopElement * contextInfo_->M * F],	555	+ DataCopyPad(expertIdsGmFStart_[idx * perLoopElement * contextInfo_->M * F], clearLocal, copyOutParams);
550	- clearLocal, copyOutParams);
551	}	556	}
552	}	557	}
553	}	558	}
	@@ -559,5 +564,5 @@ __aicore__ inline void KernelScanSortMaskMultiCoreBsK::Process()
559	VMSProcess();	564	VMSProcess();
560	SortOutProcess();	565	SortOutProcess();
561	}	566	}
562	-} // namespace FfnWbBatching	567	+} // namespace FfnWbBatching
563	-#endif // OP_KERNEL_FFN_WB_SCAN_SORT_MULTI_CORE_BSK_H	568	+#endif // OP_KERNEL_FFN_WB_SCAN_SORT_MULTI_CORE_BSK_H

Mffn/ffn_worker_batching/op_kernel/ffn_wb_scan_sort_one_core.h+23-25

	@@ -51,14 +51,11 @@ private:
51	__aicore__ inline void KernelScanSortMaskOneCore::CopyInAndClear()	51	__aicore__ inline void KernelScanSortMaskOneCore::CopyInAndClear()
52	{	52	{
53	LocalTensor<int32_t> inLocal = sortDataCopyInQueue.AllocTensor<int32_t>();	53	LocalTensor<int32_t> inLocal = sortDataCopyInQueue.AllocTensor<int32_t>();
54	- DataCopyExtParams dataCopyParams{	54	+ DataCopyExtParams dataCopyParams{static_cast<uint16_t>(contextInfo_->A),
55	- static_cast<uint16_t>(contextInfo_->A),	55	+ static_cast<uint32_t>(contextInfo_->BS * contextInfo_->K * INT32_SIZE),
56	- static_cast<uint32_t>(contextInfo_->BS * contextInfo_->K * INT32_SIZE),	56	+ static_cast<uint32_t>(((contextInfo_->M - 1) * F_ + NUM_TWO) * INT32_SIZE), 0, 0};
57	- static_cast<uint32_t>(((contextInfo_->M - 1) * F_ + NUM_TWO) * INT32_SIZE),
58	- 0, 0};
59	DataCopyPadExtParams dataCopyPadParams{true, 0, static_cast<uint8_t>(contextInfo_->BsKPaddingCount), INT_MAX};	57	DataCopyPadExtParams dataCopyPadParams{true, 0, static_cast<uint8_t>(contextInfo_->BsKPaddingCount), INT_MAX};
60	- DataCopyPad(inLocal, expertIdsGm[contextInfo_->curMicroBatchID * F_ + NUM_TWO],	58	+ DataCopyPad(inLocal, expertIdsGm[contextInfo_->curMicroBatchID * F_ + NUM_TWO], dataCopyParams, dataCopyPadParams);
61	- dataCopyParams, dataCopyPadParams);
62		59
63	LocalTensor<int32_t> rowIdsLocal = inLocal[this->sortNum];	60	LocalTensor<int32_t> rowIdsLocal = inLocal[this->sortNum];
64	ArithProgression<int32_t>(rowIdsLocal, 0, 1, this->totalLength);	61	ArithProgression<int32_t>(rowIdsLocal, 0, 1, this->totalLength);
	@@ -71,13 +68,12 @@ __aicore__ inline void KernelScanSortMaskOneCore::CopyInAndClear()
71	sortDataCopyOutQueue.EnQue(clearLocal);	68	sortDataCopyOutQueue.EnQue(clearLocal);
72		69
73	clearLocal = sortDataCopyOutQueue.DeQue<int32_t>();	70	clearLocal = sortDataCopyOutQueue.DeQue<int32_t>();
74	- DataCopyExtParams copyoutParams{static_cast<uint16_t>(contextInfo_->A),	71	+ DataCopyExtParams copyoutParams{
75	- static_cast<uint32_t>(contextInfo_->BS * contextInfo_->K * sizeof(int32_t)),	72	+ static_cast<uint16_t>(contextInfo_->A),
76	- 0,	73	+ static_cast<uint32_t>(contextInfo_->BS * contextInfo_->K * sizeof(int32_t)), 0,
77	- static_cast<uint32_t>((contextInfo_->M * F_ - contextInfo_->BS * contextInfo_->K) * sizeof(int32_t)),	74	+ static_cast<uint32_t>((contextInfo_->M * F_ - contextInfo_->BS * contextInfo_->K) * sizeof(int32_t)), 0};
78	- 0};	75	+ DataCopyExtParams copyOutParamsF{static_cast<uint16_t>(contextInfo_->A), static_cast<uint32_t>(sizeof(int32_t)), 0,
79	- DataCopyExtParams copyOutParamsF{static_cast<uint16_t>(contextInfo_->A), static_cast<uint32_t>(sizeof(int32_t)),	76	+ static_cast<uint32_t>((contextInfo_->M * F_ - 1) * sizeof(int32_t)), 0};
80	- 0, static_cast<uint32_t>((contextInfo_->M * F_ - 1) * sizeof(int32_t)), 0};
81		77
82	SetWaitFlag<HardEvent::MTE2_MTE3>(HardEvent::MTE2_MTE3);	78	SetWaitFlag<HardEvent::MTE2_MTE3>(HardEvent::MTE2_MTE3);
83	DataCopyPad(expertIdsGm[contextInfo_->curMicroBatchID * F_ + NUM_TWO], clearLocal, copyoutParams);	79	DataCopyPad(expertIdsGm[contextInfo_->curMicroBatchID * F_ + NUM_TWO], clearLocal, copyoutParams);
	@@ -101,10 +97,8 @@ __aicore__ inline void KernelScanSortMaskOneCore::SortCompute()
101	PipeBarrier<PIPE_V>();	97	PipeBarrier<PIPE_V>();
102		98
103	LocalTensor<uint8_t> maskLocalTensorUInt8 = maskLocalUInt32.ReinterpretCast<uint8_t>();	99	LocalTensor<uint8_t> maskLocalTensorUInt8 = maskLocalUInt32.ReinterpretCast<uint8_t>();
104	- AscendC::CompareScalar(maskLocalTensorUInt8,	100	+ AscendC::CompareScalar(
105	- expertIdsFp32,	101	+ maskLocalTensorUInt8, expertIdsFp32, static_cast<float>(-expertStart_), AscendC::CMPMODE::GT,
106	- static_cast<float>(-expertStart_),
107	- AscendC::CMPMODE::GT,
108	(this->totalLength + ONE_REPEAT_COMPARE_NUM - 1) / ONE_REPEAT_COMPARE_NUM * ONE_REPEAT_COMPARE_NUM);	102	(this->totalLength + ONE_REPEAT_COMPARE_NUM - 1) / ONE_REPEAT_COMPARE_NUM * ONE_REPEAT_COMPARE_NUM);
109	PipeBarrier<PIPE_V>();	103	PipeBarrier<PIPE_V>();
110		104
	@@ -173,11 +167,13 @@ __aicore__ inline void KernelScanSortMaskOneCore::CopyOut()
173	}	167	}
174		168
175	rsvdCntGm.SetValue(0, this->validCnt);	169	rsvdCntGm.SetValue(0, this->validCnt);
176	- DataCacheCleanAndInvalid<int32_t, AscendC::CacheLine::SINGLE_CACHE_LINE, AscendC::DcciDst::CACHELINE_ALL>(rsvdCntGm);	170	+ DataCacheCleanAndInvalid<int32_t, AscendC::CacheLine::SINGLE_CACHE_LINE, AscendC::DcciDst::CACHELINE_ALL>(
		171	+ rsvdCntGm);
177	}	172	}
178		173
179	__aicore__ inline void KernelScanSortMaskOneCore::Init(GM_ADDR tokenInfoGm, GM_ADDR workspace,	174	__aicore__ inline void KernelScanSortMaskOneCore::Init(GM_ADDR tokenInfoGm, GM_ADDR workspace,
180	- SortCustomTilingDataKernel tilingData, const ScheduleContextInfo contextInfo, TPipe *tPipe)	175	+ SortCustomTilingDataKernel *tilingData,
		176	+ const ScheduleContextInfo contextInfo, TPipe tPipe)
181	{	177	{
182	this->pipe = tPipe;	178	this->pipe = tPipe;
183	contextInfo_ = contextInfo;	179	contextInfo_ = contextInfo;
	@@ -190,10 +186,12 @@ __aicore__ inline void KernelScanSortMaskOneCore::Init(GM_ADDR tokenInfoGm, GM_A
190		186
191	expertIdsGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(tokenInfoGm));	187	expertIdsGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(tokenInfoGm));
192	rsvdCntGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace), OFFSET_SORTED_EXPERT_IDS);	188	rsvdCntGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace), OFFSET_SORTED_EXPERT_IDS);
193	- sortedexpertIdsGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS +	189	+ sortedexpertIdsGm.SetGlobalBuffer(
194	- contextInfo_->sortNumWorkSpace, this->totalLength);	190	+ reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS + contextInfo_->sortNumWorkSpace,
		191	+ this->totalLength);
195	sortedRowIdsGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS +	192	sortedRowIdsGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS +
196	- contextInfo_->sortNumWorkSpace + this->totalLength, this->totalLength);	193	+ contextInfo_->sortNumWorkSpace + this->totalLength,
		194	+ this->totalLength);
197		195
198	if (needCoreNum_ == 0 && GetBlockIdx() == 0) {	196	if (needCoreNum_ == 0 && GetBlockIdx() == 0) {
199	rsvdCntGm.SetValue(0, 0);	197	rsvdCntGm.SetValue(0, 0);
	@@ -218,5 +216,5 @@ __aicore__ inline void KernelScanSortMaskOneCore::Process()
218	}	216	}
219	SyncAll();	217	SyncAll();
220	}	218	}
221	-} // namespace FfnWbBatching	219	+} // namespace FfnWbBatching
222	-#endif // OP_KERNEL_FFN_WB_SCAN_SORT_ONE_CORE_H	220	+#endif // OP_KERNEL_FFN_WB_SCAN_SORT_ONE_CORE_H

Mffn/ffn_worker_batching/op_kernel/ffn_wb_sort_base.h+27-28

	@@ -27,38 +27,37 @@ namespace FfnWbBatching {
27	using namespace AscendC;	27	using namespace AscendC;
28		28
29	class SortMaskBase {	29	class SortMaskBase {
30	- public:	30	+public:
31	- __aicore__ inline SortMaskBase(){};	31	+ __aicore__ inline SortMaskBase(){};
32		32
33	- protected:	33	+protected:
34	- TPipe* pipe;	34	+ TPipe *pipe;
35	- TQue<QuePosition::VECIN, 1> sortDataCopyInQueue;	35	+ TQue<QuePosition::VECIN, 1> sortDataCopyInQueue;
36	- TQue<QuePosition::VECOUT, 1> sortDataCopyOutQueue;	36	+ TQue<QuePosition::VECOUT, 1> sortDataCopyOutQueue;
37	- TBuf<TPosition::VECCALC> tempBuffer;	37	+ TBuf<TPosition::VECCALC> tempBuffer;
38	- TBuf<TPosition::VECCALC> sortedBuffer;	38	+ TBuf<TPosition::VECCALC> sortedBuffer;
39		39
40	- GlobalTensor<int32_t> expertIdsGm;	40	+ GlobalTensor<int32_t> expertIdsGm;
41	- GlobalTensor<int32_t> rsvdCntGm;	41	+ GlobalTensor<int32_t> rsvdCntGm;
42	- GlobalTensor<int32_t> sortedexpertIdsGm;	42	+ GlobalTensor<int32_t> sortedexpertIdsGm;
43	- GlobalTensor<int32_t> sortedRowIdsGm;	43	+ GlobalTensor<int32_t> sortedRowIdsGm;
44	- GlobalTensor<int32_t> groupListTmpGm;	44	+ GlobalTensor<int32_t> groupListTmpGm;
45		45
46	- int64_t bufferNum = 1;	46	+ int64_t bufferNum = 1;
47	- int64_t totalLength = 0;	47	+ int64_t totalLength = 0;
48		48
49	- int64_t expertStart_ = 1000000;	49	+ int64_t expertStart_ = 1000000;
50	- int64_t n = 0;	50	+ int64_t n = 0;
51	- int64_t k = 0;	51	+ int64_t k = 0;
52	- int64_t rowIdxType_ = 0;	52	+ int64_t rowIdxType_ = 0;
53		53
54	- static constexpr int64_t SYNC_GM_NUM = 2;	54	+ static constexpr int64_t SYNC_GM_NUM = 2;
55	- static constexpr int64_t WORK_GM_NUM = 2;	55	+ static constexpr int64_t WORK_GM_NUM = 2;
56	- static constexpr int64_t DST_BLK_STRIDE = 1;	56	+ static constexpr int64_t DST_BLK_STRIDE = 1;
57	- static constexpr int64_t DST_REP_STRIDE = 8;	57	+ static constexpr int64_t DST_REP_STRIDE = 8;
58	- static constexpr int32_t INT_MAX = 2147483647;	58	+ static constexpr int32_t INT_MAX = 2147483647;
59	- static constexpr uint32_t BLOCK_SIZE = 32;	59	+ static constexpr uint32_t BLOCK_SIZE = 32;
60	};	60	};
61		61
62	-	62	+} // namespace FfnWbBatching
63	-} // namespace FfnWbBatching	63	+#endif // OP_KERNEL_FFN_WB_SORT_BASE_H
64	-#endif // OP_KERNEL_FFN_WB_SORT_BASE_H

Mffn/ffn_worker_batching/op_kernel/ffn_wb_sort_mrgsort.h+23-20

	@@ -36,12 +36,12 @@ struct SortCustomMrgsortParam {
36	class SortCustomMrgsort {	36	class SortCustomMrgsort {
37	public:	37	public:
38	__aicore__ inline SortCustomMrgsort(){};	38	__aicore__ inline SortCustomMrgsort(){};
39	- __aicore__ inline void Init(SortCustomMrgsortParam* param);	39	+ __aicore__ inline void Init(SortCustomMrgsortParam *param);
40	__aicore__ inline void Process();	40	__aicore__ inline void Process();
41	- __aicore__ inline void SetInput(GlobalTensor<float>& gmInput, GlobalTensor<int32_t>& gmSortNum,	41	+ __aicore__ inline void SetInput(GlobalTensor<float> &gmInput, GlobalTensor<int32_t> &gmSortNum,
42	- LocalTensor<float>& ubInput);	42	+ LocalTensor<float> &ubInput);
43	- __aicore__ inline void SetOutput(GlobalTensor<float>& gmOutput, LocalTensor<float>& ubOutput,	43	+ __aicore__ inline void SetOutput(GlobalTensor<float> &gmOutput, LocalTensor<float> &ubOutput,
44	- LocalTensor<int32_t>& outSortNumLocal);	44	+ LocalTensor<int32_t> &outSortNumLocal);
45		45
46	private:	46	private:
47	__aicore__ inline void CopyOutSortNum();	47	__aicore__ inline void CopyOutSortNum();
	@@ -53,7 +53,7 @@ private:
53	__aicore__ inline void ClearCache();	53	__aicore__ inline void ClearCache();
54		54
55	private:	55	private:
56	- SortCustomMrgsortParam* param = nullptr;	56	+ SortCustomMrgsortParam *param = nullptr;
57		57
58	GlobalTensor<float> gmInputs[4];	58	GlobalTensor<float> gmInputs[4];
59	GlobalTensor<float> gmOutput;	59	GlobalTensor<float> gmOutput;
	@@ -85,8 +85,8 @@ __aicore__ inline void SortCustomMrgsort::ClearCache()
85	this->outOffset = 0;	85	this->outOffset = 0;
86	}	86	}
87		87
88	-__aicore__ inline void SortCustomMrgsort::SetInput(GlobalTensor<float>& gmInput, GlobalTensor<int32_t>& gmSortNum,	88	+__aicore__ inline void SortCustomMrgsort::SetInput(GlobalTensor<float> &gmInput, GlobalTensor<int32_t> &gmSortNum,
89	- LocalTensor<float>& ubInput)	89	+ LocalTensor<float> &ubInput)
90	{	90	{
91	this->gmInputs[listNum] = gmInput;	91	this->gmInputs[listNum] = gmInput;
92	this->ubInputs[listNum] = ubInput;	92	this->ubInputs[listNum] = ubInput;
	@@ -94,8 +94,8 @@ __aicore__ inline void SortCustomMrgsort::SetInput(GlobalTensor<float>& gmInput,
94	this->listNum += 1;	94	this->listNum += 1;
95	}	95	}
96		96
97	-__aicore__ inline void SortCustomMrgsort::SetOutput(GlobalTensor<float>& gmOutput, LocalTensor<float>& ubOutput,	97	+__aicore__ inline void SortCustomMrgsort::SetOutput(GlobalTensor<float> &gmOutput, LocalTensor<float> &ubOutput,
98	- LocalTensor<int32_t>& outSortNumLocal)	98	+ LocalTensor<int32_t> &outSortNumLocal)
99	{	99	{
100	this->gmOutput = gmOutput;	100	this->gmOutput = gmOutput;
101	this->ubOutput = ubOutput;	101	this->ubOutput = ubOutput;
	@@ -125,7 +125,8 @@ __aicore__ inline void SortCustomMrgsort::CopyIn()
125	for (int64_t i = 0, j = 0; i < listNum; i++) {	125	for (int64_t i = 0, j = 0; i < listNum; i++) {
126	lengths[i] = Min(param->oneLoopMaxElements, listRemainElements[i]);	126	lengths[i] = Min(param->oneLoopMaxElements, listRemainElements[i]);
127	if (lengths[i] > 0) {	127	if (lengths[i] > 0) {
128	- DataCopy(this->ubInputs[i], this->gmInputs[i][offsets[i]], Align(GetSortLen<float>(lengths[i]), sizeof(float)));	128	+ DataCopy(this->ubInputs[i], this->gmInputs[i][offsets[i]],
		129	+ Align(GetSortLen<float>(lengths[i]), sizeof(float)));
129	tmpUbInputs[j] = this->ubInputs[i];	130	tmpUbInputs[j] = this->ubInputs[i];
130	elementCountListTail[j] = lengths[i];	131	elementCountListTail[j] = lengths[i];
131	this->remainListNum += 1;	132	this->remainListNum += 1;
	@@ -141,7 +142,7 @@ __aicore__ inline void SortCustomMrgsort::MrgsortCompute()
141	MrgSortSrcList sortListTail = MrgSortSrcList(tmpUbInputs[0], tmpUbInputs[1], tmpUbInputs[0], tmpUbInputs[0]);	142	MrgSortSrcList sortListTail = MrgSortSrcList(tmpUbInputs[0], tmpUbInputs[1], tmpUbInputs[0], tmpUbInputs[0]);
142	MrgSort<float, true>(this->ubOutput, sortListTail, elementCountListTail, listSortedNums, validBitTail, 1);	143	MrgSort<float, true>(this->ubOutput, sortListTail, elementCountListTail, listSortedNums, validBitTail, 1);
143	} else if (this->remainListNum == MERGE_LIST_THREE) {	144	} else if (this->remainListNum == MERGE_LIST_THREE) {
144	- MrgSortSrcList sortListTail =	145	+ MrgSortSrcList sortListTail =
145	MrgSortSrcList(tmpUbInputs[0], tmpUbInputs[1], tmpUbInputs[MERGE_LIST_IDX_TWO], tmpUbInputs[0]);	146	MrgSortSrcList(tmpUbInputs[0], tmpUbInputs[1], tmpUbInputs[MERGE_LIST_IDX_TWO], tmpUbInputs[0]);
146	MrgSort<float, true>(this->ubOutput, sortListTail, elementCountListTail, listSortedNums, validBitTail, 1);	147	MrgSort<float, true>(this->ubOutput, sortListTail, elementCountListTail, listSortedNums, validBitTail, 1);
147	} else if (this->remainListNum == MERGE_LIST_FOUR) {	148	} else if (this->remainListNum == MERGE_LIST_FOUR) {
	@@ -150,7 +151,8 @@ __aicore__ inline void SortCustomMrgsort::MrgsortCompute()
150	MrgSort<float, true>(this->ubOutput, sortListTail, elementCountListTail, listSortedNums, validBitTail, 1);	151	MrgSort<float, true>(this->ubOutput, sortListTail, elementCountListTail, listSortedNums, validBitTail, 1);
151	} else {	152	} else {
152	if (elementCountListTail[0] > 0) {	153	if (elementCountListTail[0] > 0) {
153	- DataCopy(this->ubOutput, this->tmpUbInputs[0], Align(GetSortLen<float>(elementCountListTail[0]), sizeof(float)));	154	+ DataCopy(this->ubOutput, this->tmpUbInputs[0],
		155	+ Align(GetSortLen<float>(elementCountListTail[0]), sizeof(float)));
154	}	156	}
155	listSortedNums[0] = elementCountListTail[0];	157	listSortedNums[0] = elementCountListTail[0];
156	}	158	}
	@@ -187,19 +189,20 @@ __aicore__ inline void SortCustomMrgsort::CopyOut()
187		189
188	__aicore__ inline void SortCustomMrgsort::CopyOutSortNum()	190	__aicore__ inline void SortCustomMrgsort::CopyOutSortNum()
189	{	191	{
190	- ubOutSortNum_.SetValue(0, allRemainElements);	192	+ ubOutSortNum_.SetValue(0, allRemainElements);
191	- SetWaitFlag<HardEvent::S_MTE3>(HardEvent::S_MTE3);	193	+ SetWaitFlag<HardEvent::S_MTE3>(HardEvent::S_MTE3);
192	- DataCopyExtParams copyParams{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(int32_t)), 0, 0, 0};	194	+ DataCopyExtParams copyParams{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(int32_t)), 0, 0, 0};
193	- DataCopyPad(gmActualSortNum_, ubOutSortNum_, copyParams);	195	+ DataCopyPad(gmActualSortNum_, ubOutSortNum_, copyParams);
194	}	196	}
195		197
196	-__aicore__ inline void SortCustomMrgsort::Init(SortCustomMrgsortParam* param)	198	+__aicore__ inline void SortCustomMrgsort::Init(SortCustomMrgsortParam *param)
197	{	199	{
198	- this->param= param;	200	+ this->param = param;
199	this->remainListNum = listNum;	201	this->remainListNum = listNum;
200	for (int64_t i = 0; i < listNum; i++) {	202	for (int64_t i = 0; i < listNum; i++) {
201	offsets[i] = GetSortOffset<float>(param->perListElements * i);	203	offsets[i] = GetSortOffset<float>(param->perListElements * i);
202	- DataCacheCleanAndInvalid<int32_t, AscendC::CacheLine::SINGLE_CACHE_LINE, AscendC::DcciDst::CACHELINE_ALL>(gmActualSortNum_[param->sortNumStride * i]);	204	+ DataCacheCleanAndInvalid<int32_t, AscendC::CacheLine::SINGLE_CACHE_LINE, AscendC::DcciDst::CACHELINE_ALL>(
		205	+ gmActualSortNum_[param->sortNumStride * i]);
203	listRemainElements[i] = gmActualSortNum_.GetValue(param->sortNumStride * i);	206	listRemainElements[i] = gmActualSortNum_.GetValue(param->sortNumStride * i);
204	allRemainElements += listRemainElements[i];	207	allRemainElements += listRemainElements[i];
205	}	208	}

Mffn/ffn_worker_batching/op_kernel/ffn_wb_sort_mrgsort_out.h+15-13

	@@ -35,8 +35,8 @@ public:
35	__aicore__ inline void SetInput(GlobalTensor<float> &gmInput, GlobalTensor<int32_t> &gmSortNum,	35	__aicore__ inline void SetInput(GlobalTensor<float> &gmInput, GlobalTensor<int32_t> &gmSortNum,
36	LocalTensor<float> &ubInput);	36	LocalTensor<float> &ubInput);
37	__aicore__ inline void SetOutput(GlobalTensor<int32_t> &gmOutput1, GlobalTensor<int32_t> &gmOutput2,	37	__aicore__ inline void SetOutput(GlobalTensor<int32_t> &gmOutput1, GlobalTensor<int32_t> &gmOutput2,
38	- LocalTensor<float> &ubOutput1, LocalTensor<float> &ubOutput2);	38	+ LocalTensor<float> &ubOutput1, LocalTensor<float> &ubOutput2);
39	- __aicore__ inline void SetBuffer(LocalTensor<float> &tempBuffer);	39	+ __aicore__ inline void SetBuffer(LocalTensor<float> &tempBuffer);
40		40
41	private:	41	private:
42	__aicore__ inline void CopyIn();	42	__aicore__ inline void CopyIn();
	@@ -100,13 +100,14 @@ __aicore__ inline void SortCustomMrgsortOut::SetInput(GlobalTensor<float> &gmInp
100	this->listNum += 1;	100	this->listNum += 1;
101	}	101	}
102		102
103	-__aicore__ inline void SortCustomMrgsortOut::SetOutput(GlobalTensor<int32_t> &gmOutput1, GlobalTensor<int32_t> &gmOutput2,	103	+__aicore__ inline void SortCustomMrgsortOut::SetOutput(GlobalTensor<int32_t> &gmOutput1,
104	- LocalTensor<float> &ubOutput1, LocalTensor<float> &ubOutput2)	104	+ GlobalTensor<int32_t> &gmOutput2, LocalTensor<float> &ubOutput1,
		105	+ LocalTensor<float> &ubOutput2)
105	{	106	{
106	this->gmOutput1 = gmOutput1;	107	this->gmOutput1 = gmOutput1;
107	this->ubOutput1 = ubOutput1;	108	this->ubOutput1 = ubOutput1;
108	this->ubOutputInt1 = ubOutput1.ReinterpretCast<int32_t>();	109	this->ubOutputInt1 = ubOutput1.ReinterpretCast<int32_t>();
109	-	110	+
110	this->gmOutput2 = gmOutput2;	111	this->gmOutput2 = gmOutput2;
111	this->ubOutput2 = ubOutput2.ReinterpretCast<uint32_t>();	112	this->ubOutput2 = ubOutput2.ReinterpretCast<uint32_t>();
112	this->ubOutputInt2 = ubOutput2.ReinterpretCast<int32_t>();	113	this->ubOutputInt2 = ubOutput2.ReinterpretCast<int32_t>();
	@@ -141,7 +142,7 @@ __aicore__ inline void SortCustomMrgsortOut::CopyIn()
141	lengths[i] = Min(param->oneLoopMaxElements, listRemainElements[i]);	142	lengths[i] = Min(param->oneLoopMaxElements, listRemainElements[i]);
142	if (lengths[i] > 0) {	143	if (lengths[i] > 0) {
143	DataCopy(this->ubInputs[i], this->gmInputs[i][offsets[i]],	144	DataCopy(this->ubInputs[i], this->gmInputs[i][offsets[i]],
144	- Align(GetSortLen<float>(lengths[i]), sizeof(float)));	145	+ Align(GetSortLen<float>(lengths[i]), sizeof(float)));
145	tmpUbInputs[j] = this->ubInputs[i];	146	tmpUbInputs[j] = this->ubInputs[i];
146	elementCountListTail[j] = lengths[i];	147	elementCountListTail[j] = lengths[i];
147	this->remainListNum += 1;	148	this->remainListNum += 1;
	@@ -157,7 +158,7 @@ __aicore__ inline void SortCustomMrgsortOut::MrgsortCompute()
157	MrgSortSrcList sortListTail = MrgSortSrcList(tmpUbInputs[0], tmpUbInputs[1], tmpUbInputs[0], tmpUbInputs[0]);	158	MrgSortSrcList sortListTail = MrgSortSrcList(tmpUbInputs[0], tmpUbInputs[1], tmpUbInputs[0], tmpUbInputs[0]);
158	MrgSort<float, true>(this->tempBuffer, sortListTail, elementCountListTail, listSortedNums, validBitTail, 1);	159	MrgSort<float, true>(this->tempBuffer, sortListTail, elementCountListTail, listSortedNums, validBitTail, 1);
159	} else if (this->remainListNum == MERGE_LIST_THREE) {	160	} else if (this->remainListNum == MERGE_LIST_THREE) {
160	- MrgSortSrcList sortListTail =	161	+ MrgSortSrcList sortListTail =
161	MrgSortSrcList(tmpUbInputs[0], tmpUbInputs[1], tmpUbInputs[MERGE_LIST_IDX_TWO], tmpUbInputs[0]);	162	MrgSortSrcList(tmpUbInputs[0], tmpUbInputs[1], tmpUbInputs[MERGE_LIST_IDX_TWO], tmpUbInputs[0]);
162	MrgSort<float, true>(this->tempBuffer, sortListTail, elementCountListTail, listSortedNums, validBitTail, 1);	163	MrgSort<float, true>(this->tempBuffer, sortListTail, elementCountListTail, listSortedNums, validBitTail, 1);
163	} else if (this->remainListNum == MERGE_LIST_FOUR) {	164	} else if (this->remainListNum == MERGE_LIST_FOUR) {
	@@ -167,7 +168,7 @@ __aicore__ inline void SortCustomMrgsortOut::MrgsortCompute()
167	} else {	168	} else {
168	if (elementCountListTail[0] > 0) {	169	if (elementCountListTail[0] > 0) {
169	DataCopy(this->tempBuffer, this->tmpUbInputs[0],	170	DataCopy(this->tempBuffer, this->tmpUbInputs[0],
170	- Align(GetSortLen<float>(elementCountListTail[0]), sizeof(float)));	171	+ Align(GetSortLen<float>(elementCountListTail[0]), sizeof(float)));
171	}	172	}
172	listSortedNums[0] = elementCountListTail[0];	173	listSortedNums[0] = elementCountListTail[0];
173	}	174	}
	@@ -189,11 +190,12 @@ __aicore__ inline void SortCustomMrgsortOut::UpdateSortInfo()
189	}	190	}
190	}	191	}
191	}	192	}
192	-	193	+
193	__aicore__ inline void SortCustomMrgsortOut::Extract()	194	__aicore__ inline void SortCustomMrgsortOut::Extract()
194	{	195	{
195	if (curLoopSortedNum > 0) {	196	if (curLoopSortedNum > 0) {
196	- AscendC::Extract(this->ubOutput1, this->ubOutput2, this->tempBuffer, Ceil(curLoopSortedNum, ONE_REPEAT_SORT_NUM));	197	+ AscendC::Extract(this->ubOutput1, this->ubOutput2, this->tempBuffer,
		198	+ Ceil(curLoopSortedNum, ONE_REPEAT_SORT_NUM));
197	PipeBarrier<PIPE_V>();	199	PipeBarrier<PIPE_V>();
198	Muls(this->ubOutput1, this->ubOutput1, (float)-1, curLoopSortedNum);	200	Muls(this->ubOutput1, this->ubOutput1, (float)-1, curLoopSortedNum);
199	PipeBarrier<PIPE_V>();	201	PipeBarrier<PIPE_V>();
	@@ -216,11 +218,12 @@ __aicore__ inline void SortCustomMrgsortOut::CopyOut()
216		218
217	__aicore__ inline void SortCustomMrgsortOut::Init(SortCustomMrgsortParam param, TPipe tPipe)	219	__aicore__ inline void SortCustomMrgsortOut::Init(SortCustomMrgsortParam param, TPipe tPipe)
218	{	220	{
219	- this->param= param;	221	+ this->param = param;
220	this->allRemainElements = 0;	222	this->allRemainElements = 0;
221	for (int64_t i = 0; i < listNum; i++) {	223	for (int64_t i = 0; i < listNum; i++) {
222	offsets[i] = GetSortOffset<float>(param->perListElements * i);	224	offsets[i] = GetSortOffset<float>(param->perListElements * i);
223	- DataCacheCleanAndInvalid<int32_t, AscendC::CacheLine::SINGLE_CACHE_LINE, AscendC::DcciDst::CACHELINE_ALL>(gmActualSortNum_[param->sortNumStride * i]);	225	+ DataCacheCleanAndInvalid<int32_t, AscendC::CacheLine::SINGLE_CACHE_LINE, AscendC::DcciDst::CACHELINE_ALL>(
		226	+ gmActualSortNum_[param->sortNumStride * i]);
224	listRemainElements[i] = static_cast<int64_t>(gmActualSortNum_.GetValue(param->sortNumStride * i));	227	listRemainElements[i] = static_cast<int64_t>(gmActualSortNum_.GetValue(param->sortNumStride * i));
225	allRemainElements += listRemainElements[i];	228	allRemainElements += listRemainElements[i];
226	}	229	}
	@@ -240,4 +243,3 @@ __aicore__ inline void SortCustomMrgsortOut::Process()
240	}	243	}
241	} // namespace FfnWbBatching	244	} // namespace FfnWbBatching
242	#endif // OP_KERNEL_FFN_WB_SORT_MRGSORT_OUT_H	245	#endif // OP_KERNEL_FFN_WB_SORT_MRGSORT_OUT_H
243	-

Mffn/ffn_worker_batching/op_kernel/ffn_wb_sort_multi_core.h+71-73

	@@ -45,42 +45,42 @@ private:
45	__aicore__ inline void VBSCopyIn(int64_t progress, int64_t size, int64_t sortNum);	45	__aicore__ inline void VBSCopyIn(int64_t progress, int64_t size, int64_t sortNum);
46	__aicore__ inline void UBSortCompute(int64_t progress, int64_t size, int64_t sortNum);	46	__aicore__ inline void UBSortCompute(int64_t progress, int64_t size, int64_t sortNum);
47	__aicore__ inline void VBSCopyOut(int64_t progress, int64_t size, int64_t sortNum);	47	__aicore__ inline void VBSCopyOut(int64_t progress, int64_t size, int64_t sortNum);
48	- __aicore__ inline void InitSortMaskMrgSort(SortCustomMrgsort *sorter, int64_t listNum,	48	+ __aicore__ inline void InitSortMaskMrgSort(SortCustomMrgsort *sorter, int64_t listNum, int64_t coreOffset,
49	- int64_t coreOffset, int64_t sortNumCoreOffset, int64_t loopOffset, int64_t loopIdxOffset);	49	+ int64_t sortNumCoreOffset, int64_t loopOffset, int64_t loopIdxOffset);
50	__aicore__ inline void InitSortMaskMrgSortOut(SortCustomMrgsortOut *sorter, int64_t listNum, int64_t coreOffset);	50	__aicore__ inline void InitSortMaskMrgSortOut(SortCustomMrgsortOut *sorter, int64_t listNum, int64_t coreOffset);
51	__aicore__ inline void CopyOutValidCount();	51	__aicore__ inline void CopyOutValidCount();
52		52
53	private:	53	private:
54	- GlobalTensor<float> workspaceGms[2];	54	+ GlobalTensor<float> workspaceGms[2];
55	- GlobalTensor<int32_t> workspaceSortNumGm_;	55	+ GlobalTensor<int32_t> workspaceSortNumGm_;
56		56
57	- SortCustomTilingDataKernel *tilingData_ = nullptr;	57	+ SortCustomTilingDataKernel *tilingData_ = nullptr;
58	- const ScheduleContextInfo *contextInfo_ = nullptr;	58	+ const ScheduleContextInfo *contextInfo_ = nullptr;
59		59
60	- int32_t totalValidCnt_ = 0;	60	+ int32_t totalValidCnt_ = 0;
61	- int32_t curValidCnt_ = 0;	61	+ int32_t curValidCnt_ = 0;
62		62
63	- SortCustomMrgsort mrgsorter;	63	+ SortCustomMrgsort mrgsorter;
64	- SortCustomMrgsortParam mrgsortParam;	64	+ SortCustomMrgsortParam mrgsortParam;
65		65
66	- int64_t blockIdx_ = 0;	66	+ int64_t blockIdx_ = 0;
67	- int64_t srcWsIndex = 0;	67	+ int64_t srcWsIndex = 0;
68		68
69	- int64_t listNum;	69	+ int64_t listNum;
70	- int64_t perListElements;	70	+ int64_t perListElements;
71	- int64_t lastListElements;	71	+ int64_t lastListElements;
72	- int64_t vmsSortNumStride_ = 0;	72	+ int64_t vmsSortNumStride_ = 0;
73		73
74	- int64_t sortTotalLength;	74	+ int64_t sortTotalLength;
75	- int64_t sortCoreLoops;	75	+ int64_t sortCoreLoops;
76	- int64_t sortCoreLoopElements;	76	+ int64_t sortCoreLoopElements;
77	- int64_t sortCoreLastLoopElements;	77	+ int64_t sortCoreLastLoopElements;
78		78
79	- int64_t perCoreExpert;	79	+ int64_t perCoreExpert;
80	- int64_t needInitExpertCore;	80	+ int64_t needInitExpertCore;
81	- int64_t currentCoreExpert;	81	+ int64_t currentCoreExpert;
82		82
83	- static constexpr int64_t MAX_MRGSORT_LIST = 4;	83	+ static constexpr int64_t MAX_MRGSORT_LIST = 4;
84	};	84	};
85		85
86	__aicore__ inline void KernelSortMaskMultiCore::VBSCopyIn(int64_t progress, int64_t size, int64_t sortNum)	86	__aicore__ inline void KernelSortMaskMultiCore::VBSCopyIn(int64_t progress, int64_t size, int64_t sortNum)
	@@ -112,19 +112,16 @@ __aicore__ inline void KernelSortMaskMultiCore::UBSortCompute(int64_t progress,
112	Muls(expertIdsLocalFp32, expertIdsLocalFp32, (float)-1, size);	112	Muls(expertIdsLocalFp32, expertIdsLocalFp32, (float)-1, size);
113		113
114	LocalTensor<uint8_t> maskLocalTensorUInt8 = maskLocalTensor.ReinterpretCast<uint8_t>();	114	LocalTensor<uint8_t> maskLocalTensorUInt8 = maskLocalTensor.ReinterpretCast<uint8_t>();
115	- AscendC::CompareScalar(maskLocalTensorUInt8,	115	+ AscendC::CompareScalar(maskLocalTensorUInt8, expertIdsLocalFp32, static_cast<float>(-expertStart_),
116	- expertIdsLocalFp32,	116	+ AscendC::CMPMODE::GT,
117	- static_cast<float>(-expertStart_),	117	+ (size + ONE_REPEAT_COMPARE_NUM - 1) / ONE_REPEAT_COMPARE_NUM * ONE_REPEAT_COMPARE_NUM);
118	- AscendC::CMPMODE::GT,
119	- (size + ONE_REPEAT_COMPARE_NUM - 1) / ONE_REPEAT_COMPARE_NUM * ONE_REPEAT_COMPARE_NUM);
120		118
121	GatherMaskParams gatherMaskParams;	119	GatherMaskParams gatherMaskParams;
122	gatherMaskParams.repeatTimes = 1;	120	gatherMaskParams.repeatTimes = 1;
123	gatherMaskParams.src0BlockStride = 1;	121	gatherMaskParams.src0BlockStride = 1;
124	gatherMaskParams.src0RepeatStride = 8; // 8 blocks	122	gatherMaskParams.src0RepeatStride = 8; // 8 blocks
125	gatherMaskParams.src1RepeatStride = 0;	123	gatherMaskParams.src1RepeatStride = 0;
126	- GatherMask (	124	+ GatherMask(expertIdsLocalFp32, expertIdsLocalFp32, maskLocalTensor, true, size, gatherMaskParams, rsvdCnt);
127	- expertIdsLocalFp32, expertIdsLocalFp32, maskLocalTensor, true, size, gatherMaskParams, rsvdCnt);
128	curValidCnt_ = rsvdCnt;	125	curValidCnt_ = rsvdCnt;
129	if (rsvdCnt == 0) {	126	if (rsvdCnt == 0) {
130	sortDataCopyInQueue.FreeTensor(inLocal);	127	sortDataCopyInQueue.FreeTensor(inLocal);
	@@ -147,7 +144,7 @@ __aicore__ inline void KernelSortMaskMultiCore::UBSortCompute(int64_t progress,
147	LocalTensor<float> concatLocal = expertIdsLocalFp32;	144	LocalTensor<float> concatLocal = expertIdsLocalFp32;
148	LocalTensor<float> sortedLocal = sortedBuffer.Get<float>(GetSortLen<float>(sortNum));	145	LocalTensor<float> sortedLocal = sortedBuffer.Get<float>(GetSortLen<float>(sortNum));
149	LocalTensor<float> outLocal = sortDataCopyOutQueue.AllocTensor<float>();	146	LocalTensor<float> outLocal = sortDataCopyOutQueue.AllocTensor<float>();
150	- Sort<float, true>(outLocal, concatLocal, rowIdsLocal,sortedLocal, selectedCnt / ONE_REPEAT_SORT_NUM);	147	+ Sort<float, true>(outLocal, concatLocal, rowIdsLocal, sortedLocal, selectedCnt / ONE_REPEAT_SORT_NUM);
151		148
152	sortDataCopyOutQueue.EnQue<float>(outLocal);	149	sortDataCopyOutQueue.EnQue<float>(outLocal);
153	sortDataCopyInQueue.FreeTensor(inLocal);	150	sortDataCopyInQueue.FreeTensor(inLocal);
	@@ -157,10 +154,10 @@ __aicore__ inline void KernelSortMaskMultiCore::VBSCopyOut(int64_t progress, int
157	{	154	{
158	if (curValidCnt_ > 0) {	155	if (curValidCnt_ > 0) {
159	LocalTensor<float> outLocal = sortDataCopyOutQueue.DeQue<float>();	156	LocalTensor<float> outLocal = sortDataCopyOutQueue.DeQue<float>();
160	- DataCopyExtParams copyParams{1, static_cast<uint32_t>(GetSortLen<float>(curValidCnt_) * sizeof(int32_t)),	157	+ DataCopyExtParams copyParams{1, static_cast<uint32_t>(GetSortLen<float>(curValidCnt_) * sizeof(int32_t)), 0, 0,
161	- 0, 0, 0};	158	+ 0};
162	- int64_t wkOffset = blockIdx_ * GetSortLen<float>(tilingData_->perCoreElements) +	159	+ int64_t wkOffset = blockIdx_ * GetSortLen<float>(tilingData_->perCoreElements) +
163	- GetSortLen<float>(progress * sortCoreLoopElements);	160	+ GetSortLen<float>(progress * sortCoreLoopElements);
164	DataCopyPad(workspaceGms[0][wkOffset], outLocal, copyParams);	161	DataCopyPad(workspaceGms[0][wkOffset], outLocal, copyParams);
165	sortDataCopyOutQueue.FreeTensor(outLocal);	162	sortDataCopyOutQueue.FreeTensor(outLocal);
166	}	163	}
	@@ -169,16 +166,16 @@ __aicore__ inline void KernelSortMaskMultiCore::VBSCopyOut(int64_t progress, int
169	tempTensor.SetValue(0, curValidCnt_);	166	tempTensor.SetValue(0, curValidCnt_);
170	SetWaitFlag<HardEvent::S_MTE3>(HardEvent::S_MTE3);	167	SetWaitFlag<HardEvent::S_MTE3>(HardEvent::S_MTE3);
171	DataCopyExtParams copyParams1{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(int32_t)), 0, 0, 0};	168	DataCopyExtParams copyParams1{static_cast<uint16_t>(1), static_cast<uint32_t>(sizeof(int32_t)), 0, 0, 0};
172	- DataCopyPad(workspaceSortNumGm_[blockIdx_ * tilingData_->sortNumWorkSpacePerCore + progress],	169	+ DataCopyPad(workspaceSortNumGm_[blockIdx_ * tilingData_->sortNumWorkSpacePerCore + progress], tempTensor,
173	- tempTensor, copyParams1);	170	+ copyParams1);
174	}	171	}
175		172
176	-__aicore__ inline void KernelSortMaskMultiCore::InitSortMaskMrgSort(SortCustomMrgsort * sorter, int64_t listNum,	173	+__aicore__ inline void KernelSortMaskMultiCore::InitSortMaskMrgSort(SortCustomMrgsort *sorter, int64_t listNum,
177	- int64_t coreOffset, int64_t sortNumCoreOffset, int64_t loopOffset, int64_t loopIdxOffset)	174	+ int64_t coreOffset, int64_t sortNumCoreOffset,
		175	+ int64_t loopOffset, int64_t loopIdxOffset)
178	{	176	{
179	GlobalTensor<float> srcWsGm = workspaceGms[srcWsIndex][blockIdx_ * coreOffset + loopOffset];	177	GlobalTensor<float> srcWsGm = workspaceGms[srcWsIndex][blockIdx_ * coreOffset + loopOffset];
180	- GlobalTensor<int32_t> srcSortNumGm = workspaceSortNumGm_[blockIdx_ * sortNumCoreOffset +	178	+ GlobalTensor<int32_t> srcSortNumGm = workspaceSortNumGm_[blockIdx_ * sortNumCoreOffset + loopIdxOffset];
181	- loopIdxOffset];
182	LocalTensor<float> inLocal = sortDataCopyInQueue.AllocTensor<float>();	179	LocalTensor<float> inLocal = sortDataCopyInQueue.AllocTensor<float>();
183	LocalTensor<float> outLocal = sortDataCopyOutQueue.AllocTensor<float>();	180	LocalTensor<float> outLocal = sortDataCopyOutQueue.AllocTensor<float>();
184	for (int64_t i = 0; i < listNum; i++) {	181	for (int64_t i = 0; i < listNum; i++) {
	@@ -193,8 +190,8 @@ __aicore__ inline void KernelSortMaskMultiCore::InitSortMaskMrgSort(SortCustomMr
193	tempBuffer.FreeTensor(outSortNumLocal);	190	tempBuffer.FreeTensor(outSortNumLocal);
194	}	191	}
195		192
196	-__aicore__ inline void KernelSortMaskMultiCore::InitSortMaskMrgSortOut(	193	+__aicore__ inline void KernelSortMaskMultiCore::InitSortMaskMrgSortOut(SortCustomMrgsortOut *sorter, int64_t listNum,
197	- SortCustomMrgsortOut *sorter, int64_t listNum, int64_t coreOffset)	194	+ int64_t coreOffset)
198	{	195	{
199	GlobalTensor<float> srcWsGm = workspaceGms[srcWsIndex];	196	GlobalTensor<float> srcWsGm = workspaceGms[srcWsIndex];
200	GlobalTensor<int32_t> srcSortNumGm = workspaceSortNumGm_;	197	GlobalTensor<int32_t> srcSortNumGm = workspaceSortNumGm_;
	@@ -209,15 +206,15 @@ __aicore__ inline void KernelSortMaskMultiCore::InitSortMaskMrgSortOut(
209	LocalTensor<float> outLocalV = outLocal[tilingData_->oneLoopMaxElementsMrg * MAX_MRGSORT_LIST];	206	LocalTensor<float> outLocalV = outLocal[tilingData_->oneLoopMaxElementsMrg * MAX_MRGSORT_LIST];
210	sorter->SetOutput(this->sortedexpertIdsGm, this->sortedRowIdsGm, outLocal, outLocalV);	207	sorter->SetOutput(this->sortedexpertIdsGm, this->sortedRowIdsGm, outLocal, outLocalV);
211		208
212	- LocalTensor<float> tempBuffer =	209	+ LocalTensor<float> tempBuffer =
213	sortedBuffer.Get<float>(GetSortLen<float>(tilingData_->oneLoopMaxElementsMrg) * MAX_MRGSORT_LIST);	210	sortedBuffer.Get<float>(GetSortLen<float>(tilingData_->oneLoopMaxElementsMrg) * MAX_MRGSORT_LIST);
214	sorter->SetBuffer(tempBuffer);	211	sorter->SetBuffer(tempBuffer);
215	sortDataCopyInQueue.FreeTensor(inLocal);	212	sortDataCopyInQueue.FreeTensor(inLocal);
216	sortDataCopyOutQueue.FreeTensor(outLocal);	213	sortDataCopyOutQueue.FreeTensor(outLocal);
217	}	214	}
218		215
219	-__aicore__ inline void KernelSortMaskMultiCore::OneCoreVMSProcess(	216	+__aicore__ inline void KernelSortMaskMultiCore::OneCoreVMSProcess(int64_t listNum, int64_t perListElements,
220	- int64_t listNum, int64_t perListElements, int64_t lastListElements)	217	+ int64_t lastListElements)
221	{	218	{
222	int64_t coreOffset = GetSortLen<float>(tilingData_->perCoreElements);	219	int64_t coreOffset = GetSortLen<float>(tilingData_->perCoreElements);
223	int64_t sortNumCoreOffset = tilingData_->sortNumWorkSpacePerCore;	220	int64_t sortNumCoreOffset = tilingData_->sortNumWorkSpacePerCore;
	@@ -232,16 +229,16 @@ __aicore__ inline void KernelSortMaskMultiCore::OneCoreVMSProcess(
232	mrgsortParam.sortNumStride = curSortNumStride;	229	mrgsortParam.sortNumStride = curSortNumStride;
233		230
234	int64_t loopOffset = GetSortLen<float>(mrgsortParam.perListElements * MAX_MRGSORT_LIST);	231	int64_t loopOffset = GetSortLen<float>(mrgsortParam.perListElements * MAX_MRGSORT_LIST);
235	- int64_t loopIdxOffset = mrgsortParam.sortNumStride * MAX_MRGSORT_LIST;	232	+ int64_t loopIdxOffset = mrgsortParam.sortNumStride * MAX_MRGSORT_LIST;
236	for (int64_t loop = 0; loop < loops - 1; loop++) {	233	for (int64_t loop = 0; loop < loops - 1; loop++) {
237	- InitSortMaskMrgSort(&mrgsorter, MAX_MRGSORT_LIST, coreOffset, sortNumCoreOffset,	234	+ InitSortMaskMrgSort(&mrgsorter, MAX_MRGSORT_LIST, coreOffset, sortNumCoreOffset, loop * loopOffset,
238	- loop * loopOffset, loop * loopIdxOffset);	235	+ loop * loopIdxOffset);
239	mrgsorter.Init(&mrgsortParam);	236	mrgsorter.Init(&mrgsortParam);
240	mrgsorter.Process();	237	mrgsorter.Process();
241	}	238	}
242		239
243	- InitSortMaskMrgSort(&mrgsorter, remainListNum, coreOffset, sortNumCoreOffset,	240	+ InitSortMaskMrgSort(&mrgsorter, remainListNum, coreOffset, sortNumCoreOffset, (loops - 1) * loopOffset,
244	- (loops - 1) * loopOffset, (loops - 1) * loopIdxOffset);	241	+ (loops - 1) * loopIdxOffset);
245	mrgsorter.Init(&mrgsortParam);	242	mrgsorter.Init(&mrgsortParam);
246	mrgsorter.Process();	243	mrgsorter.Process();
247		244
	@@ -346,8 +343,9 @@ __aicore__ inline void KernelSortMaskMultiCore::CopyOutValidCount()
346	sortDataCopyOutQueue.FreeTensor(outLocal);	343	sortDataCopyOutQueue.FreeTensor(outLocal);
347	}	344	}
348		345
349	-__aicore__ inline void KernelSortMaskMultiCore::Init(GM_ADDR expert_ids, GM_ADDR workspace,	346	+__aicore__ inline void KernelSortMaskMultiCore::Init(GM_ADDR expert_ids, GM_ADDR workspace,
350	- SortCustomTilingDataKernel tilingData, const ScheduleContextInfo contextInfo, TPipe *tPipe)	347	+ SortCustomTilingDataKernel *tilingData,
		348	+ const ScheduleContextInfo contextInfo, TPipe tPipe)
351	{	349	{
352	this->pipe = tPipe;	350	this->pipe = tPipe;
353	contextInfo_ = contextInfo;	351	contextInfo_ = contextInfo;
	@@ -368,17 +366,17 @@ __aicore__ inline void KernelSortMaskMultiCore::Init(GM_ADDR expert_ids, GM_ADDR
368	this->sortTotalLength = tilingData_->perCoreElements;	366	this->sortTotalLength = tilingData_->perCoreElements;
369	}	367	}
370		368
371	- expertIdsGm.SetGlobalBuffer(	369	+ expertIdsGm.SetGlobalBuffer((__gm__ int32_t )expert_ids + blockIdx_ tilingData_->perCoreElements,
372	- (__gm__ int32_t )expert_ids + blockIdx_ tilingData_->perCoreElements, this->sortTotalLength);	370	+ this->sortTotalLength);
373	- rsvdCntGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t*>(workspace), OFFSET_SORTED_EXPERT_IDS);	371	+ rsvdCntGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace), OFFSET_SORTED_EXPERT_IDS);
374	workspaceSortNumGm_.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS,	372	workspaceSortNumGm_.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS,
375	contextInfo_->sortNumWorkSpace);	373	contextInfo_->sortNumWorkSpace);
376		374
377	if (blockIdx_ == 0) {	375	if (blockIdx_ == 0) {
378	InitGlobalMemory(rsvdCntGm, OFFSET_SORTED_EXPERT_IDS, 0);	376	InitGlobalMemory(rsvdCntGm, OFFSET_SORTED_EXPERT_IDS, 0);
379	- GM_ADDR targetAddr = workspace + OFFSET_SORTED_EXPERT_IDS * sizeof(int32_t) +	377	+ GM_ADDR targetAddr = workspace + OFFSET_SORTED_EXPERT_IDS * sizeof(int32_t) +
380	contextInfo_->sortNumWorkSpace * sizeof(int32_t) * (NUM_TWO * NUM_FOUR + 1);	378	contextInfo_->sortNumWorkSpace * sizeof(int32_t) * (NUM_TWO * NUM_FOUR + 1);
381	- groupListTmpGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t*>(targetAddr), contextInfo_->expertNum);	379	+ groupListTmpGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(targetAddr), contextInfo_->expertNum);
382	InitGlobalMemory(groupListTmpGm, contextInfo_->expertNum, 0);	380	InitGlobalMemory(groupListTmpGm, contextInfo_->expertNum, 0);
383	SetWaitFlag<HardEvent::MTE3_MTE2>(HardEvent::MTE3_MTE2);	381	SetWaitFlag<HardEvent::MTE3_MTE2>(HardEvent::MTE3_MTE2);
384	}	382	}
	@@ -387,21 +385,21 @@ __aicore__ inline void KernelSortMaskMultiCore::Init(GM_ADDR expert_ids, GM_ADDR
387	sortedexpertIdsGm.SetGlobalBuffer(	385	sortedexpertIdsGm.SetGlobalBuffer(
388	reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS + contextInfo_->sortNumWorkSpace,	386	reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS + contextInfo_->sortNumWorkSpace,
389	this->totalLength);	387	this->totalLength);
390	- sortedRowIdsGm.SetGlobalBuffer(	388	+ sortedRowIdsGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS +
391	- reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS + contextInfo_->sortNumWorkSpace +	389	+ contextInfo_->sortNumWorkSpace + this->totalLength,
392	- this->totalLength, this->totalLength);	390	+ this->totalLength);
393	-	391	+
394	int64_t kvFactor = 2;	392	int64_t kvFactor = 2;
395	- workspaceGms[0].SetGlobalBuffer(	393	+ workspaceGms[0].SetGlobalBuffer(reinterpret_cast<__gm__ float *>(workspace) + OFFSET_SORTED_EXPERT_IDS +
396	- reinterpret_cast<__gm__ float *>(workspace) + OFFSET_SORTED_EXPERT_IDS + contextInfo_->sortNumWorkSpace +	394	+ contextInfo_->sortNumWorkSpace + this->totalLength * kvFactor,
397	- this->totalLength * kvFactor, this->totalLength * kvFactor);	395	+ this->totalLength * kvFactor);
398	- workspaceGms[1].SetGlobalBuffer(	396	+ workspaceGms[1].SetGlobalBuffer(reinterpret_cast<__gm__ float *>(workspace) + OFFSET_SORTED_EXPERT_IDS +
399	- reinterpret_cast<__gm__ float *>(workspace) + OFFSET_SORTED_EXPERT_IDS + contextInfo_->sortNumWorkSpace +	397	+ contextInfo_->sortNumWorkSpace + this->totalLength * (kvFactor + kvFactor),
400	- this->totalLength * (kvFactor + kvFactor), this->totalLength * kvFactor);	398	+ this->totalLength * kvFactor);
401	-	399	+
402	- int64_t bufferSize = Ceil(Max(tilingData_->oneLoopMaxElementsMrg * MAX_MRGSORT_LIST, sortCoreLoopElements),	400	+ int64_t bufferSize =
403	- ONE_REPEAT_SORT_NUM) *	401	+ Ceil(Max(tilingData_->oneLoopMaxElementsMrg * MAX_MRGSORT_LIST, sortCoreLoopElements), ONE_REPEAT_SORT_NUM) *
404	- ONE_REPEAT_SORT_NUM * sizeof(int32_t) * kvFactor;	402	+ ONE_REPEAT_SORT_NUM * sizeof(int32_t) * kvFactor;
405	pipe->InitBuffer(sortDataCopyInQueue, bufferNum, bufferSize);	403	pipe->InitBuffer(sortDataCopyInQueue, bufferNum, bufferSize);
406	pipe->InitBuffer(sortDataCopyOutQueue, bufferNum, bufferSize);	404	pipe->InitBuffer(sortDataCopyOutQueue, bufferNum, bufferSize);
407	pipe->InitBuffer(sortedBuffer, bufferSize);	405	pipe->InitBuffer(sortedBuffer, bufferSize);

Mffn/ffn_worker_batching/op_kernel/ffn_wb_sort_one_core.h+15-15

	@@ -24,7 +24,6 @@
24	#include "ffn_wb_sort_base.h"	24	#include "ffn_wb_sort_base.h"
25	#include "ffn_wb_common.h"	25	#include "ffn_wb_common.h"
26		26
27	-
28	namespace FfnWbBatching {	27	namespace FfnWbBatching {
29	using namespace AscendC;	28	using namespace AscendC;
30		29
	@@ -51,8 +50,8 @@ private:
51	__aicore__ inline void KernelSortMaskOneCore::CopyIn()	50	__aicore__ inline void KernelSortMaskOneCore::CopyIn()
52	{	51	{
53	LocalTensor<int32_t> inLocal = sortDataCopyInQueue.AllocTensor<int32_t>();	52	LocalTensor<int32_t> inLocal = sortDataCopyInQueue.AllocTensor<int32_t>();
54	- DataCopyExtParams dataCopyParams{	53	+ DataCopyExtParams dataCopyParams{static_cast<uint16_t>(1),
55	- static_cast<uint16_t>(1), static_cast<uint32_t>(this->totalLength * sizeof(int32_t)), 0, 0, 0};	54	+ static_cast<uint32_t>(this->totalLength * sizeof(int32_t)), 0, 0, 0};
56	DataCopyPadExtParams dataCopyPadParams{false, 0, 0, 0};	55	DataCopyPadExtParams dataCopyPadParams{false, 0, 0, 0};
57	DataCopyPad(inLocal[0], expertIdsGm, dataCopyParams, dataCopyPadParams);	56	DataCopyPad(inLocal[0], expertIdsGm, dataCopyParams, dataCopyPadParams);
58		57
	@@ -75,12 +74,10 @@ __aicore__ inline void KernelSortMaskOneCore::SortCompute()
75	PipeBarrier<PIPE_V>();	74	PipeBarrier<PIPE_V>();
76		75
77	LocalTensor<uint8_t> maskLocalTensorUInt8 = maskLocalUInt32.ReinterpretCast<uint8_t>();	76	LocalTensor<uint8_t> maskLocalTensorUInt8 = maskLocalUInt32.ReinterpretCast<uint8_t>();
78	- AscendC::CompareScalar(maskLocalTensorUInt8,	77	+ AscendC::CompareScalar(
79	- expertIdsFp32,	78	+ maskLocalTensorUInt8, expertIdsFp32, static_cast<float>(-expertStart_), AscendC::CMPMODE::GT,
80	- static_cast<float>(-expertStart_),
81	- AscendC::CMPMODE::GT,
82	(this->totalLength + ONE_REPEAT_COMPARE_NUM - 1) / ONE_REPEAT_COMPARE_NUM * ONE_REPEAT_COMPARE_NUM);	79	(this->totalLength + ONE_REPEAT_COMPARE_NUM - 1) / ONE_REPEAT_COMPARE_NUM * ONE_REPEAT_COMPARE_NUM);
83	- PipeBarrier<PIPE_V>();	80	+ PipeBarrier<PIPE_V>();
84		81
85	GatherMaskParams gatherMaskParams;	82	GatherMaskParams gatherMaskParams;
86	gatherMaskParams.repeatTimes = 1;	83	gatherMaskParams.repeatTimes = 1;
	@@ -144,11 +141,13 @@ __aicore__ inline void KernelSortMaskOneCore::CopyOut()
144	}	141	}
145		142
146	rsvdCntGm.SetValue(0, this->validCnt);	143	rsvdCntGm.SetValue(0, this->validCnt);
147	- DataCacheCleanAndInvalid<int32_t, AscendC::CacheLine::SINGLE_CACHE_LINE, AscendC::DcciDst::CACHELINE_ALL>(rsvdCntGm);	144	+ DataCacheCleanAndInvalid<int32_t, AscendC::CacheLine::SINGLE_CACHE_LINE, AscendC::DcciDst::CACHELINE_ALL>(
		145	+ rsvdCntGm);
148	}	146	}
149		147
150	-__aicore__ inline void KernelSortMaskOneCore::Init(GM_ADDR expert_ids, GM_ADDR workspace,	148	+__aicore__ inline void KernelSortMaskOneCore::Init(GM_ADDR expert_ids, GM_ADDR workspace,
151	- const SortCustomTilingDataKernel tilingData, const ScheduleContextInfo contextInfo, TPipe *tPipe)	149	+ const SortCustomTilingDataKernel *tilingData,
		150	+ const ScheduleContextInfo contextInfo, TPipe tPipe)
152	{	151	{
153	this->pipe = tPipe;	152	this->pipe = tPipe;
154	contextInfo_ = contextInfo;	153	contextInfo_ = contextInfo;
	@@ -159,10 +158,12 @@ __aicore__ inline void KernelSortMaskOneCore::Init(GM_ADDR expert_ids, GM_ADDR w
159		158
160	expertIdsGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(expert_ids), this->totalLength);	159	expertIdsGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(expert_ids), this->totalLength);
161	rsvdCntGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace), OFFSET_SORTED_EXPERT_IDS);	160	rsvdCntGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace), OFFSET_SORTED_EXPERT_IDS);
162	- sortedexpertIdsGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS +	161	+ sortedexpertIdsGm.SetGlobalBuffer(
163	- contextInfo_->sortNumWorkSpace, this->totalLength);	162	+ reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS + contextInfo_->sortNumWorkSpace,
		163	+ this->totalLength);
164	sortedRowIdsGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS +	164	sortedRowIdsGm.SetGlobalBuffer(reinterpret_cast<__gm__ int32_t *>(workspace) + OFFSET_SORTED_EXPERT_IDS +
165	- contextInfo_->sortNumWorkSpace + this->totalLength, this->totalLength);	165	+ contextInfo_->sortNumWorkSpace + this->totalLength,
		166	+ this->totalLength);
166		167
167	if (GetBlockIdx() == 0) {	168	if (GetBlockIdx() == 0) {
168	GM_ADDR targetAddr = workspace + OFFSET_SORTED_EXPERT_IDS * sizeof(int32_t) +	169	GM_ADDR targetAddr = workspace + OFFSET_SORTED_EXPERT_IDS * sizeof(int32_t) +
	@@ -197,4 +198,3 @@ __aicore__ inline void KernelSortMaskOneCore::Process()
197	}	198	}
198	} // namespace FfnWbBatching	199	} // namespace FfnWbBatching
199	#endif // OP_KERNEL_FFN_WB_SORT_ONE_CORE_H	200	#endif // OP_KERNEL_FFN_WB_SORT_ONE_CORE_H
200	-

Mffn/ffn_worker_scheduler/op_host/op_api/ffn_worker_scheduler.h+5-6

	@@ -17,10 +17,9 @@
17		17
18	#include "opdev/op_executor.h"	18	#include "opdev/op_executor.h"
19		19
20	-namespace l0op	20	+namespace l0op {
21	-{	21	+const aclTensor FfnWorkerScheduler(const aclTensor scheduleContext, int32_t syncGroupSize, int32_t executeMode,
22	-const aclTensor* FfnWorkerScheduler(const aclTensor* scheduleContext, int32_t syncGroupSize, int32_t executeMode,	22	+ aclTensor scheduleContextOut, aclOpExecutor executor);
23	- aclTensor* scheduleContextOut, aclOpExecutor* executor);	23	+} // namespace l0op
24	-} // namespace l0op
25		24
26	-#endif // OP_API_INC_LEVEL0_OP_FFN_WORKER_SCHEDULER_OP_H_	25	+#endif // OP_API_INC_LEVEL0_OP_FFN_WORKER_SCHEDULER_OP_H_

Mffn/ffn_worker_scheduler/op_kernel_aicpu/ffn_worker_scheduler_aicpu.h+40-42

	@@ -17,47 +17,45 @@
17		17
18	namespace aicpu {	18	namespace aicpu {
19	class FfnWorkerSchedulerKernel : public CpuKernel {	19	class FfnWorkerSchedulerKernel : public CpuKernel {
20	- public:	20	+public:
21	- FfnWorkerSchedulerKernel() = default;	21	+ FfnWorkerSchedulerKernel() = default;
22	- ~FfnWorkerSchedulerKernel() = default;	22	+ ~FfnWorkerSchedulerKernel() = default;
23	- uint32_t Compute(CpuKernelContext &ctx) override;	23	+ uint32_t Compute(CpuKernelContext &ctx) override;
24		24
25	- private:	25	+private:
26	- uint32_t DoCompute(CpuKernelContext &ctx);	26	+ uint32_t DoCompute(CpuKernelContext &ctx);
27	- uint32_t DoComputeAllSync(CpuKernelContext &ctx);	27	+ uint32_t DoComputeAllSync(CpuKernelContext &ctx);
28	- void ToOutput(CpuKernelContext &ctx, uint32_t micro_batch_id,	28	+ void ToOutput(CpuKernelContext &ctx, uint32_t micro_batch_id, uint32_t start_session_id,
29	- uint32_t start_session_id, uint32_t end_session_id) const;	29	+ uint32_t end_session_id) const;
30	- uint32_t InitAndCheckScheduleContextCommon();	30	+ uint32_t InitAndCheckScheduleContextCommon();
31	- uint32_t InitAndCheckScheduleContextFfnInput();	31	+ uint32_t InitAndCheckScheduleContextFfnInput();
32	- uint32_t InitAndCheckScheduleContextFfn();	32	+ uint32_t InitAndCheckScheduleContextFfn();
33	- uint32_t InitAndCheckScheduleContext();	33	+ uint32_t InitAndCheckScheduleContext();
34	- uint32_t InitAndCheck(CpuKernelContext &ctx);	34	+ uint32_t InitAndCheck(CpuKernelContext &ctx);
35	- bool CheckSessionsReady(uint32_t micro_batch_id, uint32_t start_session_id,	35	+ bool CheckSessionsReady(uint32_t micro_batch_id, uint32_t start_session_id, uint32_t end_session_id) const;
36	- uint32_t end_session_id) const;	36	+ void CopyAndResetSessions(uint32_t micro_batch_id, uint32_t start_session_id, uint32_t end_session_id,
37	- void CopyAndResetSessions(uint32_t micro_batch_id, uint32_t start_session_id,	37	+ uint32_t out_idx) const;
38	- uint32_t end_session_id, uint32_t out_idx) const;	38	+ bool CheckHasReadyGroup() const;
39	- bool CheckHasReadyGroup() const;	39	+ void CheckHandleSessions(CpuKernelContext &ctx, uint32_t start_session_id, uint32_t end_session_id) const;
40	- void CheckHandleSessions(CpuKernelContext &ctx, uint32_t start_session_id,	40	+ int32_t execute_mode_ = kExecuteModeOnce;
41	- uint32_t end_session_id) const;	41	+ ScheduleContext *schedule_context_ = nullptr;
42	- int32_t execute_mode_ = kExecuteModeOnce;	42	+ uint32_t session_num_ = 0;
43	- ScheduleContext *schedule_context_ = nullptr;	43	+ uint32_t micro_batch_num_ = 0;
44	- uint32_t session_num_ = 0;	44	+ uint32_t micro_batch_size_ = 0;
45	- uint32_t micro_batch_num_ = 0;	45	+ uint32_t selected_expert_num_ = 0;
46	- uint32_t micro_batch_size_ = 0;	46	+ uint32_t sync_group_size_ = 0;
47	- uint32_t selected_expert_num_ = 0;	47	+ uint32_t group_num_ = 1;
48	- uint32_t sync_group_size_ = 0;	48	+ uint64_t per_ffn_data_desc_size_ = 0;
49	- uint32_t group_num_ = 1;	49	+ int8_t *token_info_buf_ = nullptr;
50	- uint64_t per_ffn_data_desc_size_ = 0;	50	+ uint64_t token_info_buf_size_ = 0;
51	- int8_t *token_info_buf_ = nullptr;	51	+ int32_t *layer_ids_buf_ = nullptr;
52	- uint64_t token_info_buf_size_ = 0;	52	+ uint64_t layer_ids_buf_size_ = 0;
53	- int32_t *layer_ids_buf_ = nullptr;	53	+ int32_t *session_ids_buf_ = nullptr;
54	- uint64_t layer_ids_buf_size_ = 0;	54	+ uint64_t session_ids_buf_size_ = 0;
55	- int32_t *session_ids_buf_ = nullptr;	55	+ int32_t *micro_batch_ids_buf_ = nullptr;
56	- uint64_t session_ids_buf_size_ = 0;	56	+ uint64_t micro_batch_ids_buf_size_ = 0;
57	- int32_t *micro_batch_ids_buf_ = nullptr;	57	+ int32_t *expert_ids_buf_ = nullptr;
58	- uint64_t micro_batch_ids_buf_size_ = 0;	58	+ uint64_t expert_ids_buf_size_ = 0;
59	- int32_t *expert_ids_buf_ = nullptr;
60	- uint64_t expert_ids_buf_size_ = 0;
61	};	59	};
62	-} // namespace aicpu	60	+} // namespace aicpu
63	-#endif // OP_KERNEL_AICPU_FFN_WORKER_SCHEDULER_H	61	+#endif // OP_KERNEL_AICPU_FFN_WORKER_SCHEDULER_H

Mffn/swin_attention_ffn/op_host/swin_attention_ffn_tiling.h+28-29

Mffn/swin_attention_ffn/tests/ut/op_kernel/swin_attention_ffn_tiling.h+6-8

	@@ -46,24 +46,22 @@ struct SwinAttentionFFNTilingData {
46	#pragma pack()	46	#pragma pack()
47		47
48	#ifdef __NPU_TILING__	48	#ifdef __NPU_TILING__
49	-inline[aicore] void InitTilingData(const __gm__ uint8_t* tiling, SwinAttentionFFNTilingData* const_data)	49	+inline[aicore] void InitTilingData(const __gm__ uint8_t tiling, SwinAttentionFFNTilingData const_data)
50	{	50	{
51	- const __gm__ uint32_t* src = (const __gm__ uint32_t*)tiling;	51	+ const __gm__ uint32_t src = (const __gm__ uint32_t )tiling;
52	- uint32_t* dst = (uint32_t*)const_data;	52	+ uint32_t dst = (uint32_t )const_data;
53	for (auto i = 0; i < sizeof(SwinAttentionFFNTilingData) / 4; i++)	53	for (auto i = 0; i < sizeof(SwinAttentionFFNTilingData) / 4; i++)
54	(dst + i) = (src + i);	54	(dst + i) = (src + i);
55	}	55	}
56	#else	56	#else
57	-inline void InitTilingData(uint8_t* tiling, SwinAttentionFFNTilingData* const_data)	57	+inline void InitTilingData(uint8_t tiling, SwinAttentionFFNTilingData const_data)
58	{	58	{
59	memcpy(const_data, tiling, sizeof(SwinAttentionFFNTilingData));	59	memcpy(const_data, tiling, sizeof(SwinAttentionFFNTilingData));
60	}	60	}
61	#endif	61	#endif
62		62
63	-
64	#define GET_TILING_DATA(tiling_data, tiling_arg) \	63	#define GET_TILING_DATA(tiling_data, tiling_arg) \
65	- SwinAttentionFFNTilingData tiling_data; \	64	+ SwinAttentionFFNTilingData tiling_data; \
66	InitTilingData(tiling_arg, &tiling_data)	65	InitTilingData(tiling_arg, &tiling_data)
67		66
68	-	67	+#endif
69	-#endif

Mffn/swin_transformer_ln_qkv/op_host/swin_transformer_ln_qkv_tiling.h+27-28

Mffn/swin_transformer_ln_qkv/tests/ut/op_kernel/swin_transformer_ln_qkv_tiling.h+6-8

	@@ -54,24 +54,22 @@ struct SwinTransformerLnQKVTilingData {
54	#pragma pack()	54	#pragma pack()
55		55
56	#ifdef __NPU_TILING__	56	#ifdef __NPU_TILING__
57	-inline[aicore] void InitTilingData(const __gm__ uint8_t* tiling, SwinTransformerLnQKVTilingData* const_data)	57	+inline[aicore] void InitTilingData(const __gm__ uint8_t tiling, SwinTransformerLnQKVTilingData const_data)
58	{	58	{
59	- const __gm__ uint32_t* src = (const __gm__ uint32_t*)tiling;	59	+ const __gm__ uint32_t src = (const __gm__ uint32_t )tiling;
60	- uint32_t* dst = (uint32_t*)const_data;	60	+ uint32_t dst = (uint32_t )const_data;
61	for (auto i = 0; i < sizeof(SwinTransformerLnQKVTilingData) / 4; i++)	61	for (auto i = 0; i < sizeof(SwinTransformerLnQKVTilingData) / 4; i++)
62	(dst + i) = (src + i);	62	(dst + i) = (src + i);
63	}	63	}
64	#else	64	#else
65	-inline void InitTilingData(uint8_t* tiling, SwinTransformerLnQKVTilingData* const_data)	65	+inline void InitTilingData(uint8_t tiling, SwinTransformerLnQKVTilingData const_data)
66	{	66	{
67	memcpy(const_data, tiling, sizeof(SwinTransformerLnQKVTilingData));	67	memcpy(const_data, tiling, sizeof(SwinTransformerLnQKVTilingData));
68	}	68	}
69	#endif	69	#endif
70		70
71	-
72	#define GET_TILING_DATA(tiling_data, tiling_arg) \	71	#define GET_TILING_DATA(tiling_data, tiling_arg) \
73	- SwinTransformerLnQKVTilingData tiling_data; \	72	+ SwinTransformerLnQKVTilingData tiling_data; \
74	InitTilingData(tiling_arg, &tiling_data)	73	InitTilingData(tiling_arg, &tiling_data)
75		74
76	-	75	+#endif
77	-#endif

Mffn/swin_transformer_ln_qkv_quant/op_host/swin_transformer_ln_qkv_quant_tiling.h+44-44

Mffn/swin_transformer_ln_qkv_quant/op_kernel/swin_transformer_ln_qkv_quant_base.h+24-30

Mffn/swin_transformer_ln_qkv_quant/tests/ut/op_kernel/test_swin_transformer_ln_qkv_quant.h+8-8

	@@ -16,24 +16,24 @@
16		16
17	#include "kernel_tiling/kernel_tiling.h"	17	#include "kernel_tiling/kernel_tiling.h"
18		18
19	-
20	#ifdef __NPU_TILING__	19	#ifdef __NPU_TILING__
21	-inline [aicore] void InitSwinTransformerLnQkvQuantTilingData(const __gm__ uint8_t* tiling, SwinTransformerLnQkvQuantTilingData* const_data)	20	+inline[aicore] void InitSwinTransformerLnQkvQuantTilingData(const __gm__ uint8_t *tiling,
		21	+ SwinTransformerLnQkvQuantTilingData *const_data)
22	{	22	{
23	const __gm__ uint32_t src = (const __gm__ uint32_t )tiling;	23	const __gm__ uint32_t src = (const __gm__ uint32_t )tiling;
24	uint32_t dst = (uint32_t )const_data;	24	uint32_t dst = (uint32_t )const_data;
25	- for (auto i = 0; i < sizeof(SwinTransformerLnQkvQuantTilingData) / 4; i++) (dst + i) = (src + i);	25	+ for (auto i = 0; i < sizeof(SwinTransformerLnQkvQuantTilingData) / 4; i++)
		26	+ (dst + i) = (src + i);
26	}	27	}
27	#else	28	#else
28	-inline void InitSwinTransformerLnQkvQuantTilingData(uint8_t* tiling, SwinTransformerLnQkvQuantTilingData* const_data)	29	+inline void InitSwinTransformerLnQkvQuantTilingData(uint8_t tiling, SwinTransformerLnQkvQuantTilingData const_data)
29	{	30	{
30	memcpy(const_data, tiling, sizeof(SwinTransformerLnQkvQuantTilingData));	31	memcpy(const_data, tiling, sizeof(SwinTransformerLnQkvQuantTilingData));
31	}	32	}
32	#endif	33	#endif
33		34
34	-
35	#define GET_TILING_DATA(tiling_data, tiling_arg) \	35	#define GET_TILING_DATA(tiling_data, tiling_arg) \
36	-SwinTransformerLnQkvQuantTilingData tiling_data; \	36	+ SwinTransformerLnQkvQuantTilingData tiling_data; \
37	-InitSwinTransformerLnQkvQuantTilingData(tiling_arg, &tiling_data)	37	+ InitSwinTransformerLnQkvQuantTilingData(tiling_arg, &tiling_data)
38		38
39	-#endif	39	+#endif

Mmhc/mhc_post/op_host/op_api/mhc_post.h+3-4

	@@ -32,10 +32,9 @@ namespace l0op {
32	* @param [in] executor: Op executor	32	* @param [in] executor: Op executor
33	* @return aclTensor*: Output tensor x_{l+1}	33	* @return aclTensor*: Output tensor x_{l+1}
34	*/	34	*/
35	-const aclTensor MhcPost(const aclTensor x, const aclTensor *h_res,	35	+const aclTensor MhcPost(const aclTensor x, const aclTensor h_res, const aclTensor h_out, const aclTensor *h_post,
36	- const aclTensor h_out, const aclTensor h_post,
37	aclOpExecutor *executor);	36	aclOpExecutor *executor);
38		37
39	-}	38	+} // namespace l0op
40		39
41	-#endif	40	+#endif

Mmhc/mhc_post/op_host/op_tiling/mhc_post_tiling.h+1-1

	@@ -28,4 +28,4 @@ struct MhcPostCompileInfo {
28	platform_ascendc::SocVersion socVersion;	28	platform_ascendc::SocVersion socVersion;
29	NpuArch npuArch;	29	NpuArch npuArch;
30	};	30	};
31	-} // namespace optiling	31	+} // namespace optiling

Mmhc/mhc_post/op_kernel/arch22/mhc_post_tiling_key.h+4-10

	@@ -18,15 +18,9 @@
18		18
19	#include "ascendc/host_api/tiling/template_argument.h"	19	#include "ascendc/host_api/tiling/template_argument.h"
20		20
21	-ASCENDC_TPL_ARGS_DECL(MhcPost,	21	+ASCENDC_TPL_ARGS_DECL(MhcPost, ASCENDC_TPL_BOOL_DECL(USE_PERMANENT_X, 0, 1));
22	- ASCENDC_TPL_BOOL_DECL(USE_PERMANENT_X, 0, 1)
23	-);
24		22
25	-ASCENDC_TPL_SEL(	23	+ASCENDC_TPL_SEL(ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIV_ONLY),
26	- ASCENDC_TPL_ARGS_SEL(	24	+ ASCENDC_TPL_BOOL_SEL(USE_PERMANENT_X, 0, 1)));
27	- ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIV_ONLY),
28	- ASCENDC_TPL_BOOL_SEL(USE_PERMANENT_X, 0, 1)
29	- )
30	-);
31		25
32	-#endif	26	+#endif

Mmhc/mhc_post/op_kernel/arch35/mhc_post_nohres.h+27-32

	@@ -70,13 +70,14 @@ private:
70		70
71	int64_t curBS_;	71	int64_t curBS_;
72	uint32_t blockIdx_;	72	uint32_t blockIdx_;
73	- constexpr static AscendC::MicroAPI::CastTrait castB16ToB32 = { AscendC::MicroAPI::RegLayout::ZERO,	73	+ constexpr static AscendC::MicroAPI::CastTrait castB16ToB32 = {
74	- AscendC::MicroAPI::SatMode::UNKNOWN, AscendC::MicroAPI::MaskMergeMode::ZEROING, AscendC::RoundMode::UNKNOWN };	74	+ AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::UNKNOWN,
		75	+ AscendC::MicroAPI::MaskMergeMode::ZEROING, AscendC::RoundMode::UNKNOWN};
75	};	76	};
76		77
77	NOHRES_TEMPLATE_DECLARE	78	NOHRES_TEMPLATE_DECLARE
78	-__aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::Init(	79	+__aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::Init(GM_ADDR x, GM_ADDR hOut, GM_ADDR hPost, GM_ADDR output,
79	- GM_ADDR x, GM_ADDR hOut, GM_ADDR hPost, GM_ADDR output, GM_ADDR workspace)	80	+ GM_ADDR workspace)
80	{	81	{
81	blockIdx_ = GetBlockIdx();	82	blockIdx_ = GetBlockIdx();
82	if (blockIdx_ >= tilingData_->usedCoreNum) {	83	if (blockIdx_ >= tilingData_->usedCoreNum) {
	@@ -108,8 +109,8 @@ __aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::Init(
108	}	109	}
109		110
110	NOHRES_TEMPLATE_DECLARE	111	NOHRES_TEMPLATE_DECLARE
111	-__aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::DoMulAndAdd(	112	+__aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::DoMulAndAdd(LocalTensor<float> hPostUb, int64_t nNum,
112	- LocalTensor<float> hPostUb, int64_t nNum, int64_t dNum)	113	+ int64_t dNum)
113	{	114	{
114	uint16_t nTimes = static_cast<uint16_t>(nNum);	115	uint16_t nTimes = static_cast<uint16_t>(nNum);
115	uint16_t dRepeatTimes = static_cast<uint16_t>(Ops::Base::CeilDiv(dNum, static_cast<int64_t>(VL_FP32)));	116	uint16_t dRepeatTimes = static_cast<uint16_t>(Ops::Base::CeilDiv(dNum, static_cast<int64_t>(VL_FP32)));
	@@ -119,10 +120,10 @@ __aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::DoMulAndAdd(
119	LocalTensor<T> hOutUb = hOutTileQueue_.DeQue<T>();	120	LocalTensor<T> hOutUb = hOutTileQueue_.DeQue<T>();
120	LocalTensor<float> yTileBuf = yTileBuf_.Get<float>();	121	LocalTensor<float> yTileBuf = yTileBuf_.Get<float>();
121		122
122	- auto xAddr = (__ubuf__ T*)xUb.GetPhyAddr();	123	+ auto xAddr = (__ubuf__ T *)xUb.GetPhyAddr();
123	- auto hPostAddr = (__ubuf__ float*)hPostUb.GetPhyAddr();	124	+ auto hPostAddr = (__ubuf__ float *)hPostUb.GetPhyAddr();
124	- auto hOutAddr = (__ubuf__ T*)hOutUb.GetPhyAddr();	125	+ auto hOutAddr = (__ubuf__ T *)hOutUb.GetPhyAddr();
125	- auto yAddr = (__ubuf__ float*)yTileBuf.GetPhyAddr();	126	+ auto yAddr = (__ubuf__ float *)yTileBuf.GetPhyAddr();
126		127
127	__VEC_SCOPE__	128	__VEC_SCOPE__
128	{	129	{
	@@ -139,17 +140,15 @@ __aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::DoMulAndAdd(
139	uint32_t dNumU32 = static_cast<uint32_t>(dAlign);	140	uint32_t dNumU32 = static_cast<uint32_t>(dAlign);
140		141
141	// Load hPost[nIndex] to register (broadcast)	142	// Load hPost[nIndex] to register (broadcast)
142	- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(	143	+ AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(hPostReg, hPostAddr + nIndex);
143	- hPostReg, hPostAddr + nIndex);
144		144
145	for (uint16_t j = 0; j < dRepeatTimes; j++) {	145	for (uint16_t j = 0; j < dRepeatTimes; j++) {
146	pMask = AscendC::MicroAPI::UpdateMask<float>(dNumU32);	146	pMask = AscendC::MicroAPI::UpdateMask<float>(dNumU32);
147		147
148	// Load hOut[dBlock] and cast to FP32	148	// Load hOut[dBlock] and cast to FP32
149	- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(	149	+ AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(hOutReg,
150	- hOutReg, hOutAddr + j * VL_FP32);	150	+ hOutAddr + j * VL_FP32);
151	- AscendC::MicroAPI::Cast<float, T, castB16ToB32>(	151	+ AscendC::MicroAPI::Cast<float, T, castB16ToB32>(hOutRegFloat, hOutReg, pMask);
152	- hOutRegFloat, hOutReg, pMask);
153		152
154	// Post Mapping: out = hOut * hPost[nIndex]	153	// Post Mapping: out = hOut * hPost[nIndex]
155	AscendC::MicroAPI::Mul(outRegFloat, hOutRegFloat, hPostReg, pMask);	154	AscendC::MicroAPI::Mul(outRegFloat, hOutRegFloat, hPostReg, pMask);
	@@ -157,13 +156,11 @@ __aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::DoMulAndAdd(
157	// Direct add x[nIndex] (no inner j loop, no hRes)	156	// Direct add x[nIndex] (no inner j loop, no hRes)
158	AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(	157	AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
159	xReg, xAddr + nIndex * dAlign + j * VL_FP32);	158	xReg, xAddr + nIndex * dAlign + j * VL_FP32);
160	- AscendC::MicroAPI::Cast<float, T, castB16ToB32>(	159	+ AscendC::MicroAPI::Cast<float, T, castB16ToB32>(xRegFloat, xReg, pMask);
161	- xRegFloat, xReg, pMask);
162	AscendC::MicroAPI::Add(outRegFloat, outRegFloat, xRegFloat, pMask);	160	AscendC::MicroAPI::Add(outRegFloat, outRegFloat, xRegFloat, pMask);
163		161
164	// Store to yTileBuf	162	// Store to yTileBuf
165	- AscendC::MicroAPI::DataCopy(	163	+ AscendC::MicroAPI::DataCopy(yAddr + nIndex * dAlign + j * VL_FP32, outRegFloat, pMask);
166	- yAddr + nIndex * dAlign + j * VL_FP32, outRegFloat, pMask);
167	}	164	}
168	}	165	}
169	}	166	}
	@@ -211,8 +208,7 @@ __aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::Process()
211	}	208	}
212		209
213	NOHRES_TEMPLATE_DECLARE	210	NOHRES_TEMPLATE_DECLARE
214	-__aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::CopyInHOut(	211	+__aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::CopyInHOut(int64_t bsIdx, int64_t dIdx, int64_t dNum)
215	- int64_t bsIdx, int64_t dIdx, int64_t dNum)
216	{	212	{
217	int64_t hOutOffset = bsIdx * tilingData_->d + dIdx * tilingData_->dInner;	213	int64_t hOutOffset = bsIdx * tilingData_->d + dIdx * tilingData_->dInner;
218	LocalTensor<T> hOutTileLocal = hOutTileQueue_.AllocTensor<T>();	214	LocalTensor<T> hOutTileLocal = hOutTileQueue_.AllocTensor<T>();
	@@ -224,8 +220,7 @@ __aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::CopyInHOut(
224	}	220	}
225		221
226	NOHRES_TEMPLATE_DECLARE	222	NOHRES_TEMPLATE_DECLARE
227	-__aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::CopyInHPost(	223	+__aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::CopyInHPost(int64_t bsIdx, int64_t nIdx, int64_t nNum)
228	- int64_t bsIdx, int64_t nIdx, int64_t nNum)
229	{	224	{
230	int64_t hPostOffset = bsIdx * tilingData_->n + nIdx * tilingData_->nInner;	225	int64_t hPostOffset = bsIdx * tilingData_->n + nIdx * tilingData_->nInner;
231	LocalTensor<float> hPostTileLocal = hPostTileQueue_.AllocTensor<float>();	226	LocalTensor<float> hPostTileLocal = hPostTileQueue_.AllocTensor<float>();
	@@ -236,8 +231,8 @@ __aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::CopyInHPost(
236	}	231	}
237		232
238	NOHRES_TEMPLATE_DECLARE	233	NOHRES_TEMPLATE_DECLARE
239	-__aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::CopyInX(	234	+__aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::CopyInX(int64_t bsIdx, int64_t nIdx, int64_t nNum,
240	- int64_t bsIdx, int64_t nIdx, int64_t nNum, int64_t dIdx, int64_t dNum)	235	+ int64_t dIdx, int64_t dNum)
241	{	236	{
242	int64_t dStart = dIdx * tilingData_->dInner;	237	int64_t dStart = dIdx * tilingData_->dInner;
243	int64_t nStart = nIdx * tilingData_->nInner * tilingData_->d;	238	int64_t nStart = nIdx * tilingData_->nInner * tilingData_->d;
	@@ -252,22 +247,22 @@ __aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::CopyInX(
252	}	247	}
253		248
254	NOHRES_TEMPLATE_DECLARE	249	NOHRES_TEMPLATE_DECLARE
255	-__aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::CopyOutY(	250	+__aicore__ inline void MhcPostNoHRes<NOHRES_TEMPLATE_ARGS>::CopyOutY(int64_t bsIdx, int64_t nIdx, int64_t nNum,
256	- int64_t bsIdx, int64_t nIdx, int64_t nNum, int64_t dIdx, int64_t dNum)	251	+ int64_t dIdx, int64_t dNum)
257	{	252	{
258	int64_t dStart = dIdx * tilingData_->dInner;	253	int64_t dStart = dIdx * tilingData_->dInner;
259	int64_t nStart = nIdx * tilingData_->nInner * tilingData_->d;	254	int64_t nStart = nIdx * tilingData_->nInner * tilingData_->d;
260	int64_t yBase = bsIdx * tilingData_->n * tilingData_->d;	255	int64_t yBase = bsIdx * tilingData_->n * tilingData_->d;
261	int64_t yOffset = yBase + nStart + dStart;	256	int64_t yOffset = yBase + nStart + dStart;
262	LocalTensor<T> yTileLocal = yTileQueue_.DeQue<T>();	257	LocalTensor<T> yTileLocal = yTileQueue_.DeQue<T>();
263	- DataCopyExtParams copyParams = {static_cast<uint16_t>(nNum), static_cast<uint32_t>(dNum * sizeof(T)),	258	+ DataCopyExtParams copyParams = {static_cast<uint16_t>(nNum), static_cast<uint32_t>(dNum * sizeof(T)), 0,
264	- 0, static_cast<uint32_t>((tilingData_->d - dNum) * sizeof(T)), 0};	259	+ static_cast<uint32_t>((tilingData_->d - dNum) * sizeof(T)), 0};
265		260
266	DataCopyPad(outputGm_[yOffset], yTileLocal, copyParams);	261	DataCopyPad(outputGm_[yOffset], yTileLocal, copyParams);
267		262
268	yTileQueue_.FreeTensor(yTileLocal);	263	yTileQueue_.FreeTensor(yTileLocal);
269	}	264	}
270		265
271	-} // namespace MhcPost	266	+} // namespace MhcPost
272		267
273	-#endif // ASCENDC_MHC_POST_NOHRES_H	268	+#endif // ASCENDC_MHC_POST_NOHRES_H

Mmhc/mhc_post/op_kernel/arch35/mhc_post_regbase.h+43-48

	@@ -47,10 +47,10 @@ private:
47	__aicore__ inline void CopyInX(int64_t bsIdx, int64_t dIdx, int64_t dNum);	47	__aicore__ inline void CopyInX(int64_t bsIdx, int64_t dIdx, int64_t dNum);
48	__aicore__ inline void CopyInHPost(int64_t bsIdx, int64_t nIdx, int64_t nNum);	48	__aicore__ inline void CopyInHPost(int64_t bsIdx, int64_t nIdx, int64_t nNum);
49	__aicore__ inline void CopyInHRes(int64_t bsIdx, int64_t nIdx, int64_t nNum);	49	__aicore__ inline void CopyInHRes(int64_t bsIdx, int64_t nIdx, int64_t nNum);
50	- __aicore__ inline void DoMulAndAdd(LocalTensor<float> hPostUb, LocalTensor<float> hResUb,	50	+ __aicore__ inline void DoMulAndAdd(LocalTensor<float> hPostUb, LocalTensor<float> hResUb, int64_t nNum,
51	- int64_t nNum, int64_t dNum);	51	+ int64_t dNum);
52	- __aicore__ inline void DoMulAndAdd_N4(LocalTensor<float> hPostUb, LocalTensor<float> hResUb,	52	+ __aicore__ inline void DoMulAndAdd_N4(LocalTensor<float> hPostUb, LocalTensor<float> hResUb, int64_t nNum,
53	- int64_t nNum, int64_t dNum);	53	+ int64_t dNum);
54	__aicore__ inline void CopyOutY(int64_t bsIdx, int64_t nIdx, int64_t nNum, int64_t dIdx, int64_t dNum);	54	__aicore__ inline void CopyOutY(int64_t bsIdx, int64_t nIdx, int64_t nNum, int64_t dIdx, int64_t dNum);
55		55
56	private:	56	private:
	@@ -80,14 +80,14 @@ private:
80		80
81	int64_t curBS_;	81	int64_t curBS_;
82	uint32_t blockIdx_;	82	uint32_t blockIdx_;
83	- constexpr static AscendC::MicroAPI::CastTrait castB16ToB32 = { AscendC::MicroAPI::RegLayout::ZERO,	83	+ constexpr static AscendC::MicroAPI::CastTrait castB16ToB32 = {
84	- AscendC::MicroAPI::SatMode::UNKNOWN, AscendC::MicroAPI::MaskMergeMode::ZEROING, AscendC::RoundMode::UNKNOWN };	84	+ AscendC::MicroAPI::RegLayout::ZERO, AscendC::MicroAPI::SatMode::UNKNOWN,
		85	+ AscendC::MicroAPI::MaskMergeMode::ZEROING, AscendC::RoundMode::UNKNOWN};
85	};	86	};
86		87
87	REGBASE_TEMPLATE_DECLARE	88	REGBASE_TEMPLATE_DECLARE
88	-__aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::Init(	89	+__aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::Init(GM_ADDR x, GM_ADDR hRes, GM_ADDR hOut, GM_ADDR hPost,
89	- GM_ADDR x, GM_ADDR hRes, GM_ADDR hOut, GM_ADDR hPost,	90	+ GM_ADDR output, GM_ADDR workspace)
90	- GM_ADDR output, GM_ADDR workspace)
91	{	91	{
92	blockIdx_ = GetBlockIdx();	92	blockIdx_ = GetBlockIdx();
93	if (blockIdx_ >= tilingData_->usedCoreNum) {	93	if (blockIdx_ >= tilingData_->usedCoreNum) {
	@@ -122,8 +122,9 @@ __aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::Init(
122	}	122	}
123		123
124	REGBASE_TEMPLATE_DECLARE	124	REGBASE_TEMPLATE_DECLARE
125	-__aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::DoMulAndAdd(	125	+__aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::DoMulAndAdd(LocalTensor<float> hPostUb,
126	- LocalTensor<float> hPostUb, LocalTensor<float> hResUb, int64_t nNum, int64_t dNum)	126	+ LocalTensor<float> hResUb, int64_t nNum,
		127	+ int64_t dNum)
127	{	128	{
128	if (tilingData_->n == 4) {	129	if (tilingData_->n == 4) {
129	DoMulAndAdd_N4(hPostUb, hResUb, nNum, dNum);	130	DoMulAndAdd_N4(hPostUb, hResUb, nNum, dNum);
	@@ -140,11 +141,11 @@ __aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::DoMulAndAdd(
140		141
141	LocalTensor<float> yTileBuf = yTileBuf_.Get<float>();	142	LocalTensor<float> yTileBuf = yTileBuf_.Get<float>();
142		143
143	- auto xAddr = (__ubuf__ T*)xUb.GetPhyAddr();	144	+ auto xAddr = (__ubuf__ T *)xUb.GetPhyAddr();
144	- auto hResAddr = (__ubuf__ float*)hResUb.GetPhyAddr();	145	+ auto hResAddr = (__ubuf__ float *)hResUb.GetPhyAddr();
145	- auto hPostAddr = (__ubuf__ float*)hPostUb.GetPhyAddr();	146	+ auto hPostAddr = (__ubuf__ float *)hPostUb.GetPhyAddr();
146	- auto hOutAddr = (__ubuf__ T*)hOutUb.GetPhyAddr();	147	+ auto hOutAddr = (__ubuf__ T *)hOutUb.GetPhyAddr();
147	- auto yAddr = (__ubuf__ float*)yTileBuf.GetPhyAddr();	148	+ auto yAddr = (__ubuf__ float *)yTileBuf.GetPhyAddr();
148		149
149	__VEC_SCOPE__	150	__VEC_SCOPE__
150	{	151	{
	@@ -161,19 +162,16 @@ __aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::DoMulAndAdd(
161	AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(hPostReg, hPostAddr + nIndex);	162	AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(hPostReg, hPostAddr + nIndex);
162	for (uint16_t j = 0; j < dRepeatTimes; j++) {	163	for (uint16_t j = 0; j < dRepeatTimes; j++) {
163	pMask = AscendC::MicroAPI::UpdateMask<float>(dNumU32);	164	pMask = AscendC::MicroAPI::UpdateMask<float>(dNumU32);
164	- AscendC::MicroAPI::DataCopy<T,	165	+ AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(hOutReg,
165	- AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(	166	+ hOutAddr + j * VL_FP32);
166	- hOutReg, hOutAddr + j * VL_FP32);
167	AscendC::MicroAPI::Cast<float, T, castB16ToB32>(hOutRegFloat, hOutReg, pMask);	167	AscendC::MicroAPI::Cast<float, T, castB16ToB32>(hOutRegFloat, hOutReg, pMask);
168	AscendC::MicroAPI::Mul(outRegFloat, hOutRegFloat, hPostReg, pMask);	168	AscendC::MicroAPI::Mul(outRegFloat, hOutRegFloat, hPostReg, pMask);
169	for (uint16_t i = 0; i < nTotal; i++) {	169	for (uint16_t i = 0; i < nTotal; i++) {
170	- AscendC::MicroAPI::DataCopy<T,	170	+ AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
171	- AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(	171	+ xReg, xAddr + i * dAlign + j * VL_FP32);
172	- xReg, xAddr + i * dAlign + j * VL_FP32);
173	AscendC::MicroAPI::Cast<float, T, castB16ToB32>(xRegFloat, xReg, pMask);	172	AscendC::MicroAPI::Cast<float, T, castB16ToB32>(xRegFloat, xReg, pMask);
174	- AscendC::MicroAPI::DataCopy<float,	173	+ AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(
175	- AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(	174	+ hResReg, hResAddr + i * nAlign + nIndex);
176	- hResReg, hResAddr + i * nAlign + nIndex);
177	AscendC::MicroAPI::MulAddDst(outRegFloat, xRegFloat, hResReg, pMask);	175	AscendC::MicroAPI::MulAddDst(outRegFloat, xRegFloat, hResReg, pMask);
178	}	176	}
179	AscendC::MicroAPI::DataCopy(yAddr + nIndex * dAlign + j * VL_FP32, outRegFloat, pMask);	177	AscendC::MicroAPI::DataCopy(yAddr + nIndex * dAlign + j * VL_FP32, outRegFloat, pMask);
	@@ -189,8 +187,9 @@ __aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::DoMulAndAdd(
189	}	187	}
190		188
191	REGBASE_TEMPLATE_DECLARE	189	REGBASE_TEMPLATE_DECLARE
192	-__aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::DoMulAndAdd_N4(	190	+__aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::DoMulAndAdd_N4(LocalTensor<float> hPostUb,
193	- LocalTensor<float> hPostUb, LocalTensor<float> hResUb, int64_t nNum, int64_t dNum)	191	+ LocalTensor<float> hResUb, int64_t nNum,
		192	+ int64_t dNum)
194	{	193	{
195	uint16_t nTimes = static_cast<uint16_t>(nNum);	194	uint16_t nTimes = static_cast<uint16_t>(nNum);
196	uint16_t dAlign = static_cast<uint16_t>(Ops::Base::CeilAlign(dNum, static_cast<int64_t>(REG_ALIGN_D)));	195	uint16_t dAlign = static_cast<uint16_t>(Ops::Base::CeilAlign(dNum, static_cast<int64_t>(REG_ALIGN_D)));
	@@ -201,11 +200,11 @@ __aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::DoMulAndAdd_N4(
201	LocalTensor<T> hOutUb = hOutTileQueue_.DeQue<T>();	200	LocalTensor<T> hOutUb = hOutTileQueue_.DeQue<T>();
202	LocalTensor<float> yTileBuf = yTileBuf_.Get<float>();	201	LocalTensor<float> yTileBuf = yTileBuf_.Get<float>();
203		202
204	- auto xAddr = (__ubuf__ T*)xUb.GetPhyAddr();	203	+ auto xAddr = (__ubuf__ T *)xUb.GetPhyAddr();
205	- auto hResAddr = (__ubuf__ float*)hResUb.GetPhyAddr();	204	+ auto hResAddr = (__ubuf__ float *)hResUb.GetPhyAddr();
206	- auto hPostAddr = (__ubuf__ float*)hPostUb.GetPhyAddr();	205	+ auto hPostAddr = (__ubuf__ float *)hPostUb.GetPhyAddr();
207	- auto hOutAddr = (__ubuf__ T*)hOutUb.GetPhyAddr();	206	+ auto hOutAddr = (__ubuf__ T *)hOutUb.GetPhyAddr();
208	- auto yAddr = (__ubuf__ float*)yTileBuf.GetPhyAddr();	207	+ auto yAddr = (__ubuf__ float *)yTileBuf.GetPhyAddr();
209		208
210	__VEC_SCOPE__	209	__VEC_SCOPE__
211	{	210	{
	@@ -229,14 +228,13 @@ __aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::DoMulAndAdd_N4(
229	hResReg2, hResAddr + 2 * nAlign + nIndex);	228	hResReg2, hResAddr + 2 * nAlign + nIndex);
230	AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(	229	AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(
231	hResReg3, hResAddr + 3 * nAlign + nIndex);	230	hResReg3, hResAddr + 3 * nAlign + nIndex);
232	- AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(	231	+ AscendC::MicroAPI::DataCopy<float, AscendC::MicroAPI::LoadDist::DIST_BRC_B32>(hPostReg, hPostAddr + nIndex);
233	- hPostReg, hPostAddr + nIndex);
234		232
235	for (uint16_t j = 0; j < repeatTimes; j++) {	233	for (uint16_t j = 0; j < repeatTimes; j++) {
236	pMask = AscendC::MicroAPI::UpdateMask<float>(dNumU32);	234	pMask = AscendC::MicroAPI::UpdateMask<float>(dNumU32);
237		235
238	- AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(	236	+ AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(hOutReg,
239	- hOutReg, hOutAddr + j * VL_FP32);	237	+ hOutAddr + j * VL_FP32);
240	AscendC::MicroAPI::Cast<float, T, castB16ToB32>(hOutRegFloat, hOutReg, pMask);	238	AscendC::MicroAPI::Cast<float, T, castB16ToB32>(hOutRegFloat, hOutReg, pMask);
241		239
242	AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(	240	AscendC::MicroAPI::DataCopy<T, AscendC::MicroAPI::LoadDist::DIST_UNPACK_B16>(
	@@ -304,8 +302,7 @@ __aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::Process()
304	}	302	}
305		303
306	REGBASE_TEMPLATE_DECLARE	304	REGBASE_TEMPLATE_DECLARE
307	-__aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::CopyInHOut(	305	+__aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::CopyInHOut(int64_t bsIdx, int64_t dIdx, int64_t dNum)
308	- int64_t bsIdx, int64_t dIdx, int64_t dNum)
309	{	306	{
310	int64_t hOutOffset = bsIdx * tilingData_->d + dIdx * tilingData_->dInner;	307	int64_t hOutOffset = bsIdx * tilingData_->d + dIdx * tilingData_->dInner;
311	LocalTensor<T> hOutTileLocal = hOutTileQueue_.AllocTensor<T>();	308	LocalTensor<T> hOutTileLocal = hOutTileQueue_.AllocTensor<T>();
	@@ -317,8 +314,7 @@ __aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::CopyInHOut(
317	}	314	}
318		315
319	REGBASE_TEMPLATE_DECLARE	316	REGBASE_TEMPLATE_DECLARE
320	-__aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::CopyInHPost(	317	+__aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::CopyInHPost(int64_t bsIdx, int64_t nIdx, int64_t nNum)
321	- int64_t bsIdx, int64_t nIdx, int64_t nNum)
322	{	318	{
323	int64_t hPostOffset = bsIdx * tilingData_->n + nIdx * tilingData_->nInner;	319	int64_t hPostOffset = bsIdx * tilingData_->n + nIdx * tilingData_->nInner;
324	LocalTensor<float> hPostTileLocal = hPostTileQueue_.AllocTensor<float>();	320	LocalTensor<float> hPostTileLocal = hPostTileQueue_.AllocTensor<float>();
	@@ -329,8 +325,7 @@ __aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::CopyInHPost(
329	}	325	}
330		326
331	REGBASE_TEMPLATE_DECLARE	327	REGBASE_TEMPLATE_DECLARE
332	-__aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::CopyInHRes(	328	+__aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::CopyInHRes(int64_t bsIdx, int64_t nIdx, int64_t nNum)
333	- int64_t bsIdx, int64_t nIdx, int64_t nNum)
334	{	329	{
335	int64_t hResBase = bsIdx * tilingData_->n * tilingData_->n;	330	int64_t hResBase = bsIdx * tilingData_->n * tilingData_->n;
336	int64_t nStart = nIdx * tilingData_->nInner;	331	int64_t nStart = nIdx * tilingData_->nInner;
	@@ -359,22 +354,22 @@ __aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::CopyInX(int64_t bs
359	}	354	}
360		355
361	REGBASE_TEMPLATE_DECLARE	356	REGBASE_TEMPLATE_DECLARE
362	-__aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::CopyOutY(	357	+__aicore__ inline void MhcPostRegbase<REGBASE_TEMPLATE_ARGS>::CopyOutY(int64_t bsIdx, int64_t nIdx, int64_t nNum,
363	- int64_t bsIdx, int64_t nIdx, int64_t nNum, int64_t dIdx, int64_t dNum)	358	+ int64_t dIdx, int64_t dNum)
364	{	359	{
365	int64_t dStart = dIdx * tilingData_->dInner;	360	int64_t dStart = dIdx * tilingData_->dInner;
366	int64_t nStart = nIdx * tilingData_->nInner * tilingData_->d;	361	int64_t nStart = nIdx * tilingData_->nInner * tilingData_->d;
367	int64_t yBase = bsIdx * tilingData_->n * tilingData_->d;	362	int64_t yBase = bsIdx * tilingData_->n * tilingData_->d;
368	int64_t yOffset = yBase + nStart + dStart;	363	int64_t yOffset = yBase + nStart + dStart;
369	LocalTensor<T> yTileLocal = yTileQueue_.DeQue<T>();	364	LocalTensor<T> yTileLocal = yTileQueue_.DeQue<T>();
370	- DataCopyExtParams copyParams = {static_cast<uint16_t>(nNum), static_cast<uint32_t>(dNum * sizeof(T)),	365	+ DataCopyExtParams copyParams = {static_cast<uint16_t>(nNum), static_cast<uint32_t>(dNum * sizeof(T)), 0,
371	- 0, static_cast<uint32_t>((tilingData_->d - dNum) * sizeof(T)), 0};	366	+ static_cast<uint32_t>((tilingData_->d - dNum) * sizeof(T)), 0};
372		367
373	DataCopyPad(outputGm_[yOffset], yTileLocal, copyParams);	368	DataCopyPad(outputGm_[yOffset], yTileLocal, copyParams);
374		369
375	yTileQueue_.FreeTensor(yTileLocal);	370	yTileQueue_.FreeTensor(yTileLocal);
376	}	371	}
377		372
378	-} // namespace MhcPost	373	+} // namespace MhcPost
379		374
380	-#endif // ASCENDC_MHC_POST_REGBASE_H	375	+#endif // ASCENDC_MHC_POST_REGBASE_H

Mmhc/mhc_post/op_kernel/arch35/mhc_post_tiling_key.h+13-28

	@@ -23,39 +23,24 @@ namespace MhcPost {
23	constexpr uint32_t DOUBLE_BUFFER_DEPTH = 2;	23	constexpr uint32_t DOUBLE_BUFFER_DEPTH = 2;
24	constexpr uint16_t REG_ALIGN_N = 8;	24	constexpr uint16_t REG_ALIGN_N = 8;
25	constexpr uint16_t REG_ALIGN_D = 16;	25	constexpr uint16_t REG_ALIGN_D = 16;
26	-} // namespace MhcPost	26	+} // namespace MhcPost
27		27
28	-ASCENDC_TPL_ARGS_DECL(MhcPost,	28	+ASCENDC_TPL_ARGS_DECL(MhcPost, ASCENDC_TPL_BOOL_DECL(USE_PERMANENT_X, 0, 1), ASCENDC_TPL_BOOL_DECL(USE_REGBASE, 0, 1),
29	- ASCENDC_TPL_BOOL_DECL(USE_PERMANENT_X, 0, 1),	29	+ ASCENDC_TPL_BOOL_DECL(USE_H_RES, 0, 1));
30	- ASCENDC_TPL_BOOL_DECL(USE_REGBASE, 0, 1),
31	- ASCENDC_TPL_BOOL_DECL(USE_H_RES, 0, 1)
32	-);
33		30
34	ASCENDC_TPL_SEL(	31	ASCENDC_TPL_SEL(
35	- ASCENDC_TPL_ARGS_SEL(	32	+ ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIV_ONLY), ASCENDC_TPL_BOOL_SEL(USE_PERMANENT_X, 0, 1),
36	- ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIV_ONLY),	33	+ ASCENDC_TPL_BOOL_SEL(USE_REGBASE, 0), ASCENDC_TPL_BOOL_SEL(USE_H_RES, 1),
37	- ASCENDC_TPL_BOOL_SEL(USE_PERMANENT_X, 0, 1),	34	+ ASCENDC_TPL_TILING_STRUCT_SEL(MhcPostTilingData)),
38	- ASCENDC_TPL_BOOL_SEL(USE_REGBASE, 0),
39	- ASCENDC_TPL_BOOL_SEL(USE_H_RES, 1),
40	- ASCENDC_TPL_TILING_STRUCT_SEL(MhcPostTilingData)
41	- ),
42		35
43	- ASCENDC_TPL_ARGS_SEL(	36	+ ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIV_ONLY), ASCENDC_TPL_BOOL_SEL(USE_PERMANENT_X, 0),
44	- ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIV_ONLY),	37	+ ASCENDC_TPL_BOOL_SEL(USE_REGBASE, 1), ASCENDC_TPL_BOOL_SEL(USE_H_RES, 1),
45	- ASCENDC_TPL_BOOL_SEL(USE_PERMANENT_X, 0),	38	+ ASCENDC_TPL_TILING_STRUCT_SEL(MhcPostRegbaseTilingData)),
46	- ASCENDC_TPL_BOOL_SEL(USE_REGBASE, 1),
47	- ASCENDC_TPL_BOOL_SEL(USE_H_RES, 1),
48	- ASCENDC_TPL_TILING_STRUCT_SEL(MhcPostRegbaseTilingData)
49	- ),
50		39
51	- ASCENDC_TPL_ARGS_SEL(	40	+ ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIV_ONLY), ASCENDC_TPL_BOOL_SEL(USE_PERMANENT_X, 0),
52	- ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIV_ONLY),	41	+ ASCENDC_TPL_BOOL_SEL(USE_REGBASE, 1), ASCENDC_TPL_BOOL_SEL(USE_H_RES, 0),
53	- ASCENDC_TPL_BOOL_SEL(USE_PERMANENT_X, 0),	42	+ ASCENDC_TPL_TILING_STRUCT_SEL(MhcPostRegbaseTilingData)),
54	- ASCENDC_TPL_BOOL_SEL(USE_REGBASE, 1),
55	- ASCENDC_TPL_BOOL_SEL(USE_H_RES, 0),
56	- ASCENDC_TPL_TILING_STRUCT_SEL(MhcPostRegbaseTilingData)
57	- ),
58		43
59	);	44	);
60		45
61	-#endif	46	+#endif

Mmhc/mhc_post_backward/op_host/op_api/mhc_post_backward.h+5-6

	@@ -30,11 +30,10 @@ namespace l0op {
30	* @param [in] h_post: Post processing matrix, shape is [T, n] or [B, S, n], dtype is FP32	30	* @param [in] h_post: Post processing matrix, shape is [T, n] or [B, S, n], dtype is FP32
31	* @param [in] executor: Op executor	31	* @param [in] executor: Op executor
32	*/	32	*/
33	-const std::tuple<aclTensor, aclTensor, aclTensor, aclTensor> MhcPostBackward(const aclTensor *gradOutput,	33	+const std::tuple<aclTensor , aclTensor , aclTensor , aclTensor > MhcPostBackward(
34	- const aclTensor x, const aclTensor hRes,	34	+ const aclTensor gradOutput, const aclTensor x, const aclTensor hRes, const aclTensor hOut,
35	- const aclTensor hOut, const aclTensor hPost,	35	+ const aclTensor hPost, aclOpExecutor executor);
36	- aclOpExecutor *executor);
37		36
38	-}	37	+} // namespace l0op
39		38
40	-#endif	39	+#endif

Mmhc/mhc_post_backward/op_kernel/arch22/mhc_post_backward_arch22.h+84-116

	@@ -27,22 +27,19 @@ class KernelMhcPostBackward {
27	public:	27	public:
28	__aicore__ inline KernelMhcPostBackward() {}	28	__aicore__ inline KernelMhcPostBackward() {}
29		29
30	- __aicore__ inline void Init(	30	+ __aicore__ inline void Init(GM_ADDR grad_y, GM_ADDR x, GM_ADDR h_res, GM_ADDR h_out, GM_ADDR h_post, GM_ADDR grad_x,
31	- GM_ADDR grad_y, GM_ADDR x, GM_ADDR h_res, GM_ADDR h_out, GM_ADDR h_post,	31	+ GM_ADDR grad_h_res, GM_ADDR grad_h_out, GM_ADDR grad_h_post,
32	- GM_ADDR grad_x, GM_ADDR grad_h_res, GM_ADDR grad_h_out, GM_ADDR grad_h_post,	32	+ const MhcPostBackwardTilingDataArch22 &tilingData, TPipe *pipe);
33	- const MhcPostBackwardTilingDataArch22 &tilingData, TPipe *pipe);
34	__aicore__ inline void Process();	33	__aicore__ inline void Process();
35		34
36	protected:	35	protected:
37	static constexpr uint64_t BUFFER_NUM = 1;	36	static constexpr uint64_t BUFFER_NUM = 1;
38		37
39	- __aicore__ inline void VecMatmulMknk(	38	+ __aicore__ inline void VecMatmulMknk(LocalTensor<float> &A, LocalTensor<float> &B, LocalTensor<float> &C,
40	- LocalTensor<float> &A, LocalTensor<float> &B, LocalTensor<float> &C,	39	+ LocalTensor<float> &broadcastBuffer, LocalTensor<float> &reduceBuffer,
41	- LocalTensor<float> &broadcastBuffer, LocalTensor<float> &reduceBuffer, uint32_t m, uint32_t k,	40	+ uint32_t m, uint32_t k, uint32_t n, uint32_t alignN);
42	- uint32_t n, uint32_t alignN);	41	+ __aicore__ inline void VecMatmulMkkn(LocalTensor<float> &A, LocalTensor<float> &B, LocalTensor<float> &C,
43	- __aicore__ inline void VecMatmulMkkn(	42	+ LocalTensor<float> &broadcastBuffer, uint32_t m, uint32_t k, uint32_t n);
44	- LocalTensor<float> &A, LocalTensor<float> &B, LocalTensor<float> &C,
45	- LocalTensor<float> &broadcastBuffer, uint32_t m, uint32_t k, uint32_t n);
46		43
47	TBuf<TPosition::VECCALC> dFPostResCastBuf, FOutCastBuf, HLPostBuf, xLCastBuf, HLResBuf;	44	TBuf<TPosition::VECCALC> dFPostResCastBuf, FOutCastBuf, HLPostBuf, xLCastBuf, HLResBuf;
48	TBuf<TPosition::VECCALC> dHLPostBuf, dFOutCastBuf, dHLResBuf, dxLCastBuf;	45	TBuf<TPosition::VECCALC> dHLPostBuf, dFOutCastBuf, dHLResBuf, dxLCastBuf;
	@@ -88,10 +85,10 @@ protected:
88	};	85	};
89		86
90	template <typename T>	87	template <typename T>
91	-__aicore__ inline void KernelMhcPostBackward<T>::Init(	88	+__aicore__ inline void KernelMhcPostBackward<T>::Init(GM_ADDR grad_y, GM_ADDR x, GM_ADDR h_res, GM_ADDR h_out,
92	- GM_ADDR grad_y, GM_ADDR x, GM_ADDR h_res, GM_ADDR h_out, GM_ADDR h_post,	89	+ GM_ADDR h_post, GM_ADDR grad_x, GM_ADDR grad_h_res,
93	- GM_ADDR grad_x, GM_ADDR grad_h_res, GM_ADDR grad_h_out, GM_ADDR grad_h_post,	90	+ GM_ADDR grad_h_out, GM_ADDR grad_h_post,
94	- const MhcPostBackwardTilingDataArch22 &tilingData, TPipe *pipe)	91	+ const MhcPostBackwardTilingDataArch22 &tilingData, TPipe *pipe)
95	{	92	{
96	this->coreUsed = tilingData.coreUsed;	93	this->coreUsed = tilingData.coreUsed;
97	this->singleCoreBS = tilingData.singleCoreBS;	94	this->singleCoreBS = tilingData.singleCoreBS;
	@@ -158,9 +155,9 @@ __aicore__ inline void KernelMhcPostBackward<T>::Process()
158	return;	155	return;
159	}	156	}
160		157
161	- uint64_t startIdx = coreId > this->frontCore ? (coreId - this->frontCore) * this->tailBS +	158	+ uint64_t startIdx = coreId > this->frontCore ?
162	- this->frontCore * this->singleCoreBS :	159	+ (coreId - this->frontCore) * this->tailBS + this->frontCore * this->singleCoreBS :
163	- coreId * this->singleCoreBS;	160	+ coreId * this->singleCoreBS;
164		161
165	uint64_t endIdx = startIdx + ((coreId < this->frontCore) ? this->singleCoreBS : this->tailBS);	162	uint64_t endIdx = startIdx + ((coreId < this->frontCore) ? this->singleCoreBS : this->tailBS);
166		163
	@@ -212,23 +209,20 @@ __aicore__ inline void KernelMhcPostBackward<T>::Process()
212	SetFlag<HardEvent::MTE3_V>(1);	209	SetFlag<HardEvent::MTE3_V>(1);
213	WaitFlag<HardEvent::MTE3_V>(1);	210	WaitFlag<HardEvent::MTE3_V>(1);
214		211
215	- DataCopyExtParams copyParamsdFPostResUb{	212	+ DataCopyExtParams copyParamsdFPostResUb{static_cast<uint16_t>(this->n),
216	- static_cast<uint16_t>(this->n), static_cast<uint32_t>(this->blockChannel * sizeof(T)),	213	+ static_cast<uint32_t>(this->blockChannel * sizeof(T)),
217	- static_cast<uint32_t>(channelStride * sizeof(T)), 0, 0};	214	+ static_cast<uint32_t>(channelStride * sizeof(T)), 0, 0};
218	- DataCopyPad(	215	+ DataCopyPad(this->dFPostResCastUb, this->dFPostResGm[i * this->n * this->channel + j * this->blockChannel],
219	- this->dFPostResCastUb, this->dFPostResGm[i * this->n * this->channel + j * this->blockChannel],	216	+ copyParamsdFPostResUb, this->padParams);
220	- copyParamsdFPostResUb, this->padParams);
221	DataCopyExtParams copyParamsFOutUb{1, static_cast<uint32_t>(this->blockChannel * sizeof(T)), 0, 0, 0};	217	DataCopyExtParams copyParamsFOutUb{1, static_cast<uint32_t>(this->blockChannel * sizeof(T)), 0, 0, 0};
222	- DataCopyPad(	218	+ DataCopyPad(this->FOutCastUb, this->FOutGm[i * this->channel + j * this->blockChannel], copyParamsFOutUb,
223	- this->FOutCastUb, this->FOutGm[i * this->channel + j * this->blockChannel],	219	+ this->padParams);
224	- copyParamsFOutUb, this->padParams);
225		220
226	- DataCopyExtParams copyParamsXLUb{	221	+ DataCopyExtParams copyParamsXLUb{static_cast<uint16_t>(this->n),
227	- static_cast<uint16_t>(this->n), static_cast<uint32_t>(this->blockChannel * sizeof(T)),	222	+ static_cast<uint32_t>(this->blockChannel * sizeof(T)),
228	- static_cast<uint32_t>(channelStride * sizeof(T)), 0, 0};	223	+ static_cast<uint32_t>(channelStride * sizeof(T)), 0, 0};
229	- DataCopyPad(	224	+ DataCopyPad(this->xLCastUb, this->xLGm[i * this->n * this->channel + j * this->blockChannel],
230	- this->xLCastUb, this->xLGm[i * this->n * this->channel + j * this->blockChannel],	225	+ copyParamsXLUb, this->padParams);
231	- copyParamsXLUb, this->padParams);
232		226
233	SetFlag<HardEvent::MTE2_V>(0);	227	SetFlag<HardEvent::MTE2_V>(0);
234	WaitFlag<HardEvent::MTE2_V>(0);	228	WaitFlag<HardEvent::MTE2_V>(0);
	@@ -239,51 +233,44 @@ __aicore__ inline void KernelMhcPostBackward<T>::Process()
239	// dHLPost : Fout @ dFPostRes.T [1, blockChannel] @ [n, blockChannel]^T	233	// dHLPost : Fout @ dFPostRes.T [1, blockChannel] @ [n, blockChannel]^T
240	// 搬运量：[1, n, blockChannel] + [1, 1, blockChannel]	234	// 搬运量：[1, n, blockChannel] + [1, 1, blockChannel]
241	// 输出：[1, n] 驻留在ub做累加	235	// 输出：[1, n] 驻留在ub做累加
242	- Cast(	236	+ Cast(this->dFPostResUb, this->dFPostResCastUb, RoundMode::CAST_NONE,
243	- this->dFPostResUb, this->dFPostResCastUb, RoundMode::CAST_NONE,	237	+ this->n * this->blockChannel); // bf16--> fp32
244	- this->n * this->blockChannel); // bf16--> fp32
245	Cast(this->FOutUb, this->FOutCastUb, RoundMode::CAST_NONE, this->blockChannel); // bf16--> fp32	238	Cast(this->FOutUb, this->FOutCastUb, RoundMode::CAST_NONE, this->blockChannel); // bf16--> fp32
246	Cast(this->xLUb, this->xLCastUb, RoundMode::CAST_NONE, this->n * this->blockChannel); // bf16--> fp32	239	Cast(this->xLUb, this->xLCastUb, RoundMode::CAST_NONE, this->n * this->blockChannel); // bf16--> fp32
247		240
248	- VecMatmulMknk(	241	+ VecMatmulMknk(this->FOutUb, this->dFPostResUb, this->dHLPostUb, this->dHLResTmp1, this->dHLResTmp3, 1,
249	- this->FOutUb, this->dFPostResUb, this->dHLPostUb, this->dHLResTmp1,	242	+ this->blockChannel, this->n, this->alignN);
250	- this->dHLResTmp3, 1, this->blockChannel, this->n, this->alignN);
251		243
252	// dHres: dF@x^T //x@dF^T	244	// dHres: dF@x^T //x@dF^T
253	// [n, C]@[C, n] = [n, n] 驻留在ub做累加	245	// [n, C]@[C, n] = [n, n] 驻留在ub做累加
254	- VecMatmulMknk(	246	+ VecMatmulMknk(this->xLUb, this->dFPostResUb, this->dHLResUb, this->dHLResTmp1, this->dHLResTmp3, this->n,
255	- this->xLUb, this->dFPostResUb, this->dHLResUb, this->dHLResTmp1,	247	+ this->blockChannel, this->n, this->alignN);
256	- this->dHLResTmp3, this->n, this->blockChannel, this->n, this->alignN);
257		248
258	// dFout: H_post@dF	249	// dFout: H_post@dF
259	// [1, n]@[n, C] = [1, C] 直接搬出	250	// [1, n]@[n, C] = [1, C] 直接搬出
260	Duplicate(this->dFOutUb, float(0.0), this->blockChannel);	251	Duplicate(this->dFOutUb, float(0.0), this->blockChannel);
261	- VecMatmulMkkn(	252	+ VecMatmulMkkn(this->HLPostUb, this->dFPostResUb, this->dFOutUb, this->dHLResTmp2, 1, this->n,
262	- this->HLPostUb, this->dFPostResUb, this->dFOutUb, this->dHLResTmp2,	253	+ this->blockChannel);
263	- 1, this->n, this->blockChannel);
264		254
265	// dx_l: H_res^T@dF	255	// dx_l: H_res^T@dF
266	// [n, n]@[n, C] = [n, C] 直接搬出,	256	// [n, n]@[n, C] = [n, C] 直接搬出,
267	Duplicate(this->dxLUb, float(0.0), this->n * this->blockChannel);	257	Duplicate(this->dxLUb, float(0.0), this->n * this->blockChannel);
268	- VecMatmulMkkn(	258	+ VecMatmulMkkn(this->HLResUb, this->dFPostResUb, this->dxLUb, this->dHLResTmp2, this->n, this->n,
269	- this->HLResUb, this->dFPostResUb, this->dxLUb, this->dHLResTmp2,	259	+ this->blockChannel);
270	- this->n, this->n, this->blockChannel);
271		260
272	- DataCopyExtParams copyParamsdxLGm{	261	+ DataCopyExtParams copyParamsdxLGm{static_cast<uint16_t>(this->n),
273	- static_cast<uint16_t>(this->n), static_cast<uint32_t>(this->blockChannel * sizeof(T)), 0,	262	+ static_cast<uint32_t>(this->blockChannel * sizeof(T)), 0,
274	- static_cast<uint32_t>(channelStride * sizeof(T)), 0};	263	+ static_cast<uint32_t>(channelStride * sizeof(T)), 0};
275	DataCopyExtParams copyParamsdFOutGm{1, static_cast<uint32_t>(this->blockChannel * sizeof(T)), 0, 0, 0};	264	DataCopyExtParams copyParamsdFOutGm{1, static_cast<uint32_t>(this->blockChannel * sizeof(T)), 0, 0, 0};
276	Cast(this->dFOutCastUb, this->dFOutUb, RoundMode::CAST_ROUND, this->blockChannel); // float--> bf16	265	Cast(this->dFOutCastUb, this->dFOutUb, RoundMode::CAST_ROUND, this->blockChannel); // float--> bf16
277	Cast(this->dxLCastUb, this->dxLUb, RoundMode::CAST_ROUND, this->n * this->blockChannel); // float--> bf16	266	Cast(this->dxLCastUb, this->dxLUb, RoundMode::CAST_ROUND, this->n * this->blockChannel); // float--> bf16
278		267
279	SetFlag<HardEvent::V_MTE3>(0);	268	SetFlag<HardEvent::V_MTE3>(0);
280	WaitFlag<HardEvent::V_MTE3>(0);	269	WaitFlag<HardEvent::V_MTE3>(0);
281	- DataCopyPad(	270	+ DataCopyPad(this->dxLGm[i * this->n * this->channel + j * this->blockChannel], this->dxLCastUb,
282	- this->dxLGm[i * this->n * this->channel + j * this->blockChannel],	271	+ copyParamsdxLGm);
283	- this->dxLCastUb, copyParamsdxLGm);	272	+ DataCopyPad(this->dFOutGm[i * this->channel + j * this->blockChannel], this->dFOutCastUb,
284	- DataCopyPad(	273	+ copyParamsdFOutGm);
285	- this->dFOutGm[i * this->channel + j * this->blockChannel],
286	- this->dFOutCastUb, copyParamsdFOutGm);
287	}	274	}
288		275
289	if (this->tailC != 0) {	276	if (this->tailC != 0) {
	@@ -296,26 +283,21 @@ __aicore__ inline void KernelMhcPostBackward<T>::Process()
296	SetFlag<HardEvent::MTE3_V>(1);	283	SetFlag<HardEvent::MTE3_V>(1);
297	WaitFlag<HardEvent::MTE3_V>(1);	284	WaitFlag<HardEvent::MTE3_V>(1);
298		285
299	- DataCopyExtParams copyParamsdFPostResUb{	286	+ DataCopyExtParams copyParamsdFPostResUb{static_cast<uint16_t>(this->n),
300	- static_cast<uint16_t>(this->n), static_cast<uint32_t>(this->tailC * sizeof(T)),	287	+ static_cast<uint32_t>(this->tailC * sizeof(T)),
301	- static_cast<uint32_t>(channelStride * sizeof(T)), 0, 0};	288	+ static_cast<uint32_t>(channelStride * sizeof(T)), 0, 0};
302	- DataCopyPad(	289	+ DataCopyPad(this->dFPostResCastUb,
303	- this->dFPostResCastUb,	290	+ this->dFPostResGm[i * this->n * this->channel + this->loopC * this->blockChannel],
304	- this->dFPostResGm[i * this->n * this->channel + this->loopC * this->blockChannel],	291	+ copyParamsdFPostResUb, this->padParams);
305	- copyParamsdFPostResUb, this->padParams);
306		292
307	DataCopyExtParams copyParamsFOutUb{1, static_cast<uint32_t>(this->tailC * sizeof(T)), 0, 0, 0};	293	DataCopyExtParams copyParamsFOutUb{1, static_cast<uint32_t>(this->tailC * sizeof(T)), 0, 0, 0};
308	- DataCopyPad(	294	+ DataCopyPad(this->FOutCastUb, this->FOutGm[i * this->channel + this->loopC * this->blockChannel],
309	- this->FOutCastUb,	295	+ copyParamsFOutUb, this->padParams);
310	- this->FOutGm[i * this->channel + this->loopC * this->blockChannel],	296	+ DataCopyExtParams copyParamsXLUb{static_cast<uint16_t>(this->n),
311	- copyParamsFOutUb, this->padParams);	297	+ static_cast<uint32_t>(this->tailC * sizeof(T)),
312	- DataCopyExtParams copyParamsXLUb{	298	+ static_cast<uint32_t>(channelStride * sizeof(T)), 0, 0};
313	- static_cast<uint16_t>(this->n), static_cast<uint32_t>(this->tailC * sizeof(T)),	299	+ DataCopyPad(this->xLCastUb, this->xLGm[i * this->n * this->channel + this->loopC * this->blockChannel],
314	- static_cast<uint32_t>(channelStride * sizeof(T)), 0, 0};	300	+ copyParamsXLUb, this->padParams);
315	- DataCopyPad(
316	- this->xLCastUb,
317	- this->xLGm[i * this->n * this->channel + this->loopC * this->blockChannel],
318	- copyParamsXLUb, this->padParams);
319	SetFlag<HardEvent::MTE2_V>(0);	301	SetFlag<HardEvent::MTE2_V>(0);
320	WaitFlag<HardEvent::MTE2_V>(0);	302	WaitFlag<HardEvent::MTE2_V>(0);
321		303
	@@ -329,48 +311,41 @@ __aicore__ inline void KernelMhcPostBackward<T>::Process()
329	Cast(this->FOutUb, this->FOutCastUb, RoundMode::CAST_NONE, this->tailC); // bf16--> fp32	311	Cast(this->FOutUb, this->FOutCastUb, RoundMode::CAST_NONE, this->tailC); // bf16--> fp32
330	Cast(this->xLUb, this->xLCastUb, RoundMode::CAST_NONE, this->n * this->tailC); // bf16--> fp32	312	Cast(this->xLUb, this->xLCastUb, RoundMode::CAST_NONE, this->n * this->tailC); // bf16--> fp32
331		313
332	- VecMatmulMknk(	314	+ VecMatmulMknk(this->FOutUb, this->dFPostResUb, this->dHLPostUb, this->dHLResTmp1, this->dHLResTmp3, 1,
333	- this->FOutUb, this->dFPostResUb, this->dHLPostUb, this->dHLResTmp1,	315	+ this->tailC, this->n, this->alignN);
334	- this->dHLResTmp3, 1, this->tailC, this->n, this->alignN);
335		316
336	// dHres: dF@x^T	317	// dHres: dF@x^T
337	// [n, tailC]@[tailC, n] = [n, n] 驻留在ub做累加	318	// [n, tailC]@[tailC, n] = [n, n] 驻留在ub做累加
338	- VecMatmulMknk(	319	+ VecMatmulMknk(this->xLUb, this->dFPostResUb, this->dHLResUb, this->dHLResTmp1, this->dHLResTmp3, this->n,
339	- this->xLUb, this->dFPostResUb, this->dHLResUb, this->dHLResTmp1,	320	+ this->tailC, this->n, this->alignN);
340	- this->dHLResTmp3, this->n, this->tailC, this->n, this->alignN);
341		321
342	// dFout: H_post@dF	322	// dFout: H_post@dF
343	// [1, n]@[n, tailC] = [1, tailC] 直接搬出	323	// [1, n]@[n, tailC] = [1, tailC] 直接搬出
344	Duplicate(this->dFOutUb, float(0.0), this->tailC);	324	Duplicate(this->dFOutUb, float(0.0), this->tailC);
345	- VecMatmulMkkn(	325	+ VecMatmulMkkn(this->HLPostUb, this->dFPostResUb, this->dFOutUb, this->dHLResTmp2, 1, this->n, this->tailC);
346	- this->HLPostUb, this->dFPostResUb, this->dFOutUb, this->dHLResTmp2,
347	- 1, this->n, this->tailC);
348	// dx_l: H_res^T@dF	326	// dx_l: H_res^T@dF
349	// [n, n]@[n, tailC] = [n, tailC] 直接搬出	327	// [n, n]@[n, tailC] = [n, tailC] 直接搬出
350	Duplicate(this->dxLUb, float(0.0), this->n * this->tailC);	328	Duplicate(this->dxLUb, float(0.0), this->n * this->tailC);
351	- VecMatmulMkkn(	329	+ VecMatmulMkkn(this->HLResUb, this->dFPostResUb, this->dxLUb, this->dHLResTmp2, this->n, this->n,
352	- this->HLResUb, this->dFPostResUb, this->dxLUb, this->dHLResTmp2,	330	+ this->tailC);
353	- this->n, this->n, this->tailC);
354		331
355	- DataCopyExtParams copyParamsdxLGm{	332	+ DataCopyExtParams copyParamsdxLGm{static_cast<uint16_t>(this->n),
356	- static_cast<uint16_t>(this->n), static_cast<uint32_t>(this->tailC * sizeof(T)), 0,	333	+ static_cast<uint32_t>(this->tailC * sizeof(T)), 0,
357	- static_cast<uint32_t>(channelStride * sizeof(T)), 0};	334	+ static_cast<uint32_t>(channelStride * sizeof(T)), 0};
358	DataCopyExtParams copyParamsdFOutGm{1, static_cast<uint32_t>(this->tailC * sizeof(T)), 0, 0, 0};	335	DataCopyExtParams copyParamsdFOutGm{1, static_cast<uint32_t>(this->tailC * sizeof(T)), 0, 0, 0};
359	Cast(this->dFOutCastUb, this->dFOutUb, RoundMode::CAST_ROUND, this->tailC); // float--> bf16	336	Cast(this->dFOutCastUb, this->dFOutUb, RoundMode::CAST_ROUND, this->tailC); // float--> bf16
360	Cast(this->dxLCastUb, this->dxLUb, RoundMode::CAST_ROUND, this->n * this->tailC); // float--> bf16	337	Cast(this->dxLCastUb, this->dxLUb, RoundMode::CAST_ROUND, this->n * this->tailC); // float--> bf16
361		338
362	SetFlag<HardEvent::V_MTE3>(0);	339	SetFlag<HardEvent::V_MTE3>(0);
363	WaitFlag<HardEvent::V_MTE3>(0);	340	WaitFlag<HardEvent::V_MTE3>(0);
364	- DataCopyPad(	341	+ DataCopyPad(this->dxLGm[i * this->n * this->channel + this->loopC * this->blockChannel], this->dxLCastUb,
365	- this->dxLGm[i * this->n * this->channel + this->loopC * this->blockChannel],	342	+ copyParamsdxLGm);
366	- this->dxLCastUb, copyParamsdxLGm);	343	+ DataCopyPad(this->dFOutGm[i * this->channel + this->loopC * this->blockChannel], this->dFOutCastUb,
367	- DataCopyPad(	344	+ copyParamsdFOutGm);
368	- this->dFOutGm[i * this->channel + this->loopC * this->blockChannel],
369	- this->dFOutCastUb, copyParamsdFOutGm);
370	}	345	}
371		346
372	- DataCopyExtParams copyParamsdHLResGm{	347	+ DataCopyExtParams copyParamsdHLResGm{static_cast<uint16_t>(this->n),
373	- static_cast<uint16_t>(this->n), static_cast<uint32_t>(this->n * sizeof(float)), 0, 0, 0};	348	+ static_cast<uint32_t>(this->n * sizeof(float)), 0, 0, 0};
374	DataCopyExtParams copyParamsdHLPostGm{1, static_cast<uint32_t>(this->n * sizeof(float)), 0, 0, 0};	349	DataCopyExtParams copyParamsdHLPostGm{1, static_cast<uint32_t>(this->n * sizeof(float)), 0, 0, 0};
375		350
376	SetFlag<HardEvent::V_MTE3>(2);	351	SetFlag<HardEvent::V_MTE3>(2);
	@@ -381,13 +356,11 @@ __aicore__ inline void KernelMhcPostBackward<T>::Process()
381	}	356	}
382		357
383	template <typename T>	358	template <typename T>
384	-__aicore__ inline void KernelMhcPostBackward<T>::VecMatmulMknk(	359	+__aicore__ inline void KernelMhcPostBackward<T>::VecMatmulMknk(LocalTensor<float> &A, LocalTensor<float> &B,
385	- LocalTensor<float> &A,	360	+ LocalTensor<float> &C,
386	- LocalTensor<float> &B,	361	+ LocalTensor<float> &broadcastBuffer,
387	- LocalTensor<float> &C,	362	+ LocalTensor<float> &reduceBuffer, uint32_t m, uint32_t k,
388	- LocalTensor<float> &broadcastBuffer,	363	+ uint32_t n, uint32_t alignN)
389	- LocalTensor<float> &reduceBuffer,
390	- uint32_t m, uint32_t k, uint32_t n, uint32_t alignN)
391	{	364	{
392	// Compute matrix multiplication using vector instructions.	365	// Compute matrix multiplication using vector instructions.
393	uint32_t reduceShape[] = {n, k};	366	uint32_t reduceShape[] = {n, k};
	@@ -404,9 +377,8 @@ __aicore__ inline void KernelMhcPostBackward<T>::VecMatmulMknk(
404	Mul(broadcastBuffer[3 * k], A[i * k], B[3 * k], k);	377	Mul(broadcastBuffer[3 * k], A[i * k], B[3 * k], k);
405	// 沿第0维求和	378	// 沿第0维求和
406	PipeBarrier<PIPE_V>();	379	PipeBarrier<PIPE_V>();
407	- ReduceSum<float, AscendC::Pattern::Reduce::AR, isReuse>(	380	+ ReduceSum<float, AscendC::Pattern::Reduce::AR, isReuse>(reduceBuffer[i * alignN], broadcastBuffer, tempBuffer,
408	- reduceBuffer[i * alignN], broadcastBuffer,	381	+ reduceShape, true);
409	- tempBuffer, reduceShape, true);
410	}	382	}
411		383
412	// 累加到输出 C	384	// 累加到输出 C
	@@ -415,14 +387,10 @@ __aicore__ inline void KernelMhcPostBackward<T>::VecMatmulMknk(
415	}	387	}
416		388
417	template <typename T>	389	template <typename T>
418	-__aicore__ inline void KernelMhcPostBackward<T>::VecMatmulMkkn(	390	+__aicore__ inline void KernelMhcPostBackward<T>::VecMatmulMkkn(LocalTensor<float> &A, LocalTensor<float> &B,
419	- LocalTensor<float> &A,	391	+ LocalTensor<float> &C,
420	- LocalTensor<float> &B,	392	+ LocalTensor<float> &broadcastBuffer, uint32_t m,
421	- LocalTensor<float> &C,	393	+ uint32_t k, uint32_t n)
422	- LocalTensor<float> &broadcastBuffer,
423	- uint32_t m,
424	- uint32_t k,
425	- uint32_t n)
426	{	394	{
427	uint32_t broadcastSrcShape[] = {m * k, 1};	395	uint32_t broadcastSrcShape[] = {m * k, 1};
428	uint32_t broadcastDstShape[] = {m * k, n};	396	uint32_t broadcastDstShape[] = {m * k, n};

Mmhc/mhc_post_backward/op_kernel/arch22/mhc_post_backward_tiling_data_arch22.h+2-2

	@@ -7,7 +7,7 @@
7	* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.	7	* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8	* See LICENSE in the root of the software repository for the full text of the License.	8	* See LICENSE in the root of the software repository for the full text of the License.
9	*/	9	*/
10	- /*!	10	+/*!
11	* \file mhc_post_backward_tiling_data.h	11	* \file mhc_post_backward_tiling_data.h
12	* \brief mhc_post_backward	12	* \brief mhc_post_backward
13	*/	13	*/
	@@ -36,4 +36,4 @@ public:
36	uint64_t loopC{0};	36	uint64_t loopC{0};
37	};	37	};
38		38
39	-#endif // MHC_POST_BACKWARD_TILING_DATA_ARCH22_H	39	+#endif // MHC_POST_BACKWARD_TILING_DATA_ARCH22_H

Mmhc/mhc_post_backward/op_kernel/arch22/mhc_post_backward_tiling_key_arch22.h+17-23

	@@ -1,32 +1,26 @@
1	- /**	1	+/**
2	-* Copyright (c) 2026 Huawei Technologies Co., Ltd.	2	+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3	-* This program is free software, you can redistribute it and/or modify it under the terms and conditions of	3	+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4	-* CANN Open Software License Agreement Version 2.0 (the "License").	4	+ * CANN Open Software License Agreement Version 2.0 (the "License").
5	-* Please refer to the License for details. You may not use this file except in compliance with the License.	5	+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6	-* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,	6	+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7	-* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.	7	+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8	-* See LICENSE in the root of the software repository for the full text of the License.	8	+ * See LICENSE in the root of the software repository for the full text of the License.
9	-*/	9	+ */
10	/*!	10	/*!
11	-* \file mhc_post_backward_tiling_key.h	11	+ * \file mhc_post_backward_tiling_key.h
12	-* \brief mhc_post_backward tiling key declare	12	+ * \brief mhc_post_backward tiling key declare
13	-*/	13	+ */
14		14
15	#ifndef __MHC_POST_BACKWARD_TILING_KEY_ARCH22_H__	15	#ifndef __MHC_POST_BACKWARD_TILING_KEY_ARCH22_H__
16	#define __MHC_POST_BACKWARD_TILING_KEY_ARCH22_H__	16	#define __MHC_POST_BACKWARD_TILING_KEY_ARCH22_H__
17		17
18	#include "ascendc/host_api/tiling/template_argument.h"	18	#include "ascendc/host_api/tiling/template_argument.h"
19		19
20	-ASCENDC_TPL_ARGS_DECL(MhcPostBackward,	20	+ASCENDC_TPL_ARGS_DECL(MhcPostBackward, ASCENDC_TPL_UINT_DECL(SCHMODE, 1, ASCENDC_TPL_UI_LIST, 0));
21	- ASCENDC_TPL_UINT_DECL(SCHMODE, 1, ASCENDC_TPL_UI_LIST, 0)
22	-);
23		21
24	-ASCENDC_TPL_SEL(	22	+ASCENDC_TPL_SEL(ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIV_ONLY),
25	- ASCENDC_TPL_ARGS_SEL(	23	+ ASCENDC_TPL_UINT_SEL(SCHMODE, ASCENDC_TPL_UI_LIST, 0),
26	- ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIV_ONLY),	24	+ ASCENDC_TPL_TILING_STRUCT_SEL(MhcPostBackwardTilingDataArch22)));
27	- ASCENDC_TPL_UINT_SEL(SCHMODE, ASCENDC_TPL_UI_LIST, 0),
28	- ASCENDC_TPL_TILING_STRUCT_SEL(MhcPostBackwardTilingDataArch22)
29	- )
30	-);
31		25
32	-#endif	26	+#endif

Mmhc/mhc_post_backward/op_kernel/arch35/mhc_post_backward_kernel.h+15-16

	@@ -34,9 +34,7 @@ constexpr uint32_t QUEUE_DEPTH = 1; // Single Buffer depth for all queues
34	template <typename T, bool IS_HRES>	34	template <typename T, bool IS_HRES>
35	class MhcPostBackwardKernel {	35	class MhcPostBackwardKernel {
36	public:	36	public:
37	- __aicore__ inline MhcPostBackwardKernel()	37	+ __aicore__ inline MhcPostBackwardKernel() {}
38	- {
39	- }
40	__aicore__ inline void Init(GM_ADDR gradOutput, GM_ADDR x, GM_ADDR hRes, GM_ADDR hOut, GM_ADDR hPost, GM_ADDR gradX,	38	__aicore__ inline void Init(GM_ADDR gradOutput, GM_ADDR x, GM_ADDR hRes, GM_ADDR hOut, GM_ADDR hPost, GM_ADDR gradX,
41	GM_ADDR gradHRes, GM_ADDR gradHOut, GM_ADDR gradHPost, GM_ADDR workspace,	39	GM_ADDR gradHRes, GM_ADDR gradHOut, GM_ADDR gradHPost, GM_ADDR workspace,
42	const MhcPostBackwardTilingDataArch35 tilingData, TPipe tPipe);	40	const MhcPostBackwardTilingDataArch35 tilingData, TPipe tPipe);
	@@ -127,11 +125,12 @@ private:
127	};	125	};
128		126
129	template <typename T, bool IS_HRES>	127	template <typename T, bool IS_HRES>
130	-__aicore__ inline void	128	+__aicore__ inline void MhcPostBackwardKernel<T, IS_HRES>::Init(GM_ADDR gradOutput, GM_ADDR x, GM_ADDR hRes,
131	-MhcPostBackwardKernel<T, IS_HRES>::Init(GM_ADDR gradOutput, GM_ADDR x, GM_ADDR hRes, GM_ADDR hOut, GM_ADDR hPost,	129	+ GM_ADDR hOut, GM_ADDR hPost, GM_ADDR gradX,
132	- GM_ADDR gradX, GM_ADDR gradHRes, GM_ADDR gradHOut, GM_ADDR gradHPost,	130	+ GM_ADDR gradHRes, GM_ADDR gradHOut, GM_ADDR gradHPost,
133	- GM_ADDR workspace, const MhcPostBackwardTilingDataArch35 *tilingData,	131	+ GM_ADDR workspace,
134	- TPipe *tPipe)	132	+ const MhcPostBackwardTilingDataArch35 *tilingData,
		133	+ TPipe *tPipe)
135	{	134	{
136	pipe_ = tPipe;	135	pipe_ = tPipe;
137		136
	@@ -348,7 +347,6 @@ __aicore__ inline void MhcPostBackwardKernel<T, IS_HRES>::ComputeTile(uint32_t t
348	// When aligned, SetValue will fill all valid positions	347	// When aligned, SetValue will fill all valid positions
349	Duplicate(gradHPostTileSums, 0.0f, alignedN_);	348	Duplicate(gradHPostTileSums, 0.0f, alignedN_);
350		349
351	-
352	// ===== Common path for bf16 and fp16: Cast to f32 for computation =====	350	// ===== Common path for bf16 and fp16: Cast to f32 for computation =====
353	LocalTensor<float> gradOutF32 = gradOutF32Buf_.Get<float>();	351	LocalTensor<float> gradOutF32 = gradOutF32Buf_.Get<float>();
354	LocalTensor<float> hOutF32 = hOutF32Buf_.Get<float>();	352	LocalTensor<float> hOutF32 = hOutF32Buf_.Get<float>();
	@@ -465,9 +463,10 @@ __aicore__ inline float MhcPostBackwardKernel<T, IS_HRES>::HierarchicalReduceSum
465	}	463	}
466		464
467	template <typename T, bool IS_HRES>	465	template <typename T, bool IS_HRES>
468	-__aicore__ inline void	466	+__aicore__ inline void MhcPostBackwardKernel<T, IS_HRES>::ComputeGradHOutTile(LocalTensor<float> gradOutF32UB,
469	-MhcPostBackwardKernel<T, IS_HRES>::ComputeGradHOutTile(LocalTensor<float> gradOutF32UB, LocalTensor<float> hPostLocalUB,	467	+ LocalTensor<float> hPostLocalUB,
470	- LocalTensor<T> gradHOutTileUB, uint32_t actualTileD)	468	+ LocalTensor<T> gradHOutTileUB,
		469	+ uint32_t actualTileD)
471	{	470	{
472	uint32_t vfLen = 256 / sizeof(float);	471	uint32_t vfLen = 256 / sizeof(float);
473	uint16_t repeatTimes = (actualTileD + vfLen - 1) / vfLen;	472	uint16_t repeatTimes = (actualTileD + vfLen - 1) / vfLen;
	@@ -504,11 +503,11 @@ MhcPostBackwardKernel<T, IS_HRES>::ComputeGradHOutTile(LocalTensor<float> gradOu
504	}	503	}
505	}	504	}
506		505
507	-
508	template <typename T, bool IS_HRES>	506	template <typename T, bool IS_HRES>
509	-__aicore__ inline void	507	+__aicore__ inline void MhcPostBackwardKernel<T, IS_HRES>::ComputeGradXTile(LocalTensor<float> gradOutF32UB,
510	-MhcPostBackwardKernel<T, IS_HRES>::ComputeGradXTile(LocalTensor<float> gradOutF32UB, LocalTensor<float> hResLocalUB,	508	+ LocalTensor<float> hResLocalUB,
511	- LocalTensor<T> gradXTileUB, uint32_t actualTileD)	509	+ LocalTensor<T> gradXTileUB,
		510	+ uint32_t actualTileD)
512	{	511	{
513	uint32_t vfLen = 256 / sizeof(float);	512	uint32_t vfLen = 256 / sizeof(float);
514	uint16_t repeatTimes = (actualTileD + vfLen - 1) / vfLen;	513	uint16_t repeatTimes = (actualTileD + vfLen - 1) / vfLen;

Mmhc/mhc_post_backward/op_kernel/arch35/mhc_post_backward_tiling_data_arch35.h+4-4

	@@ -7,7 +7,7 @@
7	* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.	7	* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8	* See LICENSE in the root of the software repository for the full text of the License.	8	* See LICENSE in the root of the software repository for the full text of the License.
9	*/	9	*/
10	- /*!	10	+/*!
11	* \file mhc_post_backward_tiling_data.h	11	* \file mhc_post_backward_tiling_data.h
12	* \brief mhc_post_backward	12	* \brief mhc_post_backward
13	*/	13	*/
	@@ -28,11 +28,11 @@ public:
28	uint32_t nTilesD{0};	28	uint32_t nTilesD{0};
29	uint32_t alignedD{0};	29	uint32_t alignedD{0};
30	uint32_t lastTileD{0};	30	uint32_t lastTileD{0};
31	- uint32_t alignedN{0}; // n aligned to 8 for float32 vector ops	31	+ uint32_t alignedN{0}; // n aligned to 8 for float32 vector ops
32	- uint32_t alignedNN{0}; // n*n aligned to 8 for float32 vector ops	32	+ uint32_t alignedNN{0}; // n*n aligned to 8 for float32 vector ops
33	uint32_t itemsPerAic{0};	33	uint32_t itemsPerAic{0};
34	uint32_t remainderItemsAic{0};	34	uint32_t remainderItemsAic{0};
35	TCubeTiling matmulTiling{};	35	TCubeTiling matmulTiling{};
36	};	36	};
37		37
38	-#endif // MHC_POST_BACKWARD_TILING_DATA_ARCH35_H	38	+#endif // MHC_POST_BACKWARD_TILING_DATA_ARCH35_H

Mmhc/mhc_pre/op_host/op_api/mhc_pre.h+4-4

	@@ -15,10 +15,10 @@
15	#include "opdev/make_op_executor.h"	15	#include "opdev/make_op_executor.h"
16		16
17	namespace l0op {	17	namespace l0op {
18	-const std::tuple<aclTensor , aclTensor , aclTensor , aclTensor , aclTensor , aclTensor >	18	+const std::tuple<aclTensor , aclTensor , aclTensor , aclTensor , aclTensor , aclTensor > MhcPre(
19	-MhcPre(const aclTensor x, const aclTensor phi, const aclTensor alpha, const aclTensor bias,	19	+ const aclTensor x, const aclTensor phi, const aclTensor alpha, const aclTensor bias,
20	- const aclTensor *gammaOptional, int64_t outFlag, double normEps, double hcEps, int64_t opImplMode,	20	+ const aclTensor *gammaOptional, int64_t outFlag, double normEps, double hcEps, int64_t opImplMode,
21	- aclOpExecutor *executor);	21	+ aclOpExecutor *executor);
22	}	22	}
23		23
24	#endif // OP_API_MHC_PRE_H_	24	#endif // OP_API_MHC_PRE_H_

Mmhc/mhc_pre/op_kernel/arch35/mhc_pre_common.h+8-2

	@@ -99,7 +99,10 @@ constexpr uint16_t MHC_PRE_X_CONSUMED_FLAG = 9U;
99	constexpr uint16_t MHC_PRE_MM_READY_FLAG = 10U;	99	constexpr uint16_t MHC_PRE_MM_READY_FLAG = 10U;
100		100
101	// Integer shape helpers.	101	// Integer shape helpers.
102	-__aicore__ inline uint32_t BasicApiCeilDiv(uint32_t value, uint32_t div) { return (value + div - 1) / div; }	102	+__aicore__ inline uint32_t BasicApiCeilDiv(uint32_t value, uint32_t div)
		103	+{
		104	+ return (value + div - 1) / div;
		105	+}
103		106
104	__aicore__ inline uint32_t BasicApiAlign(uint32_t value, uint32_t align)	107	__aicore__ inline uint32_t BasicApiAlign(uint32_t value, uint32_t align)
105	{	108	{
	@@ -117,7 +120,10 @@ __aicore__ inline uint64_t MhcPreCeilDiv(uint64_t value, uint64_t div)
117	return div == 0U ? value : (value + div - 1U) / div;	120	return div == 0U ? value : (value + div - 1U) / div;
118	}	121	}
119		122
120	-__aicore__ inline uint64_t MhcPreAlign(uint64_t value, uint64_t align) { return MhcPreCeilDiv(value, align) * align; }	123	+__aicore__ inline uint64_t MhcPreAlign(uint64_t value, uint64_t align)
		124	+{
		125	+ return MhcPreCeilDiv(value, align) * align;
		126	+}
121		127
122	__aicore__ inline constexpr uint32_t MhcPreGetVRegSize()	128	__aicore__ inline constexpr uint32_t MhcPreGetVRegSize()
123	{	129	{

Mmhc/mhc_pre/op_kernel/arch35/mhc_pre_cube_compute.h+4-1

	@@ -51,7 +51,10 @@ public:
51		51
52	__aicore__ inline MhcPreCubeCompute() {}	52	__aicore__ inline MhcPreCubeCompute() {}
53		53
54	- __aicore__ inline uint8_t GetBL1BufferId() { return bL1BufferID_; }	54	+ __aicore__ inline uint8_t GetBL1BufferId()
		55	+ {
		56	+ return bL1BufferID_;
		57	+ }
55		58
56	__aicore__ inline void CopyInA1Nd2Nz(uint64_t m, uint64_t currentK, const AscendC::GlobalTensor<float> &aGlobal,	59	__aicore__ inline void CopyInA1Nd2Nz(uint64_t m, uint64_t currentK, const AscendC::GlobalTensor<float> &aGlobal,
57	const AscendC::LocalTensor<float> &al1Local)	60	const AscendC::LocalTensor<float> &al1Local)

Mmhc/mhc_pre/op_kernel/arch35/mhc_pre_m_k_split.h+4-1

	@@ -636,7 +636,10 @@ private:
636	return vector_.tiling_->multCoreSplitKSize >= MHC_PRE_MK_SEQUENTIAL_PARTIAL_THRESHOLD;	636	return vector_.tiling_->multCoreSplitKSize >= MHC_PRE_MK_SEQUENTIAL_PARTIAL_THRESHOLD;
637	}	637	}
638		638
639	- __aicore__ inline bool UseGmStage() const { return vector_.tiling_->mkUseGmStage != 0U; }	639	+ __aicore__ inline bool UseGmStage() const
		640	+ {
		641	+ return vector_.tiling_->mkUseGmStage != 0U;
		642	+ }
640		643
641	__aicore__ inline void ProcessAic(uint32_t mStart, uint32_t mReal, uint32_t kIndex, uint32_t kStart, uint32_t kEnd)	644	__aicore__ inline void ProcessAic(uint32_t mStart, uint32_t mReal, uint32_t kIndex, uint32_t kStart, uint32_t kEnd)
642	{	645	{

Mmhc/mhc_pre/op_kernel/arch35/mhc_pre_split_bs.h+27-36

Mmhc/mhc_pre/op_kernel/arch35/mhc_pre_tiling_key.h+23-31

Mmhc/mhc_pre_backward/op_host/op_api/mhc_pre_backward.h+5-5

Mmhc/mhc_pre_backward/op_kernel/arch35/mhc_pre_backward_kernel.h+33-29

Mmhc/mhc_pre_backward/op_kernel/arch35/mhc_pre_backward_tiling_key.h+5-10

Mmhc/mhc_pre_backward/op_kernel/arch35/mhc_pre_backward_utils.h+2-2

Mmhc/mhc_pre_sinkhorn/op_host/op_api/mhc_pre_sinkhorn.h+9-9

Mmhc/mhc_pre_sinkhorn/op_host/op_tiling/mhc_pre_sinkhorn_tiling.h+81-80

Mmhc/mhc_pre_sinkhorn/op_kernel/arch35/mhc_pre_sinkhorn_base_arch35.h+4-1

Mmhc/mhc_pre_sinkhorn/op_kernel/arch35/mhc_pre_sinkhorn_cube_compute_arch35.h+208-214

Mmhc/mhc_pre_sinkhorn/op_kernel/arch35/mhc_pre_sinkhorn_m_k_split_core_arch35.h+105-116

Mmhc/mhc_pre_sinkhorn/op_kernel/arch35/mhc_pre_sinkhorn_m_k_split_core_gradout_arch35.h+134-147

Mmhc/mhc_pre_sinkhorn/op_kernel/arch35/mhc_pre_sinkhorn_m_split_core_arch35.h+10-12

Mmhc/mhc_pre_sinkhorn/op_kernel/arch35/mhc_pre_sinkhorn_m_split_core_gradout_arch35.h+17-19

Mmhc/mhc_pre_sinkhorn/op_kernel/mhc_pre_sinkhorn_base.h+17-19

Mmhc/mhc_pre_sinkhorn/op_kernel/mhc_pre_sinkhorn_cube_compute.h+114-108

Mmhc/mhc_pre_sinkhorn/op_kernel/mhc_pre_sinkhorn_m_k_split_core.h+152-119

Mmhc/mhc_pre_sinkhorn/op_kernel/mhc_pre_sinkhorn_m_split_core.h+121-109

Mmhc/mhc_pre_sinkhorn_backward/op_host/op_tiling/arch22/mhc_pre_sinkhorn_backward_arch22_tiling.h+2-2

Mmhc/mhc_pre_sinkhorn_backward/op_host/op_tiling/arch22/mhc_pre_sinkhorn_backward_common.h+6-9

Mmhc/mhc_pre_sinkhorn_backward/op_kernel/arch22/mhc_pre_grad_kernel.h+3-4

Mmhc/mhc_pre_sinkhorn_backward/op_kernel/arch22/mhc_pre_sinkhorn_backward_key_arch22.h+9-16

Mmhc/mhc_pre_sinkhorn_backward/op_kernel/arch35/mhc_pre_sinkhorn_backward_deterministic.h+4-7

Mmhc/mhc_pre_sinkhorn_backward/op_kernel/arch35/mhc_pre_sinkhorn_backward_key_arch35.h+12-21

Mmhc/mhc_pre_sinkhorn_backward/op_kernel/arch35/mhc_pre_sinkhorn_backward_one_stage.h+126-126

Mmhc/mhc_pre_sinkhorn_backward/op_kernel/arch35/mhc_pre_sinkhorn_backward_simd_vf.h+39-42

Mmhc/mhc_pre_sinkhorn_backward/op_kernel/arch35/mhc_pre_sinkhorn_backward_simt_vf.h+17-20

Mmhc/mhc_sinkhorn/op_host/mhc_sinkhorn_tiling.h+3-4

Mmhc/mhc_sinkhorn/op_host/op_api/mhc_sinkhorn.h+1-1

Mmhc/mhc_sinkhorn/op_kernel/arch35/mhc_sinkhorn_struct.h+3-3

Mmhc/mhc_sinkhorn/op_kernel/arch35/mhc_sinkhorn_tiling_key.h+18-26

Mmhc/mhc_sinkhorn_backward/op_host/mhc_sinkhorn_backward_tiling.h+13-13

Mmhc/mhc_sinkhorn_backward/op_host/op_api/mhc_sinkhorn_backward.h+1-1

Mmhc/mhc_sinkhorn_backward/op_kernel/arch35/mhc_sinkhorn_backward_tiling_data.h+2-2

Mmhc/mhc_sinkhorn_backward/op_kernel/arch35/mhc_sinkhorn_backward_tiling_key.h+16-22

Mmhc/mhc_sinkhorn_common/op_host/arch35/mhc_sinkhorn_base_tiling.h+17-17

Mposembedding/apply_rotary_pos_emb/op_graph/fusion_pass/apply_rotary_pos_emb_tensormove_pass.h+11-10

Mposembedding/apply_rotary_pos_emb/op_host/apply_rotary_pos_emb_tiling.h+5-7

Mposembedding/apply_rotary_pos_emb/op_kernel/apply_rotary_pos_emb_base.h+1-1

Mposembedding/apply_rotary_pos_emb/op_kernel/apply_rotary_pos_emb_compute_ab.h+141-142

Mposembedding/apply_rotary_pos_emb/op_kernel/apply_rotary_pos_emb_compute_ab_cast.h+172-173

Mposembedding/apply_rotary_pos_emb/op_kernel/apply_rotary_pos_emb_small.h+44-53

Mposembedding/apply_rotary_pos_emb/op_kernel/arch35/apply_rotary_pos_emb_ab.h+60-49

Mposembedding/apply_rotary_pos_emb/op_kernel/arch35/apply_rotary_pos_emb_bab.h+58-54

Mposembedding/apply_rotary_pos_emb/op_kernel/arch35/apply_rotary_pos_emb_common.h+4-1

Mposembedding/apply_rotary_pos_emb/tests/ut/op_kernel/apply_rotary_pos_emb_tiling.h+2-2

Mposembedding/apply_rotary_pos_emb/tests/ut/op_kernel/apply_rotary_pos_emb_tiling_regbase.h+3-3

Mposembedding/dequant_rope_quant_kvcache/op_host/dequant_rope_quant_kvcache_tiling.h+2-3

Mposembedding/dequant_rope_quant_kvcache/op_kernel/dequant_rope_quant_kvcache.h+94-115

Mposembedding/dequant_rope_quant_kvcache/tests/ut/op_kernel/dequant_rope_quant_kvcache_tiling.h+3-3

Mposembedding/inplace_partial_rotary_mul/op_host/inplace_partial_rotary_mul_tiling.h+26-32

Mposembedding/inplace_partial_rotary_mul/op_kernel/arch35/inplace_partial_rotary_mul_a_and_b.h+36-51

Mposembedding/inplace_partial_rotary_mul/op_kernel/arch35/inplace_partial_rotary_mul_ab.h+21-26

Mposembedding/inplace_partial_rotary_mul/op_kernel/arch35/inplace_partial_rotary_mul_aba_and_ba.h+83-93

Mposembedding/inplace_partial_rotary_mul/op_kernel/arch35/inplace_partial_rotary_mul_arpe_common.h+98-144

Mposembedding/inplace_partial_rotary_mul/op_kernel/arch35/inplace_partial_rotary_mul_bab.h+23-21

Mposembedding/inplace_partial_rotary_mul/op_kernel/arch35/inplace_partial_rotary_mul_common.h+20-23

Mposembedding/inplace_partial_rotary_mul/op_kernel/arch35/rotary_position_embedding_a_and_b_mixed.h+30-38

Mposembedding/inplace_partial_rotary_mul/op_kernel/arch35/rotary_position_embedding_ab_mixed.h+19-26

Mposembedding/inplace_partial_rotary_mul/op_kernel/arch35/rotary_position_embedding_aba_and_ba_mixed.h+56-68

Mposembedding/inplace_partial_rotary_mul/op_kernel/arch35/rotary_position_embedding_bab_mixed.h+23-26

Mposembedding/inplace_partial_rotary_mul/op_kernel/inplace_partial_rotary_mul.h+54-68

Mposembedding/inplace_partial_rotary_mul/op_kernel/inplace_rotate_half_base.h+1-1

Mposembedding/inplace_partial_rotary_mul/op_kernel/inplace_rotate_interleaved_split_bs.h+5-5

Mposembedding/inplace_partial_rotary_mul/op_kernel/inplace_rotate_interleaved_split_bs_mixed.h+8-7

Mposembedding/inplace_partial_rotary_mul/op_kernel/inplace_rotate_interleaved_split_bs_pad.h+4-4

Mposembedding/inplace_partial_rotary_mul/op_kernel/inplace_rotate_interleaved_split_bs_pad_mixed.h+12-11

Mposembedding/inplace_partial_rotary_mul/op_kernel/inplace_rotate_interleaved_split_bsn.h+11-11

Mposembedding/inplace_partial_rotary_mul/op_kernel/inplace_rotate_interleaved_split_bsn_mixed.h+20-20

Mposembedding/inplace_partial_rotary_mul/op_kernel/inplace_rotate_interleaved_split_bsn_pad.h+13-12

Mposembedding/inplace_partial_rotary_mul/op_kernel/inplace_rotate_interleaved_split_bsn_pad_mixed.h+19-19

Mposembedding/inplace_partial_rotary_mul/op_kernel/inplace_rotate_interleaved_split_s.h+10-10

Mposembedding/inplace_partial_rotary_mul/op_kernel/inplace_rotate_interleaved_split_s_mixed.h+12-12

Mposembedding/inplace_partial_rotary_mul/op_kernel/inplace_rotate_interleaved_split_s_pad.h+9-9

Mposembedding/inplace_partial_rotary_mul/op_kernel/inplace_rotate_interleaved_split_s_pad_mixed.h+16-18

Mposembedding/inplace_partial_rotary_mul_grad/op_host/inplace_partial_rotary_mul_grad_regbase_tiling.h+24-6

Mposembedding/inplace_partial_rotary_mul_grad/op_kernel/arch35/inplace_partial_rotary_mul_grad_ab.h+12-9

Mposembedding/inplace_partial_rotary_mul_grad/op_kernel/arch35/inplace_partial_rotary_mul_grad_bab.h+9-9

Mposembedding/inplace_partial_rotary_mul_grad/op_kernel/arch35/inplace_partial_rotary_mul_grad_common.h+4-1

Mposembedding/interleave_rope/examples/aclnn_interleave_rope_example.h+6-6

Mposembedding/interleave_rope/op_host/interleave_rope_tiling.h+5-5

Mposembedding/interleave_rope/op_kernel/interleave_rope_b11d.h+22-27

Mposembedding/interleave_rope/op_kernel/interleave_rope_b1sd.h+30-39

Mposembedding/interleave_rope/op_kernel/interleave_rope_fixed_bnsd_b11d.h+34-47

Mposembedding/interleave_rope/op_kernel/interleave_rope_split_s.h+13-12

Mposembedding/interleave_rope/tests/ut/op_kernel/test_interleave_rope.h+22-22

Mposembedding/kv_rms_norm_rope_cache/op_host/kv_rms_norm_rope_cache_tiling.h+10-22

Mposembedding/kv_rms_norm_rope_cache/op_kernel/arch35/kv_rms_norm_rope_cache_regbase_base.h+62-59

Mposembedding/kv_rms_norm_rope_cache/op_kernel/arch35/kv_rms_norm_rope_cache_regbase_full_load.h+113-128

Mposembedding/kv_rms_norm_rope_cache/op_kernel/arch35/kv_rms_norm_rope_cache_regbase_recompute.h+23-33

Mposembedding/kv_rms_norm_rope_cache/op_kernel/kv_rms_norm_rope_cache_b16_b1sd.h+130-100

Mposembedding/kv_rms_norm_rope_cache/op_kernel/kv_rms_norm_rope_cache_b16_mtp.h+19-20

Mposembedding/kv_rms_norm_rope_cache/op_kernel/kv_rms_norm_rope_cache_b16_mtp_quant.h+1-2

Mposembedding/kv_rms_norm_rope_cache/op_kernel/kv_rms_norm_rope_cache_b16_pa.h+143-120

Mposembedding/kv_rms_norm_rope_cache/op_kernel/kv_rms_norm_rope_cache_b16_pa_blk_bnsd_quant.h+251-183

Mposembedding/kv_rms_norm_rope_cache/op_kernel/kv_rms_norm_rope_cache_b16_pa_blk_nz.h+139-107

Mposembedding/kv_rms_norm_rope_cache/op_kernel/kv_rms_norm_rope_cache_b16_pa_blk_nz_quant.h+221-201

Mposembedding/kv_rms_norm_rope_cache/op_kernel/kv_rms_norm_rope_cache_b16_pa_bnsd_quant.h+211-194

Mposembedding/kv_rms_norm_rope_cache/op_kernel/kv_rms_norm_rope_cache_b16_pa_nz_quant.h+234-153

Mposembedding/kv_rms_norm_rope_cache/op_kernel/kv_rms_norm_rope_cache_comm.h+6-12

Mposembedding/kv_rms_norm_rope_cache/tests/ut/op_kernel/kv_rms_norm_rope_cache_tiling_def.h+41-41

Mposembedding/norm_rope_concat/op_kernel/norm_rope_concat.h+16-17

Mposembedding/norm_rope_concat/op_kernel/norm_rope_concat_base_kernel.h+3-5

Mposembedding/norm_rope_concat/op_kernel/norm_rope_concat_tiling_key.h+1-1

Mposembedding/norm_rope_concat/tests/ut/op_kernel/test_norm_rope_concat.h+27-27

Mposembedding/norm_rope_concat_grad/op_host/norm_rope_concat_grad_base_host.h+2-1

Mposembedding/norm_rope_concat_grad/op_host/op_api/norm_rope_concat_grad.h+5-5

Mposembedding/norm_rope_concat_grad/op_kernel/norm_rope_concat_grad_kernel.h+11-13

Mposembedding/norm_rope_concat_grad/tests/ut/op_kernel/test_norm_rope_concat_grad.h+2-2

Mposembedding/qkv_rms_norm_rope_cache/op_host/op_api/qkv_rms_norm_rope_cache.h+6-6

Mposembedding/qkv_rms_norm_rope_cache/op_host/qkv_rms_norm_rope_cache_tiling.h+24-26

Mposembedding/qkv_rms_norm_rope_cache/op_kernel/qkv_rms_norm_rope_cache_b16_pa_nz_quant.h+225-156

Mposembedding/qkv_rms_norm_rope_cache/op_kernel/qkv_rms_norm_rope_cache_comm.h+24-22

Mposembedding/qkv_rms_norm_rope_cache/tests/ut/op_kernel/test_qkv_rms_norm_rope_cache_tiling.h+39-39

Mposembedding/qkv_rms_norm_rope_cache_with_k_scale/op_kernel/arch35/qkv_rms_norm_rope_cache_with_k_scale_vec.h+9-15

Mposembedding/rope_quant_kvcache/op_host/rope_quant_kvcache_tiling.h+2-3

Mposembedding/rope_quant_kvcache/op_kernel/rope_quant_kvcache.h+28-30

Mposembedding/rope_quant_kvcache/tests/ut/op_kernel/rope_quant_kvcache_tiling.h+11-11

Mposembedding/rope_with_sin_cos_cache/op_host/op_api/rope_with_sin_cos_cache.h+6-6

Mposembedding/rope_with_sin_cos_cache/op_host/rope_with_sin_cos_cache_tiling.h+1-1

Mposembedding/rope_with_sin_cos_cache/op_kernel/arch35/rope_with_sin_cos_cache_base_arch35.h+2-1

Mposembedding/rope_with_sin_cos_cache/op_kernel/arch35/rope_with_sin_cos_cache_f_bf16_arch35.h+106-127

Mposembedding/rope_with_sin_cos_cache/op_kernel/arch35/rope_with_sin_cos_cache_fp32_arch35.h+84-103

Mposembedding/rope_with_sin_cos_cache/op_kernel/rope_with_sin_cos_cache_base.h+6-6

Mposembedding/rope_with_sin_cos_cache/op_kernel/rope_with_sin_cos_cache_f_bf16.h+149-164

Mposembedding/rope_with_sin_cos_cache/op_kernel/rope_with_sin_cos_cache_fp32.h+142-154

Mposembedding/rope_with_sin_cos_cache/tests/ut/op_kernel/test_rope_with_sin_cos_cache.h+2-2

Mposembedding/rotary_position_embedding/examples/aclnn_rotary_position_embedding_example.h+4-4

Mposembedding/rotary_position_embedding/op_host/op_api/rotary_position_embedding.h+2-2

Mposembedding/rotary_position_embedding/op_host/rope_interleaved_tiling.h+6-4

Mposembedding/rotary_position_embedding/op_host/rope_rotate_half_tiling.h+5-4

Mposembedding/rotary_position_embedding/op_host/rope_rotate_matrix_tiling.h+6-6

Mposembedding/rotary_position_embedding/op_host/rotary_position_embedding_tiling.h+30-32

Mposembedding/rotary_position_embedding/op_kernel/arch35/rotary_position_embedding_ab.h+20-23

Mposembedding/rotary_position_embedding/op_kernel/arch35/rotary_position_embedding_bab.h+27-25

Mposembedding/rotary_position_embedding/op_kernel/rotate_half.h+14-15

Mposembedding/rotary_position_embedding/op_kernel/rotate_half_base.h+3-4

Mposembedding/rotary_position_embedding/op_kernel/rotate_half_bf16.h+16-20

Mposembedding/rotary_position_embedding/op_kernel/rotate_half_rope_fullLoad_xd.h+28-27

Mposembedding/rotary_position_embedding/op_kernel/rotate_interleaved_split_bsn_pad.h+4-3

Mposembedding/rotary_position_embedding/op_kernel/rotate_interleaved_split_s_pad.h+3-2

Mposembedding/rotary_position_embedding/op_kernel/rotate_matrix.h+46-37

Mposembedding/rotary_position_embedding/tests/ut/op_kernel/test_rotary_position_embedding.h+2-2

Mposembedding/rotary_position_embedding_grad/op_host/rope_half_grad_tiling.h+3-4

Mposembedding/rotary_position_embedding_grad/op_host/rope_interleaved_grad_tiling.h+4-5

Mposembedding/rotary_position_embedding_grad/op_kernel/arch35/rotary_position_embedding_grad_a_dcos_dsin.h+13-12

Mposembedding/rotary_position_embedding_grad/op_kernel/arch35/rotary_position_embedding_grad_ab.h+24-25

Mposembedding/rotary_position_embedding_grad/op_kernel/arch35/rotary_position_embedding_grad_bab.h+27-25

Mposembedding/rotary_position_embedding_grad/op_kernel/arch35/rotary_position_embedding_grad_dag.h+1-1

Mposembedding/rotary_position_embedding_grad/op_kernel/arch35/rotary_position_embedding_grad_rotary_x.h+13-12

Mposembedding/rotary_position_embedding_grad/op_kernel/arch35/rotary_position_embedding_grad_tiling_key.h+21-24

Mposembedding/rotary_position_embedding_grad/op_kernel/rope_interleaved_grad_common.h+1-1

Mposembedding/rotary_position_embedding_grad/op_kernel/rope_interleaved_grad_splits.h+67-66

Mposembedding/rotary_position_embedding_grad/op_kernel/rotate_half_grad.h+92-106

Mposembedding/rotary_position_embedding_grad/tests/ut/op_kernel/test_rotary_position_embedding_grad.h+4-4

Mposembedding/und_gen_qkv_rms_norm_rope_cache/op_host/op_api/und_gen_qkv_rms_norm_rope_cache.h+7-6

Mposembedding/und_gen_qkv_rms_norm_rope_cache/op_host/und_gen_qkv_rms_norm_rope_cache_tiling.h+16-4

Mposembedding/und_gen_qkv_rms_norm_rope_cache/op_kernel/arch35/und_gen_qkv_rms_norm_rope_cache_mrope_vf.h+35-37

Mposembedding/und_gen_qkv_rms_norm_rope_cache/op_kernel/arch35/und_gen_qkv_rms_norm_rope_cache_regbase.h+8-2

Mtests/mc2_mock_test_frame/mock_framework.h+52-13

Mtests/ut/framework_normal/common/any_value.h+8-6

Mtests/ut/framework_normal/common/hccl_stub.h+6-17

Mtests/ut/framework_normal/common/infer_shape_context_faker.h+32-20

Mtests/ut/framework_normal/common/infer_shaperange_context_faker.h+2-2

Mtests/ut/framework_normal/common/tiling_context_faker.h+72-35

Mtests/ut/framework_normal/op_api/op_api_ut_common/inc/op_api_ut_common/array_desc.h+67-69

Mtests/ut/framework_normal/op_api/op_api_ut_common/inc/op_api_ut_common/op_api_ut.h+13-6

Mtests/ut/framework_normal/op_api/op_api_ut_common/inc/op_api_ut_common/scalar_desc.h+21-13

Mtests/ut/framework_normal/op_api/stub/opdev/make_op_executor.h+4-4

Mtests/ut/framework_normal/op_api/stub/opdev/platform.h+2-1

Mtests/ut/framework_special/utest/inc/tests/utest/ts.h+0-2

Mtests/ut/framework_special/utils/inc/tests/utils/tensor_intf.h+0-1

Mtests/ut/framework_special/utils/inc/tests/utils/tests_utils_log.h+26-26

Mtorch_extension/cann_ops_transformer/common/aclnn_common.h+20-5

Mtorch_extension/cann_ops_transformer/common/hccl_common.h+8-2

	@@ -135,127 +135,126 @@ static inline bool CheckNCDimValid(const aclTensor self, const aclTensor out)
135	return true;	135	return true;
136	}	136	}
137		137
138	-	138	+#define OP_CHECK_NULL(param, retExpr) \
139	-#define OP_CHECK_NULL(param, retExpr) \	139	+ if (IsNullptr(param, #param)) { \
140	- if (IsNullptr(param, #param)) { \	140	+ retExpr; \
141	- retExpr; \
142	}	141	}
143		142
144	-#define OP_CHECK_DTYPE_NOT_SUPPORT(tensor, supportList, retExpr) \	143	+#define OP_CHECK_DTYPE_NOT_SUPPORT(tensor, supportList, retExpr) \
145	- if (!CheckType(tensor->GetDataType(), supportList)) { \	144	+ if (!CheckType(tensor->GetDataType(), supportList)) { \
146	- OP_LOGE(ACLNN_ERR_PARAM_INVALID, "Tensor %s not implemented for %s, should be in dtype support list %s.", \	145	+ OP_LOGE(ACLNN_ERR_PARAM_INVALID, "Tensor %s not implemented for %s, should be in dtype support list %s.", \
147	- #tensor, op::ToString(tensor->GetDataType()).GetString(), op::ToString(supportList).GetString()); \	146	+ #tensor, op::ToString(tensor->GetDataType()).GetString(), op::ToString(supportList).GetString()); \
148	- retExpr; \	147	+ retExpr; \
149	}	148	}
150		149
151	-#define OP_CHECK_DTYPE_NOT_MATCH(tensor, expectedDtype, retExpr) \	150	+#define OP_CHECK_DTYPE_NOT_MATCH(tensor, expectedDtype, retExpr) \
152	- if (tensor->GetDataType() != expectedDtype) { \	151	+ if (tensor->GetDataType() != expectedDtype) { \
153	- OP_LOGE(ACLNN_ERR_PARAM_INVALID, "Tensor %s expected dtype is %s but found %s.", #tensor, \	152	+ OP_LOGE(ACLNN_ERR_PARAM_INVALID, "Tensor %s expected dtype is %s but found %s.", #tensor, \
154	- op::ToString(expectedDtype).GetString(), op::ToString(tensor->GetDataType()).GetString()); \	153	+ op::ToString(expectedDtype).GetString(), op::ToString(tensor->GetDataType()).GetString()); \
155	- retExpr; \	154	+ retExpr; \
156	}	155	}
157		156
158	-#define OP_CHECK_DTYPE_NOT_SAME(tensor1, tensor2, retExpr) \	157	+#define OP_CHECK_DTYPE_NOT_SAME(tensor1, tensor2, retExpr) \
159	- if (tensor1->GetDataType() != tensor2->GetDataType()) { \	158	+ if (tensor1->GetDataType() != tensor2->GetDataType()) { \
160	- OP_LOGE(ACLNN_ERR_PARAM_INVALID, "Expected both tensors to have same dtype, but found %s %s and %s %s.", \	159	+ OP_LOGE(ACLNN_ERR_PARAM_INVALID, "Expected both tensors to have same dtype, but found %s %s and %s %s.", \
161	- #tensor1, op::ToString(tensor1->GetDataType()).GetString(), #tensor2, \	160	+ #tensor1, op::ToString(tensor1->GetDataType()).GetString(), #tensor2, \
162	- op::ToString(tensor2->GetDataType()).GetString()); \	161	+ op::ToString(tensor2->GetDataType()).GetString()); \
163	- retExpr; \	162	+ retExpr; \
164	}	163	}
165		164
166	-#define OP_CHECK_RESULT_DTYPE_CAST_FAILED(dtype, desiredDtype, retExpr) \	165	+#define OP_CHECK_RESULT_DTYPE_CAST_FAILED(dtype, desiredDtype, retExpr) \
167	- ; \	166	+ ; \
168	- if (!CanCast(dtype, desiredDtype)) { \	167	+ if (!CanCast(dtype, desiredDtype)) { \
169	- OP_LOGE(ACLNN_ERR_PARAM_INVALID, "Result type %s can't be cast to the desired output type %s.", \	168	+ OP_LOGE(ACLNN_ERR_PARAM_INVALID, "Result type %s can't be cast to the desired output type %s.", \
170	- op::ToString(dtype).GetString(), op::ToString(desiredDtype).GetString()); \	169	+ op::ToString(dtype).GetString(), op::ToString(desiredDtype).GetString()); \
171	- retExpr; \	170	+ retExpr; \
172	}	171	}
173		172
174	-#define OP_CHECK_BROADCAST(tensor1, tensor2, retExpr) \	173	+#define OP_CHECK_BROADCAST(tensor1, tensor2, retExpr) \
175	- if (!CheckBroadcastShape(tensor1->GetViewShape(), tensor2->GetViewShape())) { \	174	+ if (!CheckBroadcastShape(tensor1->GetViewShape(), tensor2->GetViewShape())) { \
176	- OP_LOGE(ACLNN_ERR_PARAM_INVALID, "The size of tensor %s %s must match the size of tensor %s %s.", #tensor1, \	175	+ OP_LOGE(ACLNN_ERR_PARAM_INVALID, "The size of tensor %s %s must match the size of tensor %s %s.", #tensor1, \
177	- op::ToString(tensor1->GetViewShape()).GetString(), #tensor2, \	176	+ op::ToString(tensor1->GetViewShape()).GetString(), #tensor2, \
178	- op::ToString(tensor2->GetViewShape()).GetString()); \	177	+ op::ToString(tensor2->GetViewShape()).GetString()); \
179	- retExpr; \	178	+ retExpr; \
180	}	179	}
181		180
182	-#define OP_CHECK_BROADCAST_WITH_SHAPE(tensor, shape, retExpr) \	181	+#define OP_CHECK_BROADCAST_WITH_SHAPE(tensor, shape, retExpr) \
183	- if (!CheckBroadcastShape(tensor->GetViewShape(), shape)) { \	182	+ if (!CheckBroadcastShape(tensor->GetViewShape(), shape)) { \
184	- OP_LOGE(ACLNN_ERR_PARAM_INVALID, "The size of tensor %s %s must match the size %s.", #tensor, \	183	+ OP_LOGE(ACLNN_ERR_PARAM_INVALID, "The size of tensor %s %s must match the size %s.", #tensor, \
185	- op::ToString(tensor->GetViewShape()).GetString(), op::ToString(shape).GetString()); \	184	+ op::ToString(tensor->GetViewShape()).GetString(), op::ToString(shape).GetString()); \
186	- retExpr; \	185	+ retExpr; \
187	}	186	}
188		187
189	-#define OP_CHECK_BROADCAST_AND_INFER_SHAPE(tensor1, tensor2, retShape, retExpr) \	188	+#define OP_CHECK_BROADCAST_AND_INFER_SHAPE(tensor1, tensor2, retShape, retExpr) \
190	- if (!BroadcastInferShape(tensor1->GetViewShape(), tensor2->GetViewShape(), retShape)) { \	189	+ if (!BroadcastInferShape(tensor1->GetViewShape(), tensor2->GetViewShape(), retShape)) { \
191	- OP_LOGE(ACLNN_ERR_PARAM_INVALID, "The size of tensor %s %s must match the size of tensor %s %s.", #tensor1, \	190	+ OP_LOGE(ACLNN_ERR_PARAM_INVALID, "The size of tensor %s %s must match the size of tensor %s %s.", #tensor1, \
192	- op::ToString(tensor1->GetViewShape()).GetString(), #tensor2, \	191	+ op::ToString(tensor1->GetViewShape()).GetString(), #tensor2, \
193	- op::ToString(tensor2->GetViewShape()).GetString()); \	192	+ op::ToString(tensor2->GetViewShape()).GetString()); \
194	- retExpr; \	193	+ retExpr; \
195	}	194	}
196		195
197	-#define OP_CHECK_SHAPE_NOT_EQUAL(tensor1, tensor2, retExpr) \	196	+#define OP_CHECK_SHAPE_NOT_EQUAL(tensor1, tensor2, retExpr) \
198	- if (tensor1->GetViewShape() != tensor2->GetViewShape()) { \	197	+ if (tensor1->GetViewShape() != tensor2->GetViewShape()) { \
199	- OP_LOGE(ACLNN_ERR_PARAM_INVALID, \	198	+ OP_LOGE(ACLNN_ERR_PARAM_INVALID, \
200	- "Expected tensor for %s to have same size as tensor for %s, but %s does not " \	199	+ "Expected tensor for %s to have same size as tensor for %s, but %s does not " \
201	- "equal %s.", \	200	+ "equal %s.", \
202	- #tensor1, #tensor2, op::ToString(tensor1->GetViewShape()).GetString(), \	201	+ #tensor1, #tensor2, op::ToString(tensor1->GetViewShape()).GetString(), \
203	- op::ToString(tensor2->GetViewShape()).GetString()); \	202	+ op::ToString(tensor2->GetViewShape()).GetString()); \
204	- retExpr; \	203	+ retExpr; \
205	}	204	}
206		205
207	-#define OP_CHECK_SHAPE_NOT_EQUAL_WITH_EXPECTED_SIZE(tensor, shape, retExpr) \	206	+#define OP_CHECK_SHAPE_NOT_EQUAL_WITH_EXPECTED_SIZE(tensor, shape, retExpr) \
208	- if (tensor->GetViewShape() != shape) { \	207	+ if (tensor->GetViewShape() != shape) { \
209	- OP_LOGE(ACLNN_ERR_PARAM_INVALID, "Expected tensor for %s to have same size as %s, but got %s.", #tensor, \	208	+ OP_LOGE(ACLNN_ERR_PARAM_INVALID, "Expected tensor for %s to have same size as %s, but got %s.", #tensor, \
210	- op::ToString(shape).GetString(), op::ToString(tensor->GetViewShape()).GetString()); \	209	+ op::ToString(shape).GetString(), op::ToString(tensor->GetViewShape()).GetString()); \
211	- retExpr; \	210	+ retExpr; \
212	}	211	}
213		212
214	-#define OP_CHECK_WRONG_DIMENSION(tensor, expectedDimNum, retExpr) \	213	+#define OP_CHECK_WRONG_DIMENSION(tensor, expectedDimNum, retExpr) \
215	- if (tensor->GetViewShape().GetDimNum() != expectedDimNum) { \	214	+ if (tensor->GetViewShape().GetDimNum() != expectedDimNum) { \
216	- OP_LOGE(ACLNN_ERR_PARAM_INVALID, "Expected %zu dimension input, but got %s with sizes %s.", \	215	+ OP_LOGE(ACLNN_ERR_PARAM_INVALID, "Expected %zu dimension input, but got %s with sizes %s.", \
217	- static_cast<size_t>(expectedDimNum), #tensor, op::ToString(tensor->GetViewShape()).GetString()); \	216	+ static_cast<size_t>(expectedDimNum), #tensor, op::ToString(tensor->GetViewShape()).GetString()); \
218	- retExpr; \	217	+ retExpr; \
219	}	218	}
220		219
221	-#define OP_CHECK_MAX_DIM(tensor, maxDim, retExpr) \	220	+#define OP_CHECK_MAX_DIM(tensor, maxDim, retExpr) \
222	- if (tensor->GetViewShape().GetDimNum() > static_cast<size_t>(maxDim)) { \	221	+ if (tensor->GetViewShape().GetDimNum() > static_cast<size_t>(maxDim)) { \
223	- OP_LOGE(ACLNN_ERR_PARAM_INVALID, "The %s tensor cannot be larger than %zu dimensions.", #tensor, \	222	+ OP_LOGE(ACLNN_ERR_PARAM_INVALID, "The %s tensor cannot be larger than %zu dimensions.", #tensor, \
224	- static_cast<size_t>(maxDim)); \	223	+ static_cast<size_t>(maxDim)); \
225	- retExpr; \	224	+ retExpr; \
226	}	225	}
227		226
228	-#define OP_CHECK_MIN_DIM(tensor, minDim, retExpr) \	227	+#define OP_CHECK_MIN_DIM(tensor, minDim, retExpr) \
229	- if (tensor->GetViewShape().GetDimNum() < static_cast<size_t>(minDim)) { \	228	+ if (tensor->GetViewShape().GetDimNum() < static_cast<size_t>(minDim)) { \
230	- OP_LOGE(ACLNN_ERR_PARAM_INVALID, "The %s tensor must have at least %zu dimensions.", #tensor, \	229	+ OP_LOGE(ACLNN_ERR_PARAM_INVALID, "The %s tensor must have at least %zu dimensions.", #tensor, \
231	- static_cast<size_t>(minDim)); \	230	+ static_cast<size_t>(minDim)); \
232	- retExpr; \	231	+ retExpr; \
233	}	232	}
234		233
235	-#define OP_CHECK_COMM_INPUT(workspaceSize, executor) \	234	+#define OP_CHECK_COMM_INPUT(workspaceSize, executor) \
236	- if (workspaceSize == nullptr \|\| executor == nullptr) { \	235	+ if (workspaceSize == nullptr \|\| executor == nullptr) { \
237	- OP_LOGE(ACLNN_ERR_PARAM_NULLPTR, "The workspaceSize or executor is nullptr."); \	236	+ OP_LOGE(ACLNN_ERR_PARAM_NULLPTR, "The workspaceSize or executor is nullptr."); \
238	- return ACLNN_ERR_PARAM_NULLPTR; \	237	+ return ACLNN_ERR_PARAM_NULLPTR; \
239	}	238	}
240		239
241	-#define OP_CHECK_ADD_TO_LAUNCHER_LIST_AICORE(cond, retExpr, errMsg, ...) \	240	+#define OP_CHECK_ADD_TO_LAUNCHER_LIST_AICORE(cond, retExpr, errMsg, ...) \
242	- if (cond) { \	241	+ if (cond) { \
243	- OP_LOGE(ACLNN_ERR_INNER_STATIC_WORKSPACE_INVALID, errMsg, ##__VA_ARGS__); \	242	+ OP_LOGE(ACLNN_ERR_INNER_STATIC_WORKSPACE_INVALID, errMsg, ##__VA_ARGS__); \
244	- retExpr; \	243	+ retExpr; \
245	}	244	}
246		245
247	-#define OP_CHECK_INFERSHAPE(cond, retExpr, errMsg, ...) \	246	+#define OP_CHECK_INFERSHAPE(cond, retExpr, errMsg, ...) \
248	- if (cond) { \	247	+ if (cond) { \
249	- OP_LOGE(ACLNN_ERR_INNER_INFERSHAPE_ERROR, errMsg, ##__VA_ARGS__); \	248	+ OP_LOGE(ACLNN_ERR_INNER_INFERSHAPE_ERROR, errMsg, ##__VA_ARGS__); \
250	- retExpr; \	249	+ retExpr; \
251	}	250	}
252		251
253	-#define OP_CHECK_TENSORLIST_SIZE_EQUAL(tensorlist1, tensorlist2, retExpr) \	252	+#define OP_CHECK_TENSORLIST_SIZE_EQUAL(tensorlist1, tensorlist2, retExpr) \
254	- if ((tensorlist1)->Size() != (tensorlist2)->Size()) { \	253	+ if ((tensorlist1)->Size() != (tensorlist2)->Size()) { \
255	- OP_LOGE(ACLNN_ERR_PARAM_INVALID, \	254	+ OP_LOGE(ACLNN_ERR_PARAM_INVALID, \
256	- "The %s tensorlist and %s tensorlist must have the same number of tensors, but got %ld and %ld.", \	255	+ "The %s tensorlist and %s tensorlist must have the same number of tensors, but got %ld and %ld.", \
257	- #tensorlist1, #tensorlist2, (tensorlist1)->Size(), (tensorlist2)->Size()); \	256	+ #tensorlist1, #tensorlist2, (tensorlist1)->Size(), (tensorlist2)->Size()); \
258	- retExpr; \	257	+ retExpr; \
259	}	258	}
260		259
261	#endif	260	#endif

	@@ -87,43 +87,43 @@ void CollectConvertedTypes(Tuple &t, std::vector<OpApiAnyValue> &params)
87	CallCollect(t, std_utils::make_index_sequence<size>{}, params);	87	CallCollect(t, std_utils::make_index_sequence<size>{}, params);
88	}	88	}
89		89
90	-#define EXEC_OPAPI_PREPARE_CMD(aclnn_api, ...) \	90	+#define EXEC_OPAPI_PREPARE_CMD(aclnn_api, ...) \
91	- ({ \	91	+ ({ \
92	- static auto ret = GRAPH_SUCCESS; \	92	+ static auto ret = GRAPH_SUCCESS; \
93	- do { \	93	+ do { \
94	- static const auto ResetCacheThreadLocalAddr = GetOpApiFuncAddr("ResetCacheThreadLocal"); \	94	+ static const auto ResetCacheThreadLocalAddr = GetOpApiFuncAddr("ResetCacheThreadLocal"); \
95	- static const auto getWorkspaceSizeFuncAddr = GetOpApiFuncAddr(#aclnn_api "GetWorkspaceSize"); \	95	+ static const auto getWorkspaceSizeFuncAddr = GetOpApiFuncAddr(#aclnn_api "GetWorkspaceSize"); \
96	- static const auto opApiFuncAddr = GetOpApiFuncAddr(#aclnn_api); \	96	+ static const auto opApiFuncAddr = GetOpApiFuncAddr(#aclnn_api); \
97	- if (getWorkspaceSizeFuncAddr == nullptr \|\| opApiFuncAddr == nullptr \|\| \	97	+ if (getWorkspaceSizeFuncAddr == nullptr \|\| opApiFuncAddr == nullptr \|\| \
98	- ResetCacheThreadLocalAddr == nullptr) { \	98	+ ResetCacheThreadLocalAddr == nullptr) { \
99	- OP_LOGE("aclnnfallback", "%s or %s not in %s or %s or ResetCacheThreadLocal not found.", \	99	+ OP_LOGE("aclnnfallback", "%s or %s not in %s or %s or ResetCacheThreadLocal not found.", \
100	- #aclnn_api "GetWorkspaceSize", #aclnn_api, GetOpApiLibName(), GetOpApiLibName()); \	100	+ #aclnn_api "GetWorkspaceSize", #aclnn_api, GetOpApiLibName(), GetOpApiLibName()); \
101	- ret = GRAPH_FAILED; \	101	+ ret = GRAPH_FAILED; \
102	- break; \	102	+ break; \
103	- } \	103	+ } \
104	- auto *op_api_params = new (std::nothrow) OpApiParams(); \	104	+ auto *op_api_params = new (std::nothrow) OpApiParams(); \
105	- auto ResetCacheThreadLocalFunc = reinterpret_cast<ResetCacheThreadLocal>(ResetCacheThreadLocalAddr); \	105	+ auto ResetCacheThreadLocalFunc = reinterpret_cast<ResetCacheThreadLocal>(ResetCacheThreadLocalAddr); \
106	- ResetCacheThreadLocalFunc(); \	106	+ ResetCacheThreadLocalFunc(); \
107	- op_api_params->op_api_func = reinterpret_cast<OpApiFunc>(opApiFuncAddr); \	107	+ op_api_params->op_api_func = reinterpret_cast<OpApiFunc>(opApiFuncAddr); \
108	- uint64_t workspace_size = 0; \	108	+ uint64_t workspace_size = 0; \
109	- uint64_t *workspace_size_addr = &workspace_size; \	109	+ uint64_t *workspace_size_addr = &workspace_size; \
110	- aclOpExecutor **executor_addr = &op_api_params->executor; \	110	+ aclOpExecutor **executor_addr = &op_api_params->executor; \
111	- auto converted_params = ConvertTypes(__VA_ARGS__, workspace_size_addr, executor_addr); \	111	+ auto converted_params = ConvertTypes(__VA_ARGS__, workspace_size_addr, executor_addr); \
112	- using TupleT = decltype(converted_params); \	112	+ using TupleT = decltype(converted_params); \
113	- constexpr size_t tuple_size = std::tuple_size<TupleT>::value; \	113	+ constexpr size_t tuple_size = std::tuple_size<TupleT>::value; \
114	- op_api_params->converted_params.reserve(tuple_size); \	114	+ op_api_params->converted_params.reserve(tuple_size); \
115	- CollectConvertedTypes(converted_params, op_api_params->converted_params); \	115	+ CollectConvertedTypes(converted_params, op_api_params->converted_params); \
116	- host_api_ctx->SetOpApiParamsWithDefaultDeleter<OpApiParams>(op_api_params); \	116	+ host_api_ctx->SetOpApiParamsWithDefaultDeleter<OpApiParams>(op_api_params); \
117	- static auto getWorkspaceSizeFunc = ConvertToOpApiFunc(converted_params, getWorkspaceSizeFuncAddr); \	117	+ static auto getWorkspaceSizeFunc = ConvertToOpApiFunc(converted_params, getWorkspaceSizeFuncAddr); \
118	- auto workspace_status = call(getWorkspaceSizeFunc, converted_params); \	118	+ auto workspace_status = call(getWorkspaceSizeFunc, converted_params); \
119	- if (workspace_status != 0) { \	119	+ if (workspace_status != 0) { \
120	- OP_LOGE("aclnnfallback", "call %s failed:", #aclnn_api); \	120	+ OP_LOGE("aclnnfallback", "call %s failed:", #aclnn_api); \
121	- ret = GRAPH_FAILED; \	121	+ ret = GRAPH_FAILED; \
122	- break; \	122	+ break; \
123	- } \	123	+ } \
124	- ret = host_api_ctx->SetWorkspaceSizes({workspace_size}); \	124	+ ret = host_api_ctx->SetWorkspaceSizes({workspace_size}); \
125	- } while (false); \	125	+ } while (false); \
126	- (ret); \	126	+ (ret); \
127	})	127	})
128		128
129	} // namespace fallback	129	} // namespace fallback

	@@ -37,9 +37,9 @@ using TilingClassCase = std::unique_ptr<TilingBaseClass> (*)(gert::TilingContext
37		37
38	class TilingCases {	38	class TilingCases {
39	public:	39	public:
40	- explicit TilingCases(std::string op_type) : op_type_(std::move(op_type))	40	+ explicit TilingCases(std::string op_type)
41	- {	41	+ : op_type_(std::move(op_type))
42	- }	42	+ {}
43		43
44	template <typename T>	44	template <typename T>
45	void AddTiling(int32_t priority)	45	void AddTiling(int32_t priority)
	@@ -147,9 +147,9 @@ private:
147		147
148	class RegisterArch {	148	class RegisterArch {
149	public:	149	public:
150	- explicit RegisterArch(std::string op_type) : op_type_(std::move(op_type))	150	+ explicit RegisterArch(std::string op_type)
151	- {	151	+ : op_type_(std::move(op_type))
152	- }	152	+ {}
153		153
154	template <typename T>	154	template <typename T>
155	RegisterArch &tiling(int32_t priority, int32_t arch)	155	RegisterArch &tiling(int32_t priority, int32_t arch)
	@@ -301,9 +301,9 @@ private:
301		301
302	class RegisterNew {	302	class RegisterNew {
303	public:	303	public:
304	- explicit RegisterNew(std::string op_type) : op_type_(std::move(op_type))	304	+ explicit RegisterNew(std::string op_type)
305	- {	305	+ : op_type_(std::move(op_type))
306	- }	306	+ {}
307		307
308	template <typename T>	308	template <typename T>
309	RegisterNew &tiling(int32_t priority, int32_t soc_version)	309	RegisterNew &tiling(int32_t priority, int32_t soc_version)
	@@ -413,9 +413,9 @@ private:
413		413
414	class Register {	414	class Register {
415	public:	415	public:
416	- explicit Register(std::string op_type) : op_type_(std::move(op_type))	416	+ explicit Register(std::string op_type)
417	- {	417	+ : op_type_(std::move(op_type))
418	- }	418	+ {}
419		419
420	template <typename T>	420	template <typename T>
421	Register &tiling(int32_t priority)	421	Register &tiling(int32_t priority)
	@@ -436,38 +436,38 @@ private:
436		436
437	// op_type: 算子名称， class_name: 注册的 tiling 类, arch：芯片架构号	437	// op_type: 算子名称， class_name: 注册的 tiling 类, arch：芯片架构号
438	// priority: tiling 类的优先级, 越小表示优先级越高, 即会优先选择这个tiling类	438	// priority: tiling 类的优先级, 越小表示优先级越高, 即会优先选择这个tiling类
439	-#define REGISTER_TILING_TEMPLATE_WITH_ARCH(op_type, class_name, archs, priority) \	439	+#define REGISTER_TILING_TEMPLATE_WITH_ARCH(op_type, class_name, archs, priority) \
440	- [[maybe_unused]] uint32_t op_impl_register_template_##op_type##_##class_name##priority; \	440	+ [[maybe_unused]] uint32_t op_impl_register_template_##op_type##_##class_name##priority; \
441	- static Ops::Transformer::OpTiling::RegisterArch VAR_UNUSED##op_type##class_name##priority_register = \	441	+ static Ops::Transformer::OpTiling::RegisterArch VAR_UNUSED##op_type##class_name##priority_register = \
442	Ops::Transformer::OpTiling::RegisterArch(#op_type).tiling<class_name>(priority, archs)	442	Ops::Transformer::OpTiling::RegisterArch(#op_type).tiling<class_name>(priority, archs)
443		443
444	// op_type: 算子名称， class_name: 注册的 tiling 类, soc_version：芯片版本号	444	// op_type: 算子名称， class_name: 注册的 tiling 类, soc_version：芯片版本号
445	// priority: tiling 类的优先级, 越小表示优先级越高, 即会优先选择这个tiling类	445	// priority: tiling 类的优先级, 越小表示优先级越高, 即会优先选择这个tiling类
446	-#define REGISTER_TILING_TEMPLATE_WITH_SOCVERSION(op_type, class_name, soc_versions, priority) \	446	+#define REGISTER_TILING_TEMPLATE_WITH_SOCVERSION(op_type, class_name, soc_versions, priority) \
447	- [[maybe_unused]] uint32_t op_impl_register_template_##op_type##_##class_name##priority; \	447	+ [[maybe_unused]] uint32_t op_impl_register_template_##op_type##_##class_name##priority; \
448	- static Ops::Transformer::OpTiling::RegisterNew VAR_UNUSED##op_type##class_name##priority_register = \	448	+ static Ops::Transformer::OpTiling::RegisterNew VAR_UNUSED##op_type##class_name##priority_register = \
449	Ops::Transformer::OpTiling::RegisterNew(#op_type).tiling<class_name>(priority, soc_versions)	449	Ops::Transformer::OpTiling::RegisterNew(#op_type).tiling<class_name>(priority, soc_versions)
450		450
451	// op_type: 算子名称， class_name: 注册的 tiling 类,	451	// op_type: 算子名称， class_name: 注册的 tiling 类,
452	// priority: tiling 类的优先级, 越小表示优先级越高, 即被选中的概率越大	452	// priority: tiling 类的优先级, 越小表示优先级越高, 即被选中的概率越大
453	-#define REGISTER_TILING_TEMPLATE(op_type, class_name, priority) \	453	+#define REGISTER_TILING_TEMPLATE(op_type, class_name, priority) \
454	- [[maybe_unused]] uint32_t op_impl_register_template_##op_type##_##class_name##priority; \	454	+ [[maybe_unused]] uint32_t op_impl_register_template_##op_type##_##class_name##priority; \
455	- static Ops::Transformer::OpTiling::Register VAR_UNUSED##op_type_##class_name##priority_register = \	455	+ static Ops::Transformer::OpTiling::Register VAR_UNUSED##op_type_##class_name##priority_register = \
456	Ops::Transformer::OpTiling::Register(op_type).tiling<class_name>(priority)	456	Ops::Transformer::OpTiling::Register(op_type).tiling<class_name>(priority)
457		457
458	// op_type: 算子名称， class_name: 注册的 tiling 类,	458	// op_type: 算子名称， class_name: 注册的 tiling 类,
459	// soc_version: soc版本，用于区分不同的soc	459	// soc_version: soc版本，用于区分不同的soc
460	// priority: tiling 类的优先级, 越小表示优先级越高, 即会优先选择这个tiling类	460	// priority: tiling 类的优先级, 越小表示优先级越高, 即会优先选择这个tiling类
461	-#define REGISTER_TILING_TEMPLATE_NEW(op_type, class_name, soc_version, priority) \	461	+#define REGISTER_TILING_TEMPLATE_NEW(op_type, class_name, soc_version, priority) \
462	- [[maybe_unused]] uint32_t op_impl_register_template_##op_type##_##class_name##priority; \	462	+ [[maybe_unused]] uint32_t op_impl_register_template_##op_type##_##class_name##priority; \
463	- static Ops::Transformer::OpTiling::RegisterNew VAR_UNUSED##op_type##class_name##priority_register = \	463	+ static Ops::Transformer::OpTiling::RegisterNew VAR_UNUSED##op_type##class_name##priority_register = \
464	Ops::Transformer::OpTiling::RegisterNew(#op_type).tiling<class_name>(priority, soc_version)	464	Ops::Transformer::OpTiling::RegisterNew(#op_type).tiling<class_name>(priority, soc_version)
465		465
466	// op_type: 算子名称， class_name: 注册的 tiling 类,	466	// op_type: 算子名称， class_name: 注册的 tiling 类,
467	// priority: tiling 类的优先级, 越小表示优先级越高, 即被选中的概率越大	467	// priority: tiling 类的优先级, 越小表示优先级越高, 即被选中的概率越大
468	// 取代 REGISTER_TILING_TEMPLATE , 传入的op_type如果是字符串常量，需要去掉引号	468	// 取代 REGISTER_TILING_TEMPLATE , 传入的op_type如果是字符串常量，需要去掉引号
469	-#define REGISTER_OPS_TILING_TEMPLATE(op_type, class_name, priority) \	469	+#define REGISTER_OPS_TILING_TEMPLATE(op_type, class_name, priority) \
470	- [[maybe_unused]] uint32_t op_impl_register_template_##op_type##_##class_name##priority; \	470	+ [[maybe_unused]] uint32_t op_impl_register_template_##op_type##_##class_name##priority; \
471	- static Ops::Transformer::OpTiling::Register \	471	+ static Ops::Transformer::OpTiling::Register \
472	- __attribute__((unused)) tiling_##op_type##_##class_name##_##priority##_register = \	472	+ __attribute__((unused)) tiling_##op_type##_##class_name##_##priority##_register = \
473	Ops::Transformer::OpTiling::Register(#op_type).tiling<class_name>(priority)	473	Ops::Transformer::OpTiling::Register(#op_type).tiling<class_name>(priority)

	@@ -15,26 +15,26 @@
15	#ifndef COMMON_NN_OP_RESOURCE_H	15	#ifndef COMMON_NN_OP_RESOURCE_H
16	#define COMMON_NN_OP_RESOURCE_H	16	#define COMMON_NN_OP_RESOURCE_H
17		17
18	-#define EXTERN_OP_RESOURCE(kernelName) \	18	+#define EXTERN_OP_RESOURCE(kernelName) \
19	- namespace l0op { \	19	+ namespace l0op { \
20	- extern void *kernelName##TilingRegisterResource(); \	20	+ extern void *kernelName##TilingRegisterResource(); \
21	- extern void *kernelName##InferShapeRegisterResource(); \	21	+ extern void *kernelName##InferShapeRegisterResource(); \
22	- extern void *kernelName##TuningRegisterResource(); \	22	+ extern void *kernelName##TuningRegisterResource(); \
23	- extern const OP_BINARY_RES &kernelName##KernelResource(); \	23	+ extern const OP_BINARY_RES &kernelName##KernelResource(); \
24	- extern const OP_RUNTIME_KB_RES &kernelName##TuningResource(); \	24	+ extern const OP_RUNTIME_KB_RES &kernelName##TuningResource(); \
25	- [[maybe_unused]] uint32_t kernelName##_kernelName_Be_Defined_Multi_Times___; \	25	+ [[maybe_unused]] uint32_t kernelName##_kernelName_Be_Defined_Multi_Times___; \
26	}	26	}
27		27
28	-#define AUTO_GEN_OP_RESOURCE(kernelName) \	28	+#define AUTO_GEN_OP_RESOURCE(kernelName) \
29	- { \	29	+ { \
30	- { \	30	+ { \
31	- #kernelName, \	31	+ #kernelName, \
32	- { \	32	+ { \
33	- {l0op::kernelName##TilingRegisterResource(), l0op::kernelName##InferShapeRegisterResource(), \	33	+ {l0op::kernelName##TilingRegisterResource(), l0op::kernelName##InferShapeRegisterResource(), \
34	- l0op::kernelName##TuningRegisterResource()}, \	34	+ l0op::kernelName##TuningRegisterResource()}, \
35	- l0op::kernelName##KernelResource(), l0op::kernelName##TuningResource() \	35	+ l0op::kernelName##KernelResource(), l0op::kernelName##TuningResource() \
36	- } \	36	+ } \
37	- } \	37	+ } \
38	}	38	}
39		39
40	#endif // COMMON_NN_OP_RESOURCE_H	40	#endif // COMMON_NN_OP_RESOURCE_H

	@@ -71,25 +71,25 @@ public:
71	GluActiveFuncPtr<T> gluFuncPointer;	71	GluActiveFuncPtr<T> gluFuncPointer;
72	};	72	};
73		73
74	-constexpr float BETA_ = 1.0; // beta param of swiglu	74	+constexpr float BETA_ = 1.0; // beta param of swiglu
75	-constexpr uint32_t MAX_EXPERT_PARALLELISM = 10; // allow `MAX_EXPERT_PARALLELISM` experts to compute together	75	+constexpr uint32_t MAX_EXPERT_PARALLELISM = 10; // allow `MAX_EXPERT_PARALLELISM` experts to compute together
76	-constexpr uint32_t UB_BLOCK_UNIT_SIZE = 32; // 32: a block has 32 bytes data	76	+constexpr uint32_t UB_BLOCK_UNIT_SIZE = 32; // 32: a block has 32 bytes data
77	-constexpr uint32_t CUBE_BASE_ALIGN_FACTOR = 16; // 16: baseM align requirement for ai cube	77	+constexpr uint32_t CUBE_BASE_ALIGN_FACTOR = 16; // 16: baseM align requirement for ai cube
78	-constexpr uint32_t CUBE_QUANT_BASE_ALIGN_FACTOR = 32; // 32: quant matmul baseM align requirement for ai cube	78	+constexpr uint32_t CUBE_QUANT_BASE_ALIGN_FACTOR = 32; // 32: quant matmul baseM align requirement for ai cube
79	-constexpr uint32_t INT8_BITS = 8; // 8: a int8 data has 8 bits	79	+constexpr uint32_t INT8_BITS = 8; // 8: a int8 data has 8 bits
80	constexpr uint32_t FP16_INT8_BEST_DATACOPY_BASE_SIZE = 512; // 512: can copy 512 elements of fp16 int8 type every time	80	constexpr uint32_t FP16_INT8_BEST_DATACOPY_BASE_SIZE = 512; // 512: can copy 512 elements of fp16 int8 type every time
81	constexpr uint32_t BF16_INT8_BEST_DATACOPY_BASE_SIZE = 256; // 256: can copy 256 elements of bf16 int8 type every time	81	constexpr uint32_t BF16_INT8_BEST_DATACOPY_BASE_SIZE = 256; // 256: can copy 256 elements of bf16 int8 type every time
82	-constexpr uint32_t INT8_SYNC_N1_SIZE = 256; // 256: when n1 is small than 256, should enable SyncbeforeMM1	82	+constexpr uint32_t INT8_SYNC_N1_SIZE = 256; // 256: when n1 is small than 256, should enable SyncbeforeMM1
83	// a unit block can contain `EXPERT_NUM_ALIGN` int64_t elements	83	// a unit block can contain `EXPERT_NUM_ALIGN` int64_t elements
84	constexpr uint32_t EXPERT_NUM_ALIGN = UB_BLOCK_UNIT_SIZE / sizeof(int64_t);	84	constexpr uint32_t EXPERT_NUM_ALIGN = UB_BLOCK_UNIT_SIZE / sizeof(int64_t);
85	constexpr uint32_t ANTIQUANT_MSD_STEP = 2;	85	constexpr uint32_t ANTIQUANT_MSD_STEP = 2;
86	constexpr uint32_t NUM_ALIGN_TO_THIRTYTWO = 31; // used to align to 32	86	constexpr uint32_t NUM_ALIGN_TO_THIRTYTWO = 31; // used to align to 32
87	constexpr uint32_t NUM_ALIGN_TO_SIXTEEN = 15; // used to align to 16	87	constexpr uint32_t NUM_ALIGN_TO_SIXTEEN = 15; // used to align to 16
88	constexpr uint32_t NUM_ALIGN_TO_ONE_HUNDRED_TWEENTY_EIGHT = 127; // used to align to 128	88	constexpr uint32_t NUM_ALIGN_TO_ONE_HUNDRED_TWEENTY_EIGHT = 127; // used to align to 128
89	-constexpr uint32_t FACTOR_FOR_FLOAT_ALIGN_TO_32BYTE = 8; // a float type num need to duplicate 8 times to align 32	89	+constexpr uint32_t FACTOR_FOR_FLOAT_ALIGN_TO_32BYTE = 8; // a float type num need to duplicate 8 times to align 32
90	-constexpr uint32_t DATASIZE_EACH_REPEAT_TIME = 256; // each repeat time can calc 256Byte data	90	+constexpr uint32_t DATASIZE_EACH_REPEAT_TIME = 256; // each repeat time can calc 256Byte data
91	-constexpr uint32_t MSD_EACH_UB_BLOCK_SIZR = 6 * 1024; // each repeat time can calc 256Byte data	91	+constexpr uint32_t MSD_EACH_UB_BLOCK_SIZR = 6 * 1024; // each repeat time can calc 256Byte data
92	-constexpr uint32_t DATABLOCK_NUM_IN_GATHER = 8; // In Gather API, each repeat collects 8 data blocks	92	+constexpr uint32_t DATABLOCK_NUM_IN_GATHER = 8; // In Gather API, each repeat collects 8 data blocks
93		93
94	template <class T>	94	template <class T>
95	__aicore__ inline constexpr uint32_t GetNumInUbBlock()	95	__aicore__ inline constexpr uint32_t GetNumInUbBlock()

	@@ -15,9 +15,9 @@
15	*/	15	*/
16		16
17	/*!	17	/*!
18	-* \file ffn_wb_group_listing.h	18	+ * \file ffn_wb_group_listing.h
19	-* \brief	19	+ * \brief
20	-*/	20	+ */
21		21
22	#ifndef OP_KERNEL_FFN_WB_GROUP_LISTING_H	22	#ifndef OP_KERNEL_FFN_WB_GROUP_LISTING_H
23	#define OP_KERNEL_FFN_WB_GROUP_LISTING_H	23	#define OP_KERNEL_FFN_WB_GROUP_LISTING_H
	@@ -25,14 +25,14 @@
25	#include "ffn_wb_common.h"	25	#include "ffn_wb_common.h"
26	#include "kernel_operator.h"	26	#include "kernel_operator.h"
27		27
28	-namespace FfnWbBatching{	28	+namespace FfnWbBatching {
29	using namespace AscendC;	29	using namespace AscendC;
30		30
31	class KernelFfnWBGroupListing {	31	class KernelFfnWBGroupListing {
32	public:	32	public:
33	__aicore__ inline KernelFfnWBGroupListing(){};	33	__aicore__ inline KernelFfnWBGroupListing(){};
34	- __aicore__ inline void Init(GM_ADDR x, GM_ADDR groupList, GM_ADDR groupListTmp,	34	+ __aicore__ inline void Init(GM_ADDR x, GM_ADDR groupList, GM_ADDR groupListTmp,
35	- ScheduleContextInfo* scheduleContext, TPipe* tPipe, int64_t groupListingDealFlag);	35	+ ScheduleContextInfo scheduleContext, TPipe tPipe, int64_t groupListingDealFlag);
36	__aicore__ inline void Process(int64_t groupListingDealFlag);	36	__aicore__ inline void Process(int64_t groupListingDealFlag);
37	__aicore__ inline void ProcessExpertCount();	37	__aicore__ inline void ProcessExpertCount();
38		38
	@@ -48,15 +48,15 @@ private:
48	__aicore__ inline void ComputeOneCore(int64_t inputNum);	48	__aicore__ inline void ComputeOneCore(int64_t inputNum);
49		49
50	private:	50	private:
51	- TPipe* pipe_;	51	+ TPipe *pipe_;
52		52
53	- GlobalTensor<int32_t> expandedExpertIdsGm; // 排序后的专家索引（全局内存）	53	+ GlobalTensor<int32_t> expandedExpertIdsGm; // 排序后的专家索引（全局内存）
54	- GlobalTensor<int64_t> groupListGm; // 最终专家Token计数（int64）	54	+ GlobalTensor<int64_t> groupListGm; // 最终专家Token计数（int64）
55	- GlobalTensor<int32_t> expertCountTempGm; // 临时专家计数（int32）	55	+ GlobalTensor<int32_t> expertCountTempGm; // 临时专家计数（int32）
56	LocalTensor<int64_t> groupListOutLocal;	56	LocalTensor<int64_t> groupListOutLocal;
57		57
58	- TQue<QuePosition::VECIN, 1> sortedExpertIdxInQueue; // 专家索引输入队列	58	+ TQue<QuePosition::VECIN, 1> sortedExpertIdxInQueue; // 专家索引输入队列
59	- TQue<QuePosition::VECOUT, 1> expertIdxCountOutQueue; // 最终计数输出队列	59	+ TQue<QuePosition::VECOUT, 1> expertIdxCountOutQueue; // 最终计数输出队列
60		60
61	int64_t blockIdx;	61	int64_t blockIdx;
62	int64_t perLoopRows;	62	int64_t perLoopRows;
	@@ -86,7 +86,8 @@ private:
86	};	86	};
87		87
88	__aicore__ inline void KernelFfnWBGroupListing::Init(GM_ADDR x, GM_ADDR groupList, GM_ADDR groupListTmp,	88	__aicore__ inline void KernelFfnWBGroupListing::Init(GM_ADDR x, GM_ADDR groupList, GM_ADDR groupListTmp,
89	- ScheduleContextInfo* scheduleContext, TPipe* tPipe, int64_t groupListingDealFlag)	89	+ ScheduleContextInfo scheduleContext, TPipe tPipe,
		90	+ int64_t groupListingDealFlag)
90	{	91	{
91	pipe_ = tPipe;	92	pipe_ = tPipe;
92	blockIdx = GetBlockIdx();	93	blockIdx = GetBlockIdx();
	@@ -125,7 +126,7 @@ __aicore__ inline void KernelFfnWBGroupListing::Init(GM_ADDR x, GM_ADDR groupLis
125	// 输入global 排序好的ExpertIds	126	// 输入global 排序好的ExpertIds
126	if (groupListingDealFlag == 1) {	127	if (groupListingDealFlag == 1) {
127	expandedExpertIdsGm.SetGlobalBuffer((__gm__ int32_t )x + (blockIdx - firstCoreIdx) perCoreElements,	128	expandedExpertIdsGm.SetGlobalBuffer((__gm__ int32_t )x + (blockIdx - firstCoreIdx) perCoreElements,
128	- curCoreElements);	129	+ curCoreElements);
129	} else {	130	} else {
130	expandedExpertIdsGm.SetGlobalBuffer((__gm__ int32_t *)x, validGatherIdxLength);	131	expandedExpertIdsGm.SetGlobalBuffer((__gm__ int32_t *)x, validGatherIdxLength);
131	}	132	}
	@@ -139,7 +140,8 @@ __aicore__ inline void KernelFfnWBGroupListing::Init(GM_ADDR x, GM_ADDR groupLis
139	pipe_->InitBuffer(expertIdxCountOutQueue, 1, cntSize);	140	pipe_->InitBuffer(expertIdxCountOutQueue, 1, cntSize);
140	}	141	}
141		142
142	-__aicore__ inline void KernelFfnWBGroupListing::Process(int64_t groupListingDealFlag) {	143	+__aicore__ inline void KernelFfnWBGroupListing::Process(int64_t groupListingDealFlag)
		144	+{
143	if ((blockIdx >= coreNum - GROUP_LISTING_MULTI_AIV_NUM) && (blockIdx < coreNum) && (groupListingDealFlag == 1)) {	145	if ((blockIdx >= coreNum - GROUP_LISTING_MULTI_AIV_NUM) && (blockIdx < coreNum) && (groupListingDealFlag == 1)) {
144	for (int64_t i = 0; i < curcoreLoopsNum; i++) {	146	for (int64_t i = 0; i < curcoreLoopsNum; i++) {
145	int64_t perLoopElements = (i == (curcoreLoopsNum - 1)) ? curCoreLastLoopElements : curCorePerLoopElements;	147	int64_t perLoopElements = (i == (curcoreLoopsNum - 1)) ? curCoreLastLoopElements : curCorePerLoopElements;
	@@ -169,8 +171,7 @@ __aicore__ inline void KernelFfnWBGroupListing::Process(int64_t groupListingDeal
169	}	171	}
170		172
171	DataCopyExtParams copyParams{static_cast<uint16_t>(1),	173	DataCopyExtParams copyParams{static_cast<uint16_t>(1),
172	- static_cast<uint32_t>(curExpertIdOffset * NUM_TWO * sizeof(int64_t)),	174	+ static_cast<uint32_t>(curExpertIdOffset * NUM_TWO * sizeof(int64_t)), 0, 0, 0};
173	- 0, 0, 0};
174	SetWaitFlag<HardEvent::S_MTE3>(HardEvent::S_MTE3);	175	SetWaitFlag<HardEvent::S_MTE3>(HardEvent::S_MTE3);
175	DataCopyPad(groupListGm, groupListOutLocal, copyParams);	176	DataCopyPad(groupListGm, groupListOutLocal, copyParams);
176		177
	@@ -178,7 +179,8 @@ __aicore__ inline void KernelFfnWBGroupListing::Process(int64_t groupListingDeal
178	}	179	}
179	}	180	}
180		181
181	-__aicore__ inline void KernelFfnWBGroupListing::ProcessExpertCount() {	182	+__aicore__ inline void KernelFfnWBGroupListing::ProcessExpertCount()
		183	+{
182	if (blockIdx == coreNum - 1) {	184	if (blockIdx == coreNum - 1) {
183	expertCountCopyIn();	185	expertCountCopyIn();
184	expertCountCompute();	186	expertCountCompute();
	@@ -186,10 +188,11 @@ __aicore__ inline void KernelFfnWBGroupListing::ProcessExpertCount() {
186	}	188	}
187	}	189	}
188		190
189	-__aicore__ inline void KernelFfnWBGroupListing::CopyIn(int64_t loop, int64_t curLoopElements) {	191	+__aicore__ inline void KernelFfnWBGroupListing::CopyIn(int64_t loop, int64_t curLoopElements)
		192	+{
190	LocalTensor<int32_t> sortedExpertIdxInLocal = sortedExpertIdxInQueue.AllocTensor<int32_t>();	193	LocalTensor<int32_t> sortedExpertIdxInLocal = sortedExpertIdxInQueue.AllocTensor<int32_t>();
191	DataCopyExtParams dataCopyParams{static_cast<uint16_t>(1), static_cast<uint32_t>(curLoopElements * sizeof(int32_t)),	194	DataCopyExtParams dataCopyParams{static_cast<uint16_t>(1), static_cast<uint32_t>(curLoopElements * sizeof(int32_t)),
192	- 0, 0, 0};	195	+ 0, 0, 0};
193	DataCopyPadExtParams dataCopyPadParams{false, 0, 0, 0};	196	DataCopyPadExtParams dataCopyPadParams{false, 0, 0, 0};
194	int64_t sortedexpertIdxOffset = loop * curCorePerLoopElements;	197	int64_t sortedexpertIdxOffset = loop * curCorePerLoopElements;
195	DataCopyPad(sortedExpertIdxInLocal, expandedExpertIdsGm[sortedexpertIdxOffset], dataCopyParams, dataCopyPadParams);	198	DataCopyPad(sortedExpertIdxInLocal, expandedExpertIdsGm[sortedexpertIdxOffset], dataCopyParams, dataCopyPadParams);
	@@ -197,11 +200,11 @@ __aicore__ inline void KernelFfnWBGroupListing::CopyIn(int64_t loop, int64_t cur
197	sortedExpertIdxInQueue.EnQue(sortedExpertIdxInLocal);	200	sortedExpertIdxInQueue.EnQue(sortedExpertIdxInLocal);
198	}	201	}
199		202
200	-__aicore__ inline void KernelFfnWBGroupListing::Compute(int64_t curLoopElements) {	203	+__aicore__ inline void KernelFfnWBGroupListing::Compute(int64_t curLoopElements)
		204	+{
201	LocalTensor<int32_t> sortedExpertIdxInLocal = sortedExpertIdxInQueue.DeQue<int32_t>();	205	LocalTensor<int32_t> sortedExpertIdxInLocal = sortedExpertIdxInQueue.DeQue<int32_t>();
202	LocalTensor<int32_t> expertCountOutLocal = expertIdxCountOutQueue.AllocTensor<int32_t>();	206	LocalTensor<int32_t> expertCountOutLocal = expertIdxCountOutQueue.AllocTensor<int32_t>();
203	- Duplicate(expertCountOutLocal.ReinterpretCast<int32_t>(), static_cast<int32_t>(0),	207	+ Duplicate(expertCountOutLocal.ReinterpretCast<int32_t>(), static_cast<int32_t>(0), static_cast<int32_t>(expertNum));
204	- static_cast<int32_t>(expertNum));
205	event_t eventIDVToS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S));	208	event_t eventIDVToS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S));
206	SetFlag<HardEvent::V_S>(eventIDVToS);	209	SetFlag<HardEvent::V_S>(eventIDVToS);
207	WaitFlag<HardEvent::V_S>(eventIDVToS);	210	WaitFlag<HardEvent::V_S>(eventIDVToS);
	@@ -226,11 +229,12 @@ __aicore__ inline void KernelFfnWBGroupListing::Compute(int64_t curLoopElements)
226	sortedExpertIdxInQueue.FreeTensor(sortedExpertIdxInLocal);	229	sortedExpertIdxInQueue.FreeTensor(sortedExpertIdxInLocal);
227	}	230	}
228		231
229	-__aicore__ inline void KernelFfnWBGroupListing::CopyOut() {	232	+__aicore__ inline void KernelFfnWBGroupListing::CopyOut()
		233	+{
230	LocalTensor<int32_t> expertCountOutLocal = expertIdxCountOutQueue.DeQue<int32_t>();	234	LocalTensor<int32_t> expertCountOutLocal = expertIdxCountOutQueue.DeQue<int32_t>();
231		235
232	- DataCopyExtParams copyParams{static_cast<uint16_t>(1), static_cast<uint32_t>((expertNum) * sizeof(int32_t)), 0,	236	+ DataCopyExtParams copyParams{static_cast<uint16_t>(1), static_cast<uint32_t>((expertNum) * sizeof(int32_t)), 0, 0,
233	- 0, 0};	237	+ 0};
234	SetAtomicAdd<int32_t>();	238	SetAtomicAdd<int32_t>();
235	DataCopyPad(expertCountTempGm, expertCountOutLocal, copyParams);	239	DataCopyPad(expertCountTempGm, expertCountOutLocal, copyParams);
236	SetAtomicNone();	240	SetAtomicNone();
	@@ -238,18 +242,20 @@ __aicore__ inline void KernelFfnWBGroupListing::CopyOut() {
238	expertIdxCountOutQueue.FreeTensor(expertCountOutLocal);	242	expertIdxCountOutQueue.FreeTensor(expertCountOutLocal);
239	}	243	}
240		244
241	-__aicore__ inline void KernelFfnWBGroupListing::expertCountCopyIn() {	245	+__aicore__ inline void KernelFfnWBGroupListing::expertCountCopyIn()
		246	+{
242	LocalTensor<int32_t> expertCountTempInLocal = sortedExpertIdxInQueue.AllocTensor<int32_t>();	247	LocalTensor<int32_t> expertCountTempInLocal = sortedExpertIdxInQueue.AllocTensor<int32_t>();
243		248
244	- DataCopyExtParams dataCopyParams{static_cast<uint16_t>(1),	249	+ DataCopyExtParams dataCopyParams{static_cast<uint16_t>(1), static_cast<uint32_t>((expertNum) * sizeof(int32_t)), 0,
245	- static_cast<uint32_t>((expertNum) * sizeof(int32_t)), 0, 0, 0};	250	+ 0, 0};
246	DataCopyPadExtParams dataCopyPadParams{false, 0, 0, 0};	251	DataCopyPadExtParams dataCopyPadParams{false, 0, 0, 0};
247	DataCopyPad(expertCountTempInLocal, expertCountTempGm, dataCopyParams, dataCopyPadParams);	252	DataCopyPad(expertCountTempInLocal, expertCountTempGm, dataCopyParams, dataCopyPadParams);
248		253
249	sortedExpertIdxInQueue.EnQue(expertCountTempInLocal);	254	sortedExpertIdxInQueue.EnQue(expertCountTempInLocal);
250	}	255	}
251		256
252	-__aicore__ inline void KernelFfnWBGroupListing::expertCountCompute() {	257	+__aicore__ inline void KernelFfnWBGroupListing::expertCountCompute()
		258	+{
253	LocalTensor<int32_t> expertCountTempInLocal = sortedExpertIdxInQueue.DeQue<int32_t>();	259	LocalTensor<int32_t> expertCountTempInLocal = sortedExpertIdxInQueue.DeQue<int32_t>();
254		260
255	LocalTensor<int64_t> expertCountOutLocal = expertIdxCountOutQueue.AllocTensor<int64_t>();	261	LocalTensor<int64_t> expertCountOutLocal = expertIdxCountOutQueue.AllocTensor<int64_t>();
	@@ -276,24 +282,28 @@ __aicore__ inline void KernelFfnWBGroupListing::expertCountCompute() {
276	sortedExpertIdxInQueue.FreeTensor(expertCountTempInLocal);	282	sortedExpertIdxInQueue.FreeTensor(expertCountTempInLocal);
277	}	283	}
278		284
279	-__aicore__ inline void KernelFfnWBGroupListing::expertCountCopyOut() {	285	+__aicore__ inline void KernelFfnWBGroupListing::expertCountCopyOut()
		286	+{
280	LocalTensor<int64_t> expertCountOutLocal = expertIdxCountOutQueue.DeQue<int64_t>();	287	LocalTensor<int64_t> expertCountOutLocal = expertIdxCountOutQueue.DeQue<int64_t>();
281	DataCopyExtParams copyParams{static_cast<uint16_t>(1), static_cast<uint32_t>(expertNum * NUM_TWO * sizeof(int64_t)),	288	DataCopyExtParams copyParams{static_cast<uint16_t>(1), static_cast<uint32_t>(expertNum * NUM_TWO * sizeof(int64_t)),
282	- 0, 0, 0};	289	+ 0, 0, 0};
283	DataCopyPad(groupListGm, expertCountOutLocal, copyParams);	290	DataCopyPad(groupListGm, expertCountOutLocal, copyParams);
284		291
285	expertIdxCountOutQueue.FreeTensor(expertCountOutLocal);	292	expertIdxCountOutQueue.FreeTensor(expertCountOutLocal);
286	}	293	}
287		294
288	-__aicore__ inline void KernelFfnWBGroupListing::CopyInOneCore(int64_t inputNum) {	295	+__aicore__ inline void KernelFfnWBGroupListing::CopyInOneCore(int64_t inputNum)
		296	+{
289	LocalTensor<int32_t> inLocal = sortedExpertIdxInQueue.AllocTensor<int32_t>();	297	LocalTensor<int32_t> inLocal = sortedExpertIdxInQueue.AllocTensor<int32_t>();
290	- DataCopyExtParams dataCopyParams{static_cast<uint16_t>(1), static_cast<uint32_t>(inputNum * sizeof(int32_t)), 0, 0, 0};	298	+ DataCopyExtParams dataCopyParams{static_cast<uint16_t>(1), static_cast<uint32_t>(inputNum * sizeof(int32_t)), 0, 0,
		299	+ 0};
291	DataCopyPadExtParams<int32_t> dataCopyPadParams{false, 0, 0, 0};	300	DataCopyPadExtParams<int32_t> dataCopyPadParams{false, 0, 0, 0};
292	DataCopyPad(inLocal, expandedExpertIdsGm, dataCopyParams, dataCopyPadParams);	301	DataCopyPad(inLocal, expandedExpertIdsGm, dataCopyParams, dataCopyPadParams);
293	sortedExpertIdxInQueue.EnQue<int32_t>(inLocal);	302	sortedExpertIdxInQueue.EnQue<int32_t>(inLocal);
294	}	303	}
295		304
296	-__aicore__ inline void KernelFfnWBGroupListing::ComputeOneCore(int64_t inputNum) {	305	+__aicore__ inline void KernelFfnWBGroupListing::ComputeOneCore(int64_t inputNum)
		306	+{
297	LocalTensor<int32_t> inLocal = sortedExpertIdxInQueue.DeQue<int32_t>();	307	LocalTensor<int32_t> inLocal = sortedExpertIdxInQueue.DeQue<int32_t>();
298	SetWaitFlag<HardEvent::MTE2_S>(HardEvent::MTE2_S);	308	SetWaitFlag<HardEvent::MTE2_S>(HardEvent::MTE2_S);
299	if (lastExpertId == -1) {	309	if (lastExpertId == -1) {
	@@ -315,5 +325,5 @@ __aicore__ inline void KernelFfnWBGroupListing::ComputeOneCore(int64_t inputNum)
315	sortedExpertIdxInQueue.FreeTensor(inLocal);	325	sortedExpertIdxInQueue.FreeTensor(inLocal);
316	}	326	}
317		327
318	-} // namespace FfnWbBatching	328	+} // namespace FfnWbBatching
319	-#endif // OP_KERNEL_FFN_WB_GROUP_LISTING_H	329	+#endif // OP_KERNEL_FFN_WB_GROUP_LISTING_H

	@@ -23,42 +23,41 @@
23	#include "op_host/tiling_base.h"	23	#include "op_host/tiling_base.h"
24	#include "util/math_util.h"	24	#include "util/math_util.h"
25		25
26	-namespace optiling	26	+namespace optiling {
27	-{
28	BEGIN_TILING_DATA_DEF(SwinAttentionFFNTilingData)	27	BEGIN_TILING_DATA_DEF(SwinAttentionFFNTilingData)
29	- TILING_DATA_FIELD_DEF(uint32_t, batchSize);	28	+TILING_DATA_FIELD_DEF(uint32_t, batchSize);
30	- TILING_DATA_FIELD_DEF(uint32_t, bmmFormerNum);	29	+TILING_DATA_FIELD_DEF(uint32_t, bmmFormerNum);
31	- TILING_DATA_FIELD_DEF(uint32_t, bmmTailNum);	30	+TILING_DATA_FIELD_DEF(uint32_t, bmmTailNum);
32	- TILING_DATA_FIELD_DEF(uint32_t, bmmFormerBatchNum);	31	+TILING_DATA_FIELD_DEF(uint32_t, bmmFormerBatchNum);
33	- TILING_DATA_FIELD_DEF(uint32_t, bmmTailBatchNum);	32	+TILING_DATA_FIELD_DEF(uint32_t, bmmTailBatchNum);
34	- TILING_DATA_FIELD_DEF(uint32_t, aivNum);	33	+TILING_DATA_FIELD_DEF(uint32_t, aivNum);
35	- TILING_DATA_FIELD_DEF(uint32_t, shift1);	34	+TILING_DATA_FIELD_DEF(uint32_t, shift1);
36	- TILING_DATA_FIELD_DEF(uint32_t, shift2);	35	+TILING_DATA_FIELD_DEF(uint32_t, shift2);
37		36
38	- TILING_DATA_FIELD_DEF(uint32_t, tpBlockSize);	37	+TILING_DATA_FIELD_DEF(uint32_t, tpBlockSize);
39	- TILING_DATA_FIELD_DEF(uint32_t, tpSpaceCnt);	38	+TILING_DATA_FIELD_DEF(uint32_t, tpSpaceCnt);
40	- TILING_DATA_FIELD_DEF(uint32_t, tpSpaceH);	39	+TILING_DATA_FIELD_DEF(uint32_t, tpSpaceH);
41	- TILING_DATA_FIELD_DEF(uint32_t, tpSpaceW);	40	+TILING_DATA_FIELD_DEF(uint32_t, tpSpaceW);
42	- TILING_DATA_FIELD_DEF(uint32_t, blockInSpace);	41	+TILING_DATA_FIELD_DEF(uint32_t, blockInSpace);
43	- TILING_DATA_FIELD_DEF(uint32_t, tpSpaceSize);	42	+TILING_DATA_FIELD_DEF(uint32_t, tpSpaceSize);
44	- TILING_DATA_FIELD_DEF(uint32_t, tpSpaceWTransposed);	43	+TILING_DATA_FIELD_DEF(uint32_t, tpSpaceWTransposed);
45	- TILING_DATA_FIELD_DEF(uint32_t, tpSpaceHTransposed);	44	+TILING_DATA_FIELD_DEF(uint32_t, tpSpaceHTransposed);
46		45
47	- TILING_DATA_FIELD_DEF_STRUCT(TCubeTiling, bmmTilingData);	46	+TILING_DATA_FIELD_DEF_STRUCT(TCubeTiling, bmmTilingData);
48	- TILING_DATA_FIELD_DEF(uint32_t, dataNumPerBatchA);	47	+TILING_DATA_FIELD_DEF(uint32_t, dataNumPerBatchA);
49	- TILING_DATA_FIELD_DEF(uint32_t, dataNumPerBatchD);	48	+TILING_DATA_FIELD_DEF(uint32_t, dataNumPerBatchD);
50	- TILING_DATA_FIELD_DEF(uint32_t, dataNumPerLoop);	49	+TILING_DATA_FIELD_DEF(uint32_t, dataNumPerLoop);
51	- TILING_DATA_FIELD_DEF(uint32_t, reserved);	50	+TILING_DATA_FIELD_DEF(uint32_t, reserved);
52	END_TILING_DATA_DEF;	51	END_TILING_DATA_DEF;
53		52
54	REGISTER_TILING_DATA_CLASS(SwinAttentionFFN, SwinAttentionFFNTilingData)	53	REGISTER_TILING_DATA_CLASS(SwinAttentionFFN, SwinAttentionFFNTilingData)
55		54
56	struct SwinAttentionFFNCompileInfo {	55	struct SwinAttentionFFNCompileInfo {
57	- uint64_t aicNum = 0;	56	+ uint64_t aicNum = 0;
58	- uint64_t ubSize = 0;	57	+ uint64_t ubSize = 0;
59	- uint64_t l1Size = 0;	58	+ uint64_t l1Size = 0;
60	- uint64_t l0cSize = 0;	59	+ uint64_t l0cSize = 0;
61	};	60	};
62		61
63	-} // namespace optiling	62	+} // namespace optiling
64	-#endif // SWIN_ATTENTION_FFN_TILING_H	63	+#endif // SWIN_ATTENTION_FFN_TILING_H

	@@ -25,50 +25,49 @@
25		25
26	namespace optiling {	26	namespace optiling {
27		27
28	-
29	BEGIN_TILING_DATA_DEF(SwinTransformerLnQKVBaseInfo)	28	BEGIN_TILING_DATA_DEF(SwinTransformerLnQKVBaseInfo)
30	- TILING_DATA_FIELD_DEF(uint32_t, inputsize); // [B,S,H]	29	+TILING_DATA_FIELD_DEF(uint32_t, inputsize); // [B,S,H]
31	- TILING_DATA_FIELD_DEF(uint32_t, hSize); // layernorm [H]	30	+TILING_DATA_FIELD_DEF(uint32_t, hSize); // layernorm [H]
32	- TILING_DATA_FIELD_DEF(uint32_t, baseLoopNum); // for every vec block	31	+TILING_DATA_FIELD_DEF(uint32_t, baseLoopNum); // for every vec block
33	- TILING_DATA_FIELD_DEF(uint32_t, remainderBlockNum); // remainder for some vec	32	+TILING_DATA_FIELD_DEF(uint32_t, remainderBlockNum); // remainder for some vec
34		33
35	END_TILING_DATA_DEF;	34	END_TILING_DATA_DEF;
36	REGISTER_TILING_DATA_CLASS(SwinTransformerLnQKVBaseInfoOp, SwinTransformerLnQKVBaseInfo)	35	REGISTER_TILING_DATA_CLASS(SwinTransformerLnQKVBaseInfoOp, SwinTransformerLnQKVBaseInfo)
37		36
38	BEGIN_TILING_DATA_DEF(SwinTransformerLnQKVLayernormTilingData)	37	BEGIN_TILING_DATA_DEF(SwinTransformerLnQKVLayernormTilingData)
39	- TILING_DATA_FIELD_DEF(uint32_t, bLength);	38	+TILING_DATA_FIELD_DEF(uint32_t, bLength);
40	- TILING_DATA_FIELD_DEF(uint32_t, sLength);	39	+TILING_DATA_FIELD_DEF(uint32_t, sLength);
41	- TILING_DATA_FIELD_DEF(uint32_t, hLength);	40	+TILING_DATA_FIELD_DEF(uint32_t, hLength);
42	- TILING_DATA_FIELD_DEF(uint32_t, bsLength);	41	+TILING_DATA_FIELD_DEF(uint32_t, bsLength);
43	- TILING_DATA_FIELD_DEF(uint32_t, shLength);	42	+TILING_DATA_FIELD_DEF(uint32_t, shLength);
44	- TILING_DATA_FIELD_DEF(uint32_t, loopSize);	43	+TILING_DATA_FIELD_DEF(uint32_t, loopSize);
45	- TILING_DATA_FIELD_DEF(uint32_t, elementPerBlock);	44	+TILING_DATA_FIELD_DEF(uint32_t, elementPerBlock);
46	- TILING_DATA_FIELD_DEF(uint32_t, remainderElementPerBlock);	45	+TILING_DATA_FIELD_DEF(uint32_t, remainderElementPerBlock);
47	- TILING_DATA_FIELD_DEF(uint32_t, innerLoopLength);	46	+TILING_DATA_FIELD_DEF(uint32_t, innerLoopLength);
48	- TILING_DATA_FIELD_DEF(uint32_t, innerLoopNum); // 8	47	+TILING_DATA_FIELD_DEF(uint32_t, innerLoopNum); // 8
49	- TILING_DATA_FIELD_DEF(uint32_t, normalBlockElementOffset);	48	+TILING_DATA_FIELD_DEF(uint32_t, normalBlockElementOffset);
50	- TILING_DATA_FIELD_DEF(uint32_t, rollOffset);	49	+TILING_DATA_FIELD_DEF(uint32_t, rollOffset);
51	END_TILING_DATA_DEF;	50	END_TILING_DATA_DEF;
52	REGISTER_TILING_DATA_CLASS(SwinTransformerLnQKVLayernormTilingDataOp, SwinTransformerLnQKVLayernormTilingData)	51	REGISTER_TILING_DATA_CLASS(SwinTransformerLnQKVLayernormTilingDataOp, SwinTransformerLnQKVLayernormTilingData)
53		52
54	BEGIN_TILING_DATA_DEF(SwinTransformerLnQKVMatmulTilingData)	53	BEGIN_TILING_DATA_DEF(SwinTransformerLnQKVMatmulTilingData)
55	- TILING_DATA_FIELD_DEF(uint32_t, bLength);	54	+TILING_DATA_FIELD_DEF(uint32_t, bLength);
56	- TILING_DATA_FIELD_DEF(uint32_t, biasLength);	55	+TILING_DATA_FIELD_DEF(uint32_t, biasLength);
57		56
58	END_TILING_DATA_DEF;	57	END_TILING_DATA_DEF;
59	REGISTER_TILING_DATA_CLASS(SwinTransformerLnQKVMatmulTilingDataOp, SwinTransformerLnQKVMatmulTilingData)	58	REGISTER_TILING_DATA_CLASS(SwinTransformerLnQKVMatmulTilingDataOp, SwinTransformerLnQKVMatmulTilingData)
60		59
61	BEGIN_TILING_DATA_DEF(SwinTransformerLnQKVTilingData)	60	BEGIN_TILING_DATA_DEF(SwinTransformerLnQKVTilingData)
62	- TILING_DATA_FIELD_DEF(uint32_t, maxCoreNum);	61	+TILING_DATA_FIELD_DEF(uint32_t, maxCoreNum);
63	- TILING_DATA_FIELD_DEF(uint32_t, useVectorNum);	62	+TILING_DATA_FIELD_DEF(uint32_t, useVectorNum);
64	- TILING_DATA_FIELD_DEF(uint32_t, workspaceSize);	63	+TILING_DATA_FIELD_DEF(uint32_t, workspaceSize);
65	- TILING_DATA_FIELD_DEF(uint32_t, inputSizeSum);	64	+TILING_DATA_FIELD_DEF(uint32_t, inputSizeSum);
66	- TILING_DATA_FIELD_DEF_STRUCT(SwinTransformerLnQKVBaseInfo, opBaseInfo);	65	+TILING_DATA_FIELD_DEF_STRUCT(SwinTransformerLnQKVBaseInfo, opBaseInfo);
67	- TILING_DATA_FIELD_DEF_STRUCT(SwinTransformerLnQKVLayernormTilingData, layernormTilingParams);	66	+TILING_DATA_FIELD_DEF_STRUCT(SwinTransformerLnQKVLayernormTilingData, layernormTilingParams);
68	- TILING_DATA_FIELD_DEF_STRUCT(TCubeTiling, mmTilingParams);	67	+TILING_DATA_FIELD_DEF_STRUCT(TCubeTiling, mmTilingParams);
69	END_TILING_DATA_DEF;	68	END_TILING_DATA_DEF;
70		69
71	REGISTER_TILING_DATA_CLASS(SwinTransformerLnQKV, SwinTransformerLnQKVTilingData)	70	REGISTER_TILING_DATA_CLASS(SwinTransformerLnQKV, SwinTransformerLnQKVTilingData)
72	-}	71	+} // namespace optiling
73		72
74	-#endif // SWIN_TRANSFORMER_LN_QKV_TILING_H_	73	+#endif // SWIN_TRANSFORMER_LN_QKV_TILING_H_

	@@ -25,62 +25,62 @@
25		25
26	namespace optiling {	26	namespace optiling {
27	BEGIN_TILING_DATA_DEF(SwinTransformerLnQkvQuantBaseInfo)	27	BEGIN_TILING_DATA_DEF(SwinTransformerLnQkvQuantBaseInfo)
28	- TILING_DATA_FIELD_DEF(uint32_t, bSize);	28	+TILING_DATA_FIELD_DEF(uint32_t, bSize);
29	- TILING_DATA_FIELD_DEF(uint32_t, sSize);	29	+TILING_DATA_FIELD_DEF(uint32_t, sSize);
30	- TILING_DATA_FIELD_DEF(uint32_t, hSize);	30	+TILING_DATA_FIELD_DEF(uint32_t, hSize);
31	- TILING_DATA_FIELD_DEF(uint32_t, headNum);	31	+TILING_DATA_FIELD_DEF(uint32_t, headNum);
32	- TILING_DATA_FIELD_DEF(uint32_t, hWinSize);	32	+TILING_DATA_FIELD_DEF(uint32_t, hWinSize);
33	- TILING_DATA_FIELD_DEF(uint32_t, wWinSize);	33	+TILING_DATA_FIELD_DEF(uint32_t, wWinSize);
34	- TILING_DATA_FIELD_DEF(uint32_t, sizePerHead);	34	+TILING_DATA_FIELD_DEF(uint32_t, sizePerHead);
35	- TILING_DATA_FIELD_DEF(uint32_t, patchHeight);	35	+TILING_DATA_FIELD_DEF(uint32_t, patchHeight);
36	- TILING_DATA_FIELD_DEF(uint32_t, patchWeight);	36	+TILING_DATA_FIELD_DEF(uint32_t, patchWeight);
37	- TILING_DATA_FIELD_DEF(uint32_t, lnBaseM);	37	+TILING_DATA_FIELD_DEF(uint32_t, lnBaseM);
38	- TILING_DATA_FIELD_DEF(uint32_t, lnBaseK);	38	+TILING_DATA_FIELD_DEF(uint32_t, lnBaseK);
39	- TILING_DATA_FIELD_DEF(uint32_t, lnBufferM);	39	+TILING_DATA_FIELD_DEF(uint32_t, lnBufferM);
40	- TILING_DATA_FIELD_DEF(uint32_t, lnBufferK);	40	+TILING_DATA_FIELD_DEF(uint32_t, lnBufferK);
41	- TILING_DATA_FIELD_DEF(uint32_t, lnMSubLoop);	41	+TILING_DATA_FIELD_DEF(uint32_t, lnMSubLoop);
42	- TILING_DATA_FIELD_DEF(uint32_t, lnKSubLoop);	42	+TILING_DATA_FIELD_DEF(uint32_t, lnKSubLoop);
43	- TILING_DATA_FIELD_DEF(uint32_t, loopNum);	43	+TILING_DATA_FIELD_DEF(uint32_t, loopNum);
44	- TILING_DATA_FIELD_DEF(uint32_t, loopSum);	44	+TILING_DATA_FIELD_DEF(uint32_t, loopSum);
45	- TILING_DATA_FIELD_DEF(uint32_t, singleCoreLnBsSize);	45	+TILING_DATA_FIELD_DEF(uint32_t, singleCoreLnBsSize);
46	- TILING_DATA_FIELD_DEF(uint32_t, lnBufferNum);	46	+TILING_DATA_FIELD_DEF(uint32_t, lnBufferNum);
47	- TILING_DATA_FIELD_DEF(uint32_t, resverd1);	47	+TILING_DATA_FIELD_DEF(uint32_t, resverd1);
48	END_TILING_DATA_DEF;	48	END_TILING_DATA_DEF;
49	REGISTER_TILING_DATA_CLASS(SwinTransformerLnQkvQuantBaseInfoOp, SwinTransformerLnQkvQuantBaseInfo)	49	REGISTER_TILING_DATA_CLASS(SwinTransformerLnQkvQuantBaseInfoOp, SwinTransformerLnQkvQuantBaseInfo)
50		50
51	BEGIN_TILING_DATA_DEF(SwinTransformerLnQkvQuantMmInfo)	51	BEGIN_TILING_DATA_DEF(SwinTransformerLnQkvQuantMmInfo)
52	- TILING_DATA_FIELD_DEF(uint32_t, mmSizeM);	52	+TILING_DATA_FIELD_DEF(uint32_t, mmSizeM);
53	- TILING_DATA_FIELD_DEF(uint32_t, mmSizeK);	53	+TILING_DATA_FIELD_DEF(uint32_t, mmSizeK);
54	- TILING_DATA_FIELD_DEF(uint32_t, mmSizeN);	54	+TILING_DATA_FIELD_DEF(uint32_t, mmSizeN);
55	- TILING_DATA_FIELD_DEF(uint32_t, dimNum);	55	+TILING_DATA_FIELD_DEF(uint32_t, dimNum);
56	- TILING_DATA_FIELD_DEF(uint32_t, mDim);	56	+TILING_DATA_FIELD_DEF(uint32_t, mDim);
57	- TILING_DATA_FIELD_DEF(uint32_t, nDim);	57	+TILING_DATA_FIELD_DEF(uint32_t, nDim);
58	- TILING_DATA_FIELD_DEF(uint32_t, shareUbForMm);	58	+TILING_DATA_FIELD_DEF(uint32_t, shareUbForMm);
59	- TILING_DATA_FIELD_DEF(uint32_t, mmLoopNum);	59	+TILING_DATA_FIELD_DEF(uint32_t, mmLoopNum);
60	END_TILING_DATA_DEF;	60	END_TILING_DATA_DEF;
61	REGISTER_TILING_DATA_CLASS(SwinTransformerLnQkvQuantMmInfoOp, SwinTransformerLnQkvQuantMmInfo)	61	REGISTER_TILING_DATA_CLASS(SwinTransformerLnQkvQuantMmInfoOp, SwinTransformerLnQkvQuantMmInfo)
62		62
63	BEGIN_TILING_DATA_DEF(SwinTransformerLnQkvQuantTilingData)	63	BEGIN_TILING_DATA_DEF(SwinTransformerLnQkvQuantTilingData)
64	- TILING_DATA_FIELD_DEF(uint32_t, size);	64	+TILING_DATA_FIELD_DEF(uint32_t, size);
65	- TILING_DATA_FIELD_DEF(uint32_t, maxCoreNum);	65	+TILING_DATA_FIELD_DEF(uint32_t, maxCoreNum);
66	- TILING_DATA_FIELD_DEF(uint32_t, lnBlockNum);	66	+TILING_DATA_FIELD_DEF(uint32_t, lnBlockNum);
67	- TILING_DATA_FIELD_DEF(uint32_t, workSpaceSize);	67	+TILING_DATA_FIELD_DEF(uint32_t, workSpaceSize);
68	- TILING_DATA_FIELD_DEF(uint32_t, inputSizeSum);	68	+TILING_DATA_FIELD_DEF(uint32_t, inputSizeSum);
69	- TILING_DATA_FIELD_DEF(uint32_t, tmpShareBufferForLn);	69	+TILING_DATA_FIELD_DEF(uint32_t, tmpShareBufferForLn);
70	- TILING_DATA_FIELD_DEF(uint32_t, tmpBufferForQuant);	70	+TILING_DATA_FIELD_DEF(uint32_t, tmpBufferForQuant);
71	- TILING_DATA_FIELD_DEF(uint32_t, weightK);	71	+TILING_DATA_FIELD_DEF(uint32_t, weightK);
72	- TILING_DATA_FIELD_DEF(uint32_t, weightN);	72	+TILING_DATA_FIELD_DEF(uint32_t, weightN);
73	- TILING_DATA_FIELD_DEF(float, epsilon);	73	+TILING_DATA_FIELD_DEF(float, epsilon);
74	- TILING_DATA_FIELD_DEF_STRUCT(SwinTransformerLnQkvQuantBaseInfo, opBaseInfo);	74	+TILING_DATA_FIELD_DEF_STRUCT(SwinTransformerLnQkvQuantBaseInfo, opBaseInfo);
75	- TILING_DATA_FIELD_DEF_STRUCT(SwinTransformerLnQkvQuantMmInfo, mmInfo);	75	+TILING_DATA_FIELD_DEF_STRUCT(SwinTransformerLnQkvQuantMmInfo, mmInfo);
76	- TILING_DATA_FIELD_DEF_STRUCT(LayerNormTiling, layernromTilingData);	76	+TILING_DATA_FIELD_DEF_STRUCT(LayerNormTiling, layernromTilingData);
77	- TILING_DATA_FIELD_DEF_STRUCT(TCubeTiling, mmTilingParams);	77	+TILING_DATA_FIELD_DEF_STRUCT(TCubeTiling, mmTilingParams);
78	END_TILING_DATA_DEF;	78	END_TILING_DATA_DEF;
79		79
80	REGISTER_TILING_DATA_CLASS(SwinTransformerLnQkvQuant, SwinTransformerLnQkvQuantTilingData)	80	REGISTER_TILING_DATA_CLASS(SwinTransformerLnQkvQuant, SwinTransformerLnQkvQuantTilingData)
81	struct SwinTransformerLnQkvQuantCompileInfo {	81	struct SwinTransformerLnQkvQuantCompileInfo {
82	uint32_t coreNum = 0;	82	uint32_t coreNum = 0;
83	};	83	};
84	-}	84	+} // namespace optiling
85		85
86	-#endif // SWIN_TRANSFORMER_LN_QKV_QUANT_TILING_H_	86	+#endif // SWIN_TRANSFORMER_LN_QKV_QUANT_TILING_H_

	@@ -13,7 +13,6 @@
13	* \brief	13	* \brief
14	*/	14	*/
15		15
16	-
17	#ifndef SWIN_TRANSFORMER_LN_QKV_QUANT_BASE_H	16	#ifndef SWIN_TRANSFORMER_LN_QKV_QUANT_BASE_H
18	#define SWIN_TRANSFORMER_LN_QKV_QUANT_BASE_H	17	#define SWIN_TRANSFORMER_LN_QKV_QUANT_BASE_H
19		18
	@@ -56,25 +55,23 @@ __aicore__ inline uint32_t DivUp(uint32_t num, uint32_t align)
56		55
57	template <typename aDType, typename bDType, typename cDType, bool aTrans, bool bTrans, bool isReuseSource = false>	56	template <typename aDType, typename bDType, typename cDType, bool aTrans, bool bTrans, bool isReuseSource = false>
58	class SwinTransformerLnQkvQuantBase {	57	class SwinTransformerLnQkvQuantBase {
59	- public:	58	+public:
60	- __aicore__ inline SwinTransformerLnQkvQuantBase() {};	59	+ __aicore__ inline SwinTransformerLnQkvQuantBase(){};
61	__aicore__ inline void CopyND2NZ(LocalTensor<int8_t> &dst, const GlobalTensor<int8_t> &src,	60	__aicore__ inline void CopyND2NZ(LocalTensor<int8_t> &dst, const GlobalTensor<int8_t> &src,
62	- LocalTensor<int8_t> &transTensor, const int row,	61	+ LocalTensor<int8_t> &transTensor, const int row, const int col, const int height,
63	- const int col, const int height, const int width, const int gCol);	62	+ const int width, const int gCol);
64	__aicore__ inline void NDPadZeros(LocalTensor<bDType> &dst, const int height, const int calcWidth, const int gCol,	63	__aicore__ inline void NDPadZeros(LocalTensor<bDType> &dst, const int height, const int calcWidth, const int gCol,
65	- const int width, bool isBankConflict);	64	+ const int width, bool isBankConflict);
66	__aicore__ inline void NDTrans2NZ(LocalTensor<bDType> &dst, LocalTensor<bDType> &src, const int calcHigh,	65	__aicore__ inline void NDTrans2NZ(LocalTensor<bDType> &dst, LocalTensor<bDType> &src, const int calcHigh,
67	- const int calcWidth, const bool isBankConflict);	66	+ const int calcWidth, const bool isBankConflict);
68	__aicore__ inline void VecND2NZ(LocalTensor<int8_t> &dst, LocalTensor<int8_t> &src, const int height,	67	__aicore__ inline void VecND2NZ(LocalTensor<int8_t> &dst, LocalTensor<int8_t> &src, const int height,
69	- const int width, const int gCol);	68	+ const int width, const int gCol);
70	};	69	};
71		70
72	-
73	template <typename aDType, typename bDType, typename cDType, bool aTrans, bool bTrans, bool isReuseSource>	71	template <typename aDType, typename bDType, typename cDType, bool aTrans, bool bTrans, bool isReuseSource>
74	-__aicore__ inline void SwinTransformerLnQkvQuantBase<aDType, bDType, cDType, aTrans, bTrans,	72	+__aicore__ inline void SwinTransformerLnQkvQuantBase<aDType, bDType, cDType, aTrans, bTrans, isReuseSource>::CopyND2NZ(
75	- isReuseSource>::CopyND2NZ(LocalTensor<int8_t> &dst, const GlobalTensor<int8_t> &src,	73	+ LocalTensor<int8_t> &dst, const GlobalTensor<int8_t> &src, LocalTensor<int8_t> &transTensor, const int row,
76	- LocalTensor<int8_t> &transTensor, const int row,	74	+ const int col, const int height, const int width, const int gCol)
77	- const int col, const int height, const int width, const int gCol)
78	{	75	{
79	auto srcOffset = ((int64_t)row * (int64_t)gCol + (int64_t)col);	76	auto srcOffset = ((int64_t)row * (int64_t)gCol + (int64_t)col);
80	bool isBankConflict = false;	77	bool isBankConflict = false;
	@@ -91,9 +88,8 @@ __aicore__ inline void SwinTransformerLnQkvQuantBase<aDType, bDType, cDType, aTr
91	}	88	}
92		89
93	template <typename aDType, typename bDType, typename cDType, bool aTrans, bool bTrans, bool isReuseSource>	90	template <typename aDType, typename bDType, typename cDType, bool aTrans, bool bTrans, bool isReuseSource>
94	-__aicore__ inline void SwinTransformerLnQkvQuantBase<aDType, bDType, cDType, aTrans, bTrans,	91	+__aicore__ inline void SwinTransformerLnQkvQuantBase<aDType, bDType, cDType, aTrans, bTrans, isReuseSource>::VecND2NZ(
95	- isReuseSource>::VecND2NZ(LocalTensor<int8_t> &dst, LocalTensor<int8_t> &src,	92	+ LocalTensor<int8_t> &dst, LocalTensor<int8_t> &src, const int height, const int width, const int gCol)
96	- const int height, const int width, const int gCol)
97	{	93	{
98	int calcHigh = DivUp(height, BLOCK_NUM_PER_FRACTAL);	94	int calcHigh = DivUp(height, BLOCK_NUM_PER_FRACTAL);
99	int calcWidth = DivUp(width, BLOCK_SIZE_32);	95	int calcWidth = DivUp(width, BLOCK_SIZE_32);
	@@ -106,9 +102,9 @@ __aicore__ inline void SwinTransformerLnQkvQuantBase<aDType, bDType, cDType, aTr
106	}	102	}
107		103
108	template <typename aDType, typename bDType, typename cDType, bool aTrans, bool bTrans, bool isReuseSource>	104	template <typename aDType, typename bDType, typename cDType, bool aTrans, bool bTrans, bool isReuseSource>
109	-__aicore__ inline void SwinTransformerLnQkvQuantBase<aDType, bDType, cDType, aTrans, bTrans,	105	+__aicore__ inline void SwinTransformerLnQkvQuantBase<aDType, bDType, cDType, aTrans, bTrans, isReuseSource>::NDPadZeros(
110	- isReuseSource>::NDPadZeros(LocalTensor<bDType> &dst, const int height,	106	+ LocalTensor<bDType> &dst, const int height, const int calcWidth, const int gCol, const int width,
111	- const int calcWidth, const int gCol, const int width, bool isBankConflict)	107	+ bool isBankConflict)
112	{	108	{
113	const int C0_SIZE = BLOCK_SIZE_32 / sizeof(int8_t);	109	const int C0_SIZE = BLOCK_SIZE_32 / sizeof(int8_t);
114	if (gCol % BLOCK_NUM_PER_FRACTAL) {	110	if (gCol % BLOCK_NUM_PER_FRACTAL) {
	@@ -127,7 +123,7 @@ __aicore__ inline void SwinTransformerLnQkvQuantBase<aDType, bDType, cDType, aTr
127	int stride = calcWidth * (C0_SIZE * sizeof(bDType) / BLOCK_SIZE_32);	123	int stride = calcWidth * (C0_SIZE * sizeof(bDType) / BLOCK_SIZE_32);
128	if (masktail != 0) {	124	if (masktail != 0) {
129	if constexpr (IsSameType<bDType, int8_t>::value) {	125	if constexpr (IsSameType<bDType, int8_t>::value) {
130	- LocalTensor <int16_t> tmpTrnasTensor = dst.template ReinterpretCast<int16_t>();	126	+ LocalTensor<int16_t> tmpTrnasTensor = dst.template ReinterpretCast<int16_t>();
131	if (stride < 32) {	127	if (stride < 32) {
132	Duplicate(tmpTrnasTensor[offset], (int16_t)0, mask, DivUp(height, 8), stride, 8 * stride);	128	Duplicate(tmpTrnasTensor[offset], (int16_t)0, mask, DivUp(height, 8), stride, 8 * stride);
133	} else {	129	} else {
	@@ -147,27 +143,26 @@ __aicore__ inline void SwinTransformerLnQkvQuantBase<aDType, bDType, cDType, aTr
147	if (tailHigh) {	143	if (tailHigh) {
148	auto dstOffset = height * calcWidth * BLOCK_CUBE;	144	auto dstOffset = height * calcWidth * BLOCK_CUBE;
149	if constexpr (IsSameType<bDType, int8_t>::value) {	145	if constexpr (IsSameType<bDType, int8_t>::value) {
150	- LocalTensor <int16_t> tmpDst = dst.template ReinterpretCast<int16_t>();	146	+ LocalTensor<int16_t> tmpDst = dst.template ReinterpretCast<int16_t>();
151	Duplicate(tmpDst[dstOffset], (int16_t)0,	147	Duplicate(tmpDst[dstOffset], (int16_t)0,
152	- (BLOCK_NUM_PER_FRACTAL - tailHigh) * calcWidth * BLOCK_NUM_PER_FRACTAL);	148	+ (BLOCK_NUM_PER_FRACTAL - tailHigh) * calcWidth * BLOCK_NUM_PER_FRACTAL);
153	} else {	149	} else {
154	Duplicate(dst[dstOffset], (bDType)0,	150	Duplicate(dst[dstOffset], (bDType)0,
155	- (BLOCK_NUM_PER_FRACTAL - tailHigh) * calcWidth * BLOCK_NUM_PER_FRACTAL);	151	+ (BLOCK_NUM_PER_FRACTAL - tailHigh) * calcWidth * BLOCK_NUM_PER_FRACTAL);
156	}	152	}
157	}	153	}
158	}	154	}
159		155
160	-
161	template <typename aDType, typename bDType, typename cDType, bool aTrans, bool bTrans, bool isReuseSource>	156	template <typename aDType, typename bDType, typename cDType, bool aTrans, bool bTrans, bool isReuseSource>
162	-__aicore__ inline void SwinTransformerLnQkvQuantBase<aDType, bDType, cDType, aTrans, bTrans,	157	+__aicore__ inline void SwinTransformerLnQkvQuantBase<aDType, bDType, cDType, aTrans, bTrans, isReuseSource>::NDTrans2NZ(
163	- isReuseSource>::NDTrans2NZ(LocalTensor<bDType> &dst, LocalTensor<bDType> &src,	158	+ LocalTensor<bDType> &dst, LocalTensor<bDType> &src, const int calcHigh, const int calcWidth,
164	- const int calcHigh, const int calcWidth, const bool isBankConflict)	159	+ const bool isBankConflict)
165	{	160	{
166	const int C0_SIZE = BLOCK_SIZE_32 / sizeof(bDType);	161	const int C0_SIZE = BLOCK_SIZE_32 / sizeof(bDType);
167	if constexpr (IsSameType<bDType, int8_t>::value) {	162	if constexpr (IsSameType<bDType, int8_t>::value) {
168	struct UnaryRepeatParams intriParams;	163	struct UnaryRepeatParams intriParams;
169	uint64_t mask[2] = {uint64_t(-1), uint64_t(-1)};	164	uint64_t mask[2] = {uint64_t(-1), uint64_t(-1)};
170	- int blkStride = isBankConflict ? calcWidth + 1: calcWidth;	165	+ int blkStride = isBankConflict ? calcWidth + 1 : calcWidth;
171	intriParams.dstBlkStride = 1;	166	intriParams.dstBlkStride = 1;
172	intriParams.srcBlkStride = blkStride;	167	intriParams.srcBlkStride = blkStride;
173	intriParams.dstRepStride = intriParams.dstBlkStride * DEFAULT_BLK_NUM;	168	intriParams.dstRepStride = intriParams.dstBlkStride * DEFAULT_BLK_NUM;
	@@ -201,5 +196,4 @@ __aicore__ inline void SwinTransformerLnQkvQuantBase<aDType, bDType, cDType, aTr
201	}	196	}
202	}	197	}
203		198
204	-	199	+#endif // SWIN_TRANSFORMER_LN_QKV_QUANT_BASE_H
205	-#endif // SWIN_TRANSFORMER_LN_QKV_QUANT_BASE_H