Star1122
2723
代码
代码
Issues222
Pull Requests411
Actions
项目讨论
Wiki
分析
Star1122
2723
已合并
attention 重复安装头文件修改 #6020
chenglongyu创建于 5月27日
attention 重复安装头文件修改 #6020
已合并
chenglongyu创建于 5月27日
从chenglongyu/ops-transformerrepeat_clean_ins合入到cann/ops-transformermaster
共 58 个文件变更+74-73
Rattention/attention_worker_combine/op_kernel/common_utils.h→attention/attention_worker_combine/op_kernel/attention_worker_combine_common_utils.h+0-0
文件重命名但无更改。
Mattention/attention_worker_combine/op_kernel/attention_worker_combine_split_bs.h+3-3
@@ -17,19 +17,19 @@
17#define OP_KERNEL_ATTENTION_WORKER_COMBINE_SPLIT_BS_H17#define OP_KERNEL_ATTENTION_WORKER_COMBINE_SPLIT_BS_H
18 18 
19#include "kernel_operator.h"19#include "kernel_operator.h"
20-#include "common_utils.h"20+#include "attention_worker_combine_common_utils.h"
21using namespace AscendC;21using namespace AscendC;
22template <typename T>22template <typename T>
23class KernelAttentionWorkerCombineSplitBS {23class KernelAttentionWorkerCombineSplitBS {
24public:24public:
25 __aicore__ inline KernelAttentionWorkerCombineSplitBS(){}25 __aicore__ inline KernelAttentionWorkerCombineSplitBS(){}
26- 26+ 
27 __aicore__ inline KernelAttentionWorkerCombineSplitBS(TPipe *pipe, const AttentionWorkerCombineTilingData *tiling)27 __aicore__ inline KernelAttentionWorkerCombineSplitBS(TPipe *pipe, const AttentionWorkerCombineTilingData *tiling)
28 : pipe_(pipe), tl_(tiling){}28 : pipe_(pipe), tl_(tiling){}
29 29 
30 __aicore__ inline void Init(GM_ADDR schedule_context, GM_ADDR expert_scales, GM_ADDR layer_id, GM_ADDR y,30 __aicore__ inline void Init(GM_ADDR schedule_context, GM_ADDR expert_scales, GM_ADDR layer_id, GM_ADDR y,
31 GM_ADDR next_layer_id);31 GM_ADDR next_layer_id);
32- 32+ 
33 __aicore__ inline void Process();33 __aicore__ inline void Process();
34 34 
35protected:35protected:
Mattention/attention_worker_combine/op_kernel/attention_worker_combine_split_h.h+3-3
@@ -17,7 +17,7 @@
17#define OP_KERNEL_ATTENTION_WORKER_COMBINE_SPLIT_H_H17#define OP_KERNEL_ATTENTION_WORKER_COMBINE_SPLIT_H_H
18 18 
19#include "kernel_operator.h"19#include "kernel_operator.h"
20-#include "common_utils.h"20+#include "attention_worker_combine_common_utils.h"
21 21 
22using namespace AscendC;22using namespace AscendC;
23template <typename T>23template <typename T>
@@ -30,7 +30,7 @@ public:
30 30 
31 __aicore__ inline void Init(GM_ADDR schedule_context, GM_ADDR expert_scales, GM_ADDR layer_id, GM_ADDR y,31 __aicore__ inline void Init(GM_ADDR schedule_context, GM_ADDR expert_scales, GM_ADDR layer_id, GM_ADDR y,
32 GM_ADDR next_layer_id);32 GM_ADDR next_layer_id);
33- 33+ 
34 __aicore__ inline void Process();34 __aicore__ inline void Process();
35 35 
36protected:36protected:
@@ -287,4 +287,4 @@ __aicore__ inline void KernelAttentionWorkerCombineSplitH<T>::SToVSync()
287 WaitFlag<HardEvent::S_V>(eventIDSToV);287 WaitFlag<HardEvent::S_V>(eventIDSToV);
288}288}
289 289 
290-#endif // OP_KERNEL_ATTENTION_WORKER_COMBINE_SPLIT_H_H290+#endif // OP_KERNEL_ATTENTION_WORKER_COMBINE_SPLIT_H_H
Mattention/attention_worker_combine/op_kernel/attention_worker_combine_split_k.h+1-1
@@ -17,7 +17,7 @@
17#define OP_KERNEL_ATTENTION_WORKER_COMBINE_SPLIT_K_H_17#define OP_KERNEL_ATTENTION_WORKER_COMBINE_SPLIT_K_H_
18 18 
19#include "kernel_operator.h"19#include "kernel_operator.h"
20-#include "common_utils.h"20+#include "attention_worker_combine_common_utils.h"
21 21 
22using namespace AscendC;22using namespace AscendC;
23template <typename T>23template <typename T>
Rattention/block_sparse_attention/op_kernel/kernel_common.hpp→attention/block_sparse_attention/op_kernel/block_sparse_attention_kernel_common.hpp+0-0
文件重命名但无更改。
Mattention/block_sparse_attention/op_kernel/block_sparse_attention_kernel_regular_arch22.h+1-1
@@ -16,7 +16,7 @@
16#ifndef BLOCK_SPARSE_ATTENTION_KERNEL_H16#ifndef BLOCK_SPARSE_ATTENTION_KERNEL_H
17#define BLOCK_SPARSE_ATTENTION_KERNEL_H17#define BLOCK_SPARSE_ATTENTION_KERNEL_H
18 18 
19-#include "kernel_common.hpp"19+#include "block_sparse_attention_kernel_common.hpp"
20 20 
21using namespace NpuArch;21using namespace NpuArch;
22using namespace RfaKenelCommon;22using namespace RfaKenelCommon;
Mattention/flash_attention_score_grad/op_kernel/arch22/basic_modules/addr_compute_det.h+1-1
@@ -16,7 +16,7 @@
16#ifndef __ADDR_COMPUTE_DET_H__16#ifndef __ADDR_COMPUTE_DET_H__
17#define __ADDR_COMPUTE_DET_H__17#define __ADDR_COMPUTE_DET_H__
18 18 
19-#include "common_header.h"19+#include "flash_attention_score_grad_common_header.h"
20using namespace AscendC;20using namespace AscendC;
21 21 
22 22 
Mattention/flash_attention_score_grad/op_kernel/arch22/basic_modules/cube_modules/cube_addr.h+1-1
@@ -14,7 +14,7 @@
14 */14 */
15#ifndef _CUBE_ADDR_H_15#ifndef _CUBE_ADDR_H_
16#define _CUBE_ADDR_H_16#define _CUBE_ADDR_H_
17-#include "../common_header.h"17+#include "../flash_attention_score_grad_common_header.h"
18 18 
19namespace CUBE_ADDR {19namespace CUBE_ADDR {
20class CubeAddr {20class CubeAddr {
Mattention/flash_attention_score_grad/op_kernel/arch22/basic_modules/cube_op.h+3-3
@@ -18,7 +18,7 @@
18#define _CUBE_OP_H_18#define _CUBE_OP_H_
19 19 
20#include "kernel_operator.h"20#include "kernel_operator.h"
21-#include "common_header.h"21+#include "flash_attention_score_grad_common_header.h"
22 22 
23using namespace AscendC;23using namespace AscendC;
24 24 
@@ -209,7 +209,7 @@ private:
209 LocalTensor<TYPE> l1_b1_pong_tensor;209 LocalTensor<TYPE> l1_b1_pong_tensor;
210 LocalTensor<TYPE> l1_b2_ping_tensor;210 LocalTensor<TYPE> l1_b2_ping_tensor;
211 LocalTensor<TYPE> l1_b2_pong_tensor;211 LocalTensor<TYPE> l1_b2_pong_tensor;
212- 212+ 
213 // L0 tensor213 // L0 tensor
214 LocalTensor<TYPE> l0_a_ping_tensor;214 LocalTensor<TYPE> l0_a_ping_tensor;
215 LocalTensor<TYPE> l0_a_pong_tensor;215 LocalTensor<TYPE> l0_a_pong_tensor;
@@ -223,7 +223,7 @@ private:
223 LocalTensor<float> l0_c1_pong_tensor;223 LocalTensor<float> l0_c1_pong_tensor;
224 LocalTensor<float> l0_c2_ping_tensor;224 LocalTensor<float> l0_c2_ping_tensor;
225 LocalTensor<float> l0_c2_pong_tensor;225 LocalTensor<float> l0_c2_pong_tensor;
226- 226+ 
227 // ping pong flag227 // ping pong flag
228 uint32_t ping_pong_flag_l1_a_ = 0;228 uint32_t ping_pong_flag_l1_a_ = 0;
229 uint32_t ping_pong_flag_l1_b_ = 0;229 uint32_t ping_pong_flag_l1_b_ = 0;
Mattention/flash_attention_score_grad/op_kernel/arch22/basic_modules/cube_op_bn2.h+1-1
@@ -16,7 +16,7 @@
16#ifndef __CUBE_OP_BN2_H__16#ifndef __CUBE_OP_BN2_H__
17#define __CUBE_OP_BN2_H__17#define __CUBE_OP_BN2_H__
18 18 
19-#include "common_header.h"19+#include "flash_attention_score_grad_common_header.h"
20#include "kernel_operator.h"20#include "kernel_operator.h"
21 21 
22using namespace AscendC;22using namespace AscendC;
Mattention/flash_attention_score_grad/op_kernel/arch22/basic_modules/cube_op_det.h+1-1
@@ -16,7 +16,7 @@
16#ifndef __CUBE_DET_OP_H__16#ifndef __CUBE_DET_OP_H__
17#define __CUBE_DET_OP_H__17#define __CUBE_DET_OP_H__
18 18 
19-#include "common_header.h"19+#include "flash_attention_score_grad_common_header.h"
20#include "kernel_operator.h"20#include "kernel_operator.h"
21 21 
22using namespace AscendC;22using namespace AscendC;
Rattention/flash_attention_score_grad/op_kernel/arch22/basic_modules/common_header.h→attention/flash_attention_score_grad/op_kernel/arch22/basic_modules/flash_attention_score_grad_common_header.h+0-0
文件重命名但无更改。
Mattention/flash_attention_score_grad/op_kernel/arch22/basic_modules/vec_modules/vec_addr.h+1-1
@@ -14,7 +14,7 @@
14 */14 */
15#ifndef _VEC_ADDR_H_15#ifndef _VEC_ADDR_H_
16#define _VEC_ADDR_H_16#define _VEC_ADDR_H_
17-#include "../common_header.h"17+#include "../flash_attention_score_grad_common_header.h"
18 18 
19namespace VEC_ADDR {19namespace VEC_ADDR {
20class VecAddr {20class VecAddr {
Mattention/flash_attention_score_grad/op_kernel/arch22/basic_modules/vec_op_det.h+1-1
@@ -17,7 +17,7 @@
17#define _VEC_OP_DET_H_17#define _VEC_OP_DET_H_
18 18 
19#include "dropmask.h"19#include "dropmask.h"
20-#include "common_header.h"20+#include "flash_attention_score_grad_common_header.h"
21#include "kernel_operator.h"21#include "kernel_operator.h"
22 22 
23template <typename FAGT>23template <typename FAGT>
Mattention/flash_attention_score_grad/op_kernel/arch22/flash_attention_score_grad_s1s2_bn2.h+1-1
@@ -20,7 +20,7 @@
20#include "kernel_operator.h"20#include "kernel_operator.h"
21#include "pse.h"21#include "pse.h"
22#include "dropmask.h"22#include "dropmask.h"
23-#include "./basic_modules/common_header.h"23+#include "./basic_modules/flash_attention_score_grad_common_header.h"
24#include "./basic_modules/cube_op_bn2.h"24#include "./basic_modules/cube_op_bn2.h"
25using matmul::Matmul;25using matmul::Matmul;
26using matmul::MatmulType;26using matmul::MatmulType;
Mattention/flash_attention_score_grad/op_kernel/arch22/flash_attention_score_grad_s1s2_bn2gs1s2_basic.h+3-3
@@ -16,7 +16,7 @@
16#define _FLASH_ATTENTION_SCORE_GRAD_S1S2_BASIC_H_16#define _FLASH_ATTENTION_SCORE_GRAD_S1S2_BASIC_H_
17 17 
18#include "kernel_operator.h"18#include "kernel_operator.h"
19-#include "./basic_modules/common_header.h"19+#include "./basic_modules/flash_attention_score_grad_common_header.h"
20#include "./basic_modules/cube_modules/cube_addr.h"20#include "./basic_modules/cube_modules/cube_addr.h"
21#include "./basic_modules/vec_modules/vec_pre.h"21#include "./basic_modules/vec_modules/vec_pre.h"
22#include "./basic_modules/vec_modules/vec_sfmg.h"22#include "./basic_modules/vec_modules/vec_sfmg.h"
@@ -147,13 +147,13 @@ __aicore__ inline void FlashAttentionScoreGradBasic<TYPE, TILING_CLASS>::CubePro
147 } else {147 } else {
148 SetFlag();148 SetFlag();
149 cubeOp.Cube2Process(cubeAddrInfo[(taskId - 1) % 2], (__gm__ TYPE *)mm1WorkSpaceAddr, (__gm__ TYPE *)key,149 cubeOp.Cube2Process(cubeAddrInfo[(taskId - 1) % 2], (__gm__ TYPE *)mm1WorkSpaceAddr, (__gm__ TYPE *)key,
150- (__gm__ float *)dqWorkSpaceAddr);150+ (__gm__ float *)dqWorkSpaceAddr);
151 WaitFlag();151 WaitFlag();
152 SetFlag();152 SetFlag();
153 cubeOp.Cube3Process(cube3AddrInfo[(taskId - 1) % 2], (__gm__ TYPE *)mm1WorkSpaceAddr, (__gm__ TYPE *)query,153 cubeOp.Cube3Process(cube3AddrInfo[(taskId - 1) % 2], (__gm__ TYPE *)mm1WorkSpaceAddr, (__gm__ TYPE *)query,
154 (__gm__ float *)dkWorkSpaceAddr);154 (__gm__ float *)dkWorkSpaceAddr);
155 cubeOp.Cube3Process(cube3AddrInfo[(taskId - 1) % 2], (__gm__ TYPE *)mm2WorkSpaceAddr, (__gm__ TYPE *)dy,155 cubeOp.Cube3Process(cube3AddrInfo[(taskId - 1) % 2], (__gm__ TYPE *)mm2WorkSpaceAddr, (__gm__ TYPE *)dy,
156- (__gm__ float *)dvWorkSpaceAddr);156+ (__gm__ float *)dvWorkSpaceAddr);
157 WaitFlag();157 WaitFlag();
158 }158 }
159 }159 }
Mattention/flash_attention_score_grad/op_kernel/arch22/flash_attention_score_grad_s1s2_bn2gs1s2_basic_det.h+1-1
@@ -17,7 +17,7 @@
17#define _FLASH_ATTENTION_SCORE_GRAD_S1S2_BASIC_DET_H_17#define _FLASH_ATTENTION_SCORE_GRAD_S1S2_BASIC_DET_H_
18 18 
19#include "kernel_operator.h"19#include "kernel_operator.h"
20-#include "./basic_modules/common_header.h"20+#include "./basic_modules/flash_attention_score_grad_common_header.h"
21#include "./basic_modules/vec_modules/vec_post_det.h"21#include "./basic_modules/vec_modules/vec_post_det.h"
22#include "./basic_modules/vec_modules/vec_pre_det.h"22#include "./basic_modules/vec_modules/vec_pre_det.h"
23#include "./basic_modules/vec_modules/vec_sfmg_det.h"23#include "./basic_modules/vec_modules/vec_sfmg_det.h"
Mattention/flash_attention_score_grad/op_kernel/arch22/flash_attention_score_grad_s1s2_bn2gs1s2_sab.h+1-1
@@ -20,7 +20,7 @@
20#include "lib/matmul_intf.h"20#include "lib/matmul_intf.h"
21#include "pse.h"21#include "pse.h"
22#include "dropmask.h"22#include "dropmask.h"
23-#include "./basic_modules/common_header.h"23+#include "./basic_modules/flash_attention_score_grad_common_header.h"
24 24 
25 25 
26using namespace matmul;26using namespace matmul;
Rattention/fused_infer_attention_score/op_kernel/kernel_common.hpp→attention/fused_infer_attention_score/op_kernel/fia_kernel_common.hpp+0-0
文件重命名但无更改。
Mattention/fused_infer_attention_score/op_kernel/flash_attention_regular.h+1-1
@@ -15,7 +15,7 @@
15#ifndef FLASH_ATTENTION_REGULAR_H15#ifndef FLASH_ATTENTION_REGULAR_H
16#define FLASH_ATTENTION_REGULAR_H16#define FLASH_ATTENTION_REGULAR_H
17 17 
18-#include "kernel_common.hpp"18+#include "fia_kernel_common.hpp"
19 19 
20using namespace NpuArch;20using namespace NpuArch;
21using namespace KernelCommon;21using namespace KernelCommon;
Mattention/fused_infer_attention_score/op_kernel/flash_attention_regular_decode.h+1-1
@@ -15,7 +15,7 @@
15#ifndef FLASH_ATTENTION_REGULAR_DECODE_H15#ifndef FLASH_ATTENTION_REGULAR_DECODE_H
16#define FLASH_ATTENTION_REGULAR_DECODE_H16#define FLASH_ATTENTION_REGULAR_DECODE_H
17 17 
18-#include "kernel_common.hpp"18+#include "fia_kernel_common.hpp"
19 19 
20using namespace NpuArch;20using namespace NpuArch;
21using namespace KernelCommon;21using namespace KernelCommon;
Mattention/mla_preprocess/op_kernel/mla_iterator.h+9-9
@@ -112,14 +112,14 @@ struct l1_to_fb {
112 uint16_t dstGap) {};112 uint16_t dstGap) {};
113};113};
114 114 
115-#include "gm_to_l1_iterator.h"115+#include "mla_preprocess_gm_to_l1_iterator.h"
116-#include "gm_to_ub_iterator.h"116+#include "mla_preprocess_gm_to_ub_iterator.h"
117-#include "l0c_to_gm_iterator.h"117+#include "mla_preprocess_l0c_to_gm_iterator.h"
118-#include "l0c_to_l1_iterator.h"118+#include "mla_preprocess_l0c_to_l1_iterator.h"
119-#include "l0c_to_ub_iterator.h"119+#include "mla_preprocess_l0c_to_ub_iterator.h"
120-#include "l1_to_bt_iterator.h"120+#include "mla_preprocess_l1_to_bt_iterator.h"
121-#include "l1_to_fb_iterator.h"121+#include "mla_preprocess_l1_to_fb_iterator.h"
122-#include "l1_to_l0_iterator.h"122+#include "mla_preprocess_l1_to_l0_iterator.h"
123-#include "l1_to_ub_iterator.h"123+#include "mla_preprocess_l1_to_ub_iterator.h"
124}124}
125#endif125#endif
Mattention/mla_preprocess/op_kernel/mla_preprocess_bf16.h+1-1
@@ -17,7 +17,7 @@
17#define MLA_PREPROCESS_BF16_H17#define MLA_PREPROCESS_BF16_H
18 18 
19#include "lib/matmul_intf.h"19#include "lib/matmul_intf.h"
20-#include "mla_common.h"20+#include "mla_preprocess_mla_common.h"
21#include "mla_iterator.h"21#include "mla_iterator.h"
22#include "mla_mem.h"22#include "mla_mem.h"
23#include "mla_mma.h"23#include "mla_mma.h"
Mattention/mla_preprocess/op_kernel/mla_preprocess_fp16.h+1-1
@@ -17,7 +17,7 @@
17#define MLA_PREPROCESS_FP16_H17#define MLA_PREPROCESS_FP16_H
18 18 
19#include "lib/matmul_intf.h"19#include "lib/matmul_intf.h"
20-#include "mla_common.h"20+#include "mla_preprocess_mla_common.h"
21#include "mla_iterator.h"21#include "mla_iterator.h"
22#include "mla_mem.h"22#include "mla_mem.h"
23#include "mla_mma.h"23#include "mla_mma.h"
Rattention/mla_preprocess/op_kernel/gm_to_l1_iterator.h→attention/mla_preprocess/op_kernel/mla_preprocess_gm_to_l1_iterator.h+0-0
文件重命名但无更改。
Rattention/mla_preprocess/op_kernel/gm_to_ub_iterator.h→attention/mla_preprocess/op_kernel/mla_preprocess_gm_to_ub_iterator.h+0-0
文件重命名但无更改。
Rattention/mla_preprocess/op_kernel/l0c_to_gm_iterator.h→attention/mla_preprocess/op_kernel/mla_preprocess_l0c_to_gm_iterator.h+0-0
文件重命名但无更改。
Rattention/mla_preprocess/op_kernel/l0c_to_l1_iterator.h→attention/mla_preprocess/op_kernel/mla_preprocess_l0c_to_l1_iterator.h+0-0
文件重命名但无更改。
Rattention/mla_preprocess/op_kernel/l0c_to_ub_iterator.h→attention/mla_preprocess/op_kernel/mla_preprocess_l0c_to_ub_iterator.h+0-0
文件重命名但无更改。
Rattention/mla_preprocess/op_kernel/l1_to_bt_iterator.h→attention/mla_preprocess/op_kernel/mla_preprocess_l1_to_bt_iterator.h+0-0
文件重命名但无更改。
Rattention/mla_preprocess/op_kernel/l1_to_fb_iterator.h→attention/mla_preprocess/op_kernel/mla_preprocess_l1_to_fb_iterator.h+0-0
文件重命名但无更改。
Rattention/mla_preprocess/op_kernel/l1_to_l0_iterator.h→attention/mla_preprocess/op_kernel/mla_preprocess_l1_to_l0_iterator.h+0-0
文件重命名但无更改。
Rattention/mla_preprocess/op_kernel/l1_to_ub_iterator.h→attention/mla_preprocess/op_kernel/mla_preprocess_l1_to_ub_iterator.h+0-0
文件重命名但无更改。
Rattention/mla_preprocess/op_kernel/mla_common.h→attention/mla_preprocess/op_kernel/mla_preprocess_mla_common.h+0-0
文件重命名但无更改。
Mattention/mla_preprocess/op_kernel/mla_preprocess_no_quant.h+1-1
@@ -17,7 +17,7 @@
17#define MLA_PREPROCESS_NO_QUANT_H17#define MLA_PREPROCESS_NO_QUANT_H
18 18 
19#include "lib/matmul_intf.h"19#include "lib/matmul_intf.h"
20-#include "mla_common.h"20+#include "mla_preprocess_mla_common.h"
21#include "mla_iterator.h"21#include "mla_iterator.h"
22#include "mla_mem.h"22#include "mla_mem.h"
23#include "mla_mma.h"23#include "mla_mma.h"
Mattention/nsa_selected_attention_grad/basic_modules/cube_op.h+1-1
@@ -13,7 +13,7 @@
13 */13 */
14#pragma once14#pragma once
15#include "kernel_operator.h"15#include "kernel_operator.h"
16-#include "common_header.h"16+#include "nsa_selected_attention_grad_common_header.h"
17using namespace AscendC;17using namespace AscendC;
18 18 
19namespace NSAG_BASIC {19namespace NSAG_BASIC {
Rattention/nsa_selected_attention_grad/basic_modules/common_header.h→attention/nsa_selected_attention_grad/basic_modules/nsa_selected_attention_grad_common_header.h+0-0
文件重命名但无更改。
Mattention/nsa_selected_attention_grad/basic_modules/vec_op.h+3-2
@@ -15,7 +15,7 @@
15 15 
16#pragma once16#pragma once
17#include "kernel_operator.h"17#include "kernel_operator.h"
18-#include "common_header.h"18+#include "nsa_selected_attention_grad_common_header.h"
19 19 
20namespace NSAG_BASIC {20namespace NSAG_BASIC {
21struct StaticParams {21struct StaticParams {
@@ -521,7 +521,8 @@ __aicore__ inline void VecOp<NSAGT>::Process(const int64_t dyGmOffset, const int
521 521 
522 for (int32_t i = 0; i < loop; i++) {522 for (int32_t i = 0; i < loop; i++) {
523 if (i == 0) {523 if (i == 0) {
524- AscendC::WaitFlag<HardEvent::MTE2_V>(static_cast<int32_t>(vWaitMte2)); // wait softmax_max and softmax_sum MTE2524+ AscendC::WaitFlag<HardEvent::MTE2_V>(
525+ static_cast<int32_t>(vWaitMte2)); // wait softmax_max and softmax_sum MTE2
525 }526 }
526 if (i == loop - 1 && tailM != 0) {527 if (i == loop - 1 && tailM != 0) {
527 processM = tailM;528 processM = tailM;
Mattention/nsa_selected_attention_grad/op_kernel/nsa_selected_attention_grad_bs1.h+1-1
@@ -16,7 +16,7 @@
16#pragma once16#pragma once
17#include "lib/matmul_intf.h"17#include "lib/matmul_intf.h"
18#include "kernel_operator.h"18#include "kernel_operator.h"
19-#include "../basic_modules/common_header.h"19+#include "../basic_modules/nsa_selected_attention_grad_common_header.h"
20using namespace AscendC;20using namespace AscendC;
21using matmul::Matmul;21using matmul::Matmul;
22using matmul::MatmulType;22using matmul::MatmulType;
Mattention/nsa_selected_attention_grad/op_kernel/nsa_selected_attention_grad_bs1_basic.h+1-1
@@ -17,7 +17,7 @@
17#include "kernel_operator.h"17#include "kernel_operator.h"
18#include "../basic_modules/cube_op.h"18#include "../basic_modules/cube_op.h"
19#include "../basic_modules/vec_op.h"19#include "../basic_modules/vec_op.h"
20-#include "../basic_modules/common_header.h"20+#include "../basic_modules/nsa_selected_attention_grad_common_header.h"
21#include "nsa_selected_attention_grad_post.h"21#include "nsa_selected_attention_grad_post.h"
22 22 
23namespace NSAG_BASIC {23namespace NSAG_BASIC {
Mattention/prompt_flash_attention/op_kernel/arch22/pfa_service_matmul_kv_nd.h+1-1
@@ -26,7 +26,7 @@
26#include "kernel_tiling/kernel_tiling.h"26#include "kernel_tiling/kernel_tiling.h"
27#include "lib/matmul_intf.h"27#include "lib/matmul_intf.h"
28#include "lib/matrix/matmul/tiling.h"28#include "lib/matrix/matmul/tiling.h"
29-#include "mla_common.h"29+#include "prompt_flash_attention_mla_common.h"
30 30 
31template <typename INPUT_T, typename T, bool pageAttention = false>31template <typename INPUT_T, typename T, bool pageAttention = false>
32class PfaMatmulKvNd {32class PfaMatmulKvNd {
Rattention/prompt_flash_attention/op_kernel/arch22/mla_common.h→attention/prompt_flash_attention/op_kernel/arch22/prompt_flash_attention_mla_common.h+0-0
文件重命名但无更改。
Mattention/prompt_flash_attention/op_kernel/arch22/prompt_flash_attention_s1s2_bns1_mla.h+1-1
@@ -17,7 +17,7 @@
17#define PROMPT_FLASH_ATTENTION_S1S2_BNS1_MLA_H17#define PROMPT_FLASH_ATTENTION_S1S2_BNS1_MLA_H
18 18 
19#include "util.h"19#include "util.h"
20-#include "mla_common.h"20+#include "prompt_flash_attention_mla_common.h"
21#include "mla_custom_matmul_policy_d192.h"21#include "mla_custom_matmul_policy_d192.h"
22#include "mla_custom_matmul_policy_d128.h"22#include "mla_custom_matmul_policy_d128.h"
23#if ASC_DEVKIT_MAJOR >= 923#if ASC_DEVKIT_MAJOR >= 9
Mattention/prompt_flash_attention/op_kernel/arch22/prompt_flash_attention_s1s2_bns1_mla_baseapi.h+1-1
@@ -17,7 +17,7 @@
17#define PROMPT_FLASH_ATTENTION_S1S2_BNS1_MLA_BASEAPI_H17#define PROMPT_FLASH_ATTENTION_S1S2_BNS1_MLA_BASEAPI_H
18 18 
19#include "util.h"19#include "util.h"
20-#include "mla_common.h"20+#include "prompt_flash_attention_mla_common.h"
21#include "mla_custom_matmul_policy_d192.h"21#include "mla_custom_matmul_policy_d192.h"
22#include "mla_custom_matmul_policy_d128.h"22#include "mla_custom_matmul_policy_d128.h"
23#if ASC_DEVKIT_MAJOR >= 923#if ASC_DEVKIT_MAJOR >= 9
Mattention/rain_fusion_attention/op_kernel/rain_fusion_attention_kernel.h+1-1
@@ -16,7 +16,7 @@
16#ifndef RAIN_FUSION_ATTENTION_KERNEL_H16#ifndef RAIN_FUSION_ATTENTION_KERNEL_H
17#define RAIN_FUSION_ATTENTION_KERNEL_H17#define RAIN_FUSION_ATTENTION_KERNEL_H
18 18 
19-#include "kernel_common.hpp"19+#include "rain_fusion_attention_kernel_common.hpp"
20 20 
21using namespace NpuArch;21using namespace NpuArch;
22using namespace RfaKenelCommon;22using namespace RfaKenelCommon;
Rattention/rain_fusion_attention/op_kernel/kernel_common.hpp→attention/rain_fusion_attention/op_kernel/rain_fusion_attention_kernel_common.hpp+0-0
文件重命名但无更改。
Rattention/sparse_flash_attention_grad/basic_modules/common_header.h→attention/sparse_flash_attention_grad/basic_modules/sparse_flash_attention_grad_common_header.h+0-0
文件重命名但无更改。
Rattention/sparse_flash_attention_grad/basic_modules/cube_op.h→attention/sparse_flash_attention_grad/basic_modules/sparse_flash_attention_grad_cube_op.h+2-2
@@ -13,8 +13,8 @@
13 */13 */
14#pragma once14#pragma once
15#include "kernel_operator.h"15#include "kernel_operator.h"
16-#include "common_header.h"16+#include "sparse_flash_attention_grad_common_header.h"
17-#include "matmul.h"17+#include "sparse_flash_attention_grad_matmul.h"
18using namespace AscendC;18using namespace AscendC;
19 19 
20namespace SFAG_BASIC {20namespace SFAG_BASIC {
Rattention/sparse_flash_attention_grad/basic_modules/matmul.h→attention/sparse_flash_attention_grad/basic_modules/sparse_flash_attention_grad_matmul.h+1-1
@@ -13,7 +13,7 @@
13 */13 */
14#pragma once14#pragma once
15#include "kernel_operator.h"15#include "kernel_operator.h"
16-#include "common_header.h"16+#include "sparse_flash_attention_grad_common_header.h"
17 17 
18using namespace AscendC;18using namespace AscendC;
19 19 
Rattention/sparse_flash_attention_grad/basic_modules/vec_op.h→attention/sparse_flash_attention_grad/basic_modules/sparse_flash_attention_grad_vec_op.h+1-1
@@ -15,7 +15,7 @@
15 15 
16#pragma once16#pragma once
17#include "kernel_operator.h"17#include "kernel_operator.h"
18-#include "common_header.h"18+#include "sparse_flash_attention_grad_common_header.h"
19 19 
20namespace SFAG_BASIC {20namespace SFAG_BASIC {
21struct StaticParams {21struct StaticParams {
Mattention/sparse_flash_attention_grad/op_kernel/arch22/sparse_flash_attention_grad_bs1_basic.h+3-3
@@ -16,9 +16,9 @@
16#pragma once16#pragma once
17#include "lib/matmul_intf.h"17#include "lib/matmul_intf.h"
18#include "kernel_operator.h"18#include "kernel_operator.h"
19-#include "../../basic_modules/cube_op.h"19+#include "../../basic_modules/sparse_flash_attention_grad_cube_op.h"
20-#include "../../basic_modules/vec_op.h"20+#include "../../basic_modules/sparse_flash_attention_grad_vec_op.h"
21-#include "../../basic_modules/common_header.h"21+#include "../../basic_modules/sparse_flash_attention_grad_common_header.h"
22#include "sparse_flash_attention_grad_post.h"22#include "sparse_flash_attention_grad_post.h"
23 23 
24namespace SFAG_BASIC {24namespace SFAG_BASIC {
Rattention/sparse_flash_mla_grad/op_kernel/arch22/basic_modules/common_header.h→attention/sparse_flash_mla_grad/op_kernel/arch22/basic_modules/sparse_flash_mla_grad_common_header.h+0-0
文件重命名但无更改。
Rattention/sparse_flash_mla_grad/op_kernel/arch22/basic_modules/cube_op.h→attention/sparse_flash_mla_grad/op_kernel/arch22/basic_modules/sparse_flash_mla_grad_cube_op.h+4-4
@@ -13,8 +13,8 @@
13 */13 */
14#pragma once14#pragma once
15#include "kernel_operator.h"15#include "kernel_operator.h"
16-#include "common_header.h"16+#include "sparse_flash_mla_grad_common_header.h"
17-#include "matmul.h"17+#include "sparse_flash_mla_grad_matmul.h"
18using namespace AscendC;18using namespace AscendC;
19 19 
20namespace SMLAG_BASIC {20namespace SMLAG_BASIC {
@@ -61,7 +61,7 @@ private:
61 const int32_t mmPingPongIdx,61 const int32_t mmPingPongIdx,
62 const int64_t lastBlockSize,62 const int64_t lastBlockSize,
63 const bool isLastBasicBlock,63 const bool isLastBasicBlock,
64- const RunInfo &runInfo);64+ const RunInfo &runInfo);
65 65 
66 __aicore__ inline __attribute__((always_inline)) void cube4Process(const int64_t dsGmOffset,66 __aicore__ inline __attribute__((always_inline)) void cube4Process(const int64_t dsGmOffset,
67 const int64_t queryGmOffset,67 const int64_t queryGmOffset,
@@ -208,7 +208,7 @@ private:
208 uint32_t cBlockIdx;208 uint32_t cBlockIdx;
209 uint32_t singleN;209 uint32_t singleN;
210 int64_t s1BasicSize;210 int64_t s1BasicSize;
211- int64_t dOriKvSize; 211+ int64_t dOriKvSize;
212};212};
213 213 
214template <typename SMLAGT>214template <typename SMLAGT>
Rattention/sparse_flash_mla_grad/op_kernel/arch22/basic_modules/matmul.h→attention/sparse_flash_mla_grad/op_kernel/arch22/basic_modules/sparse_flash_mla_grad_matmul.h+5-5
@@ -13,7 +13,7 @@
13 */13 */
14#pragma once14#pragma once
15#include "kernel_operator.h"15#include "kernel_operator.h"
16-#include "common_header.h"16+#include "sparse_flash_mla_grad_common_header.h"
17 17 
18using namespace AscendC;18using namespace AscendC;
19 19 
@@ -96,7 +96,7 @@ __aicore__ inline void FreeEventID()
96 96 
97template <typename T>97template <typename T>
98__aicore__ inline void CopyGmToL1(const LocalTensor<T> &l1Tensor, const GlobalTensor<T> &gmTensor, uint32_t srcN,98__aicore__ inline void CopyGmToL1(const LocalTensor<T> &l1Tensor, const GlobalTensor<T> &gmTensor, uint32_t srcN,
99- uint32_t srcD, uint32_t srcDstride)99+ uint32_t srcD, uint32_t srcDstride)
100{100{
101 Nd2NzParams nd2nzPara;101 Nd2NzParams nd2nzPara;
102 nd2nzPara.ndNum = 1;102 nd2nzPara.ndNum = 1;
@@ -180,7 +180,7 @@ __aicore__ inline void MmadInnerWithSync(LocalTensor<float> &l0cTensor,
180 uint32_t l0a_event = L0A_EVENTS[l0aPingPongFlag & 1];180 uint32_t l0a_event = L0A_EVENTS[l0aPingPongFlag & 1];
181 uint32_t l0b_event = L0B_EVENTS[l0bPingPongFlag & 1];181 uint32_t l0b_event = L0B_EVENTS[l0bPingPongFlag & 1];
182 uint32_t l0c_event = L0C_EVENTS[l0cPingPongFlag & 1];182 uint32_t l0c_event = L0C_EVENTS[l0cPingPongFlag & 1];
183- 183+ 
184 SetFlag<HardEvent::MTE2_MTE1>(l0b_event);184 SetFlag<HardEvent::MTE2_MTE1>(l0b_event);
185 WaitFlag<HardEvent::MTE2_MTE1>(l0b_event);185 WaitFlag<HardEvent::MTE2_MTE1>(l0b_event);
186 186 
@@ -207,7 +207,7 @@ __aicore__ inline void MmadInnerWithSync(LocalTensor<float> &l0cTensor,
207 207 
208 SetFlag<HardEvent::M_MTE1>(l0a_event);208 SetFlag<HardEvent::M_MTE1>(l0a_event);
209 SetFlag<HardEvent::M_MTE1>(l0b_event);209 SetFlag<HardEvent::M_MTE1>(l0b_event);
210- 210+ 
211 if (mmParam.isFixOut) {211 if (mmParam.isFixOut) {
212 SetFlag<HardEvent::M_FIX>(l0c_event);212 SetFlag<HardEvent::M_FIX>(l0c_event);
213 WaitFlag<HardEvent::M_FIX>(l0c_event);213 WaitFlag<HardEvent::M_FIX>(l0c_event);
@@ -232,7 +232,7 @@ __aicore__ inline void MmadInnerWithSync(LocalTensor<float> &l0cTensor,
232 if constexpr(!isScatterFixOut) {232 if constexpr(!isScatterFixOut) {
233 SetFlag<HardEvent::FIX_M>(l0c_event);233 SetFlag<HardEvent::FIX_M>(l0c_event);
234 }234 }
235- 235+ 
236 l0aPingPongFlag = 1 - l0aPingPongFlag;236 l0aPingPongFlag = 1 - l0aPingPongFlag;
237 l0bPingPongFlag = 1 - l0bPingPongFlag;237 l0bPingPongFlag = 1 - l0bPingPongFlag;
238}238}
Rattention/sparse_flash_mla_grad/op_kernel/arch22/basic_modules/vec_op.h→attention/sparse_flash_mla_grad/op_kernel/arch22/basic_modules/sparse_flash_mla_grad_vec_op.h+1-1
@@ -15,7 +15,7 @@
15 15 
16#pragma once16#pragma once
17#include "kernel_operator.h"17#include "kernel_operator.h"
18-#include "common_header.h"18+#include "sparse_flash_mla_grad_common_header.h"
19 19 
20namespace SMLAG_BASIC {20namespace SMLAG_BASIC {
21struct StaticParams {21struct StaticParams {
Mattention/sparse_flash_mla_grad/op_kernel/arch22/post.h+4-4
@@ -15,7 +15,7 @@
15 15 
16#pragma once16#pragma once
17#include "kernel_operator.h"17#include "kernel_operator.h"
18-#include "basic_modules/common_header.h"18+#include "basic_modules/sparse_flash_mla_grad_common_header.h"
19using namespace AscendC;19using namespace AscendC;
20 20 
21template <typename OUT_TYPE, typename TILING_TYPE, const bool CAST_DV, const uint32_t LAYOUT,21template <typename OUT_TYPE, typename TILING_TYPE, const bool CAST_DV, const uint32_t LAYOUT,
@@ -23,7 +23,7 @@ template <typename OUT_TYPE, typename TILING_TYPE, const bool CAST_DV, const uin
23class SparseFlashMlaGradPost {23class SparseFlashMlaGradPost {
24public:24public:
25 __aicore__ inline SparseFlashMlaGradPost(){};25 __aicore__ inline SparseFlashMlaGradPost(){};
26- __aicore__ inline void Init(__gm__ uint8_t *dq, __gm__ uint8_t *d_ori_kv, __gm__ uint8_t *d_cmp_kv, 26+ __aicore__ inline void Init(__gm__ uint8_t *dq, __gm__ uint8_t *d_ori_kv, __gm__ uint8_t *d_cmp_kv,
27 __gm__ uint8_t *workspace, const TILING_TYPE *__restrict ordTilingData, TPipe *pipe_in);27 __gm__ uint8_t *workspace, const TILING_TYPE *__restrict ordTilingData, TPipe *pipe_in);
28 __aicore__ inline void Process();28 __aicore__ inline void Process();
29 29 
@@ -121,9 +121,9 @@ __aicore__ inline void SparseFlashMlaGradPost<OUT_TYPE, TILING_TYPE, CAST_DV, LA
121 121 
122 dimDqk = tilingData->opInfo.D;122 dimDqk = tilingData->opInfo.D;
123 dOriKvSize = LAYOUT == 3 ? tilingData->opInfo.S2 * tilingData->opInfo.N2 * dimDqk : 123 dOriKvSize = LAYOUT == 3 ? tilingData->opInfo.S2 * tilingData->opInfo.N2 * dimDqk :
124- tilingData->opInfo.B * tilingData->opInfo.S2 * tilingData->opInfo.N2 * dimDqk;124+ tilingData->opInfo.B * tilingData->opInfo.S2 * tilingData->opInfo.N2 * dimDqk;
125 dCmpKvSize = LAYOUT == 3 ? tilingData->opInfo.S3 * tilingData->opInfo.N2 * dimDqk : 125 dCmpKvSize = LAYOUT == 3 ? tilingData->opInfo.S3 * tilingData->opInfo.N2 * dimDqk :
126- tilingData->opInfo.B * tilingData->opInfo.S3 * tilingData->opInfo.N2 * dimDqk;126+ tilingData->opInfo.B * tilingData->opInfo.S3 * tilingData->opInfo.N2 * dimDqk;
127 /*127 /*
128 * 初始化workspace128 * 初始化workspace
129 */129 */
Mattention/sparse_flash_mla_grad/op_kernel/arch22/sfag_basic.h+3-3
@@ -16,9 +16,9 @@
16#pragma once16#pragma once
17#include "lib/matmul_intf.h"17#include "lib/matmul_intf.h"
18#include "kernel_operator.h"18#include "kernel_operator.h"
19-#include "basic_modules/cube_op.h"19+#include "basic_modules/sparse_flash_mla_grad_cube_op.h"
20-#include "basic_modules/vec_op.h"20+#include "basic_modules/sparse_flash_mla_grad_vec_op.h"
21-#include "basic_modules/common_header.h"21+#include "basic_modules/sparse_flash_mla_grad_common_header.h"
22#include "post.h"22#include "post.h"
23 23 
24namespace SMLAG_BASIC {24namespace SMLAG_BASIC {
Mattention/sparse_flash_mla_grad/op_kernel/arch22/smlag_basic.h+3-3
@@ -16,9 +16,9 @@
16#pragma once16#pragma once
17#include "lib/matmul_intf.h"17#include "lib/matmul_intf.h"
18#include "kernel_operator.h"18#include "kernel_operator.h"
19-#include "basic_modules/cube_op.h"19+#include "basic_modules/sparse_flash_mla_grad_cube_op.h"
20-#include "basic_modules/vec_op.h"20+#include "basic_modules/sparse_flash_mla_grad_vec_op.h"
21-#include "basic_modules/common_header.h"21+#include "basic_modules/sparse_flash_mla_grad_common_header.h"
22#include "post.h"22#include "post.h"
23 23 
24namespace SMLAG_BASIC {24namespace SMLAG_BASIC {