已合并
matmul: replace include guards with #pragma once in common headers #3501
chenqi317创建于 4月2日
matmul: replace include guards with #pragma once in common headers #3501
已合并
共 296 个文件变更+1294-1679
Mmatmul/batch_mat_mul_v3/op_host/op_tiling/arch35/batch_matmul_v3_asw_al1_full_load_basic_tiling.h+2-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef __OP_HOST_BATCH_MATMUL_V3_ASW_AL1_FULL_LOAD_BASIC_TILING_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -40,4 +39,4 @@ protected: | |||
| 40 | }; | 39 | }; |
| 41 | } | 40 | } |
| 42 | } | 41 | } |
| 43 | -#endif // __OP_HOST_BATCH_MATMUL_V3_ASW_AL1_FULL_LOAD_BASIC_TILING_H__ | 42 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef __OP_HOST_BATCH_MATMUL_V3_ASW_BASIC_TILING_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -45,4 +44,4 @@ protected: | |||
| 45 | }; | 44 | }; |
| 46 | } | 45 | } |
| 47 | } | 46 | } |
| 48 | -#endif // __OP_HOST_BATCH_MATMUL_V3_ASW_BAISC_TILING_H__ | 47 | + |
Mmatmul/batch_mat_mul_v3/op_host/op_tiling/arch35/batch_matmul_v3_asw_bl1_full_load_basic_tiling.h+2-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef __OP_HOST_BATCH_MATMUL_V3_ASW_BL1_FULL_LOAD_BASIC_TILING_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -40,4 +39,4 @@ protected: | |||
| 40 | }; | 39 | }; |
| 41 | } | 40 | } |
| 42 | } | 41 | } |
| 43 | -#endif // __OP_HOST_BATCH_MATMUL_V3_ASW_BL1_FULL_LOAD_BASIC_TILING_H__ | 42 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef __OP_HOST_BATCH_MATMUL_V3_ASW_TILING_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -44,4 +43,4 @@ protected: | |||
| 44 | }; | 43 | }; |
| 45 | } | 44 | } |
| 46 | } | 45 | } |
| 47 | -#endif // __OP_HOST_BATCH_MATMUL_V3_ASW_TILING_H__ | 46 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file batch_matmul_v3_common_advanced.h | 12 | * \file batch_matmul_v3_common_advanced.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_BATCH_MATMUL_V3_COMMON_ADVANCED_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -31,4 +30,4 @@ constexpr uint64_t ND_NZ_DIM_DIFF = 2; | |||
| 31 | constexpr uint64_t BATCH_DIM_MAX = 6; | 30 | constexpr uint64_t BATCH_DIM_MAX = 6; |
| 32 | } | 31 | } |
| 33 | } | 32 | } |
| 34 | -#endif // __OP_HOST_BATCH_MATMUL_V3_COMMON_ADVANCED_H__ | 33 | + |
| @@ -14,8 +14,7 @@ | |||
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | 16 | ||
| 17 | -#ifndef __OP_HOST_BATCH_MATMUL_V3_ITERBATCH_TILING_H__ | 17 | +#pragma once |
| 18 | - | ||
| 19 | 18 | ||
| 20 | 19 | ||
| 21 | 20 | ||
| @@ -62,4 +61,4 @@ private: | |||
| 62 | }; | 61 | }; |
| 63 | } | 62 | } |
| 64 | } | 63 | } |
| 65 | -#endif // __OP_HOST_BATCH_MATMUL_V3_ITERBATCH_TILING_H__ | 64 | + |
| @@ -14,8 +14,7 @@ | |||
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | 16 | ||
| 17 | -#ifndef __OP_HOST_BATCH_MATMUL_V3_ITERBATCH_TILING_H__ | 17 | +#pragma once |
| 18 | - | ||
| 19 | 18 | ||
| 20 | 19 | ||
| 21 | 20 | ||
| @@ -49,4 +48,4 @@ private: | |||
| 49 | }; | 48 | }; |
| 50 | } | 49 | } |
| 51 | } | 50 | } |
| 52 | -#endif // __OP_HOST_BATCH_MATMUL_V3_ITERBATCH_TILING_H__ | 51 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \file batch_matmul_v3_k_equal_zero_tiling.h | 13 | * \file batch_matmul_v3_k_equal_zero_tiling.h |
| 14 | * \brief | 14 | * \brief |
| 15 | */ | 15 | */ |
| 16 | -#ifndef __OP_HOST_BATCH_MATMUL_V3_K_EQUAL_ZERO_TILING_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -40,4 +39,4 @@ protected: | |||
| 40 | }; | 39 | }; |
| 41 | } // namespace batch_matmul_v3_advanced | 40 | } // namespace batch_matmul_v3_advanced |
| 42 | } // namespace optiling | 41 | } // namespace optiling |
| 43 | -#endif | 42 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef __OP_HOST_BATCH_MATMUL_V3_MATMUL2MUL_TILING_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -38,4 +37,4 @@ protected: | |||
| 38 | }; | 37 | }; |
| 39 | } // namespace batch_matmul_v3_advanced | 38 | } // namespace batch_matmul_v3_advanced |
| 40 | } // namespace optiling | 39 | } // namespace optiling |
| 41 | -#endif | 40 | + |
| @@ -14,8 +14,7 @@ | |||
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | 16 | ||
| 17 | -#ifndef __OP_HOST_BATCH_MATMUL_V3_MERGEBATCH_TILING_H__ | 17 | +#pragma once |
| 18 | - | ||
| 19 | 18 | ||
| 20 | 19 | ||
| 21 | 20 | ||
| @@ -51,4 +50,4 @@ private: | |||
| 51 | }; | 50 | }; |
| 52 | } | 51 | } |
| 53 | } | 52 | } |
| 54 | -#endif // __OP_HOST_BATCH_MATMUL_V3_MERGEBATCH_TILING_H__ | 53 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file batch_matmul_v3_tiling_advanced.h | 12 | * \file batch_matmul_v3_tiling_advanced.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_BATCH_MATMUL_V3_TILING_ADVANCED_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -37,4 +36,4 @@ protected: | |||
| 37 | }; | 36 | }; |
| 38 | } | 37 | } |
| 39 | } | 38 | } |
| 40 | -#endif // __OP_HOST_BATCH_MATMUL_V3_TILING_ADVANCED_H__ | 39 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef __OP_HOST_BATCH_MATMUL_V3_TILING_KEY_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -29,4 +28,3 @@ public: | |||
| 29 | } // namespace matmul_v3_advanced | 28 | } // namespace matmul_v3_advanced |
| 30 | } // namespace optiling | 29 | } // namespace optiling |
| 31 | 30 | ||
| 32 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \file batch_matmul_v3_tiling_strategy.h | 13 | * \file batch_matmul_v3_tiling_strategy.h |
| 14 | * \brief | 14 | * \brief |
| 15 | */ | 15 | */ |
| 16 | -#ifndef __OP_HOST_BATCH_MATMUL_V3_TILING_STRATEGY_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -58,4 +57,3 @@ inline std::vector<int32_t> GetBatchMatMulV3Priorities(NpuArch NpuArch) | |||
| 58 | } // namespace batch_matmul_v3_advanced | 57 | } // namespace batch_matmul_v3_advanced |
| 59 | } // namespace optiling | 58 | } // namespace optiling |
| 60 | 59 | ||
| 61 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file batch_mat_mul_v3_asw_al1_full_load_kernel_advanced.h | 12 | * \file batch_mat_mul_v3_asw_al1_full_load_kernel_advanced.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef BATCH_MAT_MUL_V3_ASW_AL1_FULL_LOAD_KERNEL_ADVANCED_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -116,4 +115,3 @@ __aicore__ inline void BatchMatMulAswAL1FullLoadKernel<A_TYPE, B_TYPE, C_TYPE, B | |||
| 116 | } | 115 | } |
| 117 | } | 116 | } |
| 118 | 117 | ||
| 119 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file batch_mat_mul_v3_asw_bl1_full_load_kernel_advanced.h | 12 | * \file batch_mat_mul_v3_asw_bl1_full_load_kernel_advanced.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef BATCH_MAT_MUL_V3_ASW_BL1_FULL_LOAD_KERNEL_ADVANCED_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -117,4 +116,3 @@ __aicore__ inline void BatchMatMulAswBL1FullLoadKernel<A_TYPE, B_TYPE, C_TYPE, B | |||
| 117 | } | 116 | } |
| 118 | } | 117 | } |
| 119 | 118 | ||
| 120 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file batch_mat_mul_v3_asw_block_advanced.h | 12 | * \file batch_mat_mul_v3_asw_block_advanced.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef BATCH_MATMUL_V3_ASW_BLOCK_ADVANCED_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -278,4 +277,3 @@ __aicore__ inline void BatchMatMulAswBlock::CalcSplitKGMOffset(uint64_t splitKIn | |||
| 278 | 277 | ||
| 279 | } // namespace BatchMatMulV3Advanced | 278 | } // namespace BatchMatMulV3Advanced |
| 280 | 279 | ||
| 281 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file batch_mat_mul_v3_asw_kernel_advanced.h | 12 | * \file batch_mat_mul_v3_asw_kernel_advanced.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef BATCH_MAT_MUL_V3_ASW_KERNEL_ADVANCED_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -147,4 +146,3 @@ __aicore__ inline void BatchMatMulAswKernel<A_TYPE, B_TYPE, C_TYPE, BIAS_TYPE, B | |||
| 147 | 146 | ||
| 148 | } // namespace BatchMatMulV3Advanced | 147 | } // namespace BatchMatMulV3Advanced |
| 149 | 148 | ||
| 150 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef BATCH_MAT_MUL_V3_ITERBATCH_BASICAPI_BLOCK_SCHEDULER_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -163,4 +162,4 @@ struct BlockSchedulerSelector< | |||
| 163 | } // namespace Block | 162 | } // namespace Block |
| 164 | } // namespace Gemm | 163 | } // namespace Gemm |
| 165 | } // namespace Cmct | 164 | } // namespace Cmct |
| 166 | -#endif | 165 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file batch_mat_mul_v3_iterbatch_basicapi_cmct.h | 12 | * \file batch_mat_mul_v3_iterbatch_basicapi_cmct.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef BATCH_MAT_MUL_V3_ITERBATCH_BASICAPI_CMCT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -94,4 +93,4 @@ __aicore__ inline void BatchMatMulActIterBatchKernel( | |||
| 94 | MatmulKernel mm; | 93 | MatmulKernel mm; |
| 95 | mm(params); | 94 | mm(params); |
| 96 | } | 95 | } |
| 97 | -#endif | 96 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file batch_mat_mul_v3_iterbatch_block_advanced.h | 12 | * \file batch_mat_mul_v3_iterbatch_block_advanced.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef BATCH_MATMUL_V3_ITERBATCH_BLOCK_ADVANCED_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -132,4 +131,3 @@ __aicore__ inline void BatchMatMulMultiBatchBaseBlock::CalcGMOffset() | |||
| 132 | } | 131 | } |
| 133 | } | 132 | } |
| 134 | 133 | ||
| 135 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file batch_mat_mul_v3_iterbatch_kernel_advanced.h | 12 | * \file batch_mat_mul_v3_iterbatch_kernel_advanced.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef BATCH_MAT_MUL_V3_ITERBATCH_KERNEL_ADVANCED_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -137,4 +136,4 @@ __aicore__ inline void BatchMatMulMultiBatchKernel<A_TYPE, B_TYPE, C_TYPE, BIAS_ | |||
| 137 | mm_.SetHF32(false, 0); | 136 | mm_.SetHF32(false, 0); |
| 138 | } | 137 | } |
| 139 | } | 138 | } |
| 140 | -#endif // BATCH_MAT_MUL_V3_ITERBATCH_KERNEL_ADVANCED_H | 139 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef BATCH_MAT_MUL_V3_MATMUL2MUL_BLOCK_SCHEDULER_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -75,4 +74,3 @@ struct BlockSchedulerSelector< | |||
| 75 | } // namespace Gemm | 74 | } // namespace Gemm |
| 76 | } // namespace Cmct | 75 | } // namespace Cmct |
| 77 | 76 | ||
| 78 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef BATCH_MAT_MUL_V3_MATMUL2MUL_CMCT_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -73,4 +72,4 @@ __aicore__ inline void BatchMatMulToMulActKernel( | |||
| 73 | BatchMatmulKernel bmm; | 72 | BatchMatmulKernel bmm; |
| 74 | bmm(params); | 73 | bmm(params); |
| 75 | } | 74 | } |
| 76 | -#endif | 75 | + |
Mmatmul/batch_mat_mul_v3/op_kernel/arch35/batch_mat_mul_v3_mergebatch_basicapi_block_scheduler.h+2-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef BATCH_MAT_MUL_V3_MERGEBATCH_BASICAPI_BLOCK_SCHEDULER_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -191,4 +190,4 @@ struct BlockSchedulerSelector< | |||
| 191 | } // namespace Block | 190 | } // namespace Block |
| 192 | } // namespace Gemm | 191 | } // namespace Gemm |
| 193 | } // namespace Act | 192 | } // namespace Act |
| 194 | -#endif | 193 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file batch_mat_mul_v3_mergebatch_basicapi_cmct.h | 12 | * \file batch_mat_mul_v3_mergebatch_basicapi_cmct.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef BATCH_MAT_MUL_V3_MERGEBATCH_BASICAPI_CMCT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | using namespace Cmct; | 18 | using namespace Cmct; |
| @@ -66,4 +65,4 @@ __aicore__ inline void BatchMatMulActMergeBatchKernel(GM_ADDR aGM, GM_ADDR bGM, | |||
| 66 | MatmulKernel mm; | 65 | MatmulKernel mm; |
| 67 | mm(params); | 66 | mm(params); |
| 68 | } | 67 | } |
| 69 | -#endif | 68 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_BLOCK_BLOCK_BATCH_MATMUL_TO_MUL_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -320,4 +319,4 @@ private: | |||
| 320 | } // namespace Block | 319 | } // namespace Block |
| 321 | } // namespace Gemm | 320 | } // namespace Gemm |
| 322 | } // namespace Cmct | 321 | } // namespace Cmct |
| 323 | -#endif | 322 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file block_mmad.h | 12 | * \file block_mmad.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MATMUL_BLOCK_BLOCK_MMAD_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -105,4 +104,4 @@ public: | |||
| 105 | } // namespace Cmct | 104 | } // namespace Cmct |
| 106 | 105 | ||
| 107 | 106 | ||
| 108 | -#endif | 107 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef BLOCK_MMAD_A8W8_FIXPIPE_QUANT_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -689,4 +688,4 @@ private: | |||
| 689 | } | 688 | } |
| 690 | } | 689 | } |
| 691 | } | 690 | } |
| 692 | -#endif | 691 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef CMCT_BLOCK_BLOCK_MMAD_A_PREFETCH_B_PROLOGUE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -503,4 +502,4 @@ private: | |||
| 503 | AscendC::LocalTensor<BiasL1TensorTrait> biasL1_; | 502 | AscendC::LocalTensor<BiasL1TensorTrait> biasL1_; |
| 504 | }; | 503 | }; |
| 505 | } // namespace Cmct::Gemm::Block | 504 | } // namespace Cmct::Gemm::Block |
| 506 | -#endif | 505 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef CMCT_BLOCK_BLOCK_MMAD_B_PROLOGUE_MX_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -699,4 +698,4 @@ private: | |||
| 699 | static constexpr TEventID eventIdsFixToM_[2] = {0, 1}; | 698 | static constexpr TEventID eventIdsFixToM_[2] = {0, 1}; |
| 700 | }; | 699 | }; |
| 701 | } // namespace Cmct::Gemm::Block | 700 | } // namespace Cmct::Gemm::Block |
| 702 | -#endif | 701 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_BLOCK_BLOCK_MATMUL_BUILDER_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -169,4 +168,4 @@ public: | |||
| 169 | } // namespace Block | 168 | } // namespace Block |
| 170 | } // namespace Gemm | 169 | } // namespace Gemm |
| 171 | } // namespace Cmct | 170 | } // namespace Cmct |
| 172 | -#endif | 171 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_BLOCK_BLOCK_MMAD_ITERBATCH_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -540,4 +539,4 @@ private: | |||
| 540 | } // namespace Block | 539 | } // namespace Block |
| 541 | } // namespace Gemm | 540 | } // namespace Gemm |
| 542 | } // namespace Cmct | 541 | } // namespace Cmct |
| 543 | -#endif | 542 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_BLOCK_BLOCK_MMAD_MERGEBATCH_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -404,4 +403,4 @@ private: | |||
| 404 | } // namespace Block | 403 | } // namespace Block |
| 405 | } // namespace Gemm | 404 | } // namespace Gemm |
| 406 | } // namespace Cmct | 405 | } // namespace Cmct |
| 407 | -#endif | 406 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file block_mmad_multi_block_bias.h | 12 | * \file block_mmad_multi_block_bias.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MATMUL_BLOCK_BLOCK_MMAD_MULTI_BLOCK_BIAS_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -117,4 +116,4 @@ private: | |||
| 117 | } // namespace Block | 116 | } // namespace Block |
| 118 | } // namespace Gemm | 117 | } // namespace Gemm |
| 119 | } // namespace Cmct | 118 | } // namespace Cmct |
| 120 | -#endif | 119 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_BLOCK_BLOCK_MMAD_MULTI_BLOCK_WITH_LAYOUT_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -106,4 +105,4 @@ private: | |||
| 106 | } // namespace Block | 105 | } // namespace Block |
| 107 | } // namespace Gemm | 106 | } // namespace Gemm |
| 108 | } // namespace Cmct | 107 | } // namespace Cmct |
| 109 | -#endif | 108 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_BLOCK_MMAD_MX_QUANT_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -851,4 +850,4 @@ private: | |||
| 851 | } // namespace Block | 850 | } // namespace Block |
| 852 | } // namespace Gemm | 851 | } // namespace Gemm |
| 853 | } // namespace Cmct | 852 | } // namespace Cmct |
| 854 | -#endif | 853 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file block_mmad_pertile.h | 12 | * \file block_mmad_pertile.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MATMUL_BLOCK_BLOCK_MMAD_PERTILE_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -476,4 +475,4 @@ __aicore__ inline BlockMmadPertile<QBMM_BLOCK_MMAD_PERTILE_FUNC_LOCAL_PARAMS>::~ | |||
| 476 | } // namespace Block | 475 | } // namespace Block |
| 477 | } // namespace Gemm | 476 | } // namespace Gemm |
| 478 | } // namespace Cmct | 477 | } // namespace Cmct |
| 479 | -#endif | 478 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_BLOCK_BLOCK_MMAD_PERTILE_PARAM_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -259,4 +258,3 @@ MatMulCommonParam<aTrans, bTrans>::LoadData2dParamsB(AscendC::LoadData2DParamsV2 | |||
| 259 | } // namespace Gemm | 258 | } // namespace Gemm |
| 260 | } // namespace Cmct | 259 | } // namespace Cmct |
| 261 | 260 | ||
| 262 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_BLOCK_BLOCK_MMAD_PINGPONG_WITHOUT_QUE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -1018,4 +1017,4 @@ private: | |||
| 1018 | } // namespace Block | 1017 | } // namespace Block |
| 1019 | } // namespace Gemm | 1018 | } // namespace Gemm |
| 1020 | } // namespace Cmct | 1019 | } // namespace Cmct |
| 1021 | -#endif | 1020 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef INCLUDE_MATMUL_BLOCK_BLOCK_MMAD_STREAMK_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -463,4 +462,4 @@ private: | |||
| 463 | } // namespace Block | 462 | } // namespace Block |
| 464 | } // namespace Gemm | 463 | } // namespace Gemm |
| 465 | } // namespace Cmct | 464 | } // namespace Cmct |
| 466 | -#endif | 465 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_BLOCK_BLOCK_TO_MUL_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -273,4 +272,4 @@ private: | |||
| 273 | } // namespace Block | 272 | } // namespace Block |
| 274 | } // namespace Gemm | 273 | } // namespace Gemm |
| 275 | } // namespace Cmct | 274 | } // namespace Cmct |
| 276 | -#endif | 275 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file block_mmad_utils.h | 12 | * \file block_mmad_utils.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MATMUL_BLOCK_BLOCK_MMAD_UTILS_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -249,4 +248,4 @@ struct GetL0CAndBtType { | |||
| 249 | } // namespace Block | 248 | } // namespace Block |
| 250 | } // namespace Gemm | 249 | } // namespace Gemm |
| 251 | } // namespace Cmct | 250 | } // namespace Cmct |
| 252 | -#endif | 251 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file block_mmad_with_layout.h | 12 | * \file block_mmad_with_layout.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MATMUL_BLOCK_BLOCK_MMAD_WITH_LAYOUT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -157,4 +156,4 @@ private: | |||
| 157 | } // namespace Block | 156 | } // namespace Block |
| 158 | } // namespace Gemm | 157 | } // namespace Gemm |
| 159 | } // namespace Cmct | 158 | } // namespace Cmct |
| 160 | -#endif | 159 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file block_mmad_with_params.h | 12 | * \file block_mmad_with_params.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MATMUL_BLOCK_BLOCK_MMAD_WITH_PARAMS_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -112,4 +111,4 @@ public: | |||
| 112 | } // namespace Block | 111 | } // namespace Block |
| 113 | } // namespace Gemm | 112 | } // namespace Gemm |
| 114 | } // namespace Cmct | 113 | } // namespace Cmct |
| 115 | -#endif | 114 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \brief | 12 | * \brief |
| 13 | */ | 13 | */ |
| 14 | 14 | ||
| 15 | -#ifndef MATMUL_BLOCK_BLOCK_SCHEDULER_ITERATEK_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -126,4 +125,4 @@ struct BlockSchedulerSelector<ProblemShape_, L1TileShape_, L0TileShape_, Cmct::G | |||
| 126 | } // namespace Block | 125 | } // namespace Block |
| 127 | } // namespace Gemm | 126 | } // namespace Gemm |
| 128 | } // namespace Cmct | 127 | } // namespace Cmct |
| 129 | -#endif | 128 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_BLOCK_BLOCK_SCHEDULER_POLICY_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | namespace Cmct { | 18 | namespace Cmct { |
| 20 | namespace Gemm { | 19 | namespace Gemm { |
| @@ -37,4 +36,4 @@ struct QuantBatchMatmulV3Scheduler { | |||
| 37 | struct BuiltInMulScheduler {}; | 36 | struct BuiltInMulScheduler {}; |
| 38 | } // namespace Gemm | 37 | } // namespace Gemm |
| 39 | } // namespace Cmct | 38 | } // namespace Cmct |
| 40 | -#endif | 39 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef CMCT_QBMM_BLOCK_SCHEDULER_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -322,4 +321,4 @@ struct BlockSchedulerSelector<ProblemShape_, L1TileShape_, L0TileShape_, Cmct::G | |||
| 322 | } // namespace Block | 321 | } // namespace Block |
| 323 | } // namespace Gemm | 322 | } // namespace Gemm |
| 324 | } // namespace Cmct | 323 | } // namespace Cmct |
| 325 | -#endif | 324 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef CMCT_INCLUDE_MATMUL_BLOCK_BLOCK_SCHEDULER_SWIZZLE_IN_MN_CORE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -129,4 +128,4 @@ private: | |||
| 129 | } // namespace Block | 128 | } // namespace Block |
| 130 | } // namespace Gemm | 129 | } // namespace Gemm |
| 131 | } // namespace Cmct | 130 | } // namespace Cmct |
| 132 | -#endif | 131 | + |
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef BLOCK_SCHEDULER_TAIL_RESPLIT_EXPANDED_H | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | 13 | ||
| @@ -108,4 +107,4 @@ public: | |||
| 108 | uint64_t n2Tile; | 107 | uint64_t n2Tile; |
| 109 | }; | 108 | }; |
| 110 | } // namespace Cmct::Gemm::Block | 109 | } // namespace Cmct::Gemm::Block |
| 111 | -#endif | 110 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_BLOCK_BLOCK_SCHEDULER_UTILS_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -137,4 +136,4 @@ __host_aicore__ static Status DoCheckArgs(const ProblemShape_ &shape, int64_t l1 | |||
| 137 | } // namespace Block | 136 | } // namespace Block |
| 138 | } // namespace Gemm | 137 | } // namespace Gemm |
| 139 | } // namespace Cmct | 138 | } // namespace Cmct |
| 140 | -#endif | 139 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_BLOCK_MATMUL_IMPL_TRAITS_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -213,4 +212,4 @@ using MatmulImplTraitsT = typename MatmulImplTraits< | |||
| 213 | } // namespace Block | 212 | } // namespace Block |
| 214 | } // namespace Gemm | 213 | } // namespace Gemm |
| 215 | } // namespace Cmct | 214 | } // namespace Cmct |
| 216 | -#endif | 215 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef EPILOGUE_BLOCK_EPILOGUE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -174,4 +173,4 @@ public: | |||
| 174 | } // namespace Gemm | 173 | } // namespace Gemm |
| 175 | } // namespace Cmct | 174 | } // namespace Cmct |
| 176 | 175 | ||
| 177 | -#endif | 176 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef EPILOGUE_BLOCK_EPILOGUE_CV_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -160,4 +159,4 @@ public: | |||
| 160 | } // namespace Gemm | 159 | } // namespace Gemm |
| 161 | } // namespace Cmct | 160 | } // namespace Cmct |
| 162 | 161 | ||
| 163 | -#endif | 162 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef EPILOGUE_EPILOGUE_BLOCK_EPILOGUE_ELEMENTWISE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -164,4 +163,4 @@ public: | |||
| 164 | } // namespace Gemm | 163 | } // namespace Gemm |
| 165 | } // namespace Cmct | 164 | } // namespace Cmct |
| 166 | 165 | ||
| 167 | -#endif | 166 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef EPILOGUE_BLOCK_EPILOGUE_EMPTY_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -79,4 +78,4 @@ public: | |||
| 79 | } // namespace Block | 78 | } // namespace Block |
| 80 | } // namespace Gemm | 79 | } // namespace Gemm |
| 81 | } // namespace Cmct | 80 | } // namespace Cmct |
| 82 | -#endif | 81 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef CMCT_INCLUDE_EPILOGUE_BLOCK_EPILOGUE_FIXPIPE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -132,4 +131,4 @@ public: | |||
| 132 | } // namespace Gemm | 131 | } // namespace Gemm |
| 133 | } // namespace Cmct | 132 | } // namespace Cmct |
| 134 | 133 | ||
| 135 | -#endif | 134 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef CMCT_INCLUDE_EPILOGUE_BLOCK_EPILOGUE_ITERBATCH_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -238,4 +237,4 @@ private: | |||
| 238 | } // namespace Block | 237 | } // namespace Block |
| 239 | } // namespace Gemm | 238 | } // namespace Gemm |
| 240 | } // namespace Cmct | 239 | } // namespace Cmct |
| 241 | -#endif | 240 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef EPILOGUE_BLOCK_EPILOGUE_PERTILE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -890,4 +889,4 @@ __aicore__ inline void BlockEpiloguePertile<QBMM_BLOCK_EPILOGUE_PERTILE_FUNC_LOC | |||
| 890 | } // namespace Block | 889 | } // namespace Block |
| 891 | } // namespace Gemm | 890 | } // namespace Gemm |
| 892 | } // namespace Cmct | 891 | } // namespace Cmct |
| 893 | -#endif // EPILOGUE_BLOCK_EPILOGUE_PERTILE_H | 892 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef CMCT_INCLUDE_EPILOGUE_BLOCK_EPILOGUE_STREAMK_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -271,4 +270,4 @@ private: | |||
| 271 | } // namespace Block | 270 | } // namespace Block |
| 272 | } // namespace Gemm | 271 | } // namespace Gemm |
| 273 | } // namespace Cmct | 272 | } // namespace Cmct |
| 274 | -#endif | 273 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef EPILOGUE_FUSION_DEFAULT_FUSION_OP_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -60,4 +59,4 @@ public: | |||
| 60 | } // namespace Block | 59 | } // namespace Block |
| 61 | } // namespace Gemm | 60 | } // namespace Gemm |
| 62 | } // namespace Cmct | 61 | } // namespace Cmct |
| 63 | -#endif // EPILOGUE_FUSION_DEFAULT_FUSION_OP_H | 62 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef EPILOGUE_FUSION_EPILOGUE_FUSION_ADD_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -129,4 +128,4 @@ public: | |||
| 129 | } // namespace Block | 128 | } // namespace Block |
| 130 | } // namespace Gemm | 129 | } // namespace Gemm |
| 131 | } // namespace Cmct | 130 | } // namespace Cmct |
| 132 | -#endif // EPILOGUE_FUSION_EPILOGUE_FUSION_ADD_H | 131 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef EPILOGUE_FUSION_FUSION_GELU_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -116,4 +115,4 @@ public: | |||
| 116 | } // namespace Block | 115 | } // namespace Block |
| 117 | } // namespace Gemm | 116 | } // namespace Gemm |
| 118 | } // namespace Cmct | 117 | } // namespace Cmct |
| 119 | -#endif // CMCT_INCLUDE_EPILOGUE_FUSION_GELU_OP_H | 118 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef EPILOGUE_FUSION_FUSION_MUL_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -94,4 +93,4 @@ public: | |||
| 94 | } // namespace Block | 93 | } // namespace Block |
| 95 | } // namespace Gemm | 94 | } // namespace Gemm |
| 96 | } // namespace Cmct | 95 | } // namespace Cmct |
| 97 | -#endif // CMCT_INCLUDE_EPILOGUE_FUSION_MUL_OP_H | 96 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_KERNEL_KERNEL_BATCH_MATMUL_TO_MUL_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -246,4 +245,4 @@ public: | |||
| 246 | } // namespace Kernel | 245 | } // namespace Kernel |
| 247 | } // namespace Gemm | 246 | } // namespace Gemm |
| 248 | } // namespace Cmct | 247 | } // namespace Cmct |
| 249 | -#endif | 248 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_KERNEL_KERNEL_MATMUL_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -245,4 +244,4 @@ public: | |||
| 245 | } // namespace Kernel | 244 | } // namespace Kernel |
| 246 | } // namespace Gemm | 245 | } // namespace Gemm |
| 247 | } // namespace Cmct | 246 | } // namespace Cmct |
| 248 | -#endif | 247 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file kernel_matmul_a_prefetch_b_antiquant.h | 12 | * \file kernel_matmul_a_prefetch_b_antiquant.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef KERNEL_MATMUL_A_PREFETCH_B_ANTIQUANT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -215,4 +214,3 @@ private: | |||
| 215 | }; | 214 | }; |
| 216 | } // namespace Cmct::Gemm::Kernel | 215 | } // namespace Cmct::Gemm::Kernel |
| 217 | 216 | ||
| 218 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_KERNEL_KERNEL_MATMUL_ITERBATCH_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -320,4 +319,4 @@ public: | |||
| 320 | } // namespace Kernel | 319 | } // namespace Kernel |
| 321 | } // namespace Gemm | 320 | } // namespace Gemm |
| 322 | } // namespace Cmct | 321 | } // namespace Cmct |
| 323 | -#endif | 322 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_KERNEL_KERNEL_MATMUL_MERGE_BATCH_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -230,4 +229,4 @@ public: | |||
| 230 | } // namespace Kernel | 229 | } // namespace Kernel |
| 231 | } // namespace Gemm | 230 | } // namespace Gemm |
| 232 | } // namespace Cmct | 231 | } // namespace Cmct |
| 233 | -#endif | 232 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_KERNEL_KERNEL_MATMUL_MIX_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -312,4 +311,4 @@ public: | |||
| 312 | } // namespace Kernel | 311 | } // namespace Kernel |
| 313 | } // namespace Gemm | 312 | } // namespace Gemm |
| 314 | } // namespace Cmct | 313 | } // namespace Cmct |
| 315 | -#endif | 314 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef CMCT_INCLUDE_MATMUL_KERNEL_KERNEL_MATMUL_MIX_FIXPIPE_OPTI_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -325,4 +324,4 @@ public: | |||
| 325 | } // namespace Kernel | 324 | } // namespace Kernel |
| 326 | } // namespace Gemm | 325 | } // namespace Gemm |
| 327 | } // namespace Cmct | 326 | } // namespace Cmct |
| 328 | -#endif | 327 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef CMCT_KERNEL_KERNEL_MATMUL_MIX_WITH_WEIGHT_PRELOGUE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -150,4 +149,4 @@ private: | |||
| 150 | } // namespace Kernel | 149 | } // namespace Kernel |
| 151 | } // namespace Gemm | 150 | } // namespace Gemm |
| 152 | } // namespace Cmct | 151 | } // namespace Cmct |
| 153 | -#endif | 152 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_KERNEL_KERNEL_MATMUL_MIX_WITHOUT_QUE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -271,4 +270,4 @@ public: | |||
| 271 | } // namespace Kernel | 270 | } // namespace Kernel |
| 272 | } // namespace Gemm | 271 | } // namespace Gemm |
| 273 | } // namespace Cmct | 272 | } // namespace Cmct |
| 274 | -#endif | 273 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef CMCT_INCLUDE_MATMUL_KERNEL_KERNEL_MATMUL_STREAMK_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -326,4 +325,4 @@ public: | |||
| 326 | } // namespace Kernel | 325 | } // namespace Kernel |
| 327 | } // namespace Gemm | 326 | } // namespace Gemm |
| 328 | } // namespace Cmct | 327 | } // namespace Cmct |
| 329 | -#endif | 328 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_KERNEL_KERNEL_MATMUL_MN_EQUAL_ONE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -249,4 +248,4 @@ public: | |||
| 249 | } // namespace Kernel | 248 | } // namespace Kernel |
| 250 | } // namespace Gemm | 249 | } // namespace Gemm |
| 251 | } // namespace Cmct | 250 | } // namespace Cmct |
| 252 | -#endif | 251 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_KERNEL_KERNEL_MATMUL_WITHOUT_QUE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -293,4 +292,4 @@ public: | |||
| 293 | } // namespace Kernel | 292 | } // namespace Kernel |
| 294 | } // namespace Gemm | 293 | } // namespace Gemm |
| 295 | } // namespace Cmct | 294 | } // namespace Cmct |
| 296 | -#endif | 295 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_KERNEL_KERNEL_QBMM_CUBE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -326,4 +325,3 @@ __aicore__ inline void QuantMmBatchCube<QBMM_CUBE_KERNEL_FUN_TEM_PARAMS>::Proces | |||
| 326 | } // namespace Gemm | 325 | } // namespace Gemm |
| 327 | } // namespace Cmct | 326 | } // namespace Cmct |
| 328 | 327 | ||
| 329 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_KERNEL_KERNEL_QBMM_MX_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -352,4 +351,3 @@ __aicore__ inline void QuantMmBatchMX<QBMM_MX_KERNEL_FUN_TEM_PARAMS>::ProcessSin | |||
| 352 | } // namespace Gemm | 351 | } // namespace Gemm |
| 353 | } // namespace Cmct | 352 | } // namespace Cmct |
| 354 | 353 | ||
| 355 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_KERNEL_KERNEL_QBMM_PERTILE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -413,4 +412,3 @@ __aicore__ inline void QuantMmBatchPertile<QBMM_PERTILE_KERNEL_FUN_TEM_PARAMS>:: | |||
| 413 | } // namespace Gemm | 412 | } // namespace Gemm |
| 414 | } // namespace Cmct | 413 | } // namespace Cmct |
| 415 | 414 | ||
| 416 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file dispatch_policy.h | 12 | * \file dispatch_policy.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MATMUL_POLICY_DISPATCH_POLICY_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -330,4 +329,4 @@ struct MatmulToMul { | |||
| 330 | 329 | ||
| 331 | } // namespace Gemm | 330 | } // namespace Gemm |
| 332 | } // namespace Cmct | 331 | } // namespace Cmct |
| 333 | -#endif | 332 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef CMCT_INCLUDE_PROLOGUE_BLOCK_PROLOGUE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | namespace Cmct::Prologue { | 17 | namespace Cmct::Prologue { |
| 19 | template <class DispatchPolicy, class... Args> | 18 | template <class DispatchPolicy, class... Args> |
| 20 | class BlockPrologue { | 19 | class BlockPrologue { |
| @@ -24,4 +23,4 @@ class BlockPrologue { | |||
| 24 | 23 | ||
| 25 | 24 | ||
| 26 | 25 | ||
| 27 | -#endif | 26 | + |
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef PROLOGUE_BLOCK_BLOCK_PROLOGUE_B_ANTIQUANT_SCMC_ND_KN_H | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | 13 | ||
| @@ -474,4 +473,4 @@ private: | |||
| 474 | static constexpr QuantType ANTIQUANT_TYPE = QUANT_TYPE<decltype(LayoutScale{}.GetShape()), ElementScale>; | 473 | static constexpr QuantType ANTIQUANT_TYPE = QUANT_TYPE<decltype(LayoutScale{}.GetShape()), ElementScale>; |
| 475 | }; | 474 | }; |
| 476 | } // namespace Cmct::Prologue | 475 | } // namespace Cmct::Prologue |
| 477 | -#endif | 476 | + |
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef PROLOGUE_BLOCK_BLOCK_PROLOGUE_B_ANTIQUANT_SCMC_ND_NK_NZ_KN_H | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | 13 | ||
| @@ -656,4 +655,4 @@ private: | |||
| 656 | AscendC::LocalTensor<UbScaleAfterCastTensorTrait> ubScaleAfterCast_; | 655 | AscendC::LocalTensor<UbScaleAfterCastTensorTrait> ubScaleAfterCast_; |
| 657 | }; | 656 | }; |
| 658 | } // namespace Cmct::Prologue | 657 | } // namespace Cmct::Prologue |
| 659 | -#endif | 658 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file block_prologue_b_cast_scsc.h | 12 | * \file block_prologue_b_cast_scsc.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef CMCT_INCLUDE_PROLOGUE_BLOCK_PROLOGUE_B_CAST_SCSC_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -698,4 +697,3 @@ private: | |||
| 698 | }; | 697 | }; |
| 699 | } // namespace Cmct::Prologue | 698 | } // namespace Cmct::Prologue |
| 700 | 699 | ||
| 701 | - | ||
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef PROLOGUE_CONSTANT_H | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | 13 | ||
| @@ -61,4 +60,4 @@ template <typename Dst, typename Src = Dst> | |||
| 61 | constexpr uint32_t VECTOR_REG_SIZE = detail::VectorRegSize< | 60 | constexpr uint32_t VECTOR_REG_SIZE = detail::VectorRegSize< |
| 62 | typename AscendC::Std::remove_cvref_t<Dst>, typename AscendC::Std::remove_cvref_t<Src>>::VALUE; | 61 | typename AscendC::Std::remove_cvref_t<Dst>, typename AscendC::Std::remove_cvref_t<Src>>::VALUE; |
| 63 | } // namespace Cmct::Prologue | 62 | } // namespace Cmct::Prologue |
| 64 | -#endif | 63 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file dispatch_policy.h | 12 | * \file dispatch_policy.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef CMCT_INCLUDE_PROLOGUE_DISPATCH_POLICY_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -62,4 +61,3 @@ struct BAntiquantScmc { | |||
| 62 | }; | 61 | }; |
| 63 | } // namespace Cmct::Prologue | 62 | } // namespace Cmct::Prologue |
| 64 | 63 | ||
| 65 | - | ||
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef PROLOGUE_TILE_ANTIQUANT_ND_KN_H | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | 13 | ||
| @@ -387,4 +386,4 @@ struct AntiquantImpl< | |||
| 387 | 386 | ||
| 388 | } // namespace detail | 387 | } // namespace detail |
| 389 | } // namespace Cmct::Prologue::Tile | 388 | } // namespace Cmct::Prologue::Tile |
| 390 | -#endif | 389 | + |
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef PROLOGUE_TILE_ANTIQUANT_ND_NK_H | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | 13 | ||
| @@ -486,4 +485,4 @@ struct AntiquantImpl< | |||
| 486 | 485 | ||
| 487 | } // namespace detail | 486 | } // namespace detail |
| 488 | } // namespace Cmct::Prologue::Tile | 487 | } // namespace Cmct::Prologue::Tile |
| 489 | -#endif | 488 | + |
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef PROLOGUE_TILE_ANTIQUANT_ZN_H | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | 13 | ||
| @@ -486,4 +485,4 @@ private: | |||
| 486 | }; | 485 | }; |
| 487 | } // namespace detail | 486 | } // namespace detail |
| 488 | } // namespace Cmct::Prologue::Tile | 487 | } // namespace Cmct::Prologue::Tile |
| 489 | -#endif | 488 | + |
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef PROLOGUE_TILE_ANTIQUANT_H | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | namespace Cmct::Prologue::Tile { | 12 | namespace Cmct::Prologue::Tile { |
| 14 | 13 | ||
| @@ -58,4 +57,3 @@ __aicore__ inline void Antiquant( | |||
| 58 | 57 | ||
| 59 | 58 | ||
| 60 | 59 | ||
| 61 | - | ||
| @@ -1,35 +1,33 @@ | |||
| 1 | -/** | 1 | +/** |
| 2 | - | 2 | + |
| 3 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | 3 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. |
| 4 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | 4 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of |
| 5 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | 5 | + * CANN Open Software License Agreement Version 2.0 (the "License"). |
| 6 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | 6 | + * Please refer to the License for details. You may not use this file except in compliance with the License. |
| 7 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | 7 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, |
| 8 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 8 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 9 | - * See LICENSE in the root of the software repository for the full text of the License. | 9 | + * See LICENSE in the root of the software repository for the full text of the License. |
| 10 | - */ | 10 | + */ |
| 11 | -#ifndef PROLOGUE_TILE_CAST_H | 11 | +#pragma once |
| 12 | -#define PROLOGUE_TILE_CAST_H | 12 | + |
| 13 | - | 13 | +namespace Cmct::Prologue::Tile { |
| 14 | -namespace Cmct::Prologue::Tile { | 14 | + |
| 15 | - | 15 | +namespace detail { |
| 16 | -namespace detail { | 16 | +template <class ArchTag, class TensorOut, class TensorIn, class Shape, typename Enable = void> |
| 17 | -template <class ArchTag, class TensorOut, class TensorIn, class Shape, typename Enable = void> | 17 | +struct TileCastImpl { |
| 18 | -struct TileCastImpl { | 18 | + static_assert(AscendC::Std::always_false_v<ArchTag>, "can not find the specialization."); |
| 19 | - static_assert(AscendC::Std::always_false_v<ArchTag>, "can not find the specialization."); | 19 | + __aicore__ inline static void Run(const TensorOut& tensorOut, const TensorIn& tensorIn, const Shape& shape) = |
| 20 | - __aicore__ inline static void Run(const TensorOut& tensorOut, const TensorIn& tensorIn, const Shape& shape) = | 20 | + delete; |
| 21 | - delete; | 21 | +}; |
| 22 | -}; | 22 | +} // namespace detail |
| 23 | -} // namespace detail | 23 | + |
| 24 | - | 24 | +template <class ArchTag, class TensorOut, class TensorIn, class Shape> |
| 25 | -template <class ArchTag, class TensorOut, class TensorIn, class Shape> | 25 | +__aicore__ inline void TileCast(const TensorOut& tensorOut, const TensorIn& tensorIn, const Shape& shape) |
| 26 | -__aicore__ inline void TileCast(const TensorOut& tensorOut, const TensorIn& tensorIn, const Shape& shape) | 26 | +{ |
| 27 | -{ | 27 | + detail::TileCastImpl< |
| 28 | - detail::TileCastImpl< | 28 | + AscendC::Std::remove_cvref_t<ArchTag>, AscendC::Std::remove_cvref_t<TensorOut>, |
| 29 | - AscendC::Std::remove_cvref_t<ArchTag>, AscendC::Std::remove_cvref_t<TensorOut>, | 29 | + AscendC::Std::remove_cvref_t<TensorIn>, AscendC::Std::remove_cvref_t<Shape> >::Run(tensorOut, tensorIn, shape); |
| 30 | - AscendC::Std::remove_cvref_t<TensorIn>, AscendC::Std::remove_cvref_t<Shape> >::Run(tensorOut, tensorIn, shape); | 30 | +}; |
| 31 | -}; | 31 | +} // namespace Cmct::Prologue::Tile |
| 32 | -} // namespace Cmct::Prologue::Tile | 32 | +#include "tile_cast_b8_to_b16.h" |
| 33 | -#include "tile_cast_b8_to_b16.h" | 33 | + |
| 34 | - | ||
| 35 | - | ||
| @@ -1,137 +1,136 @@ | |||
| 1 | -/** | 1 | +/** |
| 2 | - | 2 | + |
| 3 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | 3 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. |
| 4 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | 4 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of |
| 5 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | 5 | + * CANN Open Software License Agreement Version 2.0 (the "License"). |
| 6 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | 6 | + * Please refer to the License for details. You may not use this file except in compliance with the License. |
| 7 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | 7 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, |
| 8 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 8 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 9 | - * See LICENSE in the root of the software repository for the full text of the License. | 9 | + * See LICENSE in the root of the software repository for the full text of the License. |
| 10 | - */ | 10 | + */ |
| 11 | -#ifndef PROLOGUE_TILE_CAST_MX_H | 11 | +#pragma once |
| 12 | -#define PROLOGUE_TILE_CAST_MX_H | 12 | +#if ASC_DEVKIT_MAJOR >= 9 |
| 13 | -#if ASC_DEVKIT_MAJOR >= 9 | 13 | +#include "kernel_basic_intf.h" |
| 14 | -#include "kernel_basic_intf.h" | 14 | +#else |
| 15 | -#else | 15 | +#include "kernel_operator_intf.h" |
| 16 | -#include "kernel_operator_intf.h" | 16 | +#endif |
| 17 | -#endif | 17 | +#include "../../utils/math_utils.h" |
| 18 | -#include "../../utils/math_utils.h" | 18 | +#include "../../utils/underscore.h" |
| 19 | -#include "../../utils/underscore.h" | 19 | + |
| 20 | - | 20 | +namespace Cmct::Prologue::Tile { |
| 21 | -namespace Cmct::Prologue::Tile { | 21 | +using AscendC::BLOCK_CUBE; |
| 22 | -using AscendC::BLOCK_CUBE; | 22 | +using AscendC::VECTOR_REG_WIDTH; |
| 23 | -using AscendC::VECTOR_REG_WIDTH; | 23 | +using Cmct::CeilAlign; |
| 24 | -using Cmct::CeilAlign; | 24 | +using Cmct::CeilDiv; |
| 25 | -using Cmct::CeilDiv; | 25 | +using Cmct::Gemm::Get; |
| 26 | -using Cmct::Gemm::Get; | 26 | +using Gemm::Arch::DAV3510; |
| 27 | -using Gemm::Arch::DAV3510; | 27 | +namespace MicroAPI = AscendC::MicroAPI; |
| 28 | -namespace MicroAPI = AscendC::MicroAPI; | 28 | +namespace detail { |
| 29 | -namespace detail { | 29 | + |
| 30 | - | 30 | +// ND NK |
| 31 | -// ND NK | 31 | +template <class TensorTraitOut, class TensorTraitIn, class Shape> |
| 32 | -template <class TensorTraitOut, class TensorTraitIn, class Shape> | 32 | +struct TileCastImpl< |
| 33 | -struct TileCastImpl< | 33 | + DAV3510, AscendC::LocalTensor<TensorTraitOut>, AscendC::LocalTensor<TensorTraitIn>, Shape, |
| 34 | - DAV3510, AscendC::LocalTensor<TensorTraitOut>, AscendC::LocalTensor<TensorTraitIn>, Shape, | 34 | + typename AscendC::Std::enable_if_t< |
| 35 | - typename AscendC::Std::enable_if_t< | 35 | + IsRowMajor2D<decltype(TensorTraitIn{}.GetLayout())>::value // 判断NK场景 |
| 36 | - IsRowMajor2D<decltype(TensorTraitIn{}.GetLayout())>::value // 判断NK场景 | 36 | + && AscendC::Std::is_same_v<AscendC::PrimT<TensorTraitIn>, AscendC::fp8_e8m0_t>>> { |
| 37 | - && AscendC::Std::is_same_v<AscendC::PrimT<TensorTraitIn>, AscendC::fp8_e8m0_t>>> { | 37 | + using DtypeOut = AscendC::PrimT<TensorTraitOut>; |
| 38 | - using DtypeOut = AscendC::PrimT<TensorTraitOut>; | 38 | + using DtypeIn = AscendC::PrimT<TensorTraitIn>; |
| 39 | - using DtypeIn = AscendC::PrimT<TensorTraitIn>; | 39 | + __aicore__ inline static void Run( |
| 40 | - __aicore__ inline static void Run( | 40 | + const AscendC::LocalTensor<TensorTraitOut>& tensorOut, const AscendC::LocalTensor<TensorTraitIn>& tensorIn, |
| 41 | - const AscendC::LocalTensor<TensorTraitOut>& tensorOut, const AscendC::LocalTensor<TensorTraitIn>& tensorIn, | 41 | + const Shape& shape) |
| 42 | - const Shape& shape) | 42 | + { |
| 43 | - { | 43 | + uint16_t ubLoopN = CeilDiv(Get<0>(shape), static_cast<uint64_t>(4)); |
| 44 | - uint16_t ubLoopN = CeilDiv(Get<0>(shape), static_cast<uint64_t>(4)); | 44 | + constexpr int16_t SHIFT_FOR_BF16 = 1; |
| 45 | - constexpr int16_t SHIFT_FOR_BF16 = 1; | 45 | + __ubuf__ uint8_t* antiQuantScaleBasePhyAddr = (__ubuf__ uint8_t*)tensorIn.GetPhyAddr(); |
| 46 | - __ubuf__ uint8_t* antiQuantScaleBasePhyAddr = (__ubuf__ uint8_t*)tensorIn.GetPhyAddr(); | 46 | + __ubuf__ DtypeOut* antiQuantScaleF16PhyAddr0 = (__ubuf__ DtypeOut*)tensorOut.GetPhyAddr(); |
| 47 | - __ubuf__ DtypeOut* antiQuantScaleF16PhyAddr0 = (__ubuf__ DtypeOut*)tensorOut.GetPhyAddr(); | 47 | + __ubuf__ DtypeOut* antiQuantScaleF16PhyAddr1 = antiQuantScaleF16PhyAddr0 + (VECTOR_REG_SIZE<DtypeOut>); |
| 48 | - __ubuf__ DtypeOut* antiQuantScaleF16PhyAddr1 = antiQuantScaleF16PhyAddr0 + (VECTOR_REG_SIZE<DtypeOut>); | 48 | + __VEC_SCOPE__ |
| 49 | - __VEC_SCOPE__ | 49 | + { |
| 50 | - { | 50 | + MicroAPI::RegTensor<uint8_t> antiQuantScaleE8m0Vreg0; |
| 51 | - MicroAPI::RegTensor<uint8_t> antiQuantScaleE8m0Vreg0; | 51 | + MicroAPI::RegTensor<uint8_t> antiQuantScaleE8m0Vreg1; |
| 52 | - MicroAPI::RegTensor<uint8_t> antiQuantScaleE8m0Vreg1; | 52 | + MicroAPI::RegTensor<DtypeOut> antiQuantScaleF16Vreg0; |
| 53 | - MicroAPI::RegTensor<DtypeOut> antiQuantScaleF16Vreg0; | 53 | + MicroAPI::RegTensor<DtypeOut> antiQuantScaleF16Vreg1; |
| 54 | - MicroAPI::RegTensor<DtypeOut> antiQuantScaleF16Vreg1; | 54 | + MicroAPI::MaskReg maskAll = MicroAPI::CreateMask<uint8_t, AscendC::MicroAPI::MaskPattern::ALL>(); |
| 55 | - MicroAPI::MaskReg maskAll = MicroAPI::CreateMask<uint8_t, AscendC::MicroAPI::MaskPattern::ALL>(); | 55 | + for (uint16_t ubLoopNIdx = 0; ubLoopNIdx < ubLoopN; ubLoopNIdx++) { |
| 56 | - for (uint16_t ubLoopNIdx = 0; ubLoopNIdx < ubLoopN; ubLoopNIdx++) { | 56 | + // 搬运128个E8M0的antiquantscale, 通过两倍上采样变成256个E8M0， DIST_US_B8表示搬运模式如下： |
| 57 | - // 搬运128个E8M0的antiquantscale, 通过两倍上采样变成256个E8M0， DIST_US_B8表示搬运模式如下： | 57 | + // Vn s1 s2 s3 s4 s5 s6 s7 s8 s9 ...... s125 s126 s127 s128 |
| 58 | - // Vn s1 s2 s3 s4 s5 s6 s7 s8 s9 ...... s125 s126 s127 s128 | 58 | + // Vd s1 s1 s2 s2 s3 s3 s4 s4 s5 ...... s125 s125 s126 s126 s127 s127 s128 s128 |
| 59 | - // Vd s1 s1 s2 s2 s3 s3 s4 s4 s5 ...... s125 s125 s126 s126 s127 s127 s128 s128 | 59 | + MicroAPI::LoadAlign<uint8_t, MicroAPI::LoadDist::DIST_US_B8>( |
| 60 | - MicroAPI::LoadAlign<uint8_t, MicroAPI::LoadDist::DIST_US_B8>( | 60 | + antiQuantScaleE8m0Vreg0, antiQuantScaleBasePhyAddr + ubLoopNIdx * 128); |
| 61 | - antiQuantScaleE8m0Vreg0, antiQuantScaleBasePhyAddr + ubLoopNIdx * 128); | 61 | + MicroAPI::RegTensor<uint8_t> zeroVreg; |
| 62 | - MicroAPI::RegTensor<uint8_t> zeroVreg; | 62 | + MicroAPI::Duplicate(zeroVreg, 0); |
| 63 | - MicroAPI::Duplicate(zeroVreg, 0); | 63 | + // 通过数据重排指令， 交织 antiQuantScaleE8m0Vreg0 和 zeroVreg , Interleave后变为 |
| 64 | - // 通过数据重排指令， 交织 antiQuantScaleE8m0Vreg0 和 zeroVreg , Interleave后变为 | 64 | + // antiQuantScaleE8m0Vreg0 |
| 65 | - // antiQuantScaleE8m0Vreg0 | 65 | + // Vn s1 0 s2 0 s3 0 s4 0 s5 0 s6 0 s7 0 s8 0....... s127 0 s128 0 |
| 66 | - // Vn s1 0 s2 0 s3 0 s4 0 s5 0 s6 0 s7 0 s8 0....... s127 0 s128 0 | 66 | + // antiQuantScaleE8m0Vreg1 |
| 67 | - // antiQuantScaleE8m0Vreg1 | 67 | + // Vd s128 0 s129 0 s130 0 s131 0 s132 0 s133 0....... s255 0 s256 0 |
| 68 | - // Vd s128 0 s129 0 s130 0 s131 0 s132 0 s133 0....... s255 0 s256 0 | 68 | + MicroAPI::Interleave( |
| 69 | - MicroAPI::Interleave( | 69 | + antiQuantScaleE8m0Vreg0, antiQuantScaleE8m0Vreg1, zeroVreg, antiQuantScaleE8m0Vreg0); |
| 70 | - antiQuantScaleE8m0Vreg0, antiQuantScaleE8m0Vreg1, zeroVreg, antiQuantScaleE8m0Vreg0); | 70 | + CastLowBitToF16(antiQuantScaleF16Vreg0, antiQuantScaleE8m0Vreg0, maskAll); |
| 71 | - CastLowBitToF16(antiQuantScaleF16Vreg0, antiQuantScaleE8m0Vreg0, maskAll); | 71 | + CastLowBitToF16(antiQuantScaleF16Vreg1, antiQuantScaleE8m0Vreg1, maskAll); |
| 72 | - CastLowBitToF16(antiQuantScaleF16Vreg1, antiQuantScaleE8m0Vreg1, maskAll); | 72 | + MicroAPI::StoreAlign<DtypeOut, MicroAPI::StoreDist::DIST_NORM_B16>( |
| 73 | - MicroAPI::StoreAlign<DtypeOut, MicroAPI::StoreDist::DIST_NORM_B16>( | 73 | + antiQuantScaleF16PhyAddr0 + ubLoopNIdx * AscendC::VECTOR_REG_WIDTH, antiQuantScaleF16Vreg0, |
| 74 | - antiQuantScaleF16PhyAddr0 + ubLoopNIdx * AscendC::VECTOR_REG_WIDTH, antiQuantScaleF16Vreg0, | 74 | + maskAll); |
| 75 | - maskAll); | 75 | + MicroAPI::StoreAlign<DtypeOut, MicroAPI::StoreDist::DIST_NORM_B16>( |
| 76 | - MicroAPI::StoreAlign<DtypeOut, MicroAPI::StoreDist::DIST_NORM_B16>( | 76 | + antiQuantScaleF16PhyAddr1 + ubLoopNIdx * AscendC::VECTOR_REG_WIDTH, antiQuantScaleF16Vreg1, |
| 77 | - antiQuantScaleF16PhyAddr1 + ubLoopNIdx * AscendC::VECTOR_REG_WIDTH, antiQuantScaleF16Vreg1, | 77 | + maskAll); |
| 78 | - maskAll); | 78 | + } |
| 79 | - } | 79 | + } |
| 80 | - } | 80 | + } |
| 81 | - } | 81 | +}; |
| 82 | -}; | 82 | + |
| 83 | - | 83 | +// ND/NZ KN |
| 84 | -// ND/NZ KN | 84 | +template <class TensorTraitOut, class TensorTraitIn, class Shape> |
| 85 | -template <class TensorTraitOut, class TensorTraitIn, class Shape> | 85 | +struct TileCastImpl< |
| 86 | -struct TileCastImpl< | 86 | + DAV3510, AscendC::LocalTensor<TensorTraitOut>, AscendC::LocalTensor<TensorTraitIn>, Shape, |
| 87 | - DAV3510, AscendC::LocalTensor<TensorTraitOut>, AscendC::LocalTensor<TensorTraitIn>, Shape, | 87 | + typename AscendC::Std::enable_if_t< |
| 88 | - typename AscendC::Std::enable_if_t< | 88 | + IsColumnMajor2D<decltype(TensorTraitIn{}.GetLayout())>::value // 判断KN场景 |
| 89 | - IsColumnMajor2D<decltype(TensorTraitIn{}.GetLayout())>::value // 判断KN场景 | 89 | + && AscendC::Std::is_same_v<AscendC::PrimT<TensorTraitIn>, AscendC::fp8_e8m0_t>>> { |
| 90 | - && AscendC::Std::is_same_v<AscendC::PrimT<TensorTraitIn>, AscendC::fp8_e8m0_t>>> { | 90 | + using DtypeOut = AscendC::PrimT<TensorTraitOut>; |
| 91 | - using DtypeOut = AscendC::PrimT<TensorTraitOut>; | 91 | + using DtypeIn = AscendC::PrimT<TensorTraitIn>; |
| 92 | - using DtypeIn = AscendC::PrimT<TensorTraitIn>; | 92 | + __aicore__ inline static void Run( |
| 93 | - __aicore__ inline static void Run( | 93 | + const AscendC::LocalTensor<TensorTraitOut>& tensorOut, const AscendC::LocalTensor<TensorTraitIn>& tensorIn, |
| 94 | - const AscendC::LocalTensor<TensorTraitOut>& tensorOut, const AscendC::LocalTensor<TensorTraitIn>& tensorIn, | 94 | + const Shape& shape) |
| 95 | - const Shape& shape) | 95 | + { |
| 96 | - { | 96 | + uint16_t ubLoopK = static_cast<uint16_t>(Get<1>(shape)); |
| 97 | - uint16_t ubLoopK = static_cast<uint16_t>(Get<1>(shape)); | 97 | + constexpr int16_t SHIFT_FOR_BF16 = 1; |
| 98 | - constexpr int16_t SHIFT_FOR_BF16 = 1; | 98 | + __ubuf__ uint8_t* antiQuantScaleBasePhyAddr = (__ubuf__ uint8_t*)tensorIn.GetPhyAddr(); |
| 99 | - __ubuf__ uint8_t* antiQuantScaleBasePhyAddr = (__ubuf__ uint8_t*)tensorIn.GetPhyAddr(); | 99 | + __ubuf__ DtypeOut* antiQuantScaleF16PhyAddr0 = (__ubuf__ DtypeOut*)tensorOut.GetPhyAddr(); |
| 100 | - __ubuf__ DtypeOut* antiQuantScaleF16PhyAddr0 = (__ubuf__ DtypeOut*)tensorOut.GetPhyAddr(); | 100 | + __ubuf__ DtypeOut* antiQuantScaleF16PhyAddr1 = antiQuantScaleF16PhyAddr0 + (AscendC::VECTOR_REG_WIDTH >> 1); |
| 101 | - __ubuf__ DtypeOut* antiQuantScaleF16PhyAddr1 = antiQuantScaleF16PhyAddr0 + (AscendC::VECTOR_REG_WIDTH >> 1); | 101 | + __VEC_SCOPE__ |
| 102 | - __VEC_SCOPE__ | 102 | + { |
| 103 | - { | 103 | + // KN mte2搬运的antiquantscale的标准大小为(4，256), 按照一行的粒度处理 |
| 104 | - // KN mte2搬运的antiquantscale的标准大小为(4，256), 按照一行的粒度处理 | 104 | + MicroAPI::RegTensor<uint8_t> antiQuantScaleE8m0Vreg0; |
| 105 | - MicroAPI::RegTensor<uint8_t> antiQuantScaleE8m0Vreg0; | 105 | + MicroAPI::RegTensor<uint8_t> antiQuantScaleE8m0Vreg1; |
| 106 | - MicroAPI::RegTensor<uint8_t> antiQuantScaleE8m0Vreg1; | 106 | + MicroAPI::RegTensor<DtypeOut> antiQuantScaleF16Vreg0; |
| 107 | - MicroAPI::RegTensor<DtypeOut> antiQuantScaleF16Vreg0; | 107 | + MicroAPI::RegTensor<DtypeOut> antiQuantScaleF16Vreg1; |
| 108 | - MicroAPI::RegTensor<DtypeOut> antiQuantScaleF16Vreg1; | 108 | + MicroAPI::MaskReg maskAll = MicroAPI::CreateMask<uint8_t, AscendC::MicroAPI::MaskPattern::ALL>(); |
| 109 | - MicroAPI::MaskReg maskAll = MicroAPI::CreateMask<uint8_t, AscendC::MicroAPI::MaskPattern::ALL>(); | 109 | + for (uint16_t ubLoopKIdx = 0; ubLoopKIdx < ubLoopK; ubLoopKIdx++) { |
| 110 | - for (uint16_t ubLoopKIdx = 0; ubLoopKIdx < ubLoopK; ubLoopKIdx++) { | 110 | + // 搬运256个E8M0的antiquantscale, DIST_NORM表示搬运模式如下： |
| 111 | - // 搬运256个E8M0的antiquantscale, DIST_NORM表示搬运模式如下： | 111 | + // Vn s1 s2 s3 s4 s5 s6 s7 s8 s9 ...... s254 s255 s256 |
| 112 | - // Vn s1 s2 s3 s4 s5 s6 s7 s8 s9 ...... s254 s255 s256 | 112 | + // Vd s1 s2 s3 s4 s5 s6 s7 s8 s9 ...... s254 s255 s256 |
| 113 | - // Vd s1 s2 s3 s4 s5 s6 s7 s8 s9 ...... s254 s255 s256 | 113 | + MicroAPI::LoadAlign<uint8_t, MicroAPI::LoadDist::DIST_NORM>( |
| 114 | - MicroAPI::LoadAlign<uint8_t, MicroAPI::LoadDist::DIST_NORM>( | 114 | + antiQuantScaleE8m0Vreg0, antiQuantScaleBasePhyAddr + ubLoopKIdx * VECTOR_REG_WIDTH); |
| 115 | - antiQuantScaleE8m0Vreg0, antiQuantScaleBasePhyAddr + ubLoopKIdx * VECTOR_REG_WIDTH); | 115 | + MicroAPI::RegTensor<uint8_t> zeroVreg; |
| 116 | - MicroAPI::RegTensor<uint8_t> zeroVreg; | 116 | + MicroAPI::Duplicate(zeroVreg, 0); |
| 117 | - MicroAPI::Duplicate(zeroVreg, 0); | 117 | + // 通过数据重排指令， 交织 antiQuantScaleE8m0Vreg0 和 zeroVreg , Interleave后变为 |
| 118 | - // 通过数据重排指令， 交织 antiQuantScaleE8m0Vreg0 和 zeroVreg , Interleave后变为 | 118 | + // antiQuantScaleE8m0Vreg0 |
| 119 | - // antiQuantScaleE8m0Vreg0 | 119 | + // Vn s1 0 s2 0 s3 0 s4 0 s5 0 s6 0 s7 0 s8 0....... s127 0 s128 0 |
| 120 | - // Vn s1 0 s2 0 s3 0 s4 0 s5 0 s6 0 s7 0 s8 0....... s127 0 s128 0 | 120 | + // antiQuantScaleE8m0Vreg1 |
| 121 | - // antiQuantScaleE8m0Vreg1 | 121 | + // Vd s128 0 s129 0 s130 0 s131 0 s132 0 s133 0....... s255 0 s256 0 |
| 122 | - // Vd s128 0 s129 0 s130 0 s131 0 s132 0 s133 0....... s255 0 s256 0 | 122 | + MicroAPI::Interleave( |
| 123 | - MicroAPI::Interleave( | 123 | + antiQuantScaleE8m0Vreg0, antiQuantScaleE8m0Vreg1, zeroVreg, antiQuantScaleE8m0Vreg0); |
| 124 | - antiQuantScaleE8m0Vreg0, antiQuantScaleE8m0Vreg1, zeroVreg, antiQuantScaleE8m0Vreg0); | 124 | + CastLowBitToF16(antiQuantScaleF16Vreg0, antiQuantScaleE8m0Vreg0, maskAll); |
| 125 | - CastLowBitToF16(antiQuantScaleF16Vreg0, antiQuantScaleE8m0Vreg0, maskAll); | 125 | + CastLowBitToF16(antiQuantScaleF16Vreg1, antiQuantScaleE8m0Vreg1, maskAll); |
| 126 | - CastLowBitToF16(antiQuantScaleF16Vreg1, antiQuantScaleE8m0Vreg1, maskAll); | 126 | + MicroAPI::StoreAlign<DtypeOut, MicroAPI::StoreDist::DIST_NORM_B16>( |
| 127 | - MicroAPI::StoreAlign<DtypeOut, MicroAPI::StoreDist::DIST_NORM_B16>( | 127 | + antiQuantScaleF16PhyAddr0 + ubLoopKIdx * VECTOR_REG_WIDTH, antiQuantScaleF16Vreg0, maskAll); |
| 128 | - antiQuantScaleF16PhyAddr0 + ubLoopKIdx * VECTOR_REG_WIDTH, antiQuantScaleF16Vreg0, maskAll); | 128 | + MicroAPI::StoreAlign<DtypeOut, MicroAPI::StoreDist::DIST_NORM_B16>( |
| 129 | - MicroAPI::StoreAlign<DtypeOut, MicroAPI::StoreDist::DIST_NORM_B16>( | 129 | + antiQuantScaleF16PhyAddr1 + ubLoopKIdx * VECTOR_REG_WIDTH, antiQuantScaleF16Vreg1, maskAll); |
| 130 | - antiQuantScaleF16PhyAddr1 + ubLoopKIdx * VECTOR_REG_WIDTH, antiQuantScaleF16Vreg1, maskAll); | 130 | + } |
| 131 | - } | 131 | + } |
| 132 | - } | 132 | + } |
| 133 | - } | 133 | +}; |
| 134 | -}; | 134 | +} // namespace detail |
| 135 | -} // namespace detail | 135 | +} // namespace Cmct::Prologue::Tile |
| 136 | -} // namespace Cmct::Prologue::Tile | 136 | + |
| 137 | - | ||
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef TILE_COPY_GM_TO_UB_H | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | 13 | ||
| @@ -129,4 +128,4 @@ struct CopyIfImpl< | |||
| 129 | }; | 128 | }; |
| 130 | } // namespace detail | 129 | } // namespace detail |
| 131 | } // namespace Cmct::Gemm::Tile | 130 | } // namespace Cmct::Gemm::Tile |
| 132 | -#endif // TILE_COPY_GM_TO_UB_H | 131 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file copy_out_split_m_with_params.h | 12 | * \file copy_out_split_m_with_params.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MATMUL_TILE_COPY_OUT_COPY_OUT_SPLIT_M_WITH_PARAMS_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -238,4 +237,4 @@ public: | |||
| 238 | } // namespace Tile | 237 | } // namespace Tile |
| 239 | } // namespace Gemm | 238 | } // namespace Gemm |
| 240 | } // namespace Cmct | 239 | } // namespace Cmct |
| 241 | -#endif | 240 | + |
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef TILE_COPY_UB_TO_L1_H | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | namespace Cmct::Gemm::Tile { | 13 | namespace Cmct::Gemm::Tile { |
| @@ -95,4 +94,4 @@ struct CopyIfImpl< | |||
| 95 | }; | 94 | }; |
| 96 | } // namespace detail | 95 | } // namespace detail |
| 97 | } // namespace Cmct::Gemm::Tile | 96 | } // namespace Cmct::Gemm::Tile |
| 98 | -#endif // TILE_COPY_UB_TO_L1_H | 97 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file tile_copy.h | 12 | * \file tile_copy.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MATMUL_TILE_TILE_COPY_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -291,4 +290,4 @@ inline constexpr bool HasCopyCo1ToOutV = HasCopyCo1ToOut<TileCopy>::value; | |||
| 291 | } // namespace Tile | 290 | } // namespace Tile |
| 292 | } // namespace Gemm | 291 | } // namespace Gemm |
| 293 | } // namespace Cmct | 292 | } // namespace Cmct |
| 294 | -#endif | 293 | + |
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef TILE_TILE_COPY_IF_H | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | 13 | ||
| @@ -60,4 +59,4 @@ __aicore__ inline void CopyIf(const DstTensor& dstTensor, const SrcTensor& srcTe | |||
| 60 | 59 | ||
| 61 | 60 | ||
| 62 | 61 | ||
| 63 | -#endif | 62 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file tile_copy_policy.h | 12 | * \file tile_copy_policy.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MATMUL_TILE_TILE_COPY_POLICY_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -49,4 +48,4 @@ struct Copy {}; | |||
| 49 | } // namespace Tile | 48 | } // namespace Tile |
| 50 | } // namespace Gemm | 49 | } // namespace Gemm |
| 51 | } // namespace Cmct | 50 | } // namespace Cmct |
| 52 | -#endif | 51 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef UTILS_ARCH_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | namespace Cmct { | 18 | namespace Cmct { |
| 20 | namespace Gemm { | 19 | namespace Gemm { |
| @@ -36,4 +35,4 @@ constexpr static int64_t L0C_SIZE = 128 * 1024; | |||
| 36 | constexpr static int64_t L1_SIZE = 512 * 1024; | 35 | constexpr static int64_t L1_SIZE = 512 * 1024; |
| 37 | } // namespace Gemm | 36 | } // namespace Gemm |
| 38 | } // namespace Cmct | 37 | } // namespace Cmct |
| 39 | -#endif | 38 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef UTILS_COMMON_UTILS_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -182,4 +181,4 @@ template <class Stride> | |||
| 182 | struct is_2d_nz_c0_32 : is_2d_nz_c0_32_impl<typename AscendC::Std::remove_cvref_t<Stride>> {}; | 181 | struct is_2d_nz_c0_32 : is_2d_nz_c0_32_impl<typename AscendC::Std::remove_cvref_t<Stride>> {}; |
| 183 | } // namespace Gemm | 182 | } // namespace Gemm |
| 184 | } // namespace Cmct | 183 | } // namespace Cmct |
| 185 | -#endif | 184 | + |
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef UTILS_CONSTANT_H | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | 13 | ||
| @@ -201,4 +200,4 @@ constexpr uint64_t BLK_ELEM = 32 / sizeof(T); | |||
| 201 | template <typename T> | 200 | template <typename T> |
| 202 | inline constexpr uint32_t C0 = 32 / sizeof(T); | 201 | inline constexpr uint32_t C0 = 32 / sizeof(T); |
| 203 | } // namespace Cmct::Gemm | 202 | } // namespace Cmct::Gemm |
| 204 | -#endif | 203 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef UTILS_COORD_UTILS_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -408,4 +407,4 @@ public: | |||
| 408 | }; | 407 | }; |
| 409 | } // namespace Gemm | 408 | } // namespace Gemm |
| 410 | } // namespace Cmct | 409 | } // namespace Cmct |
| 411 | -#endif | 410 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef UTILS_DEVICE_UTILS_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -40,4 +39,4 @@ __aicore__ inline int64_t AlignBlock(const int64_t& t) | |||
| 40 | 39 | ||
| 41 | } // namespace Gemm | 40 | } // namespace Gemm |
| 42 | } // namespace Cmct | 41 | } // namespace Cmct |
| 43 | -#endif | 42 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef UTILS_FILL_UTILS_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -73,4 +72,4 @@ __aicore__ inline void InitOutputWithZero(AscendC::GlobalTensor<T> yInitGlobal, | |||
| 73 | 72 | ||
| 74 | } // namespace Gemm | 73 | } // namespace Gemm |
| 75 | } // namespace Cmct | 74 | } // namespace Cmct |
| 76 | -#endif | 75 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef CMCT_INCLUDE_UTILS_GEMM_TYPE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -27,4 +26,3 @@ struct GemmType { | |||
| 27 | }; | 26 | }; |
| 28 | } // namespace Cmct::Gemm | 27 | } // namespace Cmct::Gemm |
| 29 | 28 | ||
| 30 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef UTILS_HOST_UTILS_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -45,4 +44,4 @@ static size_t GetSysWorkspaceSize() | |||
| 45 | } // namespace Gemm | 44 | } // namespace Gemm |
| 46 | } // namespace Cmct | 45 | } // namespace Cmct |
| 47 | 46 | ||
| 48 | -#endif | 47 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file integral_constant.h | 12 | * \file integral_constant.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef UTILS_INTEGRAL_CONSTANT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -195,4 +194,4 @@ __host_aicore__ inline constexpr Int<(t <= u)> operator<=(Int<t>, Int<u>) | |||
| 195 | } | 194 | } |
| 196 | } // namespace Gemm | 195 | } // namespace Gemm |
| 197 | } // namespace Cmct | 196 | } // namespace Cmct |
| 198 | -#endif | 197 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef UTILS_LAYOUT_UTILS_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | // dependency of matmul_utils.h | 18 | // dependency of matmul_utils.h |
| 20 | 19 | ||
| @@ -243,4 +242,4 @@ __aicore__ constexpr inline decltype(auto) MakeLayoutByFormat(int row, int col) | |||
| 243 | } | 242 | } |
| 244 | } // namespace Gemm | 243 | } // namespace Gemm |
| 245 | } // namespace Cmct | 244 | } // namespace Cmct |
| 246 | -#endif | 245 | + |
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef UTILS_MATH_UTILS_H | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | 13 | ||
| @@ -97,4 +96,4 @@ __aicore__ inline Int ElemToByte(AscendC::Std::integral_constant<Int, Count>) | |||
| 97 | return Count * sizeof(Dtype); | 96 | return Count * sizeof(Dtype); |
| 98 | } | 97 | } |
| 99 | } // namespace Cmct | 98 | } // namespace Cmct |
| 100 | -#endif | 99 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef UTILS_MATMUL_LAYOUT_TYPE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -49,4 +48,4 @@ using ToMatmulTypeT = typename ToMatmulType<LayoutT>::Type; | |||
| 49 | 48 | ||
| 50 | } // namespace Gemm | 49 | } // namespace Gemm |
| 51 | } // namespace Cmct | 50 | } // namespace Cmct |
| 52 | -#endif | 51 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file quant_batch_matmul_constant.h | 12 | * \file quant_batch_matmul_constant.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef UTILS_QUANT_BATCH_MATMUL_CONSTANT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | namespace Cmct { | 16 | namespace Cmct { |
| 18 | namespace Gemm { | 17 | namespace Gemm { |
| 19 | namespace QuantBatchMatmul { | 18 | namespace QuantBatchMatmul { |
| @@ -73,4 +72,4 @@ enum class QuantMode : uint32_t { | |||
| 73 | } // namespace QuantBatchMatmul | 72 | } // namespace QuantBatchMatmul |
| 74 | } // namespace Gemm | 73 | } // namespace Gemm |
| 75 | } // namespace Cmct | 74 | } // namespace Cmct |
| 76 | -#endif | 75 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef UTILS_STATUS_UTILS_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | namespace Cmct { | 18 | namespace Cmct { |
| 20 | namespace Gemm { | 19 | namespace Gemm { |
| @@ -92,4 +91,4 @@ constexpr const char* GetStatusString(Status status) | |||
| 92 | 91 | ||
| 93 | } // namespace Gemm | 92 | } // namespace Gemm |
| 94 | } // namespace Cmct | 93 | } // namespace Cmct |
| 95 | -#endif | 94 | + |
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef UTILS_TENSOR_TRAITS_H | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | 13 | ||
| @@ -66,4 +65,4 @@ struct TensorTraitL1<false, T, TPos> { | |||
| 66 | AscendC::Stride<Cmct::Gemm::_16, Cmct::Gemm::_256>, AscendC::Stride<Cmct::Gemm::_1, uint64_t>>>>; | 65 | AscendC::Stride<Cmct::Gemm::_16, Cmct::Gemm::_256>, AscendC::Stride<Cmct::Gemm::_1, uint64_t>>>>; |
| 67 | }; | 66 | }; |
| 68 | } // namespace Cmct::Gemm | 67 | } // namespace Cmct::Gemm |
| 69 | -#endif | 68 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef UTILS_TENSOR_UTILS_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -310,4 +309,4 @@ __aicore__ inline constexpr bool PosIsL0C() | |||
| 310 | 309 | ||
| 311 | } // namespace Gemm | 310 | } // namespace Gemm |
| 312 | } // namespace Cmct | 311 | } // namespace Cmct |
| 313 | -#endif | 312 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file tuple_utils.h | 12 | * \file tuple_utils.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef UTILS_TUPLE_UTILS_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -42,4 +41,4 @@ __aicore__ constexpr inline decltype(auto) GetIntegralConstant() | |||
| 42 | } | 41 | } |
| 43 | } // namespace Gemm | 42 | } // namespace Gemm |
| 44 | } // namespace Cmct | 43 | } // namespace Cmct |
| 45 | -#endif | 44 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef CMCT_INCLUDE_UTILS_UNDERSCORE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -24,4 +23,4 @@ struct Underscore : _0 {}; | |||
| 24 | static constexpr Underscore _; | 23 | static constexpr Underscore _; |
| 25 | } // namespace Gemm | 24 | } // namespace Gemm |
| 26 | } // namespace Cmct | 25 | } // namespace Cmct |
| 27 | -#endif | 26 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file math_util.h | 12 | * \file math_util.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MATMUL_COMMON_MATH_UTIL_H_ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -67,4 +66,3 @@ static T CeilDiv(T num1, T num2) | |||
| 67 | } | 66 | } |
| 68 | } | 67 | } |
| 69 | 68 | ||
| 70 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MATMUL_COMMON_INFERSHAPE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -31,4 +30,3 @@ bool CheckIsUnknownDimNum(const gert::Shape& shape); | |||
| 31 | } // namespace NN | 30 | } // namespace NN |
| 32 | } // namespace Ops | 31 | } // namespace Ops |
| 33 | 32 | ||
| 34 | - | ||
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef PTA_NPU_OP_API_INC_LEVEL0_OP_BATCHMATMUL_OP_H_ | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | 13 | ||
| @@ -38,4 +37,3 @@ const aclTensor* BatchMatMulNzFp162Fp32( | |||
| 38 | const bool adjX2, const bool offsetX, const int64_t opImplModeEnum, aclOpExecutor* executor); | 37 | const bool adjX2, const bool offsetX, const int64_t opImplModeEnum, aclOpExecutor* executor); |
| 39 | } // namespace l0op | 38 | } // namespace l0op |
| 40 | 39 | ||
| 41 | - | ||
| @@ -8,8 +8,7 @@ | |||
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | -#ifndef OP_API_SRC_LEVEL2_BATCH_MATMUL_UTIL_H_ | 11 | +#pragma once |
| 12 | - | ||
| 13 | 12 | ||
| 14 | 13 | ||
| 15 | 14 | ||
| @@ -60,4 +59,3 @@ const aclTensor* ExecFusedmmOp( | |||
| 60 | } // namespace Ops | 59 | } // namespace Ops |
| 61 | } // namespace NN | 60 | } // namespace NN |
| 62 | 61 | ||
| 63 | - | ||
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef MATMUL_COMMON_OP_API_CUBE_UTIL_H_ | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | 13 | ||
| @@ -66,4 +65,3 @@ bool CheckUnSupportDtype(const aclTensor *input, const aclTensor *weight); | |||
| 66 | } // namespace NN | 65 | } // namespace NN |
| 67 | } // namespace Ops | 66 | } // namespace Ops |
| 68 | 67 | ||
| 69 | - | ||
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef PTA_NPU_OP_API_INC_LEVEL0_OP_FUSEDMATMUL_OP_H_ | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | 13 | ||
| @@ -23,4 +22,3 @@ const aclTensor* FusedMatMul16Cast32( | |||
| 23 | 22 | ||
| 24 | } // namespace l0op | 23 | } // namespace l0op |
| 25 | 24 | ||
| 26 | - | ||
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef PTA_NPU_OP_API_INC_LEVEL0_OP_MATMUL_OP_H_ | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | 13 | ||
| @@ -77,4 +76,3 @@ const aclTensor* GemmV3NdWithAlphaBeta(const aclTensor* x1, | |||
| 77 | 76 | ||
| 78 | } // namespace l0op | 77 | } // namespace l0op |
| 79 | 78 | ||
| 80 | - | ||
| @@ -7,8 +7,7 @@ | |||
| 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | -#ifndef OP_API_SRC_LEVEL2_MATMUL_UTIL_H_ | 10 | +#pragma once |
| 11 | - | ||
| 12 | 11 | ||
| 13 | 12 | ||
| 14 | 13 | ||
| @@ -394,4 +393,3 @@ std::shared_ptr<NpuArchMatMulRuleBase> BuildRule(); | |||
| 394 | } // namespace NN | 393 | } // namespace NN |
| 395 | } // namespace Ops | 394 | } // namespace Ops |
| 396 | 395 | ||
| 397 | - | ||
| @@ -8,8 +8,7 @@ | |||
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | -#ifndef PTA_NPU_OP_API_INC_LEVEL0_OP_MATMUL_V2TOV3_H_ | 11 | +#pragma once |
| 12 | - | ||
| 13 | 12 | ||
| 14 | 13 | ||
| 15 | 14 | ||
| @@ -46,4 +45,4 @@ using l0op::MmCheckHitV3Shape; | |||
| 46 | } // namespace NN | 45 | } // namespace NN |
| 47 | } // namespace Ops | 46 | } // namespace Ops |
| 48 | 47 | ||
| 49 | -#endif // PTA_NPU_OP_API_INC_LEVEL0_OP_MATMUL_V2TOV3_H_ | 48 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file compile_info.h | 12 | * \file compile_info.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef OPS_MATMUL_COMPILE_INFO_H_ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -24,4 +23,3 @@ std::string DebugTilingData(gert::TilingContext *context); | |||
| 24 | } // namespace NN | 23 | } // namespace NN |
| 25 | } // namespace Ops | 24 | } // namespace Ops |
| 26 | 25 | ||
| 27 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file hash.h | 12 | * \file hash.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef OPS_BUILT_IN_OP_TILING_CUBE_ALGORITHM_HASH_HASH_H_ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -23,4 +22,4 @@ constexpr uint32_t kHashSeed = 271828; | |||
| 23 | uint32_t MurmurHash(const void *src, uint32_t len, uint32_t seed = kHashSeed); | 22 | uint32_t MurmurHash(const void *src, uint32_t len, uint32_t seed = kHashSeed); |
| 24 | } // namespace NN | 23 | } // namespace NN |
| 25 | } // namespace Ops | 24 | } // namespace Ops |
| 26 | -#endif // OPS_BUILT_IN_OP_TILING_CUBE_ALGORITHM_HASH_HASH_H_ | 25 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file tiling_cache.h | 12 | * \file tiling_cache.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef OPS_BUILT_IN_OP_TILING_CUBE_ALGORITHM_HASH_TILING_CACHE_H_ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -77,4 +76,4 @@ class TilingCache { | |||
| 77 | }; | 76 | }; |
| 78 | } // namespace NN | 77 | } // namespace NN |
| 79 | } // namespace Ops | 78 | } // namespace Ops |
| 80 | -#endif // OPS_BUILT_IN_OP_TILING_CUBE_ALGORITHM_HASH_TILING_CACHE_H_ | 79 | + |
Mmatmul/dual_level_quant_batch_matmul/op_kernel/arch35/dual_level_quant_batch_matmul_basic_block.h+2-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file dual_level_quant_batch_matmul_basic_block.h | 12 | * \file dual_level_quant_batch_matmul_basic_block.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef DUAL_LEVEL_QUANT_BATCH_MATMUL_BASIC_BLOCK_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -234,4 +233,4 @@ __aicore__ inline void DualLevelQuantMatmulBasicBlock<LOCAL_TEMPLATE_FUNC_PARAMS | |||
| 234 | } | 233 | } |
| 235 | 234 | ||
| 236 | } // namespace DualLevelQuantBatchMatmul::Arch35 | 235 | } // namespace DualLevelQuantBatchMatmul::Arch35 |
| 237 | -#endif // DUAL_LEVEL_QUANT_BATCH_MATMUL_BASIC_BLOCK_H | 236 | + |
Mmatmul/dual_level_quant_batch_matmul/op_kernel/arch35/dual_level_quant_batch_matmul_basic_block_controller.h+2-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef DUAL_LEVEL_QUANT_BATCH_MATMUL_BASIC_BLOCK_CONTROLLER_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -97,4 +96,4 @@ __aicore__ inline void DualLevelQuantBatchMatmulBasicBlockController<LOCAL_TEMPL | |||
| 97 | } | 96 | } |
| 98 | 97 | ||
| 99 | } // namespace DualLevelQuantBatchMatmul::Arch35 | 98 | } // namespace DualLevelQuantBatchMatmul::Arch35 |
| 100 | -#endif // DUAL_LEVEL_QUANT_BATCH_MATMUL_BASIC_BLOCK_CONTROLLER_H | 99 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef DUAL_LEVEL_QUANT_BATCH_MATMUL_BLOCK_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -241,4 +240,4 @@ __aicore__ inline void DualLevelQuantBatchMatmulBaseBlock::CalcGMOffset() | |||
| 241 | } | 240 | } |
| 242 | 241 | ||
| 243 | } // namespace DualLevelQuantBatchMatmul::Arch35 | 242 | } // namespace DualLevelQuantBatchMatmul::Arch35 |
| 244 | -#endif // DUAL_LEVEL_QUANT_BATCH_MATMUL_BLOCK_H | 243 | + |
Mmatmul/dual_level_quant_batch_matmul/op_kernel/arch35/dual_level_quant_batch_matmul_cube_compute.h+2-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef DUAL_LEVEL_QUANT_BATCH_MATMUL_CUBE_COMPUTE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -294,4 +293,4 @@ __aicore__ inline void DLQBMM_CUBE_COMPUTE_CLASS::EndSync() | |||
| 294 | } | 293 | } |
| 295 | 294 | ||
| 296 | } // namespace DualLevelQuantBatchMatmul::Arch35 | 295 | } // namespace DualLevelQuantBatchMatmul::Arch35 |
| 297 | -#endif | 296 | + |
Mmatmul/dual_level_quant_batch_matmul/op_kernel/arch35/dual_level_quant_batch_matmul_cube_compute_tools.h+2-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef DUAL_LEVEL_QUANT_BATCH_MATMUL_CUBE_COMPUTE_TOOLS_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -202,4 +201,4 @@ __aicore__ inline void FixL0CToDst( | |||
| 202 | } | 201 | } |
| 203 | 202 | ||
| 204 | } // namespace DualLevelQuantBatchMatmul::Arch35 | 203 | } // namespace DualLevelQuantBatchMatmul::Arch35 |
| 205 | -#endif | 204 | + |
Mmatmul/dual_level_quant_batch_matmul/op_kernel/arch35/dual_level_quant_batch_matmul_vec_compute.h+1-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file dual_level_quant_batch_matmul_vec_compute.h | 12 | * \file dual_level_quant_batch_matmul_vec_compute.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef DUAL_LEVEL_QUANT_BATCH_MATMUL_VEC_COMPUTE_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -360,4 +359,3 @@ __aicore__ inline void DLQBMM_VEC_COMPUTE_CLASS::SetAndWaitMte2ToV() | |||
| 360 | 359 | ||
| 361 | } // namespace DualLevelQuantBatchMatmul::Arch35 | 360 | } // namespace DualLevelQuantBatchMatmul::Arch35 |
| 362 | 361 | ||
| 363 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef DUAL_LEVEL_QUANT_BATCH_MATMUL_VF_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -158,4 +157,4 @@ __simd_vf__ inline void MulAdd( | |||
| 158 | } | 157 | } |
| 159 | 158 | ||
| 160 | } // namespace DualLevelQuantBatchMatmul::Arch35 | 159 | } // namespace DualLevelQuantBatchMatmul::Arch35 |
| 161 | -#endif // DUAL_LEVEL_QUANT_BATCH_MATMUL_VF_H | 160 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef DUAL_LEVEL_QUANT_BATCH_MATMUL_TOOL_ARCH35_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -103,4 +102,4 @@ __aicore__ inline T Min(T a, T b) | |||
| 103 | } | 102 | } |
| 104 | 103 | ||
| 105 | } // namespace DualLevelQuantBatchMatmul::Arch35 | 104 | } // namespace DualLevelQuantBatchMatmul::Arch35 |
| 106 | -#endif | 105 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file fused_matmul_asw_basic_tiling.h | 12 | * \file fused_matmul_asw_basic_tiling.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_FUSED_MATMUL_ASW_BASIC_TILING_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -35,4 +34,4 @@ protected: | |||
| 35 | }; | 34 | }; |
| 36 | } // namespace fused_matmul | 35 | } // namespace fused_matmul |
| 37 | } // namespace optiling | 36 | } // namespace optiling |
| 38 | -#endif // __OP_HOST_FUSED_MATMUL_ASW_BASIC_TILING_H__ | 37 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file fused_matmul_basic_streamk_tiling.h | 12 | * \file fused_matmul_basic_streamk_tiling.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_FUSED_MATMUL_BASIC_STREAMK_TILING_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -33,4 +32,4 @@ protected: | |||
| 33 | }; | 32 | }; |
| 34 | } // namespace fused_matmul | 33 | } // namespace fused_matmul |
| 35 | } // namespace optiling | 34 | } // namespace optiling |
| 36 | -#endif // __OP_HOST_FUSED_MATMUL_ASW_BASIC_TILING_H__ | 35 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file fused_matmul_builtin_tiling.h | 12 | * \file fused_matmul_builtin_tiling.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_FUSED_MATMUL_BUILTIN_TILING_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -44,4 +43,4 @@ private: | |||
| 44 | }; | 43 | }; |
| 45 | } // namespace fused_matmul | 44 | } // namespace fused_matmul |
| 46 | } // namespace optiling | 45 | } // namespace optiling |
| 47 | -#endif // __OP_HOST_FUSED_MATMUL_BUILTIN_TILING_H__ | 46 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file fused_matmul_builtin_tiling_strategy.h | 12 | * \file fused_matmul_builtin_tiling_strategy.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_FUSED_MATMUL_BUILTIN_TILING_STRATEGY_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -48,4 +47,3 @@ inline std::vector<int32_t> GetFusedMatMulPriorities(NpuArch npuArch) | |||
| 48 | } // namespace fused_matmul | 47 | } // namespace fused_matmul |
| 49 | } // namespace optiling | 48 | } // namespace optiling |
| 50 | 49 | ||
| 51 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file fused_matmul_common.h | 12 | * \file fused_matmul_common.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_FUSED_MATMUL_COMMON_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -64,4 +63,4 @@ const std::map<std::string, FusedOpType> FUSED_OP_TYPE_MAP = { | |||
| 64 | const std::set<std::string> FusedOpTypeSupportStreamK = {"", "relu", "16cast32"}; | 63 | const std::set<std::string> FusedOpTypeSupportStreamK = {"", "relu", "16cast32"}; |
| 65 | 64 | ||
| 66 | } // namespace optiling | 65 | } // namespace optiling |
| 67 | -#endif // __OP_HOST_FUSED_MATMUL_COMMON_H__ | 66 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef __OP_HOST_FUSED_MATMUL_ITERBATCH_BASIC_TILING_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -36,4 +35,4 @@ protected: | |||
| 36 | }; | 35 | }; |
| 37 | } // namespace fused_matmul | 36 | } // namespace fused_matmul |
| 38 | } // namespace optiling | 37 | } // namespace optiling |
| 39 | -#endif // __OP_HOST_FUSED_MATMUL_ITERBATCH_BASIC_TILING_H__ | 38 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef __OP_HOST_FUSED_MATMUL_SIMPILIFIEDKEY_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -92,4 +91,4 @@ inline ge::graphStatus GenSimplifiedKey(gert::TilingContext* context, ge::char_t | |||
| 92 | } | 91 | } |
| 93 | } // namespace fused_matmul | 92 | } // namespace fused_matmul |
| 94 | } // namespace optiling | 93 | } // namespace optiling |
| 95 | -#endif // __OP_HOST_FUSED_MATMUL_SIMPILIFIEDKEY_H__ | 94 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file fused_matmul_tiling.h | 12 | * \file fused_matmul_tiling.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_FUSED_MATMUL_TILING_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -48,4 +47,4 @@ private: | |||
| 48 | }; | 47 | }; |
| 49 | } // namespace fused_matmul | 48 | } // namespace fused_matmul |
| 50 | } // namespace optiling | 49 | } // namespace optiling |
| 51 | -#endif // __OP_HOST_FUSED_MATMUL_TILING_H__ | 50 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file fused_matmul_tiling_key.h | 12 | * \file fused_matmul_tiling_key.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_FUSED_MATMUL_TILING_KEY_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -39,4 +38,3 @@ protected: | |||
| 39 | } // namespace fused_matmul | 38 | } // namespace fused_matmul |
| 40 | } // namespace optiling | 39 | } // namespace optiling |
| 41 | 40 | ||
| 42 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file fused_mat_mul_tiling_data.h | 12 | * \file fused_mat_mul_tiling_data.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_KERNEL_FUSED_MAT_MUL_TILING_DATA_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -27,4 +26,4 @@ struct FusedMatMulTilingData { | |||
| 27 | uint32_t hasBias = 0; | 26 | uint32_t hasBias = 0; |
| 28 | }; | 27 | }; |
| 29 | 28 | ||
| 30 | -#endif // __OP_KERNEL_FUSED_MAT_MUL_TILING_DATA_H__ | 29 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file fused_mat_mul_tiling_key_public.h | 12 | * \file fused_mat_mul_tiling_key_public.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef FUSED_MAT_MUL_TILING_KEY_PUBLIC_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -35,4 +34,4 @@ | |||
| 35 | 34 | ||
| 36 | 35 | ||
| 37 | 36 | ||
| 38 | -#endif | 37 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef FUSED_MAT_MUL_TILINGKEY_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -139,4 +138,4 @@ ASCENDC_TPL_SEL( | |||
| 139 | ASCENDC_TPL_UINT_SEL(OPTYPE, ASCENDC_TPL_UI_LIST, F_OPTYPE_RELU)), | 138 | ASCENDC_TPL_UINT_SEL(OPTYPE, ASCENDC_TPL_UI_LIST, F_OPTYPE_RELU)), |
| 140 | 139 | ||
| 141 | ); | 140 | ); |
| 142 | -#endif // FUSED_MAT_MUL_TILINGKEY_H | 141 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file fused_quant_matmul_asw_tiling.h | 12 | * \file fused_quant_matmul_asw_tiling.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef FUSED_QUANT_MATMUL_ASW_TILING_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -58,4 +57,4 @@ protected: | |||
| 58 | uint64_t fusedOpType_ = 0UL; | 57 | uint64_t fusedOpType_ = 0UL; |
| 59 | }; | 58 | }; |
| 60 | } // namespace optiling | 59 | } // namespace optiling |
| 61 | -#endif // FUSED_QUANT_MATMUL_ASW_TILING_H | 60 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file fused_quant_matmul_checker.h | 12 | * \file fused_quant_matmul_checker.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef FUSED_QUANT_MATMUL_CHECKER_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -48,4 +47,4 @@ protected: | |||
| 48 | const gert::StorageShape *offsetShape, const std::vector<int64_t> &dimValueOfMKN) const; | 47 | const gert::StorageShape *offsetShape, const std::vector<int64_t> &dimValueOfMKN) const; |
| 49 | }; | 48 | }; |
| 50 | } // namespace optiling | 49 | } // namespace optiling |
| 51 | -#endif // FUSED_QUANT_MATMUL_CHECKER_H | 50 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file fused_quant_matmul_swiglu_tiling.h | 12 | * \file fused_quant_matmul_swiglu_tiling.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef FUSED_QUANT_MATMUL_SWIGLU_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | namespace optiling { | 18 | namespace optiling { |
| @@ -73,4 +72,4 @@ protected: | |||
| 73 | ge::graphStatus PostTiling() override; | 72 | ge::graphStatus PostTiling() override; |
| 74 | }; | 73 | }; |
| 75 | } | 74 | } |
| 76 | -#endif // FUSED_QUANT_MATMUL_SWIGLU_H | 75 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file fused_quant_mat_mul_tiling_data.h | 12 | * \file fused_quant_mat_mul_tiling_data.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_KERNEL_FUSED_QUANT_MAT_MUL_TILING_DATA_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -43,5 +42,3 @@ struct FusedQuantMatmulSwigluTilingData { | |||
| 43 | }; | 42 | }; |
| 44 | 43 | ||
| 45 | 44 | ||
| 46 | - | ||
| 47 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file fused_quant_mat_mul_tilingkey.h | 12 | * \file fused_quant_mat_mul_tilingkey.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef FUSED_QUANT_MAT_MUL_TILINGKEY_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -60,4 +59,4 @@ ASCENDC_TPL_SEL(ASCENDC_TPL_ARGS_SEL( | |||
| 60 | TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOBL1_WITH_MMAPI), | 59 | TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOBL1_WITH_MMAPI), |
| 61 | ASCENDC_TPL_UINT_SEL(OPTYPE, ASCENDC_TPL_UI_LIST, F_OPTYPE_NONE, F_OPTYPE_RELU, F_OPTYPE_SWIGLU)), ); | 60 | ASCENDC_TPL_UINT_SEL(OPTYPE, ASCENDC_TPL_UI_LIST, F_OPTYPE_NONE, F_OPTYPE_RELU, F_OPTYPE_SWIGLU)), ); |
| 62 | } // namespace FusedQuantMatMulTilingKey | 61 | } // namespace FusedQuantMatMulTilingKey |
| 63 | -#endif | 62 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file gemmv3_tiling.h | 12 | * \file gemmv3_tiling.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_GEMMV3_TILING_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | namespace optiling { | 18 | namespace optiling { |
| @@ -40,4 +39,4 @@ protected: | |||
| 40 | }; | 39 | }; |
| 41 | } // namespace gemmv3 | 40 | } // namespace gemmv3 |
| 42 | } // namespace optiling | 41 | } // namespace optiling |
| 43 | -#endif // __OP_HOST_TRANSPOSE_BATCH_MAT_MUL_TILING_H__ | 42 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \file gemmv3_tiling_key.h | 13 | * \file gemmv3_tiling_key.h |
| 14 | * \brief | 14 | * \brief |
| 15 | */ | 15 | */ |
| 16 | -#ifndef __OP_HOST_GEMM_V3_TILING_KEY_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -29,4 +28,3 @@ public: | |||
| 29 | } // namespace matmul_v3_advanced | 28 | } // namespace matmul_v3_advanced |
| 30 | } // namespace optiling | 29 | } // namespace optiling |
| 31 | 30 | ||
| 32 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file gemmv3_tiling_strategy.h | 12 | * \file gemmv3_tiling_strategy.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_GEMMV3_TILING_STRATEGY_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -43,4 +42,3 @@ inline std::vector<int32_t> GetGemmV3Priorities(NpuArch npuArch) | |||
| 43 | } // namespace gemmv3 | 42 | } // namespace gemmv3 |
| 44 | } // namespace optiling | 43 | } // namespace optiling |
| 45 | 44 | ||
| 46 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \file gemm_v3_tiling_key.h | 13 | * \file gemm_v3_tiling_key.h |
| 14 | * \brief | 14 | * \brief |
| 15 | */ | 15 | */ |
| 16 | -#ifndef __OP_KERNEL_GEMM_V3_TILING_KEY_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -72,4 +71,4 @@ ASCENDC_TPL_SEL( | |||
| 72 | ASCENDC_TPL_UINT_SEL(FULL_LOAD, ASCENDC_TPL_UI_LIST, MAT_MUL_NO_FULL_LOAD), | 71 | ASCENDC_TPL_UINT_SEL(FULL_LOAD, ASCENDC_TPL_UI_LIST, MAT_MUL_NO_FULL_LOAD), |
| 73 | ASCENDC_TPL_UINT_SEL(L0C2OUT_MODEL, ASCENDC_TPL_UI_LIST, MAT_MUL_ON_THE_FLY)) | 72 | ASCENDC_TPL_UINT_SEL(L0C2OUT_MODEL, ASCENDC_TPL_UI_LIST, MAT_MUL_ON_THE_FLY)) |
| 74 | ); // __OP_KERNEL_GEMM_V3_TILING_KEY_H__ | 73 | ); // __OP_KERNEL_GEMM_V3_TILING_KEY_H__ |
| 75 | -#endif | 74 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file matmul_base_tiling.h | 12 | * \file matmul_base_tiling.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_MATMUL_BASE_TILING_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -92,4 +91,3 @@ protected: | |||
| 92 | }; | 91 | }; |
| 93 | } // namespace optiling | 92 | } // namespace optiling |
| 94 | 93 | ||
| 95 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file matmul_tiling_cfg.h | 12 | * \file matmul_tiling_cfg.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_MATMUL_TILING_CFG_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -51,4 +50,3 @@ public: | |||
| 51 | }; | 50 | }; |
| 52 | } // namespace optiling | 51 | } // namespace optiling |
| 53 | 52 | ||
| 54 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef __OP_HOST_MATMUL_TILING_REGISTRY_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -188,4 +187,3 @@ private: | |||
| 188 | MMRegister(#opType).tiling<className>(static_cast<int32_t>(priority), NpuArch::npuArch) | 187 | MMRegister(#opType).tiling<className>(static_cast<int32_t>(priority), NpuArch::npuArch) |
| 189 | } // namespace optiling | 188 | } // namespace optiling |
| 190 | 189 | ||
| 191 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \file matmul_v3_asw_loadbalance_table.h | 13 | * \file matmul_v3_asw_loadbalance_table.h |
| 14 | * \brief | 14 | * \brief |
| 15 | */ | 15 | */ |
| 16 | -#ifndef __OP_HOST_MATMUL_V3_ASW_LOADBALANCE_TABLE_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -89,4 +88,4 @@ const std::vector<std::tuple<uint64_t, uint64_t, uint64_t, double, double>> BLOC | |||
| 89 | }; | 88 | }; |
| 90 | } // namespace matmul_v3_advanced | 89 | } // namespace matmul_v3_advanced |
| 91 | } // namespace optiling | 90 | } // namespace optiling |
| 92 | -#endif // __OP_HOST_MATMUL_V3_ASW_LOADBALANCE_TABLE_H__ | 91 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file matmul_v3_asw_tiling.h | 12 | * \file matmul_v3_asw_tiling.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_MATMUL_V3_ASW_TILING_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -68,4 +67,4 @@ private: | |||
| 68 | }; | 67 | }; |
| 69 | } // namespace matmul_v3_advanced | 68 | } // namespace matmul_v3_advanced |
| 70 | } // namespace optiling | 69 | } // namespace optiling |
| 71 | -#endif // __OP_HOST_MATMUL_V3_ASW_TILING_H__ | 70 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \file matmul_v3_base_tiling_advanced.h | 13 | * \file matmul_v3_base_tiling_advanced.h |
| 14 | * \brief | 14 | * \brief |
| 15 | */ | 15 | */ |
| 16 | -#ifndef __OP_HOST_MATMUL_V3_BASE_TILING_ADVANCED_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -485,4 +484,4 @@ private: | |||
| 485 | }; | 484 | }; |
| 486 | } // namespace matmul_v3 | 485 | } // namespace matmul_v3 |
| 487 | } | 486 | } |
| 488 | -#endif // __OP_HOST_MATMUL_V3_BASE_TILING_ADVANCED_H__ | 487 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \file matmul_v3_basic_aswt_tiling.h | 13 | * \file matmul_v3_basic_aswt_tiling.h |
| 14 | * \brief | 14 | * \brief |
| 15 | */ | 15 | */ |
| 16 | -#ifndef __OP_HOST_MATMUL_V3_FULL_LOAD_TILING_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -56,4 +55,4 @@ private: | |||
| 56 | }; | 55 | }; |
| 57 | } // namespace matmul_v3 | 56 | } // namespace matmul_v3 |
| 58 | } // namespace optiling | 57 | } // namespace optiling |
| 59 | -#endif // __OP_HOST_MATMUL_V3_FULL_LOAD_TILING_H__ | 58 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \file matmul_v3_basic_streamk_tiling.h | 13 | * \file matmul_v3_basic_streamk_tiling.h |
| 14 | * \brief | 14 | * \brief |
| 15 | */ | 15 | */ |
| 16 | -#ifndef __OP_HOST_MATMUL_V3_BASIC_STREAMK_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -51,4 +50,4 @@ private: | |||
| 51 | }; | 50 | }; |
| 52 | } // namespace matmul_v3 | 51 | } // namespace matmul_v3 |
| 53 | } // namespace optiling | 52 | } // namespace optiling |
| 54 | -#endif // __OP_HOST_MATMUL_V3_BASIC_STREAM_K_H__ | 53 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \file matmul_v3_common_advanced.h | 13 | * \file matmul_v3_common_advanced.h |
| 14 | * \brief | 14 | * \brief |
| 15 | */ | 15 | */ |
| 16 | -#ifndef __OP_HOST_MATMUL_V3_COMMON_ADVANCED_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -175,4 +174,4 @@ struct MatMulV3RunInfo { | |||
| 175 | }; | 174 | }; |
| 176 | } | 175 | } |
| 177 | } | 176 | } |
| 178 | -#endif // __OP_HOST_MATMUL_V3_COMMON_ADVANCED_H__ | 177 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \file matmul_v3_compile_info_advanced.h | 13 | * \file matmul_v3_compile_info_advanced.h |
| 14 | * \brief | 14 | * \brief |
| 15 | */ | 15 | */ |
| 16 | -#ifndef __OP_HOST_MATMUL_V3_COMPILE_INFO_ADVANCED_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -76,4 +75,4 @@ inline ge::graphStatus InitCompileInfo(gert::TilingParseContext *context) | |||
| 76 | } | 75 | } |
| 77 | } | 76 | } |
| 78 | } // namespace optiling::matmul_v3 | 77 | } // namespace optiling::matmul_v3 |
| 79 | -#endif // __OP_HOST_MATMUL_V3_COMPILE_INFO_ADVANCED_H__ | 78 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \file matmul_v3_k_equal_zero_tiling.h | 13 | * \file matmul_v3_k_equal_zero_tiling.h |
| 14 | * \brief | 14 | * \brief |
| 15 | */ | 15 | */ |
| 16 | -#ifndef __OP_HOST_MATMUL_V3_K_EQUAL_ZERO_TILING_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -39,4 +38,4 @@ protected: | |||
| 39 | }; | 38 | }; |
| 40 | } // namespace matmul_v3 | 39 | } // namespace matmul_v3 |
| 41 | } // namespace optiling | 40 | } // namespace optiling |
| 42 | -#endif // __OP_HOST_MATMUL_V3_K_EQUAL_ZERO_H__ | 41 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \file matmul_v3_tiling_advanced.h | 13 | * \file matmul_v3_tiling_advanced.h |
| 14 | * \brief | 14 | * \brief |
| 15 | */ | 15 | */ |
| 16 | -#ifndef __OP_HOST_MATMUL_V3_ADVANCED_TILING_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -52,4 +51,4 @@ protected: | |||
| 52 | }; | 51 | }; |
| 53 | } | 52 | } |
| 54 | } | 53 | } |
| 55 | -#endif // __OP_HOST_MATMUL_V3_ADVANCED_TILING_H__ | 54 | + |
| @@ -13,12 +13,10 @@ | |||
| 13 | * \file matmul_v3_tiling_data.h | 13 | * \file matmul_v3_tiling_data.h |
| 14 | * \brief | 14 | * \brief |
| 15 | */ | 15 | */ |
| 16 | -#ifndef __OP_HOST_MATMUL_V3_TILING_DATA_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| 21 | 20 | ||
| 22 | 21 | ||
| 23 | 22 | ||
| 24 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \file matmul_v3_tiling_helper.h | 13 | * \file matmul_v3_tiling_helper.h |
| 14 | * \brief | 14 | * \brief |
| 15 | */ | 15 | */ |
| 16 | -#ifndef __OP_HOST_MATMUL_V3_TILING_HELPER_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -44,4 +43,3 @@ public: | |||
| 44 | } | 43 | } |
| 45 | } | 44 | } |
| 46 | 45 | ||
| 47 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file matmul_v3_tiling_key.h | 12 | * \file matmul_v3_tiling_key.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef OP_HOST_MATMUL_V3_TILING_KEY_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -120,4 +119,3 @@ protected: | |||
| 120 | } // namespace matmul_v3_advanced | 119 | } // namespace matmul_v3_advanced |
| 121 | } // namespace optiling | 120 | } // namespace optiling |
| 122 | 121 | ||
| 123 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \file matmul_v3_tiling_strategy.h | 13 | * \file matmul_v3_tiling_strategy.h |
| 14 | * \brief | 14 | * \brief |
| 15 | */ | 15 | */ |
| 16 | -#ifndef __OP_HOST_MATMUL_V3_STRATEGY_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -50,4 +49,3 @@ inline std::vector<int32_t> GetMatMulV3Priorities(NpuArch npuArch) | |||
| 50 | } | 49 | } |
| 51 | } | 50 | } |
| 52 | 51 | ||
| 53 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \file matmul_v3_to_mul_tiling.h | 13 | * \file matmul_v3_to_mul_tiling.h |
| 14 | * \brief | 14 | * \brief |
| 15 | */ | 15 | */ |
| 16 | -#ifndef __OP_HOST_MATMUL_V3_TO_MUL_TILING_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -42,4 +41,4 @@ private: | |||
| 42 | }; | 41 | }; |
| 43 | } // namespace matmul_v3 | 42 | } // namespace matmul_v3 |
| 44 | } // namespace optiling | 43 | } // namespace optiling |
| 45 | -#endif // __OP_HOST_MATMUL_V3_TO_MUL_TILING_H__ | 44 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef CMCT_BLOCK_SCHEDULER_ASWT_BUILTIN_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -462,4 +461,4 @@ struct BlockSchedulerSelector< | |||
| 462 | } // namespace Block | 461 | } // namespace Block |
| 463 | } // namespace Gemm | 462 | } // namespace Gemm |
| 464 | } // namespace Cmct | 463 | } // namespace Cmct |
| 465 | -#endif | 464 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef CMCT_BLOCK_SCHEDULER_MUL_BUILTIN_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -109,4 +108,4 @@ using SchedulerOp = BlockSchedulerMulBuiltIn<ProblemShape_, L1TileShape_, L0Tile | |||
| 109 | } // namespace Block | 108 | } // namespace Block |
| 110 | } // namespace Gemm | 109 | } // namespace Gemm |
| 111 | } // namespace Cmct | 110 | } // namespace Cmct |
| 112 | -#endif | 111 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef CMCT_BLOCK_SCHEDULER_STREAMK_BUILTIN_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -209,4 +208,4 @@ using SchedulerOp = BlockSchedulerStreamKBuiltIn<ProblemShape_, L1TileShape_, L0 | |||
| 209 | } // namespace Block | 208 | } // namespace Block |
| 210 | } // namespace Gemm | 209 | } // namespace Gemm |
| 211 | } // namespace Cmct | 210 | } // namespace Cmct |
| 212 | -#endif | 211 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file mat_mul_asw_block.h | 12 | * \file mat_mul_asw_block.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MMV3_MATMUL_ASW_BLOCK_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -306,4 +305,3 @@ __aicore__ inline void MatmulAswBlock::CalcSplitKGMOffset(uint64_t splitKIndex) | |||
| 306 | 305 | ||
| 307 | } // namespace MatmulV3Advanced | 306 | } // namespace MatmulV3Advanced |
| 308 | 307 | ||
| 309 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file mat_mul_asw_kernel.h | 12 | * \file mat_mul_asw_kernel.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MMV3_MATMUL_ASW_KERNEL_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -145,4 +144,3 @@ __aicore__ inline void MatmulAswKernel<A_TYPE, B_TYPE, C_TYPE, BIAS_TYPE, BLOCK_ | |||
| 145 | 144 | ||
| 146 | } // namespace MatmulV3Advanced | 145 | } // namespace MatmulV3Advanced |
| 147 | 146 | ||
| 148 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file mat_mul_fixpipe_opti.h | 12 | * \file mat_mul_fixpipe_opti.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_KERNEL_MATMUL_V3_FIXPIPE_OPTI_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -270,4 +269,3 @@ __aicore__ inline void MatmulFixpipeOptiDualDstKernel<A_TYPE, B_TYPE, C_TYPE, BI | |||
| 270 | 269 | ||
| 271 | } // namespace MatmulV3Advanced | 270 | } // namespace MatmulV3Advanced |
| 272 | 271 | ||
| 273 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file mat_mul_fixpipe_basic_cmct.h | 12 | * \file mat_mul_fixpipe_basic_cmct.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MAT_MUL_FIXPIPE_BASIC_CMCT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -78,4 +77,4 @@ __aicore__ inline void MatMulFixpipeOptiActKernel( | |||
| 78 | mm(params); | 77 | mm(params); |
| 79 | } | 78 | } |
| 80 | } | 79 | } |
| 81 | -#endif | 80 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file mat_mul_full_load.h | 12 | * \file mat_mul_full_load.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MMV3_MATMUL_FULL_LOAD_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -272,4 +271,3 @@ __aicore__ inline void MatmulAswKernelABL1FullLoad<A_TYPE, B_TYPE, C_TYPE, BIAS_ | |||
| 272 | 271 | ||
| 273 | } // namespace MatmulV3Advanced | 272 | } // namespace MatmulV3Advanced |
| 274 | 273 | ||
| 275 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file mat_mul_input_k_eq_zero_clear_output.h | 12 | * \file mat_mul_input_k_eq_zero_clear_output.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MAT_MUL_INPUT_K_EQ_ZERO_CLEAR_OUTPUT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -64,4 +63,4 @@ __aicore__ inline void MatMulInputKEqZeroClearOutput(GM_ADDR biasGM, GM_ADDR cGM | |||
| 64 | AscendC::InitOutput<DTYPE_Y>(outputGM[coreIdx * everyAivDataCount], static_cast<uint64_t>(copyDataAmount), (DTYPE_Y)0); | 63 | AscendC::InitOutput<DTYPE_Y>(outputGM[coreIdx * everyAivDataCount], static_cast<uint64_t>(copyDataAmount), (DTYPE_Y)0); |
| 65 | } | 64 | } |
| 66 | } | 65 | } |
| 67 | -#endif | 66 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file mat_mul_mix_basic_cmct.h | 12 | * \file mat_mul_mix_basic_cmct.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef OP_KERNEL_MATMUL_MIX_BASIC_CMCT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -92,4 +91,4 @@ __aicore__ inline void MatMulMixWithoutQueActKernel(GM_ADDR aGM, GM_ADDR bGM, GM | |||
| 92 | mm(params); | 91 | mm(params); |
| 93 | } | 92 | } |
| 94 | } | 93 | } |
| 95 | -#endif // OP_KERNEL_MATMUL_MIX_BASIC_CMCT_H | 94 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file mat_mul_pingpong_basic_cmct.h | 12 | * \file mat_mul_pingpong_basic_cmct.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MAT_MUL_PINGPONG_BASIC_CMCT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -72,4 +71,4 @@ __aicore__ inline void MatMulActKernel(GM_ADDR aGM, GM_ADDR bGM, GM_ADDR biasGM, | |||
| 72 | mm(params); | 71 | mm(params); |
| 73 | } | 72 | } |
| 74 | } | 73 | } |
| 75 | -#endif | 74 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file mat_mul_stream_k_block.h | 12 | * \file mat_mul_stream_k_block.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MMV3_MATMUL_STREAM_K_BLOCK_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -267,4 +266,3 @@ __aicore__ inline void MatmulStreamKBlock::UpdateAivParams(uint64_t index, uint6 | |||
| 267 | } | 266 | } |
| 268 | } // namespace MatmulV3Advanced | 267 | } // namespace MatmulV3Advanced |
| 269 | 268 | ||
| 270 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file mat_mul_stream_k_kernel.h | 12 | * \file mat_mul_stream_k_kernel.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MMV3_MATMUL_STREAM_K_KERNEL_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -291,4 +290,3 @@ __aicore__ inline void MatmulStreamKKernel | |||
| 291 | } | 290 | } |
| 292 | } // namespace MatmulV3Advanced | 291 | } // namespace MatmulV3Advanced |
| 293 | 292 | ||
| 294 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file mat_mul_streamk_basic_cmct.h | 12 | * \file mat_mul_streamk_basic_cmct.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MAT_MUL_STREAMK_BASIC_CMCT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -75,4 +74,4 @@ __aicore__ inline void MatMulStreamKActKernel(GM_ADDR aGM, GM_ADDR bGM, GM_ADDR | |||
| 75 | MatmulKernel mm; | 74 | MatmulKernel mm; |
| 76 | mm(params); | 75 | mm(params); |
| 77 | } | 76 | } |
| 78 | -#endif | 77 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file mat_mul_tiling_data.h | 12 | * \file mat_mul_tiling_data.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_KERNEL_MATMUL_TILING_DATA_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -198,4 +197,3 @@ struct MatMulToMulBasicTilingData{ | |||
| 198 | }; | 197 | }; |
| 199 | 198 | ||
| 200 | 199 | ||
| 201 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MAT_MUL_TO_MUL_CMCT_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -73,4 +72,4 @@ __aicore__ inline void MatMulToMulActKernel( | |||
| 73 | mm(params); | 72 | mm(params); |
| 74 | } | 73 | } |
| 75 | } | 74 | } |
| 76 | -#endif | 75 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file mat_mul_v3_full_load_kernel_helper.h | 12 | * \file mat_mul_v3_full_load_kernel_helper.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_KERNEL_MAT_MUL_V3_FULL_LOAD_KERNEL_HELPER_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -226,4 +225,4 @@ __aicore__ inline void AswBL1FullLoadKernelMainLoop(MatmulImpl<A_TYPE, B_TYPE, C | |||
| 226 | } | 225 | } |
| 227 | 226 | ||
| 228 | } // namespace MatmulV3Advanced | 227 | } // namespace MatmulV3Advanced |
| 229 | -#endif // __OP_KERNEL_MAT_MUL_V3_FULL_LOAD_KERNEL_HELPER_H__ | 228 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef MAT_MUL_TILING_KEY_PUBLIC_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -89,4 +88,4 @@ enum class MatMulV3L0C2Out : std::uint8_t | |||
| 89 | ND_FIXPIPE_1_1 = MAT_MUL_1V1_ND_ALIG_FIXPIPE, | 88 | ND_FIXPIPE_1_1 = MAT_MUL_1V1_ND_ALIG_FIXPIPE, |
| 90 | ND_FIXPIPE_1_2 = MAT_MUL_1V2_ND_ALIG_FIXPIPE | 89 | ND_FIXPIPE_1_2 = MAT_MUL_1V2_ND_ALIG_FIXPIPE |
| 91 | }; | 90 | }; |
| 92 | -#endif | 91 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file mm_copy_cube_out.h | 12 | * \file mm_copy_cube_out.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MM_COPY_CUBE_OUT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | namespace MatmulCommon { | 18 | namespace MatmulCommon { |
| @@ -219,4 +218,4 @@ private: | |||
| 219 | }; | 218 | }; |
| 220 | 219 | ||
| 221 | } // namespace MatmulCommon | 220 | } // namespace MatmulCommon |
| 222 | -#endif // MM_COPY_CUBE_OUT_H | 221 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file mm_custom_mm_policy.h | 12 | * \file mm_custom_mm_policy.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MM_CUSTOM_MM_POLICY_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -26,4 +25,4 @@ public: | |||
| 26 | using CopyCubeOut = MMCustomCopyCubeOut<IMPL, A_TYPE, B_TYPE, C_TYPE, MM_CFG, McgShfMode::DUAL_DST_SPLIT_M>; | 25 | using CopyCubeOut = MMCustomCopyCubeOut<IMPL, A_TYPE, B_TYPE, C_TYPE, MM_CFG, McgShfMode::DUAL_DST_SPLIT_M>; |
| 27 | }; | 26 | }; |
| 28 | } // namespace MatmulCommon | 27 | } // namespace MatmulCommon |
| 29 | -#endif // MM_CUSTOM_MM_POLICY_H | 28 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file qbmmia_mx_basic_api_cmct.h | 12 | * \file qbmmia_mx_basic_api_cmct.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QBMMIA_MX_BAISC_API_CMCT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -87,4 +86,4 @@ __aicore__ inline void QbmmiaMxBasicApiKernel( | |||
| 87 | MatmulKernel qbmm; | 86 | MatmulKernel qbmm; |
| 88 | qbmm(params); | 87 | qbmm(params); |
| 89 | } | 88 | } |
| 90 | -#endif | 89 | + |
Mmatmul/quant_batch_matmul_inplace_add/op_kernel/arch35/quant_batch_matmul_inplace_add_tiling_data.h+2-3
| @@ -11,8 +11,7 @@ | |||
| 11 | * \file quant_matmul_inplace_add_tiling_data.h | 11 | * \file quant_matmul_inplace_add_tiling_data.h |
| 12 | * \brief | 12 | * \brief |
| 13 | */ | 13 | */ |
| 14 | -#ifndef QUANT_BATCH_MATMUL_INPLACE_ADD_TILING_DATA_H | 14 | +#pragma once |
| 15 | - | ||
| 16 | 15 | ||
| 17 | 16 | ||
| 18 | 17 | ||
| @@ -86,4 +85,4 @@ struct QuantBatchMatmulInplaceAddTilingData { | |||
| 86 | }; | 85 | }; |
| 87 | 86 | ||
| 88 | } // namespace QMMIA | 87 | } // namespace QMMIA |
| 89 | -#endif // QUANT_BATCH_MATMUL_INPLACE_ADD_TILING_DATA_H | 88 | + |
Mmatmul/quant_batch_matmul_inplace_add/op_kernel/arch35/quant_batch_matmul_inplace_add_tiling_key.h+2-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file quant_batch_matmul_inplace_add_tiling_key.h | 12 | * \file quant_batch_matmul_inplace_add_tiling_key.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QBMMIA_ARCH35_TILING_KEY_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -36,4 +35,4 @@ ASCENDC_TPL_SEL(ASCENDC_TPL_ARGS_SEL( | |||
| 36 | KERNELTYPE, ASCENDC_TPL_UI_LIST, TPL_NO_VEC_EPILOGUE_WITH_MMAPI, | 35 | KERNELTYPE, ASCENDC_TPL_UI_LIST, TPL_NO_VEC_EPILOGUE_WITH_MMAPI, |
| 37 | TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI))); | 36 | TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI))); |
| 38 | } // namespace QuantBatchMatmulInplaceAddArch35TilingKey | 37 | } // namespace QuantBatchMatmulInplaceAddArch35TilingKey |
| 39 | -#endif | 38 | + |
Mmatmul/quant_batch_matmul_v3/op_host/op_tiling/arch35/adaptive_sliding_window_basic_api_tiling.h+2-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file adaptive_sliding_window_basic_api_tiling.h | 12 | * \file adaptive_sliding_window_basic_api_tiling.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef ADAPTIVE_SLIDING_WINDOW_BASIC_API_TILING_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -57,4 +56,4 @@ protected: | |||
| 57 | DequantBmm::QuantBatchMatmulV3BasicAPITilingData &tilingData_; | 56 | DequantBmm::QuantBatchMatmulV3BasicAPITilingData &tilingData_; |
| 58 | }; | 57 | }; |
| 59 | } | 58 | } |
| 60 | -#endif // ADAPTIVE_SLIDING_WINDOW_BASIC_API_TILING_H | 59 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file adaptive_sliding_window_tiling.h | 12 | * \file adaptive_sliding_window_tiling.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef ADAPTIVE_SLIDING_WINDOW_TILING_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -147,4 +146,4 @@ protected: | |||
| 147 | uint64_t bCacheLine); | 146 | uint64_t bCacheLine); |
| 148 | }; | 147 | }; |
| 149 | } // namespace optiling | 148 | } // namespace optiling |
| 150 | -#endif // ADAPTIVE_SLIDING_WINDOW_TILING_H | 149 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file quant_batch_matmul_v3_checker.h | 12 | * \file quant_batch_matmul_v3_checker.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QUANT_BATCH_MATMUL_V3_CHECKER_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | namespace optiling { | 18 | namespace optiling { |
| @@ -71,4 +70,4 @@ protected: | |||
| 71 | bool CheckABDtypesSame() const; | 70 | bool CheckABDtypesSame() const; |
| 72 | }; | 71 | }; |
| 73 | } // namespace optiling | 72 | } // namespace optiling |
| 74 | -#endif // QUANT_BATCH_MATMUL_V3_CHECKER_H | 73 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file quant_batch_matmul_v3_checker_base.h | 12 | * \file quant_batch_matmul_v3_checker_base.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QUANT_BATCH_MATMUL_V3_CHECKER_BASE_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -81,4 +80,4 @@ protected: | |||
| 81 | } | 80 | } |
| 82 | }; | 81 | }; |
| 83 | } // namespace optiling | 82 | } // namespace optiling |
| 84 | -#endif // QUANT_BATCH_MATMUL_V3_CHECKER_BASE_H | 83 | + |
Mmatmul/quant_batch_matmul_v3/op_host/op_tiling/arch35/quant_batch_matmul_v3_checker_for_mmads8s4.h+2-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file quant_batch_matmul_v3_checker_for_mmads8s4.h | 12 | * \file quant_batch_matmul_v3_checker_for_mmads8s4.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QUANT_BATCH_MATMUL_V3_CHECKER_FOR_MMADS8S4_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -49,4 +48,4 @@ protected: | |||
| 49 | virtual bool ExtraInputCheck() const; | 48 | virtual bool ExtraInputCheck() const; |
| 50 | }; | 49 | }; |
| 51 | } // namespace optiling | 50 | } // namespace optiling |
| 52 | -#endif // QUANT_BATCH_MATMUL_V3_CHECKER_FOR_MMADS8S4_H | 51 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file quant_batch_matmul_v3_iterbatch_tiling.h | 12 | * \file quant_batch_matmul_v3_iterbatch_tiling.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QUANT_BATCH_MATMUL_V3_ITERBATCH_TILING_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -62,4 +61,4 @@ private: | |||
| 62 | void Reset(); | 61 | void Reset(); |
| 63 | }; | 62 | }; |
| 64 | } | 63 | } |
| 65 | -#endif // QUANT_BATCH_MATMUL_V3_ITERBATCH_TILING_H | 64 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file quant_batch_matmul_v3_tiling_util.h | 12 | * \file quant_batch_matmul_v3_tiling_util.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QUANT_BATCH_MATMUL_V3_TILING_UTIL_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -72,4 +71,4 @@ public: | |||
| 72 | static uint64_t GetBiasMode(const QuantBatchMatmulInfo &inputParams); | 71 | static uint64_t GetBiasMode(const QuantBatchMatmulInfo &inputParams); |
| 73 | }; | 72 | }; |
| 74 | } // namespace optiling | 73 | } // namespace optiling |
| 75 | -#endif | 74 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file qbmm_copy_cube_out.h | 12 | * \file qbmm_copy_cube_out.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QBMM_COPY_CUBE_OUT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | namespace AscendC { | 18 | namespace AscendC { |
| @@ -219,4 +218,4 @@ namespace AscendC { | |||
| 219 | }; | 218 | }; |
| 220 | 219 | ||
| 221 | } // namespace AscendC | 220 | } // namespace AscendC |
| 222 | -#endif // QBMM_COPY_CUBE_OUT_H | 221 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file qbmm_custom_mm_policy.h | 12 | * \file qbmm_custom_mm_policy.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QBMM_CUSTOM_MM_POLICY_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -26,4 +25,4 @@ public: | |||
| 26 | using CopyCubeOut = QBmmCustomCopyCubeOut<IMPL, A_TYPE, B_TYPE, C_TYPE, MM_CFG, McgShfMode::DUAL_DST_SPLIT_M>; | 25 | using CopyCubeOut = QBmmCustomCopyCubeOut<IMPL, A_TYPE, B_TYPE, C_TYPE, MM_CFG, McgShfMode::DUAL_DST_SPLIT_M>; |
| 27 | }; | 26 | }; |
| 28 | } // namespace AscendC | 27 | } // namespace AscendC |
| 29 | -#endif // QBMM_CUSTOM_MM_POLICY_H | 28 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file qbmm_api_utils.h | 12 | * \file qbmm_api_utils.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QBMM_API_UTILS_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -257,4 +256,3 @@ __aicore__ inline void ProcessWithBatch(QuantBmmAswBlock& block, T& object) | |||
| 257 | 256 | ||
| 258 | } // namespace QuantBatchMatmulV3 | 257 | } // namespace QuantBatchMatmulV3 |
| 259 | 258 | ||
| 260 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file qbmm_asw_block.h | 12 | * \file qbmm_asw_block.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QBMM_ASW_BLOCK_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -519,4 +518,3 @@ __aicore__ inline void QuantBmmAswBlock::UpdatePerBlockMmParam() | |||
| 519 | } | 518 | } |
| 520 | } // namespace QuantBatchMatmulV3 | 519 | } // namespace QuantBatchMatmulV3 |
| 521 | 520 | ||
| 522 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file qbmm_cube_basic_api_cmct.h | 12 | * \file qbmm_cube_basic_api_cmct.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QBMM_CUBE_BAISC_API_CMCT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -95,4 +94,4 @@ __aicore__ inline void QbmmCubeBasicApiKernel( | |||
| 95 | MatmulKernel qbmm; | 94 | MatmulKernel qbmm; |
| 96 | qbmm(params); | 95 | qbmm(params); |
| 97 | } | 96 | } |
| 98 | -#endif | 97 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef QBMM_CUBE_ON_THE_FLY_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -304,4 +303,4 @@ __aicore__ inline void MatMulASWKernel<LOCAL_TEMPLATE_FUNC_PARAMS>::SetMMParaAnd | |||
| 304 | mm_.GetTensorC(cGlobal_[block_.offset_.offsetC]); | 303 | mm_.GetTensorC(cGlobal_[block_.offset_.offsetC]); |
| 305 | } | 304 | } |
| 306 | } | 305 | } |
| 307 | -#endif // QBMM_CUBE_ON_THE_FLY_H | 306 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file qbmm_cube_on_the_fly_abl1_full_load.h | 12 | * \file qbmm_cube_on_the_fly_abl1_full_load.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QBMM_CUBE_ON_THE_FLY_ABL1_FULL_LOAD_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -209,4 +208,3 @@ __aicore__ inline void MatmulAswKernelABL1FullLoad<LOCAL_TEMPLATE_FUNC_PARAMS>:: | |||
| 209 | 208 | ||
| 210 | } // namespace QuantBatchMatmulV3 | 209 | } // namespace QuantBatchMatmulV3 |
| 211 | 210 | ||
| 212 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file qbmm_cube_on_the_fly_al1_full_load.h | 12 | * \file qbmm_cube_on_the_fly_al1_full_load.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QBMM_CUBE_ON_THE_FLY_AL1_FULL_LOAD_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -258,4 +257,3 @@ __aicore__ inline void MatmulAswKernelAL1FullLoad<LOCAL_TEMPLATE_FUNC_PARAMS>::P | |||
| 258 | 257 | ||
| 259 | } // namespace QuantBatchMatmulV3 | 258 | } // namespace QuantBatchMatmulV3 |
| 260 | 259 | ||
| 261 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file qbmm_cube_on_the_fly_bl1_full_load.h | 12 | * \file qbmm_cube_on_the_fly_bl1_full_load.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QBMM_CUBE_ON_THE_FLY_BL1_FULL_LOAD_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -164,4 +163,3 @@ __aicore__ inline void MatmulAswKernelBL1FullLoad<LOCAL_TEMPLATE_FUNC_PARAMS>::P | |||
| 164 | } | 163 | } |
| 165 | } // namespace QuantBatchMatmulV3 | 164 | } // namespace QuantBatchMatmulV3 |
| 166 | 165 | ||
| 167 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef QBMM_CUBE_ON_THE_FLY_ITERBATCH_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -173,4 +172,4 @@ __aicore__ inline void QbmmIterBatchKernel<LOCAL_TEMPLATE_FUNC_PARAMS_V2>::Proce | |||
| 173 | } | 172 | } |
| 174 | } | 173 | } |
| 175 | } // namespace QuantBatchMatmulV3 | 174 | } // namespace QuantBatchMatmulV3 |
| 176 | -#endif // QBMM_CUBE_ON_THE_FLY_ITERBATCH_H | 175 | + |
| @@ -1,179 +1,177 @@ | |||
| 1 | -/** | 1 | +/** |
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. |
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of |
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). |
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. |
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, |
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | 8 | + * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | - */ | 9 | + */ |
| 10 | - | 10 | + |
| 11 | -/*! | 11 | +/*! |
| 12 | - * \file qbmm_int4_to_int8_preprocess.h | 12 | + * \file qbmm_int4_to_int8_preprocess.h |
| 13 | - * \brief Preprocess class for converting int4 inputs (x1, x2) to int8 before matrix multiplication. | 13 | + * \brief Preprocess class for converting int4 inputs (x1, x2) to int8 before matrix multiplication. |
| 14 | - * Core splitting: cores are split between x1 and x2 by m:n ratio. | 14 | + * Core splitting: cores are split between x1 and x2 by m:n ratio. |
| 15 | - * Each core processes ONLY x1 or x2. | 15 | + * Each core processes ONLY x1 or x2. |
| 16 | - * Pipeline: CopyIn -> Compute (int4->half->int8) -> CopyOut | 16 | + * Pipeline: CopyIn -> Compute (int4->half->int8) -> CopyOut |
| 17 | - */ | 17 | + */ |
| 18 | - | 18 | + |
| 19 | -#ifndef QBMM_INT4_TO_INT8_PREPROCESS_H | 19 | +#pragma once |
| 20 | -#define QBMM_INT4_TO_INT8_PREPROCESS_H | 20 | + |
| 21 | - | 21 | +#include "../quant_batch_matmul_v3_base.h" |
| 22 | -#include "../quant_batch_matmul_v3_base.h" | 22 | +#include "quant_batch_matmul_v3_tiling_data.h" |
| 23 | -#include "quant_batch_matmul_v3_tiling_data.h" | 23 | + |
| 24 | - | 24 | +using namespace AscendC; |
| 25 | -using namespace AscendC; | 25 | + |
| 26 | - | 26 | +namespace { |
| 27 | -namespace { | 27 | +constexpr uint64_t ALIGN_SIZE_128 = 128u; |
| 28 | -constexpr uint64_t ALIGN_SIZE_128 = 128u; | 28 | +constexpr uint64_t TILE_ELEMS_16K = 16 * 1024u; |
| 29 | -constexpr uint64_t TILE_ELEMS_16K = 16 * 1024u; | 29 | +constexpr uint32_t ELEM_ALIGN_64 = 64u; |
| 30 | -constexpr uint32_t ELEM_ALIGN_64 = 64u; | 30 | +constexpr uint64_t NUM_2 = 2u; |
| 31 | -constexpr uint64_t NUM_2 = 2u; | 31 | +} |
| 32 | -} | 32 | + |
| 33 | - | 33 | +class QbmmInt4ToInt8Preprocess { |
| 34 | -class QbmmInt4ToInt8Preprocess { | 34 | +public: |
| 35 | -public: | 35 | + __aicore__ inline QbmmInt4ToInt8Preprocess() {} |
| 36 | - __aicore__ inline QbmmInt4ToInt8Preprocess() {} | 36 | + |
| 37 | - | 37 | + __aicore__ inline void Init(GM_ADDR x1In, GM_ADDR x2In, GM_ADDR workspace, |
| 38 | - __aicore__ inline void Init(GM_ADDR x1In, GM_ADDR x2In, GM_ADDR workspace, | 38 | + TPipe& pipe, uint64_t m, uint64_t n, |
| 39 | - TPipe& pipe, uint64_t m, uint64_t n, | 39 | + uint64_t k, uint64_t batchC); |
| 40 | - uint64_t k, uint64_t batchC); | 40 | + __aicore__ inline void Process(); |
| 41 | - __aicore__ inline void Process(); | 41 | + |
| 42 | - | 42 | +private: |
| 43 | -private: | 43 | + __aicore__ inline void CopyIn(uint32_t progress, uint32_t currentNum); |
| 44 | - __aicore__ inline void CopyIn(uint32_t progress, uint32_t currentNum); | 44 | + __aicore__ inline void Compute(uint32_t currentNum); |
| 45 | - __aicore__ inline void Compute(uint32_t currentNum); | 45 | + __aicore__ inline void CopyOut(uint32_t progress, uint32_t currentNum); |
| 46 | - __aicore__ inline void CopyOut(uint32_t progress, uint32_t currentNum); | 46 | + |
| 47 | - | 47 | +private: |
| 48 | -private: | 48 | + // Queues |
| 49 | - // Queues | 49 | + TQue<QuePosition::VECIN, BUFFER_NUM> inQueueInt4_; |
| 50 | - TQue<QuePosition::VECIN, BUFFER_NUM> inQueueInt4_; | 50 | + TQue<QuePosition::VECCALC, BUFFER_NUM> computeQueueHalf_; |
| 51 | - TQue<QuePosition::VECCALC, BUFFER_NUM> computeQueueHalf_; | 51 | + TQue<QuePosition::VECOUT, BUFFER_NUM> outQueueInt8_; |
| 52 | - TQue<QuePosition::VECOUT, BUFFER_NUM> outQueueInt8_; | 52 | + |
| 53 | - | 53 | + GlobalTensor<int8_t> srcInt4Global_; |
| 54 | - GlobalTensor<int8_t> srcInt4Global_; | 54 | + GlobalTensor<int8_t> dstInt8Global_; |
| 55 | - GlobalTensor<int8_t> dstInt8Global_; | 55 | + |
| 56 | - | 56 | + GM_ADDR x1Out_; |
| 57 | - GM_ADDR x1Out_; | 57 | + GM_ADDR x2Out_; |
| 58 | - GM_ADDR x2Out_; | 58 | + |
| 59 | - | 59 | + bool isX1Core_; |
| 60 | - bool isX1Core_; | 60 | + uint64_t blockLength_ = 0; |
| 61 | - uint64_t blockLength_ = 0; | 61 | + uint32_t ubLength_ = 0; |
| 62 | - uint32_t ubLength_ = 0; | 62 | +}; |
| 63 | -}; | 63 | + |
| 64 | - | 64 | +__aicore__ inline void QbmmInt4ToInt8Preprocess::Init(GM_ADDR x1In, GM_ADDR x2In, GM_ADDR workspace, |
| 65 | -__aicore__ inline void QbmmInt4ToInt8Preprocess::Init(GM_ADDR x1In, GM_ADDR x2In, GM_ADDR workspace, | 65 | + TPipe& pipe, uint64_t m, uint64_t n, uint64_t k, |
| 66 | - TPipe& pipe, uint64_t m, uint64_t n, uint64_t k, | 66 | + uint64_t batchC) |
| 67 | - uint64_t batchC) | 67 | +{ |
| 68 | -{ | 68 | + uint64_t x1TotalElems = batchC * m * k; |
| 69 | - uint64_t x1TotalElems = batchC * m * k; | 69 | + uint64_t x2TotalElems = k * n; |
| 70 | - uint64_t x2TotalElems = k * n; | 70 | + |
| 71 | - | 71 | + x1Out_ = workspace; |
| 72 | - x1Out_ = workspace; | 72 | + x2Out_ = workspace + DequantBmm::Align(x1TotalElems * sizeof(int8_t), ALIGN_SIZE_128); |
| 73 | - x2Out_ = workspace + DequantBmm::Align(x1TotalElems * sizeof(int8_t), ALIGN_SIZE_128); | 73 | + |
| 74 | - | 74 | + // ---- core assignment by m:n ratio ---- |
| 75 | - // ---- core assignment by m:n ratio ---- | 75 | + uint64_t totalCores = GetBlockNum(); |
| 76 | - uint64_t totalCores = GetBlockNum(); | 76 | + if ASCEND_IS_AIV { |
| 77 | - if ASCEND_IS_AIV { | 77 | + totalCores = totalCores * NUM_2; |
| 78 | - totalCores = totalCores * NUM_2; | 78 | + } |
| 79 | - } | 79 | + uint64_t coresForX1 = (totalCores * batchC * m + (batchC * m + n) / NUM_2) / (batchC * m + n); |
| 80 | - uint64_t coresForX1 = (totalCores * batchC * m + (batchC * m + n) / NUM_2) / (batchC * m + n); | 80 | + if (coresForX1 < 1) coresForX1 = 1; |
| 81 | - if (coresForX1 < 1) coresForX1 = 1; | 81 | + if (coresForX1 >= totalCores) coresForX1 = totalCores - 1; |
| 82 | - if (coresForX1 >= totalCores) coresForX1 = totalCores - 1; | 82 | + uint64_t coresForX2 = totalCores - coresForX1; |
| 83 | - uint64_t coresForX2 = totalCores - coresForX1; | 83 | + |
| 84 | - | 84 | + uint64_t coreIdx = GetBlockIdx(); |
| 85 | - uint64_t coreIdx = GetBlockIdx(); | 85 | + if (coreIdx >= totalCores) { |
| 86 | - if (coreIdx >= totalCores) { | 86 | + return; |
| 87 | - return; | 87 | + } |
| 88 | - } | 88 | + isX1Core_ = (coreIdx < coresForX1); |
| 89 | - isX1Core_ = (coreIdx < coresForX1); | 89 | + |
| 90 | - | 90 | + // ---- compute this core's element range ---- |
| 91 | - // ---- compute this core's element range ---- | 91 | + uint64_t totalElems = isX1Core_ ? x1TotalElems : x2TotalElems; |
| 92 | - uint64_t totalElems = isX1Core_ ? x1TotalElems : x2TotalElems; | 92 | + uint64_t groupCores = isX1Core_ ? coresForX1 : coresForX2; |
| 93 | - uint64_t groupCores = isX1Core_ ? coresForX1 : coresForX2; | 93 | + uint64_t localId = isX1Core_ ? coreIdx : coreIdx - coresForX1; |
| 94 | - uint64_t localId = isX1Core_ ? coreIdx : coreIdx - coresForX1; | 94 | + |
| 95 | - | 95 | + uint64_t totalBlocks = DequantBmm::Align(totalElems, static_cast<uint64_t>(ELEM_ALIGN_64)); |
| 96 | - uint64_t totalBlocks = DequantBmm::Align(totalElems, static_cast<uint64_t>(ELEM_ALIGN_64)); | 96 | + uint64_t numChunks = totalBlocks / ELEM_ALIGN_64; |
| 97 | - uint64_t numChunks = totalBlocks / ELEM_ALIGN_64; | 97 | + uint64_t baseChunks = numChunks / groupCores; |
| 98 | - uint64_t baseChunks = numChunks / groupCores; | 98 | + uint64_t remainChunks = numChunks % groupCores; |
| 99 | - uint64_t remainChunks = numChunks % groupCores; | 99 | + |
| 100 | - | 100 | + uint64_t chunkStart = localId * baseChunks + DequantBmm::Min(localId, remainChunks); |
| 101 | - uint64_t chunkStart = localId * baseChunks + DequantBmm::Min(localId, remainChunks); | 101 | + uint64_t chunkEnd = chunkStart + baseChunks + (localId < remainChunks ? 1u : 0u); |
| 102 | - uint64_t chunkEnd = chunkStart + baseChunks + (localId < remainChunks ? 1u : 0u); | 102 | + |
| 103 | - | 103 | + uint64_t elemStart = chunkStart * ELEM_ALIGN_64; |
| 104 | - uint64_t elemStart = chunkStart * ELEM_ALIGN_64; | 104 | + uint64_t elemEnd = chunkEnd * ELEM_ALIGN_64; |
| 105 | - uint64_t elemEnd = chunkEnd * ELEM_ALIGN_64; | 105 | + |
| 106 | - | 106 | + blockLength_ = elemEnd - elemStart; |
| 107 | - blockLength_ = elemEnd - elemStart; | 107 | + |
| 108 | - | 108 | + uint64_t tileSize = DequantBmm::Min(blockLength_, TILE_ELEMS_16K); |
| 109 | - uint64_t tileSize = DequantBmm::Min(blockLength_, TILE_ELEMS_16K); | 109 | + uint64_t alignUb = DequantBmm::Align(tileSize, static_cast<uint64_t>(ELEM_ALIGN_64)); |
| 110 | - uint64_t alignUb = DequantBmm::Align(tileSize, static_cast<uint64_t>(ELEM_ALIGN_64)); | 110 | + ubLength_ = DequantBmm::Max(ELEM_ALIGN_64, static_cast<uint32_t>(alignUb)); |
| 111 | - ubLength_ = DequantBmm::Max(ELEM_ALIGN_64, static_cast<uint32_t>(alignUb)); | 111 | + |
| 112 | - | 112 | + // ---- bind global buffers to this core's slice ---- |
| 113 | - // ---- bind global buffers to this core's slice ---- | 113 | + if (isX1Core_) { |
| 114 | - if (isX1Core_) { | 114 | + srcInt4Global_.SetGlobalBuffer((__gm__ int8_t*)x1In + elemStart / NUM_2, blockLength_ / NUM_2); |
| 115 | - srcInt4Global_.SetGlobalBuffer((__gm__ int8_t*)x1In + elemStart / NUM_2, blockLength_ / NUM_2); | 115 | + dstInt8Global_.SetGlobalBuffer((__gm__ int8_t*)x1Out_ + elemStart, blockLength_); |
| 116 | - dstInt8Global_.SetGlobalBuffer((__gm__ int8_t*)x1Out_ + elemStart, blockLength_); | 116 | + } else { |
| 117 | - } else { | 117 | + srcInt4Global_.SetGlobalBuffer((__gm__ int8_t*)x2In + elemStart / NUM_2, blockLength_ / NUM_2); |
| 118 | - srcInt4Global_.SetGlobalBuffer((__gm__ int8_t*)x2In + elemStart / NUM_2, blockLength_ / NUM_2); | 118 | + dstInt8Global_.SetGlobalBuffer((__gm__ int8_t*)x2Out_ + elemStart, blockLength_); |
| 119 | - dstInt8Global_.SetGlobalBuffer((__gm__ int8_t*)x2Out_ + elemStart, blockLength_); | 119 | + } |
| 120 | - } | 120 | + |
| 121 | - | 121 | + // ---- init queues ---- |
| 122 | - // ---- init queues ---- | 122 | + pipe.InitBuffer(inQueueInt4_, BUFFER_NUM, static_cast<uint32_t>(ubLength_) / NUM_2); |
| 123 | - pipe.InitBuffer(inQueueInt4_, BUFFER_NUM, static_cast<uint32_t>(ubLength_) / NUM_2); | 123 | + pipe.InitBuffer(computeQueueHalf_, BUFFER_NUM, static_cast<uint32_t>(ubLength_) * sizeof(half)); |
| 124 | - pipe.InitBuffer(computeQueueHalf_, BUFFER_NUM, static_cast<uint32_t>(ubLength_) * sizeof(half)); | 124 | + pipe.InitBuffer(outQueueInt8_, BUFFER_NUM, static_cast<uint32_t>(ubLength_) * sizeof(int8_t)); |
| 125 | - pipe.InitBuffer(outQueueInt8_, BUFFER_NUM, static_cast<uint32_t>(ubLength_) * sizeof(int8_t)); | 125 | +} |
| 126 | -} | 126 | + |
| 127 | - | 127 | +__aicore__ inline void QbmmInt4ToInt8Preprocess::Process() |
| 128 | -__aicore__ inline void QbmmInt4ToInt8Preprocess::Process() | 128 | +{ |
| 129 | -{ | 129 | + if (blockLength_ == 0) return; |
| 130 | - if (blockLength_ == 0) return; | 130 | + |
| 131 | - | 131 | + uint64_t loopCount = DequantBmm::CeilDiv(blockLength_, static_cast<uint64_t>(ubLength_)); |
| 132 | - uint64_t loopCount = DequantBmm::CeilDiv(blockLength_, static_cast<uint64_t>(ubLength_)); | 132 | + for (uint32_t i = 0; i < loopCount; i++) { |
| 133 | - for (uint32_t i = 0; i < loopCount; i++) { | 133 | + uint64_t remaining = blockLength_ - ubLength_ * i; |
| 134 | - uint64_t remaining = blockLength_ - ubLength_ * i; | 134 | + uint32_t currentNum = DequantBmm::Min(static_cast<uint32_t>(remaining), ubLength_); |
| 135 | - uint32_t currentNum = DequantBmm::Min(static_cast<uint32_t>(remaining), ubLength_); | 135 | + |
| 136 | - | 136 | + currentNum = DequantBmm::FloorAlign(currentNum, ELEM_ALIGN_64); |
| 137 | - currentNum = DequantBmm::FloorAlign(currentNum, ELEM_ALIGN_64); | 137 | + if (currentNum == 0) break; |
| 138 | - if (currentNum == 0) break; | 138 | + |
| 139 | - | 139 | + CopyIn(i, currentNum); |
| 140 | - CopyIn(i, currentNum); | 140 | + Compute(currentNum); |
| 141 | - Compute(currentNum); | 141 | + CopyOut(i, currentNum); |
| 142 | - CopyOut(i, currentNum); | 142 | + } |
| 143 | - } | 143 | +} |
| 144 | -} | 144 | + |
| 145 | - | 145 | +__aicore__ inline void QbmmInt4ToInt8Preprocess::CopyIn(uint32_t progress, uint32_t currentNum) |
| 146 | -__aicore__ inline void QbmmInt4ToInt8Preprocess::CopyIn(uint32_t progress, uint32_t currentNum) | 146 | +{ |
| 147 | -{ | 147 | + LocalTensor<int8_t> int4Local = inQueueInt4_.AllocTensor<int8_t>(); |
| 148 | - LocalTensor<int8_t> int4Local = inQueueInt4_.AllocTensor<int8_t>(); | 148 | + DataCopy(int4Local, srcInt4Global_[progress * ubLength_ / NUM_2], static_cast<uint32_t>(currentNum) / NUM_2); |
| 149 | - DataCopy(int4Local, srcInt4Global_[progress * ubLength_ / NUM_2], static_cast<uint32_t>(currentNum) / NUM_2); | 149 | + inQueueInt4_.EnQue<int8_t>(int4Local); |
| 150 | - inQueueInt4_.EnQue<int8_t>(int4Local); | 150 | +} |
| 151 | -} | 151 | + |
| 152 | - | 152 | +__aicore__ inline void QbmmInt4ToInt8Preprocess::Compute(uint32_t currentNum) |
| 153 | -__aicore__ inline void QbmmInt4ToInt8Preprocess::Compute(uint32_t currentNum) | 153 | +{ |
| 154 | -{ | 154 | + LocalTensor<int8_t> int4Local = inQueueInt4_.DeQue<int8_t>(); |
| 155 | - LocalTensor<int8_t> int4Local = inQueueInt4_.DeQue<int8_t>(); | 155 | + LocalTensor<int4b_t> int4View = int4Local.ReinterpretCast<int4b_t>(); |
| 156 | - LocalTensor<int4b_t> int4View = int4Local.ReinterpretCast<int4b_t>(); | 156 | + |
| 157 | - | 157 | + // ---- int4 -> half ---- |
| 158 | - // ---- int4 -> half ---- | 158 | + LocalTensor<half> halfLocal = computeQueueHalf_.AllocTensor<half>(); |
| 159 | - LocalTensor<half> halfLocal = computeQueueHalf_.AllocTensor<half>(); | 159 | + Cast<half, int4b_t>(halfLocal, int4View, RoundMode::CAST_NONE, static_cast<uint32_t>(currentNum)); |
| 160 | - Cast<half, int4b_t>(halfLocal, int4View, RoundMode::CAST_NONE, static_cast<uint32_t>(currentNum)); | 160 | + inQueueInt4_.FreeTensor(int4Local); |
| 161 | - inQueueInt4_.FreeTensor(int4Local); | 161 | + computeQueueHalf_.EnQue<half>(halfLocal); |
| 162 | - computeQueueHalf_.EnQue<half>(halfLocal); | 162 | + |
| 163 | - | 163 | + // ---- half -> int8 ---- |
| 164 | - // ---- half -> int8 ---- | 164 | + LocalTensor<half> halfSrc = computeQueueHalf_.DeQue<half>(); |
| 165 | - LocalTensor<half> halfSrc = computeQueueHalf_.DeQue<half>(); | 165 | + LocalTensor<int8_t> int8Local = outQueueInt8_.AllocTensor<int8_t>(); |
| 166 | - LocalTensor<int8_t> int8Local = outQueueInt8_.AllocTensor<int8_t>(); | 166 | + Cast<int8_t, half>(int8Local, halfSrc, RoundMode::CAST_ROUND, static_cast<uint32_t>(currentNum)); |
| 167 | - Cast<int8_t, half>(int8Local, halfSrc, RoundMode::CAST_ROUND, static_cast<uint32_t>(currentNum)); | 167 | + computeQueueHalf_.FreeTensor(halfSrc); |
| 168 | - computeQueueHalf_.FreeTensor(halfSrc); | 168 | + outQueueInt8_.EnQue<int8_t>(int8Local); |
| 169 | - outQueueInt8_.EnQue<int8_t>(int8Local); | 169 | +} |
| 170 | -} | 170 | + |
| 171 | - | 171 | +__aicore__ inline void QbmmInt4ToInt8Preprocess::CopyOut(uint32_t progress, uint32_t currentNum) |
| 172 | -__aicore__ inline void QbmmInt4ToInt8Preprocess::CopyOut(uint32_t progress, uint32_t currentNum) | 172 | +{ |
| 173 | -{ | 173 | + LocalTensor<int8_t> int8Local = outQueueInt8_.DeQue<int8_t>(); |
| 174 | - LocalTensor<int8_t> int8Local = outQueueInt8_.DeQue<int8_t>(); | 174 | + DataCopy(dstInt8Global_[progress * ubLength_], int8Local, static_cast<uint32_t>(currentNum * sizeof(int8_t))); |
| 175 | - DataCopy(dstInt8Global_[progress * ubLength_], int8Local, static_cast<uint32_t>(currentNum * sizeof(int8_t))); | 175 | + outQueueInt8_.FreeTensor(int8Local); |
| 176 | - outQueueInt8_.FreeTensor(int8Local); | 176 | +} |
| 177 | -} | 177 | + |
| 178 | - | ||
| 179 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef QBMM_ITERBATCH_BLOCK_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -251,4 +250,3 @@ __aicore__ inline void QbmmMultiBatchBaseBlock::CalcGMOffset() | |||
| 251 | } | 250 | } |
| 252 | } // namespace QuantBatchMatmulV3 | 251 | } // namespace QuantBatchMatmulV3 |
| 253 | 252 | ||
| 254 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file qbmm_mix_online_dynamic.h | 12 | * \file qbmm_mix_online_dynamic.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QBMM_MIX_ONLINE_DYNAMIC_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -707,4 +706,3 @@ __aicore__ inline void QuantBmmPertokenRegbaseKernel<LOCAL_TEMPLATE_FUNC_MIX_PAR | |||
| 707 | 706 | ||
| 708 | } // namespace QuantBatchMatmulV3 | 707 | } // namespace QuantBatchMatmulV3 |
| 709 | 708 | ||
| 710 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file qbmm_mix_online_dynamic_al1_full_load.h | 12 | * \file qbmm_mix_online_dynamic_al1_full_load.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QBMM_MIX_ONLINE_DYNAMIC_AL1_FULL_LOAD_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -211,4 +210,3 @@ __aicore__ inline void QuantBmmPertokenAL1FullLoad<LOCAL_TEMPLATE_FUNC_MIX_PARAM | |||
| 211 | 210 | ||
| 212 | } // namespace QuantBatchMatmulV3 | 211 | } // namespace QuantBatchMatmulV3 |
| 213 | 212 | ||
| 214 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef QBMM_MIX_PERBLOCK_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -92,4 +91,4 @@ MatMulPerBlockASW<x1Type, x2Type, biasType, yType, formatX1, formatX2, formatY, | |||
| 92 | } | 91 | } |
| 93 | } | 92 | } |
| 94 | } // namespace QuantBatchMatmulV3 | 93 | } // namespace QuantBatchMatmulV3 |
| 95 | -#endif // QBMM_MIX_PERBLOCK_H | 94 | + |
| @@ -8,8 +8,7 @@ | |||
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | -#ifndef QBMM_MIX_PERTILE_CMCT_H | 11 | +#pragma once |
| 12 | - | ||
| 13 | 12 | ||
| 14 | 13 | ||
| 15 | 14 | ||
| @@ -94,4 +93,4 @@ __aicore__ inline void QbmmCmctPertileKernel( | |||
| 94 | QbmmKernel qbmm; | 93 | QbmmKernel qbmm; |
| 95 | qbmm(params); | 94 | qbmm(params); |
| 96 | } | 95 | } |
| 97 | -#endif | 96 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file qbmm_mx_basic_api_cmct.h | 12 | * \file qbmm_mx_basic_api_cmct.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QBMM_MX_BAISC_API_CMCT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -87,4 +86,4 @@ __aicore__ inline void QbmmMxBasicApiKernel( | |||
| 87 | MatmulKernel qbmm; | 86 | MatmulKernel qbmm; |
| 88 | qbmm(params); | 87 | qbmm(params); |
| 89 | } | 88 | } |
| 90 | -#endif | 89 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | - #ifndef QBMM_PERBLOCK_API_PARAM_UTILS_H | 16 | + #pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -239,4 +238,4 @@ | |||
| 239 | } | 238 | } |
| 240 | } | 239 | } |
| 241 | } // namespace QuantBatchMatmulV3 | 240 | } // namespace QuantBatchMatmulV3 |
| 242 | - #endif // QBMM_PERBLOCK_API_PARAM_UTILS_H | 241 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \brief | 12 | * \brief |
| 13 | */ | 13 | */ |
| 14 | 14 | ||
| 15 | -#ifndef QBMM_PERBLOCK_API_UTILS_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -783,4 +782,4 @@ __aicore__ inline void MatMulPerBlock<MATMUL_PERBLOCK_FUNC_PARAMS>::AicEnd() | |||
| 783 | } | 782 | } |
| 784 | 783 | ||
| 785 | } // namespace QuantBatchMatmulV3 | 784 | } // namespace QuantBatchMatmulV3 |
| 786 | -#endif // QBMM_PERBLOCK_API_UTILS_H | 785 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file quant_batch_matmul_v3_apt_tiling_key.h | 12 | * \file quant_batch_matmul_v3_apt_tiling_key.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QBMMV3_ARCH35_TILING_KEY_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -119,4 +118,4 @@ ASCENDC_TPL_SEL( | |||
| 119 | TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI)) | 118 | TPL_NO_VEC_EPILOGUE_CUSTOM_GMTOAL1_WITH_MMAPI)) |
| 120 | ); | 119 | ); |
| 121 | } // namespace QuantBatchMatmulV3_Arch35_TilingKey | 120 | } // namespace QuantBatchMatmulV3_Arch35_TilingKey |
| 122 | -#endif | 121 | + |
| @@ -11,8 +11,7 @@ | |||
| 11 | * \file quant_batch_matmul_v3_tiling_data.h | 11 | * \file quant_batch_matmul_v3_tiling_data.h |
| 12 | * \brief | 12 | * \brief |
| 13 | */ | 13 | */ |
| 14 | -#ifndef QUANT_BATCH_MATMUL_V3_TILING_DATA_H | 14 | +#pragma once |
| 15 | - | ||
| 16 | 15 | ||
| 17 | 16 | ||
| 18 | 17 | ||
| @@ -145,4 +144,4 @@ struct QuantBatchMatmulV3BasicAPITilingData { | |||
| 145 | }; | 144 | }; |
| 146 | 145 | ||
| 147 | } // namespace DequantBmm | 146 | } // namespace DequantBmm |
| 148 | -#endif // QUANT_BATCH_MATMUL_V3_TILING_DATA_H | 147 | + |
Mmatmul/quant_batch_matmul_v4/op_host/op_tiling/arch35/adaptive_sliding_window_basic_api_v4_tiling.h+2-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file adaptive_sliding_window_basic_api_v4_tiling.h | 12 | * \file adaptive_sliding_window_basic_api_v4_tiling.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef ADAPTIVE_SLIDING_WINDOW_BASIC_API_V4_TILING_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -82,4 +81,4 @@ protected: | |||
| 82 | std::unique_ptr<QuantBatchMatmulV4CompileInfo> compileInfoPtr_; | 81 | std::unique_ptr<QuantBatchMatmulV4CompileInfo> compileInfoPtr_; |
| 83 | }; | 82 | }; |
| 84 | } // namespace optiling | 83 | } // namespace optiling |
| 85 | -#endif // ADAPTIVE_SLIDING_WINDOW_BASIC_API_V4_TILING_H | 84 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file quant_batch_matmul_v4_asw_tiling.h | 12 | * \file quant_batch_matmul_v4_asw_tiling.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QUANT_BATCH_MATMUL_V4_ASW_TILING_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -87,4 +86,4 @@ protected: | |||
| 87 | std::unique_ptr<QuantBatchMatmulV4CompileInfo> compileInfoPtr_; | 86 | std::unique_ptr<QuantBatchMatmulV4CompileInfo> compileInfoPtr_; |
| 88 | }; | 87 | }; |
| 89 | } // namespace optiling | 88 | } // namespace optiling |
| 90 | -#endif // QUANT_BATCH_MATMUL_V4_ASW_TILING_H | 89 | + |
Mmatmul/quant_batch_matmul_v4/op_host/op_tiling/arch35/quant_batch_matmul_v4_basic_block_tiling.h+1-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef QUANT_BATCH_MATMUL_V4_BASIC_BLOCK_TILING_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -336,4 +335,3 @@ protected: | |||
| 336 | } // namespace matmul_v4 | 335 | } // namespace matmul_v4 |
| 337 | } // namespace optiling | 336 | } // namespace optiling |
| 338 | 337 | ||
| 339 | - | ||
Mmatmul/quant_batch_matmul_v4/op_host/op_tiling/arch35/quant_batch_matmul_v4_checker_for_mmads8s4.h+2-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file quant_batch_matmul_v4_checker_for_mmads8s4.h | 12 | * \file quant_batch_matmul_v4_checker_for_mmads8s4.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QUANT_BATCH_MATMUL_V4_CHECKER_FOR_MMADS8S4_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | namespace optiling { | 18 | namespace optiling { |
| @@ -74,4 +73,4 @@ protected: | |||
| 74 | bool CheckOffset(const gert::StorageShape* offsetShape) const override; | 73 | bool CheckOffset(const gert::StorageShape* offsetShape) const override; |
| 75 | }; | 74 | }; |
| 76 | } // namespace optiling | 75 | } // namespace optiling |
| 77 | -#endif // QUANT_BATCH_MATMUL_V4_CHECKER_FOR_MMADS8S4_H | 76 | + |
Mmatmul/quant_batch_matmul_v4/op_host/op_tiling/arch35/quant_batch_matmul_v4_pergroup_arch35_tiling.h+1-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \brief | 12 | * \brief |
| 13 | */ | 13 | */ |
| 14 | 14 | ||
| 15 | -#ifndef QUANT_BATCH_MATMUL_V4_PERGROUP_ARCH35_TILING_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -66,4 +65,3 @@ private: | |||
| 66 | }; | 65 | }; |
| 67 | } // namespace optiling | 66 | } // namespace optiling |
| 68 | 67 | ||
| 69 | - | ||
| @@ -14,8 +14,7 @@ | |||
| 14 | * \brief | 14 | * \brief |
| 15 | */ | 15 | */ |
| 16 | 16 | ||
| 17 | -#ifndef QUANT_BATCH_MATMUL_V4_TILING_H | 17 | +#pragma once |
| 18 | - | ||
| 19 | 18 | ||
| 20 | 19 | ||
| 21 | 20 | ||
| @@ -289,4 +288,4 @@ protected: | |||
| 289 | QuantBatchMatmulV4BasicBlockTiling tilingSolver_; | 288 | QuantBatchMatmulV4BasicBlockTiling tilingSolver_; |
| 290 | }; | 289 | }; |
| 291 | } // namespace optiling | 290 | } // namespace optiling |
| 292 | -#endif // QUANT_BATCH_MATMUL_V4_TILING_H | 291 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \brief | 12 | * \brief |
| 13 | */ | 13 | */ |
| 14 | 14 | ||
| 15 | -#ifndef QUANT_BATCH_MATMUL_V4_CMCT_BLOCK_BLOCK_MMAD_MX_WEIGHT_FROM_UB_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -62,4 +61,4 @@ public: | |||
| 62 | }; | 61 | }; |
| 63 | } // namespace Block | 62 | } // namespace Block |
| 64 | } // namespace QuantBatchMatmulV4 | 63 | } // namespace QuantBatchMatmulV4 |
| 65 | -#endif | 64 | + |
Mmatmul/quant_batch_matmul_v4/op_kernel/arch35/cmct/block/block_scheduler_swizzle_in_mn_core_nn.h+2-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef QUANT_BATCH_MATMUL_V4_ARCH35_CMCT_BLOCK_BLOCK_SCHEDULER_SWIZZLE_IN_MN_CORE_NN_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | /* | 19 | /* |
| @@ -63,4 +62,4 @@ public: | |||
| 63 | }; | 62 | }; |
| 64 | } // namespace Block | 63 | } // namespace Block |
| 65 | } // namespace QuantBatchMatmulV4 | 64 | } // namespace QuantBatchMatmulV4 |
| 66 | -#endif | 65 | + |
Mmatmul/quant_batch_matmul_v4/op_kernel/arch35/cmct/kernel/kernel_matmul_mix_with_weight_prologue_nn.h+2-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef QUANT_BATCH_MATMUL_V4_ARCH35_CMCT_KERNEL_KERNEL_MATMUL_MIX_WITH_WEIGHT_PROLOGUE_NN_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -46,4 +45,4 @@ public: | |||
| 46 | }; | 45 | }; |
| 47 | } // namespace Kernel | 46 | } // namespace Kernel |
| 48 | } // namespace QuantBatchMatmulV4 | 47 | } // namespace QuantBatchMatmulV4 |
| 49 | -#endif | 48 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef QUANT_BATCH_MATMUL_V4_ARCH35_CMCT_PROLOGUE_BLOCK_PROLOGUE_B_CAST_SCSC_NN_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -55,4 +54,3 @@ public: | |||
| 55 | } // namespace Prologue | 54 | } // namespace Prologue |
| 56 | } // namespace QuantBatchMatmulV4 | 55 | } // namespace QuantBatchMatmulV4 |
| 57 | 56 | ||
| 58 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef QUANT_BATCH_MATMUL_V4_ARCH35_CMCT_CONVERTOR_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -168,4 +167,3 @@ __aicore__ inline void InvokeKernel( | |||
| 168 | 167 | ||
| 169 | x1, x2, bias, x1_scale, x2_scale, y_scale, x1_offset, x2_offset, y_offset, x2_table, y, workspace, tiling | 168 | x1, x2, bias, x1_scale, x2_scale, y_scale, x1_offset, x2_offset, y_offset, x2_table, y, workspace, tiling |
| 170 | 169 | ||
| 171 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file matmul_custom_impl.h | 12 | * \file matmul_custom_impl.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef MATMUL_CUSTOM_IMPL_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -353,4 +352,4 @@ __aicore__ inline void MatmulCustomImpl<xType, wType, biasType, yType, scaleType | |||
| 353 | } | 352 | } |
| 354 | 353 | ||
| 355 | } // namespace QuantBatchMatmulV4 | 354 | } // namespace QuantBatchMatmulV4 |
| 356 | -#endif // MATMUL_CUSTOM_IMPL_H | 355 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file quant_batch_matmul_v4_constant.h | 12 | * \file quant_batch_matmul_v4_constant.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef QUANT_BATCH_MATMUL_V4_CONSTANT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | namespace QuantBatchMatmulV4 { | 16 | namespace QuantBatchMatmulV4 { |
| 18 | enum class QuantType : uint32_t { | 17 | enum class QuantType : uint32_t { |
| 19 | NONE = 0, | 18 | NONE = 0, |
| @@ -49,4 +48,4 @@ constexpr uint32_t SHIFT_RIGHT_SIZE = 0x4; | |||
| 49 | constexpr uint32_t E2M1_AND_MASK = 0x9C; | 48 | constexpr uint32_t E2M1_AND_MASK = 0x9C; |
| 50 | constexpr uint32_t PERGROUP_NZ_MASK_REG = 32; | 49 | constexpr uint32_t PERGROUP_NZ_MASK_REG = 32; |
| 51 | } // namespace QuantBatchMatmulV4 | 50 | } // namespace QuantBatchMatmulV4 |
| 52 | -#endif // QUANT_BATCH_MATMUL_V4_CONSTANT_H | 51 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef QUANT_BATCH_MATMUL_V4_PERCHANNEL_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -347,4 +346,4 @@ QuantBatchMatmulV4PerChannelKernel<xType, wType, biasType, yType, aTrans, bTrans | |||
| 347 | } | 346 | } |
| 348 | } | 347 | } |
| 349 | } // namespace QuantBatchMatmulV4 | 348 | } // namespace QuantBatchMatmulV4 |
| 350 | -#endif // QUANT_BATCH_MATMUL_V4_PERCHANNEL_H | 349 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef QUANT_BATCH_MATMUL_V4_PERTOKEN_PERGROUP_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -561,4 +560,4 @@ private: | |||
| 561 | }; | 560 | }; |
| 562 | 561 | ||
| 563 | } // namespace AscendC | 562 | } // namespace AscendC |
| 564 | -#endif | 563 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef QUANT_BATCH_MATMUL_V4_REG_BASE_COMMON_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -1636,4 +1635,3 @@ QuantBatchMatmulV4RegBaseCommonKernel<xType, wType, biasType, yType, aTrans, bTr | |||
| 1636 | } | 1635 | } |
| 1637 | } // namespace QuantBatchMatmulV4 | 1636 | } // namespace QuantBatchMatmulV4 |
| 1638 | 1637 | ||
| 1639 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef QUANT_BATCH_MATMUL_V4_TILING_DATA_APT_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -174,6 +173,4 @@ struct QuantBatchMatmulV4TilingDataParams { | |||
| 174 | }; | 173 | }; |
| 175 | 174 | ||
| 176 | } // namespace qbmmv4_tiling | 175 | } // namespace qbmmv4_tiling |
| 177 | - | ||
| 178 | - | ||
| 179 | 176 | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef ARCH35_QUANT_BATCH_MATMUL_V4_TILING_KEY_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -104,4 +103,4 @@ ASCENDC_TPL_SEL( | |||
| 104 | ASCENDC_TPL_UINT_SEL(KERNEL_TEMPLATE_TYPE, ASCENDC_TPL_UI_LIST, QBMMV4_LUT_ASW, QBMMV4_LUT_AL1FULL), ), | 103 | ASCENDC_TPL_UINT_SEL(KERNEL_TEMPLATE_TYPE, ASCENDC_TPL_UI_LIST, QBMMV4_LUT_ASW, QBMMV4_LUT_AL1FULL), ), |
| 105 | 104 | ||
| 106 | ); | 105 | ); |
| 107 | -#endif // ARCH35_QUANT_BATCH_MATMUL_V4_TILING_KEY_H | 106 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | - #ifndef QUANT_BATCH_MATMUL_V4_VF_H | 16 | + #pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -250,4 +249,3 @@ | |||
| 250 | 249 | ||
| 251 | } // namespace QuantBatchMatmulV4 | 250 | } // namespace QuantBatchMatmulV4 |
| 252 | 251 | ||
| 253 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file tool.h | 12 | * \file tool.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef TOOL_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -239,4 +238,4 @@ public: | |||
| 239 | }; | 238 | }; |
| 240 | }; | 239 | }; |
| 241 | } | 240 | } |
| 242 | -#endif | 241 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file sparse_tensor_dense_mat_mul_b16.h | 12 | * \file sparse_tensor_dense_mat_mul_b16.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef SPARSE_TENSOR_DENSE_MAT_MUL_B16_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -212,4 +211,3 @@ __aicore__ inline void SparseTensorDenseMatMulB16<T_IDX, T_VAL, T_SUM, ADJ_A, AD | |||
| 212 | 211 | ||
| 213 | } // namespace SparseTensorDenseMatMul | 212 | } // namespace SparseTensorDenseMatMul |
| 214 | 213 | ||
| 215 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file sparse_tensor_dense_mat_mul_b32.h | 12 | * \file sparse_tensor_dense_mat_mul_b32.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef SPARSE_TENSOR_DENSE_MAT_MUL_B32_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -128,4 +127,3 @@ __aicore__ inline void SparseTensorDenseMatMulB32<T_IDX, T_VAL, ADJ_A, ADJ_B>::P | |||
| 128 | 127 | ||
| 129 | } // namespace SparseTensorDenseMatMul | 128 | } // namespace SparseTensorDenseMatMul |
| 130 | 129 | ||
| 131 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file sparse_tensor_dense_mat_mul_base.h | 12 | * \file sparse_tensor_dense_mat_mul_base.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef SPARSE_TENSOR_DENSE_MAT_MUL_BASE_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -27,4 +26,3 @@ constexpr int32_t INDICES_DIM_1 = 2; | |||
| 27 | 26 | ||
| 28 | } // namespace SparseTensorDenseMatMul | 27 | } // namespace SparseTensorDenseMatMul |
| 29 | 28 | ||
| 30 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief TilingData def for op SparseTensorDenseMatMul | 13 | * \brief TilingData def for op SparseTensorDenseMatMul |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef __SPARSE_TENSOR_DENSE_MAT_MUL_TILING_DEF_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | // 所有模板共用同一套TilingData | 18 | // 所有模板共用同一套TilingData |
| 20 | struct SparseTensorDenseMatMulTilingData { | 19 | struct SparseTensorDenseMatMulTilingData { |
| @@ -50,4 +49,3 @@ struct SparseTensorDenseMatMulTilingData { | |||
| 50 | int64_t outTailCoreUbLoopTimes; // 尾核，要多少次循环 | 49 | int64_t outTailCoreUbLoopTimes; // 尾核，要多少次循环 |
| 51 | }; | 50 | }; |
| 52 | 51 | ||
| 53 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file sparse_tensor_dense_mat_mul_zeroing.h | 12 | * \file sparse_tensor_dense_mat_mul_zeroing.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef SPARSE_TENSOR_DENSE_MAT_MUL_ZEROING_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -50,4 +49,3 @@ __aicore__ inline void SparseTensorDenseMatMulZeroing<T_VAL>::InitAndProcessZero | |||
| 50 | 49 | ||
| 51 | } // namespace SparseTensorDenseMatMul | 50 | } // namespace SparseTensorDenseMatMul |
| 52 | 51 | ||
| 53 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef __OP_HOST_TRANSPOSE_BATCH_MAT_MUL_V3_ASW_TILING_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -57,4 +56,4 @@ private: | |||
| 57 | }; | 56 | }; |
| 58 | } | 57 | } |
| 59 | } | 58 | } |
| 60 | -#endif // __OP_HOST_TRANSPOSE_BATCH_MAT_MUL_V3_ASW_TILING_H__ | 59 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file transpose_batch_mat_mul_common.h | 12 | * \file transpose_batch_mat_mul_common.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_TRANSPOSE_BATCH_MAT_MUL_COMMON_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -34,4 +33,4 @@ constexpr size_t ATTR_NUM = 5; | |||
| 34 | constexpr int64_t UNKNOWN_DIM_NUM = -2; | 33 | constexpr int64_t UNKNOWN_DIM_NUM = -2; |
| 35 | } | 34 | } |
| 36 | } // namespace | 35 | } // namespace |
| 37 | -#endif // __OP_HOST_TRANSPOSE_BATCH_MAT_MUL_COMMON_H__ | 36 | + |
Mmatmul/transpose_batch_mat_mul/op_host/op_tiling/arch35/transpose_batch_mat_mul_tiling_advanced.h+2-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file transpose_batch_mat_mul_tiling_advanced.h | 12 | * \file transpose_batch_mat_mul_tiling_advanced.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_TRANSPOSE_BATCH_MAT_MUL_TILING_ADVANCED_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | namespace optiling { | 18 | namespace optiling { |
| @@ -41,4 +40,4 @@ protected: | |||
| 41 | }; | 40 | }; |
| 42 | } | 41 | } |
| 43 | } | 42 | } |
| 44 | -#endif // __OP_HOST_TRANSPOSE_BATCH_MAT_MUL_TILING_ADVANCED_H__ | 43 | + |
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef __OP_HOST_TBMM_TILING_KEY_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -36,4 +35,4 @@ private: | |||
| 36 | }; | 35 | }; |
| 37 | } // namespace transpose_batch_mat_mul_advanced | 36 | } // namespace transpose_batch_mat_mul_advanced |
| 38 | } // namespace optiling | 37 | } // namespace optiling |
| 39 | -#endif // __OP_HOST_TBMM_TILING_KEY_H__ | 38 | + |
Mmatmul/transpose_batch_mat_mul/op_host/op_tiling/arch35/transpose_batch_mat_mul_tiling_strategy.h+1-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file transpose_batch_mat_mul_tiling_strategy.h | 12 | * \file transpose_batch_mat_mul_tiling_strategy.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_TRANSPOSE_BATCH_MAT_MUL_TILING_STRATEGY_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -43,4 +42,3 @@ inline std::vector<int32_t> GetTransposeBatchMatMulPriorities(NpuArch NpuArch) | |||
| 43 | } | 42 | } |
| 44 | } | 43 | } |
| 45 | 44 | ||
| 46 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file transpose_batch_mat_mul_asw_block_advanced.h | 12 | * \file transpose_batch_mat_mul_asw_block_advanced.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef TRANSPOSE_BATCH_MAT_MUL_ASW_BLOCK_ADVANCED_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -216,4 +215,3 @@ __aicore__ inline void TransposeBatchMatMulAswBlock::CalcGMOffset() | |||
| 216 | 215 | ||
| 217 | } // namespace TransposeBatchMatMulAdvanced | 216 | } // namespace TransposeBatchMatMulAdvanced |
| 218 | 217 | ||
| 219 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file transpose_batch_mat_mul_asw_kernel_advanced.h | 12 | * \file transpose_batch_mat_mul_asw_kernel_advanced.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef TRANSPOSE_BATCH_MAT_MUL_ASW_KERNEL_ADVANCED_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -187,4 +186,3 @@ TransposeBatchMatMulAswKernel<A_TYPE, B_TYPE, C_TYPE, BIAS_TYPE, MODE, BLOCK_TYP | |||
| 187 | 186 | ||
| 188 | } // namespace TransposeBatchMatMulAdvanced | 187 | } // namespace TransposeBatchMatMulAdvanced |
| 189 | 188 | ||
| 190 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file transpose_batch_mat_mul_tiling_key.h | 12 | * \file transpose_batch_mat_mul_tiling_key.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_KERNEL_TBMM_TILING_KEY_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -69,4 +68,4 @@ ASCENDC_TPL_SEL( | |||
| 69 | ASCENDC_TPL_UINT_SEL(PERM_X2, ASCENDC_TPL_UI_LIST, TRANSPOSE_BATCH_MAT_MUL_PERM_X2_0_2_1), | 68 | ASCENDC_TPL_UINT_SEL(PERM_X2, ASCENDC_TPL_UI_LIST, TRANSPOSE_BATCH_MAT_MUL_PERM_X2_0_2_1), |
| 70 | ASCENDC_TPL_UINT_SEL(BATCH_SPLIT, ASCENDC_TPL_UI_LIST, | 69 | ASCENDC_TPL_UINT_SEL(BATCH_SPLIT, ASCENDC_TPL_UI_LIST, |
| 71 | TRANSPOSE_BATCH_MAT_MUL_BATCH_SPLIT_TRUE))); | 70 | TRANSPOSE_BATCH_MAT_MUL_BATCH_SPLIT_TRUE))); |
| 72 | -#endif | 71 | + |
| @@ -15,8 +15,7 @@ | |||
| 15 | 15 | ||
| 16 | 16 | ||
| 17 | 17 | ||
| 18 | -#ifndef OP_KERNEL_TBMM_TILING_KEY_PUBLIC_H | 18 | +#pragma once |
| 19 | - | ||
| 20 | 19 | ||
| 21 | 20 | ||
| 22 | 21 | ||
| @@ -42,4 +41,3 @@ enum class TBMMBatchSplit : std::uint8_t { | |||
| 42 | BATCH_SPLIT_TRUE = TRANSPOSE_BATCH_MAT_MUL_BATCH_SPLIT_TRUE | 41 | BATCH_SPLIT_TRUE = TRANSPOSE_BATCH_MAT_MUL_BATCH_SPLIT_TRUE |
| 43 | }; | 42 | }; |
| 44 | 43 | ||
| 45 | - | ||
Mmatmul/transpose_quant_batch_mat_mul/op_host/op_tiling/arch35/transpose_quant_batch_mat_mul_asw_tiling.h+2-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef __TRANSPOSE_QUANT_BATCH_MAT_MUL_ASW_TILING_H__ | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -57,4 +56,4 @@ private: | |||
| 57 | }; | 56 | }; |
| 58 | } // namespace transpose_quant_batch_mat_mul_advanced | 57 | } // namespace transpose_quant_batch_mat_mul_advanced |
| 59 | } // namespace optiling | 58 | } // namespace optiling |
| 60 | -#endif // TRANSPOSE_QUANT_BATCH_MAT_MUL_ASW_TILING_H | 59 | + |
Mmatmul/transpose_quant_batch_mat_mul/op_host/op_tiling/arch35/transpose_quant_batch_mat_mul_common.h+2-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file transpose_quant_batch_mat_mul_common.h | 12 | * \file transpose_quant_batch_mat_mul_common.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_TRANSPOSE_QUANT_BATCH_MAT_MUL_COMMON_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -75,4 +74,4 @@ bool inline IsMicroScaling(const gert::CompileTimeTensorDesc* x1Scale, const ger | |||
| 75 | } | 74 | } |
| 76 | } // namespace transpose_quant_batch_mat_mul_advanced | 75 | } // namespace transpose_quant_batch_mat_mul_advanced |
| 77 | } // namespace optiling | 76 | } // namespace optiling |
| 78 | -#endif // __OP_HOST_TRANSPOSE_QUANT_BATCH_MAT_MUL_COMMON_H__ | 77 | + |
Mmatmul/transpose_quant_batch_mat_mul/op_host/op_tiling/arch35/transpose_quant_batch_mat_mul_tiling_advanced.h+2-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file transpose_quant_batch_mat_mul_tiling_advanced.h | 12 | * \file transpose_quant_batch_mat_mul_tiling_advanced.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_TRANSPOSE_QUANT_BATCH_MAT_MUL_TILING_ADVANCED_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | namespace optiling { | 18 | namespace optiling { |
| @@ -42,4 +41,4 @@ protected: | |||
| 42 | }; | 41 | }; |
| 43 | } // namespace transpose_quant_batch_mat_mul_advanced | 42 | } // namespace transpose_quant_batch_mat_mul_advanced |
| 44 | } // namespace optiling | 43 | } // namespace optiling |
| 45 | -#endif // __OP_HOST_TRANSPOSE_QUANT_BATCH_MAT_MUL_TILING_ADVANCED_H__ | 44 | + |
Mmatmul/transpose_quant_batch_mat_mul/op_host/op_tiling/arch35/transpose_quant_batch_mat_mul_tiling_strategy.h+1-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file transpose_quant_batch_mat_mul_tiling_strategy.h | 12 | * \file transpose_quant_batch_mat_mul_tiling_strategy.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_HOST_TRANSPOSE_QUANT_BATCH_MAT_MUL_TILING_STRATEGY_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -43,4 +42,3 @@ inline std::vector<int32_t> GetTransposeQuantBatchMatMulPriorities(NpuArch npuAr | |||
| 43 | } // namespace transpose_quant_batch_mat_mul_advanced | 42 | } // namespace transpose_quant_batch_mat_mul_advanced |
| 44 | } // namespace optiling | 43 | } // namespace optiling |
| 45 | 44 | ||
| 46 | - | ||
Mmatmul/transpose_quant_batch_mat_mul/op_kernel/arch35/mm_extension_interface/tqbmm_copy_cube_out.h+2-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file tqbmm_copy_cube_out.h | 12 | * \file tqbmm_copy_cube_out.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef TQBMM_COPY_CUBE_OUT_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | namespace AscendC { | 18 | namespace AscendC { |
| @@ -205,4 +204,4 @@ private: | |||
| 205 | }; | 204 | }; |
| 206 | 205 | ||
| 207 | } // namespace AscendC | 206 | } // namespace AscendC |
| 208 | -#endif // QBMM_COPY_CUBE_OUT_H | 207 | + |
Mmatmul/transpose_quant_batch_mat_mul/op_kernel/arch35/mm_extension_interface/tqbmm_custom_mm_policy.h+2-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file tqbmm_custom_mm_policy.h | 12 | * \file tqbmm_custom_mm_policy.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef TQBMM_CUSTOM_MM_POLICY_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -26,4 +25,4 @@ public: | |||
| 26 | using CopyCubeOut = TQBmmCustomCopyCubeOut<IMPL, A_TYPE, B_TYPE, C_TYPE, MM_CFG, McgShfMode::DUAL_DST_SPLIT_M>; | 25 | using CopyCubeOut = TQBmmCustomCopyCubeOut<IMPL, A_TYPE, B_TYPE, C_TYPE, MM_CFG, McgShfMode::DUAL_DST_SPLIT_M>; |
| 27 | }; | 26 | }; |
| 28 | } // namespace AscendC | 27 | } // namespace AscendC |
| 29 | -#endif // TQBMM_CUSTOM_MM_POLICY_H | 28 | + |
Mmatmul/transpose_quant_batch_mat_mul/op_kernel/arch35/transpose_quant_batch_mat_mul_asw_block_advanced.h+1-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file transpose_quant_batch_mat_mul_asw_block_advanced.h | 12 | * \file transpose_quant_batch_mat_mul_asw_block_advanced.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef TRANSPOSE_QUANT_BATCH_MAT_MUL_ASW_BLOCK_ADVANCED_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -157,4 +156,3 @@ __aicore__ inline void TransposeQuantBatchMatMulAswBlock::CalcGMOffset(bool isMx | |||
| 157 | 156 | ||
| 158 | } // namespace TransposeQuantBatchMatMulAdvanced | 157 | } // namespace TransposeQuantBatchMatMulAdvanced |
| 159 | 158 | ||
| 160 | - | ||
Mmatmul/transpose_quant_batch_mat_mul/op_kernel/arch35/transpose_quant_batch_mat_mul_asw_kernel_advanced.h+1-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file transpose_quant_batch_mat_mul_asw_kernel_advanced.h | 12 | * \file transpose_quant_batch_mat_mul_asw_kernel_advanced.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef TRANSPOSE_QUANT_BATCH_MAT_MUL_ASW_KERNEL_ADVANCED_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -447,4 +446,3 @@ __aicore__ inline void TransposeQuantBatchMatMulAswKernel<LOCAL_TEMPLATE_FUNC_MI | |||
| 447 | 446 | ||
| 448 | } // namespace TransposeQuantBatchMatMulAdvanced | 447 | } // namespace TransposeQuantBatchMatMulAdvanced |
| 449 | 448 | ||
| 450 | - | ||
Mmatmul/transpose_quant_batch_mat_mul/op_kernel/arch35/transpose_quant_batch_mat_mul_tiling_key.h+2-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file transpose_quant_batch_mat_mul_tiling_key.h | 12 | * \file transpose_quant_batch_mat_mul_tiling_key.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef __OP_KERNEL_TQBMM_TILING_KEY_H__ | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -50,4 +49,4 @@ ASCENDC_TPL_SEL( | |||
| 50 | ASCENDC_TPL_UINT_SEL(PRECISION_MODE, ASCENDC_TPL_UI_LIST, TRANSPOSE_QUANT_BATCH_MAT_MUL_MXFP8)), | 49 | ASCENDC_TPL_UINT_SEL(PRECISION_MODE, ASCENDC_TPL_UI_LIST, TRANSPOSE_QUANT_BATCH_MAT_MUL_MXFP8)), |
| 51 | 50 | ||
| 52 | ); | 51 | ); |
| 53 | -#endif | 52 | + |
Mmatmul/transpose_quant_batch_mat_mul/op_kernel/arch35/transpose_quant_batch_mat_mul_tiling_key_public.h+2-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef OP_KERNEL_TQBMM_TILING_KEY_PUBLIC_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -48,4 +47,4 @@ enum class TQBMMPrecisionMode : std::uint8_t { | |||
| 48 | PRECISION_MODE_FP8 = TRANSPOSE_QUANT_BATCH_MAT_MUL_FP8, | 47 | PRECISION_MODE_FP8 = TRANSPOSE_QUANT_BATCH_MAT_MUL_FP8, |
| 49 | PRECISION_MODE_MXFP8 = TRANSPOSE_QUANT_BATCH_MAT_MUL_MXFP8 | 48 | PRECISION_MODE_MXFP8 = TRANSPOSE_QUANT_BATCH_MAT_MUL_MXFP8 |
| 50 | }; | 49 | }; |
| 51 | -#endif | 50 | + |
Mmatmul/weight_quant_batch_matmul_v2/op_host/op_tiling/arch35/backward_compatible/weight_quant_batch_matmul_v2_tiling_custom_backward_compatible.h+2-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef WEIGHT_QUANT_BATCH_MATMUL_V2_TILING_CUSTOM_BACKWARD_COMPATIBLE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -37,4 +36,4 @@ protected: | |||
| 37 | }; | 36 | }; |
| 38 | 37 | ||
| 39 | } // namespace optiling | 38 | } // namespace optiling |
| 40 | -#endif // WEIGHT_QUANT_BATCH_MATMUL_V2_TILING_CUSTOM_BACKWARD_COMPATIBLE_H | 39 | + |
Mmatmul/weight_quant_batch_matmul_v2/op_host/op_tiling/arch35/weight_quant_batch_matmul_v2_adaptive_sliding_window_tiling.h+1-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef WEIGHT_QUANT_BATCH_MATMUL_V2_ADAPTIVE_SLIDING_WINDOW_TILING_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -102,4 +101,3 @@ private: | |||
| 102 | } // namespace weight_quant_batch_matmul_v2 | 101 | } // namespace weight_quant_batch_matmul_v2 |
| 103 | } // namespace optiling | 102 | } // namespace optiling |
| 104 | 103 | ||
| 105 | - | ||
Mmatmul/weight_quant_batch_matmul_v2/op_host/op_tiling/arch35/weight_quant_batch_matmul_v2_adaptive_split_tiling.h+1-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef WEIGHT_QUANT_BATCH_MATMUL_V2_ADAPTIVE_SPLIT_TILING_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -98,4 +97,3 @@ private: | |||
| 98 | } // namespace weight_quant_batch_matmul_v2 | 97 | } // namespace weight_quant_batch_matmul_v2 |
| 99 | } // namespace optiling | 98 | } // namespace optiling |
| 100 | 99 | ||
| 101 | - | ||
Mmatmul/weight_quant_batch_matmul_v2/op_host/op_tiling/arch35/weight_quant_batch_matmul_v2_basic_block_table.h+1-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef WEIGHT_QUANT_BATCH_MATMUL_V2_BASIC_BLOCK_TABLE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -218,4 +217,3 @@ private: | |||
| 218 | }; | 217 | }; |
| 219 | } // namespace optiling | 218 | } // namespace optiling |
| 220 | 219 | ||
| 221 | - | ||
Mmatmul/weight_quant_batch_matmul_v2/op_host/op_tiling/arch35/weight_quant_batch_matmul_v2_basic_block_tiling.h+1-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef WEIGHT_QUANT_BATCH_MATMUL_V2_BASIC_BLOCK_TILING_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -498,4 +497,3 @@ protected: | |||
| 498 | 497 | ||
| 499 | } // namespace optiling | 498 | } // namespace optiling |
| 500 | 499 | ||
| 501 | - | ||
Mmatmul/weight_quant_batch_matmul_v2/op_host/op_tiling/arch35/weight_quant_batch_matmul_v2_iterbatch_tiling.h+43-44
| @@ -1,44 +1,43 @@ | |||
| 1 | -/** | 1 | +/** |
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. |
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of |
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). |
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. |
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, |
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | 8 | + * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | - */ | 9 | + */ |
| 10 | - | 10 | + |
| 11 | -/*! | 11 | +/*! |
| 12 | - * \file weight_quant_batch_matmul_v2_iterbatch_tiling.h | 12 | + * \file weight_quant_batch_matmul_v2_iterbatch_tiling.h |
| 13 | - * \brief | 13 | + * \brief |
| 14 | - */ | 14 | + */ |
| 15 | - | 15 | + |
| 16 | -#ifndef WEIGHT_QUANT_BATCH_MATMUL_V2_ITERBATCH_TILING_H | 16 | +#pragma once |
| 17 | -#define WEIGHT_QUANT_BATCH_MATMUL_V2_ITERBATCH_TILING_H | 17 | +#include "weight_quant_batch_matmul_v2_adaptive_sliding_window_tiling.h" |
| 18 | -#include "weight_quant_batch_matmul_v2_adaptive_sliding_window_tiling.h" | 18 | +#include "../weight_quant_batch_matmul_v2_tiling.h" |
| 19 | -#include "../weight_quant_batch_matmul_v2_tiling.h" | 19 | +#include "matmul/weight_quant_batch_matmul_v2/op_kernel/arch35/weight_quant_batch_matmul_v2_arch35_tiling_data.h" |
| 20 | -#include "matmul/weight_quant_batch_matmul_v2/op_kernel/arch35/weight_quant_batch_matmul_v2_arch35_tiling_data.h" | 20 | +namespace optiling { |
| 21 | -namespace optiling { | 21 | +namespace weight_quant_batch_matmul_v2 { |
| 22 | -namespace weight_quant_batch_matmul_v2 { | 22 | + |
| 23 | - | 23 | +class WeightQuantBatchMatmulV2IterbatchTiling : public WeightQuantBatchMatmulV2TilingASW { |
| 24 | -class WeightQuantBatchMatmulV2IterbatchTiling : public WeightQuantBatchMatmulV2TilingASW { | 24 | +public: |
| 25 | -public: | 25 | + explicit WeightQuantBatchMatmulV2IterbatchTiling(gert::TilingContext* context) : |
| 26 | - explicit WeightQuantBatchMatmulV2IterbatchTiling(gert::TilingContext* context) : | 26 | + WeightQuantBatchMatmulV2TilingASW(context) {} |
| 27 | - WeightQuantBatchMatmulV2TilingASW(context) {} | 27 | + ~WeightQuantBatchMatmulV2IterbatchTiling() override = default; |
| 28 | - ~WeightQuantBatchMatmulV2IterbatchTiling() override = default; | 28 | +protected: |
| 29 | -protected: | 29 | + ge::graphStatus DoOpTiling() override; |
| 30 | - ge::graphStatus DoOpTiling() override; | 30 | + bool IsCapable() override; |
| 31 | - bool IsCapable() override; | 31 | + uint64_t GetTilingKey() const override; |
| 32 | - uint64_t GetTilingKey() const override; | 32 | + void GetBroadCastInfo(uint64_t& broadcastNum, uint64_t& innerBatchNum, bool& isBroadcastA, bool& isBroadcastB); |
| 33 | - void GetBroadCastInfo(uint64_t& broadcastNum, uint64_t& innerBatchNum, bool& isBroadcastA, bool& isBroadcastB); | 33 | + |
| 34 | - | 34 | + void CalL1Tiling(); |
| 35 | - void CalL1Tiling(); | 35 | + bool CheckBatch(); |
| 36 | - bool CheckBatch(); | 36 | + uint32_t CalcIterBatch(); |
| 37 | - uint32_t CalcIterBatch(); | 37 | + uint32_t GetGcd(uint32_t numA, uint32_t numB) const; |
| 38 | - uint32_t GetGcd(uint32_t numA, uint32_t numB) const; | 38 | +private: |
| 39 | -private: | 39 | + uint64_t leftL1Size_ = 0; |
| 40 | - uint64_t leftL1Size_ = 0; | 40 | +}; |
| 41 | -}; | 41 | +} // namespace weight_quant_batch_matmul_v2 |
| 42 | -} // namespace weight_quant_batch_matmul_v2 | 42 | +} // namespace optiling |
| 43 | -} // namespace optiling | 43 | + |
| 44 | - | ||
Mmatmul/weight_quant_batch_matmul_v2/op_host/op_tiling/arch35/weight_quant_batch_matmul_v2_reg_base_tiling.h+2-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file weight_quant_batch_matmul_v2_reg_base_tiling.h | 12 | * \file weight_quant_batch_matmul_v2_reg_base_tiling.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef WEIGHT_QUANT_BATCH_MATMUL_V2_REG_BASE_TILING_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -64,4 +63,4 @@ private: | |||
| 64 | ge::graphStatus InstantiateTilingData(); | 63 | ge::graphStatus InstantiateTilingData(); |
| 65 | }; | 64 | }; |
| 66 | } // namespace optiling | 65 | } // namespace optiling |
| 67 | -#endif // WEIGHT_QUANT_BATCH_MATMUL_V2_REG_BASE_TILING_H | 66 | + |
| @@ -8,8 +8,7 @@ | |||
| 8 | * See LICENSE in the root of the software repository for the full text of the License. | 8 | * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | */ | 9 | */ |
| 10 | 10 | ||
| 11 | -#ifndef ARCH35_CMCT_CONVERTOR_H | 11 | +#pragma once |
| 12 | - | ||
| 13 | 12 | ||
| 14 | 13 | ||
| 15 | 14 | ||
| @@ -514,4 +513,3 @@ __aicore__ inline void InvokeKernel( | |||
| 514 | 513 | ||
| 515 | 514 | ||
| 516 | 515 | ||
| 517 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file anti_quant_y_vf.h | 12 | * \file anti_quant_y_vf.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef WEIGHT_QUANT_BATCHMATMUL_V2_ANTI_QUANT_Y_VF_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -94,4 +93,4 @@ __aicore__ inline void AntiQuantYB32( | |||
| 94 | } | 93 | } |
| 95 | } | 94 | } |
| 96 | } // namespace WeightQuantBatchMatmulV2::Arch35 | 95 | } // namespace WeightQuantBatchMatmulV2::Arch35 |
| 97 | -#endif // WEIGHT_QUANT_BATCHMATMUL_V2_ANTI_QUANT_Y_VF_H | 96 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file basic_block_config.h | 12 | * \file basic_block_config.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef WEIGHT_QUANT_BATCHMATMUL_V2_BASIC_BLOCK_CONFIG_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -227,4 +226,4 @@ __aicore__ constexpr VfConfig GetVfConfig() | |||
| 227 | } | 226 | } |
| 228 | } | 227 | } |
| 229 | } // namespace WeightQuantBatchMatmulV2::Arch35 | 228 | } // namespace WeightQuantBatchMatmulV2::Arch35 |
| 230 | -#endif // WEIGHT_QUANT_BATCHMATMUL_V2_BASIC_BLOCK_CONFIG_H | 229 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file basic_block_vf_mx.h | 12 | * \file basic_block_vf_mx.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef WEIGHT_QUANT_BATCHMATMUL_V2_BASIC_BLOCK_VF_MX_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -320,4 +319,4 @@ __aicore__ inline void AntiQuantFp4NzKnVf(Fp4NzParams<xType, wType>& fp4NzParams | |||
| 320 | } | 319 | } |
| 321 | } | 320 | } |
| 322 | } // namespace WeightQuantBatchMatmulV2::Arch35 | 321 | } // namespace WeightQuantBatchMatmulV2::Arch35 |
| 323 | -#endif // WEIGHT_QUANT_BATCHMATMUL_V2_BASIC_BLOCK_VF_MX_H | 322 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file basic_block_vf_nd.h | 12 | * \file basic_block_vf_nd.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef WEIGHT_QUANT_BATCHMATMUL_V2_BASIC_BLOCK_VF_ND_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -554,4 +553,4 @@ __aicore__ inline void AntiQuantInt4NdKn( | |||
| 554 | } | 553 | } |
| 555 | 554 | ||
| 556 | } // namespace WeightQuantBatchMatmulV2::Arch35 | 555 | } // namespace WeightQuantBatchMatmulV2::Arch35 |
| 557 | -#endif // WEIGHT_QUANT_BATCHMATMUL_V2_BASIC_BLOCK_VF_ND_H | 556 | + |
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file basic_block_vf_nz.h | 12 | * \file basic_block_vf_nz.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef WEIGHT_QUANT_BATCHMATMUL_V2_BASIC_BLOCK_VF_NZ_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -165,4 +164,4 @@ __aicore__ inline void AntiQuantS8S4NzKnGroupVf(Int4NzParams<xType, wType, antiQ | |||
| 165 | } | 164 | } |
| 166 | } | 165 | } |
| 167 | } // namespace WeightQuantBatchMatmulV2::Arch35 | 166 | } // namespace WeightQuantBatchMatmulV2::Arch35 |
| 168 | -#endif // WEIGHT_QUANT_BATCHMATMUL_V2_BASIC_BLOCK_VF_NZ_H | 167 | + |
Mmatmul/weight_quant_batch_matmul_v2/op_kernel/arch35/n_first/weight_quant_batch_matmul_v2_basic_block.h+1-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file weight_quant_batch_matmul_v2_basic_block.h | 12 | * \file weight_quant_batch_matmul_v2_basic_block.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef WEIGHT_QUANT_BATCHMATMUL_V2_BASIC_BLOCK_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -343,4 +342,3 @@ WeightQuantMatmulBasicBlock<xType, wType, antiQuantScaleType, biasType, yType, w | |||
| 343 | } | 342 | } |
| 344 | } // namespace WeightQuantBatchMatmulV2::Arch35 | 343 | } // namespace WeightQuantBatchMatmulV2::Arch35 |
| 345 | 344 | ||
| 346 | - | ||
Mmatmul/weight_quant_batch_matmul_v2/op_kernel/arch35/n_first/weight_quant_batch_matmul_v2_basic_block_controller.h+2-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef WEIGHT_QUANT_BATCHMATMUL_V2_BASIC_BLOCK_CONTROLLER_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -128,4 +127,4 @@ __aicore__ inline void WeightQuantBatchMatmulV2BasicBlockController< | |||
| 128 | wqmmBasicBlock_.End(); | 127 | wqmmBasicBlock_.End(); |
| 129 | } | 128 | } |
| 130 | } // namespace WeightQuantBatchMatmulV2::Arch35 | 129 | } // namespace WeightQuantBatchMatmulV2::Arch35 |
| 131 | -#endif // WEIGHT_QUANT_BATCHMATMUL_V2_BASIC_BLOCK_CONTROLLER_H | 130 | + |
Mmatmul/weight_quant_batch_matmul_v2/op_kernel/arch35/n_first/weight_quant_batch_matmul_v2_cube_compute.h+2-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file weight_quant_batch_matmul_v2_cube_compute.h | 12 | * \file weight_quant_batch_matmul_v2_cube_compute.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef WEIGHT_QUANT_BATCHMATMUL_V2_CUBE_COMPUTE_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -426,4 +425,4 @@ __aicore__ inline void WQBMM_CUBE_COMPUTE_CLASS::GetTensorC(LocalTensor<yType>& | |||
| 426 | 425 | ||
| 427 | } | 426 | } |
| 428 | } // namespace WeightQuantBatchMatmulV2::Arch35 | 427 | } // namespace WeightQuantBatchMatmulV2::Arch35 |
| 429 | -#endif | 428 | + |
Mmatmul/weight_quant_batch_matmul_v2/op_kernel/arch35/n_first/weight_quant_batch_matmul_v2_vcv_basic_block.h+1-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file weight_quant_batch_matmul_v2_vcv_basic_block.h | 12 | * \file weight_quant_batch_matmul_v2_vcv_basic_block.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef WEIGHT_QUANT_BATCHMATMUL_V2_VCV_BASIC_BLOCK_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -298,4 +297,3 @@ __aicore__ inline void WeightQuantVcvMatmulBasicBlock<xType, wType, biasType, yT | |||
| 298 | } | 297 | } |
| 299 | } // namespace WeightQuantBatchMatmulV2::Arch35 | 298 | } // namespace WeightQuantBatchMatmulV2::Arch35 |
| 300 | 299 | ||
| 301 | - | ||
Mmatmul/weight_quant_batch_matmul_v2/op_kernel/arch35/n_first/weight_quant_batch_matmul_v2_vec_compute.h+1-3
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file weight_quant_batch_matmul_v2_vec_compute.h | 12 | * \file weight_quant_batch_matmul_v2_vec_compute.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef WEIGHT_QUANT_BATCH_MATMUL_V2_VEC_COMPUTE_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -1218,4 +1217,3 @@ BasicBlockLibVectorAntiQuantCompute<xType, wType, antiQuantScaleType, yType, wqm | |||
| 1218 | } | 1217 | } |
| 1219 | } // namespace WeightQuantBatchMatmulV2::Arch35 | 1218 | } // namespace WeightQuantBatchMatmulV2::Arch35 |
| 1220 | 1219 | ||
| 1221 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file tool_arch35.h | 12 | * \file tool_arch35.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef WEIGHT_QUANT_BATCH_MATMUL_V2_TOOL_ARCH35_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -132,4 +131,4 @@ struct MatmulL1GmType : MatmulType<POSITION, FORMAT, TYPE, ISTRANS, LAYOUT, IBSH | |||
| 132 | constexpr static TPosition srcPos = TPosition::GM; | 131 | constexpr static TPosition srcPos = TPosition::GM; |
| 133 | }; | 132 | }; |
| 134 | } // namespace WeightQuantBatchMatmulV2::Arch35 | 133 | } // namespace WeightQuantBatchMatmulV2::Arch35 |
| 135 | -#endif | 134 | + |
Mmatmul/weight_quant_batch_matmul_v2/op_kernel/arch35/weight_quant_batch_matmul_v2_adaptive_sliding_window.h+2-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef WEIGHT_QUANT_BATCH_MATMUL_V2_ADAPTIVE_SLIDING_WINDOW_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -225,4 +224,4 @@ __aicore__ inline void WeightQuantBatchMatmulV2ASWKernel<LOCAL_TEMPLATE_FUNC_PAR | |||
| 225 | mm_.GetTensorC(cGlobal_[block_.offset_.offsetC]); | 224 | mm_.GetTensorC(cGlobal_[block_.offset_.offsetC]); |
| 226 | } | 225 | } |
| 227 | } // namespace WeightQuantBatchMatmulV2::Arch35 | 226 | } // namespace WeightQuantBatchMatmulV2::Arch35 |
| 228 | -#endif // WEIGHT_QUANT_BATCH_MATMUL_V2_ADAPTIVE_SLIDING_WINDOW_H | 227 | + |
Mmatmul/weight_quant_batch_matmul_v2/op_kernel/arch35/weight_quant_batch_matmul_v2_arch35_tiling_data.h+2-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef ARCH35_WEIGHT_QUANT_BATCH_MATMUL_V2_TILING_DATA_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -161,4 +160,4 @@ struct alignas(8) WeightQuantBatchMatmulV2TilingData | |||
| 161 | }; | 160 | }; |
| 162 | 161 | ||
| 163 | } // namespace wqbmmv2_tiling | 162 | } // namespace wqbmmv2_tiling |
| 164 | -#endif // ARCH35_WEIGHT_QUANT_BATCH_MATMUL_V2_TILING_DATA_H | 163 | + |
Mmatmul/weight_quant_batch_matmul_v2/op_kernel/arch35/weight_quant_batch_matmul_v2_arch35_tiling_key.h+1-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef WEIGHT_QUANT_BATCH_MATMUL_V2_ARCH35_TILING_KEY_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -720,4 +719,3 @@ ASCENDC_TPL_SEL( | |||
| 720 | 719 | ||
| 721 | ); | 720 | ); |
| 722 | 721 | ||
| 723 | - | ||
| @@ -12,8 +12,7 @@ | |||
| 12 | * \file weight_quant_batch_matmul_v2_asw_block.h | 12 | * \file weight_quant_batch_matmul_v2_asw_block.h |
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | -#ifndef WEIGHT_QUANT_BMMV2_ASW_BLOCK_H | 15 | +#pragma once |
| 16 | - | ||
| 17 | 16 | ||
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -192,4 +191,3 @@ __aicore__ inline void WeightQuantBmmAswBlock::CalcGMOffset() | |||
| 192 | 191 | ||
| 193 | } // namespace WeightQuantBatchMatmulV2::Arch35 | 192 | } // namespace WeightQuantBatchMatmulV2::Arch35 |
| 194 | 193 | ||
| 195 | - | ||
Mmatmul/weight_quant_batch_matmul_v2/op_kernel/arch35/weight_quant_batch_matmul_v2_iterbatch.h+157-158
| @@ -1,158 +1,157 @@ | |||
| 1 | -/** | 1 | +/** |
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. |
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of |
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). |
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. |
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, |
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | 8 | + * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | - */ | 9 | + */ |
| 10 | - | 10 | + |
| 11 | -/*! | 11 | +/*! |
| 12 | - * \file weight_quant_batch_matmul_v2_iterbatch.h | 12 | + * \file weight_quant_batch_matmul_v2_iterbatch.h |
| 13 | - * \brief | 13 | + * \brief |
| 14 | - */ | 14 | + */ |
| 15 | - | 15 | + |
| 16 | -#ifndef WEIGHT_QUANT_BATCH_MATMUL_V2_ITERBATCH_H | 16 | +#pragma once |
| 17 | -#define WEIGHT_QUANT_BATCH_MATMUL_V2_ITERBATCH_H | 17 | + |
| 18 | - | 18 | +#if ASC_DEVKIT_MAJOR >= 9 |
| 19 | -#if ASC_DEVKIT_MAJOR >= 9 | 19 | +#include "kernel_basic_intf.h" |
| 20 | -#include "kernel_basic_intf.h" | 20 | +#else |
| 21 | -#else | 21 | +#include "kernel_operator.h" |
| 22 | -#include "kernel_operator.h" | 22 | +#endif |
| 23 | -#endif | 23 | +#include "lib/matmul_intf.h" |
| 24 | -#include "lib/matmul_intf.h" | 24 | +#include "../tool.h" |
| 25 | -#include "../tool.h" | 25 | +#include "weight_quant_batch_matmul_v2_iterbatch_block.h" |
| 26 | -#include "weight_quant_batch_matmul_v2_iterbatch_block.h" | 26 | +#include "../weight_quant_batch_matmul_v2_constant.h" |
| 27 | -#include "../weight_quant_batch_matmul_v2_constant.h" | 27 | +#include "weight_quant_batch_matmul_v2_arch35_tiling_data.h" |
| 28 | -#include "weight_quant_batch_matmul_v2_arch35_tiling_data.h" | 28 | + |
| 29 | - | 29 | +#define ITERBATCH_LOCAL_TEMPLATE_CLASS_PARAMS \ |
| 30 | -#define ITERBATCH_LOCAL_TEMPLATE_CLASS_PARAMS \ | 30 | + template <typename xType, typename wType, typename biasType, typename yType, bool aTrans, bool bTrans, \ |
| 31 | - template <typename xType, typename wType, typename biasType, typename yType, bool aTrans, bool bTrans, \ | 31 | + QuantType antiQuantType, bool hasAntiQuantOffset, QuantType quantType, const MatmulConfig &mmCfg> |
| 32 | - QuantType antiQuantType, bool hasAntiQuantOffset, QuantType quantType, const MatmulConfig &mmCfg> | 32 | +#define ITERBATCH_LOCAL_TEMPLATE_FUNC_PARAMS \ |
| 33 | -#define ITERBATCH_LOCAL_TEMPLATE_FUNC_PARAMS \ | 33 | + xType, wType, biasType, yType, aTrans, bTrans, antiQuantType, hasAntiQuantOffset, quantType, mmCfg |
| 34 | - xType, wType, biasType, yType, aTrans, bTrans, antiQuantType, hasAntiQuantOffset, quantType, mmCfg | 34 | + |
| 35 | - | 35 | +using AscendC::GetBlockIdx; |
| 36 | -using AscendC::GetBlockIdx; | 36 | +using AscendC::GlobalTensor; |
| 37 | -using AscendC::GlobalTensor; | 37 | +using AscendC::TPipe; |
| 38 | -using AscendC::TPipe; | 38 | +using AscendC::TPosition; |
| 39 | -using AscendC::TPosition; | 39 | +using matmul::MatmulImpl; |
| 40 | -using matmul::MatmulImpl; | 40 | +using matmul::MatmulType; |
| 41 | -using matmul::MatmulType; | 41 | +namespace WeightQuantBatchMatmulV2::Arch35 { |
| 42 | -namespace WeightQuantBatchMatmulV2::Arch35 { | 42 | + |
| 43 | - | 43 | +ITERBATCH_LOCAL_TEMPLATE_CLASS_PARAMS |
| 44 | -ITERBATCH_LOCAL_TEMPLATE_CLASS_PARAMS | 44 | +class WeightQuantBatchMatmulV2IterBatchKernel { |
| 45 | -class WeightQuantBatchMatmulV2IterBatchKernel { | 45 | +public: |
| 46 | -public: | 46 | + __aicore__ inline WeightQuantBatchMatmulV2IterBatchKernel() |
| 47 | - __aicore__ inline WeightQuantBatchMatmulV2IterBatchKernel() | 47 | + { |
| 48 | - { | 48 | + } |
| 49 | - } | 49 | + __aicore__ inline void Init(GM_ADDR x, GM_ADDR weight, GM_ADDR antiquantScale, GM_ADDR antiquantOffset, |
| 50 | - __aicore__ inline void Init(GM_ADDR x, GM_ADDR weight, GM_ADDR antiquantScale, GM_ADDR antiquantOffset, | 50 | + GM_ADDR quantScale, GM_ADDR quantOffset, GM_ADDR bias, GM_ADDR y, GM_ADDR workspace, |
| 51 | - GM_ADDR quantScale, GM_ADDR quantOffset, GM_ADDR bias, GM_ADDR y, GM_ADDR workspace, | 51 | + const void* tilingData, TPipe* tPipe); |
| 52 | - const void* tilingData, TPipe* tPipe); | 52 | + __aicore__ inline void UpdateGlobalAddr(GM_ADDR x, GM_ADDR weight, GM_ADDR antiquantScale, GM_ADDR antiquantOffset, |
| 53 | - __aicore__ inline void UpdateGlobalAddr(GM_ADDR x, GM_ADDR weight, GM_ADDR antiquantScale, GM_ADDR antiquantOffset, | 53 | + GM_ADDR quantScale, GM_ADDR quantOffset, GM_ADDR bias, GM_ADDR y, |
| 54 | - GM_ADDR quantScale, GM_ADDR quantOffset, GM_ADDR bias, GM_ADDR y, | 54 | + GM_ADDR workspace); |
| 55 | - GM_ADDR workspace); | 55 | + __aicore__ inline void Process(); |
| 56 | - __aicore__ inline void Process(); | 56 | +protected: |
| 57 | -protected: | 57 | + __aicore__ inline void CalcMmWithBatch(); |
| 58 | - __aicore__ inline void CalcMmWithBatch(); | 58 | + uint32_t blockIdx_; |
| 59 | - uint32_t blockIdx_; | 59 | + const wqbmmv2_tiling::WeightQuantBatchMatmulV2ASWTilingDataParams* tiling_; |
| 60 | - const wqbmmv2_tiling::WeightQuantBatchMatmulV2ASWTilingDataParams* tiling_; | 60 | + |
| 61 | - | 61 | + GlobalTensor<xType> aGlobal_; |
| 62 | - GlobalTensor<xType> aGlobal_; | 62 | + GlobalTensor<wType> bGlobal_; |
| 63 | - GlobalTensor<wType> bGlobal_; | 63 | + GlobalTensor<yType> cGlobal_; |
| 64 | - GlobalTensor<yType> cGlobal_; | 64 | + GlobalTensor<biasType> biasGlobal_; |
| 65 | - GlobalTensor<biasType> biasGlobal_; | 65 | + WeightQuantBmmIterBatchBlock block_; |
| 66 | - WeightQuantBmmIterBatchBlock block_; | 66 | + GlobalTensor<uint64_t> scaleGlobal_; |
| 67 | - GlobalTensor<uint64_t> scaleGlobal_; | 67 | + using aType = matmul::MatmulType<AscendC::TPosition::GM, CubeFormat::ND, xType, aTrans, LayoutMode::NORMAL>; |
| 68 | - using aType = matmul::MatmulType<AscendC::TPosition::GM, CubeFormat::ND, xType, aTrans, LayoutMode::NORMAL>; | 68 | + using bType = matmul::MatmulType<AscendC::TPosition::GM, CubeFormat::ND, wType, bTrans, LayoutMode::NORMAL>; |
| 69 | - using bType = matmul::MatmulType<AscendC::TPosition::GM, CubeFormat::ND, wType, bTrans, LayoutMode::NORMAL>; | 69 | + using cType = matmul::MatmulType<AscendC::TPosition::GM, CubeFormat::ND, yType, false, LayoutMode::NORMAL>; |
| 70 | - using cType = matmul::MatmulType<AscendC::TPosition::GM, CubeFormat::ND, yType, false, LayoutMode::NORMAL>; | 70 | + using biasMatmulType = matmul::MatmulType<AscendC::TPosition::GM, CubeFormat::ND, biasType>; |
| 71 | - using biasMatmulType = matmul::MatmulType<AscendC::TPosition::GM, CubeFormat::ND, biasType>; | 71 | + matmul::MatmulImpl<aType, bType, cType, biasMatmulType, mmCfg> mm_; |
| 72 | - matmul::MatmulImpl<aType, bType, cType, biasMatmulType, mmCfg> mm_; | 72 | +}; |
| 73 | -}; | 73 | + |
| 74 | - | 74 | +ITERBATCH_LOCAL_TEMPLATE_CLASS_PARAMS |
| 75 | -ITERBATCH_LOCAL_TEMPLATE_CLASS_PARAMS | 75 | +__aicore__ inline void WeightQuantBatchMatmulV2IterBatchKernel<ITERBATCH_LOCAL_TEMPLATE_FUNC_PARAMS>::Init( |
| 76 | -__aicore__ inline void WeightQuantBatchMatmulV2IterBatchKernel<ITERBATCH_LOCAL_TEMPLATE_FUNC_PARAMS>::Init( | 76 | + GM_ADDR x, GM_ADDR weight, GM_ADDR antiquantScale, GM_ADDR antiquantOffset, GM_ADDR quantScale, GM_ADDR quantOffset, |
| 77 | - GM_ADDR x, GM_ADDR weight, GM_ADDR antiquantScale, GM_ADDR antiquantOffset, GM_ADDR quantScale, GM_ADDR quantOffset, | 77 | + GM_ADDR bias, GM_ADDR y, GM_ADDR workspace, const void* tilingData, TPipe* tPipe) |
| 78 | - GM_ADDR bias, GM_ADDR y, GM_ADDR workspace, const void* tilingData, TPipe* tPipe) | 78 | +{ |
| 79 | -{ | 79 | + if ASCEND_IS_AIV { |
| 80 | - if ASCEND_IS_AIV { | 80 | + return; |
| 81 | - return; | 81 | + } |
| 82 | - } | 82 | + tiling_ = static_cast<const wqbmmv2_tiling::WeightQuantBatchMatmulV2ASWTilingDataParams*>(tilingData); |
| 83 | - tiling_ = static_cast<const wqbmmv2_tiling::WeightQuantBatchMatmulV2ASWTilingDataParams*>(tilingData); | 83 | + blockIdx_ = GetBlockIdx(); |
| 84 | - blockIdx_ = GetBlockIdx(); | 84 | + UpdateGlobalAddr(x, weight, antiquantScale, antiquantOffset, quantScale, quantOffset, bias, y, workspace); |
| 85 | - UpdateGlobalAddr(x, weight, antiquantScale, antiquantOffset, quantScale, quantOffset, bias, y, workspace); | 85 | + mm_.SetSubBlockIdx(0); |
| 86 | - mm_.SetSubBlockIdx(0); | 86 | + mm_.Init(&tiling_->matmulTiling, tPipe); |
| 87 | - mm_.Init(&tiling_->matmulTiling, tPipe); | 87 | +} |
| 88 | -} | 88 | + |
| 89 | - | 89 | +ITERBATCH_LOCAL_TEMPLATE_CLASS_PARAMS |
| 90 | -ITERBATCH_LOCAL_TEMPLATE_CLASS_PARAMS | 90 | +__aicore__ inline void WeightQuantBatchMatmulV2IterBatchKernel<ITERBATCH_LOCAL_TEMPLATE_FUNC_PARAMS>::UpdateGlobalAddr( |
| 91 | -__aicore__ inline void WeightQuantBatchMatmulV2IterBatchKernel<ITERBATCH_LOCAL_TEMPLATE_FUNC_PARAMS>::UpdateGlobalAddr( | 91 | + GM_ADDR x, GM_ADDR weight, GM_ADDR antiquantScale, GM_ADDR antiquantOffset, GM_ADDR quantScale, GM_ADDR quantOffset, |
| 92 | - GM_ADDR x, GM_ADDR weight, GM_ADDR antiquantScale, GM_ADDR antiquantOffset, GM_ADDR quantScale, GM_ADDR quantOffset, | 92 | + GM_ADDR bias, GM_ADDR y, GM_ADDR workspace) |
| 93 | - GM_ADDR bias, GM_ADDR y, GM_ADDR workspace) | 93 | +{ |
| 94 | -{ | 94 | + block_.Init(tiling_, blockIdx_); |
| 95 | - block_.Init(tiling_, blockIdx_); | 95 | + |
| 96 | - | 96 | + if constexpr (antiQuantType == QuantType::PER_TENSOR) { // pertensor |
| 97 | - if constexpr (antiQuantType == QuantType::PER_TENSOR) { // pertensor | 97 | + block_.offset_.scaleScalar = *((__gm__ uint64_t *)antiquantScale); |
| 98 | - block_.offset_.scaleScalar = *((__gm__ uint64_t *)antiquantScale); | 98 | + } else { |
| 99 | - } else { | 99 | + scaleGlobal_.SetGlobalBuffer((__gm__ uint64_t *)antiquantScale); |
| 100 | - scaleGlobal_.SetGlobalBuffer((__gm__ uint64_t *)antiquantScale); | 100 | + } |
| 101 | - } | 101 | + |
| 102 | - | 102 | + // update global buffer |
| 103 | - // update global buffer | 103 | + aGlobal_.SetGlobalBuffer((__gm__ xType *)x); |
| 104 | - aGlobal_.SetGlobalBuffer((__gm__ xType *)x); | 104 | + bGlobal_.SetGlobalBuffer((__gm__ wType *)weight); |
| 105 | - bGlobal_.SetGlobalBuffer((__gm__ wType *)weight); | 105 | + cGlobal_.SetGlobalBuffer((__gm__ yType *)y); |
| 106 | - cGlobal_.SetGlobalBuffer((__gm__ yType *)y); | 106 | + if (static_cast<bool>(tiling_->matmulTiling.isBias)) { |
| 107 | - if (static_cast<bool>(tiling_->matmulTiling.isBias)) { | 107 | + biasGlobal_.SetGlobalBuffer((__gm__ biasType*)bias); |
| 108 | - biasGlobal_.SetGlobalBuffer((__gm__ biasType*)bias); | 108 | + } |
| 109 | - } | 109 | +} |
| 110 | -} | 110 | + |
| 111 | - | 111 | +ITERBATCH_LOCAL_TEMPLATE_CLASS_PARAMS |
| 112 | -ITERBATCH_LOCAL_TEMPLATE_CLASS_PARAMS | 112 | +__aicore__ inline void WeightQuantBatchMatmulV2IterBatchKernel<ITERBATCH_LOCAL_TEMPLATE_FUNC_PARAMS>::Process() |
| 113 | -__aicore__ inline void WeightQuantBatchMatmulV2IterBatchKernel<ITERBATCH_LOCAL_TEMPLATE_FUNC_PARAMS>::Process() | 113 | +{ |
| 114 | -{ | 114 | + if ASCEND_IS_AIV { |
| 115 | - if ASCEND_IS_AIV { | 115 | + return; |
| 116 | - return; | 116 | + } |
| 117 | - } | 117 | + if (blockIdx_ >= block_.tilingData_->matmulTiling.usedCoreNum) { |
| 118 | - if (blockIdx_ >= block_.tilingData_->matmulTiling.usedCoreNum) { | 118 | + return; |
| 119 | - return; | 119 | + } |
| 120 | - } | 120 | + if (block_.params_.innerBatchNum == 0) { |
| 121 | - if (block_.params_.innerBatchNum == 0) { | 121 | + block_.offset_.batchCOffset = 0; |
| 122 | - block_.offset_.batchCOffset = 0; | 122 | + CalcMmWithBatch(); |
| 123 | - CalcMmWithBatch(); | 123 | + } else { |
| 124 | - } else { | 124 | + for (uint64_t b1Index = 0; b1Index < block_.params_.iterBatch1; ++b1Index) { |
| 125 | - for (uint64_t b1Index = 0; b1Index < block_.params_.iterBatch1; ++b1Index) { | 125 | + for (uint64_t b2Index = 0; b2Index < block_.params_.iterBatch2; ++b2Index) { |
| 126 | - for (uint64_t b2Index = 0; b2Index < block_.params_.iterBatch2; ++b2Index) { | 126 | + for (uint64_t b3Index = 0; b3Index < block_.params_.iterBatch3; ++b3Index) { |
| 127 | - for (uint64_t b3Index = 0; b3Index < block_.params_.iterBatch3; ++b3Index) { | 127 | + block_.GetBroadcastBatchOffset(b1Index, b2Index, b3Index); |
| 128 | - block_.GetBroadcastBatchOffset(b1Index, b2Index, b3Index); | 128 | + CalcMmWithBatch(); |
| 129 | - CalcMmWithBatch(); | 129 | + } |
| 130 | - } | 130 | + } |
| 131 | - } | 131 | + } |
| 132 | - } | 132 | + } |
| 133 | - } | 133 | +} |
| 134 | -} | 134 | + |
| 135 | - | 135 | +ITERBATCH_LOCAL_TEMPLATE_CLASS_PARAMS |
| 136 | -ITERBATCH_LOCAL_TEMPLATE_CLASS_PARAMS | 136 | +__aicore__ inline void WeightQuantBatchMatmulV2IterBatchKernel<ITERBATCH_LOCAL_TEMPLATE_FUNC_PARAMS>::CalcMmWithBatch() |
| 137 | -__aicore__ inline void WeightQuantBatchMatmulV2IterBatchKernel<ITERBATCH_LOCAL_TEMPLATE_FUNC_PARAMS>::CalcMmWithBatch() | 137 | +{ |
| 138 | -{ | 138 | + for (uint64_t loopIndex = 0; loopIndex < block_.params_.loopTimes; loopIndex++) { |
| 139 | - for (uint64_t loopIndex = 0; loopIndex < block_.params_.loopTimes; loopIndex++) { | 139 | + if constexpr(antiQuantType == QuantType::PER_TENSOR) { |
| 140 | - if constexpr(antiQuantType == QuantType::PER_TENSOR) { | 140 | + mm_.SetQuantScalar(block_.offset_.scaleScalar); |
| 141 | - mm_.SetQuantScalar(block_.offset_.scaleScalar); | 141 | + } else { |
| 142 | - } else { | 142 | + mm_.SetQuantVector(scaleGlobal_[block_.offset_.offsetScale]); |
| 143 | - mm_.SetQuantVector(scaleGlobal_[block_.offset_.offsetScale]); | 143 | + } |
| 144 | - } | 144 | + block_.GetMultiBatchInfo(loopIndex); |
| 145 | - block_.GetMultiBatchInfo(loopIndex); | 145 | + block_.CalcGMOffset(); |
| 146 | - block_.CalcGMOffset(); | 146 | + if (block_.tilingData_->matmulTiling.isBias) { |
| 147 | - if (block_.tilingData_->matmulTiling.isBias) { | 147 | + mm_.SetBias(biasGlobal_[block_.offset_.offsetBias]); |
| 148 | - mm_.SetBias(biasGlobal_[block_.offset_.offsetBias]); | 148 | + } |
| 149 | - } | 149 | + mm_.SetTensorA(aGlobal_[block_.offset_.offsetA], aTrans); |
| 150 | - mm_.SetTensorA(aGlobal_[block_.offset_.offsetA], aTrans); | 150 | + mm_.SetTensorB(bGlobal_[block_.offset_.offsetB], bTrans); |
| 151 | - mm_.SetTensorB(bGlobal_[block_.offset_.offsetB], bTrans); | 151 | + mm_.SetBatchNum(block_.params_.batchANum, block_.params_.batchBNum); |
| 152 | - mm_.SetBatchNum(block_.params_.batchANum, block_.params_.batchBNum); | 152 | + mm_.IterateBatch(cGlobal_[block_.offset_.offsetC], false, 0, false, block_.params_.singleASize, |
| 153 | - mm_.IterateBatch(cGlobal_[block_.offset_.offsetC], false, 0, false, block_.params_.singleASize, | 153 | + block_.params_.singleBSize); |
| 154 | - block_.params_.singleBSize); | 154 | + } |
| 155 | - } | 155 | +} |
| 156 | -} | 156 | +} // namespace WeightQuantBatchMatmulV2::Arch35 |
| 157 | -} // namespace WeightQuantBatchMatmulV2::Arch35 | 157 | + |
| 158 | - | ||
Mmatmul/weight_quant_batch_matmul_v2/op_kernel/arch35/weight_quant_batch_matmul_v2_iterbatch_block.h+253-254
| @@ -1,254 +1,253 @@ | |||
| 1 | -/** | 1 | +/** |
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. |
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of |
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). |
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. |
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, |
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | 8 | + * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | - */ | 9 | + */ |
| 10 | - | 10 | + |
| 11 | -/*! | 11 | +/*! |
| 12 | - * \file weight_quant_batch_matmul_v2_iterbatch_block.h | 12 | + * \file weight_quant_batch_matmul_v2_iterbatch_block.h |
| 13 | - * \brief | 13 | + * \brief |
| 14 | - */ | 14 | + */ |
| 15 | - | 15 | + |
| 16 | -#ifndef WEIGHT_QUANT_BMMV2_ASW_ITERBATCH_BLOCK_H | 16 | +#pragma once |
| 17 | -#define WEIGHT_QUANT_BMMV2_ASW_ITERBATCH_BLOCK_H | 17 | + |
| 18 | - | 18 | +#if ASC_DEVKIT_MAJOR >= 9 |
| 19 | -#if ASC_DEVKIT_MAJOR >= 9 | 19 | +#include "kernel_basic_intf.h" |
| 20 | -#include "kernel_basic_intf.h" | 20 | +#else |
| 21 | -#else | 21 | +#include "kernel_operator.h" |
| 22 | -#include "kernel_operator.h" | 22 | +#endif |
| 23 | -#endif | 23 | +#include "lib/matmul_intf.h" |
| 24 | -#include "lib/matmul_intf.h" | 24 | +#include "../tool.h" |
| 25 | -#include "../tool.h" | 25 | +#include "weight_quant_batch_matmul_v2_arch35_tiling_data.h" |
| 26 | -#include "weight_quant_batch_matmul_v2_arch35_tiling_data.h" | 26 | + |
| 27 | - | 27 | +namespace WeightQuantBatchMatmulV2::Arch35 { |
| 28 | -namespace WeightQuantBatchMatmulV2::Arch35 { | 28 | +constexpr uint64_t L0C_SIZE_256K = 256 * 1024UL; |
| 29 | -constexpr uint64_t L0C_SIZE_256K = 256 * 1024UL; | 29 | +constexpr uint8_t NO_NEED_BROADCAST = 0; |
| 30 | -constexpr uint8_t NO_NEED_BROADCAST = 0; | 30 | +constexpr uint8_t A_NEED_BROADCAST = 1; |
| 31 | -constexpr uint8_t A_NEED_BROADCAST = 1; | 31 | +constexpr uint8_t B_NEED_BROADCAST = 2; |
| 32 | -constexpr uint8_t B_NEED_BROADCAST = 2; | 32 | + |
| 33 | - | 33 | +struct IterBatchTilingParam { |
| 34 | -struct IterBatchTilingParam { | 34 | + uint64_t singleASize; |
| 35 | - uint64_t singleASize; | 35 | + uint64_t singleBSize; |
| 36 | - uint64_t singleBSize; | 36 | + uint64_t singleCSize; |
| 37 | - uint64_t singleCSize; | 37 | + uint64_t mainLoopPerCoreBatchNum; |
| 38 | - uint64_t mainLoopPerCoreBatchNum; | 38 | + uint64_t lastLoopAllBatchNum; |
| 39 | - uint64_t lastLoopAllBatchNum; | 39 | + uint64_t lastLoopPreCoreBatchNum; |
| 40 | - uint64_t lastLoopPreCoreBatchNum; | 40 | + uint64_t lastLoopBlockNum; |
| 41 | - uint64_t lastLoopBlockNum; | 41 | + uint64_t loopTimes; |
| 42 | - uint64_t loopTimes; | 42 | + uint64_t batchIndex; |
| 43 | - uint64_t batchIndex; | 43 | + uint64_t batchAIndex; |
| 44 | - uint64_t batchAIndex; | 44 | + uint64_t batchBIndex; |
| 45 | - uint64_t batchBIndex; | 45 | + uint64_t batchANum; |
| 46 | - uint64_t batchANum; | 46 | + uint64_t batchBNum; |
| 47 | - uint64_t batchBNum; | 47 | + uint64_t useCoreNum; |
| 48 | - uint64_t useCoreNum; | 48 | + uint64_t nBatchOutNum; |
| 49 | - uint64_t nBatchOutNum; | 49 | + // outshape = (batch1, batch2, batch3, batch4, m, n) |
| 50 | - // outshape = (batch1, batch2, batch3, batch4, m, n) | 50 | + uint64_t iterBatch3; |
| 51 | - uint64_t iterBatch3; | 51 | + uint64_t iterBatch2; |
| 52 | - uint64_t iterBatch2; | 52 | + uint64_t iterBatch1; |
| 53 | - uint64_t iterBatch1; | 53 | + uint64_t calcBatchNum; |
| 54 | - uint64_t calcBatchNum; | 54 | + uint64_t innerBatchNum; |
| 55 | - uint64_t innerBatchNum; | 55 | + uint64_t broadcastFlag; |
| 56 | - uint64_t broadcastFlag; | 56 | +}; |
| 57 | -}; | 57 | + |
| 58 | - | 58 | +struct IterBatchOffsetParam { |
| 59 | -struct IterBatchOffsetParam { | 59 | + uint64_t offsetA; |
| 60 | - uint64_t offsetA; | 60 | + uint64_t offsetB; |
| 61 | - uint64_t offsetB; | 61 | + uint64_t offsetC; |
| 62 | - uint64_t offsetC; | 62 | + uint64_t offsetScale; |
| 63 | - uint64_t offsetScale; | 63 | + uint64_t offsetBias; |
| 64 | - uint64_t offsetBias; | 64 | + uint64_t batchCOffset; |
| 65 | - uint64_t batchCOffset; | 65 | + uint64_t scaleScalar; |
| 66 | - uint64_t scaleScalar; | 66 | +}; |
| 67 | -}; | 67 | + |
| 68 | - | 68 | +class WeightQuantBmmIterBatchBlock { |
| 69 | -class WeightQuantBmmIterBatchBlock { | 69 | +public: |
| 70 | -public: | 70 | + __aicore__ inline WeightQuantBmmIterBatchBlock() {} |
| 71 | - __aicore__ inline WeightQuantBmmIterBatchBlock() {} | 71 | + __aicore__ inline void Init( |
| 72 | - __aicore__ inline void Init( | 72 | + const wqbmmv2_tiling::WeightQuantBatchMatmulV2ASWTilingDataParams* tilingData, uint32_t blockIdx); |
| 73 | - const wqbmmv2_tiling::WeightQuantBatchMatmulV2ASWTilingDataParams* tilingData, uint32_t blockIdx); | 73 | + __aicore__ inline void GetMultiBatchInfo(uint64_t loopIndex); |
| 74 | - __aicore__ inline void GetMultiBatchInfo(uint64_t loopIndex); | 74 | + __aicore__ inline void CalcGMOffset(); |
| 75 | - __aicore__ inline void CalcGMOffset(); | 75 | + __aicore__ inline void GetBroadcastBatchOffset(uint64_t b1Index, uint64_t b2Index, uint64_t b3Index); |
| 76 | - __aicore__ inline void GetBroadcastBatchOffset(uint64_t b1Index, uint64_t b2Index, uint64_t b3Index); | 76 | +public: |
| 77 | -public: | 77 | + IterBatchTilingParam params_; |
| 78 | - IterBatchTilingParam params_; | 78 | + IterBatchOffsetParam offset_; |
| 79 | - IterBatchOffsetParam offset_; | 79 | + const wqbmmv2_tiling::WeightQuantBatchMatmulV2ASWTilingDataParams* tilingData_; |
| 80 | - const wqbmmv2_tiling::WeightQuantBatchMatmulV2ASWTilingDataParams* tilingData_; | 80 | +protected: |
| 81 | -protected: | 81 | + __aicore__ inline void UpdateBatchInfo(); |
| 82 | - __aicore__ inline void UpdateBatchInfo(); | 82 | + __aicore__ inline void GetBatchInfo(uint64_t batchNum); |
| 83 | - __aicore__ inline void GetBatchInfo(uint64_t batchNum); | 83 | +private: |
| 84 | -private: | 84 | + uint32_t blockIdx_; |
| 85 | - uint32_t blockIdx_; | 85 | +}; |
| 86 | -}; | 86 | + |
| 87 | - | 87 | +__aicore__ inline void WeightQuantBmmIterBatchBlock::Init( |
| 88 | -__aicore__ inline void WeightQuantBmmIterBatchBlock::Init( | 88 | + const wqbmmv2_tiling::WeightQuantBatchMatmulV2ASWTilingDataParams* tilingData, uint32_t blockIdx) |
| 89 | - const wqbmmv2_tiling::WeightQuantBatchMatmulV2ASWTilingDataParams* tilingData, uint32_t blockIdx) | 89 | +{ |
| 90 | -{ | 90 | + tilingData_ = tilingData; |
| 91 | - tilingData_ = tilingData; | 91 | + blockIdx_ = blockIdx; |
| 92 | - blockIdx_ = blockIdx; | 92 | + params_.singleASize = |
| 93 | - params_.singleASize = | 93 | + static_cast<uint64_t>(tilingData_->matmulTiling.M) * static_cast<uint64_t>(tilingData_->matmulTiling.Ka); |
| 94 | - static_cast<uint64_t>(tilingData_->matmulTiling.M) * static_cast<uint64_t>(tilingData_->matmulTiling.Ka); | 94 | + params_.singleBSize = |
| 95 | - params_.singleBSize = | 95 | + static_cast<uint64_t>(tilingData_->matmulTiling.N) * static_cast<uint64_t>(tilingData_->matmulTiling.Kb); |
| 96 | - static_cast<uint64_t>(tilingData_->matmulTiling.N) * static_cast<uint64_t>(tilingData_->matmulTiling.Kb); | 96 | + params_.singleCSize = |
| 97 | - params_.singleCSize = | 97 | + static_cast<uint64_t>(tilingData_->matmulTiling.M) * static_cast<uint64_t>(tilingData_->matmulTiling.N); |
| 98 | - static_cast<uint64_t>(tilingData_->matmulTiling.M) * static_cast<uint64_t>(tilingData_->matmulTiling.N); | 98 | + |
| 99 | - | 99 | + params_.useCoreNum = tilingData_->matmulTiling.usedCoreNum; |
| 100 | - params_.useCoreNum = tilingData_->matmulTiling.usedCoreNum; | 100 | + params_.mainLoopPerCoreBatchNum = tilingData_->matmulTiling.BatchNum; |
| 101 | - params_.mainLoopPerCoreBatchNum = tilingData_->matmulTiling.BatchNum; | 101 | + params_.nBatchOutNum = Min(L0C_SIZE_256K / (tilingData_->matmulTiling.baseM * tilingData_->matmulTiling.baseN * |
| 102 | - params_.nBatchOutNum = Min(L0C_SIZE_256K / (tilingData_->matmulTiling.baseM * tilingData_->matmulTiling.baseN * | 102 | + tilingData_->matmulTiling.dbL0C * sizeof(int32_t)), |
| 103 | - tilingData_->matmulTiling.dbL0C * sizeof(int32_t)), | 103 | + static_cast<uint64_t>(tilingData_->matmulTiling.BatchNum)); |
| 104 | - static_cast<uint64_t>(tilingData_->matmulTiling.BatchNum)); | 104 | + UpdateBatchInfo(); |
| 105 | - UpdateBatchInfo(); | 105 | + params_.loopTimes = CeilDiv(params_.calcBatchNum, params_.mainLoopPerCoreBatchNum * params_.useCoreNum); |
| 106 | - params_.loopTimes = CeilDiv(params_.calcBatchNum, params_.mainLoopPerCoreBatchNum * params_.useCoreNum); | 106 | + params_.lastLoopAllBatchNum = params_.calcBatchNum % (params_.mainLoopPerCoreBatchNum * params_.useCoreNum); |
| 107 | - params_.lastLoopAllBatchNum = params_.calcBatchNum % (params_.mainLoopPerCoreBatchNum * params_.useCoreNum); | 107 | + params_.lastLoopAllBatchNum = params_.lastLoopAllBatchNum == 0 |
| 108 | - params_.lastLoopAllBatchNum = params_.lastLoopAllBatchNum == 0 | 108 | + ? params_.mainLoopPerCoreBatchNum * params_.useCoreNum |
| 109 | - ? params_.mainLoopPerCoreBatchNum * params_.useCoreNum | 109 | + : params_.lastLoopAllBatchNum; |
| 110 | - : params_.lastLoopAllBatchNum; | 110 | + |
| 111 | - | 111 | + params_.lastLoopPreCoreBatchNum = params_.lastLoopAllBatchNum / params_.useCoreNum; |
| 112 | - params_.lastLoopPreCoreBatchNum = params_.lastLoopAllBatchNum / params_.useCoreNum; | 112 | + params_.lastLoopBlockNum = params_.lastLoopAllBatchNum % params_.useCoreNum; |
| 113 | - params_.lastLoopBlockNum = params_.lastLoopAllBatchNum % params_.useCoreNum; | 113 | + params_.batchIndex = 0; |
| 114 | - params_.batchIndex = 0; | 114 | + params_.batchAIndex = 0; |
| 115 | - params_.batchAIndex = 0; | 115 | + params_.batchBIndex = 0; |
| 116 | - params_.batchBIndex = 0; | 116 | + params_.batchANum = 1; |
| 117 | - params_.batchANum = 1; | 117 | + params_.batchBNum = 1; |
| 118 | - params_.batchBNum = 1; | 118 | +} |
| 119 | -} | 119 | + |
| 120 | - | 120 | +__aicore__ inline void WeightQuantBmmIterBatchBlock::UpdateBatchInfo() |
| 121 | -__aicore__ inline void WeightQuantBmmIterBatchBlock::UpdateBatchInfo() | 121 | +{ |
| 122 | -{ | 122 | + if (tilingData_->params.batchA == 1 || tilingData_->params.batchB == 1 || |
| 123 | - if (tilingData_->params.batchA == 1 || tilingData_->params.batchB == 1 || | 123 | + tilingData_->params.batchA == tilingData_->params.batchB) { |
| 124 | - tilingData_->params.batchA == tilingData_->params.batchB) { | 124 | + // no need broadcast or batchA = 1 or batchB = 1 |
| 125 | - // no need broadcast or batchA = 1 or batchB = 1 | 125 | + params_.innerBatchNum = 0; |
| 126 | - params_.innerBatchNum = 0; | 126 | + params_.calcBatchNum = tilingData_->params.batchC; |
| 127 | - params_.calcBatchNum = tilingData_->params.batchC; | 127 | + params_.broadcastFlag = NO_NEED_BROADCAST; |
| 128 | - params_.broadcastFlag = NO_NEED_BROADCAST; | 128 | + } else if (tilingData_->params.batchA4 != tilingData_->params.batchB4) { |
| 129 | - } else if (tilingData_->params.batchA4 != tilingData_->params.batchB4) { | 129 | + // batch4 need broadcast |
| 130 | - // batch4 need broadcast | 130 | + params_.iterBatch3 = tilingData_->params.batchC3; |
| 131 | - params_.iterBatch3 = tilingData_->params.batchC3; | 131 | + params_.iterBatch2 = tilingData_->params.batchC2; |
| 132 | - params_.iterBatch2 = tilingData_->params.batchC2; | 132 | + params_.iterBatch1 = tilingData_->params.batchC1; |
| 133 | - params_.iterBatch1 = tilingData_->params.batchC1; | 133 | + params_.calcBatchNum = tilingData_->params.batchC4; |
| 134 | - params_.calcBatchNum = tilingData_->params.batchC4; | 134 | + params_.innerBatchNum = 1; |
| 135 | - params_.innerBatchNum = 1; | 135 | + params_.broadcastFlag = (tilingData_->params.batchA4 == 1) ? A_NEED_BROADCAST : B_NEED_BROADCAST; |
| 136 | - params_.broadcastFlag = (tilingData_->params.batchA4 == 1) ? A_NEED_BROADCAST : B_NEED_BROADCAST; | 136 | + } else if (tilingData_->params.batchA3 != tilingData_->params.batchB3) { |
| 137 | - } else if (tilingData_->params.batchA3 != tilingData_->params.batchB3) { | 137 | + // batch3 need broadcast |
| 138 | - // batch3 need broadcast | 138 | + params_.iterBatch3 = 1; |
| 139 | - params_.iterBatch3 = 1; | 139 | + params_.iterBatch2 = tilingData_->params.batchC2; |
| 140 | - params_.iterBatch2 = tilingData_->params.batchC2; | 140 | + params_.iterBatch1 = tilingData_->params.batchC1; |
| 141 | - params_.iterBatch1 = tilingData_->params.batchC1; | 141 | + params_.innerBatchNum = tilingData_->params.batchC4; |
| 142 | - params_.innerBatchNum = tilingData_->params.batchC4; | 142 | + params_.calcBatchNum = tilingData_->params.batchC4 * tilingData_->params.batchC3; |
| 143 | - params_.calcBatchNum = tilingData_->params.batchC4 * tilingData_->params.batchC3; | 143 | + params_.broadcastFlag = (tilingData_->params.batchA3 == 1) ? A_NEED_BROADCAST : B_NEED_BROADCAST; |
| 144 | - params_.broadcastFlag = (tilingData_->params.batchA3 == 1) ? A_NEED_BROADCAST : B_NEED_BROADCAST; | 144 | + } else if (tilingData_->params.batchA2 != tilingData_->params.batchB2) { |
| 145 | - } else if (tilingData_->params.batchA2 != tilingData_->params.batchB2) { | 145 | + // batch2 need broadcast |
| 146 | - // batch2 need broadcast | 146 | + params_.iterBatch3 = 1; |
| 147 | - params_.iterBatch3 = 1; | 147 | + params_.iterBatch2 = 1; |
| 148 | - params_.iterBatch2 = 1; | 148 | + params_.iterBatch1 = tilingData_->params.batchC1; |
| 149 | - params_.iterBatch1 = tilingData_->params.batchC1; | 149 | + params_.innerBatchNum = tilingData_->params.batchC4 * tilingData_->params.batchC3; |
| 150 | - params_.innerBatchNum = tilingData_->params.batchC4 * tilingData_->params.batchC3; | 150 | + params_.calcBatchNum = tilingData_->params.batchC4 * tilingData_->params.batchC3 * tilingData_->params.batchC2; |
| 151 | - params_.calcBatchNum = tilingData_->params.batchC4 * tilingData_->params.batchC3 * tilingData_->params.batchC2; | 151 | + params_.broadcastFlag = (tilingData_->params.batchA2 == 1) ? A_NEED_BROADCAST : B_NEED_BROADCAST; |
| 152 | - params_.broadcastFlag = (tilingData_->params.batchA2 == 1) ? A_NEED_BROADCAST : B_NEED_BROADCAST; | 152 | + } else { |
| 153 | - } else { | 153 | + // batch1 need broadcast |
| 154 | - // batch1 need broadcast | 154 | + params_.iterBatch3 = 1; |
| 155 | - params_.iterBatch3 = 1; | 155 | + params_.iterBatch2 = 1; |
| 156 | - params_.iterBatch2 = 1; | 156 | + params_.iterBatch1 = 1; |
| 157 | - params_.iterBatch1 = 1; | 157 | + params_.innerBatchNum = tilingData_->params.batchC4 * tilingData_->params.batchC3 * tilingData_->params.batchC2; |
| 158 | - params_.innerBatchNum = tilingData_->params.batchC4 * tilingData_->params.batchC3 * tilingData_->params.batchC2; | 158 | + params_.calcBatchNum = tilingData_->params.batchC; |
| 159 | - params_.calcBatchNum = tilingData_->params.batchC; | 159 | + params_.broadcastFlag = (tilingData_->params.batchA1 == 1) ? A_NEED_BROADCAST : B_NEED_BROADCAST; |
| 160 | - params_.broadcastFlag = (tilingData_->params.batchA1 == 1) ? A_NEED_BROADCAST : B_NEED_BROADCAST; | 160 | + } |
| 161 | - } | 161 | +} |
| 162 | -} | 162 | + |
| 163 | - | 163 | +__aicore__ inline void WeightQuantBmmIterBatchBlock::GetBatchInfo(uint64_t batchNum) |
| 164 | -__aicore__ inline void WeightQuantBmmIterBatchBlock::GetBatchInfo(uint64_t batchNum) | 164 | +{ |
| 165 | -{ | 165 | + if (params_.innerBatchNum == 0) { |
| 166 | - if (params_.innerBatchNum == 0) { | 166 | + params_.batchANum = Min(batchNum, static_cast<uint64_t>(tilingData_->params.batchA)); |
| 167 | - params_.batchANum = Min(batchNum, static_cast<uint64_t>(tilingData_->params.batchA)); | 167 | + params_.batchBNum = Min(batchNum, static_cast<uint64_t>(tilingData_->params.batchB)); |
| 168 | - params_.batchBNum = Min(batchNum, static_cast<uint64_t>(tilingData_->params.batchB)); | 168 | + params_.batchAIndex = Min(params_.batchIndex, static_cast<uint64_t>(tilingData_->params.batchA - 1)); |
| 169 | - params_.batchAIndex = Min(params_.batchIndex, static_cast<uint64_t>(tilingData_->params.batchA - 1)); | 169 | + params_.batchBIndex = Min(params_.batchIndex, static_cast<uint64_t>(tilingData_->params.batchB - 1)); |
| 170 | - params_.batchBIndex = Min(params_.batchIndex, static_cast<uint64_t>(tilingData_->params.batchB - 1)); | 170 | + } else if (params_.batchIndex >= (params_.innerBatchNum - 1) && params_.broadcastFlag == A_NEED_BROADCAST) { |
| 171 | - } else if (params_.batchIndex >= (params_.innerBatchNum - 1) && params_.broadcastFlag == A_NEED_BROADCAST) { | 171 | + // 当本次计算需要使用的batch数量超过需要broadcast的batch轴的内轴数量，且A矩阵的batch需要broadcast |
| 172 | - // 当本次计算需要使用的batch数量超过需要broadcast的batch轴的内轴数量，且A矩阵的batch需要broadcast | 172 | + params_.batchANum = Min(params_.innerBatchNum, batchNum); |
| 173 | - params_.batchANum = Min(params_.innerBatchNum, batchNum); | 173 | + params_.batchBNum = batchNum; |
| 174 | - params_.batchBNum = batchNum; | 174 | + params_.batchAIndex = params_.batchIndex % params_.innerBatchNum; |
| 175 | - params_.batchAIndex = params_.batchIndex % params_.innerBatchNum; | 175 | + params_.batchBIndex = params_.batchIndex; |
| 176 | - params_.batchBIndex = params_.batchIndex; | 176 | + } else if (params_.batchIndex >= (params_.innerBatchNum - 1) && params_.broadcastFlag == B_NEED_BROADCAST) { |
| 177 | - } else if (params_.batchIndex >= (params_.innerBatchNum - 1) && params_.broadcastFlag == B_NEED_BROADCAST) { | 177 | + // 当本次计算需要使用的batch数量超过需要broadcast的batch轴的内轴数量，且B矩阵的batch需要broadcast |
| 178 | - // 当本次计算需要使用的batch数量超过需要broadcast的batch轴的内轴数量，且B矩阵的batch需要broadcast | 178 | + params_.batchANum = batchNum; |
| 179 | - params_.batchANum = batchNum; | 179 | + params_.batchBNum = Min(params_.innerBatchNum, batchNum); |
| 180 | - params_.batchBNum = Min(params_.innerBatchNum, batchNum); | 180 | + params_.batchAIndex = params_.batchIndex; |
| 181 | - params_.batchAIndex = params_.batchIndex; | 181 | + params_.batchBIndex = params_.batchIndex % params_.innerBatchNum; |
| 182 | - params_.batchBIndex = params_.batchIndex % params_.innerBatchNum; | 182 | + } else { |
| 183 | - } else { | 183 | + params_.batchANum = batchNum; |
| 184 | - params_.batchANum = batchNum; | 184 | + params_.batchBNum = batchNum; |
| 185 | - params_.batchBNum = batchNum; | 185 | + params_.batchAIndex = params_.batchIndex; |
| 186 | - params_.batchAIndex = params_.batchIndex; | 186 | + params_.batchBIndex = params_.batchIndex; |
| 187 | - params_.batchBIndex = params_.batchIndex; | 187 | + } |
| 188 | - } | 188 | +} |
| 189 | -} | 189 | + |
| 190 | - | 190 | +__aicore__ inline void WeightQuantBmmIterBatchBlock::GetMultiBatchInfo(uint64_t loopIndex) |
| 191 | -__aicore__ inline void WeightQuantBmmIterBatchBlock::GetMultiBatchInfo(uint64_t loopIndex) | 191 | +{ |
| 192 | -{ | 192 | + // main loop |
| 193 | - // main loop | 193 | + if (loopIndex + 1 < params_.loopTimes) { |
| 194 | - if (loopIndex + 1 < params_.loopTimes) { | 194 | + params_.batchIndex = loopIndex * params_.mainLoopPerCoreBatchNum * params_.useCoreNum + |
| 195 | - params_.batchIndex = loopIndex * params_.mainLoopPerCoreBatchNum * params_.useCoreNum + | 195 | + blockIdx_ * params_.mainLoopPerCoreBatchNum; |
| 196 | - blockIdx_ * params_.mainLoopPerCoreBatchNum; | 196 | + GetBatchInfo(params_.mainLoopPerCoreBatchNum); |
| 197 | - GetBatchInfo(params_.mainLoopPerCoreBatchNum); | 197 | + } else if (blockIdx_ < params_.lastLoopBlockNum) { // last loop |
| 198 | - } else if (blockIdx_ < params_.lastLoopBlockNum) { // last loop | 198 | + params_.batchIndex = loopIndex * params_.mainLoopPerCoreBatchNum * params_.useCoreNum + |
| 199 | - params_.batchIndex = loopIndex * params_.mainLoopPerCoreBatchNum * params_.useCoreNum + | 199 | + blockIdx_ * (params_.lastLoopPreCoreBatchNum + 1); |
| 200 | - blockIdx_ * (params_.lastLoopPreCoreBatchNum + 1); | 200 | + GetBatchInfo(params_.lastLoopPreCoreBatchNum + 1); |
| 201 | - GetBatchInfo(params_.lastLoopPreCoreBatchNum + 1); | 201 | + } else { |
| 202 | - } else { | 202 | + params_.batchIndex = loopIndex * params_.mainLoopPerCoreBatchNum * params_.useCoreNum + |
| 203 | - params_.batchIndex = loopIndex * params_.mainLoopPerCoreBatchNum * params_.useCoreNum + | 203 | + params_.lastLoopBlockNum * (params_.lastLoopPreCoreBatchNum + 1) + |
| 204 | - params_.lastLoopBlockNum * (params_.lastLoopPreCoreBatchNum + 1) + | 204 | + (blockIdx_ - params_.lastLoopBlockNum) * params_.lastLoopPreCoreBatchNum; |
| 205 | - (blockIdx_ - params_.lastLoopBlockNum) * params_.lastLoopPreCoreBatchNum; | 205 | + |
| 206 | - | 206 | + GetBatchInfo(params_.lastLoopPreCoreBatchNum); |
| 207 | - GetBatchInfo(params_.lastLoopPreCoreBatchNum); | 207 | + } |
| 208 | - } | 208 | +} |
| 209 | -} | 209 | + |
| 210 | - | 210 | +__aicore__ inline void WeightQuantBmmIterBatchBlock::GetBroadcastBatchOffset(uint64_t b1Index, uint64_t b2Index, |
| 211 | -__aicore__ inline void WeightQuantBmmIterBatchBlock::GetBroadcastBatchOffset(uint64_t b1Index, uint64_t b2Index, | 211 | + uint64_t b3Index) |
| 212 | - uint64_t b3Index) | 212 | +{ |
| 213 | -{ | 213 | + if (params_.iterBatch1 != tilingData_->params.batchC1) { |
| 214 | - if (params_.iterBatch1 != tilingData_->params.batchC1) { | 214 | + // batch1 need broadcast |
| 215 | - // batch1 need broadcast | 215 | + offset_.batchCOffset = 0; |
| 216 | - offset_.batchCOffset = 0; | 216 | + } else if (params_.iterBatch2 != tilingData_->params.batchC2) { |
| 217 | - } else if (params_.iterBatch2 != tilingData_->params.batchC2) { | 217 | + // batch2 need broadcast |
| 218 | - // batch2 need broadcast | 218 | + offset_.batchCOffset = b1Index; |
| 219 | - offset_.batchCOffset = b1Index; | 219 | + } else if (params_.iterBatch3 != tilingData_->params.batchC3) { |
| 220 | - } else if (params_.iterBatch3 != tilingData_->params.batchC3) { | 220 | + // batch3 need broadcast |
| 221 | - // batch3 need broadcast | 221 | + offset_.batchCOffset = b2Index + b1Index * params_.iterBatch2; |
| 222 | - offset_.batchCOffset = b2Index + b1Index * params_.iterBatch2; | 222 | + } else { |
| 223 | - } else { | 223 | + // batch4 need broadcast |
| 224 | - // batch4 need broadcast | 224 | + offset_.batchCOffset = b3Index + b2Index * params_.iterBatch3 + b1Index * params_.iterBatch2; |
| 225 | - offset_.batchCOffset = b3Index + b2Index * params_.iterBatch3 + b1Index * params_.iterBatch2; | 225 | + } |
| 226 | - } | 226 | +} |
| 227 | -} | 227 | + |
| 228 | - | 228 | +__aicore__ inline void WeightQuantBmmIterBatchBlock::CalcGMOffset() |
| 229 | -__aicore__ inline void WeightQuantBmmIterBatchBlock::CalcGMOffset() | 229 | +{ |
| 230 | -{ | 230 | + if (params_.broadcastFlag == A_NEED_BROADCAST) { |
| 231 | - if (params_.broadcastFlag == A_NEED_BROADCAST) { | 231 | + offset_.offsetA = params_.batchAIndex * params_.singleASize + |
| 232 | - offset_.offsetA = params_.batchAIndex * params_.singleASize + | 232 | + offset_.batchCOffset * params_.innerBatchNum * params_.singleASize; |
| 233 | - offset_.batchCOffset * params_.innerBatchNum * params_.singleASize; | 233 | + offset_.offsetB = params_.batchBIndex * params_.singleBSize + |
| 234 | - offset_.offsetB = params_.batchBIndex * params_.singleBSize + | 234 | + offset_.batchCOffset * params_.calcBatchNum * params_.singleBSize; |
| 235 | - offset_.batchCOffset * params_.calcBatchNum * params_.singleBSize; | 235 | + offset_.offsetC = params_.batchIndex * params_.singleCSize + |
| 236 | - offset_.offsetC = params_.batchIndex * params_.singleCSize + | 236 | + offset_.batchCOffset * params_.calcBatchNum * params_.singleCSize; |
| 237 | - offset_.batchCOffset * params_.calcBatchNum * params_.singleCSize; | 237 | + } else if (params_.broadcastFlag == B_NEED_BROADCAST) { |
| 238 | - } else if (params_.broadcastFlag == B_NEED_BROADCAST) { | 238 | + offset_.offsetA = params_.batchAIndex * params_.singleASize + |
| 239 | - offset_.offsetA = params_.batchAIndex * params_.singleASize + | 239 | + offset_.batchCOffset * params_.calcBatchNum * params_.singleASize; |
| 240 | - offset_.batchCOffset * params_.calcBatchNum * params_.singleASize; | 240 | + offset_.offsetB = params_.batchBIndex * params_.singleBSize + |
| 241 | - offset_.offsetB = params_.batchBIndex * params_.singleBSize + | 241 | + offset_.batchCOffset * params_.innerBatchNum * params_.singleBSize; |
| 242 | - offset_.batchCOffset * params_.innerBatchNum * params_.singleBSize; | 242 | + offset_.offsetC = params_.batchIndex * params_.singleCSize + |
| 243 | - offset_.offsetC = params_.batchIndex * params_.singleCSize + | 243 | + offset_.batchCOffset * params_.calcBatchNum * params_.singleCSize; |
| 244 | - offset_.batchCOffset * params_.calcBatchNum * params_.singleCSize; | 244 | + } else { |
| 245 | - } else { | 245 | + offset_.offsetA = params_.batchAIndex * params_.singleASize; |
| 246 | - offset_.offsetA = params_.batchAIndex * params_.singleASize; | 246 | + offset_.offsetB = params_.batchBIndex * params_.singleBSize; |
| 247 | - offset_.offsetB = params_.batchBIndex * params_.singleBSize; | 247 | + offset_.offsetC = params_.batchIndex * params_.singleCSize; |
| 248 | - offset_.offsetC = params_.batchIndex * params_.singleCSize; | 248 | + } |
| 249 | - } | 249 | + offset_.offsetScale = 0; |
| 250 | - offset_.offsetScale = 0; | 250 | + offset_.offsetBias = 0; |
| 251 | - offset_.offsetBias = 0; | 251 | +} |
| 252 | -} | 252 | +} // namespace WeightQuantBatchMatmulV2::Arch35 |
| 253 | -} // namespace WeightQuantBatchMatmulV2::Arch35 | 253 | + |
| 254 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef WEIGHT_QUANT_BATCHMATMUL_V2_REG_BASE_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -63,4 +62,4 @@ __aicore__ inline void WeightQuantBatchMatmulV2RegBaseKernel< | |||
| 63 | this->EndSync(eventIdsMte1ToMte2, biasEventIdsMte1ToMte2); | 62 | this->EndSync(eventIdsMte1ToMte2, biasEventIdsMte1ToMte2); |
| 64 | } | 63 | } |
| 65 | } // namespace WeightQuantBatchMatmulV2::Arch35 | 64 | } // namespace WeightQuantBatchMatmulV2::Arch35 |
| 66 | -#endif // WEIGHT_QUANT_BATCHMATMUL_V2_REG_BASE_H | 65 | + |
Mmatmul/weight_quant_batch_matmul_v2/op_kernel/arch35/weight_quant_batch_matmul_v2_reg_base_common.h+1-3
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef WEIGHT_QUANT_BATCHMATMUL_V2_REG_BASE_COMMON_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -2260,4 +2259,3 @@ __aicore__ inline void WeightQuantBatchMatmulV2RegBaseCommonKernel< | |||
| 2260 | } | 2259 | } |
| 2261 | } // namespace WeightQuantBatchMatmulV2::Arch35 | 2260 | } // namespace WeightQuantBatchMatmulV2::Arch35 |
| 2262 | 2261 | ||
| 2263 | - | ||
| @@ -13,8 +13,7 @@ | |||
| 13 | * \brief | 13 | * \brief |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | -#ifndef WEIGHT_QUANT_BATCHMATMUL_V2_VF_H | 16 | +#pragma once |
| 17 | - | ||
| 18 | 17 | ||
| 19 | 18 | ||
| 20 | 19 | ||
| @@ -1034,4 +1033,3 @@ __aicore__ inline void AntiquantW4Pergroup32OddNK(ParamsGroupSize32OddNK<XType>& | |||
| 1034 | 1033 | ||
| 1035 | } // namespace WeightQuantBatchMatmulV2::Arch35 | 1034 | } // namespace WeightQuantBatchMatmulV2::Arch35 |
| 1036 | 1035 | ||
| 1037 | - | ||