Merged
fix/a6-kc-quant-core-ratio #14301
fix/a6-kc-quant-core-ratio #14301
Merged
huangyuxiaaaaaAdd on 28 days ago
Total: 3 files changed+55-36
@@ -97,12 +97,16 @@ protected:
97 __aicore__ inline void WaitForVector()97 __aicore__ inline void WaitForVector()
98 {98 {
99 AscendC::CrossCoreWaitFlag<AIC_SYNC_AIV_MODE, PIPE_FIX>(AIV_SYNC_AIC_FLAG);99 AscendC::CrossCoreWaitFlag<AIC_SYNC_AIV_MODE, PIPE_FIX>(AIV_SYNC_AIC_FLAG);
100- AscendC::CrossCoreWaitFlag<AIC_SYNC_AIV_MODE, PIPE_FIX>(AIV_SYNC_AIC_FLAG + FLAG_ID_MAX);100+ if constexpr (CV_RATIO == 2) {
101+ AscendC::CrossCoreWaitFlag<AIC_SYNC_AIV_MODE, PIPE_FIX>(AIV_SYNC_AIC_FLAG + FLAG_ID_MAX);
102+ }
101 }103 }
102 __aicore__ inline void NotifyVector()104 __aicore__ inline void NotifyVector()
103 {105 {
104 AscendC::CrossCoreSetFlag<AIC_SYNC_AIV_MODE, PIPE_FIX>(AIC_SYNC_AIV_FLAG);106 AscendC::CrossCoreSetFlag<AIC_SYNC_AIV_MODE, PIPE_FIX>(AIC_SYNC_AIV_FLAG);
105- AscendC::CrossCoreSetFlag<AIC_SYNC_AIV_MODE, PIPE_FIX>(AIC_SYNC_AIV_FLAG + FLAG_ID_MAX);107+ if constexpr (CV_RATIO == 2) {
108+ AscendC::CrossCoreSetFlag<AIC_SYNC_AIV_MODE, PIPE_FIX>(AIC_SYNC_AIV_FLAG + FLAG_ID_MAX);
109+ }
106 }110 }
107 __aicore__ inline void WaitForCube()111 __aicore__ inline void WaitForCube()
108 {112 {
@@ -156,7 +160,7 @@ __aicore__ inline void Mc2QuantBmmPertokenRegbaseKernel<LOCAL_TEMPLATE_FUNC_MIX_
156{160{
157 blockIdx_ = AscendC::GetBlockIdx();161 blockIdx_ = AscendC::GetBlockIdx();
158 if ASCEND_IS_AIV {162 if ASCEND_IS_AIV {
159- blockIdx_ = blockIdx_ / AscendC::GetTaskRation();163+ blockIdx_ = blockIdx_ / CV_RATIO;
160 subBlockIdx_ = AscendC::GetSubBlockIdx();164 subBlockIdx_ = AscendC::GetSubBlockIdx();
161 }165 }
162 pipe_ = pipe;166 pipe_ = pipe;
@@ -344,7 +348,8 @@ __aicore__ inline void Mc2QuantBmmPertokenRegbaseKernel<LOCAL_TEMPLATE_FUNC_MIX_
344LOCAL_TEMPLATE_CLASS_MIX_PARAMS348LOCAL_TEMPLATE_CLASS_MIX_PARAMS
345__aicore__ inline void Mc2QuantBmmPertokenRegbaseKernel<LOCAL_TEMPLATE_FUNC_MIX_PARAMS>::DequantCompute()349__aicore__ inline void Mc2QuantBmmPertokenRegbaseKernel<LOCAL_TEMPLATE_FUNC_MIX_PARAMS>::DequantCompute()
346{350{
347- auto halfSingleM = DequantBmm::CeilDiv(block_.params_.singleCoreM, static_cast<uint64_t>(2)); // 分配给2个AIV计算351+ auto halfSingleM =
352+ DequantBmm::CeilDiv(block_.params_.singleCoreM, static_cast<uint64_t>(CV_RATIO)); // 分配给AIV计算
348 auto singleMInVec = subBlockIdx_ == 1 ? block_.params_.singleCoreM - halfSingleM : halfSingleM;353 auto singleMInVec = subBlockIdx_ == 1 ? block_.params_.singleCoreM - halfSingleM : halfSingleM;
349 if (singleMInVec == 0) {354 if (singleMInVec == 0) {
350 return;355 return;
@@ -389,7 +394,8 @@ __aicore__ inline void Mc2QuantBmmPertokenRegbaseKernel<LOCAL_TEMPLATE_FUNC_MIX_
389LOCAL_TEMPLATE_CLASS_MIX_PARAMS394LOCAL_TEMPLATE_CLASS_MIX_PARAMS
390__aicore__ inline void Mc2QuantBmmPertokenRegbaseKernel<LOCAL_TEMPLATE_FUNC_MIX_PARAMS>::CopyDataFromGm2Ub()395__aicore__ inline void Mc2QuantBmmPertokenRegbaseKernel<LOCAL_TEMPLATE_FUNC_MIX_PARAMS>::CopyDataFromGm2Ub()
391{396{
392- auto halfSingleM = DequantBmm::CeilDiv(block_.params_.singleCoreM, static_cast<uint64_t>(2)); // 分配给2个AIV计算397+ auto halfSingleM =
398+ DequantBmm::CeilDiv(block_.params_.singleCoreM, static_cast<uint64_t>(CV_RATIO)); // 分配给AIV计算
393 auto singleMInVec = subBlockIdx_ == 1 ? block_.params_.singleCoreM - halfSingleM : halfSingleM;399 auto singleMInVec = subBlockIdx_ == 1 ? block_.params_.singleCoreM - halfSingleM : halfSingleM;
394 // scale: GM -> UB400 // scale: GM -> UB
395 if (!static_cast<bool>(tilingData_->params.isPerTensor)) {401 if (!static_cast<bool>(tilingData_->params.isPerTensor)) {
@@ -48,6 +48,11 @@ protected:
48 static constexpr uint32_t UB_DATABLOCK = 32;48 static constexpr uint32_t UB_DATABLOCK = 32;
49 static constexpr uint32_t COMPARE_ALIGN_LEN = 256;49 static constexpr uint32_t COMPARE_ALIGN_LEN = 256;
50 static constexpr uint32_t VECTOR_UB_SIZE = AscendC::TOTAL_UB_SIZE;50 static constexpr uint32_t VECTOR_UB_SIZE = AscendC::TOTAL_UB_SIZE;
51+#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201)
52+ static constexpr uint32_t AIV_TO_AIC_RATIO = ONE_FACTOR;
53+#else
54+ static constexpr uint32_t AIV_TO_AIC_RATIO = TWO_FACTOR;
55+#endif
51 56 
52 static constexpr float FP8_E5M2_MAX_VALUE = 57344.0f;57 static constexpr float FP8_E5M2_MAX_VALUE = 57344.0f;
53 static constexpr float FP8_E4M3FN_MAX_VALUE = 448.0f;58 static constexpr float FP8_E4M3FN_MAX_VALUE = 448.0f;
@@ -169,7 +174,7 @@ __aicore__ inline void Fp8DynamicQuantPertoken<quantInputDataType, quantOutputDa
169 174 
170 tPipe_->Reset();175 tPipe_->Reset();
171 176 
172- uint64_t totalCores = static_cast<uint64_t>(GetBlockNum() * TWO_FACTOR);177+ const uint64_t totalCores = static_cast<uint64_t>(GetBlockNum()) * AIV_TO_AIC_RATIO;
173 this->usedCoreAivNum_ = (context_.rowNum < totalCores) ? context_.rowNum : totalCores;178 this->usedCoreAivNum_ = (context_.rowNum < totalCores) ? context_.rowNum : totalCores;
174 // 2. 均匀分配任务到各个核179 // 2. 均匀分配任务到各个核
175 uint32_t coreIdx = GetBlockIdx();180 uint32_t coreIdx = GetBlockIdx();
@@ -36,6 +36,12 @@
36#define ADDBIAS_N_SPLIT 236#define ADDBIAS_N_SPLIT 2
37#define ADDBIAS_MN_SPLIT 337#define ADDBIAS_MN_SPLIT 3
38 38 
39+#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201)
40+#define MATMUL_ALLTOALL_ORIGINAL_KERNEL_TYPE_SEL ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_MIX_AIC_1_1),
41+#else
42+#define MATMUL_ALLTOALL_ORIGINAL_KERNEL_TYPE_SEL ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_MIX_AIC_1_2),
43+#endif
44+ 
39// 模板参数范围声明45// 模板参数范围声明
40ASCENDC_TPL_ARGS_DECL(MatmulAlltoAll,46ASCENDC_TPL_ARGS_DECL(MatmulAlltoAll,
41 ASCENDC_TPL_UINT_DECL(QUANTMODE, ASCENDC_TPL_4_BW, ASCENDC_TPL_UI_LIST, NON_QUANT_MODE,47 ASCENDC_TPL_UINT_DECL(QUANTMODE, ASCENDC_TPL_4_BW, ASCENDC_TPL_UI_LIST, NON_QUANT_MODE,
@@ -54,35 +60,37 @@ ASCENDC_TPL_ARGS_DECL(MatmulAlltoAll,
54// KC/MX_QUANT_MODE: ADDBIASSPLITMODE 恒为 OFF60// KC/MX_QUANT_MODE: ADDBIASSPLITMODE 恒为 OFF
55// MX_QUANT_MODE + USING_APACE_IMPL(1): Apace 路径, AIC only, 仅支持 X2TRANSPOSE=161// MX_QUANT_MODE + USING_APACE_IMPL(1): Apace 路径, AIC only, 仅支持 X2TRANSPOSE=1
56// 用于调用GET_TPL_TILING_KEY获取TilingKey时,接口内部校验TilingKey是否合法62// 用于调用GET_TPL_TILING_KEY获取TilingKey时,接口内部校验TilingKey是否合法
57-ASCENDC_TPL_SEL(ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_MIX_AIC_1_2),63+ASCENDC_TPL_SEL(
58- ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST, NON_QUANT_MODE),64+ ASCENDC_TPL_ARGS_SEL(MATMUL_ALLTOALL_ORIGINAL_KERNEL_TYPE_SEL ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST,
59- ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 0, 1),65+ NON_QUANT_MODE),
60- ASCENDC_TPL_UINT_SEL(DTYPEBIAS, ASCENDC_TPL_UI_LIST, DTYPE_BIAS_SAME_WITH_X,66+ ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 0, 1),
61- DTYPE_BIAS_FP32),67+ ASCENDC_TPL_UINT_SEL(DTYPEBIAS, ASCENDC_TPL_UI_LIST, DTYPE_BIAS_SAME_WITH_X, DTYPE_BIAS_FP32),
62- ASCENDC_TPL_UINT_SEL(COMMMODE, ASCENDC_TPL_UI_LIST, COMMMODE_CCU, COMMMODE_AICPU),68+ ASCENDC_TPL_UINT_SEL(COMMMODE, ASCENDC_TPL_UI_LIST, COMMMODE_CCU, COMMMODE_AICPU),
63- ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF,69+ ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF, ADDBIAS_M_SPLIT,
64- ADDBIAS_M_SPLIT, ADDBIAS_N_SPLIT, ADDBIAS_MN_SPLIT),70+ ADDBIAS_N_SPLIT, ADDBIAS_MN_SPLIT),
65- ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 0)),71+ ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 0)),
66- ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_MIX_AIC_1_2),72+ ASCENDC_TPL_ARGS_SEL(MATMUL_ALLTOALL_ORIGINAL_KERNEL_TYPE_SEL ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST,
67- ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST, KC_QUANT_MODE),73+ KC_QUANT_MODE),
68- ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 0, 1),74+ ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 0, 1),
69- ASCENDC_TPL_UINT_SEL(DTYPEBIAS, ASCENDC_TPL_UI_LIST, DTYPE_BIAS_FP32),75+ ASCENDC_TPL_UINT_SEL(DTYPEBIAS, ASCENDC_TPL_UI_LIST, DTYPE_BIAS_FP32),
70- ASCENDC_TPL_UINT_SEL(COMMMODE, ASCENDC_TPL_UI_LIST, COMMMODE_CCU, COMMMODE_AICPU),76+ ASCENDC_TPL_UINT_SEL(COMMMODE, ASCENDC_TPL_UI_LIST, COMMMODE_CCU, COMMMODE_AICPU),
71- ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF),77+ ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF),
72- ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 0)),78+ ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 0)),
73- ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_MIX_AIC_1_2),79+ ASCENDC_TPL_ARGS_SEL(MATMUL_ALLTOALL_ORIGINAL_KERNEL_TYPE_SEL ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST,
74- ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST, MX_QUANT_MODE),80+ MX_QUANT_MODE),
75- ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 0, 1),81+ ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 0, 1),
76- ASCENDC_TPL_UINT_SEL(DTYPEBIAS, ASCENDC_TPL_UI_LIST, DTYPE_BIAS_FP32),82+ ASCENDC_TPL_UINT_SEL(DTYPEBIAS, ASCENDC_TPL_UI_LIST, DTYPE_BIAS_FP32),
77- ASCENDC_TPL_UINT_SEL(COMMMODE, ASCENDC_TPL_UI_LIST, COMMMODE_CCU, COMMMODE_AICPU),83+ ASCENDC_TPL_UINT_SEL(COMMMODE, ASCENDC_TPL_UI_LIST, COMMMODE_CCU, COMMMODE_AICPU),
78- ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF),84+ ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF),
79- ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 0)),85+ ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 0)),
80- ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY),86+ ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY),
81- ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST, MX_QUANT_MODE),87+ ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST, MX_QUANT_MODE),
82- ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 1),88+ ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 1),
83- ASCENDC_TPL_UINT_SEL(DTYPEBIAS, ASCENDC_TPL_UI_LIST, DTYPE_BIAS_FP32),89+ ASCENDC_TPL_UINT_SEL(DTYPEBIAS, ASCENDC_TPL_UI_LIST, DTYPE_BIAS_FP32),
84- ASCENDC_TPL_UINT_SEL(COMMMODE, ASCENDC_TPL_UI_LIST, COMMMODE_CCU, COMMMODE_AICPU),90+ ASCENDC_TPL_UINT_SEL(COMMMODE, ASCENDC_TPL_UI_LIST, COMMMODE_CCU, COMMMODE_AICPU),
85- ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF),91+ ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF),
86- ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 1)));92+ ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 1)));
93+ 
94+#undef MATMUL_ALLTOALL_ORIGINAL_KERNEL_TYPE_SEL
87 95 
88#endif // MATMUL_ALLTO_ALL_TILING_KEY_H96#endif // MATMUL_ALLTO_ALL_TILING_KEY_H