Merged
fix/a6-kc-quant-core-ratio #14301
huangyuxiaaaaaAdd on 28 days ago
fix/a6-kc-quant-core-ratio #14301
Merged
Total: 3 files changed+55-36
| @@ -97,12 +97,16 @@ protected: | |||
| 97 | __aicore__ inline void WaitForVector() | 97 | __aicore__ inline void WaitForVector() |
| 98 | { | 98 | { |
| 99 | AscendC::CrossCoreWaitFlag<AIC_SYNC_AIV_MODE, PIPE_FIX>(AIV_SYNC_AIC_FLAG); | 99 | AscendC::CrossCoreWaitFlag<AIC_SYNC_AIV_MODE, PIPE_FIX>(AIV_SYNC_AIC_FLAG); |
| 100 | - AscendC::CrossCoreWaitFlag<AIC_SYNC_AIV_MODE, PIPE_FIX>(AIV_SYNC_AIC_FLAG + FLAG_ID_MAX); | 100 | + if constexpr (CV_RATIO == 2) { |
| 101 | + AscendC::CrossCoreWaitFlag<AIC_SYNC_AIV_MODE, PIPE_FIX>(AIV_SYNC_AIC_FLAG + FLAG_ID_MAX); | ||
| 102 | + } | ||
| 101 | } | 103 | } |
| 102 | __aicore__ inline void NotifyVector() | 104 | __aicore__ inline void NotifyVector() |
| 103 | { | 105 | { |
| 104 | AscendC::CrossCoreSetFlag<AIC_SYNC_AIV_MODE, PIPE_FIX>(AIC_SYNC_AIV_FLAG); | 106 | AscendC::CrossCoreSetFlag<AIC_SYNC_AIV_MODE, PIPE_FIX>(AIC_SYNC_AIV_FLAG); |
| 105 | - AscendC::CrossCoreSetFlag<AIC_SYNC_AIV_MODE, PIPE_FIX>(AIC_SYNC_AIV_FLAG + FLAG_ID_MAX); | 107 | + if constexpr (CV_RATIO == 2) { |
| 108 | + AscendC::CrossCoreSetFlag<AIC_SYNC_AIV_MODE, PIPE_FIX>(AIC_SYNC_AIV_FLAG + FLAG_ID_MAX); | ||
| 109 | + } | ||
| 106 | } | 110 | } |
| 107 | __aicore__ inline void WaitForCube() | 111 | __aicore__ inline void WaitForCube() |
| 108 | { | 112 | { |
| @@ -156,7 +160,7 @@ __aicore__ inline void Mc2QuantBmmPertokenRegbaseKernel<LOCAL_TEMPLATE_FUNC_MIX_ | |||
| 156 | { | 160 | { |
| 157 | blockIdx_ = AscendC::GetBlockIdx(); | 161 | blockIdx_ = AscendC::GetBlockIdx(); |
| 158 | if ASCEND_IS_AIV { | 162 | if ASCEND_IS_AIV { |
| 159 | - blockIdx_ = blockIdx_ / AscendC::GetTaskRation(); | 163 | + blockIdx_ = blockIdx_ / CV_RATIO; |
| 160 | subBlockIdx_ = AscendC::GetSubBlockIdx(); | 164 | subBlockIdx_ = AscendC::GetSubBlockIdx(); |
| 161 | } | 165 | } |
| 162 | pipe_ = pipe; | 166 | pipe_ = pipe; |
| @@ -344,7 +348,8 @@ __aicore__ inline void Mc2QuantBmmPertokenRegbaseKernel<LOCAL_TEMPLATE_FUNC_MIX_ | |||
| 344 | LOCAL_TEMPLATE_CLASS_MIX_PARAMS | 348 | LOCAL_TEMPLATE_CLASS_MIX_PARAMS |
| 345 | __aicore__ inline void Mc2QuantBmmPertokenRegbaseKernel<LOCAL_TEMPLATE_FUNC_MIX_PARAMS>::DequantCompute() | 349 | __aicore__ inline void Mc2QuantBmmPertokenRegbaseKernel<LOCAL_TEMPLATE_FUNC_MIX_PARAMS>::DequantCompute() |
| 346 | { | 350 | { |
| 347 | - auto halfSingleM = DequantBmm::CeilDiv(block_.params_.singleCoreM, static_cast<uint64_t>(2)); // 分配给2个AIV计算 | 351 | + auto halfSingleM = |
| 352 | + DequantBmm::CeilDiv(block_.params_.singleCoreM, static_cast<uint64_t>(CV_RATIO)); // 分配给AIV计算 | ||
| 348 | auto singleMInVec = subBlockIdx_ == 1 ? block_.params_.singleCoreM - halfSingleM : halfSingleM; | 353 | auto singleMInVec = subBlockIdx_ == 1 ? block_.params_.singleCoreM - halfSingleM : halfSingleM; |
| 349 | if (singleMInVec == 0) { | 354 | if (singleMInVec == 0) { |
| 350 | return; | 355 | return; |
| @@ -389,7 +394,8 @@ __aicore__ inline void Mc2QuantBmmPertokenRegbaseKernel<LOCAL_TEMPLATE_FUNC_MIX_ | |||
| 389 | LOCAL_TEMPLATE_CLASS_MIX_PARAMS | 394 | LOCAL_TEMPLATE_CLASS_MIX_PARAMS |
| 390 | __aicore__ inline void Mc2QuantBmmPertokenRegbaseKernel<LOCAL_TEMPLATE_FUNC_MIX_PARAMS>::CopyDataFromGm2Ub() | 395 | __aicore__ inline void Mc2QuantBmmPertokenRegbaseKernel<LOCAL_TEMPLATE_FUNC_MIX_PARAMS>::CopyDataFromGm2Ub() |
| 391 | { | 396 | { |
| 392 | - auto halfSingleM = DequantBmm::CeilDiv(block_.params_.singleCoreM, static_cast<uint64_t>(2)); // 分配给2个AIV计算 | 397 | + auto halfSingleM = |
| 398 | + DequantBmm::CeilDiv(block_.params_.singleCoreM, static_cast<uint64_t>(CV_RATIO)); // 分配给AIV计算 | ||
| 393 | auto singleMInVec = subBlockIdx_ == 1 ? block_.params_.singleCoreM - halfSingleM : halfSingleM; | 399 | auto singleMInVec = subBlockIdx_ == 1 ? block_.params_.singleCoreM - halfSingleM : halfSingleM; |
| 394 | // scale: GM -> UB | 400 | // scale: GM -> UB |
| 395 | if (!static_cast<bool>(tilingData_->params.isPerTensor)) { | 401 | if (!static_cast<bool>(tilingData_->params.isPerTensor)) { |
| @@ -48,6 +48,11 @@ protected: | |||
| 48 | static constexpr uint32_t UB_DATABLOCK = 32; | 48 | static constexpr uint32_t UB_DATABLOCK = 32; |
| 49 | static constexpr uint32_t COMPARE_ALIGN_LEN = 256; | 49 | static constexpr uint32_t COMPARE_ALIGN_LEN = 256; |
| 50 | static constexpr uint32_t VECTOR_UB_SIZE = AscendC::TOTAL_UB_SIZE; | 50 | static constexpr uint32_t VECTOR_UB_SIZE = AscendC::TOTAL_UB_SIZE; |
| 51 | +#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201) | ||
| 52 | + static constexpr uint32_t AIV_TO_AIC_RATIO = ONE_FACTOR; | ||
| 53 | +#else | ||
| 54 | + static constexpr uint32_t AIV_TO_AIC_RATIO = TWO_FACTOR; | ||
| 55 | +#endif | ||
| 51 | 56 | ||
| 52 | static constexpr float FP8_E5M2_MAX_VALUE = 57344.0f; | 57 | static constexpr float FP8_E5M2_MAX_VALUE = 57344.0f; |
| 53 | static constexpr float FP8_E4M3FN_MAX_VALUE = 448.0f; | 58 | static constexpr float FP8_E4M3FN_MAX_VALUE = 448.0f; |
| @@ -169,7 +174,7 @@ __aicore__ inline void Fp8DynamicQuantPertoken<quantInputDataType, quantOutputDa | |||
| 169 | 174 | ||
| 170 | tPipe_->Reset(); | 175 | tPipe_->Reset(); |
| 171 | 176 | ||
| 172 | - uint64_t totalCores = static_cast<uint64_t>(GetBlockNum() * TWO_FACTOR); | 177 | + const uint64_t totalCores = static_cast<uint64_t>(GetBlockNum()) * AIV_TO_AIC_RATIO; |
| 173 | this->usedCoreAivNum_ = (context_.rowNum < totalCores) ? context_.rowNum : totalCores; | 178 | this->usedCoreAivNum_ = (context_.rowNum < totalCores) ? context_.rowNum : totalCores; |
| 174 | // 2. 均匀分配任务到各个核 | 179 | // 2. 均匀分配任务到各个核 |
| 175 | uint32_t coreIdx = GetBlockIdx(); | 180 | uint32_t coreIdx = GetBlockIdx(); |
| @@ -36,6 +36,12 @@ | |||
| 36 | #define ADDBIAS_N_SPLIT 2 | 36 | #define ADDBIAS_N_SPLIT 2 |
| 37 | #define ADDBIAS_MN_SPLIT 3 | 37 | #define ADDBIAS_MN_SPLIT 3 |
| 38 | 38 | ||
| 39 | +#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 9201) | ||
| 40 | +#define MATMUL_ALLTOALL_ORIGINAL_KERNEL_TYPE_SEL ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_MIX_AIC_1_1), | ||
| 41 | +#else | ||
| 42 | +#define MATMUL_ALLTOALL_ORIGINAL_KERNEL_TYPE_SEL ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_MIX_AIC_1_2), | ||
| 43 | +#endif | ||
| 44 | + | ||
| 39 | // 模板参数范围声明 | 45 | // 模板参数范围声明 |
| 40 | ASCENDC_TPL_ARGS_DECL(MatmulAlltoAll, | 46 | ASCENDC_TPL_ARGS_DECL(MatmulAlltoAll, |
| 41 | ASCENDC_TPL_UINT_DECL(QUANTMODE, ASCENDC_TPL_4_BW, ASCENDC_TPL_UI_LIST, NON_QUANT_MODE, | 47 | ASCENDC_TPL_UINT_DECL(QUANTMODE, ASCENDC_TPL_4_BW, ASCENDC_TPL_UI_LIST, NON_QUANT_MODE, |
| @@ -54,35 +60,37 @@ ASCENDC_TPL_ARGS_DECL(MatmulAlltoAll, | |||
| 54 | // KC/MX_QUANT_MODE: ADDBIASSPLITMODE 恒为 OFF | 60 | // KC/MX_QUANT_MODE: ADDBIASSPLITMODE 恒为 OFF |
| 55 | // MX_QUANT_MODE + USING_APACE_IMPL(1): Apace 路径, AIC only, 仅支持 X2TRANSPOSE=1 | 61 | // MX_QUANT_MODE + USING_APACE_IMPL(1): Apace 路径, AIC only, 仅支持 X2TRANSPOSE=1 |
| 56 | // 用于调用GET_TPL_TILING_KEY获取TilingKey时,接口内部校验TilingKey是否合法 | 62 | // 用于调用GET_TPL_TILING_KEY获取TilingKey时,接口内部校验TilingKey是否合法 |
| 57 | -ASCENDC_TPL_SEL(ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_MIX_AIC_1_2), | 63 | +ASCENDC_TPL_SEL( |
| 58 | - ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST, NON_QUANT_MODE), | 64 | + ASCENDC_TPL_ARGS_SEL(MATMUL_ALLTOALL_ORIGINAL_KERNEL_TYPE_SEL ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST, |
| 59 | - ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 0, 1), | 65 | + NON_QUANT_MODE), |
| 60 | - ASCENDC_TPL_UINT_SEL(DTYPEBIAS, ASCENDC_TPL_UI_LIST, DTYPE_BIAS_SAME_WITH_X, | 66 | + ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 0, 1), |
| 61 | - DTYPE_BIAS_FP32), | 67 | + ASCENDC_TPL_UINT_SEL(DTYPEBIAS, ASCENDC_TPL_UI_LIST, DTYPE_BIAS_SAME_WITH_X, DTYPE_BIAS_FP32), |
| 62 | - ASCENDC_TPL_UINT_SEL(COMMMODE, ASCENDC_TPL_UI_LIST, COMMMODE_CCU, COMMMODE_AICPU), | 68 | + ASCENDC_TPL_UINT_SEL(COMMMODE, ASCENDC_TPL_UI_LIST, COMMMODE_CCU, COMMMODE_AICPU), |
| 63 | - ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF, | 69 | + ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF, ADDBIAS_M_SPLIT, |
| 64 | - ADDBIAS_M_SPLIT, ADDBIAS_N_SPLIT, ADDBIAS_MN_SPLIT), | 70 | + ADDBIAS_N_SPLIT, ADDBIAS_MN_SPLIT), |
| 65 | - ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 0)), | 71 | + ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 0)), |
| 66 | - ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_MIX_AIC_1_2), | 72 | + ASCENDC_TPL_ARGS_SEL(MATMUL_ALLTOALL_ORIGINAL_KERNEL_TYPE_SEL ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST, |
| 67 | - ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST, KC_QUANT_MODE), | 73 | + KC_QUANT_MODE), |
| 68 | - ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 0, 1), | 74 | + ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 0, 1), |
| 69 | - ASCENDC_TPL_UINT_SEL(DTYPEBIAS, ASCENDC_TPL_UI_LIST, DTYPE_BIAS_FP32), | 75 | + ASCENDC_TPL_UINT_SEL(DTYPEBIAS, ASCENDC_TPL_UI_LIST, DTYPE_BIAS_FP32), |
| 70 | - ASCENDC_TPL_UINT_SEL(COMMMODE, ASCENDC_TPL_UI_LIST, COMMMODE_CCU, COMMMODE_AICPU), | 76 | + ASCENDC_TPL_UINT_SEL(COMMMODE, ASCENDC_TPL_UI_LIST, COMMMODE_CCU, COMMMODE_AICPU), |
| 71 | - ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF), | 77 | + ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF), |
| 72 | - ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 0)), | 78 | + ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 0)), |
| 73 | - ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_MIX_AIC_1_2), | 79 | + ASCENDC_TPL_ARGS_SEL(MATMUL_ALLTOALL_ORIGINAL_KERNEL_TYPE_SEL ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST, |
| 74 | - ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST, MX_QUANT_MODE), | 80 | + MX_QUANT_MODE), |
| 75 | - ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 0, 1), | 81 | + ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 0, 1), |
| 76 | - ASCENDC_TPL_UINT_SEL(DTYPEBIAS, ASCENDC_TPL_UI_LIST, DTYPE_BIAS_FP32), | 82 | + ASCENDC_TPL_UINT_SEL(DTYPEBIAS, ASCENDC_TPL_UI_LIST, DTYPE_BIAS_FP32), |
| 77 | - ASCENDC_TPL_UINT_SEL(COMMMODE, ASCENDC_TPL_UI_LIST, COMMMODE_CCU, COMMMODE_AICPU), | 83 | + ASCENDC_TPL_UINT_SEL(COMMMODE, ASCENDC_TPL_UI_LIST, COMMMODE_CCU, COMMMODE_AICPU), |
| 78 | - ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF), | 84 | + ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF), |
| 79 | - ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 0)), | 85 | + ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 0)), |
| 80 | - ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY), | 86 | + ASCENDC_TPL_ARGS_SEL(ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY), |
| 81 | - ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST, MX_QUANT_MODE), | 87 | + ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST, MX_QUANT_MODE), |
| 82 | - ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 1), | 88 | + ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 1), |
| 83 | - ASCENDC_TPL_UINT_SEL(DTYPEBIAS, ASCENDC_TPL_UI_LIST, DTYPE_BIAS_FP32), | 89 | + ASCENDC_TPL_UINT_SEL(DTYPEBIAS, ASCENDC_TPL_UI_LIST, DTYPE_BIAS_FP32), |
| 84 | - ASCENDC_TPL_UINT_SEL(COMMMODE, ASCENDC_TPL_UI_LIST, COMMMODE_CCU, COMMMODE_AICPU), | 90 | + ASCENDC_TPL_UINT_SEL(COMMMODE, ASCENDC_TPL_UI_LIST, COMMMODE_CCU, COMMMODE_AICPU), |
| 85 | - ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF), | 91 | + ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF), |
| 86 | - ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 1))); | 92 | + ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 1))); |
| 93 | + | ||
| 94 | +#undef MATMUL_ALLTOALL_ORIGINAL_KERNEL_TYPE_SEL | ||
| 87 | 95 | ||
| 88 | #endif // MATMUL_ALLTO_ALL_TILING_KEY_H | 96 | #endif // MATMUL_ALLTO_ALL_TILING_KEY_H |