Merged
feat(apace): AllGather MX QuantMatmul Hcomm通算融合模板 #10588
wangfuhaoAdd on Aug 25
feat(apace): AllGather MX QuantMatmul Hcomm通算融合模板 #10588
Merged
Total: 9 files changed+375-60
| @@ -25,7 +25,7 @@ | |||
| 25 | #include "securec.h" | 25 | #include "securec.h" |
| 26 | #include "tiling/platform/platform_ascendc.h" | 26 | #include "tiling/platform/platform_ascendc.h" |
| 27 | #include "ascendc/host_api/tiling/template_argument.h" | 27 | #include "ascendc/host_api/tiling/template_argument.h" |
| 28 | -#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_tiling_data.h" | 28 | +#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_tiling_data.h" |
| 29 | #include "apace/tiling/quant_matmul_tiling_swat.h" | 29 | #include "apace/tiling/quant_matmul_tiling_swat.h" |
| 30 | #include "op_host/op_tiling/mc2_tiling_utils.h" | 30 | #include "op_host/op_tiling/mc2_tiling_utils.h" |
| 31 | #include "../../op_kernel/arch35/all_gather_matmul_v3_tiling_key.h" | 31 | #include "../../op_kernel/arch35/all_gather_matmul_v3_tiling_key.h" |
| @@ -15,7 +15,7 @@ | |||
| 15 | 15 | ||
| 16 | #include <kernel_operator.h> | 16 | #include <kernel_operator.h> |
| 17 | #include "all_gather_matmul_v3_tiling_key.h" | 17 | #include "all_gather_matmul_v3_tiling_key.h" |
| 18 | -#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_tiling_data.h" | 18 | +#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_tiling_data.h" |
| 19 | #include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_impl.h" | 19 | #include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_impl.h" |
| 20 | 20 | ||
| 21 | using namespace AscendC; | 21 | using namespace AscendC; |
| @@ -92,7 +92,7 @@ ASCENDC_TPL_SEL( | |||
| 92 | ASCENDC_TPL_UINT_SEL(COMMTYPE, ASCENDC_TPL_UI_LIST, ALL2ALL_COMM_TYPE_CCU, ALL2ALL_COMM_TYPE_AICPU), | 92 | ASCENDC_TPL_UINT_SEL(COMMTYPE, ASCENDC_TPL_UI_LIST, ALL2ALL_COMM_TYPE_CCU, ALL2ALL_COMM_TYPE_AICPU), |
| 93 | ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 0), | 93 | ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 0), |
| 94 | ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF)), | 94 | ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF)), |
| 95 | - // MX_QUANT_MODE + USING_APACE_IMPL=1: hcomm纯C通信, AIC_ONLY, ADDBIASSPLITMODE 恒为 OFF | 95 | + // MX_QUANT_MODE + USING_APACE_IMPL=1: hcomm纯C通信, ASCENDC_TPL_AIC_ONLY, ADDBIASSPLITMODE 恒为 OFF |
| 96 | ASCENDC_TPL_ARGS_SEL( | 96 | ASCENDC_TPL_ARGS_SEL( |
| 97 | ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY), | 97 | ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY), |
| 98 | ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST, MX_QUANT_MODE), ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 0, 1), | 98 | ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST, MX_QUANT_MODE), ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 0, 1), |
Amc2/common/op_kernel/apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_hcomm_impl.h+286-0
| @@ -0,0 +1,286 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | + | ||
| 11 | +/*! | ||
| 12 | + * \file all_gather_mx_matmul_hcomm_impl.h | ||
| 13 | + * \brief AllGather + QuantMatmul fusion kernel implementation (Hcomm/CCU variant) | ||
| 14 | + * | ||
| 15 | + * Init(): | ||
| 16 | + * AIC: hccl_.InitV2()/SetCcTilingV2(),批量下发 AllGather<true>(scale + dataHead + dataTail), | ||
| 17 | + * GetCommPolicy().state_ = &commState_ 绑定 WaitPolicy。 | ||
| 18 | + * | ||
| 19 | + * Run(): | ||
| 20 | + * AIC: 执行 FragmentTensor kernel,所有核 WaitTile 完成后再统一 Finalize → hccl_.Finalize。 | ||
| 21 | + * kernel 内逐 tile commPolicy_.WaitTile(dependTileIdx) → state_->hccl_.Wait(handle)。 | ||
| 22 | + */ | ||
| 23 | + | ||
| 24 | +#pragma once | ||
| 25 | + | ||
| 26 | +#include "include/tensor_api/tensor.h" | ||
| 27 | +#include "lib/hccl/hccl.h" | ||
| 28 | +#include "blaze/gemm/utils/common_utils.h" | ||
| 29 | +#include "apace/kernel/matmul/quant_batch_matmul/all_gather_qbmm_mx_kernel.h" | ||
| 30 | +#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_tiling_data.h" | ||
| 31 | +#include "apace/tiling/quant_matmul_tiling_data.h" | ||
| 32 | + | ||
| 33 | +namespace Apace { | ||
| 34 | + | ||
| 35 | +using namespace AscendC; | ||
| 36 | + | ||
| 37 | +template <typename T> | ||
| 38 | +constexpr uint64_t GetHcclDataType() | ||
| 39 | +{ | ||
| 40 | + if constexpr (AscendC::IsSameType<T, float8_e5m2_t>::value) { | ||
| 41 | + return AscendC::HCCL_DATA_TYPE_FP8E5M2; | ||
| 42 | + } else if constexpr (AscendC::IsSameType<T, fp4x2_e2m1_t>::value) { | ||
| 43 | + return AscendC::HCCL_DATA_TYPE_UINT8; | ||
| 44 | + } else { | ||
| 45 | + return AscendC::HCCL_DATA_TYPE_FP8E4M3; | ||
| 46 | + } | ||
| 47 | +} | ||
| 48 | + | ||
| 49 | +template <AscendC::HcclServerType ServerType> | ||
| 50 | +struct HcommCommState { | ||
| 51 | + AscendC::Hccl<ServerType> hccl_; | ||
| 52 | + AscendC::HcclHandle scaleHandle_{0}; | ||
| 53 | + AscendC::HcclHandle dataHeadHandle_{0}; | ||
| 54 | + AscendC::HcclHandle dataTailHandle_{0}; | ||
| 55 | + uint32_t tileCnt_{0}; | ||
| 56 | +}; | ||
| 57 | + | ||
| 58 | +template <AscendC::HcclServerType ServerType> | ||
| 59 | +struct HcommCommWaitPolicy { | ||
| 60 | + HcommCommState<ServerType> *state_{nullptr}; | ||
| 61 | + | ||
| 62 | + __aicore__ inline void WaitTile(uint32_t tileIdx) | ||
| 63 | + { | ||
| 64 | + // dependTileIdx=0 (HEAD): 本 rank 数据,无需等待通信 | ||
| 65 | + if (tileIdx == 0) { | ||
| 66 | + return; | ||
| 67 | + } | ||
| 68 | + // dependTileIdx=1 (MAIN r0): 首次远端等待,额外等 scale(全量) | ||
| 69 | + if (tileIdx == 1) { | ||
| 70 | + state_->hccl_.Wait(state_->scaleHandle_); | ||
| 71 | + } | ||
| 72 | + // dependTileIdx=1..tileCnt (MAIN): 等 data head(repeat=tileCnt,每次 Wait 消费一轮) | ||
| 73 | + if (tileIdx <= state_->tileCnt_) { | ||
| 74 | + state_->hccl_.Wait(state_->dataHeadHandle_); | ||
| 75 | + } else { | ||
| 76 | + // dependTileIdx>tileCnt (TAIL): 等 data tail | ||
| 77 | + state_->hccl_.Wait(state_->dataTailHandle_); | ||
| 78 | + } | ||
| 79 | + } | ||
| 80 | +}; | ||
| 81 | + | ||
| 82 | +template <typename AType, typename BType, typename CType, AscendC::HcclServerType ServerType, bool IsMxFp4> | ||
| 83 | +class AllGatherMxQuantMatmulHcommImpl { | ||
| 84 | +public: | ||
| 85 | + explicit __aicore__ inline AllGatherMxQuantMatmulHcommImpl(Apace::hcommAllGatherMatmulTilingData *tilingData) | ||
| 86 | + : tilingData_(tilingData) | ||
| 87 | + {} | ||
| 88 | + | ||
| 89 | + __aicore__ inline void Init(GM_ADDR aGM, GM_ADDR aScaleGM, GM_ADDR bGM, GM_ADDR bScaleGM, GM_ADDR biasGM, | ||
| 90 | + GM_ADDR cGM, GM_ADDR gatherOut, GM_ADDR workspaceGM); | ||
| 91 | + __aicore__ inline void Run(); | ||
| 92 | + | ||
| 93 | + using QuantMatmulKernelImpl = AllGatherQbmmMxKernel<AType, BType, CType, HcommCommWaitPolicy<ServerType>>; | ||
| 94 | + using KernelParams = typename QuantMatmulKernelImpl::Params; | ||
| 95 | + | ||
| 96 | + QuantMatmulKernelImpl quantMatmulKernelImpl_; | ||
| 97 | + HcommCommState<ServerType> commState_; | ||
| 98 | + Mc2Kernel::OpStateDump opStateDump_; | ||
| 99 | + | ||
| 100 | +private: | ||
| 101 | + __aicore__ inline void InitBaseParams(); | ||
| 102 | + __aicore__ inline void CommitAllGather(); | ||
| 103 | + __aicore__ inline void SetupKernelParams(KernelParams ¶ms); | ||
| 104 | + | ||
| 105 | + Apace::hcommAllGatherMatmulTilingData *tilingData_; | ||
| 106 | + | ||
| 107 | + GM_ADDR aGM_{}; | ||
| 108 | + GM_ADDR aScaleGM_{}; | ||
| 109 | + GM_ADDR bGM_{}; | ||
| 110 | + GM_ADDR bScaleGM_{}; | ||
| 111 | + GM_ADDR biasGM_{}; | ||
| 112 | + GM_ADDR cGM_{}; | ||
| 113 | + GM_ADDR workspaceGM_{}; | ||
| 114 | + GM_ADDR gatherDataAddr_{}; | ||
| 115 | + GM_ADDR gatherScaleAddr_{}; | ||
| 116 | + | ||
| 117 | + uint32_t rankId_{}; | ||
| 118 | + uint32_t rankSize_{}; | ||
| 119 | + uint32_t m_{}; | ||
| 120 | + uint32_t k_{}; | ||
| 121 | + uint32_t n_{}; | ||
| 122 | + uint32_t tileCnt_{}; | ||
| 123 | + uint32_t tileM_{}; | ||
| 124 | + uint32_t tailCnt_{}; | ||
| 125 | + uint32_t tailM_{}; | ||
| 126 | + uint32_t paddedTailM_{}; | ||
| 127 | + uint32_t commTurn_{}; | ||
| 128 | + uint64_t headRows_{}; | ||
| 129 | + uint64_t scaleKLen_{}; | ||
| 130 | + uint64_t scaleKGroups_{}; | ||
| 131 | + uint64_t kForComm_{}; // 通信用 K 元素数(FP4 时 = CeilDiv(K,2)) | ||
| 132 | + uint64_t dataBytesPerMRow_{}; | ||
| 133 | + uint64_t scaleBytesPerMRow_{}; | ||
| 134 | + uint64_t cBytesPerM_{}; | ||
| 135 | + uint64_t dataRegionBytes_{}; | ||
| 136 | + uint64_t scaleRegionBytes_{}; | ||
| 137 | + | ||
| 138 | + static constexpr uint32_t kPaddingLength = 16; | ||
| 139 | + static constexpr uint64_t scaleDataType_ = HCCL_DATA_TYPE_FP8E8M0; | ||
| 140 | + static constexpr uint64_t dataDataType_ = GetHcclDataType<AType>(); | ||
| 141 | + static constexpr uint64_t MXFP_DATA_NUM_PER_BYTE = 2UL; | ||
| 142 | +}; | ||
| 143 | + | ||
| 144 | +template <typename AType, typename BType, typename CType, AscendC::HcclServerType ServerType, bool IsMxFp4> | ||
| 145 | +__aicore__ inline void AllGatherMxQuantMatmulHcommImpl<AType, BType, CType, ServerType, IsMxFp4>::InitBaseParams() | ||
| 146 | +{ | ||
| 147 | + const auto &ct = tilingData_->commTile; | ||
| 148 | + tileCnt_ = static_cast<uint32_t>(ct.splitAxisTileCnt); | ||
| 149 | + tileM_ = static_cast<uint32_t>(ct.splitAxisTileSize); | ||
| 150 | + tailCnt_ = static_cast<uint32_t>(ct.splitAxisTailCnt); | ||
| 151 | + tailM_ = static_cast<uint32_t>(ct.splitAxisTailSize); | ||
| 152 | + k_ = tilingData_->mmTile.k; | ||
| 153 | + n_ = tilingData_->mmTile.n; | ||
| 154 | + m_ = static_cast<uint32_t>(ct.splitAxisTileSize * ct.splitAxisTileCnt + ct.splitAxisTailSize * ct.splitAxisTailCnt); | ||
| 155 | + commTurn_ = tileCnt_ + tailCnt_; | ||
| 156 | + paddedTailM_ = (tailM_ > 0) ? ((tailM_ + kPaddingLength - 1) / kPaddingLength * kPaddingLength) : 0U; | ||
| 157 | + headRows_ = static_cast<uint64_t>(tileCnt_) * tileM_; | ||
| 158 | + | ||
| 159 | + scaleKGroups_ = Blaze::Gemm::CeilDiv(static_cast<uint64_t>(k_), Blaze::Gemm::MXFP_DIVISOR_SIZE); | ||
| 160 | + scaleKLen_ = scaleKGroups_ * static_cast<uint64_t>(Blaze::Gemm::MXFP_MULTI_BASE_SIZE); | ||
| 161 | + // FP4 (fp4x2) 1 字节装 2 个元素:每行字节数 = k/2;FP8 等 = k * sizeof(AType)(与 URMA impl 契约一致) | ||
| 162 | + // 以类模板参数 IsMxFp4 为唯一判断源(与 kForComm_ 一致),保证所有实例化场景 AType 与 IsMxFp4 配对 | ||
| 163 | + dataBytesPerMRow_ = IsMxFp4 ? (static_cast<uint64_t>(k_) >> 1) : static_cast<uint64_t>(k_) * sizeof(AType); | ||
| 164 | + scaleBytesPerMRow_ = | ||
| 165 | + scaleKGroups_ * static_cast<uint64_t>(Blaze::Gemm::MXFP_MULTI_BASE_SIZE) * sizeof(AscendC::fp8_e8m0_t); | ||
| 166 | + cBytesPerM_ = static_cast<uint64_t>(n_) * sizeof(CType); | ||
| 167 | + // FP4: 2 个元素打包为 1 字节,通信 count/stride 需用 CeilDiv(K, 2) 减半 | ||
| 168 | + kForComm_ = static_cast<uint64_t>(k_); | ||
| 169 | + if constexpr (IsMxFp4) { | ||
| 170 | + kForComm_ = Blaze::Gemm::CeilDiv(kForComm_, MXFP_DATA_NUM_PER_BYTE); | ||
| 171 | + } | ||
| 172 | +} | ||
| 173 | + | ||
| 174 | +template <typename AType, typename BType, typename CType, AscendC::HcclServerType ServerType, bool IsMxFp4> | ||
| 175 | +__aicore__ inline void AllGatherMxQuantMatmulHcommImpl<AType, BType, CType, ServerType, IsMxFp4>::Init( | ||
| 176 | + GM_ADDR aGM, GM_ADDR aScaleGM, GM_ADDR bGM, GM_ADDR bScaleGM, GM_ADDR biasGM, GM_ADDR cGM, GM_ADDR gatherOut, | ||
| 177 | + GM_ADDR workspaceGM) | ||
| 178 | +{ | ||
| 179 | + aGM_ = aGM; | ||
| 180 | + aScaleGM_ = aScaleGM; | ||
| 181 | + bGM_ = bGM; | ||
| 182 | + bScaleGM_ = bScaleGM; | ||
| 183 | + biasGM_ = biasGM; | ||
| 184 | + cGM_ = cGM; | ||
| 185 | + workspaceGM_ = workspaceGM; | ||
| 186 | + | ||
| 187 | + InitBaseParams(); | ||
| 188 | + | ||
| 189 | + // HCCL initialization | ||
| 190 | + commState_.hccl_.InitV2(AscendC::GetHcclContext<0>(), &(tilingData_->mc2InitTiling)); | ||
| 191 | + commState_.hccl_.SetCcTilingV2(static_cast<uint64_t>(offsetof(Apace::hcommAllGatherMatmulTilingData, mc2CcTiling))); | ||
| 192 | + rankId_ = commState_.hccl_.GetRankId(); | ||
| 193 | + rankSize_ = commState_.hccl_.GetRankDim(); | ||
| 194 | + | ||
| 195 | + // Gather buffer: scale 始终用 workspace; data 用 gatherOut(当可用)或 workspace | ||
| 196 | + scaleRegionBytes_ = static_cast<uint64_t>(rankSize_) * m_ * scaleBytesPerMRow_; | ||
| 197 | + dataRegionBytes_ = static_cast<uint64_t>(rankSize_) * m_ * dataBytesPerMRow_; | ||
| 198 | + gatherScaleAddr_ = workspaceGM_; | ||
| 199 | + if (tilingData_->gatherLen == 0 && gatherOut != 0) { | ||
| 200 | + gatherDataAddr_ = gatherOut; | ||
| 201 | + } else { | ||
| 202 | + gatherDataAddr_ = workspaceGM_ + scaleRegionBytes_; | ||
| 203 | + } | ||
| 204 | + | ||
| 205 | + // Submit AllGather (non-blocking <true>) | ||
| 206 | + CommitAllGather(); | ||
| 207 | + | ||
| 208 | +#if MC2_DFX_ENABLE | ||
| 209 | + opStateDump_.Init(workspaceGM_, &tilingData_->dumpInfo.workspaceLayout, tilingData_->mmTile.usedCoreNum); | ||
| 210 | +#endif | ||
| 211 | + // Bind CommPolicy to CommState | ||
| 212 | + quantMatmulKernelImpl_.GetCommPolicy().state_ = &commState_; | ||
| 213 | + commState_.tileCnt_ = tileCnt_; | ||
| 214 | +} | ||
| 215 | + | ||
| 216 | +template <typename AType, typename BType, typename CType, AscendC::HcclServerType ServerType, bool IsMxFp4> | ||
| 217 | +__aicore__ inline void AllGatherMxQuantMatmulHcommImpl<AType, BType, CType, ServerType, IsMxFp4>::CommitAllGather() | ||
| 218 | +{ | ||
| 219 | + // Scale: 全量一次性提交(repeat=1) | ||
| 220 | + commState_.scaleHandle_ = | ||
| 221 | + commState_.hccl_.template AllGather<true>(aScaleGM_, gatherScaleAddr_, static_cast<uint64_t>(m_) * scaleKLen_, | ||
| 222 | + static_cast<AscendC::HcclDataType>(scaleDataType_), | ||
| 223 | + static_cast<uint64_t>(m_) * scaleKLen_, static_cast<uint8_t>(1)); | ||
| 224 | + | ||
| 225 | + // Data Head: 批量提交所有 head tile(repeat=tileCnt);纯尾块路径(tileCnt=0)跳过 | ||
| 226 | + // count/stride 用 kForComm_(FP4 = K/2 打包字节),地址偏移用 dataBytesPerMRow_(数值与 kForComm_ 一致) | ||
| 227 | + if (tileCnt_ > 0) { | ||
| 228 | + commState_.dataHeadHandle_ = commState_.hccl_.template AllGather<true>( | ||
| 229 | + aGM_, gatherDataAddr_, static_cast<uint64_t>(tileM_) * kForComm_, | ||
| 230 | + static_cast<AscendC::HcclDataType>(dataDataType_), static_cast<uint64_t>(m_) * kForComm_, | ||
| 231 | + static_cast<uint8_t>(tileCnt_)); | ||
| 232 | + } | ||
| 233 | + | ||
| 234 | + // Data Tail: 批量提交所有 tail tile(repeat=tailCnt) | ||
| 235 | + if (tailCnt_ > 0) { | ||
| 236 | + commState_.dataTailHandle_ = commState_.hccl_.template AllGather<true>( | ||
| 237 | + aGM_ + headRows_ * dataBytesPerMRow_, gatherDataAddr_ + headRows_ * dataBytesPerMRow_, | ||
| 238 | + static_cast<uint64_t>(tailM_) * kForComm_, static_cast<AscendC::HcclDataType>(dataDataType_), | ||
| 239 | + static_cast<uint64_t>(m_) * kForComm_, static_cast<uint8_t>(tailCnt_)); | ||
| 240 | + } | ||
| 241 | +} | ||
| 242 | + | ||
| 243 | +template <typename AType, typename BType, typename CType, AscendC::HcclServerType ServerType, bool IsMxFp4> | ||
| 244 | +__aicore__ inline void AllGatherMxQuantMatmulHcommImpl<AType, BType, CType, ServerType, IsMxFp4>::SetupKernelParams( | ||
| 245 | + KernelParams ¶ms) | ||
| 246 | +{ | ||
| 247 | + params.mmTile = &(tilingData_->mmTile); | ||
| 248 | + params.qbmmParams = {tilingData_->mmTile.baseM, tilingData_->mmTile.baseN, tilingData_->mmTile.baseK, | ||
| 249 | + tilingData_->mmTile.dbL0c}; | ||
| 250 | + params.fragParams = {tileCnt_, | ||
| 251 | + tileM_, | ||
| 252 | + tailCnt_, | ||
| 253 | + tailM_, | ||
| 254 | + paddedTailM_, | ||
| 255 | + commTurn_, | ||
| 256 | + headRows_, | ||
| 257 | + rankId_, | ||
| 258 | + rankSize_, | ||
| 259 | + m_, | ||
| 260 | + static_cast<uint64_t>(k_), | ||
| 261 | + static_cast<uint64_t>(n_), | ||
| 262 | + scaleKLen_}; | ||
| 263 | + params.aGM = aGM_; | ||
| 264 | + params.aScaleGM = aScaleGM_; | ||
| 265 | + params.bGM = bGM_; | ||
| 266 | + params.bScaleGM = bScaleGM_; | ||
| 267 | + params.biasGM = biasGM_; | ||
| 268 | + params.isBias = (biasGM_ != nullptr); // bias optional 输入缺失时框架传 nullptr(同 gatherOut 判空惯例) | ||
| 269 | + params.cGM = cGM_; | ||
| 270 | + params.winDataBase = gatherDataAddr_; | ||
| 271 | + params.winScaleBase = gatherScaleAddr_; | ||
| 272 | + params.dataBytesPerMRow = dataBytesPerMRow_; | ||
| 273 | + params.scaleBytesPerMRow = scaleBytesPerMRow_; | ||
| 274 | + params.cBytesPerM = cBytesPerM_; | ||
| 275 | +} | ||
| 276 | + | ||
| 277 | +template <typename AType, typename BType, typename CType, AscendC::HcclServerType ServerType, bool IsMxFp4> | ||
| 278 | +__aicore__ inline void AllGatherMxQuantMatmulHcommImpl<AType, BType, CType, ServerType, IsMxFp4>::Run() | ||
| 279 | +{ | ||
| 280 | + KernelParams params; | ||
| 281 | + SetupKernelParams(params); | ||
| 282 | + quantMatmulKernelImpl_(params, opStateDump_); | ||
| 283 | + commState_.hccl_.Finalize(); | ||
| 284 | +} | ||
| 285 | + | ||
| 286 | +} // namespace Apace | ||
Rmc2/common/op_kernel/apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_tiling_data.h→mc2/common/op_kernel/apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_tiling_data.h+26-10
| @@ -1,21 +1,24 @@ | |||
| 1 | /** | 1 | /** |
| 2 | - * Copyright (c) 2026 Huawei Technologies Co., Ltd. | 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. |
| 3 | - * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of |
| 4 | - * CANN Open Software License Agreement Version 2.0 (the "License"). | 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). |
| 5 | - * Please refer to the License for details. You may not use this file except in compliance with the License. | 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. |
| 6 | - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, |
| 7 | - * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 8 | - * See LICENSE in the root of the software repository for the full text of the License. | 8 | + * See LICENSE in the root of the software repository for the full text of the License. |
| 9 | - */ | 9 | + */ |
| 10 | 10 | ||
| 11 | /*! | 11 | /*! |
| 12 | - * \file all_gather_mx_matmul_urma_tiling_data.h | 12 | + * \file all_gather_mx_matmul_tiling_data.h |
| 13 | - * \brief Tiling data for AllGather + QuantMatmul fusion kernel (prefill variant) | 13 | + * \brief Tiling data for AllGather + QuantMatmul fusion kernel (URMA + Hcomm/CCU variants) |
| 14 | */ | 14 | */ |
| 15 | 15 | ||
| 16 | #pragma once | 16 | #pragma once |
| 17 | 17 | ||
| 18 | +#ifndef __CCE_AICORE__ | ||
| 18 | #include <cstdint> | 19 | #include <cstdint> |
| 20 | +#endif | ||
| 21 | +#include "kernel_tiling/kernel_tiling.h" | ||
| 19 | #include "apace/tiling/quant_matmul_tiling_data.h" | 22 | #include "apace/tiling/quant_matmul_tiling_data.h" |
| 20 | #include "apace/tiling/comm_tiling_data.h" | 23 | #include "apace/tiling/comm_tiling_data.h" |
| 21 | #include "apace/core/aiv_comm/collective_comm_context.h" | 24 | #include "apace/core/aiv_comm/collective_comm_context.h" |
| @@ -41,3 +44,16 @@ struct alignas(8) AllGatherMxMatmulUrmaTilingData { | |||
| 41 | uint8_t isBias{0}; | 44 | uint8_t isBias{0}; |
| 42 | }; | 45 | }; |
| 43 | #pragma pack(pop) | 46 | #pragma pack(pop) |
| 47 | + | ||
| 48 | +// Hcomm/CCU 路径 tiling 结构体(apace 自有,不依赖外部算子 tiling) | ||
| 49 | +namespace Apace { | ||
| 50 | +struct hcommAllGatherMatmulTilingData { | ||
| 51 | + // DFX 头部约定: dumpInfo 必须是第一个成员(offset 0) | ||
| 52 | + Utils::DfxDumpInfo dumpInfo{}; | ||
| 53 | + Mc2InitTiling mc2InitTiling; // HCCL 初始化参数 | ||
| 54 | + Mc2CcTiling mc2CcTiling; // HCCL CC task 配置 | ||
| 55 | + CommTilingData commTile; // AllGather 切分参数(splitAxis=M_per_rank, nonSplitAxis=K) | ||
| 56 | + QuantMatmulTilingData mmTile; // Matmul tiling 参数 | ||
| 57 | + uint64_t gatherLen; // 0: gather_out 输出可用,通信写入 gatherOut; >0: 用 workspace | ||
| 58 | +}; | ||
| 59 | +} // namespace Apace | ||
Mmc2/common/op_kernel/apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_impl.h+9-2
| @@ -17,7 +17,7 @@ | |||
| 17 | 17 | ||
| 18 | #include "kernel_basic_intf.h" | 18 | #include "kernel_basic_intf.h" |
| 19 | #include "kernel_tiling/kernel_tiling.h" | 19 | #include "kernel_tiling/kernel_tiling.h" |
| 20 | -#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_tiling_data.h" | 20 | +#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_tiling_data.h" |
| 21 | #include "apace/kernel/matmul/quant_batch_matmul/all_gather_qbmm_mx_kernel.h" | 21 | #include "apace/kernel/matmul/quant_batch_matmul/all_gather_qbmm_mx_kernel.h" |
| 22 | #include "adv_api/hcomm/hcomm.h" | 22 | #include "adv_api/hcomm/hcomm.h" |
| 23 | #include "apace/core/aiv_comm/collective_comm_context.h" | 23 | #include "apace/core/aiv_comm/collective_comm_context.h" |
| @@ -37,6 +37,13 @@ using LayoutB = asc::te::dn_ext_layout_ptn; | |||
| 37 | using LayoutC = asc::te::nd_ext_layout_ptn; | 37 | using LayoutC = asc::te::nd_ext_layout_ptn; |
| 38 | using ProblemShape = asc::te::shape<int64_t, int64_t, int64_t, int64_t>; | 38 | using ProblemShape = asc::te::shape<int64_t, int64_t, int64_t, int64_t>; |
| 39 | 39 | ||
| 40 | +struct UrmaCommWaitPolicy { | ||
| 41 | + __aicore__ inline void WaitTile(uint32_t tileIdx) | ||
| 42 | + { | ||
| 43 | + AscendC::CrossCoreWaitFlag<0x2, PIPE_MTE2>(tileIdx); | ||
| 44 | + } | ||
| 45 | +}; | ||
| 46 | + | ||
| 40 | template <typename AType, typename BType, typename CType> | 47 | template <typename AType, typename BType, typename CType> |
| 41 | class AllGatherMxMatmulUrmaImpl { | 48 | class AllGatherMxMatmulUrmaImpl { |
| 42 | public: | 49 | public: |
| @@ -46,7 +53,7 @@ public: | |||
| 46 | const AllGatherMxMatmulUrmaTilingData *tilingData); | 53 | const AllGatherMxMatmulUrmaTilingData *tilingData); |
| 47 | __aicore__ inline void Process(); | 54 | __aicore__ inline void Process(); |
| 48 | 55 | ||
| 49 | - using QuantMatmulKernelImpl = AllGatherQbmmMxKernel<AType, BType, CType>; | 56 | + using QuantMatmulKernelImpl = AllGatherQbmmMxKernel<AType, BType, CType, UrmaCommWaitPolicy>; |
| 50 | using KernelParams = typename QuantMatmulKernelImpl::Params; | 57 | using KernelParams = typename QuantMatmulKernelImpl::Params; |
| 51 | using QBMMTiling = typename QuantMatmulKernelImpl::QBMMTiling; | 58 | using QBMMTiling = typename QuantMatmulKernelImpl::QBMMTiling; |
| 52 | using FragmentParams = typename QuantMatmulKernelImpl::FragmentParams; | 59 | using FragmentParams = typename QuantMatmulKernelImpl::FragmentParams; |
| @@ -24,14 +24,14 @@ | |||
| 24 | * Run(): | 24 | * Run(): |
| 25 | * AIC: local块前置 — 若 localMatmul != 0,先执行 MatmulProcess(LOCAL) 计算本 rank 数据, | 25 | * AIC: local块前置 — 若 localMatmul != 0,先执行 MatmulProcess(LOCAL) 计算本 rank 数据, |
| 26 | * 再执行 MatmulProcess(REMOTE) 计算通信收到的远端数据,以 local 计算掩盖通信延迟; | 26 | * 再执行 MatmulProcess(REMOTE) 计算通信收到的远端数据,以 local 计算掩盖通信延迟; |
| 27 | - * 最后 commState_.hccl_.Finalize()。 | 27 | + * 最后所有核 WaitTile 完成后再统一 Finalize。 |
| 28 | * kernel 内逐 tile 通过 commPolicy_.WaitTile() → state_->hccl_.Wait(handle) 等待通信完成, | 28 | * kernel 内逐 tile 通过 commPolicy_.WaitTile() → state_->hccl_.Wait(handle) 等待通信完成, |
| 29 | - * 首次 wait 紧挨 data wait 之前执行 hccl->Wait(scaleHandle_)(每核仅一次,掩盖 matmul 头开销)。 | 29 | + * 首次 wait 紧挨 data wait 之前执行 hccl->Wait(scaleHandle_)。 |
| 30 | * | 30 | * |
| 31 | * AIC side: 通信下发 + 计算 + per-tile wait 均在 AIC 核内完成,无 AIV↔AIC 跨核 flag 同步. | 31 | * AIC side: 通信下发 + 计算 + per-tile wait 均在 AIC 核内完成,无 AIV↔AIC 跨核 flag 同步. |
| 32 | * | 32 | * |
| 33 | - * \note 本实现仅启动 AIC 核(cube-only)。若需启动 AIV 核,须为 Init/Run 中的通信下发、 | 33 | + * \note 本实现仅启动 AIC 核。若需启动 AIV 核,须为 Init/Run 中的通信下发、 |
| 34 | - * Wait、Finalize 及 SyncAll 增加 AIC 守卫(if ASCEND_IS_AIC),否则 AIV 误参与 HCCL | 34 | + * Wait、Finalize 及核间屏障增加 AIC 守卫(if ASCEND_IS_AIC),否则 AIV 误参与 HCCL |
| 35 | * 调用将导致 Prepare/Wait 失配或死锁。 | 35 | * 调用将导致 Prepare/Wait 失配或死锁。 |
| 36 | */ | 36 | */ |
| 37 | 37 | ||
| @@ -47,7 +47,7 @@ enum RegionTag { | |||
| 47 | TAIL | 47 | TAIL |
| 48 | }; | 48 | }; |
| 49 | 49 | ||
| 50 | -template <typename AType, typename BType, typename CType> | 50 | +template <typename AType, typename BType, typename CType, typename CommPolicy> |
| 51 | class AllGatherQbmmMxKernel { | 51 | class AllGatherQbmmMxKernel { |
| 52 | public: | 52 | public: |
| 53 | __aicore__ inline AllGatherQbmmMxKernel() {} | 53 | __aicore__ inline AllGatherQbmmMxKernel() {} |
| @@ -151,6 +151,10 @@ public: | |||
| 151 | { | 151 | { |
| 152 | Run(params, opStateDump); | 152 | Run(params, opStateDump); |
| 153 | } | 153 | } |
| 154 | + __aicore__ inline CommPolicy &GetCommPolicy() | ||
| 155 | + { | ||
| 156 | + return commPolicy_; | ||
| 157 | + } | ||
| 154 | 158 | ||
| 155 | private: | 159 | private: |
| 156 | __aicore__ inline void Init(const Params ¶ms); | 160 | __aicore__ inline void Init(const Params ¶ms); |
| @@ -220,6 +224,7 @@ private: | |||
| 220 | GM_ADDR tailAddrListScale_[MAX_FRAG]{}; | 224 | GM_ADDR tailAddrListScale_[MAX_FRAG]{}; |
| 221 | GM_ADDR tailAddrListC_[MAX_FRAG]{}; | 225 | GM_ADDR tailAddrListC_[MAX_FRAG]{}; |
| 222 | 226 | ||
| 227 | + CommPolicy commPolicy_; | ||
| 223 | uint32_t curMainRoundIdx_{0xFFFFFFFF}; | 228 | uint32_t curMainRoundIdx_{0xFFFFFFFF}; |
| 224 | bool winBasesReady_{false}; | 229 | bool winBasesReady_{false}; |
| 225 | bool tailBuilt_{false}; | 230 | bool tailBuilt_{false}; |
| @@ -233,8 +238,8 @@ private: | |||
| 233 | uint64_t tileMCStride_{}; | 238 | uint64_t tileMCStride_{}; |
| 234 | }; | 239 | }; |
| 235 | 240 | ||
| 236 | -template <typename AType, typename BType, typename CType> | 241 | +template <typename AType, typename BType, typename CType, typename CommPolicy> |
| 237 | -__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::Init(const Params ¶ms) | 242 | +__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::Init(const Params ¶ms) |
| 238 | { | 243 | { |
| 239 | if ASCEND_IS_AIV { | 244 | if ASCEND_IS_AIV { |
| 240 | return; | 245 | return; |
| @@ -247,11 +252,10 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::Init(const Pa | |||
| 247 | biasGmAddr_ = reinterpret_cast<__gm__ float *>(params.biasGM); | 252 | biasGmAddr_ = reinterpret_cast<__gm__ float *>(params.biasGM); |
| 248 | } | 253 | } |
| 249 | 254 | ||
| 250 | -template <typename AType, typename BType, typename CType> | 255 | +template <typename AType, typename BType, typename CType, typename CommPolicy> |
| 251 | template <typename TensorB, typename TensorScaleB> | 256 | template <typename TensorB, typename TensorScaleB> |
| 252 | -__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::SetL2Cache(const ProblemShape &problemShape, | 257 | +__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::SetL2Cache( |
| 253 | - int64_t baseM, int64_t baseN, | 258 | + const ProblemShape &problemShape, int64_t baseM, int64_t baseN, TensorB &gmB, TensorScaleB &gmScaleB) |
| 254 | - TensorB &gmB, TensorScaleB &gmScaleB) | ||
| 255 | { | 259 | { |
| 256 | const bool fullMBlock = (baseM >= asc::te::get<Blaze::Gemm::MNK_M>(problemShape)); | 260 | const bool fullMBlock = (baseM >= asc::te::get<Blaze::Gemm::MNK_M>(problemShape)); |
| 257 | 261 | ||
| @@ -274,9 +278,9 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::SetL2Cache(co | |||
| 274 | asc::te::cache_mode::normal); | 278 | asc::te::cache_mode::normal); |
| 275 | } | 279 | } |
| 276 | 280 | ||
| 277 | -template <typename AType, typename BType, typename CType> | 281 | +template <typename AType, typename BType, typename CType, typename CommPolicy> |
| 278 | -__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::Run(const Params ¶ms, | 282 | +__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::Run(const Params ¶ms, |
| 279 | - Mc2Kernel::OpStateDump &opStateDump) | 283 | + Mc2Kernel::OpStateDump &opStateDump) |
| 280 | { | 284 | { |
| 281 | Init(params); | 285 | Init(params); |
| 282 | 286 | ||
| @@ -320,12 +324,10 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::Run(const Par | |||
| 320 | Process(params, problemShape, sch, mmadFrag, opStateDump); | 324 | Process(params, problemShape, sch, mmadFrag, opStateDump); |
| 321 | } | 325 | } |
| 322 | 326 | ||
| 323 | -template <typename AType, typename BType, typename CType> | 327 | +template <typename AType, typename BType, typename CType, typename CommPolicy> |
| 324 | -__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::Process(const Params ¶ms, | 328 | +__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::Process( |
| 325 | - const ProblemShape &problemShape, | 329 | + const Params ¶ms, const ProblemShape &problemShape, BlockScheduler &sch, BlockMmadFragC &mmadFrag, |
| 326 | - BlockScheduler &sch, | 330 | + Mc2Kernel::OpStateDump &opStateDump) |
| 327 | - BlockMmadFragC &mmadFrag, | ||
| 328 | - Mc2Kernel::OpStateDump &opStateDump) | ||
| 329 | { | 331 | { |
| 330 | const auto &mmT = *params.mmTile; | 332 | const auto &mmT = *params.mmTile; |
| 331 | const auto &fp = params.fragParams; | 333 | const auto &fp = params.fragParams; |
| @@ -353,7 +355,8 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::Process(const | |||
| 353 | sch.UpdateTailTile(mTailTile, nTailTile); | 355 | sch.UpdateTailTile(mTailTile, nTailTile); |
| 354 | } | 356 | } |
| 355 | uint32_t readyTileIdx = 0; | 357 | uint32_t readyTileIdx = 0; |
| 356 | - CrossCoreWaitFlag<0x2, PIPE_MTE2>(0); // dependTileIdx=0 由 AIV 预触发,无需等待。 | 358 | + commPolicy_.WaitTile(0); |
| 359 | + // dependTileIdx=0 为本 rank 数据,无需等待通信。 | ||
| 357 | opStateDump.DoDump(DUMP_FIELD_WAIT); | 360 | opStateDump.DoDump(DUMP_FIELD_WAIT); |
| 358 | 361 | ||
| 359 | asc::te::coord<int64_t, int64_t, int64_t, int64_t> blockIdx; | 362 | asc::te::coord<int64_t, int64_t, int64_t, int64_t> blockIdx; |
| @@ -372,9 +375,10 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::Process(const | |||
| 372 | sch.GetTileCoord(blockIdx, mPos, nPos); | 375 | sch.GetTileCoord(blockIdx, mPos, nPos); |
| 373 | 376 | ||
| 374 | auto ctx = ResolveTileCtx(mPos, headMainRows, mainRoundRows, mainSectionRows, fp.rankSize, fp.commTurn); | 377 | auto ctx = ResolveTileCtx(mPos, headMainRows, mainRoundRows, mainSectionRows, fp.rankSize, fp.commTurn); |
| 378 | + | ||
| 375 | while (readyTileIdx < ctx.dependTileIdx) { | 379 | while (readyTileIdx < ctx.dependTileIdx) { |
| 376 | readyTileIdx++; | 380 | readyTileIdx++; |
| 377 | - CrossCoreWaitFlag<0x2, PIPE_MTE2>(readyTileIdx); | 381 | + commPolicy_.WaitTile(readyTileIdx); |
| 378 | opStateDump.DoDump(DUMP_FIELD_WAIT); | 382 | opStateDump.DoDump(DUMP_FIELD_WAIT); |
| 379 | } | 383 | } |
| 380 | 384 | ||
| @@ -414,15 +418,15 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::Process(const | |||
| 414 | // 确保所有 dependTileIdx 均已 wait(收尾清理)。 | 418 | // 确保所有 dependTileIdx 均已 wait(收尾清理)。 |
| 415 | while (readyTileIdx < fp.commTurn) { | 419 | while (readyTileIdx < fp.commTurn) { |
| 416 | readyTileIdx++; | 420 | readyTileIdx++; |
| 417 | - CrossCoreWaitFlag<0x2, PIPE_MTE2>(readyTileIdx); | 421 | + commPolicy_.WaitTile(readyTileIdx); |
| 418 | opStateDump.DoDump(DUMP_FIELD_WAIT); | 422 | opStateDump.DoDump(DUMP_FIELD_WAIT); |
| 419 | } | 423 | } |
| 420 | } | 424 | } |
| 421 | 425 | ||
| 422 | -template <typename AType, typename BType, typename CType> | 426 | +template <typename AType, typename BType, typename CType, typename CommPolicy> |
| 423 | -__aicore__ inline Apace::Basic::FragmentParam<AllGatherQbmmMxKernel<AType, BType, CType>::DIMS_NUM> | 427 | +__aicore__ inline Apace::Basic::FragmentParam<AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::DIMS_NUM> |
| 424 | -AllGatherQbmmMxKernel<AType, BType, CType>::MakeFragParam(uint64_t fragSize, uint64_t realFragSize, uint32_t fragCnt, | 428 | +AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::MakeFragParam(uint64_t fragSize, uint64_t realFragSize, |
| 425 | - uint64_t shape1) const | 429 | + uint32_t fragCnt, uint64_t shape1) const |
| 426 | { | 430 | { |
| 427 | Apace::Basic::FragmentParam<DIMS_NUM> param{}; | 431 | Apace::Basic::FragmentParam<DIMS_NUM> param{}; |
| 428 | param.assembleAxis = 0; | 432 | param.assembleAxis = 0; |
| @@ -434,8 +438,9 @@ AllGatherQbmmMxKernel<AType, BType, CType>::MakeFragParam(uint64_t fragSize, uin | |||
| 434 | return param; | 438 | return param; |
| 435 | } | 439 | } |
| 436 | 440 | ||
| 437 | -template <typename AType, typename BType, typename CType> | 441 | +template <typename AType, typename BType, typename CType, typename CommPolicy> |
| 438 | -__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::BuildFragmentTensors(const Params ¶ms) | 442 | +__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::BuildFragmentTensors( |
| 443 | + const Params ¶ms) | ||
| 439 | { | 444 | { |
| 440 | const auto &fp = params.fragParams; | 445 | const auto &fp = params.fragParams; |
| 441 | 446 | ||
| @@ -460,8 +465,9 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::BuildFragment | |||
| 460 | // main / tail 的 fragment tensor 延迟到首次使用时构建,构建开销被 comm wait 掩盖。 | 465 | // main / tail 的 fragment tensor 延迟到首次使用时构建,构建开销被 comm wait 掩盖。 |
| 461 | } | 466 | } |
| 462 | 467 | ||
| 463 | -template <typename AType, typename BType, typename CType> | 468 | +template <typename AType, typename BType, typename CType, typename CommPolicy> |
| 464 | -__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::EnsureWinRankBasesReady(const Params ¶ms) | 469 | +__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::EnsureWinRankBasesReady( |
| 470 | + const Params ¶ms) | ||
| 465 | { | 471 | { |
| 466 | if (winBasesReady_) { | 472 | if (winBasesReady_) { |
| 467 | return; | 473 | return; |
| @@ -474,9 +480,9 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::EnsureWinRank | |||
| 474 | winBasesReady_ = true; | 480 | winBasesReady_ = true; |
| 475 | } | 481 | } |
| 476 | 482 | ||
| 477 | -template <typename AType, typename BType, typename CType> | 483 | +template <typename AType, typename BType, typename CType, typename CommPolicy> |
| 478 | -__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::BuildMainFragment(const Params ¶ms, | 484 | +__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::BuildMainFragment(const Params ¶ms, |
| 479 | - uint32_t roundIdx) | 485 | + uint32_t roundIdx) |
| 480 | { | 486 | { |
| 481 | EnsureWinRankBasesReady(params); | 487 | EnsureWinRankBasesReady(params); |
| 482 | const auto &fp = params.fragParams; | 488 | const auto &fp = params.fragParams; |
| @@ -504,8 +510,8 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::BuildMainFrag | |||
| 504 | curMainRoundIdx_ = roundIdx; | 510 | curMainRoundIdx_ = roundIdx; |
| 505 | } | 511 | } |
| 506 | 512 | ||
| 507 | -template <typename AType, typename BType, typename CType> | 513 | +template <typename AType, typename BType, typename CType, typename CommPolicy> |
| 508 | -__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::BuildTailFragment(const Params ¶ms) | 514 | +__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::BuildTailFragment(const Params ¶ms) |
| 509 | { | 515 | { |
| 510 | EnsureWinRankBasesReady(params); | 516 | EnsureWinRankBasesReady(params); |
| 511 | const auto &fp = params.fragParams; | 517 | const auto &fp = params.fragParams; |
| @@ -533,9 +539,9 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::BuildTailFrag | |||
| 533 | MakeFragParam(fp.paddedTailM, fp.tailM, fp.rankSize, fp.n), tailAddrListC_); | 539 | MakeFragParam(fp.paddedTailM, fp.tailM, fp.rankSize, fp.n), tailAddrListC_); |
| 534 | } | 540 | } |
| 535 | 541 | ||
| 536 | -template <typename AType, typename BType, typename CType> | 542 | +template <typename AType, typename BType, typename CType, typename CommPolicy> |
| 537 | -__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::UpdateMainRoundAddrs(const Params ¶ms, | 543 | +__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::UpdateMainRoundAddrs( |
| 538 | - uint32_t roundIdx) | 544 | + const Params ¶ms, uint32_t roundIdx) |
| 539 | { | 545 | { |
| 540 | const auto &fp = params.fragParams; | 546 | const auto &fp = params.fragParams; |
| 541 | uint32_t delta = roundIdx - curMainRoundIdx_; | 547 | uint32_t delta = roundIdx - curMainRoundIdx_; |
| @@ -559,11 +565,11 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::UpdateMainRou | |||
| 559 | curMainRoundIdx_ = roundIdx; | 565 | curMainRoundIdx_ = roundIdx; |
| 560 | } | 566 | } |
| 561 | 567 | ||
| 562 | -template <typename AType, typename BType, typename CType> | 568 | +template <typename AType, typename BType, typename CType, typename CommPolicy> |
| 563 | -__aicore__ inline typename AllGatherQbmmMxKernel<AType, BType, CType>::TileCtx | 569 | +__aicore__ inline typename AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::TileCtx |
| 564 | -AllGatherQbmmMxKernel<AType, BType, CType>::ResolveTileCtx(int64_t mPos, int64_t headMainRows, int64_t mainRoundRows, | 570 | +AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::ResolveTileCtx(int64_t mPos, int64_t headMainRows, |
| 565 | - int64_t mainSectionRows, uint32_t rankSize, | 571 | + int64_t mainRoundRows, int64_t mainSectionRows, |
| 566 | - uint32_t commTurn) const | 572 | + uint32_t rankSize, uint32_t commTurn) const |
| 567 | { | 573 | { |
| 568 | TileCtx ctx{}; | 574 | TileCtx ctx{}; |
| 569 | if (mPos < headMainRows) { | 575 | if (mPos < headMainRows) { |
| @@ -32,7 +32,7 @@ | |||
| 32 | #include "apace_st_utils.h" | 32 | #include "apace_st_utils.h" |
| 33 | #include "apace/utils/apace_constant.h" | 33 | #include "apace/utils/apace_constant.h" |
| 34 | #include "apace/tiling/quant_matmul_tiling_swat.h" | 34 | #include "apace/tiling/quant_matmul_tiling_swat.h" |
| 35 | -#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_tiling_data.h" | 35 | +#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_tiling_data.h" |
| 36 | #include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_impl.h" | 36 | #include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_impl.h" |
| 37 | #include "apace/core/aiv_comm/collective_comm_context.h" | 37 | #include "apace/core/aiv_comm/collective_comm_context.h" |
| 38 | #include "apace/utils/comm_channel_builder.h" | 38 | #include "apace/utils/comm_channel_builder.h" |
注释没对上,81行=1,这里还是 = 1