Merged
feat(apace): AllGather MX QuantMatmul Hcomm通算融合模板 #10588
feat(apace): AllGather MX QuantMatmul Hcomm通算融合模板 #10588
Merged
wangfuhaoAdd on Aug 25
Total: 9 files changed+375-60
@@ -25,7 +25,7 @@
25#include "securec.h"25#include "securec.h"
26#include "tiling/platform/platform_ascendc.h"26#include "tiling/platform/platform_ascendc.h"
27#include "ascendc/host_api/tiling/template_argument.h"27#include "ascendc/host_api/tiling/template_argument.h"
28-#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_tiling_data.h"28+#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_tiling_data.h"
29#include "apace/tiling/quant_matmul_tiling_swat.h"29#include "apace/tiling/quant_matmul_tiling_swat.h"
30#include "op_host/op_tiling/mc2_tiling_utils.h"30#include "op_host/op_tiling/mc2_tiling_utils.h"
31#include "../../op_kernel/arch35/all_gather_matmul_v3_tiling_key.h"31#include "../../op_kernel/arch35/all_gather_matmul_v3_tiling_key.h"
@@ -15,7 +15,7 @@
15 15 
16#include <kernel_operator.h>16#include <kernel_operator.h>
17#include "all_gather_matmul_v3_tiling_key.h"17#include "all_gather_matmul_v3_tiling_key.h"
18-#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_tiling_data.h"18+#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_tiling_data.h"
19#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_impl.h"19#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_impl.h"
20 20 
21using namespace AscendC;21using namespace AscendC;
@@ -92,7 +92,7 @@ ASCENDC_TPL_SEL(
92 ASCENDC_TPL_UINT_SEL(COMMTYPE, ASCENDC_TPL_UI_LIST, ALL2ALL_COMM_TYPE_CCU, ALL2ALL_COMM_TYPE_AICPU),92 ASCENDC_TPL_UINT_SEL(COMMTYPE, ASCENDC_TPL_UI_LIST, ALL2ALL_COMM_TYPE_CCU, ALL2ALL_COMM_TYPE_AICPU),
93 ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 0),93 ASCENDC_TPL_BOOL_SEL(USING_APACE_IMPL, 0),
94 ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF)),94 ASCENDC_TPL_UINT_SEL(ADDBIASSPLITMODE, ASCENDC_TPL_UI_LIST, ADDBIAS_OFF)),
95- // MX_QUANT_MODE + USING_APACE_IMPL=1: hcomm纯C通信, AIC_ONLY, ADDBIASSPLITMODE 恒为 OFF95+ // MX_QUANT_MODE + USING_APACE_IMPL=1: hcomm纯C通信, ASCENDC_TPL_AIC_ONLY, ADDBIASSPLITMODE 恒为 OFF
96 ASCENDC_TPL_ARGS_SEL(96 ASCENDC_TPL_ARGS_SEL(
97 ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY),97 ASCENDC_TPL_KERNEL_TYPE_SEL(ASCENDC_TPL_AIC_ONLY),
98 ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST, MX_QUANT_MODE), ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 0, 1),98 ASCENDC_TPL_UINT_SEL(QUANTMODE, ASCENDC_TPL_UI_LIST, MX_QUANT_MODE), ASCENDC_TPL_BOOL_SEL(X2TRANSPOSE, 0, 1),
@@ -0,0 +1,286 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file all_gather_mx_matmul_hcomm_impl.h
13+ * \brief AllGather + QuantMatmul fusion kernel implementation (Hcomm/CCU variant)
14+ *
15+ * Init():
16+ * AIC: hccl_.InitV2()/SetCcTilingV2(),批量下发 AllGather<true>(scale + dataHead + dataTail),
17+ * GetCommPolicy().state_ = &commState_ 绑定 WaitPolicy。
18+ *
19+ * Run():
20+ * AIC: 执行 FragmentTensor kernel,所有核 WaitTile 完成后再统一 Finalize → hccl_.Finalize。
21+ * kernel 内逐 tile commPolicy_.WaitTile(dependTileIdx) → state_->hccl_.Wait(handle)。
22+ */
23+ 
24+#pragma once
25+ 
26+#include "include/tensor_api/tensor.h"
27+#include "lib/hccl/hccl.h"
28+#include "blaze/gemm/utils/common_utils.h"
29+#include "apace/kernel/matmul/quant_batch_matmul/all_gather_qbmm_mx_kernel.h"
30+#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_tiling_data.h"
31+#include "apace/tiling/quant_matmul_tiling_data.h"
32+ 
33+namespace Apace {
34+ 
35+using namespace AscendC;
36+ 
37+template <typename T>
38+constexpr uint64_t GetHcclDataType()
39+{
40+ if constexpr (AscendC::IsSameType<T, float8_e5m2_t>::value) {
41+ return AscendC::HCCL_DATA_TYPE_FP8E5M2;
42+ } else if constexpr (AscendC::IsSameType<T, fp4x2_e2m1_t>::value) {
43+ return AscendC::HCCL_DATA_TYPE_UINT8;
44+ } else {
45+ return AscendC::HCCL_DATA_TYPE_FP8E4M3;
46+ }
47+}
48+ 
49+template <AscendC::HcclServerType ServerType>
50+struct HcommCommState {
51+ AscendC::Hccl<ServerType> hccl_;
52+ AscendC::HcclHandle scaleHandle_{0};
53+ AscendC::HcclHandle dataHeadHandle_{0};
54+ AscendC::HcclHandle dataTailHandle_{0};
55+ uint32_t tileCnt_{0};
56+};
57+ 
58+template <AscendC::HcclServerType ServerType>
59+struct HcommCommWaitPolicy {
60+ HcommCommState<ServerType> *state_{nullptr};
61+ 
62+ __aicore__ inline void WaitTile(uint32_t tileIdx)
63+ {
64+ // dependTileIdx=0 (HEAD): 本 rank 数据,无需等待通信
65+ if (tileIdx == 0) {
66+ return;
67+ }
68+ // dependTileIdx=1 (MAIN r0): 首次远端等待,额外等 scale(全量)
69+ if (tileIdx == 1) {
70+ state_->hccl_.Wait(state_->scaleHandle_);
71+ }
72+ // dependTileIdx=1..tileCnt (MAIN): 等 data head(repeat=tileCnt,每次 Wait 消费一轮)
物华
物华物华Sep 10

注释没对上,81行=1,这里还是 = 1

likedislike
73+ if (tileIdx <= state_->tileCnt_) {
74+ state_->hccl_.Wait(state_->dataHeadHandle_);
75+ } else {
76+ // dependTileIdx>tileCnt (TAIL): 等 data tail
77+ state_->hccl_.Wait(state_->dataTailHandle_);
78+ }
79+ }
80+};
81+ 
82+template <typename AType, typename BType, typename CType, AscendC::HcclServerType ServerType, bool IsMxFp4>
83+class AllGatherMxQuantMatmulHcommImpl {
84+public:
85+ explicit __aicore__ inline AllGatherMxQuantMatmulHcommImpl(Apace::hcommAllGatherMatmulTilingData *tilingData)
86+ : tilingData_(tilingData)
87+ {}
88+ 
89+ __aicore__ inline void Init(GM_ADDR aGM, GM_ADDR aScaleGM, GM_ADDR bGM, GM_ADDR bScaleGM, GM_ADDR biasGM,
90+ GM_ADDR cGM, GM_ADDR gatherOut, GM_ADDR workspaceGM);
91+ __aicore__ inline void Run();
92+ 
93+ using QuantMatmulKernelImpl = AllGatherQbmmMxKernel<AType, BType, CType, HcommCommWaitPolicy<ServerType>>;
94+ using KernelParams = typename QuantMatmulKernelImpl::Params;
95+ 
96+ QuantMatmulKernelImpl quantMatmulKernelImpl_;
97+ HcommCommState<ServerType> commState_;
98+ Mc2Kernel::OpStateDump opStateDump_;
99+ 
100+private:
101+ __aicore__ inline void InitBaseParams();
102+ __aicore__ inline void CommitAllGather();
103+ __aicore__ inline void SetupKernelParams(KernelParams &params);
104+ 
105+ Apace::hcommAllGatherMatmulTilingData *tilingData_;
106+ 
107+ GM_ADDR aGM_{};
108+ GM_ADDR aScaleGM_{};
109+ GM_ADDR bGM_{};
110+ GM_ADDR bScaleGM_{};
111+ GM_ADDR biasGM_{};
112+ GM_ADDR cGM_{};
113+ GM_ADDR workspaceGM_{};
114+ GM_ADDR gatherDataAddr_{};
115+ GM_ADDR gatherScaleAddr_{};
116+ 
117+ uint32_t rankId_{};
118+ uint32_t rankSize_{};
119+ uint32_t m_{};
120+ uint32_t k_{};
121+ uint32_t n_{};
122+ uint32_t tileCnt_{};
123+ uint32_t tileM_{};
124+ uint32_t tailCnt_{};
125+ uint32_t tailM_{};
126+ uint32_t paddedTailM_{};
127+ uint32_t commTurn_{};
128+ uint64_t headRows_{};
129+ uint64_t scaleKLen_{};
130+ uint64_t scaleKGroups_{};
131+ uint64_t kForComm_{}; // 通信用 K 元素数(FP4 时 = CeilDiv(K,2))
132+ uint64_t dataBytesPerMRow_{};
133+ uint64_t scaleBytesPerMRow_{};
134+ uint64_t cBytesPerM_{};
135+ uint64_t dataRegionBytes_{};
136+ uint64_t scaleRegionBytes_{};
137+ 
138+ static constexpr uint32_t kPaddingLength = 16;
139+ static constexpr uint64_t scaleDataType_ = HCCL_DATA_TYPE_FP8E8M0;
140+ static constexpr uint64_t dataDataType_ = GetHcclDataType<AType>();
141+ static constexpr uint64_t MXFP_DATA_NUM_PER_BYTE = 2UL;
142+};
143+ 
144+template <typename AType, typename BType, typename CType, AscendC::HcclServerType ServerType, bool IsMxFp4>
145+__aicore__ inline void AllGatherMxQuantMatmulHcommImpl<AType, BType, CType, ServerType, IsMxFp4>::InitBaseParams()
146+{
147+ const auto &ct = tilingData_->commTile;
148+ tileCnt_ = static_cast<uint32_t>(ct.splitAxisTileCnt);
149+ tileM_ = static_cast<uint32_t>(ct.splitAxisTileSize);
150+ tailCnt_ = static_cast<uint32_t>(ct.splitAxisTailCnt);
151+ tailM_ = static_cast<uint32_t>(ct.splitAxisTailSize);
152+ k_ = tilingData_->mmTile.k;
153+ n_ = tilingData_->mmTile.n;
154+ m_ = static_cast<uint32_t>(ct.splitAxisTileSize * ct.splitAxisTileCnt + ct.splitAxisTailSize * ct.splitAxisTailCnt);
155+ commTurn_ = tileCnt_ + tailCnt_;
156+ paddedTailM_ = (tailM_ > 0) ? ((tailM_ + kPaddingLength - 1) / kPaddingLength * kPaddingLength) : 0U;
157+ headRows_ = static_cast<uint64_t>(tileCnt_) * tileM_;
158+ 
159+ scaleKGroups_ = Blaze::Gemm::CeilDiv(static_cast<uint64_t>(k_), Blaze::Gemm::MXFP_DIVISOR_SIZE);
160+ scaleKLen_ = scaleKGroups_ * static_cast<uint64_t>(Blaze::Gemm::MXFP_MULTI_BASE_SIZE);
161+ // FP4 (fp4x2) 1 字节装 2 个元素:每行字节数 = k/2;FP8 等 = k * sizeof(AType)(与 URMA impl 契约一致)
162+ // 以类模板参数 IsMxFp4 为唯一判断源(与 kForComm_ 一致),保证所有实例化场景 AType 与 IsMxFp4 配对
163+ dataBytesPerMRow_ = IsMxFp4 ? (static_cast<uint64_t>(k_) >> 1) : static_cast<uint64_t>(k_) * sizeof(AType);
164+ scaleBytesPerMRow_ =
165+ scaleKGroups_ * static_cast<uint64_t>(Blaze::Gemm::MXFP_MULTI_BASE_SIZE) * sizeof(AscendC::fp8_e8m0_t);
166+ cBytesPerM_ = static_cast<uint64_t>(n_) * sizeof(CType);
167+ // FP4: 2 个元素打包为 1 字节,通信 count/stride 需用 CeilDiv(K, 2) 减半
168+ kForComm_ = static_cast<uint64_t>(k_);
169+ if constexpr (IsMxFp4) {
170+ kForComm_ = Blaze::Gemm::CeilDiv(kForComm_, MXFP_DATA_NUM_PER_BYTE);
171+ }
172+}
173+ 
174+template <typename AType, typename BType, typename CType, AscendC::HcclServerType ServerType, bool IsMxFp4>
175+__aicore__ inline void AllGatherMxQuantMatmulHcommImpl<AType, BType, CType, ServerType, IsMxFp4>::Init(
176+ GM_ADDR aGM, GM_ADDR aScaleGM, GM_ADDR bGM, GM_ADDR bScaleGM, GM_ADDR biasGM, GM_ADDR cGM, GM_ADDR gatherOut,
177+ GM_ADDR workspaceGM)
178+{
179+ aGM_ = aGM;
180+ aScaleGM_ = aScaleGM;
181+ bGM_ = bGM;
182+ bScaleGM_ = bScaleGM;
183+ biasGM_ = biasGM;
184+ cGM_ = cGM;
185+ workspaceGM_ = workspaceGM;
186+ 
187+ InitBaseParams();
188+ 
189+ // HCCL initialization
190+ commState_.hccl_.InitV2(AscendC::GetHcclContext<0>(), &(tilingData_->mc2InitTiling));
191+ commState_.hccl_.SetCcTilingV2(static_cast<uint64_t>(offsetof(Apace::hcommAllGatherMatmulTilingData, mc2CcTiling)));
192+ rankId_ = commState_.hccl_.GetRankId();
193+ rankSize_ = commState_.hccl_.GetRankDim();
194+ 
195+ // Gather buffer: scale 始终用 workspace; data 用 gatherOut(当可用)或 workspace
196+ scaleRegionBytes_ = static_cast<uint64_t>(rankSize_) * m_ * scaleBytesPerMRow_;
197+ dataRegionBytes_ = static_cast<uint64_t>(rankSize_) * m_ * dataBytesPerMRow_;
198+ gatherScaleAddr_ = workspaceGM_;
199+ if (tilingData_->gatherLen == 0 && gatherOut != 0) {
200+ gatherDataAddr_ = gatherOut;
201+ } else {
202+ gatherDataAddr_ = workspaceGM_ + scaleRegionBytes_;
203+ }
204+ 
205+ // Submit AllGather (non-blocking <true>)
206+ CommitAllGather();
207+ 
208+#if MC2_DFX_ENABLE
209+ opStateDump_.Init(workspaceGM_, &tilingData_->dumpInfo.workspaceLayout, tilingData_->mmTile.usedCoreNum);
210+#endif
211+ // Bind CommPolicy to CommState
212+ quantMatmulKernelImpl_.GetCommPolicy().state_ = &commState_;
213+ commState_.tileCnt_ = tileCnt_;
214+}
215+ 
216+template <typename AType, typename BType, typename CType, AscendC::HcclServerType ServerType, bool IsMxFp4>
217+__aicore__ inline void AllGatherMxQuantMatmulHcommImpl<AType, BType, CType, ServerType, IsMxFp4>::CommitAllGather()
218+{
219+ // Scale: 全量一次性提交(repeat=1)
220+ commState_.scaleHandle_ =
221+ commState_.hccl_.template AllGather<true>(aScaleGM_, gatherScaleAddr_, static_cast<uint64_t>(m_) * scaleKLen_,
222+ static_cast<AscendC::HcclDataType>(scaleDataType_),
223+ static_cast<uint64_t>(m_) * scaleKLen_, static_cast<uint8_t>(1));
224+ 
225+ // Data Head: 批量提交所有 head tile(repeat=tileCnt);纯尾块路径(tileCnt=0)跳过
226+ // count/stride 用 kForComm_(FP4 = K/2 打包字节),地址偏移用 dataBytesPerMRow_(数值与 kForComm_ 一致)
227+ if (tileCnt_ > 0) {
228+ commState_.dataHeadHandle_ = commState_.hccl_.template AllGather<true>(
229+ aGM_, gatherDataAddr_, static_cast<uint64_t>(tileM_) * kForComm_,
230+ static_cast<AscendC::HcclDataType>(dataDataType_), static_cast<uint64_t>(m_) * kForComm_,
231+ static_cast<uint8_t>(tileCnt_));
232+ }
233+ 
234+ // Data Tail: 批量提交所有 tail tile(repeat=tailCnt)
235+ if (tailCnt_ > 0) {
236+ commState_.dataTailHandle_ = commState_.hccl_.template AllGather<true>(
237+ aGM_ + headRows_ * dataBytesPerMRow_, gatherDataAddr_ + headRows_ * dataBytesPerMRow_,
238+ static_cast<uint64_t>(tailM_) * kForComm_, static_cast<AscendC::HcclDataType>(dataDataType_),
239+ static_cast<uint64_t>(m_) * kForComm_, static_cast<uint8_t>(tailCnt_));
240+ }
241+}
242+ 
243+template <typename AType, typename BType, typename CType, AscendC::HcclServerType ServerType, bool IsMxFp4>
244+__aicore__ inline void AllGatherMxQuantMatmulHcommImpl<AType, BType, CType, ServerType, IsMxFp4>::SetupKernelParams(
245+ KernelParams &params)
246+{
247+ params.mmTile = &(tilingData_->mmTile);
248+ params.qbmmParams = {tilingData_->mmTile.baseM, tilingData_->mmTile.baseN, tilingData_->mmTile.baseK,
249+ tilingData_->mmTile.dbL0c};
250+ params.fragParams = {tileCnt_,
251+ tileM_,
252+ tailCnt_,
253+ tailM_,
254+ paddedTailM_,
255+ commTurn_,
256+ headRows_,
257+ rankId_,
258+ rankSize_,
259+ m_,
260+ static_cast<uint64_t>(k_),
261+ static_cast<uint64_t>(n_),
262+ scaleKLen_};
263+ params.aGM = aGM_;
264+ params.aScaleGM = aScaleGM_;
265+ params.bGM = bGM_;
266+ params.bScaleGM = bScaleGM_;
267+ params.biasGM = biasGM_;
268+ params.isBias = (biasGM_ != nullptr); // bias optional 输入缺失时框架传 nullptr(同 gatherOut 判空惯例)
269+ params.cGM = cGM_;
270+ params.winDataBase = gatherDataAddr_;
271+ params.winScaleBase = gatherScaleAddr_;
272+ params.dataBytesPerMRow = dataBytesPerMRow_;
273+ params.scaleBytesPerMRow = scaleBytesPerMRow_;
274+ params.cBytesPerM = cBytesPerM_;
275+}
276+ 
277+template <typename AType, typename BType, typename CType, AscendC::HcclServerType ServerType, bool IsMxFp4>
278+__aicore__ inline void AllGatherMxQuantMatmulHcommImpl<AType, BType, CType, ServerType, IsMxFp4>::Run()
279+{
280+ KernelParams params;
281+ SetupKernelParams(params);
282+ quantMatmulKernelImpl_(params, opStateDump_);
283+ commState_.hccl_.Finalize();
284+}
285+ 
286+} // namespace Apace
Rmc2/common/op_kernel/apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_tiling_data.h→mc2/common/op_kernel/apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_tiling_data.h+26-10
@@ -1,21 +1,24 @@
1/**1/**
2- * Copyright (c) 2026 Huawei Technologies Co., Ltd.2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3- * This program is free software, you can redistribute it and/or modify it under the terms and conditions of3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4- * CANN Open Software License Agreement Version 2.0 (the "License").4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5- * Please refer to the License for details. You may not use this file except in compliance with the License.5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6- * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7- * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8- * See LICENSE in the root of the software repository for the full text of the License.8+ * See LICENSE in the root of the software repository for the full text of the License.
9- */9+ */
10 10 
11/*!11/*!
12- * \file all_gather_mx_matmul_urma_tiling_data.h12+ * \file all_gather_mx_matmul_tiling_data.h
13- * \brief Tiling data for AllGather + QuantMatmul fusion kernel (prefill variant)13+ * \brief Tiling data for AllGather + QuantMatmul fusion kernel (URMA + Hcomm/CCU variants)
14 */14 */
15 15 
16#pragma once16#pragma once
17 17 
18+#ifndef __CCE_AICORE__
18#include <cstdint>19#include <cstdint>
20+#endif
21+#include "kernel_tiling/kernel_tiling.h"
19#include "apace/tiling/quant_matmul_tiling_data.h"22#include "apace/tiling/quant_matmul_tiling_data.h"
20#include "apace/tiling/comm_tiling_data.h"23#include "apace/tiling/comm_tiling_data.h"
21#include "apace/core/aiv_comm/collective_comm_context.h"24#include "apace/core/aiv_comm/collective_comm_context.h"
@@ -41,3 +44,16 @@ struct alignas(8) AllGatherMxMatmulUrmaTilingData {
41 uint8_t isBias{0};44 uint8_t isBias{0};
42};45};
43#pragma pack(pop)46#pragma pack(pop)
47+ 
48+// Hcomm/CCU 路径 tiling 结构体(apace 自有,不依赖外部算子 tiling)
49+namespace Apace {
50+struct hcommAllGatherMatmulTilingData {
51+ // DFX 头部约定: dumpInfo 必须是第一个成员(offset 0)
52+ Utils::DfxDumpInfo dumpInfo{};
53+ Mc2InitTiling mc2InitTiling; // HCCL 初始化参数
54+ Mc2CcTiling mc2CcTiling; // HCCL CC task 配置
55+ CommTilingData commTile; // AllGather 切分参数(splitAxis=M_per_rank, nonSplitAxis=K)
56+ QuantMatmulTilingData mmTile; // Matmul tiling 参数
57+ uint64_t gatherLen; // 0: gather_out 输出可用,通信写入 gatherOut; >0: 用 workspace
58+};
59+} // namespace Apace
@@ -17,7 +17,7 @@
17 17 
18#include "kernel_basic_intf.h"18#include "kernel_basic_intf.h"
19#include "kernel_tiling/kernel_tiling.h"19#include "kernel_tiling/kernel_tiling.h"
20-#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_tiling_data.h"20+#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_tiling_data.h"
21#include "apace/kernel/matmul/quant_batch_matmul/all_gather_qbmm_mx_kernel.h"21#include "apace/kernel/matmul/quant_batch_matmul/all_gather_qbmm_mx_kernel.h"
22#include "adv_api/hcomm/hcomm.h"22#include "adv_api/hcomm/hcomm.h"
23#include "apace/core/aiv_comm/collective_comm_context.h"23#include "apace/core/aiv_comm/collective_comm_context.h"
@@ -37,6 +37,13 @@ using LayoutB = asc::te::dn_ext_layout_ptn;
37using LayoutC = asc::te::nd_ext_layout_ptn;37using LayoutC = asc::te::nd_ext_layout_ptn;
38using ProblemShape = asc::te::shape<int64_t, int64_t, int64_t, int64_t>;38using ProblemShape = asc::te::shape<int64_t, int64_t, int64_t, int64_t>;
39 39 
40+struct UrmaCommWaitPolicy {
41+ __aicore__ inline void WaitTile(uint32_t tileIdx)
42+ {
43+ AscendC::CrossCoreWaitFlag<0x2, PIPE_MTE2>(tileIdx);
44+ }
45+};
46+ 
40template <typename AType, typename BType, typename CType>47template <typename AType, typename BType, typename CType>
41class AllGatherMxMatmulUrmaImpl {48class AllGatherMxMatmulUrmaImpl {
42public:49public:
@@ -46,7 +53,7 @@ public:
46 const AllGatherMxMatmulUrmaTilingData *tilingData);53 const AllGatherMxMatmulUrmaTilingData *tilingData);
47 __aicore__ inline void Process();54 __aicore__ inline void Process();
48 55 
49- using QuantMatmulKernelImpl = AllGatherQbmmMxKernel<AType, BType, CType>;56+ using QuantMatmulKernelImpl = AllGatherQbmmMxKernel<AType, BType, CType, UrmaCommWaitPolicy>;
50 using KernelParams = typename QuantMatmulKernelImpl::Params;57 using KernelParams = typename QuantMatmulKernelImpl::Params;
51 using QBMMTiling = typename QuantMatmulKernelImpl::QBMMTiling;58 using QBMMTiling = typename QuantMatmulKernelImpl::QBMMTiling;
52 using FragmentParams = typename QuantMatmulKernelImpl::FragmentParams;59 using FragmentParams = typename QuantMatmulKernelImpl::FragmentParams;
@@ -24,14 +24,14 @@
24 * Run():24 * Run():
25 * AIC: local块前置 — 若 localMatmul != 0,先执行 MatmulProcess(LOCAL) 计算本 rank 数据,25 * AIC: local块前置 — 若 localMatmul != 0,先执行 MatmulProcess(LOCAL) 计算本 rank 数据,
26 * 再执行 MatmulProcess(REMOTE) 计算通信收到的远端数据,以 local 计算掩盖通信延迟;26 * 再执行 MatmulProcess(REMOTE) 计算通信收到的远端数据,以 local 计算掩盖通信延迟;
27- * 最后 commState_.hccl_.Finalize()。27+ * 最后所有核 WaitTile 完成后再统一 Finalize。
28 * kernel 内逐 tile 通过 commPolicy_.WaitTile() → state_->hccl_.Wait(handle) 等待通信完成,28 * kernel 内逐 tile 通过 commPolicy_.WaitTile() → state_->hccl_.Wait(handle) 等待通信完成,
29- * 首次 wait 紧挨 data wait 之前执行 hccl->Wait(scaleHandle_)(每核仅一次,掩盖 matmul 头开销)。29+ * 首次 wait 紧挨 data wait 之前执行 hccl->Wait(scaleHandle_)。
30 *30 *
31 * AIC side: 通信下发 + 计算 + per-tile wait 均在 AIC 核内完成,无 AIV↔AIC 跨核 flag 同步.31 * AIC side: 通信下发 + 计算 + per-tile wait 均在 AIC 核内完成,无 AIV↔AIC 跨核 flag 同步.
32 *32 *
33- * \note 本实现仅启动 AIC 核(cube-only)。若需启动 AIV 核,须为 Init/Run 中的通信下发、33+ * \note 本实现仅启动 AIC 核。若需启动 AIV 核,须为 Init/Run 中的通信下发、
34- * Wait、Finalize 及 SyncAll 增加 AIC 守卫(if ASCEND_IS_AIC),否则 AIV 误参与 HCCL34+ * Wait、Finalize 及核间屏障增加 AIC 守卫(if ASCEND_IS_AIC),否则 AIV 误参与 HCCL
35 * 调用将导致 Prepare/Wait 失配或死锁。35 * 调用将导致 Prepare/Wait 失配或死锁。
36 */36 */
37 37 
@@ -47,7 +47,7 @@ enum RegionTag {
47 TAIL47 TAIL
48};48};
49 49 
50-template <typename AType, typename BType, typename CType>50+template <typename AType, typename BType, typename CType, typename CommPolicy>
51class AllGatherQbmmMxKernel {51class AllGatherQbmmMxKernel {
52public:52public:
53 __aicore__ inline AllGatherQbmmMxKernel() {}53 __aicore__ inline AllGatherQbmmMxKernel() {}
@@ -151,6 +151,10 @@ public:
151 {151 {
152 Run(params, opStateDump);152 Run(params, opStateDump);
153 }153 }
154+ __aicore__ inline CommPolicy &GetCommPolicy()
155+ {
156+ return commPolicy_;
157+ }
154 158 
155private:159private:
156 __aicore__ inline void Init(const Params &params);160 __aicore__ inline void Init(const Params &params);
@@ -220,6 +224,7 @@ private:
220 GM_ADDR tailAddrListScale_[MAX_FRAG]{};224 GM_ADDR tailAddrListScale_[MAX_FRAG]{};
221 GM_ADDR tailAddrListC_[MAX_FRAG]{};225 GM_ADDR tailAddrListC_[MAX_FRAG]{};
222 226 
227+ CommPolicy commPolicy_;
223 uint32_t curMainRoundIdx_{0xFFFFFFFF};228 uint32_t curMainRoundIdx_{0xFFFFFFFF};
224 bool winBasesReady_{false};229 bool winBasesReady_{false};
225 bool tailBuilt_{false};230 bool tailBuilt_{false};
@@ -233,8 +238,8 @@ private:
233 uint64_t tileMCStride_{};238 uint64_t tileMCStride_{};
234};239};
235 240 
236-template <typename AType, typename BType, typename CType>241+template <typename AType, typename BType, typename CType, typename CommPolicy>
237-__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::Init(const Params &params)242+__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::Init(const Params &params)
238{243{
239 if ASCEND_IS_AIV {244 if ASCEND_IS_AIV {
240 return;245 return;
@@ -247,11 +252,10 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::Init(const Pa
247 biasGmAddr_ = reinterpret_cast<__gm__ float *>(params.biasGM);252 biasGmAddr_ = reinterpret_cast<__gm__ float *>(params.biasGM);
248}253}
249 254 
250-template <typename AType, typename BType, typename CType>255+template <typename AType, typename BType, typename CType, typename CommPolicy>
251template <typename TensorB, typename TensorScaleB>256template <typename TensorB, typename TensorScaleB>
252-__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::SetL2Cache(const ProblemShape &problemShape,257+__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::SetL2Cache(
253- int64_t baseM, int64_t baseN,258+ const ProblemShape &problemShape, int64_t baseM, int64_t baseN, TensorB &gmB, TensorScaleB &gmScaleB)
254- TensorB &gmB, TensorScaleB &gmScaleB)
255{259{
256 const bool fullMBlock = (baseM >= asc::te::get<Blaze::Gemm::MNK_M>(problemShape));260 const bool fullMBlock = (baseM >= asc::te::get<Blaze::Gemm::MNK_M>(problemShape));
257 261 
@@ -274,9 +278,9 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::SetL2Cache(co
274 asc::te::cache_mode::normal);278 asc::te::cache_mode::normal);
275}279}
276 280 
277-template <typename AType, typename BType, typename CType>281+template <typename AType, typename BType, typename CType, typename CommPolicy>
278-__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::Run(const Params &params,282+__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::Run(const Params &params,
279- Mc2Kernel::OpStateDump &opStateDump)283+ Mc2Kernel::OpStateDump &opStateDump)
280{284{
281 Init(params);285 Init(params);
282 286 
@@ -320,12 +324,10 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::Run(const Par
320 Process(params, problemShape, sch, mmadFrag, opStateDump);324 Process(params, problemShape, sch, mmadFrag, opStateDump);
321}325}
322 326 
323-template <typename AType, typename BType, typename CType>327+template <typename AType, typename BType, typename CType, typename CommPolicy>
324-__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::Process(const Params &params,328+__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::Process(
325- const ProblemShape &problemShape,329+ const Params &params, const ProblemShape &problemShape, BlockScheduler &sch, BlockMmadFragC &mmadFrag,
326- BlockScheduler &sch,330+ Mc2Kernel::OpStateDump &opStateDump)
327- BlockMmadFragC &mmadFrag,
328- Mc2Kernel::OpStateDump &opStateDump)
329{331{
330 const auto &mmT = *params.mmTile;332 const auto &mmT = *params.mmTile;
331 const auto &fp = params.fragParams;333 const auto &fp = params.fragParams;
@@ -353,7 +355,8 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::Process(const
353 sch.UpdateTailTile(mTailTile, nTailTile);355 sch.UpdateTailTile(mTailTile, nTailTile);
354 }356 }
355 uint32_t readyTileIdx = 0;357 uint32_t readyTileIdx = 0;
356- CrossCoreWaitFlag<0x2, PIPE_MTE2>(0); // dependTileIdx=0 由 AIV 预触发,无需等待。358+ commPolicy_.WaitTile(0);
359+ // dependTileIdx=0 为本 rank 数据,无需等待通信。
357 opStateDump.DoDump(DUMP_FIELD_WAIT);360 opStateDump.DoDump(DUMP_FIELD_WAIT);
358 361 
359 asc::te::coord<int64_t, int64_t, int64_t, int64_t> blockIdx;362 asc::te::coord<int64_t, int64_t, int64_t, int64_t> blockIdx;
@@ -372,9 +375,10 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::Process(const
372 sch.GetTileCoord(blockIdx, mPos, nPos);375 sch.GetTileCoord(blockIdx, mPos, nPos);
373 376 
374 auto ctx = ResolveTileCtx(mPos, headMainRows, mainRoundRows, mainSectionRows, fp.rankSize, fp.commTurn);377 auto ctx = ResolveTileCtx(mPos, headMainRows, mainRoundRows, mainSectionRows, fp.rankSize, fp.commTurn);
378+ 
375 while (readyTileIdx < ctx.dependTileIdx) {379 while (readyTileIdx < ctx.dependTileIdx) {
376 readyTileIdx++;380 readyTileIdx++;
377- CrossCoreWaitFlag<0x2, PIPE_MTE2>(readyTileIdx);381+ commPolicy_.WaitTile(readyTileIdx);
378 opStateDump.DoDump(DUMP_FIELD_WAIT);382 opStateDump.DoDump(DUMP_FIELD_WAIT);
379 }383 }
380 384 
@@ -414,15 +418,15 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::Process(const
414 // 确保所有 dependTileIdx 均已 wait(收尾清理)。418 // 确保所有 dependTileIdx 均已 wait(收尾清理)。
415 while (readyTileIdx < fp.commTurn) {419 while (readyTileIdx < fp.commTurn) {
416 readyTileIdx++;420 readyTileIdx++;
417- CrossCoreWaitFlag<0x2, PIPE_MTE2>(readyTileIdx);421+ commPolicy_.WaitTile(readyTileIdx);
418 opStateDump.DoDump(DUMP_FIELD_WAIT);422 opStateDump.DoDump(DUMP_FIELD_WAIT);
419 }423 }
420}424}
421 425 
422-template <typename AType, typename BType, typename CType>426+template <typename AType, typename BType, typename CType, typename CommPolicy>
423-__aicore__ inline Apace::Basic::FragmentParam<AllGatherQbmmMxKernel<AType, BType, CType>::DIMS_NUM>427+__aicore__ inline Apace::Basic::FragmentParam<AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::DIMS_NUM>
424-AllGatherQbmmMxKernel<AType, BType, CType>::MakeFragParam(uint64_t fragSize, uint64_t realFragSize, uint32_t fragCnt,428+AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::MakeFragParam(uint64_t fragSize, uint64_t realFragSize,
425- uint64_t shape1) const429+ uint32_t fragCnt, uint64_t shape1) const
426{430{
427 Apace::Basic::FragmentParam<DIMS_NUM> param{};431 Apace::Basic::FragmentParam<DIMS_NUM> param{};
428 param.assembleAxis = 0;432 param.assembleAxis = 0;
@@ -434,8 +438,9 @@ AllGatherQbmmMxKernel<AType, BType, CType>::MakeFragParam(uint64_t fragSize, uin
434 return param;438 return param;
435}439}
436 440 
437-template <typename AType, typename BType, typename CType>441+template <typename AType, typename BType, typename CType, typename CommPolicy>
438-__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::BuildFragmentTensors(const Params &params)442+__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::BuildFragmentTensors(
443+ const Params &params)
439{444{
440 const auto &fp = params.fragParams;445 const auto &fp = params.fragParams;
441 446 
@@ -460,8 +465,9 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::BuildFragment
460 // main / tail 的 fragment tensor 延迟到首次使用时构建,构建开销被 comm wait 掩盖。465 // main / tail 的 fragment tensor 延迟到首次使用时构建,构建开销被 comm wait 掩盖。
461}466}
462 467 
463-template <typename AType, typename BType, typename CType>468+template <typename AType, typename BType, typename CType, typename CommPolicy>
464-__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::EnsureWinRankBasesReady(const Params &params)469+__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::EnsureWinRankBasesReady(
470+ const Params &params)
465{471{
466 if (winBasesReady_) {472 if (winBasesReady_) {
467 return;473 return;
@@ -474,9 +480,9 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::EnsureWinRank
474 winBasesReady_ = true;480 winBasesReady_ = true;
475}481}
476 482 
477-template <typename AType, typename BType, typename CType>483+template <typename AType, typename BType, typename CType, typename CommPolicy>
478-__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::BuildMainFragment(const Params &params,484+__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::BuildMainFragment(const Params &params,
479- uint32_t roundIdx)485+ uint32_t roundIdx)
480{486{
481 EnsureWinRankBasesReady(params);487 EnsureWinRankBasesReady(params);
482 const auto &fp = params.fragParams;488 const auto &fp = params.fragParams;
@@ -504,8 +510,8 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::BuildMainFrag
504 curMainRoundIdx_ = roundIdx;510 curMainRoundIdx_ = roundIdx;
505}511}
506 512 
507-template <typename AType, typename BType, typename CType>513+template <typename AType, typename BType, typename CType, typename CommPolicy>
508-__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::BuildTailFragment(const Params &params)514+__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::BuildTailFragment(const Params &params)
509{515{
510 EnsureWinRankBasesReady(params);516 EnsureWinRankBasesReady(params);
511 const auto &fp = params.fragParams;517 const auto &fp = params.fragParams;
@@ -533,9 +539,9 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::BuildTailFrag
533 MakeFragParam(fp.paddedTailM, fp.tailM, fp.rankSize, fp.n), tailAddrListC_);539 MakeFragParam(fp.paddedTailM, fp.tailM, fp.rankSize, fp.n), tailAddrListC_);
534}540}
535 541 
536-template <typename AType, typename BType, typename CType>542+template <typename AType, typename BType, typename CType, typename CommPolicy>
537-__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::UpdateMainRoundAddrs(const Params &params,543+__aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::UpdateMainRoundAddrs(
538- uint32_t roundIdx)544+ const Params &params, uint32_t roundIdx)
539{545{
540 const auto &fp = params.fragParams;546 const auto &fp = params.fragParams;
541 uint32_t delta = roundIdx - curMainRoundIdx_;547 uint32_t delta = roundIdx - curMainRoundIdx_;
@@ -559,11 +565,11 @@ __aicore__ inline void AllGatherQbmmMxKernel<AType, BType, CType>::UpdateMainRou
559 curMainRoundIdx_ = roundIdx;565 curMainRoundIdx_ = roundIdx;
560}566}
561 567 
562-template <typename AType, typename BType, typename CType>568+template <typename AType, typename BType, typename CType, typename CommPolicy>
563-__aicore__ inline typename AllGatherQbmmMxKernel<AType, BType, CType>::TileCtx569+__aicore__ inline typename AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::TileCtx
564-AllGatherQbmmMxKernel<AType, BType, CType>::ResolveTileCtx(int64_t mPos, int64_t headMainRows, int64_t mainRoundRows,570+AllGatherQbmmMxKernel<AType, BType, CType, CommPolicy>::ResolveTileCtx(int64_t mPos, int64_t headMainRows,
565- int64_t mainSectionRows, uint32_t rankSize,571+ int64_t mainRoundRows, int64_t mainSectionRows,
566- uint32_t commTurn) const572+ uint32_t rankSize, uint32_t commTurn) const
567{573{
568 TileCtx ctx{};574 TileCtx ctx{};
569 if (mPos < headMainRows) {575 if (mPos < headMainRows) {
@@ -32,7 +32,7 @@
32#include "apace_st_utils.h"32#include "apace_st_utils.h"
33#include "apace/utils/apace_constant.h"33#include "apace/utils/apace_constant.h"
34#include "apace/tiling/quant_matmul_tiling_swat.h"34#include "apace/tiling/quant_matmul_tiling_swat.h"
35-#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_tiling_data.h"35+#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_tiling_data.h"
36#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_impl.h"36#include "apace/kernel/fusions/all_gather_quant_matmul/all_gather_mx_matmul_urma_impl.h"
37#include "apace/core/aiv_comm/collective_comm_context.h"37#include "apace/core/aiv_comm/collective_comm_context.h"
38#include "apace/utils/comm_channel_builder.h"38#include "apace/utils/comm_channel_builder.h"