Merged
fix/apply-came-part1-narrow-reduction #13736
fix/apply-came-part1-narrow-reduction #13736
Merged
zsq_888Add on 16 days ago
Total: 12 files changed+319-116
@@ -22,6 +22,7 @@
22#include "register/op_impl_registry.h"22#include "register/op_impl_registry.h"
23#include "tiling/platform/platform_ascendc.h"23#include "tiling/platform/platform_ascendc.h"
24#include "apply_came_part1_tiling_arch35.h"24#include "apply_came_part1_tiling_arch35.h"
25+#include "apply_came_part1_ub_plan.h"
25 26 
26namespace optiling {27namespace optiling {
27 28 
@@ -129,15 +130,15 @@ static ge::graphStatus ApplyCamePart1SetTilingData(gert::TilingContext* context,
129static inline ge::graphStatus CalcNdimTiling(const gert::TilingContext* context, ApplyCamePart1TilingData& tilingData)130static inline ge::graphStatus CalcNdimTiling(const gert::TilingContext* context, ApplyCamePart1TilingData& tilingData)
130{131{
131 OP_LOGD(context->GetNodeName(), "TilingApplyCamePart1 Enter CalcNdimTiling.");132 OP_LOGD(context->GetNodeName(), "TilingApplyCamePart1 Enter CalcNdimTiling.");
132- int64_t nNormalCoreNum = std::max(CeilMul(tilingData.get_N(), N_NUM_PER_CORE, tilingData.get_totalCoreNum()),133+ int64_t nNormalCoreNum = std::max(
133- static_cast<int64_t>(1));134+ CeilMul(tilingData.get_N(), tilingData.get_tileSize(), tilingData.get_totalCoreNum()), static_cast<int64_t>(1));
134 int64_t nTailCoreNum = tilingData.get_N() % nNormalCoreNum;135 int64_t nTailCoreNum = tilingData.get_N() % nNormalCoreNum;
135 if (nTailCoreNum == 0) {136 if (nTailCoreNum == 0) {
136 nTailCoreNum = nNormalCoreNum;137 nTailCoreNum = nNormalCoreNum;
137 }138 }
138 int64_t nCoreNum = CeilDiv(tilingData.get_N(), nNormalCoreNum);139 int64_t nCoreNum = CeilDiv(tilingData.get_N(), nNormalCoreNum);
139- int64_t nLoopNormCore = CeilDiv(nNormalCoreNum, N_NUM_PER_CORE);140+ int64_t nLoopNormCore = CeilDiv(nNormalCoreNum, tilingData.get_tileSize());
140- int64_t nLoopTailCore = CeilDiv(nTailCoreNum, N_NUM_PER_CORE);141+ int64_t nLoopTailCore = CeilDiv(nTailCoreNum, tilingData.get_tileSize());
141 tilingData.set_nNormalCoreNum(nNormalCoreNum);142 tilingData.set_nNormalCoreNum(nNormalCoreNum);
142 tilingData.set_nTailCoreNum(nTailCoreNum);143 tilingData.set_nTailCoreNum(nTailCoreNum);
143 tilingData.set_nCoreNum(nCoreNum);144 tilingData.set_nCoreNum(nCoreNum);
@@ -173,15 +174,15 @@ static inline ge::graphStatus CalcMdimTiling(const gert::TilingContext* context,
173 VECTOR_INNER_ERR_REPORT_TILIING(context->GetNodeName(), "TilingApplyCamePart1 nCoreNum is invalid."),174 VECTOR_INNER_ERR_REPORT_TILIING(context->GetNodeName(), "TilingApplyCamePart1 nCoreNum is invalid."),
174 return ge::GRAPH_FAILED);175 return ge::GRAPH_FAILED);
175 176 
176- const int64_t columnTileCount = CeilDiv(tilingData.get_M(), M_NUM_PER_CORE);177+ const int64_t columnTileCount = CeilDiv(tilingData.get_M(), tilingData.get_tileSize());
177 const int64_t availableMCoreNum = std::max<int64_t>(1, tilingData.get_totalCoreNum() / nCoreNum);178 const int64_t availableMCoreNum = std::max<int64_t>(1, tilingData.get_totalCoreNum() / nCoreNum);
178 int64_t mCoreNum = std::min(columnTileCount, availableMCoreNum);179 int64_t mCoreNum = std::min(columnTileCount, availableMCoreNum);
179 const int64_t columnsPerCore = CeilDiv(tilingData.get_M(), mCoreNum);180 const int64_t columnsPerCore = CeilDiv(tilingData.get_M(), mCoreNum);
180- const int64_t mNormalCoreNum = CeilAlign(columnsPerCore, M_NUM_PER_CORE);181+ const int64_t mNormalCoreNum = CeilAlign(columnsPerCore, tilingData.get_tileSize());
181 mCoreNum = CeilDiv(tilingData.get_M(), mNormalCoreNum);182 mCoreNum = CeilDiv(tilingData.get_M(), mNormalCoreNum);
182 const int64_t mTailCoreNum = tilingData.get_M() - mNormalCoreNum * (mCoreNum - 1);183 const int64_t mTailCoreNum = tilingData.get_M() - mNormalCoreNum * (mCoreNum - 1);
183 184 
184- tilingData.set_mLoopNumCore(CeilDiv(mNormalCoreNum, M_NUM_PER_CORE));185+ tilingData.set_mLoopNumCore(CeilDiv(mNormalCoreNum, tilingData.get_tileSize()));
185 tilingData.set_mNormalCoreNum(mNormalCoreNum);186 tilingData.set_mNormalCoreNum(mNormalCoreNum);
186 tilingData.set_mTailCoreNum(mTailCoreNum);187 tilingData.set_mTailCoreNum(mTailCoreNum);
187 tilingData.set_mCoreNum(mCoreNum);188 tilingData.set_mCoreNum(mCoreNum);
@@ -232,6 +233,29 @@ static ge::graphStatus CheckParamsShape(const gert::TilingContext* context)
232 return ge::GRAPH_SUCCESS;233 return ge::GRAPH_SUCCESS;
233}234}
234 235 
236+// Size buffers from the maximum concurrent live set in each phase.
237+static ge::graphStatus PlanUbBuffers(gert::TilingContext* context, ApplyCamePart1TilingData& data)
238+{
239+ auto platformInfo = context->GetPlatformInfo();
240+ OPS_CHECK_NULL_WITH_CONTEXT(context, platformInfo);
241+ auto platform = platform_ascendc::PlatformAscendC(platformInfo);
242+ uint64_t ubBytes = 0;
243+ platform.GetCoreMemSize(platform_ascendc::CoreMemType::UB, ubBytes);
244+ const uint64_t elementBytes = context->GetInputDesc(0)->GetDataType() == ge::DT_FLOAT ? 4 : 2;
245+ const auto plan = apply_came_ub::MakePlan(ubBytes, elementBytes, data.get_totalCoreNum());
246+ OP_TILING_CHECK(plan.tile == 0 || plan.directElements == 0,
247+ VECTOR_INNER_ERR_REPORT_TILIING(context, "UB cannot hold the peak live buffer set."),
248+ return ge::GRAPH_FAILED);
249+ data.set_tileSize(plan.tile);
BartonFang
BartonFangBartonFang16 days ago

【review】 问题分类:可测 位置:optim/apply_came_part1/op_host/arch35/apply_came_part1_tiling_arch35.cpp:249,代码:data.set_tileSize(plan.tile); 问题:本次将原来固定 64 的生产阶段分块改成 MakePlan 选择的 64/128,并让 FP16/FP32 生产与 post 阶段共用 arena。按现有 tiling UT 的 UB=248 KiB、48 核配置核对账本,FP16/BF16 和 FP32 都会选择 128;但完整 tests 目录只有一个实际 kernel ST,输入为 BF16 [65,67],两个轴均不足 128,只能执行尾块。其余 UT 只调用 InferShape/Tiling,也没有断言 tileSize、arena 容量和 post 分区。因此新引入的 128 行、128 列完整块搬运及相邻完整块切换没有任何设备用例或有效结果断言覆盖。 可能导致的结果:若 128 完整块的行跨度、数据搬运长度或共享缓冲区偏移发生错误,现有 [65,67] 尾块测试和主机 UT 仍无法识别,大矩阵的三个归约输出可能在回归时静默出错。 建议的修改方法:补充确认 tileSize=128 的规划/tiling 断言,并增加实际执行 kernel 的 [256,256] 及 [257,259] 等完整块与尾块混合回归;逐项校验 sum_grad_r、sum_grad_c、sum_grad_rc 与 FP64 参考,同时覆盖规划选择 64 的较小 UB 配置。

likedislike
zsq_888
zsq_888
16 days ago Comments:
likedislike
250+ data.set_ubArenaBytes(plan.arenaBytes);
251+ data.set_postUbElements(plan.postElements);
252+ data.set_directUbElements(plan.directElements);
253+ OP_LOGI(context->GetNodeName(),
254+ "UB plan: total=%lu tile=%lu persistent=%lu producerPeak=%lu arena=%lu postElements=%lu", ubBytes,
255+ plan.tile, plan.persistentBytes, plan.producerPeakBytes, plan.arenaBytes, plan.postElements);
256+ return ge::GRAPH_SUCCESS;
257+}
258+ 
235static bool ShouldUseDirectBatch(const ApplyCamePart1TilingData& tilingData)259static bool ShouldUseDirectBatch(const ApplyCamePart1TilingData& tilingData)
236{260{
237 const int64_t n = tilingData.get_N();261 const int64_t n = tilingData.get_N();
@@ -240,7 +264,7 @@ static bool ShouldUseDirectBatch(const ApplyCamePart1TilingData& tilingData)
240 return false;264 return false;
241 }265 }
242 const int64_t matrixElements = n * m;266 const int64_t matrixElements = n * m;
243- if (matrixElements > MAX_DIRECT_MATRIX_ELEMENTS) {267+ if (matrixElements > tilingData.get_directUbElements()) {
244 return false;268 return false;
245 }269 }
246 const bool singleVectorTile = n <= N_NUM_PER_CORE && m <= M_NUM_PER_CORE;270 const bool singleVectorTile = n <= N_NUM_PER_CORE && m <= M_NUM_PER_CORE;
@@ -287,7 +311,7 @@ static ge::graphStatus CalculateApplyCamePart1Workspace(gert::TilingContext* con
287 size_t nLoopCount = 0;311 size_t nLoopCount = 0;
288 size_t rElements = 0;312 size_t rElements = 0;
289 size_t rcElements = 0;313 size_t rcElements = 0;
290- const size_t totalColumnTileCount = static_cast<size_t>(CeilDiv(tilingData.get_M(), M_NUM_PER_CORE));314+ const size_t totalColumnTileCount = static_cast<size_t>(CeilDiv(tilingData.get_M(), tilingData.get_tileSize()));
291 OP_TILING_CHECK(315 OP_TILING_CHECK(
292 !CheckedMultiply(nCoreNum - 1, nLoopNormCore, normalLoopCount) ||316 !CheckedMultiply(nCoreNum - 1, nLoopNormCore, normalLoopCount) ||
293 !CheckedAdd(normalLoopCount, nLoopTailCore, nLoopCount) ||317 !CheckedAdd(normalLoopCount, nLoopTailCore, nLoopCount) ||
@@ -308,8 +332,10 @@ static ge::graphStatus CalculateApplyCamePart1Workspace(gert::TilingContext* con
308 size_t cWorkspaceBytes = 0;332 size_t cWorkspaceBytes = 0;
309 OP_TILING_CHECK(333 OP_TILING_CHECK(
310 !CheckedMultiply(rcElements, rcSlotSize, rcStorageElements) ||334 !CheckedMultiply(rcElements, rcSlotSize, rcStorageElements) ||
311- !CheckedMultiply(rElements, static_cast<size_t>(N_NUM_PER_CORE * sizeof(float)), rWorkspaceBytes) ||335+ !CheckedMultiply(rElements, static_cast<size_t>(tilingData.get_tileSize() * sizeof(float)),
312- !CheckedMultiply(rcElements, static_cast<size_t>(M_NUM_PER_CORE * sizeof(float)), cWorkspaceBytes),336+ rWorkspaceBytes) ||
337+ !CheckedMultiply(rcElements, static_cast<size_t>(tilingData.get_tileSize() * sizeof(float)),
338+ cWorkspaceBytes),
313 VECTOR_INNER_ERR_REPORT_TILIING(context->GetNodeName(), "ApplyCamePart1 workspace byte size overflows."),339 VECTOR_INNER_ERR_REPORT_TILIING(context->GetNodeName(), "ApplyCamePart1 workspace byte size overflows."),
314 return ge::GRAPH_FAILED);340 return ge::GRAPH_FAILED);
315 341 
@@ -442,6 +468,9 @@ static ge::graphStatus InitializeApplyCamePart1Tiling(gert::TilingContext* conte
442 468 
443static ge::graphStatus CalculateApplyCamePart1Tiling(gert::TilingContext* context, ApplyCamePart1TilingData& tilingData)469static ge::graphStatus CalculateApplyCamePart1Tiling(gert::TilingContext* context, ApplyCamePart1TilingData& tilingData)
444{470{
471+ if (PlanUbBuffers(context, tilingData) != ge::GRAPH_SUCCESS) {
472+ return ge::GRAPH_FAILED;
473+ }
445 if (ShouldUseDirectBatch(tilingData)) {474 if (ShouldUseDirectBatch(tilingData)) {
446 tilingData.set_directBatchMode(1);475 tilingData.set_directBatchMode(1);
447 tilingData.set_nNormalCoreNum(tilingData.get_N());476 tilingData.set_nNormalCoreNum(tilingData.get_N());
@@ -0,0 +1,63 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+#ifndef APPLY_CAME_PART1_UB_PLAN_H
11+#define APPLY_CAME_PART1_UB_PLAN_H
12+#include <cstdint>
13+namespace apply_came_ub {
14+struct Plan {
15+ uint64_t tile = 0;
16+ uint64_t persistentBytes = 0;
17+ uint64_t arenaBytes = 0;
18+ uint64_t producerPeakBytes = 0;
19+ uint64_t postElements = 0;
20+ uint64_t directElements = 0;
21+};
22+constexpr uint64_t kReserveBytes = 8192;
23+constexpr uint64_t AlignDown(uint64_t n, uint64_t a) { return n / a * a; }
24+// Lifetimes in the reusable arena:
25+// elementwise: values[t*t], rowAccumulator[2*t], cast[t*t]
26+// R/C/RC: values[t*t], rowAccumulator[2*t], scratch[12*t]
27+// post: six FP32 pair-reduction nodes plus one 64B scalar-output slot.
28+// Input/output queues remain physically allocated throughout all phases.
29+inline Plan MakePlan(uint64_t ub, uint64_t dtypeBytes, uint64_t cores)
30+{
31+ Plan p;
32+ if ((dtypeBytes != 2 && dtypeBytes != 4) || cores == 0 || cores > 4096 || ub <= kReserveBytes)
33+ return p;
34+ // Power-of-two tiles preserve alignment and the vector reduction contract.
35+ for (uint64_t t = 64; t <= 128; t *= 2) {
36+ const uint64_t fixed = t * t * dtypeBytes + 32 + 2 * t * 4 + 16 * 4 + cores * 32;
37+ const uint64_t work = t * t > 12 * t ? t * t : 12 * t;
38+ const uint64_t producerPeak = (t * t + 2 * t + work) * 4;
39+ if (ub <= kReserveBytes + fixed)
40+ continue;
41+ const uint64_t arena = AlignDown(ub - kReserveBytes - fixed, 256);
42+ if (arena < producerPeak || arena <= 64)
43+ continue;
44+ const uint64_t post = AlignDown((arena - 64) / 4, t);
45+ if (post < 2 * t || post < 6 * 8)
46+ continue;
47+ p.tile = t;
48+ p.persistentBytes = fixed;
49+ p.arenaBytes = arena;
50+ p.producerPeakBytes = producerPeak;
51+ p.postElements = post;
52+ }
53+ // Direct: cast and output have disjoint lifetimes and share one slot.
54+ // grad(T), values(FP32), cast/output(FP32), plus fixed queue/scratch slots.
55+ constexpr uint64_t fixedDirect = 32 + 2 * 64 * 4 + 12 * 64 * 4 + (2 * 32 + 3 * 8) * 4;
56+ if (ub > kReserveBytes + fixedDirect) {
57+ const uint64_t count = AlignDown((ub - kReserveBytes - fixedDirect) / (dtypeBytes + 8), 64);
58+ p.directElements = count < 8192 ? count : 8192;
59+ }
60+ return p;
61+}
62+} // namespace apply_came_ub
63+#endif
@@ -49,7 +49,8 @@ __global__ __aicore__ void apply_came_part1(GM_ADDR grad, GM_ADDR eps, GM_ADDR s
49 49 
50 if (tilingData.directBatchMode != 0) {50 if (tilingData.directBatchMode != 0) {
51 ApplyCamePart1::ApplyCamePart1Batch<D_T> batchOp;51 ApplyCamePart1::ApplyCamePart1Batch<D_T> batchOp;
52- batchOp.Init(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, tilingData.N, tilingData.M);52+ batchOp.Init(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, tilingData.N, tilingData.M,
53+ tilingData.directUbElements);
53 batchOp.Process(tilingData.batchCount, tilingData.usedCoreNum);54 batchOp.Process(tilingData.batchCount, tilingData.usedCoreNum);
54 return;55 return;
55 }56 }
@@ -58,7 +59,7 @@ __global__ __aicore__ void apply_came_part1(GM_ADDR grad, GM_ADDR eps, GM_ADDR s
58 ApplyCamePart1::ApplyCamePart1FP32<float> op;59 ApplyCamePart1::ApplyCamePart1FP32<float> op;
59 op.InitOnce(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, userWS, &tilingData);60 op.InitOnce(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, userWS, &tilingData);
60 ApplyCamePart1::ApplyCamePart1Post<float> opPost;61 ApplyCamePart1::ApplyCamePart1Post<float> opPost;
61- opPost.Init(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, userWS, &tilingData);62+ opPost.Init(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, userWS, &tilingData, op.GetUbArena());
62 // Clear outputs and workspace before concurrent atomic updates.63 // Clear outputs and workspace before concurrent atomic updates.
63 op.ZeroWorkspace(userWS);64 op.ZeroWorkspace(userWS);
64 op.ZeroOutputs(sum_grad_r, sum_grad_c, sum_grad_rc, tilingData.batchCount);65 op.ZeroOutputs(sum_grad_r, sum_grad_c, sum_grad_rc, tilingData.batchCount);
@@ -81,7 +82,7 @@ __global__ __aicore__ void apply_came_part1(GM_ADDR grad, GM_ADDR eps, GM_ADDR s
81 ApplyCamePart1::ApplyCamePart1FP16<D_T> op;82 ApplyCamePart1::ApplyCamePart1FP16<D_T> op;
82 op.InitOnce(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, userWS, &tilingData);83 op.InitOnce(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, userWS, &tilingData);
83 ApplyCamePart1::ApplyCamePart1Post<float> opPost;84 ApplyCamePart1::ApplyCamePart1Post<float> opPost;
84- opPost.Init(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, userWS, &tilingData);85+ opPost.Init(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, userWS, &tilingData, op.GetUbArena());
85 op.ZeroWorkspace(userWS);86 op.ZeroWorkspace(userWS);
86 op.ZeroOutputs(sum_grad_r, sum_grad_c, sum_grad_rc, tilingData.batchCount);87 op.ZeroOutputs(sum_grad_r, sum_grad_c, sum_grad_rc, tilingData.batchCount);
87 SyncAll();88 SyncAll();
@@ -21,8 +21,10 @@ template <typename T>
21class ApplyCamePart1Batch {21class ApplyCamePart1Batch {
22public:22public:
23 __aicore__ inline void Init(GM_ADDR grad, GM_ADDR eps, GM_ADDR sumGradR, GM_ADDR sumGradC, GM_ADDR sumGradRC,23 __aicore__ inline void Init(GM_ADDR grad, GM_ADDR eps, GM_ADDR sumGradR, GM_ADDR sumGradC, GM_ADDR sumGradRC,
24- int64_t n, int64_t m)24+ int64_t n, int64_t m, int64_t ubElements)
25 {25 {
26+ maxInputElements_ = ubElements;
27+ maxOutputElements_ = ubElements + 2 * kMaxTileBatches + 3 * kOutputAlignmentElements;
26 n_ = n;28 n_ = n;
27 m_ = m;29 m_ = m;
28 matrixElements_ = n * m;30 matrixElements_ = n * m;
@@ -32,13 +34,13 @@ public:
32 sumGradCGm_.SetGlobalBuffer(reinterpret_cast<__gm__ float*>(sumGradC));34 sumGradCGm_.SetGlobalBuffer(reinterpret_cast<__gm__ float*>(sumGradC));
33 sumGradRCGm_.SetGlobalBuffer(reinterpret_cast<__gm__ float*>(sumGradRC));35 sumGradRCGm_.SetGlobalBuffer(reinterpret_cast<__gm__ float*>(sumGradRC));
34 36 
35- pipe_.InitBuffer(gradQueue_, 1, kMaxInputElements * sizeof(T));37+ pipe_.InitBuffer(gradQueue_, 1, maxInputElements_ * sizeof(T));
36 pipe_.InitBuffer(epsQueue_, 1, kBlockFloats * sizeof(float));38 pipe_.InitBuffer(epsQueue_, 1, kBlockFloats * sizeof(float));
37 pipe_.InitBuffer(sumGradRQueue_, 1, kVectorTileWidth * sizeof(float));39 pipe_.InitBuffer(sumGradRQueue_, 1, kVectorTileWidth * sizeof(float));
38 pipe_.InitBuffer(sumGradCQueue_, 1, kVectorTileWidth * sizeof(float));40 pipe_.InitBuffer(sumGradCQueue_, 1, kVectorTileWidth * sizeof(float));
39- pipe_.InitBuffer(castBuf_, kMaxInputElements * sizeof(float));41+ // Cast dies before output production, so both use this physical slot.
40- pipe_.InitBuffer(valueBuf_, kMaxInputElements * sizeof(float));42+ pipe_.InitBuffer(castOutputBuf_, maxOutputElements_ * sizeof(float));
41- pipe_.InitBuffer(outputBuf_, kMaxOutputElements * sizeof(float));43+ pipe_.InitBuffer(valueBuf_, maxInputElements_ * sizeof(float));
42 pipe_.InitBuffer(reduceBuf_, kReduceScratchElements * sizeof(float));44 pipe_.InitBuffer(reduceBuf_, kReduceScratchElements * sizeof(float));
43 }45 }
44 46 
@@ -92,9 +94,9 @@ private:
92 94 
93 __aicore__ inline int64_t GetScalarTileBatchCount() const95 __aicore__ inline int64_t GetScalarTileBatchCount() const
94 {96 {
95- int64_t byInput = kMaxInputElements / matrixElements_;97+ int64_t byInput = maxInputElements_ / matrixElements_;
96 const int64_t outputElementsPerBatch = n_ + m_ + 1;98 const int64_t outputElementsPerBatch = n_ + m_ + 1;
97- int64_t byOutput = kMaxOutputElements / outputElementsPerBatch;99+ int64_t byOutput = maxOutputElements_ / outputElementsPerBatch;
98 int64_t tileBatches = byInput < byOutput ? byInput : byOutput;100 int64_t tileBatches = byInput < byOutput ? byInput : byOutput;
99 tileBatches = tileBatches < kMaxTileBatches ? tileBatches : kMaxTileBatches;101 tileBatches = tileBatches < kMaxTileBatches ? tileBatches : kMaxTileBatches;
100 return tileBatches > 0 ? tileBatches : 1;102 return tileBatches > 0 ? tileBatches : 1;
@@ -121,9 +123,9 @@ private:
121 gradQueue_.EnQue(gradLocal);123 gradQueue_.EnQue(gradLocal);
122 gradLocal = gradQueue_.DeQue<T>();124 gradLocal = gradQueue_.DeQue<T>();
123 125 
124- LocalTensor<float> castLocal = castBuf_.Get<float>();126+ LocalTensor<float> castLocal = castOutputBuf_.Get<float>();
125 LocalTensor<float> valueLocal = valueBuf_.Get<float>();127 LocalTensor<float> valueLocal = valueBuf_.Get<float>();
126- LocalTensor<float> outputLocal = outputBuf_.Get<float>();128+ LocalTensor<float> outputLocal = castOutputBuf_.Get<float>();
127 event_t eventMte2ToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V));129 event_t eventMte2ToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V));
128 SetFlag<HardEvent::MTE2_V>(eventMte2ToV);130 SetFlag<HardEvent::MTE2_V>(eventMte2ToV);
129 WaitFlag<HardEvent::MTE2_V>(eventMte2ToV);131 WaitFlag<HardEvent::MTE2_V>(eventMte2ToV);
@@ -212,7 +214,7 @@ private:
212 LocalTensor<float>& sumGradRCLocal)214 LocalTensor<float>& sumGradRCLocal)
213 {215 {
214 const int64_t paddedElements = n_ * kVectorTileWidth;216 const int64_t paddedElements = n_ * kVectorTileWidth;
215- LocalTensor<float> castLocal = castBuf_.Get<float>();217+ LocalTensor<float> castLocal = castOutputBuf_.Get<float>();
216 LocalTensor<float> valueLocal = valueBuf_.Get<float>();218 LocalTensor<float> valueLocal = valueBuf_.Get<float>();
217 LocalTensor<float> reductionScratch = reduceBuf_.Get<float>();219 LocalTensor<float> reductionScratch = reduceBuf_.Get<float>();
218 event_t eventMte2ToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V));220 event_t eventMte2ToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V));
@@ -236,8 +238,14 @@ private:
236 ComputeCApplyCamePart1(sumGradCQueue_, n_, valueLocal, 1, 1, kVectorTileWidth, m_, 0);238 ComputeCApplyCamePart1(sumGradCQueue_, n_, valueLocal, 1, 1, kVectorTileWidth, m_, 0);
237 sumGradRLocal = sumGradRQueue_.DeQue<float>();239 sumGradRLocal = sumGradRQueue_.DeQue<float>();
238 sumGradCLocal = sumGradCQueue_.DeQue<float>();240 sumGradCLocal = sumGradCQueue_.DeQue<float>();
239- sumGradRCLocal = outputBuf_.Get<float>();241+ sumGradRCLocal = castOutputBuf_.Get<float>();
240 FinishApplyCamePart1Reduction(sumGradRCLocal, sumGradCLocal, reductionScratch, kVectorTileWidth);242 FinishApplyCamePart1Reduction(sumGradRCLocal, sumGradCLocal, reductionScratch, kVectorTileWidth);
243+ // The producer/post path carries this low slot through workspace.
244+ // Direct writes RC immediately, so fold it before the one-float copy.
245+ // Both slots already live in castOutputBuf_; no new UB allocation.
246+ constexpr int64_t kRcLowOffset = 8;
247+ Add(sumGradRCLocal, sumGradRCLocal, sumGradRCLocal[kRcLowOffset], 1);
248+ PipeBarrier<PIPE_V>();
241 }249 }
242 250 
243 __aicore__ inline void CopyOutVectorMatrix(int64_t batchIdx, LocalTensor<T> gradLocal,251 __aicore__ inline void CopyOutVectorMatrix(int64_t batchIdx, LocalTensor<T> gradLocal,
@@ -303,6 +311,8 @@ private:
303 3 * kOutputAlignmentElements;311 3 * kOutputAlignmentElements;
304 static constexpr int64_t kReduceScratchElements = 12 * kVectorTileWidth;312 static constexpr int64_t kReduceScratchElements = 12 * kVectorTileWidth;
305 313 
314+ int64_t maxInputElements_{8192};
315+ int64_t maxOutputElements_{8280};
306 int64_t n_{0};316 int64_t n_{0};
307 int64_t m_{0};317 int64_t m_{0};
308 int64_t matrixElements_{0};318 int64_t matrixElements_{0};
@@ -311,9 +321,8 @@ private:
311 TQue<QuePosition::VECIN, 1> epsQueue_;321 TQue<QuePosition::VECIN, 1> epsQueue_;
312 TQue<QuePosition::VECOUT, 1> sumGradRQueue_;322 TQue<QuePosition::VECOUT, 1> sumGradRQueue_;
313 TQue<QuePosition::VECOUT, 1> sumGradCQueue_;323 TQue<QuePosition::VECOUT, 1> sumGradCQueue_;
314- TBuf<QuePosition::VECCALC> castBuf_;324+ TBuf<QuePosition::VECCALC> castOutputBuf_;
315 TBuf<QuePosition::VECCALC> valueBuf_;325 TBuf<QuePosition::VECCALC> valueBuf_;
316- TBuf<QuePosition::VECCALC> outputBuf_;
317 TBuf<QuePosition::VECCALC> reduceBuf_;326 TBuf<QuePosition::VECCALC> reduceBuf_;
318 GlobalTensor<T> gradGm_;327 GlobalTensor<T> gradGm_;
319 GlobalTensor<float> epsGm_;328 GlobalTensor<float> epsGm_;
@@ -64,9 +64,8 @@ __aicore__ inline int64_t CeilDivApplyCamePart1(int64_t value, int64_t factor)
64 64 
65// Apply eps to the valid columns in each row.65// Apply eps to the valid columns in each row.
66__aicore__ inline void AddEpsApplyCamePart1(LocalTensor<float> values, float eps, int64_t tileColumns,66__aicore__ inline void AddEpsApplyCamePart1(LocalTensor<float> values, float eps, int64_t tileColumns,
67- int64_t curRepeatTimes)67+ int64_t curRepeatTimes, int64_t tileSize = 64)
68{68{
69- constexpr int64_t tileSize = 64;
70 if (tileColumns >= tileSize) {69 if (tileColumns >= tileSize) {
71 Adds(values, values, eps, curRepeatTimes * tileSize);70 Adds(values, values, eps, curRepeatTimes * tileSize);
72 return;71 return;
@@ -36,6 +36,8 @@ public:
36 __aicore__ inline void ZeroWorkspace(GM_ADDR workspace);36 __aicore__ inline void ZeroWorkspace(GM_ADDR workspace);
37 __aicore__ inline void ProcessTile(int64_t nLoopIdx, int64_t mLoopIdx, int64_t curRepeatTimes);37 __aicore__ inline void ProcessTile(int64_t nLoopIdx, int64_t mLoopIdx, int64_t curRepeatTimes);
38 38 
39+ __aicore__ inline LocalTensor<float> GetUbArena() { return phaseArena_.Get<float>(); }
40+ 
39private:41private:
40 __aicore__ inline void ParseTilingData(const ApplyCamePart1TilingData* tilingData);42 __aicore__ inline void ParseTilingData(const ApplyCamePart1TilingData* tilingData);
41 __aicore__ inline void ClearAcculateMatrix();43 __aicore__ inline void ClearAcculateMatrix();
@@ -51,6 +53,8 @@ private:
51 __aicore__ inline void ComputeC(int64_t mLoopIdx, int64_t curRepeatTimes, LocalTensor<float> gradSqrtTmpUb);53 __aicore__ inline void ComputeC(int64_t mLoopIdx, int64_t curRepeatTimes, LocalTensor<float> gradSqrtTmpUb);
52 54 
53private:55private:
56+ TBuf<QuePosition::VECCALC> phaseArena_;
57+ int64_t arenaBytes_{0};
54 TPipe pipe;58 TPipe pipe;
55 TQue<QuePosition::VECIN, 1> gradQueue_;59 TQue<QuePosition::VECIN, 1> gradQueue_;
56 TQue<QuePosition::VECIN, 1> epsQueue_;60 TQue<QuePosition::VECIN, 1> epsQueue_;
@@ -59,11 +63,6 @@ private:
59 TQue<QuePosition::VECOUT, 1> sumGradCQueue_;63 TQue<QuePosition::VECOUT, 1> sumGradCQueue_;
60 TQue<QuePosition::VECOUT, 1> sumGradRCQueue_;64 TQue<QuePosition::VECOUT, 1> sumGradRCQueue_;
61 65 
62- TBuf<QuePosition::VECCALC> accuComTmpBuf_;
63- TBuf<QuePosition::VECCALC> gradCastTmpBuf_;
64- TBuf<QuePosition::VECCALC> gradSqrtTmpBuf_;
65- TBuf<QuePosition::VECCALC> mComTmpBuf_;
66- 
67 GlobalTensor<T> gmGrad_;66 GlobalTensor<T> gmGrad_;
68 GlobalTensor<float> gmEps_;67 GlobalTensor<float> gmEps_;
69 GlobalTensor<float> gmSumGradR_;68 GlobalTensor<float> gmSumGradR_;
@@ -114,7 +113,7 @@ private:
114 GM_ADDR sumGradCAddrBatch_{0};113 GM_ADDR sumGradCAddrBatch_{0};
115 GM_ADDR sumGradRCAddrBatch_{0};114 GM_ADDR sumGradRCAddrBatch_{0};
116 115 
117- const int64_t ONCE_HANDLE_NUM64{64};116+ int64_t ONCE_HANDLE_NUM64{64};
118 const int64_t ONCE_ONE_SIZE8{8};117 const int64_t ONCE_ONE_SIZE8{8};
119};118};
120 119 
@@ -122,6 +121,8 @@ template <typename T>
122__aicore__ inline void ApplyCamePart1FP16<T>::ParseTilingData(const ApplyCamePart1TilingData* tilingData)121__aicore__ inline void ApplyCamePart1FP16<T>::ParseTilingData(const ApplyCamePart1TilingData* tilingData)
123{122{
124 // 总维度[N, M]123 // 总维度[N, M]
124+ ONCE_HANDLE_NUM64 = tilingData->tileSize;
125+ arenaBytes_ = tilingData->ubArenaBytes;
125 N = tilingData->N;126 N = tilingData->N;
126 M = tilingData->M;127 M = tilingData->M;
127 128 
@@ -192,7 +193,7 @@ template <typename T>
192__aicore__ inline void ApplyCamePart1FP16<T>::InitBatch(int64_t batchIdx)193__aicore__ inline void ApplyCamePart1FP16<T>::InitBatch(int64_t batchIdx)
193{194{
194 // The post stage reuses UB on the group leader, so each batch starts with a fresh row accumulator.195 // The post stage reuses UB on the group leader, so each batch starts with a fresh row accumulator.
195- LocalTensor<float> rowAccumulator = accuComTmpBuf_.Get<float>();196+ LocalTensor<float> rowAccumulator = phaseArena_.Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64];
196 Duplicate(rowAccumulator, static_cast<float>(0), 2 * ONCE_HANDLE_NUM64);197 Duplicate(rowAccumulator, static_cast<float>(0), 2 * ONCE_HANDLE_NUM64);
197 PipeBarrier<PIPE_V>();198 PipeBarrier<PIPE_V>();
198 199 
@@ -215,7 +216,7 @@ template <typename T>
215__aicore__ inline void ApplyCamePart1FP16<T>::ZeroOutputs(GM_ADDR sum_grad_r, GM_ADDR sum_grad_c, GM_ADDR sum_grad_rc,216__aicore__ inline void ApplyCamePart1FP16<T>::ZeroOutputs(GM_ADDR sum_grad_r, GM_ADDR sum_grad_c, GM_ADDR sum_grad_rc,
216 int64_t batchCount)217 int64_t batchCount)
217{218{
218- LocalTensor<float> zeroUb = gradCastTmpBuf_.Get<float>();219+ LocalTensor<float> zeroUb = phaseArena_.Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 + 2 * ONCE_HANDLE_NUM64];
219 Duplicate(zeroUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);220 Duplicate(zeroUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);
220 PipeBarrier<PIPE_V>();221 PipeBarrier<PIPE_V>();
221 event_t eventIdVToMte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3));222 event_t eventIdVToMte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3));
@@ -249,7 +250,8 @@ __aicore__ inline void ApplyCamePart1FP16<T>::ZeroWorkspace(GM_ADDR workspace)
249 return;250 return;
250 }251 }
251 }252 }
252- LocalTensor<float> zeroUb = gradCastTmpBuf_.Get<float>();253+ LocalTensor<float> zeroUb = phaseArena_
254+ .Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 + 2 * ONCE_HANDLE_NUM64];
253 Duplicate(zeroUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);255 Duplicate(zeroUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);
254 PipeBarrier<PIPE_V>();256 PipeBarrier<PIPE_V>();
255 event_t eventIdVToMte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3));257 event_t eventIdVToMte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3));
@@ -311,10 +313,12 @@ __aicore__ inline void ApplyCamePart1FP16<T>::Compute(int64_t nLoopIdx, int64_t
311 LocalTensor<T> gradLocal = gradQueue_.DeQue<T>();313 LocalTensor<T> gradLocal = gradQueue_.DeQue<T>();
312 LocalTensor<float> epsLocal = epsQueue_.DeQue<float>();314 LocalTensor<float> epsLocal = epsQueue_.DeQue<float>();
313 315 
314- LocalTensor<float> gradCastTmpUb = gradCastTmpBuf_.Get<float>();316+ LocalTensor<float> gradCastTmpUb = phaseArena_
315- LocalTensor<float> gradSqrtTmpUb = gradSqrtTmpBuf_.Get<float>();317+ .Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 + 2 * ONCE_HANDLE_NUM64];
316- LocalTensor<float> accuComTmpUb = accuComTmpBuf_.Get<float>();318+ LocalTensor<float> gradSqrtTmpUb = phaseArena_.Get<float>();
317- LocalTensor<float> reductionTmpUb = mComTmpBuf_.Get<float>();319+ LocalTensor<float> accuComTmpUb = phaseArena_.Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64];
320+ LocalTensor<float>
321+ reductionTmpUb = phaseArena_.Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 + 2 * ONCE_HANDLE_NUM64];
318 Duplicate(gradCastTmpUb, (float)0.0, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);322 Duplicate(gradCastTmpUb, (float)0.0, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);
319 PipeBarrier<PIPE_V>();323 PipeBarrier<PIPE_V>();
320 event_t eventIdMte2ToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V));324 event_t eventIdMte2ToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V));
@@ -334,7 +338,8 @@ __aicore__ inline void ApplyCamePart1FP16<T>::Compute(int64_t nLoopIdx, int64_t
334 Mul(gradSqrtTmpUb, gradCastTmpUb, gradCastTmpUb, calCount);338 Mul(gradSqrtTmpUb, gradCastTmpUb, gradCastTmpUb, calCount);
335 339 
336 PipeBarrier<PIPE_V>();340 PipeBarrier<PIPE_V>();
337- AddEpsApplyCamePart1(gradSqrtTmpUb, eps, coreColumns_ - mLoopIdx * ONCE_HANDLE_NUM64, curRepeatTimes);341+ AddEpsApplyCamePart1(gradSqrtTmpUb, eps, coreColumns_ - mLoopIdx * ONCE_HANDLE_NUM64, curRepeatTimes,
342+ ONCE_HANDLE_NUM64);
338 343 
339 PipeBarrier<PIPE_V>();344 PipeBarrier<PIPE_V>();
340 (void)accuComTmpUb;345 (void)accuComTmpUb;
@@ -347,7 +352,7 @@ __aicore__ inline void ApplyCamePart1FP16<T>::Compute(int64_t nLoopIdx, int64_t
347 352 
348 if (mLoopIdx == (mLoopCount_ - 1)) {353 if (mLoopIdx == (mLoopCount_ - 1)) {
349 PipeBarrier<PIPE_V>();354 PipeBarrier<PIPE_V>();
350- Duplicate(accuComTmpUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);355+ Duplicate(accuComTmpUb, static_cast<float>(0), 2 * ONCE_HANDLE_NUM64);
351 Duplicate(reductionTmpUb, static_cast<float>(0), 4 * ONCE_HANDLE_NUM64);356 Duplicate(reductionTmpUb, static_cast<float>(0), 4 * ONCE_HANDLE_NUM64);
352 }357 }
353 358 
@@ -418,23 +423,9 @@ __aicore__ inline void ApplyCamePart1FP16<T>::ProcessTile(int64_t nLoopIdx, int6
418template <typename T>423template <typename T>
419__aicore__ inline void ApplyCamePart1FP16<T>::ClearAcculateMatrix()424__aicore__ inline void ApplyCamePart1FP16<T>::ClearAcculateMatrix()
420{425{
421- constexpr float scalarValue = 0;426+ pipe.InitBuffer(phaseArena_, arenaBytes_);
422- 427+ LocalTensor<float> arena = phaseArena_.Get<float>();
423- pipe.InitBuffer(gradCastTmpBuf_, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 * sizeof(float));428+ Duplicate(arena, static_cast<float>(0), arenaBytes_ / sizeof(float));
424- LocalTensor<float> gradCastTmpUb = gradCastTmpBuf_.Get<float>(ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);
425- Duplicate(gradCastTmpUb, scalarValue, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);
426- 
427- pipe.InitBuffer(gradSqrtTmpBuf_, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 * sizeof(float));
428- LocalTensor<float> gradSqrtTmpUb = gradSqrtTmpBuf_.Get<float>(ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);
429- Duplicate(gradSqrtTmpUb, scalarValue, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);
430- 
431- pipe.InitBuffer(accuComTmpBuf_, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 * sizeof(float));
432- LocalTensor<float> accuComTmpUb = accuComTmpBuf_.Get<float>(ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);
433- Duplicate(accuComTmpUb, scalarValue, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);
434- 
435- pipe.InitBuffer(mComTmpBuf_, (4 + 8) * ONCE_HANDLE_NUM64 * sizeof(float));
436- LocalTensor<float> reductionTmpUb = mComTmpBuf_.Get<float>();
437- Duplicate(reductionTmpUb, scalarValue, 4 * ONCE_HANDLE_NUM64);
438}429}
439 430 
440} // namespace ApplyCamePart1431} // namespace ApplyCamePart1
@@ -35,6 +35,8 @@ public:
35 __aicore__ inline void Process();35 __aicore__ inline void Process();
36 __aicore__ inline void ProcessTile(int64_t nLoopIdx, int64_t mLoopIdx, int64_t curRepeatTimes);36 __aicore__ inline void ProcessTile(int64_t nLoopIdx, int64_t mLoopIdx, int64_t curRepeatTimes);
37 37 
38+ __aicore__ inline LocalTensor<float> GetUbArena() { return phaseArena_.Get<float>(); }
39+ 
38private:40private:
39 __aicore__ inline void ParseTilingData(const ApplyCamePart1TilingData* tilingData);41 __aicore__ inline void ParseTilingData(const ApplyCamePart1TilingData* tilingData);
40 __aicore__ inline void ClearAcculateMatrix();42 __aicore__ inline void ClearAcculateMatrix();
@@ -53,6 +55,8 @@ private:
53 __aicore__ inline void ComputeC(int64_t mLoopIdx, int64_t curRepeatTimes, LocalTensor<float> gradSqrtTmpUb);55 __aicore__ inline void ComputeC(int64_t mLoopIdx, int64_t curRepeatTimes, LocalTensor<float> gradSqrtTmpUb);
54 56 
55private:57private:
58+ TBuf<QuePosition::VECCALC> phaseArena_;
59+ int64_t arenaBytes_{0};
56 TPipe pipe;60 TPipe pipe;
57 TQue<QuePosition::VECIN, 1> gradQueue_;61 TQue<QuePosition::VECIN, 1> gradQueue_;
58 TQue<QuePosition::VECIN, 1> epsQueue_;62 TQue<QuePosition::VECIN, 1> epsQueue_;
@@ -61,11 +65,6 @@ private:
61 TQue<QuePosition::VECOUT, 1> sumGradCQueue_;65 TQue<QuePosition::VECOUT, 1> sumGradCQueue_;
62 TQue<QuePosition::VECOUT, 1> sumGradRCQueue_;66 TQue<QuePosition::VECOUT, 1> sumGradRCQueue_;
63 67 
64- TBuf<QuePosition::VECCALC> accuComTmpBuf_;
65- TBuf<QuePosition::VECCALC> gradCastTmpBuf_;
66- TBuf<QuePosition::VECCALC> gradSqrtTmpBuf_;
67- TBuf<QuePosition::VECCALC> mComTmpBuf_;
68- 
69 GlobalTensor<float> gmGrad_;68 GlobalTensor<float> gmGrad_;
70 GlobalTensor<float> gmEps_;69 GlobalTensor<float> gmEps_;
71 GlobalTensor<float> gmSumGradR_;70 GlobalTensor<float> gmSumGradR_;
@@ -117,7 +116,7 @@ private:
117 GM_ADDR sumGradCAddrBatch_{0};116 GM_ADDR sumGradCAddrBatch_{0};
118 GM_ADDR sumGradRCAddrBatch_{0};117 GM_ADDR sumGradRCAddrBatch_{0};
119 118 
120- const int64_t ONCE_HANDLE_NUM64{64};119+ int64_t ONCE_HANDLE_NUM64{64};
121 const int64_t ONCE_ONE_SIZE8{8};120 const int64_t ONCE_ONE_SIZE8{8};
122};121};
123 122 
@@ -125,6 +124,8 @@ template <typename T>
125__aicore__ inline void ApplyCamePart1FP32<T>::ParseTilingData(const ApplyCamePart1TilingData* tilingData)124__aicore__ inline void ApplyCamePart1FP32<T>::ParseTilingData(const ApplyCamePart1TilingData* tilingData)
126{125{
127 // 总维度[N, M]126 // 总维度[N, M]
127+ ONCE_HANDLE_NUM64 = tilingData->tileSize;
128+ arenaBytes_ = tilingData->ubArenaBytes;
128 N = tilingData->N;129 N = tilingData->N;
129 M = tilingData->M;130 M = tilingData->M;
130 131 
@@ -195,7 +196,7 @@ template <typename T>
195__aicore__ inline void ApplyCamePart1FP32<T>::InitBatch(int64_t batchIdx)196__aicore__ inline void ApplyCamePart1FP32<T>::InitBatch(int64_t batchIdx)
196{197{
197 // The post stage reuses UB on the group leader, so each batch starts with a fresh row accumulator.198 // The post stage reuses UB on the group leader, so each batch starts with a fresh row accumulator.
198- LocalTensor<float> rowAccumulator = accuComTmpBuf_.Get<float>();199+ LocalTensor<float> rowAccumulator = phaseArena_.Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64];
199 Duplicate(rowAccumulator, static_cast<float>(0), 2 * ONCE_HANDLE_NUM64);200 Duplicate(rowAccumulator, static_cast<float>(0), 2 * ONCE_HANDLE_NUM64);
200 PipeBarrier<PIPE_V>();201 PipeBarrier<PIPE_V>();
201 202 
@@ -218,7 +219,7 @@ template <typename T>
218__aicore__ inline void ApplyCamePart1FP32<T>::ZeroOutputs(GM_ADDR sum_grad_r, GM_ADDR sum_grad_c, GM_ADDR sum_grad_rc,219__aicore__ inline void ApplyCamePart1FP32<T>::ZeroOutputs(GM_ADDR sum_grad_r, GM_ADDR sum_grad_c, GM_ADDR sum_grad_rc,
219 int64_t batchCount)220 int64_t batchCount)
220{221{
221- LocalTensor<float> zeroUb = gradCastTmpBuf_.Get<float>();222+ LocalTensor<float> zeroUb = phaseArena_.Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 + 2 * ONCE_HANDLE_NUM64];
222 Duplicate(zeroUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);223 Duplicate(zeroUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);
223 PipeBarrier<PIPE_V>();224 PipeBarrier<PIPE_V>();
224 event_t eventIdVToMte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3));225 event_t eventIdVToMte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3));
@@ -252,7 +253,8 @@ __aicore__ inline void ApplyCamePart1FP32<T>::ZeroWorkspace(GM_ADDR workspace)
252 return;253 return;
253 }254 }
254 }255 }
255- LocalTensor<float> zeroUb = gradCastTmpBuf_.Get<float>();256+ LocalTensor<float> zeroUb = phaseArena_
257+ .Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 + 2 * ONCE_HANDLE_NUM64];
256 Duplicate(zeroUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);258 Duplicate(zeroUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);
257 PipeBarrier<PIPE_V>();259 PipeBarrier<PIPE_V>();
258 event_t eventIdVToMte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3));260 event_t eventIdVToMte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3));
@@ -334,7 +336,8 @@ __aicore__ inline void ApplyCamePart1FP32<T>::ComputeElementwise(
334 Mul(gradSqrtTmpUb, gradCastTmpUb, gradCastTmpUb, calCount);336 Mul(gradSqrtTmpUb, gradCastTmpUb, gradCastTmpUb, calCount);
335 337 
336 PipeBarrier<PIPE_V>();338 PipeBarrier<PIPE_V>();
337- AddEpsApplyCamePart1(gradSqrtTmpUb, eps, coreColumns_ - mLoopIdx * ONCE_HANDLE_NUM64, curRepeatTimes);339+ AddEpsApplyCamePart1(gradSqrtTmpUb, eps, coreColumns_ - mLoopIdx * ONCE_HANDLE_NUM64, curRepeatTimes,
340+ ONCE_HANDLE_NUM64);
338 341 
339 PipeBarrier<PIPE_V>();342 PipeBarrier<PIPE_V>();
340 (void)accuComTmpUb;343 (void)accuComTmpUb;
@@ -345,10 +348,12 @@ __aicore__ inline void ApplyCamePart1FP32<T>::Compute(int64_t nLoopIdx, int64_t
345{348{
346 LocalTensor<float> gradLocal = gradQueue_.DeQue<float>();349 LocalTensor<float> gradLocal = gradQueue_.DeQue<float>();
347 LocalTensor<float> epsLocal = epsQueue_.DeQue<float>();350 LocalTensor<float> epsLocal = epsQueue_.DeQue<float>();
348- LocalTensor<float> gradCastTmpUb = gradCastTmpBuf_.Get<float>();351+ LocalTensor<float> gradCastTmpUb = phaseArena_
349- LocalTensor<float> gradSqrtTmpUb = gradSqrtTmpBuf_.Get<float>();352+ .Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 + 2 * ONCE_HANDLE_NUM64];
350- LocalTensor<float> accuComTmpUb = accuComTmpBuf_.Get<float>();353+ LocalTensor<float> gradSqrtTmpUb = phaseArena_.Get<float>();
351- LocalTensor<float> reductionTmpUb = mComTmpBuf_.Get<float>();354+ LocalTensor<float> accuComTmpUb = phaseArena_.Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64];
355+ LocalTensor<float>
356+ reductionTmpUb = phaseArena_.Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 + 2 * ONCE_HANDLE_NUM64];
352 ComputeElementwise(mLoopIdx, curRepeatTimes, gradLocal, epsLocal, gradCastTmpUb, gradSqrtTmpUb, accuComTmpUb);357 ComputeElementwise(mLoopIdx, curRepeatTimes, gradLocal, epsLocal, gradCastTmpUb, gradSqrtTmpUb, accuComTmpUb);
353 358 
354 LocalTensor<float> sumGradRCLocal = sumGradRCQueue_.AllocTensor<float>();359 LocalTensor<float> sumGradRCLocal = sumGradRCQueue_.AllocTensor<float>();
@@ -360,7 +365,7 @@ __aicore__ inline void ApplyCamePart1FP32<T>::Compute(int64_t nLoopIdx, int64_t
360 365 
361 if (mLoopIdx == (mLoopCount_ - 1)) {366 if (mLoopIdx == (mLoopCount_ - 1)) {
362 PipeBarrier<PIPE_V>();367 PipeBarrier<PIPE_V>();
363- Duplicate(accuComTmpUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);368+ Duplicate(accuComTmpUb, static_cast<float>(0), 2 * ONCE_HANDLE_NUM64);
364 Duplicate(reductionTmpUb, static_cast<float>(0), 4 * ONCE_HANDLE_NUM64);369 Duplicate(reductionTmpUb, static_cast<float>(0), 4 * ONCE_HANDLE_NUM64);
365 }370 }
366 sumGradCQueue_.EnQue<float>(sumGradCLocal);371 sumGradCQueue_.EnQue<float>(sumGradCLocal);
@@ -431,23 +436,9 @@ __aicore__ inline void ApplyCamePart1FP32<T>::ProcessTile(int64_t nLoopIdx, int6
431template <typename T>436template <typename T>
432__aicore__ inline void ApplyCamePart1FP32<T>::ClearAcculateMatrix()437__aicore__ inline void ApplyCamePart1FP32<T>::ClearAcculateMatrix()
433{438{
434- constexpr float scalarValue = 0;439+ pipe.InitBuffer(phaseArena_, arenaBytes_);
435- 440+ LocalTensor<float> arena = phaseArena_.Get<float>();
436- pipe.InitBuffer(gradCastTmpBuf_, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 * sizeof(float));441+ Duplicate(arena, static_cast<float>(0), arenaBytes_ / sizeof(float));
437- LocalTensor<float> gradCastTmpUb = gradCastTmpBuf_.Get<float>(ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);
438- Duplicate(gradCastTmpUb, scalarValue, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);
439- 
440- pipe.InitBuffer(gradSqrtTmpBuf_, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 * sizeof(float));
441- LocalTensor<float> gradSqrtTmpUb = gradSqrtTmpBuf_.Get<float>(ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);
442- Duplicate(gradSqrtTmpUb, scalarValue, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);
443- 
444- pipe.InitBuffer(accuComTmpBuf_, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 * sizeof(float));
445- LocalTensor<float> accuComTmpUb = accuComTmpBuf_.Get<float>(ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);
446- Duplicate(accuComTmpUb, scalarValue, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64);
447- 
448- pipe.InitBuffer(mComTmpBuf_, (4 + 8) * ONCE_HANDLE_NUM64 * sizeof(float));
449- LocalTensor<float> reductionTmpUb = mComTmpBuf_.Get<float>();
450- Duplicate(reductionTmpUb, scalarValue, 4 * ONCE_HANDLE_NUM64);
451}442}
452 443 
453} // namespace ApplyCamePart1444} // namespace ApplyCamePart1
@@ -29,12 +29,15 @@ class ApplyCamePart1Post {
29public:29public:
30 __aicore__ inline ApplyCamePart1Post(){};30 __aicore__ inline ApplyCamePart1Post(){};
31 __aicore__ inline void Init(GM_ADDR grad, GM_ADDR eps, GM_ADDR sum_grad_r, GM_ADDR sum_grad_c, GM_ADDR sum_grad_rc,31 __aicore__ inline void Init(GM_ADDR grad, GM_ADDR eps, GM_ADDR sum_grad_r, GM_ADDR sum_grad_c, GM_ADDR sum_grad_rc,
32- GM_ADDR workspace, const ApplyCamePart1TilingData* tilingData);32+ GM_ADDR workspace, const ApplyCamePart1TilingData* tilingData,
33+ LocalTensor<float> arena);
33 __aicore__ inline void ProcessBatch(int64_t batchIdx);34 __aicore__ inline void ProcessBatch(int64_t batchIdx);
34 35 
35private:36private:
36 __aicore__ inline void ParseTilingData(const ApplyCamePart1TilingData* tilingData);37 __aicore__ inline void ParseTilingData(const ApplyCamePart1TilingData* tilingData);
37 __aicore__ inline void Pre_Core_Compute(uint64_t gmOffsets, uint64_t cal_m, uint64_t base_m);38 __aicore__ inline void Pre_Core_Compute(uint64_t gmOffsets, uint64_t cal_m, uint64_t base_m);
39+ __aicore__ inline void ReduceNarrowC(uint64_t gmOffsets, uint64_t outputCount, uint64_t rowCount,
40+ uint64_t rowsPerChunk);
38 __aicore__ inline void ReduceWorkspaceRPartial(LocalTensor<float> inputLocal, int64_t base, int64_t partialRows);41 __aicore__ inline void ReduceWorkspaceRPartial(LocalTensor<float> inputLocal, int64_t base, int64_t partialRows);
39 __aicore__ inline void ReduceWorkspaceR();42 __aicore__ inline void ReduceWorkspaceR();
40 __aicore__ inline void ReduceWorkspaceRCPair(LocalTensor<float> high, LocalTensor<float> highRight,43 __aicore__ inline void ReduceWorkspaceRCPair(LocalTensor<float> high, LocalTensor<float> highRight,
@@ -51,11 +54,12 @@ private:
51 LocalTensor<float> scratch);54 LocalTensor<float> scratch);
52 55 
53private:56private:
54- TPipe pipe;57+ int64_t postUbElements_{32768};
55- TBuf<QuePosition::VECCALC> inputBuf_;58+ LocalTensor<float> inputLocal_;
59+ LocalTensor<float> outputScratchLocal_;
60+ 
56 // Keep GM read-modify-write scratch separate from reduction input. Reusing61 // Keep GM read-modify-write scratch separate from reduction input. Reusing
57 // inputBuf_ here would overwrite values that the caller still needs to read.62 // inputBuf_ here would overwrite values that the caller still needs to read.
58- TBuf<QuePosition::VECCALC> outputScratchBuf_;
59 63 
60 GlobalTensor<float> gmGrad_;64 GlobalTensor<float> gmGrad_;
61 GlobalTensor<float> gmEps_;65 GlobalTensor<float> gmEps_;
@@ -95,7 +99,7 @@ private:
95 bool IsContainsTailM{false};99 bool IsContainsTailM{false};
96 int64_t batchIdx_{0};100 int64_t batchIdx_{0};
97 101 
98- const int64_t ONCE_HANDLE_NUM64{64};102+ int64_t ONCE_HANDLE_NUM64{64};
99 const int64_t ONCE_HANDLE_NUM512{512};103 const int64_t ONCE_HANDLE_NUM512{512};
100 const int64_t ONCE_ONE_SIZE8{8};104 const int64_t ONCE_ONE_SIZE8{8};
101};105};
@@ -104,6 +108,8 @@ template <typename T>
104__aicore__ inline void ApplyCamePart1Post<T>::ParseTilingData(const ApplyCamePart1TilingData* tilingData)108__aicore__ inline void ApplyCamePart1Post<T>::ParseTilingData(const ApplyCamePart1TilingData* tilingData)
105{109{
106 // 总维度[N, M]110 // 总维度[N, M]
111+ ONCE_HANDLE_NUM64 = tilingData->tileSize;
112+ postUbElements_ = tilingData->postUbElements;
107 N = tilingData->N;113 N = tilingData->N;
108 M = tilingData->M;114 M = tilingData->M;
109 115 
@@ -133,7 +139,7 @@ __aicore__ inline void ApplyCamePart1Post<T>::ParseTilingData(const ApplyCamePar
133template <typename T>139template <typename T>
134__aicore__ inline void ApplyCamePart1Post<T>::Init(GM_ADDR grad, GM_ADDR eps, GM_ADDR sum_grad_r, GM_ADDR sum_grad_c,140__aicore__ inline void ApplyCamePart1Post<T>::Init(GM_ADDR grad, GM_ADDR eps, GM_ADDR sum_grad_r, GM_ADDR sum_grad_c,
135 GM_ADDR sum_grad_rc, GM_ADDR workspace,141 GM_ADDR sum_grad_rc, GM_ADDR workspace,
136- const ApplyCamePart1TilingData* tilingData)142+ const ApplyCamePart1TilingData* tilingData, LocalTensor<float> arena)
137{143{
138 (void)grad;144 (void)grad;
139 (void)eps;145 (void)eps;
@@ -169,8 +175,8 @@ __aicore__ inline void ApplyCamePart1Post<T>::Init(GM_ADDR grad, GM_ADDR eps, GM
169 workspaceSumGradC_.SetGlobalBuffer(regionBase + 2 * rcOffsets + rOffsets);175 workspaceSumGradC_.SetGlobalBuffer(regionBase + 2 * rcOffsets + rOffsets);
170 }176 }
171 177 
172- pipe.InitBuffer(inputBuf_, ONCE_HANDLE_NUM512 * ONCE_HANDLE_NUM64 * sizeof(float));178+ inputLocal_ = arena;
173- pipe.InitBuffer(outputScratchBuf_, 2 * ONCE_ONE_SIZE8 * sizeof(float));179+ outputScratchLocal_ = arena[postUbElements_];
174}180}
175 181 
176template <typename T>182template <typename T>
@@ -192,7 +198,8 @@ __aicore__ inline void ApplyCamePart1Post<T>::ProcessBatch(int64_t batchIdx)
192 // 产者/归约阶段的同步由内核入口统一负责, 此处不再做全局屏障.198 // 产者/归约阶段的同步由内核入口统一负责, 此处不再做全局屏障.
193 batchIdx_ = batchIdx;199 batchIdx_ = batchIdx;
194 ReduceWorkspaceR();200 ReduceWorkspaceR();
195- uint64_t core_loop = (totalColumnTileCount_ + ONCE_HANDLE_NUM512 - 1) / ONCE_HANDLE_NUM512;201+ const uint64_t columnTilesPerChunk = postUbElements_ / ONCE_HANDLE_NUM64;
202+ uint64_t core_loop = (totalColumnTileCount_ + columnTilesPerChunk - 1) / columnTilesPerChunk;
196 uint64_t pre_core_m = (totalColumnTileCount_ + core_loop - 1) / core_loop * ONCE_HANDLE_NUM64;203 uint64_t pre_core_m = (totalColumnTileCount_ + core_loop - 1) / core_loop * ONCE_HANDLE_NUM64;
197 uint64_t last_core_m = (totalColumnTileCount_ - pre_core_m * (core_loop - 1) / ONCE_HANDLE_NUM64) *204 uint64_t last_core_m = (totalColumnTileCount_ - pre_core_m * (core_loop - 1) / ONCE_HANDLE_NUM64) *
198 ONCE_HANDLE_NUM64;205 ONCE_HANDLE_NUM64;
@@ -213,8 +220,8 @@ template <typename T>
213__aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceRPartial(LocalTensor<float> inputLocal, int64_t base,220__aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceRPartial(LocalTensor<float> inputLocal, int64_t base,
214 int64_t partialRows)221 int64_t partialRows)
215{222{
216- constexpr int64_t rowWidth = 64;223+ const int64_t rowWidth = ONCE_HANDLE_NUM64;
217- constexpr int64_t maxRowsPerCopy = 256;224+ const int64_t maxRowsPerCopy = postUbElements_ / (2 * rowWidth);
218 int64_t activeRows = partialRows;225 int64_t activeRows = partialRows;
219 while (activeRows > 1) {226 while (activeRows > 1) {
220 const int64_t rightRows = (activeRows + 1) / 2;227 const int64_t rightRows = (activeRows + 1) / 2;
@@ -254,8 +261,8 @@ __aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceRPartial(LocalTenso
254template <typename T>261template <typename T>
255__aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceR()262__aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceR()
256{263{
257- LocalTensor<float> inputLocal = inputBuf_.Get<float>(ONCE_HANDLE_NUM512 * ONCE_HANDLE_NUM64);264+ LocalTensor<float> inputLocal = inputLocal_;
258- constexpr int64_t rowWidth = 64;265+ const int64_t rowWidth = ONCE_HANDLE_NUM64;
259 const int64_t partialRows = mCoreNum_;266 const int64_t partialRows = mCoreNum_;
260 267 
261 for (int64_t nCoreIdx = 0; nCoreIdx < nCoreNum_; ++nCoreIdx) {268 for (int64_t nCoreIdx = 0; nCoreIdx < nCoreNum_; ++nCoreIdx) {
@@ -275,7 +282,7 @@ __aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceR()
275 SetFlag<HardEvent::MTE2_S>(eventIdMte2ToS);282 SetFlag<HardEvent::MTE2_S>(eventIdMte2ToS);
276 WaitFlag<HardEvent::MTE2_S>(eventIdMte2ToS);283 WaitFlag<HardEvent::MTE2_S>(eventIdMte2ToS);
277 for (int64_t i = 0; i < rowCount; ++i) {284 for (int64_t i = 0; i < rowCount; ++i) {
278- StoreFloatAligned(gmSumGradR_, rowOffset + i, inputLocal.GetValue(i), outputScratchBuf_.Get<float>());285+ StoreFloatAligned(gmSumGradR_, rowOffset + i, inputLocal.GetValue(i), outputScratchLocal_);
279 }286 }
280 event_t eventIdSToMte2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE2));287 event_t eventIdSToMte2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE2));
281 SetFlag<HardEvent::S_MTE2>(eventIdSToMte2);288 SetFlag<HardEvent::S_MTE2>(eventIdSToMte2);
@@ -359,7 +366,7 @@ __aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceRCFinal(LocalTensor
359 event_t eventIdVToS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S));366 event_t eventIdVToS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S));
360 SetFlag<HardEvent::V_S>(eventIdVToS);367 SetFlag<HardEvent::V_S>(eventIdVToS);
361 WaitFlag<HardEvent::V_S>(eventIdVToS);368 WaitFlag<HardEvent::V_S>(eventIdVToS);
362- StoreFloatAligned(gmSumGradRC_, batchIdx_, inputLocal.GetValue(0), outputScratchBuf_.Get<float>());369+ StoreFloatAligned(gmSumGradRC_, batchIdx_, inputLocal.GetValue(0), outputScratchLocal_);
363 event_t eventIdSToMte2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE2));370 event_t eventIdSToMte2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE2));
364 SetFlag<HardEvent::S_MTE2>(eventIdSToMte2);371 SetFlag<HardEvent::S_MTE2>(eventIdSToMte2);
365 WaitFlag<HardEvent::S_MTE2>(eventIdSToMte2);372 WaitFlag<HardEvent::S_MTE2>(eventIdSToMte2);
@@ -368,10 +375,10 @@ __aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceRCFinal(LocalTensor
368template <typename T>375template <typename T>
369__aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceRC()376__aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceRC()
370{377{
371- LocalTensor<float> inputLocal = inputBuf_.Get<float>(ONCE_HANDLE_NUM512 * ONCE_HANDLE_NUM64);378+ LocalTensor<float> inputLocal = inputLocal_;
372 constexpr uint64_t kScalarSlotSize = 8;379 constexpr uint64_t kScalarSlotSize = 8;
373- constexpr uint64_t kMaxPairCount = 512;380+ const uint64_t kMaxPairCount = static_cast<uint64_t>(postUbElements_) / (6 * 8);
374- constexpr uint64_t kMaxPairElements = kMaxPairCount * kScalarSlotSize;381+ const uint64_t kMaxPairElements = kMaxPairCount * kScalarSlotSize;
375 LocalTensor<float> high = inputLocal;382 LocalTensor<float> high = inputLocal;
376 LocalTensor<float> highRight = inputLocal[kMaxPairElements];383 LocalTensor<float> highRight = inputLocal[kMaxPairElements];
377 LocalTensor<float> low = inputLocal[2 * kMaxPairElements];384 LocalTensor<float> low = inputLocal[2 * kMaxPairElements];
@@ -407,14 +414,67 @@ __aicore__ inline void ApplyCamePart1Post<T>::CopyWorkspaceCChunk(LocalTensor<fl
407 }414 }
408}415}
409 416 
417+template <typename T>
418+__aicore__ inline void ApplyCamePart1Post<T>::ReduceNarrowC(uint64_t gmOffsets, uint64_t outputCount, uint64_t rowCount,
419+ uint64_t rowsPerChunk)
420+{
421+ const uint64_t width = static_cast<uint64_t>(ONCE_HANDLE_NUM64);
422+ LocalTensor<float> input = inputLocal_;
423+ LocalTensor<float> high = input[rowsPerChunk * width];
424+ LocalTensor<float> low = input[(rowsPerChunk + 1) * width];
425+ Duplicate(high, 0.0f, width);
426+ Duplicate(low, 0.0f, width);
427+ for (uint64_t start = 0; start < rowCount; start += rowsPerChunk) {
428+ Duplicate(input, 0.0f, rowsPerChunk * width);
429+ event_t vToMte2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE2));
430+ SetFlag<HardEvent::V_MTE2>(vToMte2);
431+ WaitFlag<HardEvent::V_MTE2>(vToMte2);
432+ const uint64_t count = rowCount - start < rowsPerChunk ? rowCount - start : rowsPerChunk;
433+ CopyWorkspaceCChunk(input, start, count, gmOffsets, width);
434+ event_t mte2ToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V));
435+ SetFlag<HardEvent::MTE2_V>(mte2ToV);
436+ WaitFlag<HardEvent::MTE2_V>(mte2ToV);
437+ ReduceAdd(input, rowsPerChunk, width);
438+ event_t vToS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S));
439+ SetFlag<HardEvent::V_S>(vToS);
440+ WaitFlag<HardEvent::V_S>(vToS);
441+ for (uint64_t col = 0; col < outputCount; ++col) {
442+ float h = 0.0f;
443+ float l = 0.0f;
444+ CompensatedAddApplyCamePart1(high.GetValue(col), low.GetValue(col), input.GetValue(col), 0.0f, h, l);
445+ high.SetValue(col, h);
446+ low.SetValue(col, l);
447+ }
448+ event_t sToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V));
449+ SetFlag<HardEvent::S_V>(sToV);
450+ WaitFlag<HardEvent::S_V>(sToV);
451+ }
452+ Add(input, high, low, width);
453+ CopyOutCReduceResult(gmSumGradC_, input, batchIdx_ * M + gmOffsets, outputCount);
454+}
455+ 
410template <typename T>456template <typename T>
411__aicore__ inline void ApplyCamePart1Post<T>::Pre_Core_Compute(uint64_t gmOffsets, uint64_t cal_m, uint64_t base_m)457__aicore__ inline void ApplyCamePart1Post<T>::Pre_Core_Compute(uint64_t gmOffsets, uint64_t cal_m, uint64_t base_m)
412{458{
413 uint64_t total_n = (nCoreNum_ - 1) * nLoopNormCore_ + nLoopTailCore_;459 uint64_t total_n = (nCoreNum_ - 1) * nLoopNormCore_ + nLoopTailCore_;
460+ if (cal_m == static_cast<uint64_t>(ONCE_HANDLE_NUM64) && total_n > 512) {
461+ // Reserve two aligned rows for compensated accumulation in the same arena.
462+ const uint64_t width = static_cast<uint64_t>(ONCE_HANDLE_NUM64);
463+ const uint64_t availableRows = static_cast<uint64_t>(postUbElements_) / width;
464+ if (availableRows >= 4) {
465+ uint64_t rowsPerChunk = 1;
466+ while (rowsPerChunk < 256 && rowsPerChunk * 2 <= availableRows - 2) {
467+ rowsPerChunk *= 2;
468+ }
469+ const uint64_t outputCount = (M - gmOffsets < base_m) ? M - gmOffsets : base_m;
470+ ReduceNarrowC(gmOffsets, outputCount, total_n, rowsPerChunk);
471+ return;
472+ }
473+ }
atomgit-bot
atomgit-botatomgit-bot16 days ago
Outdated

🟡 Medium Priority

变更行(post.h:456-460)新增了整条补偿归约快路径并 return,但仓内没有任何用例能进入它。

可达条件推导:ProcessBatch 中 core_loop = ceil(totalColumnTileCount_/512)、pre_core_m = ceil(totalColumnTileCount_/core_loop)64、last_core_m = (totalColumnTileCount_ - pre_core_m(core_loop-1)/64)*64。只有当 totalColumnTileCount_ == 1(即 M ≤ 64)时 core_loop==1、last_core_m==64,cal_m 才可能等于 ONCE_HANDLE_NUM64;因此新路径的触发域是「M ≤ 64 且 total_n > 512(n 向行块数 > 512)」。

覆盖现状:tests/st/arch35/ttk_apply_came_part1_bf16.csv 只有一条数据用例 (65,67),M=67 → totalColumnTileCount_=2 → cal_m=128,永远不会走 ReduceNarrowC;UT 也仅覆盖 (2,2)/(4,8)/(65,67) 等小微 shape。同时新增的 tests/assets/golden_fp64.py 未被任何 csv/配置引用。

失败模式:新代码里有多个极易写错且只影响该触发域的点——high/low 槽位取址 input[25664]/input[25764]、尾块(count<rowsPerChunk)依赖 Duplicate 补零才成立、结束后一次性 Add(high,low) 再单次原子回写。上述任一处出错(例如槽位重叠、尾块未补零导致把残留数据累加进去、final Add 的 count 用错),在 N 很大且 M≤64 的输入上会给 sum_grad_c 造成静默精度偏差,而 CI 完全无法发现。

建议:为新增分支补充能真正进入 ReduceNarrowC 的用例(M ≤ 64 且 total_n > 512),同时覆盖 rowCount 非 rowsPerChunk 整数倍(尾块补零)与 outputCount < 64(M 非 64 对齐)两种边界,并用 FP64 参考校验精度。

likedislike
Incorrect?
zsq_888
zsq_888
16 days ago Comments:
likedislike
414 uint64_t pre_loop_n = 1;474 uint64_t pre_loop_n = 1;
415 while (pre_loop_n < total_n) {475 while (pre_loop_n < total_n) {
416 pre_loop_n = pre_loop_n << 1;476 pre_loop_n = pre_loop_n << 1;
417- if (pre_loop_n * cal_m > ONCE_HANDLE_NUM512 * ONCE_HANDLE_NUM64) {477+ if (pre_loop_n * cal_m > static_cast<uint64_t>(postUbElements_)) {
418 pre_loop_n = pre_loop_n >> 1;478 pre_loop_n = pre_loop_n >> 1;
419 break;479 break;
420 }480 }
@@ -424,7 +484,7 @@ __aicore__ inline void ApplyCamePart1Post<T>::Pre_Core_Compute(uint64_t gmOffset
424 }484 }
425 uint64_t loop_time = (total_n + pre_loop_n - 1) / pre_loop_n;485 uint64_t loop_time = (total_n + pre_loop_n - 1) / pre_loop_n;
426 uint64_t last_loop_n = total_n - (loop_time - 1) * pre_loop_n;486 uint64_t last_loop_n = total_n - (loop_time - 1) * pre_loop_n;
427- LocalTensor<float> inputLocal = inputBuf_.Get<float>(ONCE_HANDLE_NUM512 * ONCE_HANDLE_NUM64);487+ LocalTensor<float> inputLocal = inputLocal_;
428 event_t eventIdSToMte2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE2));488 event_t eventIdSToMte2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE2));
429 SetFlag<HardEvent::S_MTE2>(eventIdSToMte2);489 SetFlag<HardEvent::S_MTE2>(eventIdSToMte2);
430 WaitFlag<HardEvent::S_MTE2>(eventIdSToMte2);490 WaitFlag<HardEvent::S_MTE2>(eventIdSToMte2);
@@ -29,6 +29,10 @@ struct ApplyCamePart1TilingData {
29 int64_t mCoreNum = 0;29 int64_t mCoreNum = 0;
30 int64_t batchCoreNum = 1;30 int64_t batchCoreNum = 1;
31 int64_t directBatchMode = 0;31 int64_t directBatchMode = 0;
32+ int64_t tileSize = 64;
33+ int64_t ubArenaBytes = 131072;
34+ int64_t postUbElements = 32768;
35+ int64_t directUbElements = 8192;
32#define APPLY_CAME_TILING_ACCESSOR(name) \36#define APPLY_CAME_TILING_ACCESSOR(name) \
33 int64_t get_##name() const { return name; } \37 int64_t get_##name() const { return name; } \
34 void set_##name(int64_t value) { name = value; }38 void set_##name(int64_t value) { name = value; }
@@ -48,6 +52,10 @@ struct ApplyCamePart1TilingData {
48 APPLY_CAME_TILING_ACCESSOR(mCoreNum)52 APPLY_CAME_TILING_ACCESSOR(mCoreNum)
49 APPLY_CAME_TILING_ACCESSOR(batchCoreNum)53 APPLY_CAME_TILING_ACCESSOR(batchCoreNum)
50 APPLY_CAME_TILING_ACCESSOR(directBatchMode)54 APPLY_CAME_TILING_ACCESSOR(directBatchMode)
55+ APPLY_CAME_TILING_ACCESSOR(tileSize)
56+ APPLY_CAME_TILING_ACCESSOR(ubArenaBytes)
57+ APPLY_CAME_TILING_ACCESSOR(postUbElements)
58+ APPLY_CAME_TILING_ACCESSOR(directUbElements)
51#undef APPLY_CAME_TILING_ACCESSOR59#undef APPLY_CAME_TILING_ACCESSOR
52};60};
53#endif // OPS_NN_APPLY_CAME_PART1_TILING_DATA_H61#endif // OPS_NN_APPLY_CAME_PART1_TILING_DATA_H
@@ -8,7 +8,6 @@
8# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.8# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
9# See LICENSE in the root of the software repository for the full text of the License.9# See LICENSE in the root of the software repository for the full text of the License.
10 10 
11-import ml_dtypes
12import numpy as np11import numpy as np
13import torch12import torch
14 13 
@@ -51,6 +50,8 @@ def _to_numpy(outputs):
51 for output in outputs:50 for output in outputs:
52 result = output.detach().cpu().contiguous()51 result = output.detach().cpu().contiguous()
53 if result.dtype == torch.bfloat16:52 if result.dtype == torch.bfloat16:
53+ import ml_dtypes
54+ 
54 bits = result.view(torch.uint16).numpy().copy()55 bits = result.view(torch.uint16).numpy().copy()
55 arrays.append(bits.view(ml_dtypes.bfloat16))56 arrays.append(bits.view(ml_dtypes.bfloat16))
56 else:57 else:
@@ -59,8 +60,17 @@ def _to_numpy(outputs):
59 60 
60 61 
61def apply_came_part1_golden(grad, eps, **kwargs):62def apply_came_part1_golden(grad, eps, **kwargs):
62- """Use FP32 accumulation for low-precision input."""63+ """Compute the CPU reference entirely in FP64 and retain FP64 outputs."""
63- return _to_numpy(_compute(grad, eps, match_kernel=True))64+ grad64 = _to_torch(grad).to(dtype=torch.float64, device="cpu")
65+ eps64 = _to_torch(eps).reshape(-1)[0].to(dtype=torch.float64, device="cpu")
66+ values64 = torch.square(grad64) + eps64
67+ return _to_numpy(
68+ [
69+ torch.sum(values64, dim=-1),
70+ torch.sum(values64, dim=-2),
71+ torch.sum(values64, dim=(-2, -1)),
72+ ]
73+ )
64 74 
65 75 
66class _ApplyCamePart1Compose:76class _ApplyCamePart1Compose:
@@ -1,2 +1,8 @@
1testcase_name,op_name,input_dtypes,input_formats,input_shapes,output_shapes,output_dtypes,output_formats,attributes,input_data_ranges,absolute_precision1testcase_name,op_name,input_dtypes,input_formats,input_shapes,output_shapes,output_dtypes,output_formats,attributes,input_data_ranges,absolute_precision
2apply_came_part1_bf16_tail,apply_came_part1,"('bfloat16','float32')","('ND','ND')","((65,67),(1,))","((65,),(67,),())","('float32','float32','float32')","('ND','ND','ND')",{},"((-1,1),(0.001,0.001))",0.0012apply_came_part1_bf16_tail,apply_came_part1,"('bfloat16','float32')","('ND','ND')","((65,67),(1,))","((65,),(67,),())","('float32','float32','float32')","('ND','ND','ND')",{},"((-1,1),(0.001,0.001))",0.001
3+apply_came_part1_float16_256_256,apply_came_part1,"('float16','float32')","('ND','ND')","((256,256),(1,))","((256,),(256,),())","('float32','float32','float32')","('ND','ND','ND')",{},"((-1,1),(0.001,0.001))",0.001
4+apply_came_part1_float16_257_259,apply_came_part1,"('float16','float32')","('ND','ND')","((257,259),(1,))","((257,),(259,),())","('float32','float32','float32')","('ND','ND','ND')",{},"((-1,1),(0.001,0.001))",0.001
5+apply_came_part1_bfloat16_256_256,apply_came_part1,"('bfloat16','float32')","('ND','ND')","((256,256),(1,))","((256,),(256,),())","('float32','float32','float32')","('ND','ND','ND')",{},"((-1,1),(0.001,0.001))",0.001
6+apply_came_part1_bfloat16_257_259,apply_came_part1,"('bfloat16','float32')","('ND','ND')","((257,259),(1,))","((257,),(259,),())","('float32','float32','float32')","('ND','ND','ND')",{},"((-1,1),(0.001,0.001))",0.001
7+apply_came_part1_float32_256_256,apply_came_part1,"('float32','float32')","('ND','ND')","((256,256),(1,))","((256,),(256,),())","('float32','float32','float32')","('ND','ND','ND')",{},"((-1,1),(0.001,0.001))",0.001
8+apply_came_part1_float32_257_259,apply_came_part1,"('float32','float32')","('ND','ND')","((257,259),(1,))","((257,),(259,),())","('float32','float32','float32')","('ND','ND','ND')",{},"((-1,1),(0.001,0.001))",0.001
@@ -10,11 +10,13 @@
10 10 
11#include <cstring>11#include <cstring>
12#include <limits>12#include <limits>
13+#include <utility>
13#include <vector>14#include <vector>
14#include <gtest/gtest.h>15#include <gtest/gtest.h>
15#include "tiling_case_executor.h"16#include "tiling_case_executor.h"
16#include "tiling_context_faker.h"17#include "tiling_context_faker.h"
17#include "../../../../op_host/arch35/apply_came_part1_tiling_arch35.h"18#include "../../../../op_host/arch35/apply_came_part1_tiling_arch35.h"
19+#include "../../../../op_host/arch35/apply_came_part1_ub_plan.h"
18 20 
19namespace ApplyCamePart1TilingUT {21namespace ApplyCamePart1TilingUT {
20namespace {22namespace {
@@ -89,6 +91,40 @@ bool TryTiling(const std::vector<int64_t>& gradShape, ge::DataType gradType)
89}91}
90} // namespace92} // namespace
91 93 
94+TEST(ApplyCamePart1Tiling, UbPlannerSelectsTileAndBoundsLiveBuffers)
95+{
96+ for (const uint64_t dtypeBytes : {2UL, 4UL}) {
97+ for (const auto& config : {std::pair<uint64_t, uint64_t>{kUbSize, 128}, {64UL * 1024UL, 64}}) {
98+ SCOPED_TRACE(::testing::Message() << "dtypeBytes=" << dtypeBytes << " ub=" << config.first);
99+ const auto plan = apply_came_ub::MakePlan(config.first, dtypeBytes, kCoreNum);
100+ ASSERT_EQ(plan.tile, config.second);
101+ EXPECT_GT(plan.directElements, 0U);
102+ EXPECT_LE(apply_came_ub::kReserveBytes + plan.persistentBytes + plan.arenaBytes, config.first);
103+ EXPECT_LE(plan.producerPeakBytes, plan.arenaBytes);
104+ EXPECT_LE(plan.postElements * sizeof(float) + 64, plan.arenaBytes);
105+ EXPECT_EQ(plan.postElements % plan.tile, 0U);
106+ EXPECT_GE(plan.postElements, 2 * plan.tile);
107+ // RC post uses six arrays of 8-float pairs and a separate 64-byte output slot.
108+ EXPECT_LE((plan.postElements / (6 * 8)) * 6 * 8 * sizeof(float) + 64, plan.arenaBytes);
109+ }
110+ }
111+}
112+ 
113+TEST(ApplyCamePart1Tiling, LargeTilesUsePlannedArenaForEachDtype)
114+{
115+ for (const auto dtype : {ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT}) {
116+ SCOPED_TRACE(static_cast<int>(dtype));
117+ const auto result = RunTiling({257, 259}, dtype);
118+ const uint64_t dtypeBytes = dtype == ge::DT_FLOAT ? 4 : 2;
119+ const auto plan = apply_came_ub::MakePlan(kUbSize, dtypeBytes, kCoreNum);
120+ ASSERT_EQ(plan.tile, 128U);
121+ EXPECT_EQ(result.data.tileSize, plan.tile);
122+ EXPECT_EQ(result.data.ubArenaBytes, plan.arenaBytes);
123+ EXPECT_EQ(result.data.postUbElements, plan.postElements);
124+ EXPECT_EQ(result.data.directBatchMode, 0);
125+ }
126+}
127+ 
92TEST(ApplyCamePart1Tiling, Fp16UsesGeneralBatchDirectPath)128TEST(ApplyCamePart1Tiling, Fp16UsesGeneralBatchDirectPath)
93{129{
94 const auto result = RunTiling({12562512, 2, 2}, ge::DT_FLOAT16);130 const auto result = RunTiling({12562512, 2, 2}, ge::DT_FLOAT16);