| @@ -22,6 +22,7 @@ | |||
| 22 | #include "register/op_impl_registry.h" | 22 | #include "register/op_impl_registry.h" |
| 23 | #include "tiling/platform/platform_ascendc.h" | 23 | #include "tiling/platform/platform_ascendc.h" |
| 24 | #include "apply_came_part1_tiling_arch35.h" | 24 | #include "apply_came_part1_tiling_arch35.h" |
| 25 | +#include "apply_came_part1_ub_plan.h" | ||
| 25 | 26 | ||
| 26 | namespace optiling { | 27 | namespace optiling { |
| 27 | 28 | ||
| @@ -129,15 +130,15 @@ static ge::graphStatus ApplyCamePart1SetTilingData(gert::TilingContext* context, | |||
| 129 | static inline ge::graphStatus CalcNdimTiling(const gert::TilingContext* context, ApplyCamePart1TilingData& tilingData) | 130 | static inline ge::graphStatus CalcNdimTiling(const gert::TilingContext* context, ApplyCamePart1TilingData& tilingData) |
| 130 | { | 131 | { |
| 131 | OP_LOGD(context->GetNodeName(), "TilingApplyCamePart1 Enter CalcNdimTiling."); | 132 | OP_LOGD(context->GetNodeName(), "TilingApplyCamePart1 Enter CalcNdimTiling."); |
| 132 | - int64_t nNormalCoreNum = std::max(CeilMul(tilingData.get_N(), N_NUM_PER_CORE, tilingData.get_totalCoreNum()), | 133 | + int64_t nNormalCoreNum = std::max( |
| 133 | - static_cast<int64_t>(1)); | 134 | + CeilMul(tilingData.get_N(), tilingData.get_tileSize(), tilingData.get_totalCoreNum()), static_cast<int64_t>(1)); |
| 134 | int64_t nTailCoreNum = tilingData.get_N() % nNormalCoreNum; | 135 | int64_t nTailCoreNum = tilingData.get_N() % nNormalCoreNum; |
| 135 | if (nTailCoreNum == 0) { | 136 | if (nTailCoreNum == 0) { |
| 136 | nTailCoreNum = nNormalCoreNum; | 137 | nTailCoreNum = nNormalCoreNum; |
| 137 | } | 138 | } |
| 138 | int64_t nCoreNum = CeilDiv(tilingData.get_N(), nNormalCoreNum); | 139 | int64_t nCoreNum = CeilDiv(tilingData.get_N(), nNormalCoreNum); |
| 139 | - int64_t nLoopNormCore = CeilDiv(nNormalCoreNum, N_NUM_PER_CORE); | 140 | + int64_t nLoopNormCore = CeilDiv(nNormalCoreNum, tilingData.get_tileSize()); |
| 140 | - int64_t nLoopTailCore = CeilDiv(nTailCoreNum, N_NUM_PER_CORE); | 141 | + int64_t nLoopTailCore = CeilDiv(nTailCoreNum, tilingData.get_tileSize()); |
| 141 | tilingData.set_nNormalCoreNum(nNormalCoreNum); | 142 | tilingData.set_nNormalCoreNum(nNormalCoreNum); |
| 142 | tilingData.set_nTailCoreNum(nTailCoreNum); | 143 | tilingData.set_nTailCoreNum(nTailCoreNum); |
| 143 | tilingData.set_nCoreNum(nCoreNum); | 144 | tilingData.set_nCoreNum(nCoreNum); |
| @@ -173,15 +174,15 @@ static inline ge::graphStatus CalcMdimTiling(const gert::TilingContext* context, | |||
| 173 | VECTOR_INNER_ERR_REPORT_TILIING(context->GetNodeName(), "TilingApplyCamePart1 nCoreNum is invalid."), | 174 | VECTOR_INNER_ERR_REPORT_TILIING(context->GetNodeName(), "TilingApplyCamePart1 nCoreNum is invalid."), |
| 174 | return ge::GRAPH_FAILED); | 175 | return ge::GRAPH_FAILED); |
| 175 | 176 | ||
| 176 | - const int64_t columnTileCount = CeilDiv(tilingData.get_M(), M_NUM_PER_CORE); | 177 | + const int64_t columnTileCount = CeilDiv(tilingData.get_M(), tilingData.get_tileSize()); |
| 177 | const int64_t availableMCoreNum = std::max<int64_t>(1, tilingData.get_totalCoreNum() / nCoreNum); | 178 | const int64_t availableMCoreNum = std::max<int64_t>(1, tilingData.get_totalCoreNum() / nCoreNum); |
| 178 | int64_t mCoreNum = std::min(columnTileCount, availableMCoreNum); | 179 | int64_t mCoreNum = std::min(columnTileCount, availableMCoreNum); |
| 179 | const int64_t columnsPerCore = CeilDiv(tilingData.get_M(), mCoreNum); | 180 | const int64_t columnsPerCore = CeilDiv(tilingData.get_M(), mCoreNum); |
| 180 | - const int64_t mNormalCoreNum = CeilAlign(columnsPerCore, M_NUM_PER_CORE); | 181 | + const int64_t mNormalCoreNum = CeilAlign(columnsPerCore, tilingData.get_tileSize()); |
| 181 | mCoreNum = CeilDiv(tilingData.get_M(), mNormalCoreNum); | 182 | mCoreNum = CeilDiv(tilingData.get_M(), mNormalCoreNum); |
| 182 | const int64_t mTailCoreNum = tilingData.get_M() - mNormalCoreNum * (mCoreNum - 1); | 183 | const int64_t mTailCoreNum = tilingData.get_M() - mNormalCoreNum * (mCoreNum - 1); |
| 183 | 184 | ||
| 184 | - tilingData.set_mLoopNumCore(CeilDiv(mNormalCoreNum, M_NUM_PER_CORE)); | 185 | + tilingData.set_mLoopNumCore(CeilDiv(mNormalCoreNum, tilingData.get_tileSize())); |
| 185 | tilingData.set_mNormalCoreNum(mNormalCoreNum); | 186 | tilingData.set_mNormalCoreNum(mNormalCoreNum); |
| 186 | tilingData.set_mTailCoreNum(mTailCoreNum); | 187 | tilingData.set_mTailCoreNum(mTailCoreNum); |
| 187 | tilingData.set_mCoreNum(mCoreNum); | 188 | tilingData.set_mCoreNum(mCoreNum); |
| @@ -232,6 +233,29 @@ static ge::graphStatus CheckParamsShape(const gert::TilingContext* context) | |||
| 232 | return ge::GRAPH_SUCCESS; | 233 | return ge::GRAPH_SUCCESS; |
| 233 | } | 234 | } |
| 234 | 235 | ||
| 236 | +// Size buffers from the maximum concurrent live set in each phase. | ||
| 237 | +static ge::graphStatus PlanUbBuffers(gert::TilingContext* context, ApplyCamePart1TilingData& data) | ||
| 238 | +{ | ||
| 239 | + auto platformInfo = context->GetPlatformInfo(); | ||
| 240 | + OPS_CHECK_NULL_WITH_CONTEXT(context, platformInfo); | ||
| 241 | + auto platform = platform_ascendc::PlatformAscendC(platformInfo); | ||
| 242 | + uint64_t ubBytes = 0; | ||
| 243 | + platform.GetCoreMemSize(platform_ascendc::CoreMemType::UB, ubBytes); | ||
| 244 | + const uint64_t elementBytes = context->GetInputDesc(0)->GetDataType() == ge::DT_FLOAT ? 4 : 2; | ||
| 245 | + const auto plan = apply_came_ub::MakePlan(ubBytes, elementBytes, data.get_totalCoreNum()); | ||
| 246 | + OP_TILING_CHECK(plan.tile == 0 || plan.directElements == 0, | ||
| 247 | + VECTOR_INNER_ERR_REPORT_TILIING(context, "UB cannot hold the peak live buffer set."), | ||
| 248 | + return ge::GRAPH_FAILED); | ||
| 249 | + data.set_tileSize(plan.tile); | ||
| 250 | + data.set_ubArenaBytes(plan.arenaBytes); | ||
| 251 | + data.set_postUbElements(plan.postElements); | ||
| 252 | + data.set_directUbElements(plan.directElements); | ||
| 253 | + OP_LOGI(context->GetNodeName(), | ||
| 254 | + "UB plan: total=%lu tile=%lu persistent=%lu producerPeak=%lu arena=%lu postElements=%lu", ubBytes, | ||
| 255 | + plan.tile, plan.persistentBytes, plan.producerPeakBytes, plan.arenaBytes, plan.postElements); | ||
| 256 | + return ge::GRAPH_SUCCESS; | ||
| 257 | +} | ||
| 258 | + | ||
| 235 | static bool ShouldUseDirectBatch(const ApplyCamePart1TilingData& tilingData) | 259 | static bool ShouldUseDirectBatch(const ApplyCamePart1TilingData& tilingData) |
| 236 | { | 260 | { |
| 237 | const int64_t n = tilingData.get_N(); | 261 | const int64_t n = tilingData.get_N(); |
| @@ -240,7 +264,7 @@ static bool ShouldUseDirectBatch(const ApplyCamePart1TilingData& tilingData) | |||
| 240 | return false; | 264 | return false; |
| 241 | } | 265 | } |
| 242 | const int64_t matrixElements = n * m; | 266 | const int64_t matrixElements = n * m; |
| 243 | - if (matrixElements > MAX_DIRECT_MATRIX_ELEMENTS) { | 267 | + if (matrixElements > tilingData.get_directUbElements()) { |
| 244 | return false; | 268 | return false; |
| 245 | } | 269 | } |
| 246 | const bool singleVectorTile = n <= N_NUM_PER_CORE && m <= M_NUM_PER_CORE; | 270 | const bool singleVectorTile = n <= N_NUM_PER_CORE && m <= M_NUM_PER_CORE; |
| @@ -287,7 +311,7 @@ static ge::graphStatus CalculateApplyCamePart1Workspace(gert::TilingContext* con | |||
| 287 | size_t nLoopCount = 0; | 311 | size_t nLoopCount = 0; |
| 288 | size_t rElements = 0; | 312 | size_t rElements = 0; |
| 289 | size_t rcElements = 0; | 313 | size_t rcElements = 0; |
| 290 | - const size_t totalColumnTileCount = static_cast<size_t>(CeilDiv(tilingData.get_M(), M_NUM_PER_CORE)); | 314 | + const size_t totalColumnTileCount = static_cast<size_t>(CeilDiv(tilingData.get_M(), tilingData.get_tileSize())); |
| 291 | OP_TILING_CHECK( | 315 | OP_TILING_CHECK( |
| 292 | !CheckedMultiply(nCoreNum - 1, nLoopNormCore, normalLoopCount) || | 316 | !CheckedMultiply(nCoreNum - 1, nLoopNormCore, normalLoopCount) || |
| 293 | !CheckedAdd(normalLoopCount, nLoopTailCore, nLoopCount) || | 317 | !CheckedAdd(normalLoopCount, nLoopTailCore, nLoopCount) || |
| @@ -308,8 +332,10 @@ static ge::graphStatus CalculateApplyCamePart1Workspace(gert::TilingContext* con | |||
| 308 | size_t cWorkspaceBytes = 0; | 332 | size_t cWorkspaceBytes = 0; |
| 309 | OP_TILING_CHECK( | 333 | OP_TILING_CHECK( |
| 310 | !CheckedMultiply(rcElements, rcSlotSize, rcStorageElements) || | 334 | !CheckedMultiply(rcElements, rcSlotSize, rcStorageElements) || |
| 311 | - !CheckedMultiply(rElements, static_cast<size_t>(N_NUM_PER_CORE * sizeof(float)), rWorkspaceBytes) || | 335 | + !CheckedMultiply(rElements, static_cast<size_t>(tilingData.get_tileSize() * sizeof(float)), |
| 312 | - !CheckedMultiply(rcElements, static_cast<size_t>(M_NUM_PER_CORE * sizeof(float)), cWorkspaceBytes), | 336 | + rWorkspaceBytes) || |
| 337 | + !CheckedMultiply(rcElements, static_cast<size_t>(tilingData.get_tileSize() * sizeof(float)), | ||
| 338 | + cWorkspaceBytes), | ||
| 313 | VECTOR_INNER_ERR_REPORT_TILIING(context->GetNodeName(), "ApplyCamePart1 workspace byte size overflows."), | 339 | VECTOR_INNER_ERR_REPORT_TILIING(context->GetNodeName(), "ApplyCamePart1 workspace byte size overflows."), |
| 314 | return ge::GRAPH_FAILED); | 340 | return ge::GRAPH_FAILED); |
| 315 | 341 | ||
| @@ -442,6 +468,9 @@ static ge::graphStatus InitializeApplyCamePart1Tiling(gert::TilingContext* conte | |||
| 442 | 468 | ||
| 443 | static ge::graphStatus CalculateApplyCamePart1Tiling(gert::TilingContext* context, ApplyCamePart1TilingData& tilingData) | 469 | static ge::graphStatus CalculateApplyCamePart1Tiling(gert::TilingContext* context, ApplyCamePart1TilingData& tilingData) |
| 444 | { | 470 | { |
| 471 | + if (PlanUbBuffers(context, tilingData) != ge::GRAPH_SUCCESS) { | ||
| 472 | + return ge::GRAPH_FAILED; | ||
| 473 | + } | ||
| 445 | if (ShouldUseDirectBatch(tilingData)) { | 474 | if (ShouldUseDirectBatch(tilingData)) { |
| 446 | tilingData.set_directBatchMode(1); | 475 | tilingData.set_directBatchMode(1); |
| 447 | tilingData.set_nNormalCoreNum(tilingData.get_N()); | 476 | tilingData.set_nNormalCoreNum(tilingData.get_N()); |
| @@ -0,0 +1,63 @@ | |||
| 1 | +/** | ||
| 2 | + * Copyright (c) 2026 Huawei Technologies Co., Ltd. | ||
| 3 | + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of | ||
| 4 | + * CANN Open Software License Agreement Version 2.0 (the "License"). | ||
| 5 | + * Please refer to the License for details. You may not use this file except in compliance with the License. | ||
| 6 | + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, | ||
| 7 | + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | ||
| 8 | + * See LICENSE in the root of the software repository for the full text of the License. | ||
| 9 | + */ | ||
| 10 | +#ifndef APPLY_CAME_PART1_UB_PLAN_H | ||
| 11 | +#define APPLY_CAME_PART1_UB_PLAN_H | ||
| 12 | +#include <cstdint> | ||
| 13 | +namespace apply_came_ub { | ||
| 14 | +struct Plan { | ||
| 15 | + uint64_t tile = 0; | ||
| 16 | + uint64_t persistentBytes = 0; | ||
| 17 | + uint64_t arenaBytes = 0; | ||
| 18 | + uint64_t producerPeakBytes = 0; | ||
| 19 | + uint64_t postElements = 0; | ||
| 20 | + uint64_t directElements = 0; | ||
| 21 | +}; | ||
| 22 | +constexpr uint64_t kReserveBytes = 8192; | ||
| 23 | +constexpr uint64_t AlignDown(uint64_t n, uint64_t a) { return n / a * a; } | ||
| 24 | +// Lifetimes in the reusable arena: | ||
| 25 | +// elementwise: values[t*t], rowAccumulator[2*t], cast[t*t] | ||
| 26 | +// R/C/RC: values[t*t], rowAccumulator[2*t], scratch[12*t] | ||
| 27 | +// post: six FP32 pair-reduction nodes plus one 64B scalar-output slot. | ||
| 28 | +// Input/output queues remain physically allocated throughout all phases. | ||
| 29 | +inline Plan MakePlan(uint64_t ub, uint64_t dtypeBytes, uint64_t cores) | ||
| 30 | +{ | ||
| 31 | + Plan p; | ||
| 32 | + if ((dtypeBytes != 2 && dtypeBytes != 4) || cores == 0 || cores > 4096 || ub <= kReserveBytes) | ||
| 33 | + return p; | ||
| 34 | + // Power-of-two tiles preserve alignment and the vector reduction contract. | ||
| 35 | + for (uint64_t t = 64; t <= 128; t *= 2) { | ||
| 36 | + const uint64_t fixed = t * t * dtypeBytes + 32 + 2 * t * 4 + 16 * 4 + cores * 32; | ||
| 37 | + const uint64_t work = t * t > 12 * t ? t * t : 12 * t; | ||
| 38 | + const uint64_t producerPeak = (t * t + 2 * t + work) * 4; | ||
| 39 | + if (ub <= kReserveBytes + fixed) | ||
| 40 | + continue; | ||
| 41 | + const uint64_t arena = AlignDown(ub - kReserveBytes - fixed, 256); | ||
| 42 | + if (arena < producerPeak || arena <= 64) | ||
| 43 | + continue; | ||
| 44 | + const uint64_t post = AlignDown((arena - 64) / 4, t); | ||
| 45 | + if (post < 2 * t || post < 6 * 8) | ||
| 46 | + continue; | ||
| 47 | + p.tile = t; | ||
| 48 | + p.persistentBytes = fixed; | ||
| 49 | + p.arenaBytes = arena; | ||
| 50 | + p.producerPeakBytes = producerPeak; | ||
| 51 | + p.postElements = post; | ||
| 52 | + } | ||
| 53 | + // Direct: cast and output have disjoint lifetimes and share one slot. | ||
| 54 | + // grad(T), values(FP32), cast/output(FP32), plus fixed queue/scratch slots. | ||
| 55 | + constexpr uint64_t fixedDirect = 32 + 2 * 64 * 4 + 12 * 64 * 4 + (2 * 32 + 3 * 8) * 4; | ||
| 56 | + if (ub > kReserveBytes + fixedDirect) { | ||
| 57 | + const uint64_t count = AlignDown((ub - kReserveBytes - fixedDirect) / (dtypeBytes + 8), 64); | ||
| 58 | + p.directElements = count < 8192 ? count : 8192; | ||
| 59 | + } | ||
| 60 | + return p; | ||
| 61 | +} | ||
| 62 | +} // namespace apply_came_ub | ||
| 63 | +#endif | ||
| @@ -49,7 +49,8 @@ __global__ __aicore__ void apply_came_part1(GM_ADDR grad, GM_ADDR eps, GM_ADDR s | |||
| 49 | 49 | ||
| 50 | if (tilingData.directBatchMode != 0) { | 50 | if (tilingData.directBatchMode != 0) { |
| 51 | ApplyCamePart1::ApplyCamePart1Batch<D_T> batchOp; | 51 | ApplyCamePart1::ApplyCamePart1Batch<D_T> batchOp; |
| 52 | - batchOp.Init(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, tilingData.N, tilingData.M); | 52 | + batchOp.Init(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, tilingData.N, tilingData.M, |
| 53 | + tilingData.directUbElements); | ||
| 53 | batchOp.Process(tilingData.batchCount, tilingData.usedCoreNum); | 54 | batchOp.Process(tilingData.batchCount, tilingData.usedCoreNum); |
| 54 | return; | 55 | return; |
| 55 | } | 56 | } |
| @@ -58,7 +59,7 @@ __global__ __aicore__ void apply_came_part1(GM_ADDR grad, GM_ADDR eps, GM_ADDR s | |||
| 58 | ApplyCamePart1::ApplyCamePart1FP32<float> op; | 59 | ApplyCamePart1::ApplyCamePart1FP32<float> op; |
| 59 | op.InitOnce(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, userWS, &tilingData); | 60 | op.InitOnce(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, userWS, &tilingData); |
| 60 | ApplyCamePart1::ApplyCamePart1Post<float> opPost; | 61 | ApplyCamePart1::ApplyCamePart1Post<float> opPost; |
| 61 | - opPost.Init(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, userWS, &tilingData); | 62 | + opPost.Init(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, userWS, &tilingData, op.GetUbArena()); |
| 62 | // Clear outputs and workspace before concurrent atomic updates. | 63 | // Clear outputs and workspace before concurrent atomic updates. |
| 63 | op.ZeroWorkspace(userWS); | 64 | op.ZeroWorkspace(userWS); |
| 64 | op.ZeroOutputs(sum_grad_r, sum_grad_c, sum_grad_rc, tilingData.batchCount); | 65 | op.ZeroOutputs(sum_grad_r, sum_grad_c, sum_grad_rc, tilingData.batchCount); |
| @@ -81,7 +82,7 @@ __global__ __aicore__ void apply_came_part1(GM_ADDR grad, GM_ADDR eps, GM_ADDR s | |||
| 81 | ApplyCamePart1::ApplyCamePart1FP16<D_T> op; | 82 | ApplyCamePart1::ApplyCamePart1FP16<D_T> op; |
| 82 | op.InitOnce(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, userWS, &tilingData); | 83 | op.InitOnce(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, userWS, &tilingData); |
| 83 | ApplyCamePart1::ApplyCamePart1Post<float> opPost; | 84 | ApplyCamePart1::ApplyCamePart1Post<float> opPost; |
| 84 | - opPost.Init(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, userWS, &tilingData); | 85 | + opPost.Init(grad, eps, sum_grad_r, sum_grad_c, sum_grad_rc, userWS, &tilingData, op.GetUbArena()); |
| 85 | op.ZeroWorkspace(userWS); | 86 | op.ZeroWorkspace(userWS); |
| 86 | op.ZeroOutputs(sum_grad_r, sum_grad_c, sum_grad_rc, tilingData.batchCount); | 87 | op.ZeroOutputs(sum_grad_r, sum_grad_c, sum_grad_rc, tilingData.batchCount); |
| 87 | SyncAll(); | 88 | SyncAll(); |
| @@ -21,8 +21,10 @@ template <typename T> | |||
| 21 | class ApplyCamePart1Batch { | 21 | class ApplyCamePart1Batch { |
| 22 | public: | 22 | public: |
| 23 | __aicore__ inline void Init(GM_ADDR grad, GM_ADDR eps, GM_ADDR sumGradR, GM_ADDR sumGradC, GM_ADDR sumGradRC, | 23 | __aicore__ inline void Init(GM_ADDR grad, GM_ADDR eps, GM_ADDR sumGradR, GM_ADDR sumGradC, GM_ADDR sumGradRC, |
| 24 | - int64_t n, int64_t m) | 24 | + int64_t n, int64_t m, int64_t ubElements) |
| 25 | { | 25 | { |
| 26 | + maxInputElements_ = ubElements; | ||
| 27 | + maxOutputElements_ = ubElements + 2 * kMaxTileBatches + 3 * kOutputAlignmentElements; | ||
| 26 | n_ = n; | 28 | n_ = n; |
| 27 | m_ = m; | 29 | m_ = m; |
| 28 | matrixElements_ = n * m; | 30 | matrixElements_ = n * m; |
| @@ -32,13 +34,13 @@ public: | |||
| 32 | sumGradCGm_.SetGlobalBuffer(reinterpret_cast<__gm__ float*>(sumGradC)); | 34 | sumGradCGm_.SetGlobalBuffer(reinterpret_cast<__gm__ float*>(sumGradC)); |
| 33 | sumGradRCGm_.SetGlobalBuffer(reinterpret_cast<__gm__ float*>(sumGradRC)); | 35 | sumGradRCGm_.SetGlobalBuffer(reinterpret_cast<__gm__ float*>(sumGradRC)); |
| 34 | 36 | ||
| 35 | - pipe_.InitBuffer(gradQueue_, 1, kMaxInputElements * sizeof(T)); | 37 | + pipe_.InitBuffer(gradQueue_, 1, maxInputElements_ * sizeof(T)); |
| 36 | pipe_.InitBuffer(epsQueue_, 1, kBlockFloats * sizeof(float)); | 38 | pipe_.InitBuffer(epsQueue_, 1, kBlockFloats * sizeof(float)); |
| 37 | pipe_.InitBuffer(sumGradRQueue_, 1, kVectorTileWidth * sizeof(float)); | 39 | pipe_.InitBuffer(sumGradRQueue_, 1, kVectorTileWidth * sizeof(float)); |
| 38 | pipe_.InitBuffer(sumGradCQueue_, 1, kVectorTileWidth * sizeof(float)); | 40 | pipe_.InitBuffer(sumGradCQueue_, 1, kVectorTileWidth * sizeof(float)); |
| 39 | - pipe_.InitBuffer(castBuf_, kMaxInputElements * sizeof(float)); | 41 | + // Cast dies before output production, so both use this physical slot. |
| 40 | - pipe_.InitBuffer(valueBuf_, kMaxInputElements * sizeof(float)); | 42 | + pipe_.InitBuffer(castOutputBuf_, maxOutputElements_ * sizeof(float)); |
| 41 | - pipe_.InitBuffer(outputBuf_, kMaxOutputElements * sizeof(float)); | 43 | + pipe_.InitBuffer(valueBuf_, maxInputElements_ * sizeof(float)); |
| 42 | pipe_.InitBuffer(reduceBuf_, kReduceScratchElements * sizeof(float)); | 44 | pipe_.InitBuffer(reduceBuf_, kReduceScratchElements * sizeof(float)); |
| 43 | } | 45 | } |
| 44 | 46 | ||
| @@ -92,9 +94,9 @@ private: | |||
| 92 | 94 | ||
| 93 | __aicore__ inline int64_t GetScalarTileBatchCount() const | 95 | __aicore__ inline int64_t GetScalarTileBatchCount() const |
| 94 | { | 96 | { |
| 95 | - int64_t byInput = kMaxInputElements / matrixElements_; | 97 | + int64_t byInput = maxInputElements_ / matrixElements_; |
| 96 | const int64_t outputElementsPerBatch = n_ + m_ + 1; | 98 | const int64_t outputElementsPerBatch = n_ + m_ + 1; |
| 97 | - int64_t byOutput = kMaxOutputElements / outputElementsPerBatch; | 99 | + int64_t byOutput = maxOutputElements_ / outputElementsPerBatch; |
| 98 | int64_t tileBatches = byInput < byOutput ? byInput : byOutput; | 100 | int64_t tileBatches = byInput < byOutput ? byInput : byOutput; |
| 99 | tileBatches = tileBatches < kMaxTileBatches ? tileBatches : kMaxTileBatches; | 101 | tileBatches = tileBatches < kMaxTileBatches ? tileBatches : kMaxTileBatches; |
| 100 | return tileBatches > 0 ? tileBatches : 1; | 102 | return tileBatches > 0 ? tileBatches : 1; |
| @@ -121,9 +123,9 @@ private: | |||
| 121 | gradQueue_.EnQue(gradLocal); | 123 | gradQueue_.EnQue(gradLocal); |
| 122 | gradLocal = gradQueue_.DeQue<T>(); | 124 | gradLocal = gradQueue_.DeQue<T>(); |
| 123 | 125 | ||
| 124 | - LocalTensor<float> castLocal = castBuf_.Get<float>(); | 126 | + LocalTensor<float> castLocal = castOutputBuf_.Get<float>(); |
| 125 | LocalTensor<float> valueLocal = valueBuf_.Get<float>(); | 127 | LocalTensor<float> valueLocal = valueBuf_.Get<float>(); |
| 126 | - LocalTensor<float> outputLocal = outputBuf_.Get<float>(); | 128 | + LocalTensor<float> outputLocal = castOutputBuf_.Get<float>(); |
| 127 | event_t eventMte2ToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V)); | 129 | event_t eventMte2ToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V)); |
| 128 | SetFlag<HardEvent::MTE2_V>(eventMte2ToV); | 130 | SetFlag<HardEvent::MTE2_V>(eventMte2ToV); |
| 129 | WaitFlag<HardEvent::MTE2_V>(eventMte2ToV); | 131 | WaitFlag<HardEvent::MTE2_V>(eventMte2ToV); |
| @@ -212,7 +214,7 @@ private: | |||
| 212 | LocalTensor<float>& sumGradRCLocal) | 214 | LocalTensor<float>& sumGradRCLocal) |
| 213 | { | 215 | { |
| 214 | const int64_t paddedElements = n_ * kVectorTileWidth; | 216 | const int64_t paddedElements = n_ * kVectorTileWidth; |
| 215 | - LocalTensor<float> castLocal = castBuf_.Get<float>(); | 217 | + LocalTensor<float> castLocal = castOutputBuf_.Get<float>(); |
| 216 | LocalTensor<float> valueLocal = valueBuf_.Get<float>(); | 218 | LocalTensor<float> valueLocal = valueBuf_.Get<float>(); |
| 217 | LocalTensor<float> reductionScratch = reduceBuf_.Get<float>(); | 219 | LocalTensor<float> reductionScratch = reduceBuf_.Get<float>(); |
| 218 | event_t eventMte2ToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V)); | 220 | event_t eventMte2ToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V)); |
| @@ -236,8 +238,14 @@ private: | |||
| 236 | ComputeCApplyCamePart1(sumGradCQueue_, n_, valueLocal, 1, 1, kVectorTileWidth, m_, 0); | 238 | ComputeCApplyCamePart1(sumGradCQueue_, n_, valueLocal, 1, 1, kVectorTileWidth, m_, 0); |
| 237 | sumGradRLocal = sumGradRQueue_.DeQue<float>(); | 239 | sumGradRLocal = sumGradRQueue_.DeQue<float>(); |
| 238 | sumGradCLocal = sumGradCQueue_.DeQue<float>(); | 240 | sumGradCLocal = sumGradCQueue_.DeQue<float>(); |
| 239 | - sumGradRCLocal = outputBuf_.Get<float>(); | 241 | + sumGradRCLocal = castOutputBuf_.Get<float>(); |
| 240 | FinishApplyCamePart1Reduction(sumGradRCLocal, sumGradCLocal, reductionScratch, kVectorTileWidth); | 242 | FinishApplyCamePart1Reduction(sumGradRCLocal, sumGradCLocal, reductionScratch, kVectorTileWidth); |
| 243 | + // The producer/post path carries this low slot through workspace. | ||
| 244 | + // Direct writes RC immediately, so fold it before the one-float copy. | ||
| 245 | + // Both slots already live in castOutputBuf_; no new UB allocation. | ||
| 246 | + constexpr int64_t kRcLowOffset = 8; | ||
| 247 | + Add(sumGradRCLocal, sumGradRCLocal, sumGradRCLocal[kRcLowOffset], 1); | ||
| 248 | + PipeBarrier<PIPE_V>(); | ||
| 241 | } | 249 | } |
| 242 | 250 | ||
| 243 | __aicore__ inline void CopyOutVectorMatrix(int64_t batchIdx, LocalTensor<T> gradLocal, | 251 | __aicore__ inline void CopyOutVectorMatrix(int64_t batchIdx, LocalTensor<T> gradLocal, |
| @@ -303,6 +311,8 @@ private: | |||
| 303 | 3 * kOutputAlignmentElements; | 311 | 3 * kOutputAlignmentElements; |
| 304 | static constexpr int64_t kReduceScratchElements = 12 * kVectorTileWidth; | 312 | static constexpr int64_t kReduceScratchElements = 12 * kVectorTileWidth; |
| 305 | 313 | ||
| 314 | + int64_t maxInputElements_{8192}; | ||
| 315 | + int64_t maxOutputElements_{8280}; | ||
| 306 | int64_t n_{0}; | 316 | int64_t n_{0}; |
| 307 | int64_t m_{0}; | 317 | int64_t m_{0}; |
| 308 | int64_t matrixElements_{0}; | 318 | int64_t matrixElements_{0}; |
| @@ -311,9 +321,8 @@ private: | |||
| 311 | TQue<QuePosition::VECIN, 1> epsQueue_; | 321 | TQue<QuePosition::VECIN, 1> epsQueue_; |
| 312 | TQue<QuePosition::VECOUT, 1> sumGradRQueue_; | 322 | TQue<QuePosition::VECOUT, 1> sumGradRQueue_; |
| 313 | TQue<QuePosition::VECOUT, 1> sumGradCQueue_; | 323 | TQue<QuePosition::VECOUT, 1> sumGradCQueue_; |
| 314 | - TBuf<QuePosition::VECCALC> castBuf_; | 324 | + TBuf<QuePosition::VECCALC> castOutputBuf_; |
| 315 | TBuf<QuePosition::VECCALC> valueBuf_; | 325 | TBuf<QuePosition::VECCALC> valueBuf_; |
| 316 | - TBuf<QuePosition::VECCALC> outputBuf_; | ||
| 317 | TBuf<QuePosition::VECCALC> reduceBuf_; | 326 | TBuf<QuePosition::VECCALC> reduceBuf_; |
| 318 | GlobalTensor<T> gradGm_; | 327 | GlobalTensor<T> gradGm_; |
| 319 | GlobalTensor<float> epsGm_; | 328 | GlobalTensor<float> epsGm_; |
| @@ -64,9 +64,8 @@ __aicore__ inline int64_t CeilDivApplyCamePart1(int64_t value, int64_t factor) | |||
| 64 | 64 | ||
| 65 | // Apply eps to the valid columns in each row. | 65 | // Apply eps to the valid columns in each row. |
| 66 | __aicore__ inline void AddEpsApplyCamePart1(LocalTensor<float> values, float eps, int64_t tileColumns, | 66 | __aicore__ inline void AddEpsApplyCamePart1(LocalTensor<float> values, float eps, int64_t tileColumns, |
| 67 | - int64_t curRepeatTimes) | 67 | + int64_t curRepeatTimes, int64_t tileSize = 64) |
| 68 | { | 68 | { |
| 69 | - constexpr int64_t tileSize = 64; | ||
| 70 | if (tileColumns >= tileSize) { | 69 | if (tileColumns >= tileSize) { |
| 71 | Adds(values, values, eps, curRepeatTimes * tileSize); | 70 | Adds(values, values, eps, curRepeatTimes * tileSize); |
| 72 | return; | 71 | return; |
| @@ -36,6 +36,8 @@ public: | |||
| 36 | __aicore__ inline void ZeroWorkspace(GM_ADDR workspace); | 36 | __aicore__ inline void ZeroWorkspace(GM_ADDR workspace); |
| 37 | __aicore__ inline void ProcessTile(int64_t nLoopIdx, int64_t mLoopIdx, int64_t curRepeatTimes); | 37 | __aicore__ inline void ProcessTile(int64_t nLoopIdx, int64_t mLoopIdx, int64_t curRepeatTimes); |
| 38 | 38 | ||
| 39 | + __aicore__ inline LocalTensor<float> GetUbArena() { return phaseArena_.Get<float>(); } | ||
| 40 | + | ||
| 39 | private: | 41 | private: |
| 40 | __aicore__ inline void ParseTilingData(const ApplyCamePart1TilingData* tilingData); | 42 | __aicore__ inline void ParseTilingData(const ApplyCamePart1TilingData* tilingData); |
| 41 | __aicore__ inline void ClearAcculateMatrix(); | 43 | __aicore__ inline void ClearAcculateMatrix(); |
| @@ -51,6 +53,8 @@ private: | |||
| 51 | __aicore__ inline void ComputeC(int64_t mLoopIdx, int64_t curRepeatTimes, LocalTensor<float> gradSqrtTmpUb); | 53 | __aicore__ inline void ComputeC(int64_t mLoopIdx, int64_t curRepeatTimes, LocalTensor<float> gradSqrtTmpUb); |
| 52 | 54 | ||
| 53 | private: | 55 | private: |
| 56 | + TBuf<QuePosition::VECCALC> phaseArena_; | ||
| 57 | + int64_t arenaBytes_{0}; | ||
| 54 | TPipe pipe; | 58 | TPipe pipe; |
| 55 | TQue<QuePosition::VECIN, 1> gradQueue_; | 59 | TQue<QuePosition::VECIN, 1> gradQueue_; |
| 56 | TQue<QuePosition::VECIN, 1> epsQueue_; | 60 | TQue<QuePosition::VECIN, 1> epsQueue_; |
| @@ -59,11 +63,6 @@ private: | |||
| 59 | TQue<QuePosition::VECOUT, 1> sumGradCQueue_; | 63 | TQue<QuePosition::VECOUT, 1> sumGradCQueue_; |
| 60 | TQue<QuePosition::VECOUT, 1> sumGradRCQueue_; | 64 | TQue<QuePosition::VECOUT, 1> sumGradRCQueue_; |
| 61 | 65 | ||
| 62 | - TBuf<QuePosition::VECCALC> accuComTmpBuf_; | ||
| 63 | - TBuf<QuePosition::VECCALC> gradCastTmpBuf_; | ||
| 64 | - TBuf<QuePosition::VECCALC> gradSqrtTmpBuf_; | ||
| 65 | - TBuf<QuePosition::VECCALC> mComTmpBuf_; | ||
| 66 | - | ||
| 67 | GlobalTensor<T> gmGrad_; | 66 | GlobalTensor<T> gmGrad_; |
| 68 | GlobalTensor<float> gmEps_; | 67 | GlobalTensor<float> gmEps_; |
| 69 | GlobalTensor<float> gmSumGradR_; | 68 | GlobalTensor<float> gmSumGradR_; |
| @@ -114,7 +113,7 @@ private: | |||
| 114 | GM_ADDR sumGradCAddrBatch_{0}; | 113 | GM_ADDR sumGradCAddrBatch_{0}; |
| 115 | GM_ADDR sumGradRCAddrBatch_{0}; | 114 | GM_ADDR sumGradRCAddrBatch_{0}; |
| 116 | 115 | ||
| 117 | - const int64_t ONCE_HANDLE_NUM64{64}; | 116 | + int64_t ONCE_HANDLE_NUM64{64}; |
| 118 | const int64_t ONCE_ONE_SIZE8{8}; | 117 | const int64_t ONCE_ONE_SIZE8{8}; |
| 119 | }; | 118 | }; |
| 120 | 119 | ||
| @@ -122,6 +121,8 @@ template <typename T> | |||
| 122 | __aicore__ inline void ApplyCamePart1FP16<T>::ParseTilingData(const ApplyCamePart1TilingData* tilingData) | 121 | __aicore__ inline void ApplyCamePart1FP16<T>::ParseTilingData(const ApplyCamePart1TilingData* tilingData) |
| 123 | { | 122 | { |
| 124 | // 总维度[N, M] | 123 | // 总维度[N, M] |
| 124 | + ONCE_HANDLE_NUM64 = tilingData->tileSize; | ||
| 125 | + arenaBytes_ = tilingData->ubArenaBytes; | ||
| 125 | N = tilingData->N; | 126 | N = tilingData->N; |
| 126 | M = tilingData->M; | 127 | M = tilingData->M; |
| 127 | 128 | ||
| @@ -192,7 +193,7 @@ template <typename T> | |||
| 192 | __aicore__ inline void ApplyCamePart1FP16<T>::InitBatch(int64_t batchIdx) | 193 | __aicore__ inline void ApplyCamePart1FP16<T>::InitBatch(int64_t batchIdx) |
| 193 | { | 194 | { |
| 194 | // The post stage reuses UB on the group leader, so each batch starts with a fresh row accumulator. | 195 | // The post stage reuses UB on the group leader, so each batch starts with a fresh row accumulator. |
| 195 | - LocalTensor<float> rowAccumulator = accuComTmpBuf_.Get<float>(); | 196 | + LocalTensor<float> rowAccumulator = phaseArena_.Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64]; |
| 196 | Duplicate(rowAccumulator, static_cast<float>(0), 2 * ONCE_HANDLE_NUM64); | 197 | Duplicate(rowAccumulator, static_cast<float>(0), 2 * ONCE_HANDLE_NUM64); |
| 197 | PipeBarrier<PIPE_V>(); | 198 | PipeBarrier<PIPE_V>(); |
| 198 | 199 | ||
| @@ -215,7 +216,7 @@ template <typename T> | |||
| 215 | __aicore__ inline void ApplyCamePart1FP16<T>::ZeroOutputs(GM_ADDR sum_grad_r, GM_ADDR sum_grad_c, GM_ADDR sum_grad_rc, | 216 | __aicore__ inline void ApplyCamePart1FP16<T>::ZeroOutputs(GM_ADDR sum_grad_r, GM_ADDR sum_grad_c, GM_ADDR sum_grad_rc, |
| 216 | int64_t batchCount) | 217 | int64_t batchCount) |
| 217 | { | 218 | { |
| 218 | - LocalTensor<float> zeroUb = gradCastTmpBuf_.Get<float>(); | 219 | + LocalTensor<float> zeroUb = phaseArena_.Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 + 2 * ONCE_HANDLE_NUM64]; |
| 219 | Duplicate(zeroUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | 220 | Duplicate(zeroUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); |
| 220 | PipeBarrier<PIPE_V>(); | 221 | PipeBarrier<PIPE_V>(); |
| 221 | event_t eventIdVToMte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); | 222 | event_t eventIdVToMte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); |
| @@ -249,7 +250,8 @@ __aicore__ inline void ApplyCamePart1FP16<T>::ZeroWorkspace(GM_ADDR workspace) | |||
| 249 | return; | 250 | return; |
| 250 | } | 251 | } |
| 251 | } | 252 | } |
| 252 | - LocalTensor<float> zeroUb = gradCastTmpBuf_.Get<float>(); | 253 | + LocalTensor<float> zeroUb = phaseArena_ |
| 254 | + .Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 + 2 * ONCE_HANDLE_NUM64]; | ||
| 253 | Duplicate(zeroUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | 255 | Duplicate(zeroUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); |
| 254 | PipeBarrier<PIPE_V>(); | 256 | PipeBarrier<PIPE_V>(); |
| 255 | event_t eventIdVToMte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); | 257 | event_t eventIdVToMte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); |
| @@ -311,10 +313,12 @@ __aicore__ inline void ApplyCamePart1FP16<T>::Compute(int64_t nLoopIdx, int64_t | |||
| 311 | LocalTensor<T> gradLocal = gradQueue_.DeQue<T>(); | 313 | LocalTensor<T> gradLocal = gradQueue_.DeQue<T>(); |
| 312 | LocalTensor<float> epsLocal = epsQueue_.DeQue<float>(); | 314 | LocalTensor<float> epsLocal = epsQueue_.DeQue<float>(); |
| 313 | 315 | ||
| 314 | - LocalTensor<float> gradCastTmpUb = gradCastTmpBuf_.Get<float>(); | 316 | + LocalTensor<float> gradCastTmpUb = phaseArena_ |
| 315 | - LocalTensor<float> gradSqrtTmpUb = gradSqrtTmpBuf_.Get<float>(); | 317 | + .Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 + 2 * ONCE_HANDLE_NUM64]; |
| 316 | - LocalTensor<float> accuComTmpUb = accuComTmpBuf_.Get<float>(); | 318 | + LocalTensor<float> gradSqrtTmpUb = phaseArena_.Get<float>(); |
| 317 | - LocalTensor<float> reductionTmpUb = mComTmpBuf_.Get<float>(); | 319 | + LocalTensor<float> accuComTmpUb = phaseArena_.Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64]; |
| 320 | + LocalTensor<float> | ||
| 321 | + reductionTmpUb = phaseArena_.Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 + 2 * ONCE_HANDLE_NUM64]; | ||
| 318 | Duplicate(gradCastTmpUb, (float)0.0, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | 322 | Duplicate(gradCastTmpUb, (float)0.0, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); |
| 319 | PipeBarrier<PIPE_V>(); | 323 | PipeBarrier<PIPE_V>(); |
| 320 | event_t eventIdMte2ToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V)); | 324 | event_t eventIdMte2ToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V)); |
| @@ -334,7 +338,8 @@ __aicore__ inline void ApplyCamePart1FP16<T>::Compute(int64_t nLoopIdx, int64_t | |||
| 334 | Mul(gradSqrtTmpUb, gradCastTmpUb, gradCastTmpUb, calCount); | 338 | Mul(gradSqrtTmpUb, gradCastTmpUb, gradCastTmpUb, calCount); |
| 335 | 339 | ||
| 336 | PipeBarrier<PIPE_V>(); | 340 | PipeBarrier<PIPE_V>(); |
| 337 | - AddEpsApplyCamePart1(gradSqrtTmpUb, eps, coreColumns_ - mLoopIdx * ONCE_HANDLE_NUM64, curRepeatTimes); | 341 | + AddEpsApplyCamePart1(gradSqrtTmpUb, eps, coreColumns_ - mLoopIdx * ONCE_HANDLE_NUM64, curRepeatTimes, |
| 342 | + ONCE_HANDLE_NUM64); | ||
| 338 | 343 | ||
| 339 | PipeBarrier<PIPE_V>(); | 344 | PipeBarrier<PIPE_V>(); |
| 340 | (void)accuComTmpUb; | 345 | (void)accuComTmpUb; |
| @@ -347,7 +352,7 @@ __aicore__ inline void ApplyCamePart1FP16<T>::Compute(int64_t nLoopIdx, int64_t | |||
| 347 | 352 | ||
| 348 | if (mLoopIdx == (mLoopCount_ - 1)) { | 353 | if (mLoopIdx == (mLoopCount_ - 1)) { |
| 349 | PipeBarrier<PIPE_V>(); | 354 | PipeBarrier<PIPE_V>(); |
| 350 | - Duplicate(accuComTmpUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | 355 | + Duplicate(accuComTmpUb, static_cast<float>(0), 2 * ONCE_HANDLE_NUM64); |
| 351 | Duplicate(reductionTmpUb, static_cast<float>(0), 4 * ONCE_HANDLE_NUM64); | 356 | Duplicate(reductionTmpUb, static_cast<float>(0), 4 * ONCE_HANDLE_NUM64); |
| 352 | } | 357 | } |
| 353 | 358 | ||
| @@ -418,23 +423,9 @@ __aicore__ inline void ApplyCamePart1FP16<T>::ProcessTile(int64_t nLoopIdx, int6 | |||
| 418 | template <typename T> | 423 | template <typename T> |
| 419 | __aicore__ inline void ApplyCamePart1FP16<T>::ClearAcculateMatrix() | 424 | __aicore__ inline void ApplyCamePart1FP16<T>::ClearAcculateMatrix() |
| 420 | { | 425 | { |
| 421 | - constexpr float scalarValue = 0; | 426 | + pipe.InitBuffer(phaseArena_, arenaBytes_); |
| 422 | - | 427 | + LocalTensor<float> arena = phaseArena_.Get<float>(); |
| 423 | - pipe.InitBuffer(gradCastTmpBuf_, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 * sizeof(float)); | 428 | + Duplicate(arena, static_cast<float>(0), arenaBytes_ / sizeof(float)); |
| 424 | - LocalTensor<float> gradCastTmpUb = gradCastTmpBuf_.Get<float>(ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | ||
| 425 | - Duplicate(gradCastTmpUb, scalarValue, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | ||
| 426 | - | ||
| 427 | - pipe.InitBuffer(gradSqrtTmpBuf_, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 * sizeof(float)); | ||
| 428 | - LocalTensor<float> gradSqrtTmpUb = gradSqrtTmpBuf_.Get<float>(ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | ||
| 429 | - Duplicate(gradSqrtTmpUb, scalarValue, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | ||
| 430 | - | ||
| 431 | - pipe.InitBuffer(accuComTmpBuf_, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 * sizeof(float)); | ||
| 432 | - LocalTensor<float> accuComTmpUb = accuComTmpBuf_.Get<float>(ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | ||
| 433 | - Duplicate(accuComTmpUb, scalarValue, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | ||
| 434 | - | ||
| 435 | - pipe.InitBuffer(mComTmpBuf_, (4 + 8) * ONCE_HANDLE_NUM64 * sizeof(float)); | ||
| 436 | - LocalTensor<float> reductionTmpUb = mComTmpBuf_.Get<float>(); | ||
| 437 | - Duplicate(reductionTmpUb, scalarValue, 4 * ONCE_HANDLE_NUM64); | ||
| 438 | } | 429 | } |
| 439 | 430 | ||
| 440 | } // namespace ApplyCamePart1 | 431 | } // namespace ApplyCamePart1 |
| @@ -35,6 +35,8 @@ public: | |||
| 35 | __aicore__ inline void Process(); | 35 | __aicore__ inline void Process(); |
| 36 | __aicore__ inline void ProcessTile(int64_t nLoopIdx, int64_t mLoopIdx, int64_t curRepeatTimes); | 36 | __aicore__ inline void ProcessTile(int64_t nLoopIdx, int64_t mLoopIdx, int64_t curRepeatTimes); |
| 37 | 37 | ||
| 38 | + __aicore__ inline LocalTensor<float> GetUbArena() { return phaseArena_.Get<float>(); } | ||
| 39 | + | ||
| 38 | private: | 40 | private: |
| 39 | __aicore__ inline void ParseTilingData(const ApplyCamePart1TilingData* tilingData); | 41 | __aicore__ inline void ParseTilingData(const ApplyCamePart1TilingData* tilingData); |
| 40 | __aicore__ inline void ClearAcculateMatrix(); | 42 | __aicore__ inline void ClearAcculateMatrix(); |
| @@ -53,6 +55,8 @@ private: | |||
| 53 | __aicore__ inline void ComputeC(int64_t mLoopIdx, int64_t curRepeatTimes, LocalTensor<float> gradSqrtTmpUb); | 55 | __aicore__ inline void ComputeC(int64_t mLoopIdx, int64_t curRepeatTimes, LocalTensor<float> gradSqrtTmpUb); |
| 54 | 56 | ||
| 55 | private: | 57 | private: |
| 58 | + TBuf<QuePosition::VECCALC> phaseArena_; | ||
| 59 | + int64_t arenaBytes_{0}; | ||
| 56 | TPipe pipe; | 60 | TPipe pipe; |
| 57 | TQue<QuePosition::VECIN, 1> gradQueue_; | 61 | TQue<QuePosition::VECIN, 1> gradQueue_; |
| 58 | TQue<QuePosition::VECIN, 1> epsQueue_; | 62 | TQue<QuePosition::VECIN, 1> epsQueue_; |
| @@ -61,11 +65,6 @@ private: | |||
| 61 | TQue<QuePosition::VECOUT, 1> sumGradCQueue_; | 65 | TQue<QuePosition::VECOUT, 1> sumGradCQueue_; |
| 62 | TQue<QuePosition::VECOUT, 1> sumGradRCQueue_; | 66 | TQue<QuePosition::VECOUT, 1> sumGradRCQueue_; |
| 63 | 67 | ||
| 64 | - TBuf<QuePosition::VECCALC> accuComTmpBuf_; | ||
| 65 | - TBuf<QuePosition::VECCALC> gradCastTmpBuf_; | ||
| 66 | - TBuf<QuePosition::VECCALC> gradSqrtTmpBuf_; | ||
| 67 | - TBuf<QuePosition::VECCALC> mComTmpBuf_; | ||
| 68 | - | ||
| 69 | GlobalTensor<float> gmGrad_; | 68 | GlobalTensor<float> gmGrad_; |
| 70 | GlobalTensor<float> gmEps_; | 69 | GlobalTensor<float> gmEps_; |
| 71 | GlobalTensor<float> gmSumGradR_; | 70 | GlobalTensor<float> gmSumGradR_; |
| @@ -117,7 +116,7 @@ private: | |||
| 117 | GM_ADDR sumGradCAddrBatch_{0}; | 116 | GM_ADDR sumGradCAddrBatch_{0}; |
| 118 | GM_ADDR sumGradRCAddrBatch_{0}; | 117 | GM_ADDR sumGradRCAddrBatch_{0}; |
| 119 | 118 | ||
| 120 | - const int64_t ONCE_HANDLE_NUM64{64}; | 119 | + int64_t ONCE_HANDLE_NUM64{64}; |
| 121 | const int64_t ONCE_ONE_SIZE8{8}; | 120 | const int64_t ONCE_ONE_SIZE8{8}; |
| 122 | }; | 121 | }; |
| 123 | 122 | ||
| @@ -125,6 +124,8 @@ template <typename T> | |||
| 125 | __aicore__ inline void ApplyCamePart1FP32<T>::ParseTilingData(const ApplyCamePart1TilingData* tilingData) | 124 | __aicore__ inline void ApplyCamePart1FP32<T>::ParseTilingData(const ApplyCamePart1TilingData* tilingData) |
| 126 | { | 125 | { |
| 127 | // 总维度[N, M] | 126 | // 总维度[N, M] |
| 127 | + ONCE_HANDLE_NUM64 = tilingData->tileSize; | ||
| 128 | + arenaBytes_ = tilingData->ubArenaBytes; | ||
| 128 | N = tilingData->N; | 129 | N = tilingData->N; |
| 129 | M = tilingData->M; | 130 | M = tilingData->M; |
| 130 | 131 | ||
| @@ -195,7 +196,7 @@ template <typename T> | |||
| 195 | __aicore__ inline void ApplyCamePart1FP32<T>::InitBatch(int64_t batchIdx) | 196 | __aicore__ inline void ApplyCamePart1FP32<T>::InitBatch(int64_t batchIdx) |
| 196 | { | 197 | { |
| 197 | // The post stage reuses UB on the group leader, so each batch starts with a fresh row accumulator. | 198 | // The post stage reuses UB on the group leader, so each batch starts with a fresh row accumulator. |
| 198 | - LocalTensor<float> rowAccumulator = accuComTmpBuf_.Get<float>(); | 199 | + LocalTensor<float> rowAccumulator = phaseArena_.Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64]; |
| 199 | Duplicate(rowAccumulator, static_cast<float>(0), 2 * ONCE_HANDLE_NUM64); | 200 | Duplicate(rowAccumulator, static_cast<float>(0), 2 * ONCE_HANDLE_NUM64); |
| 200 | PipeBarrier<PIPE_V>(); | 201 | PipeBarrier<PIPE_V>(); |
| 201 | 202 | ||
| @@ -218,7 +219,7 @@ template <typename T> | |||
| 218 | __aicore__ inline void ApplyCamePart1FP32<T>::ZeroOutputs(GM_ADDR sum_grad_r, GM_ADDR sum_grad_c, GM_ADDR sum_grad_rc, | 219 | __aicore__ inline void ApplyCamePart1FP32<T>::ZeroOutputs(GM_ADDR sum_grad_r, GM_ADDR sum_grad_c, GM_ADDR sum_grad_rc, |
| 219 | int64_t batchCount) | 220 | int64_t batchCount) |
| 220 | { | 221 | { |
| 221 | - LocalTensor<float> zeroUb = gradCastTmpBuf_.Get<float>(); | 222 | + LocalTensor<float> zeroUb = phaseArena_.Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 + 2 * ONCE_HANDLE_NUM64]; |
| 222 | Duplicate(zeroUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | 223 | Duplicate(zeroUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); |
| 223 | PipeBarrier<PIPE_V>(); | 224 | PipeBarrier<PIPE_V>(); |
| 224 | event_t eventIdVToMte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); | 225 | event_t eventIdVToMte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); |
| @@ -252,7 +253,8 @@ __aicore__ inline void ApplyCamePart1FP32<T>::ZeroWorkspace(GM_ADDR workspace) | |||
| 252 | return; | 253 | return; |
| 253 | } | 254 | } |
| 254 | } | 255 | } |
| 255 | - LocalTensor<float> zeroUb = gradCastTmpBuf_.Get<float>(); | 256 | + LocalTensor<float> zeroUb = phaseArena_ |
| 257 | + .Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 + 2 * ONCE_HANDLE_NUM64]; | ||
| 256 | Duplicate(zeroUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | 258 | Duplicate(zeroUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); |
| 257 | PipeBarrier<PIPE_V>(); | 259 | PipeBarrier<PIPE_V>(); |
| 258 | event_t eventIdVToMte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); | 260 | event_t eventIdVToMte3 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3)); |
| @@ -334,7 +336,8 @@ __aicore__ inline void ApplyCamePart1FP32<T>::ComputeElementwise( | |||
| 334 | Mul(gradSqrtTmpUb, gradCastTmpUb, gradCastTmpUb, calCount); | 336 | Mul(gradSqrtTmpUb, gradCastTmpUb, gradCastTmpUb, calCount); |
| 335 | 337 | ||
| 336 | PipeBarrier<PIPE_V>(); | 338 | PipeBarrier<PIPE_V>(); |
| 337 | - AddEpsApplyCamePart1(gradSqrtTmpUb, eps, coreColumns_ - mLoopIdx * ONCE_HANDLE_NUM64, curRepeatTimes); | 339 | + AddEpsApplyCamePart1(gradSqrtTmpUb, eps, coreColumns_ - mLoopIdx * ONCE_HANDLE_NUM64, curRepeatTimes, |
| 340 | + ONCE_HANDLE_NUM64); | ||
| 338 | 341 | ||
| 339 | PipeBarrier<PIPE_V>(); | 342 | PipeBarrier<PIPE_V>(); |
| 340 | (void)accuComTmpUb; | 343 | (void)accuComTmpUb; |
| @@ -345,10 +348,12 @@ __aicore__ inline void ApplyCamePart1FP32<T>::Compute(int64_t nLoopIdx, int64_t | |||
| 345 | { | 348 | { |
| 346 | LocalTensor<float> gradLocal = gradQueue_.DeQue<float>(); | 349 | LocalTensor<float> gradLocal = gradQueue_.DeQue<float>(); |
| 347 | LocalTensor<float> epsLocal = epsQueue_.DeQue<float>(); | 350 | LocalTensor<float> epsLocal = epsQueue_.DeQue<float>(); |
| 348 | - LocalTensor<float> gradCastTmpUb = gradCastTmpBuf_.Get<float>(); | 351 | + LocalTensor<float> gradCastTmpUb = phaseArena_ |
| 349 | - LocalTensor<float> gradSqrtTmpUb = gradSqrtTmpBuf_.Get<float>(); | 352 | + .Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 + 2 * ONCE_HANDLE_NUM64]; |
| 350 | - LocalTensor<float> accuComTmpUb = accuComTmpBuf_.Get<float>(); | 353 | + LocalTensor<float> gradSqrtTmpUb = phaseArena_.Get<float>(); |
| 351 | - LocalTensor<float> reductionTmpUb = mComTmpBuf_.Get<float>(); | 354 | + LocalTensor<float> accuComTmpUb = phaseArena_.Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64]; |
| 355 | + LocalTensor<float> | ||
| 356 | + reductionTmpUb = phaseArena_.Get<float>()[ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 + 2 * ONCE_HANDLE_NUM64]; | ||
| 352 | ComputeElementwise(mLoopIdx, curRepeatTimes, gradLocal, epsLocal, gradCastTmpUb, gradSqrtTmpUb, accuComTmpUb); | 357 | ComputeElementwise(mLoopIdx, curRepeatTimes, gradLocal, epsLocal, gradCastTmpUb, gradSqrtTmpUb, accuComTmpUb); |
| 353 | 358 | ||
| 354 | LocalTensor<float> sumGradRCLocal = sumGradRCQueue_.AllocTensor<float>(); | 359 | LocalTensor<float> sumGradRCLocal = sumGradRCQueue_.AllocTensor<float>(); |
| @@ -360,7 +365,7 @@ __aicore__ inline void ApplyCamePart1FP32<T>::Compute(int64_t nLoopIdx, int64_t | |||
| 360 | 365 | ||
| 361 | if (mLoopIdx == (mLoopCount_ - 1)) { | 366 | if (mLoopIdx == (mLoopCount_ - 1)) { |
| 362 | PipeBarrier<PIPE_V>(); | 367 | PipeBarrier<PIPE_V>(); |
| 363 | - Duplicate(accuComTmpUb, static_cast<float>(0), ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | 368 | + Duplicate(accuComTmpUb, static_cast<float>(0), 2 * ONCE_HANDLE_NUM64); |
| 364 | Duplicate(reductionTmpUb, static_cast<float>(0), 4 * ONCE_HANDLE_NUM64); | 369 | Duplicate(reductionTmpUb, static_cast<float>(0), 4 * ONCE_HANDLE_NUM64); |
| 365 | } | 370 | } |
| 366 | sumGradCQueue_.EnQue<float>(sumGradCLocal); | 371 | sumGradCQueue_.EnQue<float>(sumGradCLocal); |
| @@ -431,23 +436,9 @@ __aicore__ inline void ApplyCamePart1FP32<T>::ProcessTile(int64_t nLoopIdx, int6 | |||
| 431 | template <typename T> | 436 | template <typename T> |
| 432 | __aicore__ inline void ApplyCamePart1FP32<T>::ClearAcculateMatrix() | 437 | __aicore__ inline void ApplyCamePart1FP32<T>::ClearAcculateMatrix() |
| 433 | { | 438 | { |
| 434 | - constexpr float scalarValue = 0; | 439 | + pipe.InitBuffer(phaseArena_, arenaBytes_); |
| 435 | - | 440 | + LocalTensor<float> arena = phaseArena_.Get<float>(); |
| 436 | - pipe.InitBuffer(gradCastTmpBuf_, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 * sizeof(float)); | 441 | + Duplicate(arena, static_cast<float>(0), arenaBytes_ / sizeof(float)); |
| 437 | - LocalTensor<float> gradCastTmpUb = gradCastTmpBuf_.Get<float>(ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | ||
| 438 | - Duplicate(gradCastTmpUb, scalarValue, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | ||
| 439 | - | ||
| 440 | - pipe.InitBuffer(gradSqrtTmpBuf_, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 * sizeof(float)); | ||
| 441 | - LocalTensor<float> gradSqrtTmpUb = gradSqrtTmpBuf_.Get<float>(ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | ||
| 442 | - Duplicate(gradSqrtTmpUb, scalarValue, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | ||
| 443 | - | ||
| 444 | - pipe.InitBuffer(accuComTmpBuf_, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64 * sizeof(float)); | ||
| 445 | - LocalTensor<float> accuComTmpUb = accuComTmpBuf_.Get<float>(ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | ||
| 446 | - Duplicate(accuComTmpUb, scalarValue, ONCE_HANDLE_NUM64 * ONCE_HANDLE_NUM64); | ||
| 447 | - | ||
| 448 | - pipe.InitBuffer(mComTmpBuf_, (4 + 8) * ONCE_HANDLE_NUM64 * sizeof(float)); | ||
| 449 | - LocalTensor<float> reductionTmpUb = mComTmpBuf_.Get<float>(); | ||
| 450 | - Duplicate(reductionTmpUb, scalarValue, 4 * ONCE_HANDLE_NUM64); | ||
| 451 | } | 442 | } |
| 452 | 443 | ||
| 453 | } // namespace ApplyCamePart1 | 444 | } // namespace ApplyCamePart1 |
| @@ -29,12 +29,15 @@ class ApplyCamePart1Post { | |||
| 29 | public: | 29 | public: |
| 30 | __aicore__ inline ApplyCamePart1Post(){}; | 30 | __aicore__ inline ApplyCamePart1Post(){}; |
| 31 | __aicore__ inline void Init(GM_ADDR grad, GM_ADDR eps, GM_ADDR sum_grad_r, GM_ADDR sum_grad_c, GM_ADDR sum_grad_rc, | 31 | __aicore__ inline void Init(GM_ADDR grad, GM_ADDR eps, GM_ADDR sum_grad_r, GM_ADDR sum_grad_c, GM_ADDR sum_grad_rc, |
| 32 | - GM_ADDR workspace, const ApplyCamePart1TilingData* tilingData); | 32 | + GM_ADDR workspace, const ApplyCamePart1TilingData* tilingData, |
| 33 | + LocalTensor<float> arena); | ||
| 33 | __aicore__ inline void ProcessBatch(int64_t batchIdx); | 34 | __aicore__ inline void ProcessBatch(int64_t batchIdx); |
| 34 | 35 | ||
| 35 | private: | 36 | private: |
| 36 | __aicore__ inline void ParseTilingData(const ApplyCamePart1TilingData* tilingData); | 37 | __aicore__ inline void ParseTilingData(const ApplyCamePart1TilingData* tilingData); |
| 37 | __aicore__ inline void Pre_Core_Compute(uint64_t gmOffsets, uint64_t cal_m, uint64_t base_m); | 38 | __aicore__ inline void Pre_Core_Compute(uint64_t gmOffsets, uint64_t cal_m, uint64_t base_m); |
| 39 | + __aicore__ inline void ReduceNarrowC(uint64_t gmOffsets, uint64_t outputCount, uint64_t rowCount, | ||
| 40 | + uint64_t rowsPerChunk); | ||
| 38 | __aicore__ inline void ReduceWorkspaceRPartial(LocalTensor<float> inputLocal, int64_t base, int64_t partialRows); | 41 | __aicore__ inline void ReduceWorkspaceRPartial(LocalTensor<float> inputLocal, int64_t base, int64_t partialRows); |
| 39 | __aicore__ inline void ReduceWorkspaceR(); | 42 | __aicore__ inline void ReduceWorkspaceR(); |
| 40 | __aicore__ inline void ReduceWorkspaceRCPair(LocalTensor<float> high, LocalTensor<float> highRight, | 43 | __aicore__ inline void ReduceWorkspaceRCPair(LocalTensor<float> high, LocalTensor<float> highRight, |
| @@ -51,11 +54,12 @@ private: | |||
| 51 | LocalTensor<float> scratch); | 54 | LocalTensor<float> scratch); |
| 52 | 55 | ||
| 53 | private: | 56 | private: |
| 54 | - TPipe pipe; | 57 | + int64_t postUbElements_{32768}; |
| 55 | - TBuf<QuePosition::VECCALC> inputBuf_; | 58 | + LocalTensor<float> inputLocal_; |
| 59 | + LocalTensor<float> outputScratchLocal_; | ||
| 60 | + | ||
| 56 | // Keep GM read-modify-write scratch separate from reduction input. Reusing | 61 | // Keep GM read-modify-write scratch separate from reduction input. Reusing |
| 57 | // inputBuf_ here would overwrite values that the caller still needs to read. | 62 | // inputBuf_ here would overwrite values that the caller still needs to read. |
| 58 | - TBuf<QuePosition::VECCALC> outputScratchBuf_; | ||
| 59 | 63 | ||
| 60 | GlobalTensor<float> gmGrad_; | 64 | GlobalTensor<float> gmGrad_; |
| 61 | GlobalTensor<float> gmEps_; | 65 | GlobalTensor<float> gmEps_; |
| @@ -95,7 +99,7 @@ private: | |||
| 95 | bool IsContainsTailM{false}; | 99 | bool IsContainsTailM{false}; |
| 96 | int64_t batchIdx_{0}; | 100 | int64_t batchIdx_{0}; |
| 97 | 101 | ||
| 98 | - const int64_t ONCE_HANDLE_NUM64{64}; | 102 | + int64_t ONCE_HANDLE_NUM64{64}; |
| 99 | const int64_t ONCE_HANDLE_NUM512{512}; | 103 | const int64_t ONCE_HANDLE_NUM512{512}; |
| 100 | const int64_t ONCE_ONE_SIZE8{8}; | 104 | const int64_t ONCE_ONE_SIZE8{8}; |
| 101 | }; | 105 | }; |
| @@ -104,6 +108,8 @@ template <typename T> | |||
| 104 | __aicore__ inline void ApplyCamePart1Post<T>::ParseTilingData(const ApplyCamePart1TilingData* tilingData) | 108 | __aicore__ inline void ApplyCamePart1Post<T>::ParseTilingData(const ApplyCamePart1TilingData* tilingData) |
| 105 | { | 109 | { |
| 106 | // 总维度[N, M] | 110 | // 总维度[N, M] |
| 111 | + ONCE_HANDLE_NUM64 = tilingData->tileSize; | ||
| 112 | + postUbElements_ = tilingData->postUbElements; | ||
| 107 | N = tilingData->N; | 113 | N = tilingData->N; |
| 108 | M = tilingData->M; | 114 | M = tilingData->M; |
| 109 | 115 | ||
| @@ -133,7 +139,7 @@ __aicore__ inline void ApplyCamePart1Post<T>::ParseTilingData(const ApplyCamePar | |||
| 133 | template <typename T> | 139 | template <typename T> |
| 134 | __aicore__ inline void ApplyCamePart1Post<T>::Init(GM_ADDR grad, GM_ADDR eps, GM_ADDR sum_grad_r, GM_ADDR sum_grad_c, | 140 | __aicore__ inline void ApplyCamePart1Post<T>::Init(GM_ADDR grad, GM_ADDR eps, GM_ADDR sum_grad_r, GM_ADDR sum_grad_c, |
| 135 | GM_ADDR sum_grad_rc, GM_ADDR workspace, | 141 | GM_ADDR sum_grad_rc, GM_ADDR workspace, |
| 136 | - const ApplyCamePart1TilingData* tilingData) | 142 | + const ApplyCamePart1TilingData* tilingData, LocalTensor<float> arena) |
| 137 | { | 143 | { |
| 138 | (void)grad; | 144 | (void)grad; |
| 139 | (void)eps; | 145 | (void)eps; |
| @@ -169,8 +175,8 @@ __aicore__ inline void ApplyCamePart1Post<T>::Init(GM_ADDR grad, GM_ADDR eps, GM | |||
| 169 | workspaceSumGradC_.SetGlobalBuffer(regionBase + 2 * rcOffsets + rOffsets); | 175 | workspaceSumGradC_.SetGlobalBuffer(regionBase + 2 * rcOffsets + rOffsets); |
| 170 | } | 176 | } |
| 171 | 177 | ||
| 172 | - pipe.InitBuffer(inputBuf_, ONCE_HANDLE_NUM512 * ONCE_HANDLE_NUM64 * sizeof(float)); | 178 | + inputLocal_ = arena; |
| 173 | - pipe.InitBuffer(outputScratchBuf_, 2 * ONCE_ONE_SIZE8 * sizeof(float)); | 179 | + outputScratchLocal_ = arena[postUbElements_]; |
| 174 | } | 180 | } |
| 175 | 181 | ||
| 176 | template <typename T> | 182 | template <typename T> |
| @@ -192,7 +198,8 @@ __aicore__ inline void ApplyCamePart1Post<T>::ProcessBatch(int64_t batchIdx) | |||
| 192 | // 产者/归约阶段的同步由内核入口统一负责, 此处不再做全局屏障. | 198 | // 产者/归约阶段的同步由内核入口统一负责, 此处不再做全局屏障. |
| 193 | batchIdx_ = batchIdx; | 199 | batchIdx_ = batchIdx; |
| 194 | ReduceWorkspaceR(); | 200 | ReduceWorkspaceR(); |
| 195 | - uint64_t core_loop = (totalColumnTileCount_ + ONCE_HANDLE_NUM512 - 1) / ONCE_HANDLE_NUM512; | 201 | + const uint64_t columnTilesPerChunk = postUbElements_ / ONCE_HANDLE_NUM64; |
| 202 | + uint64_t core_loop = (totalColumnTileCount_ + columnTilesPerChunk - 1) / columnTilesPerChunk; | ||
| 196 | uint64_t pre_core_m = (totalColumnTileCount_ + core_loop - 1) / core_loop * ONCE_HANDLE_NUM64; | 203 | uint64_t pre_core_m = (totalColumnTileCount_ + core_loop - 1) / core_loop * ONCE_HANDLE_NUM64; |
| 197 | uint64_t last_core_m = (totalColumnTileCount_ - pre_core_m * (core_loop - 1) / ONCE_HANDLE_NUM64) * | 204 | uint64_t last_core_m = (totalColumnTileCount_ - pre_core_m * (core_loop - 1) / ONCE_HANDLE_NUM64) * |
| 198 | ONCE_HANDLE_NUM64; | 205 | ONCE_HANDLE_NUM64; |
| @@ -213,8 +220,8 @@ template <typename T> | |||
| 213 | __aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceRPartial(LocalTensor<float> inputLocal, int64_t base, | 220 | __aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceRPartial(LocalTensor<float> inputLocal, int64_t base, |
| 214 | int64_t partialRows) | 221 | int64_t partialRows) |
| 215 | { | 222 | { |
| 216 | - constexpr int64_t rowWidth = 64; | 223 | + const int64_t rowWidth = ONCE_HANDLE_NUM64; |
| 217 | - constexpr int64_t maxRowsPerCopy = 256; | 224 | + const int64_t maxRowsPerCopy = postUbElements_ / (2 * rowWidth); |
| 218 | int64_t activeRows = partialRows; | 225 | int64_t activeRows = partialRows; |
| 219 | while (activeRows > 1) { | 226 | while (activeRows > 1) { |
| 220 | const int64_t rightRows = (activeRows + 1) / 2; | 227 | const int64_t rightRows = (activeRows + 1) / 2; |
| @@ -254,8 +261,8 @@ __aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceRPartial(LocalTenso | |||
| 254 | template <typename T> | 261 | template <typename T> |
| 255 | __aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceR() | 262 | __aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceR() |
| 256 | { | 263 | { |
| 257 | - LocalTensor<float> inputLocal = inputBuf_.Get<float>(ONCE_HANDLE_NUM512 * ONCE_HANDLE_NUM64); | 264 | + LocalTensor<float> inputLocal = inputLocal_; |
| 258 | - constexpr int64_t rowWidth = 64; | 265 | + const int64_t rowWidth = ONCE_HANDLE_NUM64; |
| 259 | const int64_t partialRows = mCoreNum_; | 266 | const int64_t partialRows = mCoreNum_; |
| 260 | 267 | ||
| 261 | for (int64_t nCoreIdx = 0; nCoreIdx < nCoreNum_; ++nCoreIdx) { | 268 | for (int64_t nCoreIdx = 0; nCoreIdx < nCoreNum_; ++nCoreIdx) { |
| @@ -275,7 +282,7 @@ __aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceR() | |||
| 275 | SetFlag<HardEvent::MTE2_S>(eventIdMte2ToS); | 282 | SetFlag<HardEvent::MTE2_S>(eventIdMte2ToS); |
| 276 | WaitFlag<HardEvent::MTE2_S>(eventIdMte2ToS); | 283 | WaitFlag<HardEvent::MTE2_S>(eventIdMte2ToS); |
| 277 | for (int64_t i = 0; i < rowCount; ++i) { | 284 | for (int64_t i = 0; i < rowCount; ++i) { |
| 278 | - StoreFloatAligned(gmSumGradR_, rowOffset + i, inputLocal.GetValue(i), outputScratchBuf_.Get<float>()); | 285 | + StoreFloatAligned(gmSumGradR_, rowOffset + i, inputLocal.GetValue(i), outputScratchLocal_); |
| 279 | } | 286 | } |
| 280 | event_t eventIdSToMte2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE2)); | 287 | event_t eventIdSToMte2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE2)); |
| 281 | SetFlag<HardEvent::S_MTE2>(eventIdSToMte2); | 288 | SetFlag<HardEvent::S_MTE2>(eventIdSToMte2); |
| @@ -359,7 +366,7 @@ __aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceRCFinal(LocalTensor | |||
| 359 | event_t eventIdVToS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | 366 | event_t eventIdVToS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); |
| 360 | SetFlag<HardEvent::V_S>(eventIdVToS); | 367 | SetFlag<HardEvent::V_S>(eventIdVToS); |
| 361 | WaitFlag<HardEvent::V_S>(eventIdVToS); | 368 | WaitFlag<HardEvent::V_S>(eventIdVToS); |
| 362 | - StoreFloatAligned(gmSumGradRC_, batchIdx_, inputLocal.GetValue(0), outputScratchBuf_.Get<float>()); | 369 | + StoreFloatAligned(gmSumGradRC_, batchIdx_, inputLocal.GetValue(0), outputScratchLocal_); |
| 363 | event_t eventIdSToMte2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE2)); | 370 | event_t eventIdSToMte2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE2)); |
| 364 | SetFlag<HardEvent::S_MTE2>(eventIdSToMte2); | 371 | SetFlag<HardEvent::S_MTE2>(eventIdSToMte2); |
| 365 | WaitFlag<HardEvent::S_MTE2>(eventIdSToMte2); | 372 | WaitFlag<HardEvent::S_MTE2>(eventIdSToMte2); |
| @@ -368,10 +375,10 @@ __aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceRCFinal(LocalTensor | |||
| 368 | template <typename T> | 375 | template <typename T> |
| 369 | __aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceRC() | 376 | __aicore__ inline void ApplyCamePart1Post<T>::ReduceWorkspaceRC() |
| 370 | { | 377 | { |
| 371 | - LocalTensor<float> inputLocal = inputBuf_.Get<float>(ONCE_HANDLE_NUM512 * ONCE_HANDLE_NUM64); | 378 | + LocalTensor<float> inputLocal = inputLocal_; |
| 372 | constexpr uint64_t kScalarSlotSize = 8; | 379 | constexpr uint64_t kScalarSlotSize = 8; |
| 373 | - constexpr uint64_t kMaxPairCount = 512; | 380 | + const uint64_t kMaxPairCount = static_cast<uint64_t>(postUbElements_) / (6 * 8); |
| 374 | - constexpr uint64_t kMaxPairElements = kMaxPairCount * kScalarSlotSize; | 381 | + const uint64_t kMaxPairElements = kMaxPairCount * kScalarSlotSize; |
| 375 | LocalTensor<float> high = inputLocal; | 382 | LocalTensor<float> high = inputLocal; |
| 376 | LocalTensor<float> highRight = inputLocal[kMaxPairElements]; | 383 | LocalTensor<float> highRight = inputLocal[kMaxPairElements]; |
| 377 | LocalTensor<float> low = inputLocal[2 * kMaxPairElements]; | 384 | LocalTensor<float> low = inputLocal[2 * kMaxPairElements]; |
| @@ -407,14 +414,67 @@ __aicore__ inline void ApplyCamePart1Post<T>::CopyWorkspaceCChunk(LocalTensor<fl | |||
| 407 | } | 414 | } |
| 408 | } | 415 | } |
| 409 | 416 | ||
| 417 | +template <typename T> | ||
| 418 | +__aicore__ inline void ApplyCamePart1Post<T>::ReduceNarrowC(uint64_t gmOffsets, uint64_t outputCount, uint64_t rowCount, | ||
| 419 | + uint64_t rowsPerChunk) | ||
| 420 | +{ | ||
| 421 | + const uint64_t width = static_cast<uint64_t>(ONCE_HANDLE_NUM64); | ||
| 422 | + LocalTensor<float> input = inputLocal_; | ||
| 423 | + LocalTensor<float> high = input[rowsPerChunk * width]; | ||
| 424 | + LocalTensor<float> low = input[(rowsPerChunk + 1) * width]; | ||
| 425 | + Duplicate(high, 0.0f, width); | ||
| 426 | + Duplicate(low, 0.0f, width); | ||
| 427 | + for (uint64_t start = 0; start < rowCount; start += rowsPerChunk) { | ||
| 428 | + Duplicate(input, 0.0f, rowsPerChunk * width); | ||
| 429 | + event_t vToMte2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE2)); | ||
| 430 | + SetFlag<HardEvent::V_MTE2>(vToMte2); | ||
| 431 | + WaitFlag<HardEvent::V_MTE2>(vToMte2); | ||
| 432 | + const uint64_t count = rowCount - start < rowsPerChunk ? rowCount - start : rowsPerChunk; | ||
| 433 | + CopyWorkspaceCChunk(input, start, count, gmOffsets, width); | ||
| 434 | + event_t mte2ToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_V)); | ||
| 435 | + SetFlag<HardEvent::MTE2_V>(mte2ToV); | ||
| 436 | + WaitFlag<HardEvent::MTE2_V>(mte2ToV); | ||
| 437 | + ReduceAdd(input, rowsPerChunk, width); | ||
| 438 | + event_t vToS = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S)); | ||
| 439 | + SetFlag<HardEvent::V_S>(vToS); | ||
| 440 | + WaitFlag<HardEvent::V_S>(vToS); | ||
| 441 | + for (uint64_t col = 0; col < outputCount; ++col) { | ||
| 442 | + float h = 0.0f; | ||
| 443 | + float l = 0.0f; | ||
| 444 | + CompensatedAddApplyCamePart1(high.GetValue(col), low.GetValue(col), input.GetValue(col), 0.0f, h, l); | ||
| 445 | + high.SetValue(col, h); | ||
| 446 | + low.SetValue(col, l); | ||
| 447 | + } | ||
| 448 | + event_t sToV = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_V)); | ||
| 449 | + SetFlag<HardEvent::S_V>(sToV); | ||
| 450 | + WaitFlag<HardEvent::S_V>(sToV); | ||
| 451 | + } | ||
| 452 | + Add(input, high, low, width); | ||
| 453 | + CopyOutCReduceResult(gmSumGradC_, input, batchIdx_ * M + gmOffsets, outputCount); | ||
| 454 | +} | ||
| 455 | + | ||
| 410 | template <typename T> | 456 | template <typename T> |
| 411 | __aicore__ inline void ApplyCamePart1Post<T>::Pre_Core_Compute(uint64_t gmOffsets, uint64_t cal_m, uint64_t base_m) | 457 | __aicore__ inline void ApplyCamePart1Post<T>::Pre_Core_Compute(uint64_t gmOffsets, uint64_t cal_m, uint64_t base_m) |
| 412 | { | 458 | { |
| 413 | uint64_t total_n = (nCoreNum_ - 1) * nLoopNormCore_ + nLoopTailCore_; | 459 | uint64_t total_n = (nCoreNum_ - 1) * nLoopNormCore_ + nLoopTailCore_; |
| 460 | + if (cal_m == static_cast<uint64_t>(ONCE_HANDLE_NUM64) && total_n > 512) { | ||
| 461 | + // Reserve two aligned rows for compensated accumulation in the same arena. | ||
| 462 | + const uint64_t width = static_cast<uint64_t>(ONCE_HANDLE_NUM64); | ||
| 463 | + const uint64_t availableRows = static_cast<uint64_t>(postUbElements_) / width; | ||
| 464 | + if (availableRows >= 4) { | ||
| 465 | + uint64_t rowsPerChunk = 1; | ||
| 466 | + while (rowsPerChunk < 256 && rowsPerChunk * 2 <= availableRows - 2) { | ||
| 467 | + rowsPerChunk *= 2; | ||
| 468 | + } | ||
| 469 | + const uint64_t outputCount = (M - gmOffsets < base_m) ? M - gmOffsets : base_m; | ||
| 470 | + ReduceNarrowC(gmOffsets, outputCount, total_n, rowsPerChunk); | ||
| 471 | + return; | ||
| 472 | + } | ||
| 473 | + } | ||
Outdated 🟡 Medium Priority 变更行(post.h:456-460)新增了整条补偿归约快路径并 return,但仓内没有任何用例能进入它。 可达条件推导:ProcessBatch 中 core_loop = ceil(totalColumnTileCount_/512)、pre_core_m = ceil(totalColumnTileCount_/core_loop)64、last_core_m = (totalColumnTileCount_ - pre_core_m(core_loop-1)/64)*64。只有当 totalColumnTileCount_ == 1(即 M ≤ 64)时 core_loop==1、last_core_m==64,cal_m 才可能等于 ONCE_HANDLE_NUM64;因此新路径的触发域是「M ≤ 64 且 total_n > 512(n 向行块数 > 512)」。 覆盖现状:tests/st/arch35/ttk_apply_came_part1_bf16.csv 只有一条数据用例 (65,67),M=67 → totalColumnTileCount_=2 → cal_m=128,永远不会走 ReduceNarrowC;UT 也仅覆盖 (2,2)/(4,8)/(65,67) 等小微 shape。同时新增的 tests/assets/golden_fp64.py 未被任何 csv/配置引用。 失败模式:新代码里有多个极易写错且只影响该触发域的点——high/low 槽位取址 input[25664]/input[25764]、尾块(count<rowsPerChunk)依赖 Duplicate 补零才成立、结束后一次性 Add(high,low) 再单次原子回写。上述任一处出错(例如槽位重叠、尾块未补零导致把残留数据累加进去、final Add 的 count 用错),在 N 很大且 M≤64 的输入上会给 sum_grad_c 造成静默精度偏差,而 CI 完全无法发现。 建议:为新增分支补充能真正进入 ReduceNarrowC 的用例(M ≤ 64 且 total_n > 512),同时覆盖 rowCount 非 rowsPerChunk 整数倍(尾块补零)与 outputCount < 64(M 非 64 对齐)两种边界,并用 FP64 参考校验精度。 ![]() ![]() Incorrect? | |||
| 414 | uint64_t pre_loop_n = 1; | 474 | uint64_t pre_loop_n = 1; |
| 415 | while (pre_loop_n < total_n) { | 475 | while (pre_loop_n < total_n) { |
| 416 | pre_loop_n = pre_loop_n << 1; | 476 | pre_loop_n = pre_loop_n << 1; |
| 417 | - if (pre_loop_n * cal_m > ONCE_HANDLE_NUM512 * ONCE_HANDLE_NUM64) { | 477 | + if (pre_loop_n * cal_m > static_cast<uint64_t>(postUbElements_)) { |
| 418 | pre_loop_n = pre_loop_n >> 1; | 478 | pre_loop_n = pre_loop_n >> 1; |
| 419 | break; | 479 | break; |
| 420 | } | 480 | } |
| @@ -424,7 +484,7 @@ __aicore__ inline void ApplyCamePart1Post<T>::Pre_Core_Compute(uint64_t gmOffset | |||
| 424 | } | 484 | } |
| 425 | uint64_t loop_time = (total_n + pre_loop_n - 1) / pre_loop_n; | 485 | uint64_t loop_time = (total_n + pre_loop_n - 1) / pre_loop_n; |
| 426 | uint64_t last_loop_n = total_n - (loop_time - 1) * pre_loop_n; | 486 | uint64_t last_loop_n = total_n - (loop_time - 1) * pre_loop_n; |
| 427 | - LocalTensor<float> inputLocal = inputBuf_.Get<float>(ONCE_HANDLE_NUM512 * ONCE_HANDLE_NUM64); | 487 | + LocalTensor<float> inputLocal = inputLocal_; |
| 428 | event_t eventIdSToMte2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE2)); | 488 | event_t eventIdSToMte2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::S_MTE2)); |
| 429 | SetFlag<HardEvent::S_MTE2>(eventIdSToMte2); | 489 | SetFlag<HardEvent::S_MTE2>(eventIdSToMte2); |
| 430 | WaitFlag<HardEvent::S_MTE2>(eventIdSToMte2); | 490 | WaitFlag<HardEvent::S_MTE2>(eventIdSToMte2); |
| @@ -29,6 +29,10 @@ struct ApplyCamePart1TilingData { | |||
| 29 | int64_t mCoreNum = 0; | 29 | int64_t mCoreNum = 0; |
| 30 | int64_t batchCoreNum = 1; | 30 | int64_t batchCoreNum = 1; |
| 31 | int64_t directBatchMode = 0; | 31 | int64_t directBatchMode = 0; |
| 32 | + int64_t tileSize = 64; | ||
| 33 | + int64_t ubArenaBytes = 131072; | ||
| 34 | + int64_t postUbElements = 32768; | ||
| 35 | + int64_t directUbElements = 8192; | ||
| 32 | #define APPLY_CAME_TILING_ACCESSOR(name) \ | 36 | #define APPLY_CAME_TILING_ACCESSOR(name) \ |
| 33 | int64_t get_##name() const { return name; } \ | 37 | int64_t get_##name() const { return name; } \ |
| 34 | void set_##name(int64_t value) { name = value; } | 38 | void set_##name(int64_t value) { name = value; } |
| @@ -48,6 +52,10 @@ struct ApplyCamePart1TilingData { | |||
| 48 | APPLY_CAME_TILING_ACCESSOR(mCoreNum) | 52 | APPLY_CAME_TILING_ACCESSOR(mCoreNum) |
| 49 | APPLY_CAME_TILING_ACCESSOR(batchCoreNum) | 53 | APPLY_CAME_TILING_ACCESSOR(batchCoreNum) |
| 50 | APPLY_CAME_TILING_ACCESSOR(directBatchMode) | 54 | APPLY_CAME_TILING_ACCESSOR(directBatchMode) |
| 55 | + APPLY_CAME_TILING_ACCESSOR(tileSize) | ||
| 56 | + APPLY_CAME_TILING_ACCESSOR(ubArenaBytes) | ||
| 57 | + APPLY_CAME_TILING_ACCESSOR(postUbElements) | ||
| 58 | + APPLY_CAME_TILING_ACCESSOR(directUbElements) | ||
| 51 | #undef APPLY_CAME_TILING_ACCESSOR | 59 | #undef APPLY_CAME_TILING_ACCESSOR |
| 52 | }; | 60 | }; |
| 53 | #endif // OPS_NN_APPLY_CAME_PART1_TILING_DATA_H | 61 | #endif // OPS_NN_APPLY_CAME_PART1_TILING_DATA_H |
| @@ -8,7 +8,6 @@ | |||
| 8 | # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. | 8 | # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. |
| 9 | # See LICENSE in the root of the software repository for the full text of the License. | 9 | # See LICENSE in the root of the software repository for the full text of the License. |
| 10 | 10 | ||
| 11 | -import ml_dtypes | ||
| 12 | import numpy as np | 11 | import numpy as np |
| 13 | import torch | 12 | import torch |
| 14 | 13 | ||
| @@ -51,6 +50,8 @@ def _to_numpy(outputs): | |||
| 51 | for output in outputs: | 50 | for output in outputs: |
| 52 | result = output.detach().cpu().contiguous() | 51 | result = output.detach().cpu().contiguous() |
| 53 | if result.dtype == torch.bfloat16: | 52 | if result.dtype == torch.bfloat16: |
| 53 | + import ml_dtypes | ||
| 54 | + | ||
| 54 | bits = result.view(torch.uint16).numpy().copy() | 55 | bits = result.view(torch.uint16).numpy().copy() |
| 55 | arrays.append(bits.view(ml_dtypes.bfloat16)) | 56 | arrays.append(bits.view(ml_dtypes.bfloat16)) |
| 56 | else: | 57 | else: |
| @@ -59,8 +60,17 @@ def _to_numpy(outputs): | |||
| 59 | 60 | ||
| 60 | 61 | ||
| 61 | def apply_came_part1_golden(grad, eps, **kwargs): | 62 | def apply_came_part1_golden(grad, eps, **kwargs): |
| 62 | - """Use FP32 accumulation for low-precision input.""" | 63 | + """Compute the CPU reference entirely in FP64 and retain FP64 outputs.""" |
| 63 | - return _to_numpy(_compute(grad, eps, match_kernel=True)) | 64 | + grad64 = _to_torch(grad).to(dtype=torch.float64, device="cpu") |
| 65 | + eps64 = _to_torch(eps).reshape(-1)[0].to(dtype=torch.float64, device="cpu") | ||
| 66 | + values64 = torch.square(grad64) + eps64 | ||
| 67 | + return _to_numpy( | ||
| 68 | + [ | ||
| 69 | + torch.sum(values64, dim=-1), | ||
| 70 | + torch.sum(values64, dim=-2), | ||
| 71 | + torch.sum(values64, dim=(-2, -1)), | ||
| 72 | + ] | ||
| 73 | + ) | ||
| 64 | 74 | ||
| 65 | 75 | ||
| 66 | class _ApplyCamePart1Compose: | 76 | class _ApplyCamePart1Compose: |
| @@ -1,2 +1,8 @@ | |||
| 1 | testcase_name,op_name,input_dtypes,input_formats,input_shapes,output_shapes,output_dtypes,output_formats,attributes,input_data_ranges,absolute_precision | 1 | testcase_name,op_name,input_dtypes,input_formats,input_shapes,output_shapes,output_dtypes,output_formats,attributes,input_data_ranges,absolute_precision |
| 2 | apply_came_part1_bf16_tail,apply_came_part1,"('bfloat16','float32')","('ND','ND')","((65,67),(1,))","((65,),(67,),())","('float32','float32','float32')","('ND','ND','ND')",{},"((-1,1),(0.001,0.001))",0.001 | 2 | apply_came_part1_bf16_tail,apply_came_part1,"('bfloat16','float32')","('ND','ND')","((65,67),(1,))","((65,),(67,),())","('float32','float32','float32')","('ND','ND','ND')",{},"((-1,1),(0.001,0.001))",0.001 |
| 3 | +apply_came_part1_float16_256_256,apply_came_part1,"('float16','float32')","('ND','ND')","((256,256),(1,))","((256,),(256,),())","('float32','float32','float32')","('ND','ND','ND')",{},"((-1,1),(0.001,0.001))",0.001 | ||
| 4 | +apply_came_part1_float16_257_259,apply_came_part1,"('float16','float32')","('ND','ND')","((257,259),(1,))","((257,),(259,),())","('float32','float32','float32')","('ND','ND','ND')",{},"((-1,1),(0.001,0.001))",0.001 | ||
| 5 | +apply_came_part1_bfloat16_256_256,apply_came_part1,"('bfloat16','float32')","('ND','ND')","((256,256),(1,))","((256,),(256,),())","('float32','float32','float32')","('ND','ND','ND')",{},"((-1,1),(0.001,0.001))",0.001 | ||
| 6 | +apply_came_part1_bfloat16_257_259,apply_came_part1,"('bfloat16','float32')","('ND','ND')","((257,259),(1,))","((257,),(259,),())","('float32','float32','float32')","('ND','ND','ND')",{},"((-1,1),(0.001,0.001))",0.001 | ||
| 7 | +apply_came_part1_float32_256_256,apply_came_part1,"('float32','float32')","('ND','ND')","((256,256),(1,))","((256,),(256,),())","('float32','float32','float32')","('ND','ND','ND')",{},"((-1,1),(0.001,0.001))",0.001 | ||
| 8 | +apply_came_part1_float32_257_259,apply_came_part1,"('float32','float32')","('ND','ND')","((257,259),(1,))","((257,),(259,),())","('float32','float32','float32')","('ND','ND','ND')",{},"((-1,1),(0.001,0.001))",0.001 | ||
| @@ -10,11 +10,13 @@ | |||
| 10 | 10 | ||
| 11 | #include <cstring> | 11 | #include <cstring> |
| 12 | #include <limits> | 12 | #include <limits> |
| 13 | +#include <utility> | ||
| 13 | #include <vector> | 14 | #include <vector> |
| 14 | #include <gtest/gtest.h> | 15 | #include <gtest/gtest.h> |
| 15 | #include "tiling_case_executor.h" | 16 | #include "tiling_case_executor.h" |
| 16 | #include "tiling_context_faker.h" | 17 | #include "tiling_context_faker.h" |
| 17 | #include "../../../../op_host/arch35/apply_came_part1_tiling_arch35.h" | 18 | #include "../../../../op_host/arch35/apply_came_part1_tiling_arch35.h" |
| 19 | +#include "../../../../op_host/arch35/apply_came_part1_ub_plan.h" | ||
| 18 | 20 | ||
| 19 | namespace ApplyCamePart1TilingUT { | 21 | namespace ApplyCamePart1TilingUT { |
| 20 | namespace { | 22 | namespace { |
| @@ -89,6 +91,40 @@ bool TryTiling(const std::vector<int64_t>& gradShape, ge::DataType gradType) | |||
| 89 | } | 91 | } |
| 90 | } // namespace | 92 | } // namespace |
| 91 | 93 | ||
| 94 | +TEST(ApplyCamePart1Tiling, UbPlannerSelectsTileAndBoundsLiveBuffers) | ||
| 95 | +{ | ||
| 96 | + for (const uint64_t dtypeBytes : {2UL, 4UL}) { | ||
| 97 | + for (const auto& config : {std::pair<uint64_t, uint64_t>{kUbSize, 128}, {64UL * 1024UL, 64}}) { | ||
| 98 | + SCOPED_TRACE(::testing::Message() << "dtypeBytes=" << dtypeBytes << " ub=" << config.first); | ||
| 99 | + const auto plan = apply_came_ub::MakePlan(config.first, dtypeBytes, kCoreNum); | ||
| 100 | + ASSERT_EQ(plan.tile, config.second); | ||
| 101 | + EXPECT_GT(plan.directElements, 0U); | ||
| 102 | + EXPECT_LE(apply_came_ub::kReserveBytes + plan.persistentBytes + plan.arenaBytes, config.first); | ||
| 103 | + EXPECT_LE(plan.producerPeakBytes, plan.arenaBytes); | ||
| 104 | + EXPECT_LE(plan.postElements * sizeof(float) + 64, plan.arenaBytes); | ||
| 105 | + EXPECT_EQ(plan.postElements % plan.tile, 0U); | ||
| 106 | + EXPECT_GE(plan.postElements, 2 * plan.tile); | ||
| 107 | + // RC post uses six arrays of 8-float pairs and a separate 64-byte output slot. | ||
| 108 | + EXPECT_LE((plan.postElements / (6 * 8)) * 6 * 8 * sizeof(float) + 64, plan.arenaBytes); | ||
| 109 | + } | ||
| 110 | + } | ||
| 111 | +} | ||
| 112 | + | ||
| 113 | +TEST(ApplyCamePart1Tiling, LargeTilesUsePlannedArenaForEachDtype) | ||
| 114 | +{ | ||
| 115 | + for (const auto dtype : {ge::DT_FLOAT16, ge::DT_BF16, ge::DT_FLOAT}) { | ||
| 116 | + SCOPED_TRACE(static_cast<int>(dtype)); | ||
| 117 | + const auto result = RunTiling({257, 259}, dtype); | ||
| 118 | + const uint64_t dtypeBytes = dtype == ge::DT_FLOAT ? 4 : 2; | ||
| 119 | + const auto plan = apply_came_ub::MakePlan(kUbSize, dtypeBytes, kCoreNum); | ||
| 120 | + ASSERT_EQ(plan.tile, 128U); | ||
| 121 | + EXPECT_EQ(result.data.tileSize, plan.tile); | ||
| 122 | + EXPECT_EQ(result.data.ubArenaBytes, plan.arenaBytes); | ||
| 123 | + EXPECT_EQ(result.data.postUbElements, plan.postElements); | ||
| 124 | + EXPECT_EQ(result.data.directBatchMode, 0); | ||
| 125 | + } | ||
| 126 | +} | ||
| 127 | + | ||
| 92 | TEST(ApplyCamePart1Tiling, Fp16UsesGeneralBatchDirectPath) | 128 | TEST(ApplyCamePart1Tiling, Fp16UsesGeneralBatchDirectPath) |
| 93 | { | 129 | { |
| 94 | const auto result = RunTiling({12562512, 2, 2}, ge::DT_FLOAT16); | 130 | const auto result = RunTiling({12562512, 2, 2}, ge::DT_FLOAT16); |


【review】 问题分类:可测 位置:optim/apply_came_part1/op_host/arch35/apply_came_part1_tiling_arch35.cpp:249,代码:data.set_tileSize(plan.tile); 问题:本次将原来固定 64 的生产阶段分块改成 MakePlan 选择的 64/128,并让 FP16/FP32 生产与 post 阶段共用 arena。按现有 tiling UT 的 UB=248 KiB、48 核配置核对账本,FP16/BF16 和 FP32 都会选择 128;但完整 tests 目录只有一个实际 kernel ST,输入为 BF16 [65,67],两个轴均不足 128,只能执行尾块。其余 UT 只调用 InferShape/Tiling,也没有断言 tileSize、arena 容量和 post 分区。因此新引入的 128 行、128 列完整块搬运及相邻完整块切换没有任何设备用例或有效结果断言覆盖。 可能导致的结果:若 128 完整块的行跨度、数据搬运长度或共享缓冲区偏移发生错误,现有 [65,67] 尾块测试和主机 UT 仍无法识别,大矩阵的三个归约输出可能在回归时静默出错。 建议的修改方法:补充确认 tileSize=128 的规划/tiling 断言,并增加实际执行 kernel 的 [256,256] 及 [257,259] 等完整块与尾块混合回归;逐项校验 sum_grad_r、sum_grad_c、sum_grad_rc 与 FP64 参考,同时覆盖规划选择 64 的较小 UB 配置。