Star1291
2052
代码
代码
Issues636
Pull Requests579
Actions
项目讨论
Wiki
分析
Star1291
2052
已合并
fix: Conv3DBackpropInputV2 vec 兜底分支 fp32 累加改为补偿累加 #10796
weike创建于 9月20日
fix: Conv3DBackpropInputV2 vec 兜底分支 fp32 累加改为补偿累加 #10796
已合并
weike创建于 9月20日
从m0_55003149/ops-nnfix/conv3d-dx-vec-fp32-compensated-accum合入到cann/ops-nnmaster
共 3 个文件变更+244-21
Mconv/conv3d_backprop_input_v2/op_host/op_tiling/arch22/conv3d_dx_v2_vec_tiling.cpp+7-5
@@ -32,6 +32,7 @@ constexpr uint32_t FP32_BITS = 3;
32constexpr uint32_t UB_BUDGET_DIVISOR = 2; // UB 总容量按 2 分配，留一半给其他缓冲32constexpr uint32_t UB_BUDGET_DIVISOR = 2; // UB 总容量按 2 分配，留一半给其他缓冲
33constexpr uint32_t KB_UNIT = 1024; // 日志中 KB 换算单位33constexpr uint32_t KB_UNIT = 1024; // 日志中 KB 换算单位
34constexpr uint32_t SCALAR_ACC_ROW_BYTES = 8; // rowAcc(FP32) + outH(保守按 FP32) 每元素字节34constexpr uint32_t SCALAR_ACC_ROW_BYTES = 8; // rowAcc(FP32) + outH(保守按 FP32) 每元素字节
35+constexpr uint32_t FP32_VEC_ROW_BYTES = 20; // rowAcc/prod/gradRow/outH/rowComp 各 4B 每元素字节
35constexpr uint32_t DATACOPY_PAD_MAX_ELEMS = 255; // DataCopyPad 单侧填充元素上限（uint8_t）36constexpr uint32_t DATACOPY_PAD_MAX_ELEMS = 255; // DataCopyPad 单侧填充元素上限（uint8_t）
36const size_t FILTER_INDEX = 1;37const size_t FILTER_INDEX = 1;
37const size_t OUT_BACKPROP_INDEX = 2;38const size_t OUT_BACKPROP_INDEX = 2;
@@ -242,10 +243,10 @@ void Conv3DDXV2VecTiling::CalcFp32UbBudget(uint64_t ubSize)
242{243{
243 auto& dx = tilingData_.conv3DDxTiling;244 auto& dx = tilingData_.conv3DDxTiling;
244 const uint64_t ubBudget = ubSize / UB_BUDGET_DIVISOR;245 const uint64_t ubBudget = ubSize / UB_BUDGET_DIVISOR;
245- // FP32 vector path UB budget check（与 kernel InitBuffer 一致的口径）：246+ // FP32 行缓冲 UB 预算（与 kernel InitBuffer 口径一致）：
246- // 向量快路径（CanUseBf16VecRow 同款条件）行缓冲 rowAcc/prod/gradRow/outH = alignedWi*16B；247+ // 向量快路径行缓冲 rowAcc/prod/gradRow/outH/rowComp = alignedWi*20B；
247- // strideW>1 或 pad 超限时降级 ComputeRowScalarAcc，只需 rowAcc/outH = alignedWi*8B；248+ // 降级 ComputeRowScalarAcc 只需 rowAcc/outH = alignedWi*8B；
248- // weightDilated 按 dilatedHk*alignedDilatedW*4B。超限降级为纯标量 ComputeRow。249+ // 再加 weightDilated（dilatedHk*alignedDilatedW*4B）。超限则降级为纯标量 ComputeRow。
249 if (vecDtype_ == ge::DT_FLOAT) {250 if (vecDtype_ == ge::DT_FLOAT) {
250 const uint32_t alignedWi = ((static_cast<uint32_t>(dx.wi) + dx.dataPerBlock - 1) / dx.dataPerBlock) *251 const uint32_t alignedWi = ((static_cast<uint32_t>(dx.wi) + dx.dataPerBlock - 1) / dx.dataPerBlock) *
251 dx.dataPerBlock;252 dx.dataPerBlock;
@@ -256,7 +257,8 @@ void Conv3DDXV2VecTiling::CalcFp32UbBudget(uint64_t ubSize)
256 dx.dilatedWk <= static_cast<uint32_t>(DATACOPY_PAD_MAX_ELEMS) && dx.wi > 0 &&257 dx.dilatedWk <= static_cast<uint32_t>(DATACOPY_PAD_MAX_ELEMS) && dx.wi > 0 &&
257 dx.wo > 0 && static_cast<int64_t>(dx.padLDx) * sizeof(float) <= BYTE_BLOCK &&258 dx.wo > 0 && static_cast<int64_t>(dx.padLDx) * sizeof(float) <= BYTE_BLOCK &&
258 (rightPadMax <= 0 || rightPadMax * sizeof(float) <= BYTE_BLOCK) && alignedWi <= 0xFFFFU;259 (rightPadMax <= 0 || rightPadMax * sizeof(float) <= BYTE_BLOCK) && alignedWi <= 0xFFFFU;
259- uint64_t fp32UbNeed = static_cast<uint64_t>(alignedWi) * (canVecRow ? 16 : 8) +260+ uint64_t fp32UbNeed = static_cast<uint64_t>(alignedWi) *
261+ (canVecRow ? FP32_VEC_ROW_BYTES : SCALAR_ACC_ROW_BYTES) +
260 static_cast<uint64_t>(dx.dilatedHk) * dx.alignedDilatedW * sizeof(float);262 static_cast<uint64_t>(dx.dilatedHk) * dx.alignedDilatedW * sizeof(float);
261 if (fp32UbNeed > ubBudget) {263 if (fp32UbNeed > ubBudget) {
262 OP_LOGI(opName_,264 OP_LOGI(opName_,
Mconv/conv3d_backprop_input_v2/op_kernel/arch22/conv3d_dx_v2_vec_impl.h+118-9
@@ -28,6 +28,9 @@ constexpr int32_t VEC_BLOCK_BYTES = 32;
28constexpr int32_t DATACOPY_PAD_MAX_ELEMS = 255;28constexpr int32_t DATACOPY_PAD_MAX_ELEMS = 255;
29} // namespace29} // namespace
30 30 
31+constexpr uint64_t FP32_VEC_ROW_BYTES = 20ULL;
32+constexpr uint64_t FP32_ROW_UB_LIMIT = 64ULL * 1024ULL;
33+ 
31template <typename T>34template <typename T>
32__aicore__ inline float VecToFloat(T value)35__aicore__ inline float VecToFloat(T value)
33{36{
@@ -150,18 +153,56 @@ private:
150 dataPerBlock == 0) {153 dataPerBlock == 0) {
151 return false;154 return false;
152 }155 }
153- // DataCopyPad 左右填充按字节计不能超过 32B（bf16/fp16 元素上限不够）156+ if constexpr (!std::is_same<T, float>::value) {
154- if (static_cast<int64_t>(padLDx) * sizeof(T) > VEC_BLOCK_BYTES) {157+ // BF16/FP16 维持既有 32B 填充约束
155- return false;158+ if (static_cast<int64_t>(padLDx) * sizeof(T) > VEC_BLOCK_BYTES) {
156- }159+ return false;
157- const int64_t rightPadMax = static_cast<int64_t>(gradInW) - static_cast<int64_t>(gradOutW) +160+ }
158- (static_cast<int64_t>(kernelW) - 1) * dilationW - padLDx;161+ const int64_t rightPadMax = Fp32RightPadMax();
159- if (rightPadMax > 0 && rightPadMax * sizeof(T) > VEC_BLOCK_BYTES) {162+ if (rightPadMax > 0 && rightPadMax * sizeof(T) > VEC_BLOCK_BYTES) {
160- return false;163+ return false;
164+ }
161 }165 }
162 return alignedWi <= 0xFFFFU;166 return alignedWi <= 0xFFFFU;
163 }167 }
164 168 
169+ __aicore__ inline int64_t Fp32RightPadMax() const
170+ {
171+ return static_cast<int64_t>(gradInW) - static_cast<int64_t>(gradOutW) +
172+ (static_cast<int64_t>(kernelW) - 1) * static_cast<int64_t>(dilationW) - static_cast<int64_t>(padLDx);
173+ }
174+ 
175+ __aicore__ inline bool Fp32PadExceedsVecLimit() const
176+ {
177+ if (static_cast<int64_t>(padLDx) * static_cast<int64_t>(sizeof(float)) > VEC_BLOCK_BYTES) {
178+ return true;
179+ }
180+ const int64_t rightPadMax = Fp32RightPadMax();
181+ return rightPadMax > 0 && rightPadMax * static_cast<int64_t>(sizeof(float)) > VEC_BLOCK_BYTES;
182+ }
183+ 
184+ __aicore__ inline bool CanUseFp32VecRow() const
185+ {
186+ if (!CanUseBf16VecRow()) {
187+ return false;
188+ }
189+ return !Fp32PadExceedsVecLimit() || UseDhBlockedAcc();
190+ }
191+ 
192+ __aicore__ inline bool UseDhBlockedAcc() const
193+ {
194+ if constexpr (!std::is_same<T, float>::value) {
195+ return false;
196+ } else {
197+ if (!Fp32PadExceedsVecLimit()) {
198+ return false;
199+ }
200+ const uint64_t need = static_cast<uint64_t>(alignedWi) * (FP32_VEC_ROW_BYTES + sizeof(float)) +
201+ static_cast<uint64_t>(dilatedHk) * alignedDilatedW * sizeof(T);
202+ return need <= FP32_ROW_UB_LIMIT;
203+ }
204+ }
205+ 
165 // 仅 BF16：strideW>1 按相位分解向量化，每相位 tap 仍是一条连续 DataCopyPad206 // 仅 BF16：strideW>1 按相位分解向量化，每相位 tap 仍是一条连续 DataCopyPad
166 __aicore__ inline bool CanUseBf16VecStridedRow() const207 __aicore__ inline bool CanUseBf16VecStridedRow() const
167 {208 {
@@ -260,7 +301,19 @@ private:
260 Muls(prod, prod, wv, alignedWi);301 Muls(prod, prod, wv, alignedWi);
261 SetFlag<HardEvent::V_S>(evtVToS_); // V->S 屏障：Muls 完成后 Add 才能读 prod302 SetFlag<HardEvent::V_S>(evtVToS_); // V->S 屏障：Muls 完成后 Add 才能读 prod
262 WaitFlag<HardEvent::V_S>(evtVToS_);303 WaitFlag<HardEvent::V_S>(evtVToS_);
263- Add(rowAcc, rowAcc, prod, alignedWi);304+ if constexpr (std::is_same<T, float>::value) {
305+ // FP32 用 Kahan 补偿累加，抑制长链舍入漂移
306+ LocalTensor<float> tmpT = gradRow;
307+ LocalTensor<float> tmpE = tmpBufOutH.Get<float>();
308+ LocalTensor<float> comp = tmpBufRowComp.Get<float>();
309+ Add(tmpT, rowAcc, prod, alignedWi);
310+ Sub(tmpE, rowAcc, tmpT, alignedWi);
311+ Add(tmpE, tmpE, prod, alignedWi);
312+ Add(comp, comp, tmpE, alignedWi);
313+ Adds(rowAcc, tmpT, 0.0f, alignedWi);
314+ } else {
315+ Add(rowAcc, rowAcc, prod, alignedWi);
316+ }
264 }317 }
265 318 
266 // FP32 下 Cast 同型不被支持，用 Adds+0 等价拷贝319 // FP32 下 Cast 同型不被支持，用 Adds+0 等价拷贝
@@ -273,6 +326,54 @@ private:
273 }326 }
274 }327 }
275 328 
329+ __aicore__ inline void AccumulateBf16TapBlocked(uint64_t goRowBase, uint32_t dw, float wv,
330+ LocalTensor<float>& rowAcc, LocalTensor<float>& prod,
331+ LocalTensor<T>& gradRow)
332+ {
333+ const int32_t delta = static_cast<int32_t>(dw) - padLDx;
334+ const uint32_t pFirst = delta < 0 ? static_cast<uint32_t>(-delta) : 0U;
335+ const uint32_t m0 = delta > 0 ? static_cast<uint32_t>(delta) : 0U;
336+ if (m0 >= gradOutW || pFirst >= gradInW) {
337+ return;
338+ }
339+ const uint32_t availOut = static_cast<uint32_t>(gradOutW) - m0;
340+ const uint32_t availIn = static_cast<uint32_t>(gradInW) - pFirst;
341+ const uint32_t validLen = (availOut < availIn) ? availOut : availIn;
342+ if (validLen == 0) {
343+ return;
344+ }
345+ const uint32_t dstOffset = (pFirst / dataPerBlock) * dataPerBlock;
346+ const uint32_t leftPad = pFirst - dstOffset;
347+ const uint32_t written = pFirst + validLen;
348+ const uint32_t writtenAligned = ((written + dataPerBlock - 1) / dataPerBlock) * dataPerBlock;
349+ uint32_t rightPad = writtenAligned - written;
350+ if (rightPad > dataPerBlock) {
351+ rightPad = dataPerBlock;
352+ }
353+ 
354+ Duplicate(gradRow, static_cast<T>(0), alignedWi);
355+ SetFlag<HardEvent::V_MTE2>(evtVToMte2_);
356+ WaitFlag<HardEvent::V_MTE2>(evtVToMte2_);
357+ DataCopyExtParams tapParams(1, validLen * sizeof(T), 0, 0, 0);
358+ DataCopyPadExtParams<T> tapPad(true, static_cast<uint8_t>(leftPad), static_cast<uint8_t>(rightPad),
359+ static_cast<T>(0));
360+ DataCopyPad<T>(gradRow[dstOffset], gradOutputGm[goRowBase + m0], tapParams, tapPad);
361+ SetFlag<HardEvent::MTE2_V>(evtMte2ToV_);
362+ WaitFlag<HardEvent::MTE2_V>(evtMte2ToV_);
363+ 
364+ if constexpr (std::is_same<T, float>::value) {
365+ Adds(prod, gradRow, 0.0f, alignedWi);
366+ } else {
367+ Cast(prod, gradRow, RoundMode::CAST_NONE, alignedWi);
368+ }
369+ SetFlag<HardEvent::V_S>(evtVToS_);
370+ WaitFlag<HardEvent::V_S>(evtVToS_);
371+ Muls(prod, prod, wv, alignedWi);
372+ SetFlag<HardEvent::V_S>(evtVToS_);
373+ WaitFlag<HardEvent::V_S>(evtVToS_);
374+ Add(rowAcc, rowAcc, prod, alignedWi);
375+ }
376+ 
276 // 相位分解 tap 累加：与 AccumulateBf16Tap 同构，作用于紧凑类缓冲377 // 相位分解 tap 累加：与 AccumulateBf16Tap 同构，作用于紧凑类缓冲
277 __aicore__ inline void AccumulateBf16TapClass(uint64_t goRowBase, int32_t delta, float wv,378 __aicore__ inline void AccumulateBf16TapClass(uint64_t goRowBase, int32_t delta, float wv,
278 LocalTensor<float>& rowAcc, LocalTensor<float>& prod,379 LocalTensor<float>& rowAcc, LocalTensor<float>& prod,
@@ -336,6 +437,9 @@ private:
336 __aicore__ inline void AccumulateBf16VecRowTaps(uint64_t goPlaneBase, uint32_t hi, LocalTensor<float>& rowAcc,437 __aicore__ inline void AccumulateBf16VecRowTaps(uint64_t goPlaneBase, uint32_t hi, LocalTensor<float>& rowAcc,
337 LocalTensor<float>& prod, LocalTensor<T>& gradRow,438 LocalTensor<float>& prod, LocalTensor<T>& gradRow,
338 LocalTensor<T>& weightDilated);439 LocalTensor<T>& weightDilated);
440+ __aicore__ inline void AccumulateBf16VecRowTapsBlocked(uint64_t goPlaneBase, uint32_t hi,
441+ LocalTensor<float>& rowAcc, LocalTensor<float>& prod,
442+ LocalTensor<T>& gradRow, LocalTensor<T>& weightDilated);
339 __aicore__ inline void AccumulateBf16VecStridedTaps(uint32_t ciLocal, uint32_t coStart, uint32_t coEnd, uint32_t n,443 __aicore__ inline void AccumulateBf16VecStridedTaps(uint32_t ciLocal, uint32_t coStart, uint32_t coEnd, uint32_t n,
340 uint32_t diRow, uint32_t hi, uint32_t kMax, uint32_t kAligned,444 uint32_t diRow, uint32_t hi, uint32_t kMax, uint32_t kAligned,
341 int32_t dw0, int32_t c0, uint32_t sW,445 int32_t dw0, int32_t c0, uint32_t sW,
@@ -349,6 +453,7 @@ private:
349 __aicore__ inline void AccumulateScalarRowTaps(uint64_t goPlaneBase, uint32_t hi, uint64_t rowBase,453 __aicore__ inline void AccumulateScalarRowTaps(uint64_t goPlaneBase, uint32_t hi, uint64_t rowBase,
350 LocalTensor<float>& rowAcc, LocalTensor<T>& weightDilated);454 LocalTensor<float>& rowAcc, LocalTensor<T>& weightDilated);
351 __aicore__ inline void ComputeRowBf16Vec(uint64_t r);455 __aicore__ inline void ComputeRowBf16Vec(uint64_t r);
456+ __aicore__ inline void ComputeRowBf16VecPlain(uint64_t r);
352 __aicore__ inline void ComputeRowBf16VecStrided(uint64_t r);457 __aicore__ inline void ComputeRowBf16VecStrided(uint64_t r);
353 __aicore__ inline void ComputeRowScalarAcc(uint64_t r);458 __aicore__ inline void ComputeRowScalarAcc(uint64_t r);
354 __aicore__ inline void ComputeRow(uint64_t r);459 __aicore__ inline void ComputeRow(uint64_t r);
@@ -364,11 +469,15 @@ private:
364 uint32_t dataPerBlock, alignedWi, classAlignedWi;469 uint32_t dataPerBlock, alignedWi, classAlignedWi;
365 uint32_t vecScalarOnly = 0;470 uint32_t vecScalarOnly = 0;
366 uint32_t useScalarAcc = 0;471 uint32_t useScalarAcc = 0;
472+ uint32_t dhBlocked = 0;
473+ // 标量 UB 累加（UseUbAcc=true）在 T=float 时使用的补偿数组（复用 outH，行末折回 rowAcc）
474+ LocalTensor<float> scalarComp;
367 uint32_t blockDim;475 uint32_t blockDim;
368 476 
369 TPipe pipe;477 TPipe pipe;
370 TBuf<QuePosition::VECCALC> tmpBufWeightDilated;478 TBuf<QuePosition::VECCALC> tmpBufWeightDilated;
371 TBuf<QuePosition::VECCALC> tmpBufRowAcc;479 TBuf<QuePosition::VECCALC> tmpBufRowAcc;
480+ TBuf<QuePosition::VECCALC> tmpBufRowComp; // FP32 补偿缓冲
372 TBuf<QuePosition::VECCALC> tmpBufProd;481 TBuf<QuePosition::VECCALC> tmpBufProd;
373 TBuf<QuePosition::VECCALC> tmpBufGradRow;482 TBuf<QuePosition::VECCALC> tmpBufGradRow;
374 TBuf<QuePosition::VECCALC> tmpBufOutH;483 TBuf<QuePosition::VECCALC> tmpBufOutH;
Mconv/conv3d_backprop_input_v2/op_kernel/arch22/conv3d_dx_v2_vec_impl_row.h+119-7
@@ -61,11 +61,12 @@ __aicore__ inline void KernelConv3dBackpropInputVecImpl<T>::InitBuffers()
61 // FP32：strideW==1 走 W 维向量化快路径，否则降级 ComputeRowScalarAcc61 // FP32：strideW==1 走 W 维向量化快路径，否则降级 ComputeRowScalarAcc
62 if (vecScalarOnly) {62 if (vecScalarOnly) {
63 // 极端大 wi 时 tiling 已降级：仅保留 weightDilated，走 ComputeRow 标量路径63 // 极端大 wi 时 tiling 已降级：仅保留 weightDilated，走 ComputeRow 标量路径
64- } else if (CanUseBf16VecRow()) {64+ } else if (CanUseFp32VecRow()) {
65 pipe.InitBuffer(tmpBufRowAcc, alignedWi * sizeof(float));65 pipe.InitBuffer(tmpBufRowAcc, alignedWi * sizeof(float));
66 pipe.InitBuffer(tmpBufProd, alignedWi * sizeof(float));66 pipe.InitBuffer(tmpBufProd, alignedWi * sizeof(float));
67 pipe.InitBuffer(tmpBufGradRow, alignedWi * sizeof(T));67 pipe.InitBuffer(tmpBufGradRow, alignedWi * sizeof(T));
68 pipe.InitBuffer(tmpBufOutH, alignedWi * sizeof(T));68 pipe.InitBuffer(tmpBufOutH, alignedWi * sizeof(T));
69+ pipe.InitBuffer(tmpBufRowComp, alignedWi * sizeof(float));
69 } else {70 } else {
70 pipe.InitBuffer(tmpBufRowAcc, alignedWi * sizeof(float));71 pipe.InitBuffer(tmpBufRowAcc, alignedWi * sizeof(float));
71 pipe.InitBuffer(tmpBufOutH, alignedWi * sizeof(T));72 pipe.InitBuffer(tmpBufOutH, alignedWi * sizeof(T));
@@ -94,7 +95,7 @@ __aicore__ inline void KernelConv3dBackpropInputVecImpl<T>::ProcessRow(uint64_t
94 ComputeRow(r);95 ComputeRow(r);
95 }96 }
96 } else if (CanUseBf16VecRow()) {97 } else if (CanUseBf16VecRow()) {
97- ComputeRowBf16Vec(r);98+ ComputeRowBf16VecPlain(r);
98 } else if (CanUseBf16VecStridedRow()) {99 } else if (CanUseBf16VecStridedRow()) {
99 ComputeRowBf16VecStrided(r);100 ComputeRowBf16VecStrided(r);
100 } else {101 } else {
@@ -109,7 +110,7 @@ __aicore__ inline void KernelConv3dBackpropInputVecImpl<T>::ProcessRow(uint64_t
109 ComputeRow(r);110 ComputeRow(r);
110 }111 }
111 } else if (CanUseBf16VecRow()) {112 } else if (CanUseBf16VecRow()) {
112- ComputeRowBf16Vec(r);113+ ComputeRowBf16VecPlain(r);
113 } else {114 } else {
114 ComputeRowScalarAcc(r);115 ComputeRowScalarAcc(r);
115 }116 }
@@ -117,8 +118,12 @@ __aicore__ inline void KernelConv3dBackpropInputVecImpl<T>::ProcessRow(uint64_t
117 // FP32：累加顺序与 ComputeRow 一致，向量化无精度变化，避免大 shape 超时118 // FP32：累加顺序与 ComputeRow 一致，向量化无精度变化，避免大 shape 超时
118 if (vecScalarOnly) {119 if (vecScalarOnly) {
119 ComputeRow(r);120 ComputeRow(r);
120- } else if (CanUseBf16VecRow()) {121+ } else if (CanUseFp32VecRow()) {
121- ComputeRowBf16Vec(r);122+ if (UseDhBlockedAcc()) {
123+ ComputeRowBf16Vec(r);
124+ } else {
125+ ComputeRowBf16VecPlain(r);
126+ }
122 } else {127 } else {
123 ComputeRowScalarAcc(r);128 ComputeRowScalarAcc(r);
124 }129 }
@@ -302,7 +307,16 @@ __aicore__ inline void KernelConv3dBackpropInputVecImpl<T>::AccumulateScalarRowT
302 }307 }
303 const float gVal = VecToFloat(gradOutputGm.GetValue(goRowBase + static_cast<uint32_t>(wo)));308 const float gVal = VecToFloat(gradOutputGm.GetValue(goRowBase + static_cast<uint32_t>(wo)));
304 if constexpr (UseUbAcc) {309 if constexpr (UseUbAcc) {
305- rowAcc.SetValue(wi, rowAcc.GetValue(wi) + gVal * wv);310+ if constexpr (std::is_same<T, float>::value) {
311+ const float prod = gVal * wv;
312+ const float acc = rowAcc.GetValue(wi);
313+ const float t = acc + prod;
314+ const float e = (acc - t) + prod;
315+ scalarComp.SetValue(wi, scalarComp.GetValue(wi) + e);
316+ rowAcc.SetValue(wi, t);
317+ } else {
318+ rowAcc.SetValue(wi, rowAcc.GetValue(wi) + gVal * wv);
319+ }
306 } else {320 } else {
307 const float cur = VecToFloat(gradInputGm.GetValue(rowBase + wi));321 const float cur = VecToFloat(gradInputGm.GetValue(rowBase + wi));
308 gradInputGm.SetValue(rowBase + wi, VecFromFloat<T>(cur + gVal * wv));322 gradInputGm.SetValue(rowBase + wi, VecFromFloat<T>(cur + gVal * wv));
@@ -314,7 +328,7 @@ __aicore__ inline void KernelConv3dBackpropInputVecImpl<T>::AccumulateScalarRowT
314 328 
315// BF16/FP16 快路径：整行 W 维向量化累加，行末单次 CAST_RINT329// BF16/FP16 快路径：整行 W 维向量化累加，行末单次 CAST_RINT
316template <typename T>330template <typename T>
317-__aicore__ inline void KernelConv3dBackpropInputVecImpl<T>::ComputeRowBf16Vec(uint64_t r)331+__aicore__ inline void KernelConv3dBackpropInputVecImpl<T>::ComputeRowBf16VecPlain(uint64_t r)
318{332{
319 uint32_t n, ci, diRow, hi;333 uint32_t n, ci, diRow, hi;
320 DecomposeRow(r, n, ci, diRow, hi);334 DecomposeRow(r, n, ci, diRow, hi);
@@ -327,6 +341,9 @@ __aicore__ inline void KernelConv3dBackpropInputVecImpl<T>::ComputeRowBf16Vec(ui
327 LocalTensor<T> weightDilated = tmpBufWeightDilated.Get<T>();341 LocalTensor<T> weightDilated = tmpBufWeightDilated.Get<T>();
328 342 
329 Duplicate(rowAcc, 0.0f, alignedWi);343 Duplicate(rowAcc, 0.0f, alignedWi);
344+ if constexpr (std::is_same<T, float>::value) {
345+ Duplicate(tmpBufRowComp.Get<float>(), 0.0f, alignedWi);
346+ }
330 347 
331 uint32_t ciLocal, coStart, coEnd;348 uint32_t ciLocal, coStart, coEnd;
332 GetGroupRange(ci, ciLocal, coStart, coEnd);349 GetGroupRange(ci, ciLocal, coStart, coEnd);
@@ -344,6 +361,9 @@ __aicore__ inline void KernelConv3dBackpropInputVecImpl<T>::ComputeRowBf16Vec(ui
344 }361 }
345 }362 }
346 363 
364+ if constexpr (std::is_same<T, float>::value) {
365+ Add(rowAcc, rowAcc, tmpBufRowComp.Get<float>(), alignedWi);
366+ }
347 CastRowAccToOutH(rowAcc, outH, alignedWi);367 CastRowAccToOutH(rowAcc, outH, alignedWi);
348 SetFlag<HardEvent::V_MTE3>(evtVToMte3_);368 SetFlag<HardEvent::V_MTE3>(evtVToMte3_);
349 WaitFlag<HardEvent::V_MTE3>(evtVToMte3_);369 WaitFlag<HardEvent::V_MTE3>(evtVToMte3_);
@@ -356,6 +376,91 @@ __aicore__ inline void KernelConv3dBackpropInputVecImpl<T>::ComputeRowBf16Vec(ui
356 WaitFlag<HardEvent::MTE3_V>(evtMte3ToV_);376 WaitFlag<HardEvent::MTE3_V>(evtMte3ToV_);
357}377}
358 378 
379+template <typename T>
380+__aicore__ inline void KernelConv3dBackpropInputVecImpl<T>::ComputeRowBf16Vec(uint64_t r)
381+{
382+ if constexpr (std::is_same<T, float>::value) {
383+ uint32_t n, ci, diRow, hi;
384+ DecomposeRow(r, n, ci, diRow, hi);
385+ const uint64_t rowBase = GetRowBase(n, ci, diRow, hi);
386+ 
387+ LocalTensor<float> rowAcc = tmpBufRowAcc.Get<float>();
388+ LocalTensor<float> prod = tmpBufProd.Get<float>();
389+ LocalTensor<T> gradRow = tmpBufGradRow.Get<T>();
390+ LocalTensor<T> outH = tmpBufOutH.Get<T>();
391+ LocalTensor<T> weightDilated = tmpBufWeightDilated.Get<T>();
392+ 
393+ Duplicate(rowAcc, 0.0f, alignedWi);
394+ dhBlocked = UseDhBlockedAcc() ? 1U : 0U;
395+ if (dhBlocked != 0) {
396+ Duplicate(outH, static_cast<T>(0), alignedWi);
397+ } else {
398+ Duplicate(tmpBufRowComp.Get<float>(), 0.0f, alignedWi);
399+ }
400+ 
401+ uint32_t ciLocal, coStart, coEnd;
402+ GetGroupRange(ci, ciLocal, coStart, coEnd);
403+ for (uint32_t co = coStart; co < coEnd; co++) {
404+ for (uint32_t doIdx = 0; doIdx < gradOutD; doIdx++) {
405+ for (uint32_t dk = 0; dk < kernelD; dk++) {
406+ const int32_t di = static_cast<int32_t>(doIdx) * strideD + static_cast<int32_t>(dk) * dilationD -
407+ padFront;
408+ if (di != static_cast<int32_t>(diRow)) {
409+ continue;
410+ }
411+ BuildWeightDilated(co, ciLocal, dk, weightDilated);
412+ if (dhBlocked != 0) {
413+ AccumulateBf16VecRowTapsBlocked(GetGoPlaneBase(n, co, doIdx), hi, rowAcc, prod, gradRow,
414+ weightDilated);
415+ } else {
416+ AccumulateBf16VecRowTaps(GetGoPlaneBase(n, co, doIdx), hi, rowAcc, prod, gradRow,
417+ weightDilated);
418+ }
419+ }
420+ }
421+ }
422+ 
423+ if (dhBlocked != 0) {
424+ Add(rowAcc, rowAcc, outH, alignedWi);
425+ } else {
426+ Add(rowAcc, rowAcc, tmpBufRowComp.Get<float>(), alignedWi);
427+ }
428+ CastRowAccToOutH(rowAcc, outH, alignedWi);
429+ SetFlag<HardEvent::V_MTE3>(evtVToMte3_);
430+ WaitFlag<HardEvent::V_MTE3>(evtVToMte3_);
431+ 
432+ DataCopyExtParams outParams(1, static_cast<uint32_t>(gradInW) * sizeof(T), 0, 0, 0);
433+ DataCopyPad<T>(gradInputGm[rowBase], outH, outParams);
434+ 
435+ SetFlag<HardEvent::MTE3_V>(evtMte3ToV_);
436+ WaitFlag<HardEvent::MTE3_V>(evtMte3ToV_);
437+ }
438+}
439+ 
440+template <typename T>
441+__aicore__ inline void KernelConv3dBackpropInputVecImpl<T>::AccumulateBf16VecRowTapsBlocked(
442+ uint64_t goPlaneBase, uint32_t hi, LocalTensor<float>& rowAcc, LocalTensor<float>& prod, LocalTensor<T>& gradRow,
443+ LocalTensor<T>& weightDilated)
444+{
445+ for (uint32_t dh = 0; dh < dilatedHk; dh++) {
446+ uint32_t ho;
447+ if (!GetHoIndex(dh, hi, ho)) {
448+ continue;
449+ }
450+ const uint64_t goRowBase = goPlaneBase + static_cast<uint64_t>(ho) * gradOutW;
451+ for (uint32_t dw = 0; dw < dilatedWk; dw++) {
452+ const float wv = VecToFloat(weightDilated.GetValue(dh * alignedDilatedW + dw));
453+ if (wv == 0.0f) {
454+ continue;
455+ }
456+ AccumulateBf16TapBlocked(goRowBase, dw, wv, rowAcc, prod, gradRow);
457+ }
458+ LocalTensor<float> bucket = tmpBufOutH.Get<float>();
459+ Add(bucket, bucket, rowAcc, alignedWi);
460+ Duplicate(rowAcc, 0.0f, alignedWi);
461+ }
462+}
463+ 
359// BF16 strided-W：按相位分解向量化，rowAcc 紧凑类缓冲全程 fp32，行末单次 CAST_RINT464// BF16 strided-W：按相位分解向量化，rowAcc 紧凑类缓冲全程 fp32，行末单次 CAST_RINT
360template <typename T>465template <typename T>
361__aicore__ inline void KernelConv3dBackpropInputVecImpl<T>::ComputeRowBf16VecStrided(uint64_t r)466__aicore__ inline void KernelConv3dBackpropInputVecImpl<T>::ComputeRowBf16VecStrided(uint64_t r)
@@ -418,6 +523,10 @@ __aicore__ inline void KernelConv3dBackpropInputVecImpl<T>::ComputeRowScalarAcc(
418 LocalTensor<T> weightDilated = tmpBufWeightDilated.Get<T>();523 LocalTensor<T> weightDilated = tmpBufWeightDilated.Get<T>();
419 524 
420 Duplicate(rowAcc, 0.0f, alignedWi);525 Duplicate(rowAcc, 0.0f, alignedWi);
526+ if constexpr (std::is_same<T, float>::value) {
527+ scalarComp = outH;
528+ Duplicate(scalarComp, 0.0f, alignedWi);
529+ }
421 // V->S 同步：清零完成后标量才能读 rowAcc530 // V->S 同步：清零完成后标量才能读 rowAcc
422 SetFlag<HardEvent::V_S>(evtVToS_);531 SetFlag<HardEvent::V_S>(evtVToS_);
423 WaitFlag<HardEvent::V_S>(evtVToS_);532 WaitFlag<HardEvent::V_S>(evtVToS_);
@@ -442,6 +551,9 @@ __aicore__ inline void KernelConv3dBackpropInputVecImpl<T>::ComputeRowScalarAcc(
442 SetFlag<HardEvent::S_V>(evtSToV_);551 SetFlag<HardEvent::S_V>(evtSToV_);
443 WaitFlag<HardEvent::S_V>(evtSToV_);552 WaitFlag<HardEvent::S_V>(evtSToV_);
444 553 
554+ if constexpr (std::is_same<T, float>::value) {
555+ Add(rowAcc, rowAcc, scalarComp, alignedWi);
556+ }
445 CastRowAccToOutH(rowAcc, outH, alignedWi);557 CastRowAccToOutH(rowAcc, outH, alignedWi);
446 SetFlag<HardEvent::V_MTE3>(evtVToMte3_);558 SetFlag<HardEvent::V_MTE3>(evtVToMte3_);
447 WaitFlag<HardEvent::V_MTE3>(evtVToMte3_);559 WaitFlag<HardEvent::V_MTE3>(evtVToMte3_);