Star908
1826
代码
代码
Issues226
Pull Requests308
Actions
项目讨论
Wiki
分析
Star908
1826
已合并
Conv:AscendC Api整改 #9114
高维健创建于 13 天前
Conv:AscendC Api整改 #9114
已合并
高维健创建于 13 天前
从weijian317/ops-nnasc合入到cann/ops-nnmaster
共 10 个文件变更+354-427
Mconv/common/op_kernel/arch35/conv_instr_hw_mode_impl.h+7-6
@@ -194,7 +194,7 @@ public:
194 currentNL0_ = n;194 currentNL0_ = n;
195 }195 }
196 196 
197- __aicore__ inline void SetFixpipeIntriParams(FixpipeParamsC310<CO2Layout::COLUMN_MAJOR>& intriParams)197+ __aicore__ inline void SetFixpipeIntriParams(FixpipeParamsArch3510<CO2Layout::COLUMN_MAJOR>& intriParams)
198 {198 {
199 if constexpr (Intf::isDmaFlag) {199 if constexpr (Intf::isDmaFlag) {
200 intriParams.mSize = self_->ctx.currentWoL0;200 intriParams.mSize = self_->ctx.currentWoL0;
@@ -226,7 +226,7 @@ public:
226 SetBaseParams<CO2Layout::COLUMN_MAJOR>(intriParams);226 SetBaseParams<CO2Layout::COLUMN_MAJOR>(intriParams);
227 }227 }
228 228 
229- __aicore__ inline void SetFixpipeIntriParamsHWC(FixpipeParamsC310<CO2Layout::ROW_MAJOR>& intriParams)229+ __aicore__ inline void SetFixpipeIntriParamsHWC(FixpipeParamsArch3510<CO2Layout::ROW_MAJOR>& intriParams)
230 {230 {
231 if constexpr (Intf::isDmaFlag) {231 if constexpr (Intf::isDmaFlag) {
232 intriParams.mSize = self_->ctx.currentWoL0;232 intriParams.mSize = self_->ctx.currentWoL0;
@@ -252,7 +252,8 @@ public:
252 }252 }
253 253 
254 template <CO2Layout format>254 template <CO2Layout format>
255- __aicore__ inline void SetFixpipeIntriParamsUb(FixpipeParamsC310<format>& intriParams, CopyUbInfo* ubInfo = nullptr)255+ __aicore__ inline void SetFixpipeIntriParamsUb(FixpipeParamsArch3510<format>& intriParams,
256+ CopyUbInfo* ubInfo = nullptr)
256 {257 {
257 if (ubInfo == nullptr) {258 if (ubInfo == nullptr) {
258 return;259 return;
@@ -314,7 +315,7 @@ public:
314 }315 }
315 316 
316 template <CO2Layout format>317 template <CO2Layout format>
317- __aicore__ inline void SetBaseParams(FixpipeParamsC310<format>& intriParams)318+ __aicore__ inline void SetBaseParams(FixpipeParamsArch3510<format>& intriParams)
318 {319 {
319 intriParams.quantPre = GetQuantPre<Intf, OutputT, FixpipeIdx>(self_);320 intriParams.quantPre = GetQuantPre<Intf, OutputT, FixpipeIdx>(self_);
320 if (self_->ctx.convTilingData->hasScale == 0) {321 if (self_->ctx.convTilingData->hasScale == 0) {
@@ -410,7 +411,7 @@ public:
410 411 
411 template <template <typename> class TensorTypeT, const FixpipeConfig& config>412 template <template <typename> class TensorTypeT, const FixpipeConfig& config>
412 __aicore__ inline void ExtendConv2DFixpipe(const TensorTypeT<OutputT>& output,413 __aicore__ inline void ExtendConv2DFixpipe(const TensorTypeT<OutputT>& output,
413- FixpipeParamsC310<config.format>& intriParams, uint64_t offset)414+ FixpipeParamsArch3510<config.format>& intriParams, uint64_t offset)
414 {415 {
415 if (self_->ctx.enableVectorQuant) {416 if (self_->ctx.enableVectorQuant) {
416 if constexpr (FixpipeIdx == 0) {417 if constexpr (FixpipeIdx == 0) {
@@ -446,7 +447,7 @@ public:
446 offset = CalcFixpipeOffset();447 offset = CalcFixpipeOffset();
447 }448 }
448 449 
449- FixpipeParamsC310<config.format> intriParams;450+ FixpipeParamsArch3510<config.format> intriParams;
450#if defined(__DAV_35_FAMILY__)451#if defined(__DAV_35_FAMILY__)
451 if constexpr (Intf::isFixedPoint) {452 if constexpr (Intf::isFixedPoint) {
452 intriParams.fixShiftVal = FIX_SHIFT_VAL_LEN_A16W16 - self_->ctx.convTilingData->fixedShiftValue;453 intriParams.fixShiftVal = FIX_SHIFT_VAL_LEN_A16W16 - self_->ctx.convTilingData->fixedShiftValue;
Mconv/common/op_kernel/arch35/conv_instr_m_mode_impl.h+10-9
@@ -186,7 +186,7 @@ public:
186 if constexpr (Intf::isInnerBatchFlag) {186 if constexpr (Intf::isInnerBatchFlag) {
187 CopyOutInnerBatch<TensorTypeT, config.format, config>(output, offset, ubInfo);187 CopyOutInnerBatch<TensorTypeT, config.format, config>(output, offset, ubInfo);
188 } else {188 } else {
189- FixpipeParamsC310<config.format> intriParams;189+ FixpipeParamsArch3510<config.format> intriParams;
190#if defined(__DAV_35_FAMILY__)190#if defined(__DAV_35_FAMILY__)
191 if constexpr (Intf::isFixedPoint) {191 if constexpr (Intf::isFixedPoint) {
192 intriParams.fixShiftVal = FIX_SHIFT_VAL_LEN_A16W16 - self_->ctx.convTilingData->fixedShiftValue;192 intriParams.fixShiftVal = FIX_SHIFT_VAL_LEN_A16W16 - self_->ctx.convTilingData->fixedShiftValue;
@@ -223,7 +223,7 @@ private:
223 __aicore__ inline void CopyOutInnerBatch(const TensorTypeT<OutputT>& output, uint64_t offset,223 __aicore__ inline void CopyOutInnerBatch(const TensorTypeT<OutputT>& output, uint64_t offset,
224 CopyUbInfo* ubInfo = nullptr)224 CopyUbInfo* ubInfo = nullptr)
225 {225 {
226- FixpipeParamsC310<format> intriParams;226+ FixpipeParamsArch3510<format> intriParams;
227#if defined(__DAV_35_FAMILY__)227#if defined(__DAV_35_FAMILY__)
228 if constexpr (Intf::isFixedPoint) {228 if constexpr (Intf::isFixedPoint) {
229 intriParams.fixShiftVal = FIX_SHIFT_VAL_LEN_A16W16 - self_->ctx.convTilingData->fixedShiftValue;229 intriParams.fixShiftVal = FIX_SHIFT_VAL_LEN_A16W16 - self_->ctx.convTilingData->fixedShiftValue;
@@ -255,7 +255,7 @@ private:
255 }255 }
256 }256 }
257 257 
258- __aicore__ inline void InnerBatchParamsCHW(FixpipeParamsC310<CO2Layout::COLUMN_MAJOR>& intriParams)258+ __aicore__ inline void InnerBatchParamsCHW(FixpipeParamsArch3510<CO2Layout::COLUMN_MAJOR>& intriParams)
259 {259 {
260 intriParams.mSize = currentML0_;260 intriParams.mSize = currentML0_;
261 intriParams.nSize = currentNL0_;261 intriParams.nSize = currentNL0_;
@@ -278,7 +278,7 @@ private:
278 intriParams.dstStride = valueHoWo_;278 intriParams.dstStride = valueHoWo_;
279 }279 }
280 280 
281- __aicore__ inline void InnerBatchParamsHWC(FixpipeParamsC310<CO2Layout::ROW_MAJOR>& intriParams)281+ __aicore__ inline void InnerBatchParamsHWC(FixpipeParamsArch3510<CO2Layout::ROW_MAJOR>& intriParams)
282 {282 {
283 intriParams.mSize = currentML0_;283 intriParams.mSize = currentML0_;
284 intriParams.nSize = currentNL0_;284 intriParams.nSize = currentNL0_;
@@ -294,7 +294,7 @@ private:
294 intriParams.dstStride = self_->ctx.convTilingData->orgCo;294 intriParams.dstStride = self_->ctx.convTilingData->orgCo;
295 }295 }
296 296 
297- __aicore__ inline void SetFixpipeIntriParamsHWC(FixpipeParamsC310<CO2Layout::ROW_MAJOR>& intriParams)297+ __aicore__ inline void SetFixpipeIntriParamsHWC(FixpipeParamsArch3510<CO2Layout::ROW_MAJOR>& intriParams)
298 {298 {
299 intriParams.nSize = currentNL0_;299 intriParams.nSize = currentNL0_;
300 intriParams.mSize = currentML0_;300 intriParams.mSize = currentML0_;
@@ -306,7 +306,7 @@ private:
306 SetBaseParams<CO2Layout::ROW_MAJOR>(intriParams);306 SetBaseParams<CO2Layout::ROW_MAJOR>(intriParams);
307 }307 }
308 308 
309- __aicore__ inline void SetFixpipeIntriParams(FixpipeParamsC310<CO2Layout::COLUMN_MAJOR>& intriParams)309+ __aicore__ inline void SetFixpipeIntriParams(FixpipeParamsArch3510<CO2Layout::COLUMN_MAJOR>& intriParams)
310 {310 {
311 intriParams.nSize = currentNL0_;311 intriParams.nSize = currentNL0_;
312 intriParams.mSize = currentML0_;312 intriParams.mSize = currentML0_;
@@ -355,7 +355,7 @@ private:
355 }355 }
356 356 
357 template <CO2Layout format>357 template <CO2Layout format>
358- __aicore__ inline void SetBaseParams(FixpipeParamsC310<format>& intriParams)358+ __aicore__ inline void SetBaseParams(FixpipeParamsArch3510<format>& intriParams)
359 {359 {
360 intriParams.quantPre = GetQuantPre<Intf, OutputT, FixpipeIdx>(self_);360 intriParams.quantPre = GetQuantPre<Intf, OutputT, FixpipeIdx>(self_);
361 if (self_->ctx.convTilingData->hasScale == 0) {361 if (self_->ctx.convTilingData->hasScale == 0) {
@@ -408,7 +408,8 @@ private:
408 }408 }
409 409 
410 template <CO2Layout format>410 template <CO2Layout format>
411- __aicore__ inline void SetFixpipeIntriParamsUb(FixpipeParamsC310<format>& intriParams, CopyUbInfo* ubInfo = nullptr)411+ __aicore__ inline void SetFixpipeIntriParamsUb(FixpipeParamsArch3510<format>& intriParams,
412+ CopyUbInfo* ubInfo = nullptr)
412 {413 {
413 if (ubInfo == nullptr) {414 if (ubInfo == nullptr) {
414 return;415 return;
@@ -500,7 +501,7 @@ private:
500 501 
501 template <template <typename> class TensorTypeT, const FixpipeConfig& config>502 template <template <typename> class TensorTypeT, const FixpipeConfig& config>
502 __aicore__ inline void ExtendConv2DFixpipe(const TensorTypeT<OutputT>& output,503 __aicore__ inline void ExtendConv2DFixpipe(const TensorTypeT<OutputT>& output,
503- FixpipeParamsC310<config.format>& intriParams, uint64_t offset)504+ FixpipeParamsArch3510<config.format>& intriParams, uint64_t offset)
504 {505 {
505 if (!self_->ctx.enableVectorQuant) {506 if (!self_->ctx.enableVectorQuant) {
506 Fixpipe<OutputT, typename Intf::L0cT, config>(output[offset], self_->ctx.cl0, intriParams);507 Fixpipe<OutputT, typename Intf::L0cT, config>(output[offset], self_->ctx.cl0, intriParams);
Aconv/common/op_kernel/arch35/conv_instr_nd2nz_vf.h+166-0
@@ -0,0 +1,166 @@
1+/**
2+ * Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+ * This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+ * CANN Open Software License Agreement Version 2.0 (the "License").
5+ * Please refer to the License for details. You may not use this file except in compliance with the License.
6+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+ * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+ * See LICENSE in the root of the software repository for the full text of the License.
9+ */
10+ 
11+/*!
12+ * \file conv_instr_nd2nz_vf.h
13+ * \brief ND2NZ vector function helpers shared by conv weight trans kernels.
14+ */
15+ 
16+#ifndef CONV_INSTR_ND2NZ_VF_H
17+#define CONV_INSTR_ND2NZ_VF_H
18+ 
19+#include "conv_util.h"
20+ 
21+namespace conv {
22+using namespace AscendC;
23+ 
24+template <typename SrcT, typename DstT, typename IndexT>
25+struct TransND2NZVfParams {
26+ uint16_t ciLoopTimes;
27+ uint16_t khkwLoopTimes;
28+ uint16_t coLoopTimes;
29+ uint32_t srcCiStride;
30+ uint32_t srcKhKwStride;
31+ uint32_t srcCoStride;
32+ uint32_t dstCiStride;
33+ uint32_t dstKhKwStride;
34+ uint32_t dstCoStride;
35+ __ubuf__ SrcT* srcAddr;
36+ __ubuf__ DstT* dstAddr;
37+ __ubuf__ IndexT* indexAddr;
38+};
39+ 
40+struct TransND2NZKdVfParams {
41+ uint16_t kdLoopTimes;
42+ uint32_t srcKdStride;
43+ uint32_t dstKdStride;
44+};
45+ 
46+template <typename T, typename IndexT>
47+struct TransFractalZVfParams {
48+ uint16_t kLoopTimes;
49+ uint16_t nLoopTimes;
50+ uint32_t srcKStride;
51+ uint32_t srcNStride;
52+ uint32_t dstKStride;
53+ uint32_t dstNStride;
54+ __ubuf__ T* srcAddr;
55+ __ubuf__ T* dstAddr;
56+ __ubuf__ IndexT* indexAddr;
57+};
58+ 
59+template <typename IndexT>
60+__simd_vf__ inline void SetIndexVf(__ubuf__ IndexT* indexAddr, uint16_t repeatTimes, IndexT nStride, uint8_t k0)
61+{
62+ Reg::RegTensor<IndexT> indexReg;
63+ Reg::LoadAlign<IndexT>(indexReg, indexAddr);
64+ uint32_t maskL = k0;
65+ Reg::MaskReg maskReg = Reg::UpdateMask<IndexT>(maskL);
66+ 
67+ uint8_t dstOffset = k0;
68+ uint8_t elesPerRepeat = k0;
69+ for (uint16_t repeat = 0; repeat < repeatTimes; ++repeat) {
70+ Reg::Adds<IndexT, IndexT>(indexReg, indexReg, nStride, maskReg);
71+ Reg::StoreAlign<IndexT>(indexAddr + dstOffset, indexReg, maskReg);
72+ dstOffset += elesPerRepeat;
73+ }
74+}
75+ 
76+template <typename SrcT, typename DstT, typename RegT, typename IndexT, bool isQuantScene>
77+__simd_callee__ inline void GatherStoreNz(__ubuf__ SrcT* srcPtr, __ubuf__ DstT* dstPtr,
78+ Reg::RegTensor<IndexT>& indexReg, Reg::MaskReg& gatherMaskReg,
79+ Reg::MaskReg& vstsMaskReg)
80+{
81+ Reg::RegTensor<RegT> gatherReg;
82+ Reg::Gather<RegT, SrcT, IndexT>(gatherReg, srcPtr, indexReg, gatherMaskReg);
83+ if constexpr (isQuantScene) {
84+ // Remove the higher zeros of the int16_t data gathered by the Micro Gather instr
85+ Reg::Pack<uint8_t, RegT, Reg::HighLowPart::LOWEST>((Reg::RegTensor<uint8_t>&)gatherReg, gatherReg);
86+ }
87+ Reg::StoreAlign<DstT>(dstPtr, (Reg::RegTensor<DstT>&)gatherReg, vstsMaskReg);
88+}
89+ 
90+template <typename SrcT, typename DstT, typename RegT, typename IndexT, bool isQuantScene>
91+__simd_vf__ inline void TransND2NZVf(const TransND2NZVfParams<SrcT, DstT, IndexT> params)
92+{
93+ Reg::RegTensor<IndexT> indexReg;
94+ Reg::MaskReg gatherMaskReg = Reg::CreateMask<RegT, Reg::MaskPattern::ALL>();
95+ Reg::MaskReg vstsMaskReg;
96+ if constexpr (isQuantScene) {
97+ vstsMaskReg = Reg::CreateMask<DstT, Reg::MaskPattern::H>();
98+ } else {
99+ vstsMaskReg = Reg::CreateMask<DstT, Reg::MaskPattern::ALL>();
100+ }
101+ Reg::LoadAlign<IndexT>(indexReg, params.indexAddr);
102+ 
103+ for (uint16_t ci1OptIndex = 0; ci1OptIndex < params.ciLoopTimes; ++ci1OptIndex) {
104+ for (uint16_t khkwIndex = 0; khkwIndex < params.khkwLoopTimes; ++khkwIndex) {
105+ for (uint16_t coOptIndex = 0; coOptIndex < params.coLoopTimes; ++coOptIndex) {
106+ uint32_t srcOffset = ci1OptIndex * params.srcCiStride + khkwIndex * params.srcKhKwStride +
107+ coOptIndex * params.srcCoStride;
108+ uint32_t dstOffset = ci1OptIndex * params.dstCiStride + khkwIndex * params.dstKhKwStride +
109+ coOptIndex * params.dstCoStride;
110+ GatherStoreNz<SrcT, DstT, RegT, IndexT, isQuantScene>(
111+ params.srcAddr + srcOffset, params.dstAddr + dstOffset, indexReg, gatherMaskReg, vstsMaskReg);
112+ }
113+ }
114+ }
115+}
116+ 
117+template <typename SrcT, typename DstT, typename RegT, typename IndexT, bool isQuantScene>
118+__simd_vf__ inline void TransND2NZKdVf(const TransND2NZVfParams<SrcT, DstT, IndexT> params,
119+ const TransND2NZKdVfParams kdParams)
120+{
121+ Reg::RegTensor<IndexT> indexReg;
122+ Reg::MaskReg gatherMaskReg = Reg::CreateMask<RegT, Reg::MaskPattern::ALL>();
123+ Reg::MaskReg vstsMaskReg;
124+ if constexpr (isQuantScene) {
125+ vstsMaskReg = Reg::CreateMask<DstT, Reg::MaskPattern::H>();
126+ } else {
127+ vstsMaskReg = Reg::CreateMask<DstT, Reg::MaskPattern::ALL>();
128+ }
129+ Reg::LoadAlign<IndexT>(indexReg, params.indexAddr);
130+ 
131+ for (uint16_t kdIndex = 0; kdIndex < kdParams.kdLoopTimes; ++kdIndex) {
132+ for (uint16_t ci1OptIndex = 0; ci1OptIndex < params.ciLoopTimes; ++ci1OptIndex) {
133+ for (uint16_t khkwIndex = 0; khkwIndex < params.khkwLoopTimes; ++khkwIndex) {
134+ for (uint16_t coOptIndex = 0; coOptIndex < params.coLoopTimes; ++coOptIndex) {
135+ uint32_t srcOffset = kdIndex * kdParams.srcKdStride + ci1OptIndex * params.srcCiStride +
136+ khkwIndex * params.srcKhKwStride + coOptIndex * params.srcCoStride;
137+ uint32_t dstOffset = kdIndex * kdParams.dstKdStride + ci1OptIndex * params.dstCiStride +
138+ khkwIndex * params.dstKhKwStride + coOptIndex * params.dstCoStride;
139+ GatherStoreNz<SrcT, DstT, RegT, IndexT, isQuantScene>(
140+ params.srcAddr + srcOffset, params.dstAddr + dstOffset, indexReg, gatherMaskReg, vstsMaskReg);
141+ }
142+ }
143+ }
144+ }
145+}
146+ 
147+template <typename T, typename IndexT>
148+__simd_vf__ inline void TransFractalZVf(const TransFractalZVfParams<T, IndexT> params)
149+{
150+ Reg::RegTensor<IndexT> indexReg;
151+ Reg::MaskReg maskReg = Reg::CreateMask<T, Reg::MaskPattern::ALL>();
152+ Reg::LoadAlign<IndexT>(indexReg, params.indexAddr);
153+ 
154+ for (uint16_t kIndex = 0; kIndex < params.kLoopTimes; ++kIndex) {
155+ for (uint16_t nIndex = 0; nIndex < params.nLoopTimes; ++nIndex) {
156+ uint32_t srcOffset = kIndex * params.srcKStride + nIndex * params.srcNStride;
157+ uint32_t dstOffset = kIndex * params.dstKStride + nIndex * params.dstNStride;
158+ GatherStoreNz<T, T, T, IndexT, false>(params.srcAddr + srcOffset, params.dstAddr + dstOffset, indexReg,
159+ maskReg, maskReg);
160+ }
161+ }
162+}
163+ 
164+} // namespace conv
165+ 
166+#endif // CONV_INSTR_ND2NZ_VF_H
Mconv/common/op_kernel/arch35/conv_instr_opt_group_impl.h+52-230
@@ -18,6 +18,7 @@
18 18 
19#include "conv_config.h"19#include "conv_config.h"
20#include "conv_util.h"20#include "conv_util.h"
21+#include "conv_instr_nd2nz_vf.h"
21 22 
22namespace ConvFunc {23namespace ConvFunc {
23using namespace AscendC;24using namespace AscendC;
@@ -116,8 +117,8 @@ private:
116 117 
117private:118private:
118 Intf* self_ = nullptr;119 Intf* self_ = nullptr;
119- MultiCopyParams<typename Intf::WeightT, NDDMA_DIMS> copyParams;120+ NdDmaParams<typename Intf::WeightT, NDDMA_DIMS> copyParams;
120- MultiCopyParams<typename Intf::WeightT, NDDMA_HWC_DIMS> copyParamsHWC;121+ NdDmaParams<typename Intf::WeightT, NDDMA_HWC_DIMS> copyParamsHWC;
121 LocalTensor<typename Intf::WeightT> ndTensor;122 LocalTensor<typename Intf::WeightT> ndTensor;
122};123};
123 124 
@@ -176,259 +177,67 @@ private:
176 SetFlag<HardEvent::S_V>(eventId);177 SetFlag<HardEvent::S_V>(eventId);
177 WaitFlag<HardEvent::S_V>(eventId);178 WaitFlag<HardEvent::S_V>(eventId);
178 179 
179- __local_mem__ IndexT* indexAddr = (__local_mem__ IndexT*)indexTensor.GetPhyAddr();180+ __ubuf__ IndexT* indexAddr = (__ubuf__ IndexT*)indexTensor.GetPhyAddr();
180 uint16_t repeatTimes = static_cast<uint16_t>(REG_SIZE / sizeof(IndexT) / Intf::k0 - 1);181 uint16_t repeatTimes = static_cast<uint16_t>(REG_SIZE / sizeof(IndexT) / Intf::k0 - 1);
181- uint8_t dstOffset = Intf::k0;
182- uint8_t elesPerRepeat = Intf::k0;
183- uint32_t maskL = Intf::k0;
184 IndexT nStride;182 IndexT nStride;
185 if constexpr (Intf::formatOutput == ConvFormat::NCDHW || Intf::formatOutput == ConvFormat::NCHW) {183 if constexpr (Intf::formatOutput == ConvFormat::NCDHW || Intf::formatOutput == ConvFormat::NCHW) {
186 nStride = static_cast<IndexT>(self_->ctx.kUbSize);184 nStride = static_cast<IndexT>(self_->ctx.kUbSize);
187 } else {185 } else {
188 nStride = static_cast<IndexT>(1);186 nStride = static_cast<IndexT>(1);
189 }187 }
190- 188+ SetIndexVf(indexAddr, repeatTimes, nStride, static_cast<uint8_t>(Intf::k0));
191- __VEC_SCOPE__
192- {
193- MicroAPI::RegTensor<IndexT> indexReg;
194- MicroAPI::DataCopy<IndexT>(indexReg, indexAddr);
195- MicroAPI::MaskReg maskReg = MicroAPI::UpdateMask<IndexT>(maskL);
196- 
197- for (uint16_t repeat = 0; repeat < repeatTimes; ++repeat) {
198- MicroAPI::Adds<IndexT, IndexT>(indexReg, indexReg, nStride, maskReg);
199- MicroAPI::DataCopy<IndexT>(indexAddr + dstOffset, indexReg, maskReg);
200- dstOffset += elesPerRepeat;
201- }
202- }
203 }189 }
204 190 
205 __aicore__ inline void TransNCHW2NZ()191 __aicore__ inline void TransNCHW2NZ()
206 {192 {
207- uint16_t ciLoopTimes = self_->ctx.ci1Opt;193+ TransND2NZVfParams<SrcT, DstT, IndexT> params;
208- uint16_t coLoopTimes = coOptLoopTimes;194+ FillCommonNd2NzParams(params);
209- uint16_t khkwLoopTimes = self_->ctx.convTilingData->kernelHxkernelW;195+ params.srcCiStride = self_->ctx.convTilingData->kernelHxkernelW * Intf::k0;
210- uint32_t srcCiStride = self_->ctx.convTilingData->kernelHxkernelW * Intf::k0;196+ params.srcKhKwStride = 1;
211- uint32_t srcCoStride = coPerReg * self_->ctx.kUbSize;197+ params.srcCoStride = coPerReg * self_->ctx.kUbSize;
212- uint32_t dstCiStride = self_->ctx.convTilingData->kernelHxkernelW * Intf::k0 * self_->ctx.coOptAlign;198+ TransND2NZVf<SrcT, DstT, RegT, IndexT, Intf::isQuantScene>(params);
213- uint32_t dstKhKwStride = Intf::k0 * self_->ctx.coOptAlign;
214- uint32_t dstCoStride = coPerReg * Intf::k0;
215- 
216- __VEC_SCOPE__
217- {
218- MicroAPI::RegTensor<RegT> gatherReg;
219- MicroAPI::RegTensor<IndexT> indexReg;
220- MicroAPI::MaskReg gatherMaskReg = MicroAPI::CreateMask<RegT, MicroAPI::MaskPattern::ALL>();
221- MicroAPI::MaskReg vstsMaskReg;
222- if constexpr (Intf::isQuantScene) {
223- vstsMaskReg = MicroAPI::CreateMask<DstT, MicroAPI::MaskPattern::H>();
224- } else {
225- vstsMaskReg = MicroAPI::CreateMask<DstT, MicroAPI::MaskPattern::ALL>();
226- }
227- 
228- __local_mem__ SrcT* srcAddr = (__local_mem__ SrcT*)ndTensor.GetPhyAddr();
229- __local_mem__ DstT* dstAddr = (__local_mem__ DstT*)self_->ctx.nzTensor.GetPhyAddr();
230- __local_mem__ IndexT* indexAddr = (__local_mem__ IndexT*)indexTensor.GetPhyAddr();
231- 
232- MicroAPI::DataCopy<IndexT>(indexReg, indexAddr);
233- 
234- for (uint16_t ci1OptIndex = 0; ci1OptIndex < ciLoopTimes; ++ci1OptIndex) {
235- for (uint16_t khkwIndex = 0; khkwIndex < khkwLoopTimes; ++khkwIndex) {
236- for (uint16_t coOptIndex = 0; coOptIndex < coLoopTimes; ++coOptIndex) {
237- uint32_t srcOffset = ci1OptIndex * srcCiStride + khkwIndex + coOptIndex * srcCoStride;
238- uint32_t dstOffset = ci1OptIndex * dstCiStride + khkwIndex * dstKhKwStride +
239- coOptIndex * dstCoStride;
240- 
241- MicroAPI::DataCopyGather<RegT, SrcT, IndexT>(gatherReg, srcAddr + srcOffset, indexReg,
242- gatherMaskReg);
243- 
244- if constexpr (Intf::isQuantScene) {
245- // Remove the higher zeros of the int16_t data gathered by the Micro Gather instr
246- MicroAPI::Pack<uint8_t, RegT, MicroAPI::HighLowPart::LOWEST>(
247- (MicroAPI::RegTensor<uint8_t>&)gatherReg, gatherReg);
248- MicroAPI::DataCopy<DstT>(dstAddr + dstOffset, (MicroAPI::RegTensor<DstT>&)gatherReg,
249- vstsMaskReg);
250- } else {
251- MicroAPI::DataCopy<DstT>(dstAddr + dstOffset, (MicroAPI::RegTensor<DstT>&)gatherReg,
252- vstsMaskReg);
253- }
254- }
255- }
256- }
257- }
258 }199 }
259 200 
260 __aicore__ inline void TransNCDHW2NZ()201 __aicore__ inline void TransNCDHW2NZ()
261 {202 {
262- uint16_t kdLoopTimes = self_->ctx.convTilingData->kernelD;203+ TransND2NZVfParams<SrcT, DstT, IndexT> params;
263- uint16_t ciLoopTimes = self_->ctx.ci1Opt;204+ FillCommonNd2NzParams(params);
264- uint16_t coLoopTimes = coOptLoopTimes;205+ params.srcCiStride = self_->ctx.convTilingData->kernelHxkernelWxkernelD * Intf::k0;
265- uint16_t khkwLoopTimes = self_->ctx.convTilingData->kernelHxkernelW;206+ params.srcKhKwStride = 1;
266- uint32_t srcCiStride = self_->ctx.convTilingData->kernelHxkernelWxkernelD * Intf::k0;207+ params.srcCoStride = coPerReg * self_->ctx.kUbSize;
267- uint32_t srcCoStride = coPerReg * self_->ctx.kUbSize;
268- uint32_t dstCiStride = self_->ctx.convTilingData->kernelHxkernelW * Intf::k0 * self_->ctx.coOptAlign;
269- uint32_t dstKhKwStride = Intf::k0 * self_->ctx.coOptAlign;
270- uint32_t dstCoStride = coPerReg * Intf::k0;
271- uint32_t srcKdStride = self_->ctx.convTilingData->kernelHxkernelW;
272- uint32_t dstKdStride = self_->ctx.ci1Opt * dstCiStride;
273 208 
274- __VEC_SCOPE__209+ TransND2NZKdVfParams kdParams;
275- {210+ kdParams.kdLoopTimes = self_->ctx.convTilingData->kernelD;
276- MicroAPI::RegTensor<RegT> gatherReg;211+ kdParams.srcKdStride = self_->ctx.convTilingData->kernelHxkernelW;
277- MicroAPI::RegTensor<IndexT> indexReg;212+ kdParams.dstKdStride = self_->ctx.ci1Opt * params.dstCiStride;
278- MicroAPI::MaskReg gatherMaskReg = MicroAPI::CreateMask<RegT, MicroAPI::MaskPattern::ALL>();213+ TransND2NZKdVf<SrcT, DstT, RegT, IndexT, Intf::isQuantScene>(params, kdParams);
279- MicroAPI::MaskReg vstsMaskReg;
280- if constexpr (Intf::isQuantScene) {
281- vstsMaskReg = MicroAPI::CreateMask<DstT, MicroAPI::MaskPattern::H>();
282- } else {
283- vstsMaskReg = MicroAPI::CreateMask<DstT, MicroAPI::MaskPattern::ALL>();
284- }
285- 
286- __local_mem__ SrcT* srcAddr = (__local_mem__ SrcT*)ndTensor.GetPhyAddr();
287- __local_mem__ DstT* dstAddr = (__local_mem__ DstT*)self_->ctx.nzTensor.GetPhyAddr();
288- __local_mem__ IndexT* indexAddr = (__local_mem__ IndexT*)indexTensor.GetPhyAddr();
289- 
290- MicroAPI::DataCopy<IndexT>(indexReg, indexAddr);
291- 
292- for (uint16_t kdIndex = 0; kdIndex < kdLoopTimes; ++kdIndex) {
293- for (uint16_t ci1OptIndex = 0; ci1OptIndex < ciLoopTimes; ++ci1OptIndex) {
294- for (uint16_t khkwIndex = 0; khkwIndex < khkwLoopTimes; ++khkwIndex) {
295- for (uint16_t coOptIndex = 0; coOptIndex < coLoopTimes; ++coOptIndex) {
296- uint32_t srcOffset = kdIndex * srcKdStride + ci1OptIndex * srcCiStride + khkwIndex +
297- coOptIndex * srcCoStride;
298- uint32_t dstOffset = kdIndex * dstKdStride + ci1OptIndex * dstCiStride +
299- khkwIndex * dstKhKwStride + coOptIndex * dstCoStride;
300- 
301- MicroAPI::DataCopyGather<RegT, SrcT, IndexT>(gatherReg, srcAddr + srcOffset, indexReg,
302- gatherMaskReg);
303- 
304- if constexpr (Intf::isQuantScene) {
305- MicroAPI::Pack<uint8_t, RegT, MicroAPI::HighLowPart::LOWEST>(
306- (MicroAPI::RegTensor<uint8_t>&)gatherReg, gatherReg);
307- MicroAPI::DataCopy<DstT>(dstAddr + dstOffset, (MicroAPI::RegTensor<DstT>&)gatherReg,
308- vstsMaskReg);
309- } else {
310- MicroAPI::DataCopy<DstT>(dstAddr + dstOffset, gatherReg, vstsMaskReg);
311- }
312- }
313- }
314- }
315- }
316- }
317 }214 }
318 215 
319 __aicore__ inline void TransNDHWC2NZ()216 __aicore__ inline void TransNDHWC2NZ()
320 {217 {
321- uint16_t kdLoopTimes = self_->ctx.convTilingData->kernelD;218+ TransND2NZVfParams<SrcT, DstT, IndexT> params;
322- uint16_t ciLoopTimes = self_->ctx.ci1Opt;219+ FillCommonNd2NzParams(params);
323- uint16_t coLoopTimes = coOptLoopTimes;
324- uint16_t khkwLoopTimes = self_->ctx.convTilingData->kernelHxkernelW;
325 uint32_t srcGroupOptSize = self_->ctx.coOptAlign * self_->ctx.ciOptAlign;220 uint32_t srcGroupOptSize = self_->ctx.coOptAlign * self_->ctx.ciOptAlign;
326- uint32_t srcCiStride = self_->ctx.coOptAlign * Intf::k0;221+ params.srcCiStride = self_->ctx.coOptAlign * Intf::k0;
327- uint32_t srcCoStride = coPerReg;222+ params.srcKhKwStride = srcGroupOptSize;
328- uint32_t dstCiStride = self_->ctx.convTilingData->kernelHxkernelW * Intf::k0 * self_->ctx.coOptAlign;223+ params.srcCoStride = coPerReg;
329- uint32_t dstKhKwStride = Intf::k0 * self_->ctx.coOptAlign;224+ 
330- uint32_t srcKhKwStride = srcGroupOptSize;225+ TransND2NZKdVfParams kdParams;
331- uint32_t dstCoStride = coPerReg * Intf::k0;226+ kdParams.kdLoopTimes = self_->ctx.convTilingData->kernelD;
332- uint32_t srcKdStride = self_->ctx.convTilingData->kernelHxkernelW * srcGroupOptSize;227+ kdParams.srcKdStride = self_->ctx.convTilingData->kernelHxkernelW * srcGroupOptSize;
333- uint32_t dstKdStride = self_->ctx.convTilingData->kernelHxkernelW * self_->ctx.coOptAlign * self_->ctx.ci1Opt *228+ kdParams.dstKdStride = self_->ctx.convTilingData->kernelHxkernelW * self_->ctx.coOptAlign * self_->ctx.ci1Opt *
334 Intf::k0; // ci1Opt has updated in groupOptTail229 Intf::k0; // ci1Opt has updated in groupOptTail
335- __VEC_SCOPE__230+ TransND2NZKdVf<SrcT, DstT, RegT, IndexT, Intf::isQuantScene>(params, kdParams);
336- {
337- MicroAPI::RegTensor<RegT> gatherReg;
338- MicroAPI::RegTensor<IndexT> indexReg;
339- MicroAPI::MaskReg gatherMaskReg = MicroAPI::CreateMask<RegT, MicroAPI::MaskPattern::ALL>();
340- MicroAPI::MaskReg vstsMaskReg;
341- if constexpr (Intf::isQuantScene) {
342- vstsMaskReg = MicroAPI::CreateMask<DstT, MicroAPI::MaskPattern::H>();
343- } else {
344- vstsMaskReg = MicroAPI::CreateMask<DstT, MicroAPI::MaskPattern::ALL>();
345- }
346- 
347- __local_mem__ SrcT* srcAddr = (__local_mem__ SrcT*)ndTensor.GetPhyAddr();
348- __local_mem__ DstT* dstAddr = (__local_mem__ DstT*)self_->ctx.nzTensor.GetPhyAddr();
349- __local_mem__ IndexT* indexAddr = (__local_mem__ IndexT*)indexTensor.GetPhyAddr();
350- 
351- MicroAPI::DataCopy<IndexT>(indexReg, indexAddr);
352- 
353- for (uint16_t kdIndex = 0; kdIndex < kdLoopTimes; ++kdIndex) {
354- for (uint16_t ci1OptIndex = 0; ci1OptIndex < ciLoopTimes; ++ci1OptIndex) {
355- for (uint16_t khkwIndex = 0; khkwIndex < khkwLoopTimes; ++khkwIndex) {
356- for (uint16_t coOptIndex = 0; coOptIndex < coLoopTimes; ++coOptIndex) {
357- uint32_t srcOffset = kdIndex * srcKdStride + ci1OptIndex * srcCiStride +
358- khkwIndex * srcKhKwStride + coOptIndex * srcCoStride;
359- uint32_t dstOffset = kdIndex * dstKdStride + ci1OptIndex * dstCiStride +
360- khkwIndex * dstKhKwStride + coOptIndex * dstCoStride;
361- 
362- MicroAPI::DataCopyGather<RegT, SrcT, IndexT>(gatherReg, srcAddr + srcOffset, indexReg,
363- gatherMaskReg);
364- if constexpr (Intf::isQuantScene) {
365- // Remove the higher zeros of the int16_t data gathered by the Micro Gather instr
366- MicroAPI::Pack<uint8_t, RegT, MicroAPI::HighLowPart::LOWEST>(
367- (MicroAPI::RegTensor<uint8_t>&)gatherReg, gatherReg);
368- MicroAPI::DataCopy<DstT>(dstAddr + dstOffset, (MicroAPI::RegTensor<DstT>&)gatherReg,
369- vstsMaskReg);
370- } else {
371- MicroAPI::DataCopy<DstT>(dstAddr + dstOffset, gatherReg, vstsMaskReg);
372- }
373- }
374- }
375- }
376- }
377- }
378 }231 }
379 232 
380 __aicore__ inline void TransNHWC2NZ()233 __aicore__ inline void TransNHWC2NZ()
381 {234 {
382- uint16_t ciLoopTimes = self_->ctx.ci1Opt;235+ TransND2NZVfParams<SrcT, DstT, IndexT> params;
383- uint16_t khkwLoopTimes = self_->ctx.convTilingData->kernelHxkernelW;236+ FillCommonNd2NzParams(params);
384- uint16_t coLoopTimes = coOptLoopTimes;237+ params.srcCiStride = self_->ctx.coOptAlign * Intf::k0;
385- uint32_t srcCiStride = self_->ctx.coOptAlign * Intf::k0;238+ params.srcKhKwStride = self_->ctx.coOptAlign * self_->ctx.ciOptAlign;
386- uint32_t srcKhKwStride = self_->ctx.coOptAlign * self_->ctx.ciOptAlign;239+ params.srcCoStride = coPerReg;
387- uint32_t srcCoStride = coPerReg;240+ TransND2NZVf<SrcT, DstT, RegT, IndexT, Intf::isQuantScene>(params);
388- uint32_t dstCiStride = self_->ctx.convTilingData->kernelHxkernelW * Intf::k0 * self_->ctx.coOptAlign;
389- uint32_t dstKhKwStride = Intf::k0 * self_->ctx.coOptAlign;
390- uint32_t dstCoStride = coPerReg * Intf::k0;
391- 
392- __VEC_SCOPE__
393- {
394- MicroAPI::RegTensor<RegT> gatherReg;
395- MicroAPI::RegTensor<IndexT> indexReg;
396- MicroAPI::MaskReg gatherMaskReg = MicroAPI::CreateMask<RegT, MicroAPI::MaskPattern::ALL>();
397- MicroAPI::MaskReg vstsMaskReg;
398- if constexpr (Intf::isQuantScene) {
399- vstsMaskReg = MicroAPI::CreateMask<DstT, MicroAPI::MaskPattern::H>();
400- } else {
401- vstsMaskReg = MicroAPI::CreateMask<DstT, MicroAPI::MaskPattern::ALL>();
402- }
403- 
404- __local_mem__ SrcT* srcAddr = (__local_mem__ SrcT*)ndTensor.GetPhyAddr();
405- __local_mem__ DstT* dstAddr = (__local_mem__ DstT*)self_->ctx.nzTensor.GetPhyAddr();
406- __local_mem__ IndexT* indexAddr = (__local_mem__ IndexT*)indexTensor.GetPhyAddr();
407- 
408- MicroAPI::DataCopy<IndexT>(indexReg, indexAddr);
409- 
410- for (uint16_t ci1OptIndex = 0; ci1OptIndex < ciLoopTimes; ++ci1OptIndex) {
411- for (uint16_t khkwIndex = 0; khkwIndex < khkwLoopTimes; ++khkwIndex) {
412- for (uint16_t coOptIndex = 0; coOptIndex < coLoopTimes; ++coOptIndex) {
413- uint32_t srcOffset = ci1OptIndex * srcCiStride + khkwIndex * srcKhKwStride +
414- coOptIndex * srcCoStride;
415- uint32_t dstOffset = ci1OptIndex * dstCiStride + khkwIndex * dstKhKwStride +
416- coOptIndex * dstCoStride;
417- MicroAPI::DataCopyGather<RegT, SrcT, IndexT>(gatherReg, srcAddr + srcOffset, indexReg,
418- gatherMaskReg);
419- if constexpr (Intf::isQuantScene) {
420- // Remove the higher zeros of the int16_t data gathered by the Micro Gather instr
421- MicroAPI::Pack<uint8_t, RegT, MicroAPI::HighLowPart::LOWEST>(
422- (MicroAPI::RegTensor<uint8_t>&)gatherReg, gatherReg);
423- MicroAPI::DataCopy<DstT>(dstAddr + dstOffset, (MicroAPI::RegTensor<DstT>&)gatherReg,
424- vstsMaskReg);
425- } else {
426- MicroAPI::DataCopy<DstT>(dstAddr + dstOffset, gatherReg, vstsMaskReg);
427- }
428- }
429- }
430- }
431- }
432 }241 }
433 242 
434private:243private:
@@ -445,6 +254,19 @@ private:
445 254 
446 uint16_t coOptLoopTimes = 0;255 uint16_t coOptLoopTimes = 0;
447 uint16_t coPerReg = 0;256 uint16_t coPerReg = 0;
257+ 
258+ __aicore__ inline void FillCommonNd2NzParams(TransND2NZVfParams<SrcT, DstT, IndexT>& params)
259+ {
260+ params.ciLoopTimes = self_->ctx.ci1Opt;
261+ params.coLoopTimes = coOptLoopTimes;
262+ params.khkwLoopTimes = self_->ctx.convTilingData->kernelHxkernelW;
263+ params.dstCiStride = self_->ctx.convTilingData->kernelHxkernelW * Intf::k0 * self_->ctx.coOptAlign;
264+ params.dstKhKwStride = Intf::k0 * self_->ctx.coOptAlign;
265+ params.dstCoStride = coPerReg * Intf::k0;
266+ params.srcAddr = (__ubuf__ SrcT*)ndTensor.GetPhyAddr();
267+ params.dstAddr = (__ubuf__ DstT*)self_->ctx.nzTensor.GetPhyAddr();
268+ params.indexAddr = (__ubuf__ IndexT*)indexTensor.GetPhyAddr();
269+ }
448};270};
449 271 
450template <class Intf>272template <class Intf>
@@ -529,4 +351,4 @@ private:
529 351 
530}; // namespace ConvFunc352}; // namespace ConvFunc
531 353 
532-#endif // CONV_INSTR_OPT_GROUP_IMPL_H354+#endif // CONV_INSTR_OPT_GROUP_IMPL_H
Mconv/conv2d_v2/op_kernel/arch35/conv2d_small_kernel.h+1-1
@@ -1174,7 +1174,7 @@ __aicore__ inline void Conv2dSmallKernel<FmapType, weightType, biasType, out0Typ
1174 GlobalTensor<OutputT> outputGm;1174 GlobalTensor<OutputT> outputGm;
1175 outputGm.SetGlobalBuffer(reinterpret_cast<__gm__ OutputT*>(yAddr) + batchOutOff + nOutOff);1175 outputGm.SetGlobalBuffer(reinterpret_cast<__gm__ OutputT*>(yAddr) + batchOutOff + nOutOff);
1176 1176 
1177- FixpipeParamsC310<Layout> fp;1177+ FixpipeParamsArch3510<Layout> fp;
1178#if defined(__DAV_35_FAMILY__)1178#if defined(__DAV_35_FAMILY__)
1179 if constexpr (AscendC::IsSameType<weightType, half>::value) {1179 if constexpr (AscendC::IsSameType<weightType, half>::value) {
1180 fp.fixShiftVal = FIX_SHIFT_VAL_LEN_A16W16 - tiling_->fixedShiftValue;1180 fp.fixShiftVal = FIX_SHIFT_VAL_LEN_A16W16 - tiling_->fixedShiftValue;
Mconv/conv2d_v2/op_kernel/arch35/conv2d_v2_depthwise_simplify.h+77-55
@@ -69,6 +69,61 @@ constexpr uint16_t G_MTE1_EVENT_BASE = 6;
69__aicore__ inline uint32_t GCeilDiv(uint32_t a, uint32_t b) { return (b == 0) ? 0 : (a + b - 1) / b; }69__aicore__ inline uint32_t GCeilDiv(uint32_t a, uint32_t b) { return (b == 0) ? 0 : (a + b - 1) / b; }
70__aicore__ inline uint32_t GAlignUp(uint32_t a, uint32_t b) { return GCeilDiv(a, b) * b; }70__aicore__ inline uint32_t GAlignUp(uint32_t a, uint32_t b) { return GCeilDiv(a, b) * b; }
71 71 
72+template <typename IndexT>
73+__simd_vf__ inline void DwSetIndexVf(__ubuf__ IndexT* indexAddr, uint16_t repeatTimes, IndexT nStride, uint8_t k0)
74+{
75+ Reg::RegTensor<IndexT> indexReg;
76+ Reg::LoadAlign<IndexT>(indexReg, indexAddr);
77+ uint32_t maskL = k0;
78+ Reg::MaskReg maskReg = Reg::UpdateMask<IndexT>(maskL);
79+ 
80+ uint8_t dstOffset = k0;
81+ uint8_t elesPerRepeat = k0;
82+ for (uint16_t repeat = 0; repeat < repeatTimes; ++repeat) {
83+ Reg::Adds<IndexT, IndexT>(indexReg, indexReg, nStride, maskReg);
84+ Reg::StoreAlign<IndexT>(indexAddr + dstOffset, indexReg, maskReg);
85+ dstOffset += elesPerRepeat;
86+ }
87+}
88+ 
89+template <typename T, typename IndexT>
90+struct DwTransND2NZVfParams {
91+ uint16_t ciLoopTimes;
92+ uint16_t khkwLoopTimes;
93+ uint16_t coLoopTimes;
94+ uint32_t srcCiStride;
95+ uint32_t srcKhKwStride;
96+ uint32_t srcCoStride;
97+ uint32_t dstCiStride;
98+ uint32_t dstKhKwStride;
99+ uint32_t dstCoStride;
100+ __ubuf__ T* srcAddr;
101+ __ubuf__ T* dstAddr;
102+ __ubuf__ IndexT* indexAddr;
103+};
104+ 
105+template <typename T, typename IndexT>
106+__simd_vf__ inline void DwTransND2NZVf(const DwTransND2NZVfParams<T, IndexT> params)
107+{
108+ Reg::RegTensor<IndexT> indexReg;
109+ Reg::MaskReg maskReg = Reg::CreateMask<T, Reg::MaskPattern::ALL>();
110+ Reg::LoadAlign<IndexT>(indexReg, params.indexAddr);
111+ 
112+ for (uint16_t ci1OptIndex = 0; ci1OptIndex < params.ciLoopTimes; ++ci1OptIndex) {
113+ for (uint16_t khkwIndex = 0; khkwIndex < params.khkwLoopTimes; ++khkwIndex) {
114+ for (uint16_t coOptIndex = 0; coOptIndex < params.coLoopTimes; ++coOptIndex) {
115+ uint32_t srcOffset = ci1OptIndex * params.srcCiStride + khkwIndex * params.srcKhKwStride +
116+ coOptIndex * params.srcCoStride;
117+ uint32_t dstOffset = ci1OptIndex * params.dstCiStride + khkwIndex * params.dstKhKwStride +
118+ coOptIndex * params.dstCoStride;
119+ Reg::RegTensor<T> gatherReg;
120+ Reg::Gather<T, T, IndexT>(gatherReg, params.srcAddr + srcOffset, indexReg, maskReg);
121+ Reg::StoreAlign<T>(params.dstAddr + dstOffset, gatherReg, maskReg);
122+ }
123+ }
124+ }
125+}
126+ 
72// Default CONV_CFG for standalone direct-invoke builds (OPT_GROUP + M_MODE)127// Default CONV_CFG for standalone direct-invoke builds (OPT_GROUP + M_MODE)
73struct DefaultConvCfg {128struct DefaultConvCfg {
74 static constexpr int8_t groupType = 3; // OPT_GROUP_CONV129 static constexpr int8_t groupType = 3; // OPT_GROUP_CONV
@@ -352,7 +407,7 @@ __aicore__ inline void DepthwiseConv2dSimplifiedKernel<CONV_CFG, DTYPE, FmapForm
352 if constexpr (aFormat == ConvFormat::NHWC) {407 if constexpr (aFormat == ConvFormat::NHWC) {
353 uint32_t co1Opt = GCeilDiv(t.coutOpt, GN0);408 uint32_t co1Opt = GCeilDiv(t.coutOpt, GN0);
354 uint32_t coOptAlign = co1Opt * GN0;409 uint32_t coOptAlign = co1Opt * GN0;
355- MultiCopyParams<DTYPE, G_NDDMA_HWC_DIMS> copyParamsHWC;410+ NdDmaParams<DTYPE, G_NDDMA_HWC_DIMS> copyParamsHWC;
356 copyParamsHWC.loopInfo.loopSize[G_NDDMA_LOOP0_INDEX] = (t.cout / t.groups);411 copyParamsHWC.loopInfo.loopSize[G_NDDMA_LOOP0_INDEX] = (t.cout / t.groups);
357 copyParamsHWC.loopInfo.loopSrcStride[G_NDDMA_LOOP0_INDEX] = 1;412 copyParamsHWC.loopInfo.loopSrcStride[G_NDDMA_LOOP0_INDEX] = 1;
358 copyParamsHWC.loopInfo.loopDstStride[G_NDDMA_LOOP0_INDEX] = 1;413 copyParamsHWC.loopInfo.loopDstStride[G_NDDMA_LOOP0_INDEX] = 1;
@@ -371,7 +426,7 @@ __aicore__ inline void DepthwiseConv2dSimplifiedKernel<CONV_CFG, DTYPE, FmapForm
371 copyParamsHWC.loopInfo.loopDstStride[G_NDDMA_LOOP3_INDEX] = coOptAlign * cinAligned_;426 copyParamsHWC.loopInfo.loopDstStride[G_NDDMA_LOOP3_INDEX] = coOptAlign * cinAligned_;
372 DataCopy<DTYPE, G_NDDMA_HWC_DIMS, kDefaultMultiCopyConfig>(ubNd, filterGm_[0], copyParamsHWC);427 DataCopy<DTYPE, G_NDDMA_HWC_DIMS, kDefaultMultiCopyConfig>(ubNd, filterGm_[0], copyParamsHWC);
373 } else {428 } else {
374- MultiCopyParams<DTYPE, G_NDDMA_DIMS> copyParams;429+ NdDmaParams<DTYPE, G_NDDMA_DIMS> copyParams;
375 uint64_t srcKSize = (t.cin / t.groups) * khkw;430 uint64_t srcKSize = (t.cin / t.groups) * khkw;
376 copyParams.loopInfo.loopSize[G_NDDMA_LOOP0_INDEX] = static_cast<uint32_t>(srcKSize);431 copyParams.loopInfo.loopSize[G_NDDMA_LOOP0_INDEX] = static_cast<uint32_t>(srcKSize);
377 copyParams.loopInfo.loopSrcStride[G_NDDMA_LOOP0_INDEX] = 1;432 copyParams.loopInfo.loopSrcStride[G_NDDMA_LOOP0_INDEX] = 1;
@@ -517,25 +572,10 @@ __aicore__ inline void DepthwiseConv2dSimplifiedKernel<CONV_CFG, DTYPE, FmapForm
517 SetFlag<HardEvent::S_V>(EVENT_ID0);572 SetFlag<HardEvent::S_V>(EVENT_ID0);
518 WaitFlag<HardEvent::S_V>(EVENT_ID0);573 WaitFlag<HardEvent::S_V>(EVENT_ID0);
519 574 
520- __local_mem__ IndexT* indexAddr = (__local_mem__ IndexT*)indexTensor.GetPhyAddr();575+ __ubuf__ IndexT* indexAddr = (__ubuf__ IndexT*)indexTensor.GetPhyAddr();
521 uint16_t repeatTimes = static_cast<uint16_t>(G_REG_SIZE / sizeof(IndexT) / K0_VAL - 1);576 uint16_t repeatTimes = static_cast<uint16_t>(G_REG_SIZE / sizeof(IndexT) / K0_VAL - 1);
522- uint8_t dstOffset = K0_VAL;577+ IndexT nStride = static_cast<IndexT>((aFormat == ConvFormat::NHWC) ? 1 : kUbSize_);
523- uint8_t elesPerRepeat = K0_VAL;578+ DwSetIndexVf(indexAddr, repeatTimes, nStride, static_cast<uint8_t>(K0_VAL));
524- uint32_t maskL = K0_VAL;
525- uint16_t nStride = static_cast<uint16_t>((aFormat == ConvFormat::NHWC) ? 1 : kUbSize_);
526- 
527- __VEC_SCOPE__
528- {
529- MicroAPI::RegTensor<IndexT> indexReg;
530- MicroAPI::MaskReg maskReg = MicroAPI::UpdateMask<IndexT>(maskL);
531- MicroAPI::DataCopy<IndexT>(indexReg, indexAddr);
532- 
533- for (uint16_t repeat = 0; repeat < repeatTimes; ++repeat) {
534- MicroAPI::Adds<IndexT, IndexT>(indexReg, indexReg, nStride, maskReg);
535- MicroAPI::DataCopy<IndexT>(indexAddr + dstOffset, indexReg, maskReg);
536- dstOffset += elesPerRepeat;
537- }
538- }
539}579}
540 580 
541template <class CONV_CFG, typename DTYPE, ConvFormat FmapFormat>581template <class CONV_CFG, typename DTYPE, ConvFormat FmapFormat>
@@ -564,40 +604,22 @@ __aicore__ inline void DepthwiseConv2dSimplifiedKernel<CONV_CFG, DTYPE, FmapForm
564 srcKhKwStride = 1;604 srcKhKwStride = 1;
565 srcCoStride = coPerReg * kUbSize_;605 srcCoStride = coPerReg * kUbSize_;
566 }606 }
567- uint32_t dstCiStride = khkw * K0_VAL * coOptAlign;
568- uint32_t dstKhKwStride = K0_VAL * coOptAlign;
569- uint32_t dstCoStride = coPerReg * K0_VAL;
570 607 
571- __VEC_SCOPE__608+ DwTransND2NZVfParams<DTYPE, IndexT> params;
572- {609+ params.ciLoopTimes = ciLoopTimes;
573- MicroAPI::RegTensor<DTYPE> gatherReg;610+ params.khkwLoopTimes = khkwLoopTimes;
574- MicroAPI::RegTensor<IndexT> indexReg;611+ params.coLoopTimes = coLoopTimes;
575- MicroAPI::MaskReg gatherMaskReg = MicroAPI::CreateMask<DTYPE, MicroAPI::MaskPattern::ALL>();612+ params.srcCiStride = srcCiStride;
576- MicroAPI::MaskReg vstsMaskReg = MicroAPI::CreateMask<DTYPE, MicroAPI::MaskPattern::ALL>();613+ params.srcKhKwStride = srcKhKwStride;
577- 614+ params.srcCoStride = srcCoStride;
578- __local_mem__ DTYPE* srcAddr = (__local_mem__ DTYPE*)ndTensor.GetPhyAddr();615+ params.dstCiStride = khkw * K0_VAL * coOptAlign;
579- LocalTensor<DTYPE> nzTmp(TPosition::VECIN, ubNzOffBytes_, nzBufElems_);616+ params.dstKhKwStride = K0_VAL * coOptAlign;
580- __local_mem__ DTYPE* dstAddr = (__local_mem__ DTYPE*)nzTmp.GetPhyAddr();617+ params.dstCoStride = coPerReg * K0_VAL;
581- __local_mem__ IndexT* indexAddr = (__local_mem__ IndexT*)indexTensor.GetPhyAddr();618+ params.srcAddr = (__ubuf__ DTYPE*)ndTensor.GetPhyAddr();
582- 619+ LocalTensor<DTYPE> nzTmp(TPosition::VECIN, ubNzOffBytes_, nzBufElems_);
583- MicroAPI::DataCopy<IndexT>(indexReg, indexAddr);620+ params.dstAddr = (__ubuf__ DTYPE*)nzTmp.GetPhyAddr();
584- 621+ params.indexAddr = (__ubuf__ IndexT*)indexTensor.GetPhyAddr();
585- for (uint16_t ci1OptIndex = 0; ci1OptIndex < ciLoopTimes; ++ci1OptIndex) {622+ DwTransND2NZVf(params);
586- for (uint16_t khkwIndex = 0; khkwIndex < khkwLoopTimes; ++khkwIndex) {
587- for (uint16_t coOptIndex = 0; coOptIndex < coLoopTimes; ++coOptIndex) {
588- uint32_t srcOffset = ci1OptIndex * srcCiStride + khkwIndex * srcKhKwStride +
589- coOptIndex * srcCoStride;
590- uint32_t dstOffset = ci1OptIndex * dstCiStride + khkwIndex * dstKhKwStride +
591- coOptIndex * dstCoStride;
592- 
593- MicroAPI::DataCopyGather<DTYPE, DTYPE, IndexT>(gatherReg, srcAddr + srcOffset, indexReg,
594- gatherMaskReg);
595- 
596- MicroAPI::DataCopy<DTYPE>(dstAddr + dstOffset, (MicroAPI::RegTensor<DTYPE>&)gatherReg, vstsMaskReg);
597- }
598- }
599- }
600- }
601}623}
602 624 
603template <class CONV_CFG, typename DTYPE, ConvFormat FmapFormat>625template <class CONV_CFG, typename DTYPE, ConvFormat FmapFormat>
@@ -877,7 +899,7 @@ __aicore__ inline void DepthwiseConv2dSimplifiedKernel<CONV_CFG, DTYPE, FmapForm
877 uint64_t hwOut = static_cast<uint64_t>(t.hout) * t.wout;899 uint64_t hwOut = static_cast<uint64_t>(t.hout) * t.wout;
878 if constexpr (aFormat == ConvFormat::NHWC) {900 if constexpr (aFormat == ConvFormat::NHWC) {
879 uint64_t outOff = static_cast<uint64_t>(mGlobal) * t.cout + nStart;901 uint64_t outOff = static_cast<uint64_t>(mGlobal) * t.cout + nStart;
880- FixpipeParamsC310<CO2Layout::ROW_MAJOR> fp;902+ FixpipeParamsArch3510<CO2Layout::ROW_MAJOR> fp;
881 fp.nSize = curN;903 fp.nSize = curN;
882 fp.mSize = curM;904 fp.mSize = curM;
883 fp.srcStride = mAligned;905 fp.srcStride = mAligned;
@@ -898,7 +920,7 @@ __aicore__ inline void DepthwiseConv2dSimplifiedKernel<CONV_CFG, DTYPE, FmapForm
898 return;920 return;
899 }921 }
900 uint64_t outOff = static_cast<uint64_t>(nStart) * hwOut + mGlobal;922 uint64_t outOff = static_cast<uint64_t>(nStart) * hwOut + mGlobal;
901- FixpipeParamsC310<CO2Layout::COLUMN_MAJOR> fp;923+ FixpipeParamsArch3510<CO2Layout::COLUMN_MAJOR> fp;
902 fp.nSize = curN;924 fp.nSize = curN;
903 fp.mSize = curM;925 fp.mSize = curM;
904 fp.srcStride = mAligned;926 fp.srcStride = mAligned;
Mconv/conv2d_v2/op_kernel/arch35/conv2d_v2_instr_c04_impl.h+16-54
@@ -18,6 +18,7 @@
18 18 
19#include "conv2d_v2_config.h"19#include "conv2d_v2_config.h"
20#include "conv2d_v2_util.h"20#include "conv2d_v2_util.h"
21+#include "../../common/arch35/conv_instr_nd2nz_vf.h"
21 22 
22namespace Conv2dFunc {23namespace Conv2dFunc {
23using namespace AscendC;24using namespace AscendC;
@@ -124,7 +125,7 @@ private:
124 125 
125private:126private:
126 Intf* self_ = nullptr;127 Intf* self_ = nullptr;
127- MultiCopyParams<typename Intf::WeightT, NDDMA_DIMS> copyParams;128+ NdDmaParams<typename Intf::WeightT, NDDMA_DIMS> copyParams;
128 uint64_t srcOffset = 0;129 uint64_t srcOffset = 0;
129};130};
130 131 
@@ -146,41 +147,17 @@ public:
146 SetIndex();147 SetIndex();
147 }148 }
148 149 
149- uint16_t kLoopTimes = self_->ctx.convTilingData->kBL1 / Intf::k0;150+ TransFractalZVfParams<typename Intf::WeightT, IndexT> params;
150- uint16_t nLoopTimes = self_->ctx.currentUbNStepAilgn / BLOCK_L0_N * CO0_LOOP_TIMES;151+ params.kLoopTimes = self_->ctx.convTilingData->kBL1 / Intf::k0;
151- 152+ params.nLoopTimes = self_->ctx.currentUbNStepAilgn / BLOCK_L0_N * CO0_LOOP_TIMES;
152- uint32_t srcKStride = Intf::k0;153+ params.srcKStride = Intf::k0;
153- uint32_t srcNStride = coPerReg * self_->ctx.convTilingData->kBL1;154+ params.srcNStride = coPerReg * self_->ctx.convTilingData->kBL1;
154- uint32_t dstKStride = Intf::k0 * self_->ctx.currentUbNStepAilgn;155+ params.dstKStride = Intf::k0 * self_->ctx.currentUbNStepAilgn;
155- uint32_t dstNStride = Intf::k0 * coPerReg;156+ params.dstNStride = Intf::k0 * coPerReg;
156- 157+ params.srcAddr = (__ubuf__ typename Intf::WeightT*)self_->ctx.ndTensor.GetPhyAddr();
157- __VEC_SCOPE__158+ params.dstAddr = (__ubuf__ typename Intf::WeightT*)self_->ctx.nzTensor.GetPhyAddr();
158- {159+ params.indexAddr = (__ubuf__ IndexT*)indexTensor.GetPhyAddr();
159- MicroAPI::RegTensor<typename Intf::WeightT> gatherReg;160+ TransFractalZVf(params);
160- MicroAPI::RegTensor<IndexT> indexReg;
161- MicroAPI::MaskReg maskReg = MicroAPI::CreateMask<typename Intf::WeightT, MicroAPI::MaskPattern::ALL>();
162- 
163- __local_mem__ typename Intf::WeightT* srcAddr = (__local_mem__
164- typename Intf::WeightT*)self_->ctx.ndTensor.GetPhyAddr();
165- __local_mem__ typename Intf::WeightT* dstAddr = (__local_mem__
166- typename Intf::WeightT*)self_->ctx.nzTensor.GetPhyAddr();
167- __local_mem__ IndexT* indexAddr = (__local_mem__ IndexT*)indexTensor.GetPhyAddr();
168- 
169- MicroAPI::DataCopy<IndexT>(indexReg, indexAddr);
170- 
171- for (uint16_t kIndex = 0; kIndex < kLoopTimes; ++kIndex) {
172- for (uint16_t nIndex = 0; nIndex < nLoopTimes; ++nIndex) {
173- uint32_t srcOffset = kIndex * srcKStride + nIndex * srcNStride;
174- uint32_t dstOffset = kIndex * dstKStride + nIndex * dstNStride;
175- 
176- MicroAPI::DataCopyGather<typename Intf::WeightT, typename Intf::WeightT, IndexT>(
177- gatherReg, srcAddr + srcOffset, indexReg, maskReg);
178- 
179- MicroAPI::DataCopy<typename Intf::WeightT>(
180- dstAddr + dstOffset, (MicroAPI::RegTensor<typename Intf::WeightT>&)gatherReg, maskReg);
181- }
182- }
183- }
184 }161 }
185 162 
186private:163private:
@@ -195,25 +172,10 @@ private:
195 SetFlag<HardEvent::S_V>(eventId);172 SetFlag<HardEvent::S_V>(eventId);
196 WaitFlag<HardEvent::S_V>(eventId);173 WaitFlag<HardEvent::S_V>(eventId);
197 174 
198- __local_mem__ IndexT* indexAddr = (__local_mem__ IndexT*)indexTensor.GetPhyAddr();175+ __ubuf__ IndexT* indexAddr = (__ubuf__ IndexT*)indexTensor.GetPhyAddr();
199 uint16_t repeatTimes = static_cast<uint16_t>(REG_SIZE / sizeof(IndexT) / Intf::k0 - 1);176 uint16_t repeatTimes = static_cast<uint16_t>(REG_SIZE / sizeof(IndexT) / Intf::k0 - 1);
200- uint8_t dstOffset = Intf::k0;
201- uint8_t elesPerRepeat = Intf::k0;
202- uint32_t maskL = Intf::k0;
203 IndexT nStride = static_cast<IndexT>(self_->ctx.convTilingData->kBL1);177 IndexT nStride = static_cast<IndexT>(self_->ctx.convTilingData->kBL1);
204- 178+ SetIndexVf(indexAddr, repeatTimes, nStride, static_cast<uint8_t>(Intf::k0));
205- __VEC_SCOPE__
206- {
207- MicroAPI::RegTensor<IndexT> indexReg;
208- MicroAPI::DataCopy<IndexT>(indexReg, indexAddr);
209- MicroAPI::MaskReg maskReg = MicroAPI::UpdateMask<IndexT>(maskL);
210- 
211- for (uint16_t repeat = 0; repeat < repeatTimes; ++repeat) {
212- MicroAPI::Adds<IndexT, IndexT>(indexReg, indexReg, nStride, maskReg);
213- MicroAPI::DataCopy<IndexT>(indexAddr + dstOffset, indexReg, maskReg);
214- dstOffset += elesPerRepeat;
215- }
216- }
217 }179 }
218 180 
219private:181private:
@@ -263,4 +225,4 @@ private:
263 225 
264}; // namespace Conv2dFunc226}; // namespace Conv2dFunc
265 227 
266-#endif // CONV2D_2V_INSTR_C04_IMPL_H228+#endif // CONV2D_2V_INSTR_C04_IMPL_H
Mconv/conv2d_v2/op_kernel/arch35/conv2d_v2_instr_dma_impl.h+3-3
@@ -222,8 +222,8 @@ private:
222 222 
223private:223private:
224 Intf* self_ = nullptr;224 Intf* self_ = nullptr;
225- // MultiCopyParams<typename Intf::FmapT, NDDMA_DIMS_BASE> copyParams;225+ // NdDmaParams<typename Intf::FmapT, NDDMA_DIMS_BASE> copyParams;
226- MultiCopyParams<typename Intf::FmapT, NDDMA_DIMS_LOAD_FMAP> copyParams;226+ NdDmaParams<typename Intf::FmapT, NDDMA_DIMS_LOAD_FMAP> copyParams;
227 uint32_t srcCiStride = 0;227 uint32_t srcCiStride = 0;
228 uint32_t srcKhStride = 0;228 uint32_t srcKhStride = 0;
229 uint32_t srcKwStride = 0;229 uint32_t srcKwStride = 0;
@@ -297,4 +297,4 @@ private:
297 297 
298}; // namespace Conv2dFunc298}; // namespace Conv2dFunc
299 299 
300-#endif // CONV2D_V2_INSTR_DMA_IMPL_H300+#endif // CONV2D_V2_INSTR_DMA_IMPL_H
Mconv/conv2d_v2/op_kernel/arch35/conv2d_v2_instr_weight_ub_trans_impl.h+20-67
@@ -18,9 +18,11 @@
18 18 
19#include "conv2d_v2_config.h"19#include "conv2d_v2_config.h"
20#include "conv2d_v2_util.h"20#include "conv2d_v2_util.h"
21+#include "../../common/arch35/conv_instr_nd2nz_vf.h"
21 22 
22namespace Conv2dFunc {23namespace Conv2dFunc {
23using namespace AscendC;24using namespace AscendC;
25+using namespace conv;
24 26 
25template <class Intf>27template <class Intf>
26class WeightLoadGM2UBTools {28class WeightLoadGM2UBTools {
@@ -102,7 +104,7 @@ private:
102 using NddmaT = typename Conditional<(sizeof(typename Intf::WeightT) == DTYPE_SIZE_B8), uint8_t,104 using NddmaT = typename Conditional<(sizeof(typename Intf::WeightT) == DTYPE_SIZE_B8), uint8_t,
103 typename Intf::WeightT>::type;105 typename Intf::WeightT>::type;
104 DataCopyParams repeatParams;106 DataCopyParams repeatParams;
105- MultiCopyParams<NddmaT, NDDMA_DIMS> copyParams;107+ NdDmaParams<NddmaT, NDDMA_DIMS> copyParams;
106};108};
107 109 
108template <class Intf>110template <class Intf>
@@ -123,55 +125,21 @@ public:
123 SetIndex();125 SetIndex();
124 }126 }
125 127 
126- uint16_t ciLoopTimes = self_->ctx.convTilingData->bUbKStep / self_->ctx.convTilingData->kernelHxkernelW /128+ TransND2NZVfParams<SrcT, DstT, IndexT> params;
127- Intf::k0;129+ params.ciLoopTimes = self_->ctx.convTilingData->bUbKStep / self_->ctx.convTilingData->kernelHxkernelW /
128- uint16_t coLoopTimes = self_->ctx.currentUbNStepAilgn / BLOCK_L0_N * co0LoopTimes;130+ Intf::k0;
129- uint16_t khkwLoopTimes = self_->ctx.convTilingData->kernelHxkernelW;131+ params.coLoopTimes = self_->ctx.currentUbNStepAilgn / BLOCK_L0_N * co0LoopTimes;
130- uint32_t srcCiStride = self_->ctx.convTilingData->kernelHxkernelW * Intf::k0;132+ params.khkwLoopTimes = self_->ctx.convTilingData->kernelHxkernelW;
131- uint32_t srcCoStride = coPerReg * self_->ctx.convTilingData->bUbKStep;133+ params.srcCiStride = self_->ctx.convTilingData->kernelHxkernelW * Intf::k0;
132- uint32_t dstCiStride = self_->ctx.convTilingData->kernelHxkernelW * Intf::k0 * self_->ctx.currentUbNStepAilgn;134+ params.srcKhKwStride = 1;
133- uint32_t dstKhKwStride = Intf::k0 * self_->ctx.currentUbNStepAilgn;135+ params.srcCoStride = coPerReg * self_->ctx.convTilingData->bUbKStep;
134- uint32_t dstCoStride = coPerReg * Intf::k0;136+ params.dstCiStride = self_->ctx.convTilingData->kernelHxkernelW * Intf::k0 * self_->ctx.currentUbNStepAilgn;
135- 137+ params.dstKhKwStride = Intf::k0 * self_->ctx.currentUbNStepAilgn;
136- __VEC_SCOPE__138+ params.dstCoStride = coPerReg * Intf::k0;
137- {139+ params.srcAddr = (__ubuf__ SrcT*)self_->ctx.ndTensor.GetPhyAddr();
138- MicroAPI::RegTensor<RegT> gatherReg;140+ params.dstAddr = (__ubuf__ DstT*)self_->ctx.nzTensor.GetPhyAddr();
139- MicroAPI::RegTensor<IndexT> indexReg;141+ params.indexAddr = (__ubuf__ IndexT*)indexTensor.GetPhyAddr();
140- MicroAPI::MaskReg gatherMaskReg = MicroAPI::CreateMask<RegT, MicroAPI::MaskPattern::ALL>();142+ TransND2NZVf<SrcT, DstT, RegT, IndexT, Intf::isQuantScene>(params);
141- MicroAPI::MaskReg vstsMaskReg;
142- if constexpr (Intf::isQuantScene) {
143- vstsMaskReg = MicroAPI::CreateMask<DstT, MicroAPI::MaskPattern::H>();
144- } else {
145- vstsMaskReg = MicroAPI::CreateMask<DstT, MicroAPI::MaskPattern::ALL>();
146- }
147- 
148- __local_mem__ SrcT* srcAddr = (__local_mem__ SrcT*)self_->ctx.ndTensor.GetPhyAddr();
149- __local_mem__ DstT* dstAddr = (__local_mem__ DstT*)self_->ctx.nzTensor.GetPhyAddr();
150- __local_mem__ IndexT* indexAddr = (__local_mem__ IndexT*)indexTensor.GetPhyAddr();
151- 
152- MicroAPI::DataCopy<IndexT>(indexReg, indexAddr);
153- 
154- for (uint16_t ci1OptIndex = 0; ci1OptIndex < ciLoopTimes; ++ci1OptIndex) {
155- for (uint16_t khkwIndex = 0; khkwIndex < khkwLoopTimes; ++khkwIndex) {
156- for (uint16_t coOptIndex = 0; coOptIndex < coLoopTimes; ++coOptIndex) {
157- uint32_t srcOffset = ci1OptIndex * srcCiStride + khkwIndex + coOptIndex * srcCoStride;
158- uint32_t dstOffset = ci1OptIndex * dstCiStride + khkwIndex * dstKhKwStride +
159- coOptIndex * dstCoStride;
160- 
161- MicroAPI::DataCopyGather<RegT, SrcT, IndexT>(gatherReg, srcAddr + srcOffset, indexReg,
162- gatherMaskReg);
163- 
164- if constexpr (Intf::isQuantScene) {
165- // Remove the higher zeros of the int16_t data gathered by the Micro Gather instr
166- MicroAPI::Pack<uint8_t, RegT, MicroAPI::HighLowPart::LOWEST>(
167- (MicroAPI::RegTensor<uint8_t>&)gatherReg, gatherReg);
168- }
169- MicroAPI::DataCopy<DstT>(dstAddr + dstOffset, (MicroAPI::RegTensor<DstT>&)gatherReg,
170- vstsMaskReg);
171- }
172- }
173- }
174- }
175 }143 }
176 144 
177private:145private:
@@ -186,25 +154,10 @@ private:
186 SetFlag<HardEvent::S_V>(eventId);154 SetFlag<HardEvent::S_V>(eventId);
187 WaitFlag<HardEvent::S_V>(eventId);155 WaitFlag<HardEvent::S_V>(eventId);
188 156 
189- __local_mem__ IndexT* indexAddr = (__local_mem__ IndexT*)indexTensor.GetPhyAddr();157+ __ubuf__ IndexT* indexAddr = (__ubuf__ IndexT*)indexTensor.GetPhyAddr();
190 uint16_t repeatTimes = static_cast<uint16_t>(REG_SIZE / sizeof(IndexT) / Intf::k0 - 1);158 uint16_t repeatTimes = static_cast<uint16_t>(REG_SIZE / sizeof(IndexT) / Intf::k0 - 1);
191- uint8_t dstOffset = Intf::k0;
192- uint8_t elesPerRepeat = Intf::k0;
193- uint32_t maskL = Intf::k0;
194 IndexT nStride = static_cast<IndexT>(self_->ctx.convTilingData->bUbKStep);159 IndexT nStride = static_cast<IndexT>(self_->ctx.convTilingData->bUbKStep);
195- 160+ SetIndexVf(indexAddr, repeatTimes, nStride, static_cast<uint8_t>(Intf::k0));
196- __VEC_SCOPE__
197- {
198- MicroAPI::RegTensor<IndexT> indexReg;
199- MicroAPI::DataCopy<IndexT>(indexReg, indexAddr);
200- MicroAPI::MaskReg maskReg = MicroAPI::UpdateMask<IndexT>(maskL);
201- 
202- for (uint16_t repeat = 0; repeat < repeatTimes; ++repeat) {
203- MicroAPI::Adds<IndexT, IndexT>(indexReg, indexReg, nStride, maskReg);
204- MicroAPI::DataCopy<IndexT>(indexAddr + dstOffset, indexReg, maskReg);
205- dstOffset += elesPerRepeat;
206- }
207- }
208 }161 }
209 162 
210private:163private:
Mconv/conv3d_v2/op_kernel/arch35/conv3d_v2_instr_dequant_impl.h+2-2
@@ -91,7 +91,7 @@ public:
91private:91private:
92 Intf* self_ = nullptr;92 Intf* self_ = nullptr;
93 93 
94- FixpipeParamsC310<CFG_ROW_MAJOR_UB.format> fixpipeParams;94+ FixpipeParamsArch3510<CFG_ROW_MAJOR_UB.format> fixpipeParams;
95 95 
96 uint32_t currentML0 = 0;96 uint32_t currentML0 = 0;
97 uint32_t currentNL0 = 0;97 uint32_t currentNL0 = 0;
@@ -298,4 +298,4 @@ private:
298 298 
299} // namespace Conv3dFunc299} // namespace Conv3dFunc
300 300 
301-#endif // CONV3D_V2_INSTR_DEQUANT_IMPL_H301+#endif // CONV3D_V2_INSTR_DEQUANT_IMPL_H